mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 00:18:27 +00:00
Reverts #6736. Its end-of-clip read fails intermittently on Windows, and the fallback then expects frames past the clip's end.
400 lines
14 KiB
TypeScript
400 lines
14 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
|
|
import { processCompositionAudio } from "./audioMixer.js";
|
|
import { extractMediaMetadata, resolvePlayableVideoDuration } from "../utils/ffprobe.js";
|
|
import {
|
|
extractVideoFramesRange,
|
|
getFrameAtTime,
|
|
resolveFinalFrameExtractionWindow,
|
|
resolveVideoExtractionWindow,
|
|
} from "./videoFrameExtractor.js";
|
|
|
|
const FFMPEG = getFfmpegBinary();
|
|
const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;
|
|
const SAMPLE_RATE = 48_000;
|
|
const LEAD_SECONDS = 7;
|
|
const CLIP_SECONDS = 12;
|
|
const PICTURE_SECONDS = 20;
|
|
const BOTH_LATE_SECONDS = 2;
|
|
|
|
function ffmpeg(args: string[]): void {
|
|
const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args], {
|
|
encoding: "utf-8",
|
|
});
|
|
if (result.status !== 0) throw new Error(result.stderr);
|
|
}
|
|
|
|
/** Per output second, the source second heard (the tone is 200 + 100 * source second Hz), or null for silence. */
|
|
function sourceSecondsHeard(path: string, seconds: number): Array<number | null> {
|
|
const decoded = spawnSync(
|
|
FFMPEG,
|
|
["-v", "error", "-i", path, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"],
|
|
{ maxBuffer: 1 << 28 },
|
|
);
|
|
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
|
|
const pcm = decoded.stdout;
|
|
const heard: Array<number | null> = [];
|
|
const edge = 2_000;
|
|
for (let second = 0; second < seconds; second += 1) {
|
|
let crossings = 0;
|
|
let energy = 0;
|
|
let previous = 0;
|
|
for (let i = second * SAMPLE_RATE + edge; i < (second + 1) * SAMPLE_RATE - edge; i += 1) {
|
|
const sample = pcm.readInt16LE(i * 2);
|
|
energy += sample * sample;
|
|
if (i > second * SAMPLE_RATE + edge && previous < 0 !== sample < 0) crossings += 1;
|
|
previous = sample;
|
|
}
|
|
const windowSeconds = (SAMPLE_RATE - 2 * edge) / SAMPLE_RATE;
|
|
const rms = Math.sqrt(energy / (SAMPLE_RATE - 2 * edge));
|
|
heard.push(rms < 30 ? null : Math.round((crossings / 2 / windowSeconds - 200) / 100) + 0);
|
|
}
|
|
return heard;
|
|
}
|
|
|
|
describe.skipIf(!HAS_FFMPEG)(
|
|
"sources whose audio and video streams start at different times",
|
|
() => {
|
|
const dir = mkdtempSync(join(tmpdir(), "hf-stream-start-"));
|
|
|
|
beforeAll(() => {
|
|
ffmpeg([
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`testsrc=size=64x36:rate=30:duration=${PICTURE_SECONDS}`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-g",
|
|
"300",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
join(dir, "picture.mp4"),
|
|
]);
|
|
ffmpeg([
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`aevalsrc='0.1*sin(2*PI*(200+100*floor(t))*t)':s=${SAMPLE_RATE}:d=20`,
|
|
"-c:a",
|
|
"aac",
|
|
join(dir, "steps.m4a"),
|
|
]);
|
|
const lead = String(LEAD_SECONDS);
|
|
const picture = join(dir, "picture.mp4");
|
|
const steps = join(dir, "steps.m4a");
|
|
const mux = ["-map", "0:v", "-map", "1:a", "-c", "copy"];
|
|
ffmpeg(["-itsoffset", lead, "-i", picture, "-i", steps, ...mux, join(dir, "video-late.mp4")]);
|
|
ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);
|
|
const both = String(BOTH_LATE_SECONDS);
|
|
ffmpeg([
|
|
"-i",
|
|
picture,
|
|
"-i",
|
|
steps,
|
|
...mux,
|
|
"-output_ts_offset",
|
|
both,
|
|
join(dir, "both-late.mp4"),
|
|
]);
|
|
// MPEG-TS starts its clock at 1.4 s, so no stream starts at 0.
|
|
ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);
|
|
}, 60_000);
|
|
|
|
afterAll(() => rmSync(dir, { recursive: true, force: true }), 30_000);
|
|
|
|
const flatLane = {
|
|
target: "rate",
|
|
points: [
|
|
{ t: 0, v: 1 },
|
|
{ t: CLIP_SECONDS, v: 1 },
|
|
],
|
|
};
|
|
it.each([
|
|
{ source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 0 },
|
|
{ source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3 },
|
|
{ source: "video-late.mp4", audioLead: 0, type: "audio", mediaStart: 0 },
|
|
{ source: "video-late.ts", audioLead: 0, type: "video", mediaStart: 0 },
|
|
{ source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3, rate: flatLane },
|
|
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },
|
|
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },
|
|
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "audio", mediaStart: 0 },
|
|
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "audio", mediaStart: 0 },
|
|
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "video", mediaStart: 1 },
|
|
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "audio", mediaStart: 3 },
|
|
{
|
|
source: "audio-late.mp4",
|
|
audioLead: LEAD_SECONDS,
|
|
type: "video",
|
|
mediaStart: 0,
|
|
seconds: 5,
|
|
},
|
|
] as const)(
|
|
"%#: $source as <$type> from $mediaStart s plays each second where the browser plays it",
|
|
async ({ source, audioLead, type, mediaStart, ...rest }) => {
|
|
const seconds = "seconds" in rest ? rest.seconds : CLIP_SECONDS;
|
|
const workDir = mkdtempSync(join(dir, "work-"));
|
|
const outputPath = `${workDir}.m4a`;
|
|
const result = await processCompositionAudio(
|
|
[
|
|
{
|
|
id: "clip",
|
|
src: source,
|
|
start: 0,
|
|
end: seconds,
|
|
mediaStart,
|
|
...("rate" in rest ? { playbackRate: rest.rate } : {}),
|
|
layer: 0,
|
|
volume: 1,
|
|
type,
|
|
},
|
|
],
|
|
dir,
|
|
workDir,
|
|
outputPath,
|
|
seconds,
|
|
);
|
|
expect(result.success, JSON.stringify(result)).toBe(true);
|
|
const expected = Array.from({ length: seconds }, (_, second) => {
|
|
const mediaTime = mediaStart + second;
|
|
return mediaTime < audioLead ? null : mediaTime - audioLead;
|
|
});
|
|
expect(sourceSecondsHeard(outputPath, seconds)).toEqual(expected);
|
|
},
|
|
30_000,
|
|
);
|
|
|
|
// Chrome, seeking a late video stream, paints nothing until its first frame is under 1 s away.
|
|
it("shows no video frame while a late video stream's first frame is a second or more away", async () => {
|
|
const extracted = await extractVideoFramesRange(
|
|
join(dir, "video-late.mp4"),
|
|
"video-late",
|
|
0,
|
|
CLIP_SECONDS,
|
|
{ fps: 30, outputDir: join(dir, "frames") },
|
|
);
|
|
expect(getFrameAtTime(extracted, 1, 0)).toBeNull();
|
|
expect(getFrameAtTime(extracted, LEAD_SECONDS - 1.1, 0)).toBeNull();
|
|
expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.9, 0)).not.toBeNull();
|
|
expect(getFrameAtTime(extracted, 8, 0)).not.toBeNull();
|
|
}, 30_000);
|
|
|
|
it("shows the picture of a file whose streams all start late on the clock its audio plays on", async () => {
|
|
const frames = join(dir, "frames");
|
|
const source = await extractVideoFramesRange(join(dir, "picture.mp4"), "picture", 0, 5, {
|
|
fps: 30,
|
|
outputDir: frames,
|
|
});
|
|
const late = await extractVideoFramesRange(
|
|
join(dir, "both-late.mp4"),
|
|
"both-late",
|
|
0,
|
|
5 + BOTH_LATE_SECONDS,
|
|
{ fps: 30, outputDir: frames },
|
|
);
|
|
for (const second of [0.5, 3.5]) {
|
|
expect(readFileSync(getFrameAtTime(late, BOTH_LATE_SECONDS + second, 0)!)).toEqual(
|
|
readFileSync(getFrameAtTime(source, second, 0)!),
|
|
);
|
|
}
|
|
}, 30_000);
|
|
|
|
it("keeps every frame of a video stream that starts first", async () => {
|
|
const extracted = await extractVideoFramesRange(
|
|
join(dir, "audio-late.mp4"),
|
|
"audio-late",
|
|
0,
|
|
CLIP_SECONDS,
|
|
{ fps: 30, outputDir: join(dir, "frames") },
|
|
);
|
|
expect(getFrameAtTime(extracted, 0, 0)).not.toBeNull();
|
|
expect(getFrameAtTime(extracted, 1, 0)).not.toBeNull();
|
|
}, 30_000);
|
|
|
|
it("hides a late video stream's lead in a transport whose clock starts at 1.4 s", async () => {
|
|
const extracted = await extractVideoFramesRange(
|
|
join(dir, "video-late.ts"),
|
|
"video-late-ts",
|
|
0,
|
|
CLIP_SECONDS,
|
|
{ fps: 30, outputDir: join(dir, "frames") },
|
|
);
|
|
expect(getFrameAtTime(extracted, 1, 0)).toBeNull();
|
|
expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.5, 0)).not.toBeNull();
|
|
}, 30_000);
|
|
|
|
it("plays a late video stream to its end on the media timeline and loops on its full length", async () => {
|
|
const source = join(dir, "video-late.mp4");
|
|
const mediaSeconds = LEAD_SECONDS + PICTURE_SECONDS;
|
|
const window = resolveVideoExtractionWindow(
|
|
{ start: 0, end: mediaSeconds, mediaStart: 0, loop: false },
|
|
await extractMediaMetadata(source),
|
|
mediaSeconds,
|
|
);
|
|
expect(window.durationSeconds).toBeCloseTo(mediaSeconds, 1);
|
|
|
|
const extracted = await extractVideoFramesRange(source, "video-late-full", 0, mediaSeconds, {
|
|
fps: 10,
|
|
outputDir: join(dir, "frames"),
|
|
});
|
|
expect(getFrameAtTime(extracted, PICTURE_SECONDS + 1.5, 0, true)).not.toBeNull();
|
|
expect(getFrameAtTime(extracted, mediaSeconds + 1.5, 0, true)).toBeNull();
|
|
}, 30_000);
|
|
|
|
it.each([6.5, 10])(
|
|
"shows a late video stream from a trim at %s s on the media timeline",
|
|
async (mediaStart) => {
|
|
const extracted = await extractVideoFramesRange(
|
|
join(dir, "video-late.mp4"),
|
|
`video-late-from-${mediaStart}`,
|
|
mediaStart,
|
|
5,
|
|
{ fps: 10, outputDir: join(dir, "frames") },
|
|
);
|
|
expect(getFrameAtTime(extracted, 0.05, 0, false, mediaStart)).not.toBeNull();
|
|
},
|
|
30_000,
|
|
);
|
|
|
|
it("holds a late video stream's last frame past its end", async () => {
|
|
const source = join(dir, "video-late.mp4");
|
|
const metadata = await extractMediaMetadata(source);
|
|
const video = {
|
|
start: 0,
|
|
end: 5,
|
|
mediaStart: LEAD_SECONDS + PICTURE_SECONDS + 0.5,
|
|
loop: false,
|
|
};
|
|
const window = await resolveFinalFrameExtractionWindow(
|
|
source,
|
|
video,
|
|
metadata,
|
|
resolveVideoExtractionWindow(video, metadata, 5),
|
|
);
|
|
expect(window.extractionMediaStart).toBeGreaterThan(LEAD_SECONDS + PICTURE_SECONDS - 0.1);
|
|
}, 30_000);
|
|
},
|
|
);
|
|
|
|
/** Moves every fragment of a fragmented MP4 later, as cutting it from the middle of a longer stream does. */
|
|
function delayFragments(path: string, ticks: number): void {
|
|
const data = readFileSync(path);
|
|
const walk = (start: number, end: number): void => {
|
|
for (let at = start; at + 8 <= end; at += data.readUInt32BE(at)) {
|
|
const type = data.toString("latin1", at + 4, at + 8);
|
|
if (type === "moof" || type === "traf") walk(at + 8, at + data.readUInt32BE(at));
|
|
if (type !== "tfdt") continue;
|
|
if (data[at + 8] === 1) {
|
|
data.writeBigUInt64BE(data.readBigUInt64BE(at + 12) + BigInt(ticks), at + 12);
|
|
} else {
|
|
data.writeUInt32BE(data.readUInt32BE(at + 12) + ticks, at + 12);
|
|
}
|
|
}
|
|
};
|
|
walk(0, data.length);
|
|
writeFileSync(path, data);
|
|
}
|
|
|
|
describe.skipIf(!HAS_FFMPEG)("MP4 and WebM sources whose first timestamp is after 0", () => {
|
|
const dir = mkdtempSync(join(tmpdir(), "hf-late-first-frame-"));
|
|
// Every frame a different grey, so two frames are equal only when they show the same source frame.
|
|
const picture = (seconds: number) => [
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`nullsrc=size=64x36:rate=24:duration=${seconds},geq=lum='mod(N*37\\,256)':cb=128:cr=128`,
|
|
];
|
|
const h264 = ["-c:v", "libx264", "-bf", "3", "-pix_fmt", "yuv420p"];
|
|
|
|
beforeAll(() => {
|
|
const fragmented = join(dir, "fragment-cut.mp4");
|
|
ffmpeg([
|
|
...picture(3.125),
|
|
...h264,
|
|
"-video_track_timescale",
|
|
"12288",
|
|
"-movflags",
|
|
"cmaf",
|
|
fragmented,
|
|
]);
|
|
delayFragments(fragmented, 2048);
|
|
ffmpeg([
|
|
...picture(3),
|
|
"-c:v",
|
|
"libvpx-vp9",
|
|
"-deadline",
|
|
"realtime",
|
|
"-output_ts_offset",
|
|
"0.52",
|
|
join(dir, "late.webm"),
|
|
]);
|
|
ffmpeg([...picture(3.125), ...h264, "-output_ts_offset", "1.5", join(dir, "edit-list.mp4")]);
|
|
}, 60_000);
|
|
|
|
afterAll(() => rmSync(dir, { recursive: true, force: true }), 30_000);
|
|
|
|
const frameBytes = (
|
|
extracted: Awaited<ReturnType<typeof extractVideoFramesRange>>,
|
|
time: number,
|
|
) => {
|
|
const path = getFrameAtTime(extracted, time, 0);
|
|
return path === null ? null : readFileSync(path);
|
|
};
|
|
|
|
// The browser counts these files from timestamp 0; FFmpeg counts from the first one.
|
|
it.each([
|
|
{ source: "fragment-cut.mp4", firstFrameAt: 0.25 },
|
|
{ source: "late.webm", firstFrameAt: 0.52 },
|
|
{ source: "edit-list.mp4", firstFrameAt: 1.5 },
|
|
])(
|
|
"$source extracts every frame the browser plays, each at its timestamp",
|
|
async ({ source, firstFrameAt }) => {
|
|
const path = join(dir, source);
|
|
const playable = resolvePlayableVideoDuration(await extractMediaMetadata(path));
|
|
const extracted = await extractVideoFramesRange(path, source, 0, playable, {
|
|
fps: 30,
|
|
outputDir: join(dir, `frames-${source}`),
|
|
});
|
|
expect(extracted.totalFrames).toBe(Math.round(playable * 30));
|
|
const sourceFrameAt = (slot: number) =>
|
|
Math.max(0, Math.floor((slot / 30 - firstFrameAt) * 24 + 1e-6));
|
|
const slots = Array.from({ length: extracted.totalFrames }, (_, slot) =>
|
|
readFileSync(extracted.framePaths.get(slot)!),
|
|
);
|
|
for (let slot = 1; slot < slots.length; slot += 1) {
|
|
expect(slots[slot]!.equals(slots[slot - 1]!), `slot ${slot}`).toBe(
|
|
sourceFrameAt(slot) === sourceFrameAt(slot - 1),
|
|
);
|
|
}
|
|
|
|
const first = frameBytes(extracted, firstFrameAt + 0.03);
|
|
if (firstFrameAt < 1) {
|
|
expect(frameBytes(extracted, 0)).toEqual(first);
|
|
} else {
|
|
expect(frameBytes(extracted, firstFrameAt - 1.1)).toBeNull();
|
|
expect(frameBytes(extracted, firstFrameAt - 0.9)).toEqual(first);
|
|
}
|
|
|
|
const ranged = await extractVideoFramesRange(path, `${source}-ranged`, 0, playable, {
|
|
fps: 30,
|
|
outputDir: join(dir, `ranged-${source}`),
|
|
frameRanges: [
|
|
{ firstFrame: 0, frames: 5 },
|
|
{ firstFrame: 40, frames: 10 },
|
|
],
|
|
});
|
|
expect(ranged.framePaths.size).toBe(15);
|
|
for (const [index, framePath] of ranged.framePaths) {
|
|
expect(readFileSync(framePath), `frame ${index}`).toEqual(
|
|
readFileSync(extracted.framePaths.get(index)!),
|
|
);
|
|
}
|
|
},
|
|
30_000,
|
|
);
|
|
});
|