mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
389 lines
14 KiB
TypeScript
389 lines
14 KiB
TypeScript
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { spawnSync } from "node:child_process";
|
|
import {
|
|
extractAllVideoFrames,
|
|
extractVideoFramesRange,
|
|
type VideoElement,
|
|
type ExtractedFrames,
|
|
} from "./videoFrameExtractor.js";
|
|
import { ffmpegStatsReader, runFfmpeg, type RunFfmpegOptions } from "../utils/runFfmpeg.js";
|
|
|
|
// Pass-through, so a test can tell whether extraction ran ffmpeg.
|
|
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
|
|
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
|
|
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
|
|
});
|
|
|
|
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
|
|
|
// Each output slot shows the frame on screen at its time, as the preview does.
|
|
describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-sampling-"));
|
|
const WIDTH = 32;
|
|
const HEIGHT = 16;
|
|
// Matroska stores whole-millisecond timestamps (a 30 fps frame at 66.67 ms reads 67 ms);
|
|
// a 1/30 timescale stores each frame exactly on a coarse tick.
|
|
const SOURCES = {
|
|
"mp4-60": { fps: 60, file: "index-60fps.mp4", muxer: [] },
|
|
"mkv-30": { fps: 30, file: "index-30fps.mkv", muxer: [] },
|
|
"mp4-30-timescale-30": {
|
|
fps: 30,
|
|
file: "index-30fps-ts30.mp4",
|
|
muxer: ["-video_track_timescale", "30"],
|
|
},
|
|
} as const;
|
|
type SourceName = keyof typeof SOURCES;
|
|
const sourcePath = (name: SourceName) => join(FIXTURE_DIR, SOURCES[name].file);
|
|
|
|
beforeAll(async () => {
|
|
for (const name of Object.keys(SOURCES) as SourceName[]) {
|
|
// Frame k carries luma 16 + 2k, so each extracted frame names its source index.
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`nullsrc=s=${WIDTH}x${HEIGHT}:r=${SOURCES[name].fps}:d=1.5,geq=lum='16+2*N':cb=128:cr=128`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-qp",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
...SOURCES[name].muxer,
|
|
sourcePath(name),
|
|
]);
|
|
if (!result.success) throw new Error(`index fixture synthesis failed: ${result.stderr}`);
|
|
}
|
|
}, 30_000);
|
|
|
|
afterAll(() => {
|
|
rmSync(FIXTURE_DIR, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
function sourceIndexes(extracted: ExtractedFrames): number[] {
|
|
const decoded = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
join(extracted.outputDir, extracted.framePattern),
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"gray",
|
|
"pipe:1",
|
|
]);
|
|
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
|
|
const indexes: number[] = [];
|
|
for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) {
|
|
indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2));
|
|
}
|
|
return indexes;
|
|
}
|
|
|
|
it.each([
|
|
{ source: "mp4-60", fps: 24, startTime: 0, duration: 0.5 },
|
|
{ source: "mp4-60", fps: 10, startTime: 0.37, duration: 0.6 },
|
|
{ source: "mkv-30", fps: 30, startTime: 0, duration: 0.5 },
|
|
{ source: "mp4-30-timescale-30", fps: 60, startTime: 0, duration: 0.5 },
|
|
] as const)(
|
|
"$source: samples the frame on screen at each $fps fps slot from $startTime s",
|
|
async (c) => {
|
|
const extracted = await extractVideoFramesRange(
|
|
sourcePath(c.source),
|
|
`${c.source}-${c.fps}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "png" },
|
|
);
|
|
const onScreen = Array.from({ length: Math.round(c.duration * c.fps) }, (_, i) =>
|
|
Math.floor((c.startTime + i / c.fps) * SOURCES[c.source].fps + 1e-9),
|
|
);
|
|
expect(sourceIndexes(extracted)).toEqual(onScreen);
|
|
},
|
|
30_000,
|
|
);
|
|
|
|
// #5260: one ffmpeg process over a 26-minute clip hit ffmpegProcessTimeout.
|
|
it.each([
|
|
// 260 frames: one 240-frame segment, then 20 more.
|
|
{ startTime: 0.5, duration: 130, frames: 260 },
|
|
// 200.3 - 80.3 is 120.00000000000001 s: exactly one segment, not a second near-empty one.
|
|
{ startTime: 80.3, duration: 200.3 - 80.3, frames: 240 },
|
|
])(
|
|
"extracts $duration s from $startTime s at 2 fps across segments without a gap or repeat",
|
|
async (c) => {
|
|
const source = join(FIXTURE_DIR, "index-2fps-long.mp4");
|
|
if (!existsSync(source)) {
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`nullsrc=s=${WIDTH}x${HEIGHT}:r=2:d=205,geq=lum='16+2*mod(N\\,100)':cb=128:cr=128`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-qp",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
source,
|
|
]);
|
|
if (!synth.success) throw new Error(`long fixture synthesis failed: ${synth.stderr}`);
|
|
}
|
|
const extracted = await extractVideoFramesRange(
|
|
source,
|
|
`long-2fps-${c.startTime}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: 2, outputDir: FIXTURE_DIR, format: "png" },
|
|
);
|
|
const onScreen = Array.from(
|
|
{ length: c.frames },
|
|
(_, i) => Math.floor((c.startTime + i / 2) * 2 + 1e-9) % 100,
|
|
);
|
|
expect(extracted.totalFrames).toBe(c.frames);
|
|
expect(sourceIndexes(extracted)).toEqual(onScreen);
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("stops extraction only on a stall, and its ffmpeg reports each frame it writes", async () => {
|
|
const source = join(FIXTURE_DIR, "paced-2s.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
|
|
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=2`, "-pix_fmt", "yuv420p", source],
|
|
]);
|
|
if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`);
|
|
const { runFfmpeg: real } =
|
|
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
|
|
let requested: RunFfmpegOptions | undefined;
|
|
const reported: number[] = [];
|
|
const readStats = ffmpegStatsReader(({ frames }) => {
|
|
if (frames) reported.push(frames);
|
|
});
|
|
// Real-time pacing spreads the stats over the run. The stall clock itself is runFfmpeg's to
|
|
// test; here it is lifted so a starved runner cannot race it.
|
|
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
|
|
requested = opts;
|
|
return real(["-stats_period", "0.1", "-re", ...args], {
|
|
...opts,
|
|
timeout: 60_000,
|
|
onStderr: readStats,
|
|
});
|
|
});
|
|
try {
|
|
const extracted = await extractVideoFramesRange(
|
|
source,
|
|
"paced-stall-only",
|
|
0,
|
|
2,
|
|
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
|
|
undefined,
|
|
{ ffmpegProcessTimeout: 2000 },
|
|
);
|
|
expect(extracted.totalFrames).toBe(20);
|
|
expect(requested).toMatchObject({ timeout: 2000 });
|
|
expect(new Set(reported).size).toBeGreaterThan(1);
|
|
} finally {
|
|
vi.mocked(runFfmpeg).mockImplementation(real);
|
|
}
|
|
}, 30_000);
|
|
|
|
it.each([
|
|
{ name: "one keyframe", keyint: "infinite", gapSeconds: 8 },
|
|
{ name: "a keyframe every second", keyint: "30", gapSeconds: 0 },
|
|
])(
|
|
"gives the stall clock the time to decode up to the seek point ($name)",
|
|
async ({ keyint, gapSeconds }) => {
|
|
const source = join(FIXTURE_DIR, `gop-${keyint}.mp4`);
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
|
|
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=10`, "-pix_fmt", "yuv420p"],
|
|
...["-x264-params", `keyint=${keyint}:scenecut=0`, source],
|
|
]);
|
|
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
|
|
const { runFfmpeg: real } =
|
|
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
|
|
let requested: RunFfmpegOptions | undefined;
|
|
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
|
|
requested = opts;
|
|
return real(args, opts);
|
|
});
|
|
try {
|
|
await extractVideoFramesRange(
|
|
source,
|
|
`stall-seek-${keyint}`,
|
|
8,
|
|
1,
|
|
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
|
|
undefined,
|
|
{ ffmpegProcessTimeout: 2000 },
|
|
);
|
|
expect(requested?.timeout).toBe(2000 + gapSeconds * 1000);
|
|
} finally {
|
|
vi.mocked(runFfmpeg).mockImplementation(real);
|
|
}
|
|
},
|
|
30_000,
|
|
);
|
|
|
|
it("stops a run that writes no new frame for the stall timeout", async () => {
|
|
// stdin stays open and empty, so ffmpeg waits for its first frame forever.
|
|
const stalled = await runFfmpeg(
|
|
["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"],
|
|
{ timeout: 500 },
|
|
);
|
|
expect(stalled.terminationReason).toBe("inactivity");
|
|
}, 30_000);
|
|
|
|
// A distributed chunk extracts only the frames it renders; they must be the full extraction's.
|
|
describe("partial extraction", () => {
|
|
const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4");
|
|
|
|
beforeAll(async () => {
|
|
// Real motion and long GOPs, so range seeks land mid-GOP.
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc2=s=64x36:r=30:d=20",
|
|
"-c:v",
|
|
"libx264",
|
|
"-g",
|
|
"250",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
gopSource,
|
|
]);
|
|
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
|
|
}, 30_000);
|
|
|
|
function frameBytes(dir: string): Map<string, Buffer> {
|
|
return new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))]));
|
|
}
|
|
|
|
it.each([
|
|
{ fps: 24, startTime: 1.3, duration: 17 },
|
|
{ fps: 60, startTime: 0, duration: 20 },
|
|
])(
|
|
"frameRanges at $fps fps write the same files as a full extraction",
|
|
async (c) => {
|
|
const full = await extractVideoFramesRange(
|
|
gopSource,
|
|
`full-${c.fps}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg" },
|
|
);
|
|
const last = full.totalFrames;
|
|
const ranges = [
|
|
{ firstFrame: 0, frames: 5 },
|
|
{ firstFrame: 97, frames: 40 },
|
|
{ firstFrame: last - 3, frames: 3 },
|
|
];
|
|
const partial = await extractVideoFramesRange(
|
|
gopSource,
|
|
`partial-${c.fps}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg", frameRanges: ranges },
|
|
);
|
|
const fullFrames = frameBytes(full.outputDir);
|
|
const partialFrames = frameBytes(partial.outputDir);
|
|
expect(partialFrames.size).toBe(48);
|
|
for (const [name, bytes] of partialFrames) {
|
|
expect(fullFrames.get(name)?.equals(bytes), name).toBe(true);
|
|
}
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it.each([
|
|
{ fps: 24, mediaStart: 3.3 },
|
|
{ fps: 30, mediaStart: 0 },
|
|
{ fps: 60, mediaStart: 7.77 },
|
|
])(
|
|
"a deferred window to the source end at $fps fps reports the full frame count",
|
|
async (c) => {
|
|
const video = (): VideoElement => ({
|
|
id: `defer-${c.fps}`,
|
|
src: gopSource,
|
|
start: 0,
|
|
end: Infinity,
|
|
mediaStart: c.mediaStart,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
const full = await extractAllVideoFrames([video()], FIXTURE_DIR, {
|
|
fps: c.fps,
|
|
outputDir: join(FIXTURE_DIR, `defer-full-${c.fps}`),
|
|
});
|
|
const deferred = await extractAllVideoFrames([video()], FIXTURE_DIR, {
|
|
fps: c.fps,
|
|
outputDir: join(FIXTURE_DIR, `defer-${c.fps}`),
|
|
deferRangeExtraction: true,
|
|
});
|
|
expect(deferred.errors).toEqual([]);
|
|
const result = deferred.extracted[0]!;
|
|
expect(result.totalFrames).toBe(full.extracted[0]!.totalFrames);
|
|
expect(result.framePaths.size).toBe(0);
|
|
expect(readdirSync(result.outputDir)).toEqual([]);
|
|
expect(result.deferredRange).toEqual({
|
|
startTime: c.mediaStart,
|
|
durationSeconds: expect.any(Number),
|
|
format: "jpg",
|
|
});
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("halves frames while they still cover the box a video is shown in", async () => {
|
|
const firstFrameSize = async (
|
|
videoId: string,
|
|
shownBox: { width: number; height: number },
|
|
) => {
|
|
const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, {
|
|
fps: 30,
|
|
outputDir: FIXTURE_DIR,
|
|
format: "jpg",
|
|
frameRanges: [{ firstFrame: 0, frames: 1 }],
|
|
shownBox,
|
|
});
|
|
const [frame] = result.framePaths.values();
|
|
return spawnSync("ffprobe", [
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"stream=width,height",
|
|
"-of",
|
|
"csv=p=0",
|
|
"--",
|
|
frame!,
|
|
])
|
|
.stdout.toString()
|
|
.trim();
|
|
};
|
|
// 64x36 source: halved as often as the box allows (even sides round up); never below the box.
|
|
expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10");
|
|
expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18");
|
|
// Exactly half (a 4K source in a 1080p box): one halving lands on the box and still covers it.
|
|
expect(await firstFrameSize("shown-half", { width: 32, height: 18 })).toBe("32,18");
|
|
expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36");
|
|
}, 30_000);
|
|
});
|
|
});
|