mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(engine): short clips from Apple-encoded videos keep their last frame (#7469)
* fix(engine): short clips from Apple-encoded videos keep their last frame * refactor(engine): name the end-of-source hold and frame bound instead of commenting them * fix(engine): frame ranges past a source's end stay empty with the held final frame * test(engine): move the final-frame test into its own file
This commit is contained in:
@@ -67,8 +67,8 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s
|
||||
}
|
||||
|
||||
describe("extractionCache constants", () => {
|
||||
it("exposes the v7 schema prefix", () => {
|
||||
expect(SCHEMA_PREFIX).toBe("hfcache-v8-");
|
||||
it("exposes the v9 schema prefix", () => {
|
||||
expect(SCHEMA_PREFIX).toBe("hfcache-v9-");
|
||||
});
|
||||
|
||||
it("exposes the frame filename prefix shared with the extractor", () => {
|
||||
|
||||
@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";
|
||||
* Current schema version. Bump it whenever extraction writes different frames for the same key,
|
||||
* or warm entries keep serving the old frames across a deploy. Each bump's commit says why.
|
||||
*/
|
||||
export const SCHEMA_PREFIX = "hfcache-v8-";
|
||||
export const SCHEMA_PREFIX = "hfcache-v9-";
|
||||
|
||||
/** Truncated hex chars of SHA-256 used for the entry directory name. */
|
||||
const KEY_HEX_CHARS = 16;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
import { extractVideoFramesRange, type ExtractedFrames } from "./videoFrameExtractor.js";
|
||||
|
||||
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
||||
const WIDTH = 32;
|
||||
const HEIGHT = 16;
|
||||
|
||||
// B-frames at a 1/600 timescale (what Apple devices write) store each frame one tick long.
|
||||
describe.skipIf(!HAS_FFMPEG)("a window that ends at the source's end", () => {
|
||||
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-final-frame-"));
|
||||
const source = join(fixtureDir, "index-30fps-bframes-ts600.mov");
|
||||
|
||||
beforeAll(async () => {
|
||||
// Frame k carries luma 16 + 2k, so each extracted frame names its source index.
|
||||
const synth = await runFfmpeg([
|
||||
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i"],
|
||||
`nullsrc=s=${WIDTH}x${HEIGHT}:r=30:d=1.5,geq=lum='16+2*N':cb=128:cr=128`,
|
||||
...["-c:v", "libx264", "-qp", "1", "-bf", "2", "-pix_fmt", "yuv420p"],
|
||||
...["-video_track_timescale", "600", source],
|
||||
]);
|
||||
if (!synth.success) throw new Error(`b-frame fixture synthesis failed: ${synth.stderr}`);
|
||||
}, 30_000);
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function sourceIndexes(extracted: ExtractedFrames): number[] {
|
||||
const decoded = spawnSync("ffmpeg", [
|
||||
...["-v", "error", "-i", join(extracted.outputDir, extracted.framePattern)],
|
||||
...["-f", "rawvideo", "-pix_fmt", "gray", "pipe:1"],
|
||||
]);
|
||||
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
|
||||
const indexes: number[] = [];
|
||||
for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) {
|
||||
indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2));
|
||||
}
|
||||
return indexes;
|
||||
}
|
||||
|
||||
it("keeps the last frame of a window that ends at the source's end", async () => {
|
||||
const extracted = await extractVideoFramesRange(source, "bframes-tail", 37 / 30, 8 / 30, {
|
||||
fps: 30,
|
||||
outputDir: fixtureDir,
|
||||
format: "png",
|
||||
});
|
||||
expect(sourceIndexes(extracted)).toEqual([37, 38, 39, 40, 41, 42, 43, 44]);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -1036,10 +1036,16 @@ async function leadInSecondsUnlessAborted(
|
||||
function stallTimeoutMs(baseMs: number, skippedSeconds: number, sourceFps: number): number {
|
||||
return baseMs + Math.ceil((skippedSeconds * sourceFps * 1000) / SKIP_DECODE_FPS_FLOOR);
|
||||
}
|
||||
const KEEP_END_PAST_A_ONE_TICK_FINAL_FRAME_THROUGH_THE_SNAP = "tpad=stop_mode=clone:stop=1";
|
||||
/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
|
||||
function undoTickRoundingOnTheSourceFrameGrid(seek: number, lead: number): string[] {
|
||||
function undoTickRoundingOnTheSourceFrameGrid(
|
||||
seek: number,
|
||||
lead: number,
|
||||
sourceFps: number,
|
||||
): string[] {
|
||||
const sinceFileStart = `(PTS*TB+${seek}-${lead})`;
|
||||
return [
|
||||
...(sourceFps > 0 ? [KEEP_END_PAST_A_ONE_TICK_FINAL_FRAME_THROUGH_THE_SNAP] : []),
|
||||
"settb=AVTB",
|
||||
`setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,
|
||||
];
|
||||
@@ -1077,13 +1083,18 @@ function framesWrittenReader(onProgress: ExtractionOptions["onProgress"]) {
|
||||
}
|
||||
|
||||
/** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */
|
||||
function splitFrameRanges(ranges: readonly FrameRange[], segmentFrames: number): FrameRange[] {
|
||||
function splitFrameRanges(
|
||||
ranges: readonly FrameRange[],
|
||||
segmentFrames: number,
|
||||
frameLimit: number,
|
||||
): FrameRange[] {
|
||||
const segments: FrameRange[] = [];
|
||||
for (const range of ranges) {
|
||||
for (let offset = 0; offset < range.frames; offset += segmentFrames) {
|
||||
const frames = Math.min(range.frames, frameLimit - range.firstFrame);
|
||||
for (let offset = 0; offset < frames; offset += segmentFrames) {
|
||||
segments.push({
|
||||
firstFrame: range.firstFrame + offset,
|
||||
frames: Math.min(segmentFrames, range.frames - offset),
|
||||
frames: Math.min(segmentFrames, frames - offset),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1205,7 +1216,11 @@ function videoFilterChain(input: VideoFilterChainInput): (seek: number) => strin
|
||||
? [
|
||||
...(metadata.isVFR
|
||||
? []
|
||||
: undoTickRoundingOnTheSourceFrameGrid(seek, videoStartOnMediaTimeline(metadata))),
|
||||
: undoTickRoundingOnTheSourceFrameGrid(
|
||||
seek,
|
||||
videoStartOnMediaTimeline(metadata),
|
||||
metadata.fps,
|
||||
)),
|
||||
"settb=intb/2",
|
||||
"setpts=PTS-lte(TB\\,0.001)",
|
||||
`fps=${ffmpegFps}:start_time=0:round=up`,
|
||||
@@ -1419,11 +1434,18 @@ async function extractVideoFramesRangeUnredacted(
|
||||
return filters.length > 0 ? ["-vf", filters.join(",")] : [];
|
||||
};
|
||||
const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
|
||||
const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
|
||||
const windowFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
|
||||
const framesBeforeSourceEnd = extractionFrameCountForDuration(
|
||||
playableDuration - startTime,
|
||||
normalizedFps,
|
||||
false,
|
||||
);
|
||||
const totalFrames = metadata.isVFR ? windowFrames : Math.min(windowFrames, framesBeforeSourceEnd);
|
||||
// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
|
||||
// emits that last slot: the fps filter ends a `-t` window at the nearest slot. A
|
||||
// decimal rate counts zero frames here and keeps the `-t` window.
|
||||
const boundVfrFrameCount = metadata.isVFR && totalFrames > 0;
|
||||
const frameCap = sampleAtOutputFps && totalFrames > 0 ? ["-frames:v", String(totalFrames)] : [];
|
||||
const extract = (hwaccel: string[]) =>
|
||||
sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
|
||||
? runSegmentedExtraction({
|
||||
@@ -1438,23 +1460,27 @@ async function extractVideoFramesRangeUnredacted(
|
||||
segments: splitFrameRanges(
|
||||
options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }],
|
||||
segmentFrames,
|
||||
totalFrames > 0 ? totalFrames : Number.POSITIVE_INFINITY,
|
||||
),
|
||||
runOptions,
|
||||
onProgress: options.onProgress,
|
||||
})
|
||||
: inExtractionSlot(
|
||||
async () =>
|
||||
runFfmpeg([...hwaccel, ...args, ...filterArgsFor(startTime), ...encodeArgs], {
|
||||
...runOptions,
|
||||
onStderr: framesWrittenReader(options.onProgress),
|
||||
timeout: stallTimeoutMs(
|
||||
ffmpegProcessTimeout,
|
||||
options.finalFrameOnly
|
||||
? startTime
|
||||
: await leadInSecondsUnlessAborted(videoPath, startTime, signal),
|
||||
metadata.fps,
|
||||
),
|
||||
}),
|
||||
runFfmpeg(
|
||||
[...hwaccel, ...args, ...frameCap, ...filterArgsFor(startTime), ...encodeArgs],
|
||||
{
|
||||
...runOptions,
|
||||
onStderr: framesWrittenReader(options.onProgress),
|
||||
timeout: stallTimeoutMs(
|
||||
ffmpegProcessTimeout,
|
||||
options.finalFrameOnly
|
||||
? startTime
|
||||
: await leadInSecondsUnlessAborted(videoPath, startTime, signal),
|
||||
metadata.fps,
|
||||
),
|
||||
},
|
||||
),
|
||||
signal,
|
||||
);
|
||||
options.onProgress?.({
|
||||
|
||||
Reference in New Issue
Block a user