diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index f5da23c5e4..075adb086b 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1929,12 +1929,29 @@ mismatched encodes. The directory is deleted after a successful render unless mp4 and mov only; png-sequence, gif, webm, HDR and shader-transition renders keep their existing routes. +By default every frame of every source video is extracted before capture +starts. With `HF_VIDEO_FRAME_WINDOW=true`, each segment instead extracts only +the source frames it shows and deletes them when it closes. Things to know +before switching it on: + +- Segment files are deleted as soon as they are joined, unless `--keep-segments` + is set. A failure after the join therefore recaptures everything on `--resume`. +- Only mp4 and mov renders that segment honour the flag. Motion blur, variable + frame rate sources and HDR-transformed sources stay fully extracted. The + render log says when the flag was declined and how many videos stayed fully + extracted. +- Source-frame disk use is bounded by roughly workers x `HF_SEGMENT_FRAMES` + frames, and only for videos that can be windowed. +- A windowed render skips the extraction cache, so a re-export gets no + warm-cache speedup. + | Environment variable | Default | Effect | | ------------------------------ | ---------- | ------------------------------------------------------------------------------------------ | | `HF_SEGMENTED_CAPTURE` | unset | `true` segments any eligible render; `false` is the kill switch | | `HF_SEGMENTED_MIN_SECONDS` | off | Segment renders at least this long. Unset means duration never routes on its own | | `HF_SEGMENT_FRAMES` | `3000` | Frames per segment (~100 s at 30 fps); minimum 30 | | `HF_SEGMENT_BROWSER_RECYCLE` | `3` | Restart the browser every N segments; `0` keeps one session for the whole render | +| `HF_VIDEO_FRAME_WINDOW` | unset | `true` extracts each segment's source frames when it starts and deletes them when it closes | #### Render the same composition with different content diff --git a/packages/producer/scripts/integration-tests.json b/packages/producer/scripts/integration-tests.json index 08c5c33794..5fcc1febe8 100644 --- a/packages/producer/scripts/integration-tests.json +++ b/packages/producer/scripts/integration-tests.json @@ -40,6 +40,7 @@ "src/services/render/renderCancel.integration.test.ts", "src/services/render/segmentedColorGrading.integration.test.ts", "src/services/render/variableMountMedia.integration.test.ts", + "src/services/render/windowedFrames.integration.test.ts", "src/services/render/wrappedMountTiming.integration.test.ts", "src/services/render/stages/compileStage.mediaType.test.ts", "src/services/render/stages/encodeWhileCapturing.test.ts", @@ -49,5 +50,6 @@ "src/utils/streamDurationParity.test.ts", "src/services/distributed/referenceBackground.integration.test.ts", "src/services/render/planReferenceBackground.polling.test.ts", - "src/services/render/deferredVideoFrames.integration.test.ts" + "src/services/render/deferredVideoFrames.integration.test.ts", + "src/services/render/stages/extractVideosStage.integration.test.ts" ] diff --git a/packages/producer/src/services/render/deferredVideoFrames.integration.test.ts b/packages/producer/src/services/render/deferredVideoFrames.integration.test.ts index b88b532b61..f85080e088 100644 --- a/packages/producer/src/services/render/deferredVideoFrames.integration.test.ts +++ b/packages/producer/src/services/render/deferredVideoFrames.integration.test.ts @@ -1,10 +1,14 @@ import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "bun:test"; -import type { ExtractedFrames, VideoElement } from "@hyperframes/engine"; -import { extractDeferredFrames } from "./deferredVideoFrames.js"; +import { + extractAllVideoFrames, + type ExtractedFrames, + type VideoElement, +} from "@hyperframes/engine"; +import { extractAllDeferredFrames, extractDeferredFrames } from "./deferredVideoFrames.js"; const metadata = { durationSeconds: 4, @@ -32,6 +36,11 @@ const deferredVideo = (videoId: string, totalFrames: number): ExtractedFrames => deferredRange: { startTime: 1.5, durationSeconds: totalFrames / 30, format: "jpg" }, }); +const fullyExtracted = (videoId: string): ExtractedFrames => { + const { deferredRange: _deferred, ...rest } = deferredVideo(videoId, 90); + return rest; +}; + const element = (id: string, start: number, end: number): VideoElement => ({ id, src: `${id}.mp4`, @@ -95,4 +104,48 @@ describe("extractDeferredFrames", () => { await expect(extraction).rejects.toBe(missingError); expect(reported).toEqual([{ videoId: "clip", index: 45 }]); }); + describe("extractAllDeferredFrames", () => { + const framesOf = (dir: string) => + new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))] as const)); + + it("fills a deferred video in place with the frames a full extraction writes, byte for byte", async () => { + const outputDir = join(root, "all", "clip"); + mkdirSync(outputDir, { recursive: true }); + const video: ExtractedFrames = { + ...deferredVideo("clip", 90), + srcPath: sourcePath, + outputDir, + deferredRange: { startTime: 0, durationSeconds: 3, format: "jpg" }, + }; + const untouched = fullyExtracted("whole"); + untouched.framePaths.set(0, "/kept/frame_00001.jpg"); + const entries = [video, untouched]; + + const progress: Array<[number, number]> = []; + await extractAllDeferredFrames(entries, { + fps, + cfg: {}, + onProgress: (done, total) => progress.push([done, total]), + }); + expect(progress).toEqual([[90, 90]]); + + expect(entries[0]).toBe(video); + expect(video.deferredRange).toBeUndefined(); + expect(video.outputDir).toBe(outputDir); + expect(video.framePaths.size).toBe(90); + expect([...video.framePaths.values()].every((path) => existsSync(path))).toBe(true); + expect(untouched.framePaths.get(0)).toBe("/kept/frame_00001.jpg"); + + const reference = await extractAllVideoFrames( + [{ ...element("ref", 0, 3), src: sourcePath, mediaStart: 0 }], + root, + { fps, outputDir: join(root, "all-reference") }, + ); + const expected = framesOf(reference.extracted[0]!.outputDir); + const actual = framesOf(outputDir); + expect([...actual.keys()]).toEqual([...expected.keys()]); + for (const [name, bytes] of expected) + expect(actual.get(name)?.equals(bytes), name).toBe(true); + }, 60_000); + }); }); diff --git a/packages/producer/src/services/render/deferredVideoFrames.test.ts b/packages/producer/src/services/render/deferredVideoFrames.test.ts index 3074cf3f11..16d7281d66 100644 --- a/packages/producer/src/services/render/deferredVideoFrames.test.ts +++ b/packages/producer/src/services/render/deferredVideoFrames.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "bun:test"; import type { ExtractedFrames, VideoElement } from "@hyperframes/engine"; -import { shownDeferredFrameIndexes } from "./deferredVideoFrames.js"; +import { extractAllDeferredFrames, shownDeferredFrameIndexes } from "./deferredVideoFrames.js"; const metadata = { durationSeconds: 4, @@ -28,6 +28,11 @@ const deferredVideo = (videoId: string, totalFrames: number): ExtractedFrames => deferredRange: { startTime: 1.5, durationSeconds: totalFrames / 30, format: "jpg" }, }); +const fullyExtracted = (videoId: string): ExtractedFrames => { + const { deferredRange: _deferred, ...rest } = deferredVideo(videoId, 90); + return rest; +}; + const element = (id: string, start: number, end: number): VideoElement => ({ id, src: `${id}.mp4`, @@ -57,3 +62,17 @@ describe("shownDeferredFrameIndexes", () => { expect(shown.has("later")).toBe(false); }); }); + +describe("extractAllDeferredFrames", () => { + it("reads nothing when no video is deferred", async () => { + const whole = fullyExtracted("whole"); + await extractAllDeferredFrames([whole], { + fps, + cfg: {}, + sourceOf: () => { + throw new Error("a fully extracted video must not be read again"); + }, + }); + expect(whole.framePaths.size).toBe(0); + }); +}); diff --git a/packages/producer/src/services/render/deferredVideoFrames.ts b/packages/producer/src/services/render/deferredVideoFrames.ts index 08cec3503b..5b19102c98 100644 --- a/packages/producer/src/services/render/deferredVideoFrames.ts +++ b/packages/producer/src/services/render/deferredVideoFrames.ts @@ -1,10 +1,12 @@ -import { join } from "node:path"; +import { mkdirSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; import { createFrameLookupTable, type EngineConfig, type ExtractedFrames, extractVideoFramesRange, type FrameRange, + runVideoExtractionWithRetry, type ShownBox, type VideoElement, } from "@hyperframes/engine"; @@ -96,3 +98,59 @@ export async function extractDeferredFrames(input: { }), ); } + +/** + * Fills every deferred video in place with the frames a full extraction writes, into the folder it + * already has. For a route that cannot read frames per segment. + */ +export async function extractAllDeferredFrames( + extracted: ExtractedFrames[], + input: { + fps: { num: number; den: number }; + cfg: Pick; + signal?: AbortSignal; + sourceOf?: (video: ExtractedFrames) => string; + maxTransientRetries?: number; + onProgress?: (framesDone: number, framesTotal: number) => void; + }, +): Promise { + const framesTotal = extracted.reduce( + (sum, video) => sum + (video.deferredRange ? video.totalFrames : 0), + 0, + ); + let framesDone = 0; + for (const video of extracted) { + const range = video.deferredRange; + if (!range) continue; + const { result: filled } = await runVideoExtractionWithRetry( + () => + extractVideoFramesRange( + input.sourceOf?.(video) ?? video.srcPath, + video.videoId, + range.startTime, + range.durationSeconds, + { fps: input.fps, outputDir: dirname(video.outputDir), format: range.format }, + input.signal, + input.cfg, + ), + { + signal: input.signal, + maxTransientRetries: input.maxTransientRetries, + onRetry: () => { + rmSync(video.outputDir, { recursive: true, force: true }); + mkdirSync(video.outputDir, { recursive: true }); + }, + }, + ); + if (filled.framePaths.size < video.totalFrames) { + throw new Error( + `[Render] video ${JSON.stringify(video.videoId)} has ${filled.framePaths.size} frames; ` + + `it was expected to have ${video.totalFrames} frames.`, + ); + } + video.framePaths = filled.framePaths; + delete video.deferredRange; + framesDone += video.totalFrames; + input.onProgress?.(framesDone, framesTotal); + } +} diff --git a/packages/producer/src/services/render/observability.ts b/packages/producer/src/services/render/observability.ts index 0d39dfead1..442fd28f17 100644 --- a/packages/producer/src/services/render/observability.ts +++ b/packages/producer/src/services/render/observability.ts @@ -209,6 +209,8 @@ export interface RenderCaptureObservability { /** Segmented capture only (Phase 2): current segment and retries so far. */ segmentIndex?: number; segmentRetries?: number; + videoFrameWindow?: "on" | "off"; + videoFrameWindowPeakBytes?: number; } export interface RenderExtractionObservability { diff --git a/packages/producer/src/services/render/segmentVideoFrames.test.ts b/packages/producer/src/services/render/segmentVideoFrames.test.ts index 1dc657db4b..ca74803117 100644 --- a/packages/producer/src/services/render/segmentVideoFrames.test.ts +++ b/packages/producer/src/services/render/segmentVideoFrames.test.ts @@ -142,6 +142,38 @@ describe("createSegmentVideoFrames", () => { expect(served).toContain(join(folder, "clip", "frame_00032.jpg")); }); + it("forFrames extracts just the given frames ahead of time and release removes them", async () => { + const { dir, served, extractor, segmentVideoFrames, page } = setup(); + const frames = await segmentVideoFrames.forFrames([0, 40, 74], "calibration"); + + const folder = join(dir, "segment-frames", "calibration"); + expect(extractor.calls[0]!.outputRoot).toBe(folder); + expect(extractor.calls[0]!.frames).toEqual([0, 40, 74]); + expect(readdirSync(join(folder, "clip")).sort()).toEqual([ + "frame_00001.jpg", + "frame_00041.jpg", + "frame_00075.jpg", + ]); + + await frames.injector()!(page, 40 / 30); + expect(served).toContain(join(folder, "clip", "frame_00041.jpg")); + + frames.release(); + expect(existsSync(folder)).toBe(false); + expect(() => frames.release()).not.toThrow(); + }); + + it("forFrames leaves no folder behind when its extraction fails", async () => { + const { dir, segmentVideoFrames } = setup({ + extractFrames: async (input) => { + mkdirSync(join(input.outputRoot, "clip"), { recursive: true }); + throw new Error("aborted"); + }, + }); + await expect(segmentVideoFrames.forFrames([0], "calibration")).rejects.toThrow("aborted"); + expect(existsSync(join(dir, "segment-frames", "calibration"))).toBe(false); + }); + it("release removes the folder and a second release does nothing", async () => { const { dir, workerFrames } = setup(); await workerFrames.prepare(slice(2, 60, 75)); diff --git a/packages/producer/src/services/render/segmentVideoFrames.ts b/packages/producer/src/services/render/segmentVideoFrames.ts index 4117b98e29..d3377edc3f 100644 --- a/packages/producer/src/services/render/segmentVideoFrames.ts +++ b/packages/producer/src/services/render/segmentVideoFrames.ts @@ -28,6 +28,11 @@ export interface SegmentWorkerFrames { export interface SegmentVideoFrames { forWorker(): SegmentWorkerFrames; + /** Extracts the given output frames into `segment-frames//` ahead of time; `release()` is idempotent. */ + forFrames( + frames: readonly number[], + name: string, + ): Promise<{ injector(): BeforeCaptureHook | null; release(): void }>; } type FrameExtractor = typeof extractDeferredFrames; @@ -53,8 +58,8 @@ export function createSegmentVideoFrames(input: { const sourceOf = input.sourceOf ?? ((video: ExtractedFrames) => video.srcPath); const fullyExtracted = input.extracted.filter((video) => !video.deferredRange); - /** Extracts the given output frames into `dir` and returns an injector over them plus the fully extracted videos. */ - const injectorFor = async (frames: number[], dir: string): Promise => { + /** Extracts the given output frames into `dir` and returns a lookup over them plus the fully extracted videos. */ + const tableFor = async (frames: number[], dir: string) => { const sliced = await extractFrames({ videos: input.videos, extracted: input.extracted, @@ -70,13 +75,14 @@ export function createSegmentVideoFrames(input: { `it was expected to have ${video.totalFrames} frames.`, ), }); - const table = createFrameLookupTable( + return createFrameLookupTable( input.videos, [...fullyExtracted, ...sliced], resolveRenderFpsConfig(input.fps).value, ); - return createVideoFrameInjector(table, input.injectorOptions); }; + const injectorFor = async (frames: number[], dir: string): Promise => + createVideoFrameInjector(await tableFor(frames, dir), input.injectorOptions); /** * The drawElement self-verify runs the hook at session open and after every @@ -106,6 +112,19 @@ export function createSegmentVideoFrames(input: { ); return { + async forFrames(frames, name) { + const dir = join(input.rootDir, "segment-frames", name); + const release = () => rmSync(dir, { recursive: true, force: true }); + try { + release(); + mkdirSync(dir, { recursive: true }); + const table = await tableFor([...frames], dir); + return { injector: () => createVideoFrameInjector(table, input.injectorOptions), release }; + } catch (error) { + release(); + throw error; + } + }, forWorker() { let injector: BeforeCaptureHook | null = null; let folder: string | null = null; diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts index 6947a2319b..13a1842a1b 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts @@ -1339,6 +1339,52 @@ describe("runCaptureSegmentedStage with segmentVideoFrames", () => { expect(removed).toEqual([]); }); + it("with keepSegments, a windowed render keeps its segment files", async () => { + const removed: string[] = []; + const input = fakeStageInput({ totalFrames: 4 }); + + const result = await runCaptureSegmentedStage({ + ...input, + job: { ...input.job, config: { ...input.job.config, keepSegments: true } }, + segmentFrames: 2, + segmentDir: join(fixtureRoot, "windowed-keep-segments"), + segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames, + deps: stageDeps({ + spawnEncoder: loggingEncoder([]), + removeFile: (path: string) => removed.push(path), + }), + }); + + expect(result.success).toBe(true); + expect(removed).toEqual([]); + }); + + it("a segment file that will not delete is logged and does not fail the stage", async () => { + const attempted: string[] = []; + const warnings: string[] = []; + const segmentDir = join(fixtureRoot, "windowed-remove-throws"); + const input = fakeStageInput({ totalFrames: 6 }); + + const result = await runCaptureSegmentedStage({ + ...input, + log: { ...input.log, warn: (message: string) => void warnings.push(message) }, + segmentFrames: 2, + segmentDir, + segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames, + deps: stageDeps({ + spawnEncoder: loggingEncoder([]), + removeFile: (path: string) => { + attempted.push(path); + if (path === segmentOutputPath(segmentDir, 0)) throw new Error("EPERM"); + }, + }), + }); + + expect(result.success).toBe(true); + expect(attempted).toEqual([0, 1, 2].map((i) => segmentOutputPath(segmentDir, i))); + expect(warnings).toEqual(["[Render] could not remove a joined segment file"]); + }); + it("without segmentVideoFrames, segment files are kept", async () => { const removed: string[] = []; diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.ts index c5d4de6fb0..bbe60ec348 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.ts @@ -386,10 +386,22 @@ async function openSegmentedSession( ); } -/** Once joined, the windowed route has no use for the segments; keeping them would hold the whole video twice. */ +/** + * Joined segments would hold the video twice, so the windowed route deletes them unless `--keep-segments`. + * A file that will not delete costs disk, not the finished render: log it and try the rest. + */ function removeJoinedSegments(run: SegmentRun, segmentPaths: readonly string[]): void { - if (!run.input.segmentVideoFrames) return; - for (const path of segmentPaths) run.deps.removeFile(path); + if (!run.input.segmentVideoFrames || run.input.job.config.keepSegments === true) return; + for (const path of segmentPaths) { + try { + run.deps.removeFile(path); + } catch (err) { + run.input.log.warn("[Render] could not remove a joined segment file", { + path, + error: err instanceof Error ? err.message : String(err), + }); + } + } } /** Stream-copy the finished segments into the video-only file, or throw. */ diff --git a/packages/producer/src/services/render/stages/extractVideosStage.integration.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.integration.test.ts new file mode 100644 index 0000000000..6773673cdc --- /dev/null +++ b/packages/producer/src/services/render/stages/extractVideosStage.integration.test.ts @@ -0,0 +1,96 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { VideoElement } from "@hyperframes/engine"; +import { resolveConfig } from "@hyperframes/engine"; +import { runExtractVideosStage } from "./extractVideosStage.js"; +import { createRenderJob } from "../../renderOrchestrator.js"; + +describe("deferring range extraction beside a variable-frame-rate source", () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-defer-vfr-")); + const synthesize = (name: string, input: string, filters: string[] = []) => { + const result = spawnSync("ffmpeg", [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + input, + ...filters, + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + join(projectDir, name), + ]); + if (result.status !== 0) throw new Error(result.stderr.toString()); + }; + + beforeAll(() => { + synthesize("cfr.mp4", "testsrc2=s=64x36:r=30:d=6"); + synthesize("vfr.mp4", "testsrc2=s=64x36:d=10:rate=1/2", [ + "-vf", + "select='eq(n,0)+eq(n,2)'", + "-fps_mode", + "vfr", + ]); + }, 30_000); + afterAll(() => rmSync(projectDir, { recursive: true, force: true })); + + const clip = (id: string): VideoElement => ({ + id, + src: `${id}.mp4`, + start: 0, + end: 3, + mediaStart: 0, + loop: false, + hasAudio: false, + }); + + const extract = (deferRangeExtraction: boolean | undefined) => { + const compiledDir = join(projectDir, `compiled-${String(deferRangeExtraction)}`); + mkdirSync(compiledDir, { recursive: true }); + return runExtractVideosStage({ + projectDir, + compiledDir, + job: createRenderJob({ fps: 30, quality: "standard", hdrMode: "force-sdr" }), + cfg: resolveConfig({}), + composition: { + duration: 3, + width: 64, + height: 36, + videos: [clip("cfr"), clip("vfr")], + audios: [], + images: [], + }, + abortSignal: undefined, + assertNotAborted: () => {}, + deferRangeExtraction, + }); + }; + + it("a VFR source beside a CFR one: the CFR source defers and the VFR source extracts in full", async () => { + const { extractionResult } = await extract(true); + const byId = new Map(extractionResult!.extracted.map((video) => [video.videoId, video])); + + const cfr = byId.get("cfr")!; + expect(cfr.deferredRange).toBeDefined(); + expect(cfr.framePaths.size).toBe(0); + expect(cfr.totalFrames).toBe(90); + + const vfr = byId.get("vfr")!; + expect(vfr.metadata.isVFR).toBe(true); + expect(vfr.deferredRange).toBeUndefined(); + expect(vfr.framePaths.size).toBeGreaterThan(0); + }, 60_000); + + it("extracts every source in full when range extraction is not deferred", async () => { + const { extractionResult } = await extract(undefined); + expect(extractionResult!.extracted.some((video) => video.deferredRange)).toBe(false); + const cfr = extractionResult!.extracted.find((video) => video.videoId === "cfr")!; + expect(cfr.framePaths.size).toBe(90); + }, 60_000); +}); diff --git a/packages/producer/src/services/render/videoFrameCoverage.test.ts b/packages/producer/src/services/render/videoFrameCoverage.test.ts index 1af2d990f7..4332e77f0e 100644 --- a/packages/producer/src/services/render/videoFrameCoverage.test.ts +++ b/packages/producer/src/services/render/videoFrameCoverage.test.ts @@ -229,6 +229,18 @@ describe("computeVideoFrameCoverage", () => { }); }); + it("coverage does not report a deferred video as uncovered", () => { + const video = makeVideo({ id: "long", start: 0, end: 10 }); + const deferred: ExtractedFrames = { + ...makeExtracted("long", 0, { durationSeconds: 10 }), + totalFrames: 300, + deferredRange: { startTime: 0, durationSeconds: 10, format: "jpg" }, + }; + const [report] = computeVideoFrameCoverage([video], [deferred], 30); + expect(report).toMatchObject({ capturedFrames: 300, expectedFrames: 300, ratio: 1 }); + expect(() => assertVideoFrameCoverage([report!], 0.95)).not.toThrow(); + }); + it("uses delivered framePaths.size, not the possibly-stale totalFrames field", () => { const videos = [makeVideo({ id: "a", start: 0, end: 1 })]; // Simulate an extractor whose totalFrames reports 30 but only 5 frames diff --git a/packages/producer/src/services/render/videoFrameCoverage.ts b/packages/producer/src/services/render/videoFrameCoverage.ts index a64dc88703..8bac1c296a 100644 --- a/packages/producer/src/services/render/videoFrameCoverage.ts +++ b/packages/producer/src/services/render/videoFrameCoverage.ts @@ -205,11 +205,15 @@ export function computeVideoFrameCoverage( for (const video of videos) { const entry = byId.get(video.id); const expectedFrames = expectedFramesForVideo(video, entry, fps); - // framePaths is a Map — `size` is the number of distinct captured frames - // delivered to the runtime injector, which is the load-bearing count - // (some extractors report a total that includes cache-hit-skipped frames - // via a stale `totalFrames`, so we trust the delivered-path count). - const capturedFrames = entry ? entry.framePaths.size : 0; + // framePaths is a Map — `size` is the number of distinct captured frames delivered to the + // runtime injector (some extractors report a stale `totalFrames` that includes cache-hit-skipped + // frames, so we trust the delivered-path count). + // A deferred video has none yet by design; its `totalFrames` is exact. + const capturedFrames = entry + ? entry.deferredRange + ? entry.totalFrames + : entry.framePaths.size + : 0; const ratio = expectedFrames === 0 ? 1 : capturedFrames / expectedFrames; reports.push({ videoId: video.id, diff --git a/packages/producer/src/services/render/videoFrameWindow.test.ts b/packages/producer/src/services/render/videoFrameWindow.test.ts new file mode 100644 index 0000000000..3230d81643 --- /dev/null +++ b/packages/producer/src/services/render/videoFrameWindow.test.ts @@ -0,0 +1,55 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "bun:test"; +import type { SegmentVideoFrames } from "./segmentVideoFrames.js"; +import { trackSegmentFramePeak } from "./videoFrameWindow.js"; + +const root = mkdtempSync(join(tmpdir(), "hf-video-frame-window-")); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +/** Writes `bytes` into the segment's folder on prepare and deletes it on release, as the real frames do. */ +function framesOfSize(rootDir: string, bytes: number): SegmentVideoFrames { + return { + forWorker() { + let folder: string | null = null; + return { + hook: Object.assign(async () => {}, {}), + async prepare(segment) { + folder = join(rootDir, "segment-frames", `segment_${segment.index}`, "clip"); + mkdirSync(folder, { recursive: true }); + writeFileSync(join(folder, "frame_00001.jpg"), Buffer.alloc(bytes)); + }, + release() { + if (folder) rmSync(join(folder, ".."), { recursive: true, force: true }); + }, + }; + }, + }; +} + +describe("trackSegmentFramePeak", () => { + it("reports the most bytes segment-frames held at once, across workers", async () => { + const dir = mkdtempSync(join(root, "work-")); + const peaks: number[] = []; + const tracked = trackSegmentFramePeak(framesOfSize(dir, 1000), dir, (bytes) => + peaks.push(bytes), + ); + const a = tracked.forWorker(); + const b = tracked.forWorker(); + + await a.prepare({ index: 0, startFrame: 0, endFrame: 10 }); + await b.prepare({ index: 1, startFrame: 10, endFrame: 20 }); + a.release(); + await a.prepare({ index: 2, startFrame: 20, endFrame: 30 }); + + expect(peaks).toEqual([1000, 2000]); + }); + + it("reports nothing before any segment lands", () => { + const dir = mkdtempSync(join(root, "work-")); + const peaks: number[] = []; + trackSegmentFramePeak(framesOfSize(dir, 1), dir, (bytes) => peaks.push(bytes)).forWorker(); + expect(peaks).toEqual([]); + }); +}); diff --git a/packages/producer/src/services/render/videoFrameWindow.ts b/packages/producer/src/services/render/videoFrameWindow.ts new file mode 100644 index 0000000000..49f653eb09 --- /dev/null +++ b/packages/producer/src/services/render/videoFrameWindow.ts @@ -0,0 +1,60 @@ +import { readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import type { SegmentVideoFrames } from "./segmentVideoFrames.js"; + +/** + * Whether source-video frames are read one capture segment at a time. Off unless + * `HF_VIDEO_FRAME_WINDOW` reads `true`, and only for a render that segments, because only + * the segmented stage knows which frames a segment shows and when it is done with them. + */ +export function shouldWindowVideoFrames(args: { + env: Readonly>; + segmented: boolean; +}): boolean { + return args.segmented && args.env.HF_VIDEO_FRAME_WINDOW?.trim().toLowerCase() === "true"; +} + +function directoryBytes(path: string): number { + let bytes = 0; + let entries; + try { + entries = readdirSync(path, { withFileTypes: true }); + } catch { + return 0; + } + for (const entry of entries) { + const child = join(path, entry.name); + try { + bytes += entry.isDirectory() ? directoryBytes(child) : statSync(child).size; + } catch { + // A segment closing on another worker removes its files mid-walk. + } + } + return bytes; +} + +/** Reports the most bytes `segment-frames` has held at once, sampled each time a segment's frames land. */ +export function trackSegmentFramePeak( + frames: SegmentVideoFrames, + rootDir: string, + onPeak: (bytes: number) => void, +): SegmentVideoFrames { + let peak = 0; + return { + forFrames: (frameIndexes, name) => frames.forFrames(frameIndexes, name), + forWorker() { + const worker = frames.forWorker(); + return { + ...worker, + async prepare(segment) { + await worker.prepare(segment); + const held = directoryBytes(join(rootDir, "segment-frames")); + if (held > peak) { + peak = held; + onPeak(peak); + } + }, + }; + }, + }; +} diff --git a/packages/producer/src/services/render/windowedFrames.integration.test.ts b/packages/producer/src/services/render/windowedFrames.integration.test.ts new file mode 100644 index 0000000000..7326699f20 --- /dev/null +++ b/packages/producer/src/services/render/windowedFrames.integration.test.ts @@ -0,0 +1,572 @@ +// Windowed source-video frames (HF_VIDEO_FRAME_WINDOW), rendered for real: each case renders twice +// on the segmented route, once reading a segment's frames at a time and once extracting whole +// videos, and the two outputs must match frame for frame. Needs Chrome + ffmpeg: integration lane. +// Known gap: this test pins `browserGpuMode: "software"`, so every render here takes the screenshot +// path. Windowed drawElement output parity was measured by hand on real footage and has no automated +// guard. +// +// HF_TEST_RETRY_BROWSER_STALL=1 renders once more after Chrome dies, for an overloaded local host. +// Never set it in CI: a windowed render that needs the retry alone fails the case. + +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { resolveConfig, type MotionBlurOptions } from "@hyperframes/engine"; +import { + createRenderJob, + executeRenderJob, + resolveRenderWorkDirPrefix, + type RenderJob, +} from "../renderOrchestrator.js"; +import { FFMPEG, HAS_FFMPEG } from "./frameLevelsTestFixtures.js"; + +const WIDTH = 160; +const HEIGHT = 90; +const SEGMENT_FRAMES = 60; +const LOOSENESS = 1.2; + +interface Clip { + id: string; + src: string; + start: number; + duration: number; + mediaStart?: number; + loop?: boolean; + /** A `data-automation` rate lane, as the speed-ramp regression fixture writes one. */ + rate?: Array<{ t: number; v: number }>; + /** Left and width in px; the whole frame when absent. */ + box?: { left: number; width: number }; +} + +interface Case { + name: string; + fps: { num: number; den: number }; + fpsAttribute: string; + seconds: number; + clips: Clip[]; + /** Absent: the orchestrator picks, and measures capture speed first. */ + workers?: number; + motionBlur?: MotionBlurOptions; + /** Whether the windowed render is expected to read frames a segment at a time. */ + windowed: boolean; + /** Frames of the videos that are never deferred, which the windowed render still extracts whole. */ + upFrontFrames?: number; + /** Set when the orchestrator measures capture speed before it captures. */ + calibrates?: boolean; + /** Caps the worker count the orchestrator picks, to keep the Chromes it opens few. */ + maxWorkers?: number; +} + +interface Run { + frames: string[]; + mode: string; + capturePath: string | undefined; + videoFrameWindow: string | undefined; + workerCount: number | undefined; + /** The render died on Chrome and was rendered again (HF_TEST_RETRY_BROWSER_STALL). */ + retried: boolean; + peakBytes: number | undefined; + extractedFrames: number | undefined; + calibration: string; + /** Windowed renders only: what a poll of `segment-frames` saw while the render ran. */ + sampled: { peakBytes: number; largestFileBytes: number }; + /** `keepWorkDir` renders only: the entries left in `segment-frames` once the render closed. */ + leftInSegmentFrames: string[] | undefined; +} + +let root: string; + +function ffmpeg(args: string[]): void { + const result = spawnSync(FFMPEG, ["-v", "error", "-y", ...args], { encoding: "utf-8" }); + if (result.status !== 0) throw new Error(`fixture failed: ${result.stderr}`); +} + +/** testsrc2 changes every frame, so a window off by one frame shows. */ +function makeSource(name: string, rate: string, seconds: number, filter?: string): void { + const path = join(root, "assets", name); + ffmpeg([ + "-f", + "lavfi", + "-i", + `testsrc2=size=${WIDTH}x${HEIGHT}:rate=${rate}:duration=${seconds}`, + ...(filter ? ["-vf", filter, "-fps_mode", "vfr"] : []), + "-pix_fmt", + "yuv420p", + "-c:v", + "libx264", + "-g", + "1", + path, + ]); +} + +function composition(testCase: Case): string { + const videos = testCase.clips.map((clip, track) => { + const box = clip.box ?? { left: 0, width: WIDTH }; + const attributes = [ + `id="${clip.id}"`, + `class="clip"`, + `src="${clip.src}"`, + `data-start="${clip.start}"`, + `data-duration="${clip.duration}"`, + `data-track-index="${track}"`, + clip.mediaStart === undefined ? "" : `data-media-start="${clip.mediaStart}"`, + clip.loop ? "loop" : "", + clip.rate + ? `data-automation='${JSON.stringify({ version: 1, lanes: [{ target: "rate", points: clip.rate }] })}'` + : "", + `style="left:${box.left}px;width:${box.width}px"`, + "muted playsinline", + ]; + return ` `; + }); + return ` + + + + + + +
+${videos.join("\n")} +
+ + +`; +} + +function framemd5(path: string): string[] { + const result = spawnSync(FFMPEG, ["-v", "error", "-i", path, "-f", "framemd5", "-"], { + encoding: "utf-8", + maxBuffer: 1 << 26, + }); + if (result.status !== 0) throw new Error(`framemd5 failed: ${result.stderr}`); + return result.stdout.split("\n").filter((line) => line !== "" && !line.startsWith("#")); +} + +function directoryBytes(path: string, onFile: (bytes: number) => void): number { + let bytes = 0; + let entries; + try { + entries = readdirSync(path, { withFileTypes: true }); + } catch { + return 0; + } + for (const entry of entries) { + const child = join(path, entry.name); + try { + if (entry.isDirectory()) bytes += directoryBytes(child, onFile); + else { + const size = statSync(child).size; + bytes += size; + onFile(size); + } + } catch { + // A segment closing removes its files mid-walk. + } + } + return bytes; +} + +function firstDifference(left: string[], right: string[]): number { + const length = Math.max(left.length, right.length); + for (let i = 0; i < length; i++) if (left[i] !== right[i]) return i; + return -1; +} + +const ENV_KEYS = [ + "HF_SEGMENTED_CAPTURE", + "HF_SEGMENT_FRAMES", + "HF_VIDEO_FRAME_WINDOW", + "KEEP_TEMP", +] as const; + +/** Sets the render's env and returns the call that puts the caller's back. */ +function setRenderEnv(windowed: boolean, keepWorkDir: boolean | undefined): () => void { + const saved = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]])); + process.env.HF_SEGMENTED_CAPTURE = "true"; + process.env.HF_SEGMENT_FRAMES = String(SEGMENT_FRAMES); + if (windowed) process.env.HF_VIDEO_FRAME_WINDOW = "true"; + else delete process.env.HF_VIDEO_FRAME_WINDOW; + if (keepWorkDir) process.env.KEEP_TEMP = "1"; + else delete process.env.KEEP_TEMP; + return () => { + for (const key of ENV_KEYS) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + }; +} + +function ownedByThisProcess(dir: string): boolean { + try { + return JSON.parse(readFileSync(join(dir, "owner.json"), "utf-8")).pid === process.pid; + } catch { + return false; + } +} + +/** + * This render's work dir, where the orchestrator puts it on this platform: named by the job on + * Linux, and a shared `hf-render-` prefix in the system temp dir elsewhere. The owner record's pid + * rules out another process's render; renders in this process run serially and each snapshots the + * directory before it starts, which is what tells one of them from another. + */ +function watchWorkDirs(output: string, jobId: string) { + const prefix = resolveRenderWorkDirPrefix(output, jobId); + const parent = dirname(prefix); + const before = new Set(readdirSync(parent)); + const own = () => + readdirSync(parent) + .filter((name) => name.startsWith(basename(prefix)) && !before.has(name)) + .map((name) => join(parent, name)) + .filter(ownedByThisProcess); + return { own }; +} + +/** Polls `segment-frames` in the render's work dir for the most it holds at once. */ +function sampleSegmentFrames(own: () => string[]) { + const sampled = { peakBytes: 0, largestFileBytes: 0 }; + const poll = setInterval(() => { + for (const workDir of own()) { + const held = directoryBytes(join(workDir, "segment-frames"), (bytes) => { + sampled.largestFileBytes = Math.max(sampled.largestFileBytes, bytes); + }); + sampled.peakBytes = Math.max(sampled.peakBytes, held); + } + }, 50); + return { sampled, stop: () => clearInterval(poll) }; +} + +// Flat field mapping: the `?.` and `??` are fallbacks for a summary that may lack a field, not logic. +// fallow-ignore-next-line complexity +function describeRun( + job: RenderJob, + output: string, + trace: Array>, +): Omit { + const perf = job.perfSummary; + const capture = perf?.observability?.capture; + const calibration = trace.find( + (event) => event.phase === "capture_calibration" && event.status === "checkpoint", + ); + return { + frames: framemd5(output), + mode: [ + capture?.captureMode, + `drawElement=${perf?.drawElement?.mode ?? "none"}`, + `forceScreenshot=${String(capture?.forceScreenshot)}`, + `selfVerifyFallback=${String(perf?.drawElement?.selfVerifyFallback ?? false)}`, + ].join(" "), + capturePath: capture?.capturePath, + videoFrameWindow: capture?.videoFrameWindow, + workerCount: capture?.workerCount, + retried: false, + peakBytes: capture?.videoFrameWindowPeakBytes, + extractedFrames: perf?.observability?.extraction?.totalFramesExtracted, + calibration: String(calibration?.message ?? "not reached"), + }; +} + +const RETRY_BROWSER_DEATH = process.env.HF_TEST_RETRY_BROWSER_STALL === "1"; +const BROWSER_DEATH = /Connection closed|Chrome started and kept running/; + +interface RenderOptions { + /** Leaves the work dir behind (KEEP_TEMP) and reports what `segment-frames` held at the end. */ + keepWorkDir?: boolean; + /** Polls `segment-frames` while the render runs. */ + sampleFolder?: boolean; +} + +/** A render; with HF_TEST_RETRY_BROWSER_STALL=1, rendered once more if Chrome died under it. */ +async function render( + testCase: Case, + windowed: boolean, + options: RenderOptions = {}, +): Promise { + try { + return await renderOnce(testCase, windowed, options); + } catch (error) { + if (!RETRY_BROWSER_DEATH || !BROWSER_DEATH.test(String(error))) throw error; + console.log(`[windowedFrames] ${String(error).slice(0, 160)}; rendering it again`); + return { ...(await renderOnce(testCase, windowed, options)), retried: true }; + } +} + +function createJob(testCase: Case, trace: Array>): RenderJob { + return createRenderJob({ + fps: testCase.fps, + quality: "draft", + format: "mp4", + hdrMode: "force-sdr", + workers: testCase.workers, + motionBlur: testCase.motionBlur, + logger: { + info(message: string, meta?: Record) { + if (message === "[Render:trace]" && meta) trace.push(meta); + }, + warn() {}, + error() {}, + debug() {}, + }, + producerConfig: resolveConfig({ + browserGpuMode: "software", + concurrency: testCase.maxWorkers ?? "auto", + }), + }); +} + +const NO_SAMPLER = { sampled: { peakBytes: 0, largestFileBytes: 0 }, stop: () => {} }; + +/** What a kept work dir's `segment-frames` still holds, or undefined when there is no such folder. */ +function leftInSegmentFrames(workDirs: string[]): string[] | undefined { + const folder = workDirs.map((workDir) => join(workDir, "segment-frames")).find(existsSync); + return folder ? readdirSync(folder) : undefined; +} + +/** Renders the case on the segmented route with or without the window. */ +async function renderOnce(testCase: Case, windowed: boolean, options: RenderOptions): Promise { + const label = `${testCase.name.replace(/\W+/g, "-")}-${windowed ? "windowed" : "full"}`; + writeFileSync(join(root, "index.html"), composition(testCase)); + const trace: Array> = []; + const job = createJob(testCase, trace); + const restoreEnv = setRenderEnv(windowed, options.keepWorkDir); + const output = join(root, `${label}.mp4`); + const { own } = watchWorkDirs(output, job.id); + const sampler = options.sampleFolder ? sampleSegmentFrames(own) : NO_SAMPLER; + const kept: string[] = []; + try { + await executeRenderJob(job, root, output).catch((error: unknown) => { + throw new Error(`${label} failed: ${String(error)}`, { cause: error }); + }); + if (options.keepWorkDir) kept.push(...own()); + return { + ...describeRun(job, output, trace), + sampled: sampler.sampled, + leftInSegmentFrames: leftInSegmentFrames(kept), + }; + } finally { + sampler.stop(); + restoreEnv(); + for (const workDir of kept) rmSync(workDir, { recursive: true, force: true }); + } +} + +const rendered = new Map>(); + +/** Both renders of a case, once per run of this file. Serial: Chrome is the machine's to share. */ +function renderPair(testCase: Case): Promise<{ windowed: Run; full: Run }> { + let pair = rendered.get(testCase.name); + if (!pair) { + pair = (async () => { + const full = await render(testCase, false); + const windowed = await render(testCase, true, { + keepWorkDir: testCase.name === PEAK_CASE, + sampleFolder: testCase.name === PEAK_CASE, + }); + if (windowed.retried && !full.retried) { + throw new Error(`${testCase.name}: only the windowed render needed a retry`); + } + console.log( + `[windowedFrames] ${testCase.name}: retried windowed=${windowed.retried} full=${full.retried}; mode=${windowed.mode} (full: ${full.mode}); ` + + `window=${windowed.videoFrameWindow}; workers=${windowed.workerCount}; calibration=${windowed.calibration}; ` + + `frames=${windowed.frames.length}; peak=${windowed.peakBytes ?? "n/a"} B; ` + + `extracted frames windowed=${windowed.extractedFrames} full=${full.extractedFrames}`, + ); + return { windowed, full }; + })(); + rendered.set(testCase.name, pair); + } + return pair; +} + +const PEAK_CASE = "2 workers"; +// Four segments for two workers, so the window has something to hold back. +const PEAK_SECONDS = 8; +const CFR = "assets/cfr.mp4"; + +function cases(): Case[] { + const thirty = { num: 30, den: 1 }; + const base = { fps: thirty, fpsAttribute: "30", workers: 1, windowed: true }; + return [ + { + ...base, + name: "a mid-clip start", + seconds: 5, + clips: [{ id: "clip", src: CFR, start: 1.2, duration: 3, mediaStart: 2 }], + }, + { + ...base, + name: "NTSC 24000/1001", + fps: { num: 24000, den: 1001 }, + fpsAttribute: "24000/1001", + seconds: 5, + clips: [{ id: "clip", src: "assets/ntsc.mp4", start: 0.5, duration: 3.5, mediaStart: 0.7 }], + }, + { + ...base, + name: "a loop crossing a slice edge", + seconds: 5.5, + clips: [{ id: "clip", src: "assets/short.mp4", start: 0, duration: 5.5, loop: true }], + }, + { + ...base, + name: "a playback-rate ramp", + seconds: 3, + clips: [ + { + id: "clip", + src: CFR, + start: 0, + duration: 3, + rate: [ + { t: 0, v: 1 }, + { t: 1.5, v: 3 }, + { t: 3, v: 0.5 }, + ], + }, + ], + }, + { + ...base, + name: "a held tail", + seconds: 4, + clips: [{ id: "clip", src: "assets/short.mp4", start: 0, duration: 4 }], + }, + { + ...base, + name: "two clips of one file in different slices", + seconds: 5, + clips: [ + { id: "first", src: CFR, start: 0, duration: 2 }, + { id: "second", src: CFR, start: 3, duration: 2, mediaStart: 3 }, + ], + }, + { + ...base, + name: "a VFR source beside a CFR one", + upFrontFrames: 90, + seconds: 3, + clips: [ + { id: "vfr", src: "assets/vfr.mp4", start: 0, duration: 3, box: { left: 0, width: 80 } }, + { id: "cfr", src: CFR, start: 0, duration: 3, box: { left: 80, width: 80 } }, + ], + }, + { + ...base, + name: PEAK_CASE, + workers: 2, + seconds: PEAK_SECONDS, + clips: [{ id: "clip", src: "assets/long.mp4", start: 0, duration: PEAK_SECONDS }], + }, + { + ...base, + name: "motion blur, which declines the window", + motionBlur: { samplesPerFrame: 2 }, + windowed: false, + seconds: 3, + clips: [{ id: "clip", src: CFR, start: 0, duration: 3 }], + }, + { + ...base, + name: "the orchestrator choosing workers", + workers: undefined, + calibrates: true, + maxWorkers: 2, + seconds: 4, + clips: [{ id: "clip", src: CFR, start: 0, duration: 4 }], + }, + ]; +} + +/** What the windowed render read up front, and whether capture calibration ran, against the case. */ +function expectWindowing(testCase: Case, windowed: Run, full: Run): void { + if (testCase.windowed) { + expect(windowed.peakBytes ?? 0).toBeGreaterThan(0); + // Deferred videos are left out of the count, so a window that deferred nothing would tie. + expect(windowed.extractedFrames ?? 0).toBeLessThan(full.extractedFrames ?? 0); + // What is still extracted up front is exactly the videos that cannot be deferred. + expect(windowed.extractedFrames).toBe(testCase.upFrontFrames ?? 0); + } + if (testCase.calibrates) { + expect(windowed.calibration).toBe("resolved"); + expect(full.calibration).toBe("resolved"); + } +} + +describe.skipIf(!HAS_FFMPEG)("windowed source-video frames — real render", () => { + beforeAll(() => { + root = mkdtempSync(join(tmpdir(), "hf-windowed-frames-")); + mkdirSync(join(root, "assets"), { recursive: true }); + makeSource("cfr.mp4", "30", 6); + makeSource("short.mp4", "30", 2.5); + makeSource("long.mp4", "30", PEAK_SECONDS); + makeSource("ntsc.mp4", "24000/1001", 5); + // Half the frames dropped in each second keeps the timestamps' gaps: r_frame_rate 30, average 15. + makeSource("vfr.mp4", "30", 6, "select='lt(mod(n\\,30)\\,15)'"); + }); + + afterAll(() => { + if (root) rmSync(root, { recursive: true, force: true }); + }); + + for (const testCase of cases()) { + it(`renders ${testCase.name} frame for frame like a full extraction`, async () => { + const { windowed, full } = await renderPair(testCase); + + const difference = firstDifference(windowed.frames, full.frames); + expect(difference === -1 ? "identical" : `first differs at output frame ${difference}`).toBe( + "identical", + ); + expect(windowed.frames).toEqual(full.frames); + expect(full.frames.length).toBeGreaterThan(30); + + for (const run of [windowed, full]) expect(run.capturePath).toBe("segmented"); + expect(full.videoFrameWindow).toBe("off"); + expect(windowed.videoFrameWindow).toBe(testCase.windowed ? "on" : "off"); + // A fall back to a different capture mode is the failure a parity pass would hide. + expect(windowed.mode).toBe(full.mode); + expectWindowing(testCase, windowed, full); + }, 600_000); + } + + it("never holds more than workers x segment frames of source frames", async () => { + const testCase = cases().find((candidate) => candidate.name === PEAK_CASE)!; + const { windowed } = await renderPair(testCase); + const { sampled } = windowed; + + // A poll that saw no frame file would pass every bound below. + expect(sampled.largestFileBytes).toBeGreaterThan(0); + // The count the render ran with, which a clamp could have lowered from the one asked for. + expect(windowed.workerCount).toBe(testCase.workers); + const bound = windowed.workerCount! * SEGMENT_FRAMES * sampled.largestFileBytes * LOOSENESS; + expect(sampled.peakBytes).toBeGreaterThan(0); + expect(sampled.peakBytes).toBeLessThanOrEqual(bound); + expect(windowed.peakBytes ?? 0).toBeGreaterThan(0); + expect(windowed.peakBytes!).toBeLessThanOrEqual(bound); + // The whole clip is what a full extraction holds; the bound must sit below it to mean anything. + expect(bound).toBeLessThan(PEAK_SECONDS * 30 * sampled.largestFileBytes); + expect(windowed.leftInSegmentFrames).toEqual([]); + console.log( + `[windowedFrames] ${PEAK_CASE}: sampled peak=${sampled.peakBytes} B, ` + + `reported peak=${windowed.peakBytes} B, largest frame file=${sampled.largestFileBytes} B, ` + + `bound=${bound} B`, + ); + }, 600_000); +}); diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index e00e5963e3..de892d005f 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -73,6 +73,7 @@ import { createRenderJob, RenderQualityError, } from "./renderOrchestrator.js"; +import { shouldWindowVideoFrames } from "./render/videoFrameWindow.js"; import { probeRequiresBrowser } from "./render/stages/probeStage.js"; import { EncoderInterruptedError } from "./render/encoderInterruption.js"; import { ensureFrameWritten } from "./render/stages/captureHdrFrameShared.js"; @@ -3600,6 +3601,33 @@ describe("resolveParallelCaptureMode", () => { }); }); +describe("shouldWindowVideoFrames", () => { + const segmentsAt = (env: Record, outputFormat: string) => + shouldSegmentCapture({ + env: { HF_SEGMENTED_CAPTURE: "true", ...env }, + durationSeconds: 900, + outputFormat, + layeredOrEffectRoute: false, + streamingOk: true, + }); + const windows = (env: Record, outputFormat = "mp4") => + shouldWindowVideoFrames({ env, segmented: segmentsAt(env, outputFormat) }); + + it("windows only when the flag is true and the render segments", () => { + expect(windows({ HF_VIDEO_FRAME_WINDOW: "true" })).toBe(true); + expect(windows({ HF_VIDEO_FRAME_WINDOW: " TRUE " }, "mov")).toBe(true); + for (const off of ["false", "0", "off", "no", "", "1", "yes", undefined]) { + expect(windows({ HF_VIDEO_FRAME_WINDOW: off })).toBe(false); + } + // Excluded outputs never segment, so the flag has nothing to window. + expect(windows({ HF_VIDEO_FRAME_WINDOW: "true" }, "webm")).toBe(false); + expect(windows({ HF_VIDEO_FRAME_WINDOW: "true" }, "png-sequence")).toBe(false); + expect( + shouldWindowVideoFrames({ env: { HF_VIDEO_FRAME_WINDOW: "true" }, segmented: false }), + ).toBe(false); + }); +}); + describe("shouldSegmentCapture", () => { const base = { env: {} as Record, diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 83ec68edc1..94349c15a5 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -143,6 +143,7 @@ import { describeWorkerPhases, forgetWorkerPhases, type MemorySampler, + reportExtractProgress, updateJobStatus, } from "./render/shared.js"; import { frameCacheOptions } from "./render/frameCacheBudget.js"; @@ -173,6 +174,9 @@ import { } from "./render/renderEtaGuard.js"; import { checkPageNeverReady } from "./render/pageNeverReady.js"; import { resolveSegmentFrames } from "./render/segmentPlan.js"; +import { extractAllDeferredFrames } from "./render/deferredVideoFrames.js"; +import { createSegmentVideoFrames } from "./render/segmentVideoFrames.js"; +import { shouldWindowVideoFrames, trackSegmentFramePeak } from "./render/videoFrameWindow.js"; import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js"; import { computeSegmentPlanHash, @@ -202,6 +206,7 @@ import { buildHeapAdvisoryWarning, resolveRenderWorkerCount, runCaptureCalibration, + selectCaptureCalibrationFrames, } from "./render/captureCost.js"; import { computeCompositionObservabilityHash, @@ -227,7 +232,10 @@ import { import { runCompileStage } from "./render/stages/compileStage.js"; import { runProbeStage } from "./render/stages/probeStage.js"; import { validateRenderDuration } from "./render/planValidation.js"; -import { runExtractVideosStage } from "./render/stages/extractVideosStage.js"; +import { + resolveVideoExtractionPolicy, + runExtractVideosStage, +} from "./render/stages/extractVideosStage.js"; import { rendersSdrOnly } from "./render/hdrMode.js"; import { planVideoLayerPassthrough, @@ -3522,6 +3530,34 @@ async function executeRenderPipeline(input: { }); const compiledDir = join(workDir, "compiled"); + // Settled before extraction, from what is known then. HDR content, which turns a + // segmented render into a layered one, only shows up once the videos are probed; + // the final plan is checked against this below. + const segmentsCapture = (layeredOrEffectRoute: boolean): boolean => + shouldSegmentCapture({ + env: process.env, + durationSeconds: probeResult.duration, + outputFormat, + layeredOrEffectRoute, + streamingOk: shouldUseStreamingEncode(cfg, outputFormat, 1, probeResult.duration), + }); + // A blur sample at a clip's edge shows a source frame the output frames do not, and a window + // holds only the frames the output frames show (videoFrameInjector.ts, `heldVideoTime`). + const segmentedForWindow = segmentsCapture(compiled.hasShaderTransitions); + const windowVideoFrames = + job.config.motionBlur === undefined && + shouldWindowVideoFrames({ env: process.env, segmented: segmentedForWindow }); + if (!windowVideoFrames && shouldWindowVideoFrames({ env: process.env, segmented: true })) { + const reason = + job.config.motionBlur !== undefined + ? "motion blur" + : outputFormat !== "mp4" && outputFormat !== "mov" + ? `${outputFormat} output` + : "capture is not segmented"; + log.info("[Render] HF_VIDEO_FRAME_WINDOW is set but this render extracts every frame", { + reason, + }); + } const extractResult = await observeRenderStage( observability, "video_extract", @@ -3539,6 +3575,7 @@ async function executeRenderPipeline(input: { assertNotAborted, // Local staging can use links; distributed plan() alone requires real copies. materializeSymlinks: false, + deferRangeExtraction: windowVideoFrames ? true : undefined, onProgress, }), ); @@ -3795,8 +3832,58 @@ async function executeRenderPipeline(input: { // can occupy all HTTP connections during native-video drift recovery. // The byte ceiling bounds the Node LRU; Chrome receives short URLs so // its resource cache does not retain the corresponding inline strings. + // True while some video's frames are still to be read per segment. + let videoFrameWindow = + windowVideoFrames && + (extractionResult?.extracted.some((video) => video.deferredRange) ?? false); + if (windowVideoFrames) { + const extracted = extractionResult?.extracted ?? []; + log.info("[Render] Video frame window", { + windowed: extracted.filter((video) => video.deferredRange).length, + fullyExtracted: extracted.filter((video) => !video.deferredRange).length, + }); + } + const buildSegmentVideoFrames = () => + trackSegmentFramePeak( + createSegmentVideoFrames({ + videos: composition.videos, + extracted: extractionResult?.extracted ?? [], + fps: job.config.fps, + rootDir: workDir, + cfg, + injectorOptions: frameCacheOptions(cfg, workerCount), + abortSignal: executionSignal, + }), + workDir, + (bytes) => updateCaptureObservability({ videoFrameWindowPeakBytes: bytes }), + ); const createRenderVideoFrameInjector = (): BeforeCaptureHook | null => createVideoFrameInjector(frameLookup, frameCacheOptions(cfg, workerCount)); + // Every route but the segmented one reads whole videos, so it needs the frames a window would have held back. + const extractDeferredVideoFrames = async (): Promise => { + videoFrameWindow = false; + updateCaptureObservability({ videoFrameWindow: "off" }); + const extracted = extractionResult?.extracted ?? []; + const videoCount = extracted.filter((video) => video.deferredRange).length; + const startedAt = Date.now(); + log.info("[Render] Extracting the frames of the videos a window would have held back", { + videoCount, + }); + await observeRenderStage(observability, "video_extract_deferred", { videoCount }, () => + extractAllDeferredFrames(extracted, { + fps: job.config.fps, + cfg, + signal: executionSignal, + maxTransientRetries: resolveVideoExtractionPolicy().maxTransientRetries, + onProgress: (done, total) => reportExtractProgress(job, done, total, onProgress), + }), + ); + assertNotAborted(); + log.info("[Render] Extracted the deferred video frames", { + videoCount, + elapsedMs: Date.now() - startedAt, + }); + }; let captureCalibration: | { @@ -4106,33 +4193,48 @@ async function executeRenderPipeline(input: { updateJobStatus(job, "rendering", "Measuring capture speed", 25, onProgress, { code: "start_browsers", }); - const outcome = await observeRenderStage( - observability, - "capture_calibration", - captureStageObservationData({ - forceScreenshot: captureForceScreenshot, - // Override the default `capturing` — calibration writes probe - // frames only, not `job.framesRendered`, so heartbeats reporting - // `framesCompleted: 0` misread as broken. Field signal - // ts=1784019503. - stagePhase: "calibrating", - }), - () => - runCaptureCalibration({ - cfg: calibrationCaptureConfig(cfg), - fileServer: activeFileServer, - workDir, - log, - job, - totalFrames, + // Calibration samples fixed frames, so the windowed videos it shows are extracted for just + // those here, outside its timing, and removed when it ends however it ends. + const calibrationFrames = videoFrameWindow + ? await buildSegmentVideoFrames().forFrames( + selectCaptureCalibrationFrames(totalFrames), + "calibration", + ) + : undefined; + let outcome: Awaited>; + try { + outcome = await observeRenderStage( + observability, + "capture_calibration", + captureStageObservationData({ forceScreenshot: captureForceScreenshot, - probeSession, - buildCaptureOptions: buildPrepCaptureOptions, - createRenderVideoFrameInjector, - assertNotAborted, + // Override the default `capturing` — calibration writes probe + // frames only, not `job.framesRendered`, so heartbeats reporting + // `framesCompleted: 0` misread as broken. Field signal + // ts=1784019503. + stagePhase: "calibrating", }), - { heartbeatMessage: "browser calibrating (frames not started)" }, - ); + () => + runCaptureCalibration({ + cfg: calibrationCaptureConfig(cfg), + fileServer: activeFileServer, + workDir, + log, + job, + totalFrames, + forceScreenshot: captureForceScreenshot, + probeSession, + buildCaptureOptions: buildPrepCaptureOptions, + createRenderVideoFrameInjector: calibrationFrames + ? calibrationFrames.injector + : createRenderVideoFrameInjector, + assertNotAborted, + }), + { heartbeatMessage: "browser calibrating (frames not started)" }, + ); + } finally { + calibrationFrames?.release(); + } captureCalibration = outcome.calibration; calibrationEvaluateTimedOut = outcome.calibrationEvaluateTimedOut; captureForceScreenshot = outcome.forceScreenshot; @@ -4596,13 +4698,7 @@ async function executeRenderPipeline(input: { // excluded routes are the ones whose concat-copy or capture loop the // segment contract does not cover (webm VP9 concat is fragile, HDR and // shader transitions run their own compositor). - useSegmentedCapture: shouldSegmentCapture({ - env: process.env, - durationSeconds: job.duration, - outputFormat, - layeredOrEffectRoute: hasHdrContent || compiled.hasShaderTransitions, - streamingOk: shouldUseStreamingEncode(cfg, outputFormat, 1, job.duration), - }), + useSegmentedCapture: segmentsCapture(hasHdrContent || compiled.hasShaderTransitions), useLayeredComposite, usePageSideCompositing: usePageSideCompositingForTransitions, hasHdrContent, @@ -4648,10 +4744,14 @@ async function executeRenderPipeline(input: { deWorkerInversion, deParallelRouter, }); + if (videoFrameWindow && capturePlan.kind !== "sdr_segmented") + await extractDeferredVideoFrames(); + updateCaptureObservability({ videoFrameWindow: videoFrameWindow ? "on" : "off" }); observability.checkpoint("capture_strategy", "resolved", { plan: capturePlan.kind, workerCount: capturePlan.workerCount, forceScreenshot: capturePlan.forceScreenshot, + videoFrameWindow: videoFrameWindow ? "on" : "off", captureBeyondViewport: resolvedCaptureBeyondViewport ?? null, useStreamingEncode: capturePlan.kind === "sdr_streaming", useLayeredComposite: capturePlan.kind === "hdr_layered", @@ -4904,6 +5004,7 @@ async function executeRenderPipeline(input: { }, buildCaptureOptions: buildWorkerCaptureOptions, createRenderVideoFrameInjector, + segmentVideoFrames: videoFrameWindow ? buildSegmentVideoFrames() : undefined, abortSignal: executionSignal, assertNotAborted, onProgress, @@ -5019,6 +5120,7 @@ async function executeRenderPipeline(input: { // the branch below run the render normally. capturePlan = replanAfterFailure(capturePlan, { kind: "streaming_unavailable" }); syncCapturePlan(); + if (videoFrameWindow) await extractDeferredVideoFrames(); // The stage closed every session it opened in its own finally, // including a reused probe. Drop the reference so the streaming // branch below opens a fresh session instead of re-initialising a diff --git a/packages/producer/src/services/renderOrchestrator.videoFrameWindow.test.ts b/packages/producer/src/services/renderOrchestrator.videoFrameWindow.test.ts new file mode 100644 index 0000000000..b5256cb336 --- /dev/null +++ b/packages/producer/src/services/renderOrchestrator.videoFrameWindow.test.ts @@ -0,0 +1,436 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DrawElementCaptureError, resolveConfig, type ExtractedFrames } from "@hyperframes/engine"; +import { selectCaptureCalibrationFrames } from "./render/captureCost.js"; +import { CaptureStageError } from "./render/captureStageError.js"; +import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js"; + +// Entry: executeRenderJob with one long video whose frames are deferred when the extract stage is +// asked to. Every capture stage is scripted, so a test ends at the first stage that is reached. +const { state, REACHED } = vi.hoisted(() => ({ + state: { + events: [] as string[], + deferRequests: [] as Array, + segmentedWindow: [] as boolean[], + segmentedOutcomes: [] as Array<"spawn_failed" | "drawelement_failure">, + offerDeferredFrames: true, + nativeHdr: false, + extractions: [] as Array<{ frames: number[]; outputRoot: string }>, + tables: [] as Array, + calibrations: [] as Array<{ + folderExisted: boolean; + frameIndex: number | undefined; + frameOnDisk: boolean; + extractionsBefore: number; + }>, + folderAtNextStage: [] as boolean[], + logs: [] as string[], + stages: [] as string[], + }, + REACHED: "stage reached", +})); + +vi.mock("@hyperframes/engine", async (importOriginal) => ({ + ...(await importOriginal()), + assertConfiguredFfmpegBinariesExist: () => {}, + resolveBrowserGpuMode: async () => "software", + createVideoFrameInjector: (table: FrameLookupTable | null) => { + state.tables.push(table); + return Object.assign(async () => {}, { fresh: () => null }); + }, + createCaptureSession: async () => ({ + browserConsoleBuffer: [], + options: {}, + isInitialized: false, + }), + initializeSession: async () => {}, + closeCaptureSession: async () => {}, +})); +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => ({ + url: "http://127.0.0.1:1", + port: 1, + close: () => {}, + addPreHeadScript: () => {}, + }), +})); +vi.mock("./render/stages/compileStage.js", () => ({ + runCompileStage: async () => ({ + compiled: { + html: "
", + renderModeHints: { reasons: [] }, + hasShaderTransitions: false, + unresolvedCompositions: [], + }, + composition: { + width: 320, + height: 180, + duration: 40, + videos: [ + { + id: "long", + src: "long.mp4", + start: 0, + end: 40, + mediaStart: 0, + loop: false, + hasAudio: false, + }, + ], + images: [], + audios: [], + }, + deviceScaleFactor: 1, + outputWidth: 320, + outputHeight: 180, + compileOnlyMs: 0, + forceScreenshot: false, + }), +})); +vi.mock("./render/stages/probeStage.js", () => ({ + runProbeStage: async ({ compiled }: { compiled: object }) => ({ + compiled, + fileServer: null, + probeSession: null, + lastBrowserConsole: [], + duration: 40, + totalFrames: 1200, + browserProbeMs: 0, + }), +})); +vi.mock("./render/stages/extractVideosStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runExtractVideosStage: async (input: { deferRangeExtraction?: boolean }) => { + state.deferRequests.push(input.deferRangeExtraction); + const deferred = input.deferRangeExtraction === true && state.offerDeferredFrames; + const frame: ExtractedFrames = { + videoId: "long", + srcPath: "/project/long.mp4", + outputDir: "/work/compiled/__hyperframes_video_frames/long", + framePattern: "frame_%05d.jpg", + fps: 30, + totalFrames: 1200, + metadata: { + durationSeconds: 40, + videoStreamDurationSeconds: 40, + videoStreamStartSeconds: 0, + width: 320, + height: 180, + fps: 30, + videoCodec: "h264", + hasAudio: false, + isVFR: false, + hasAlpha: false, + colorSpace: null, + }, + framePaths: new Map( + deferred ? [] : Array.from({ length: 1200 }, (_, i) => [i, `/frames/${i}.jpg`] as const), + ), + ...(deferred + ? { deferredRange: { startTime: 0, durationSeconds: 40, format: "jpg" as const } } + : {}), + }; + return { + extractionResult: { + success: true, + errors: [], + extracted: [frame], + totalFramesExtracted: deferred ? 0 : 1200, + durationMs: 1, + phaseBreakdown: {}, + }, + frameLookup: null, + videoReadinessSkipIds: [], + videoMetadataHints: [], + nativeHdrVideoIds: new Set(state.nativeHdr ? ["long"] : []), + videoTransfers: new Map(), + nativeHdrImageIds: new Set(), + imageTransfers: new Map(), + hdrImageSrcPaths: new Map(), + imageColorSpaces: [], + videoExtractMs: 0, + failureToEnforce: null, + }; + }, +})); +vi.mock("./render/deferredVideoFrames.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + extractAllDeferredFrames: async ( + extracted: ExtractedFrames[], + input: { onProgress?: (done: number, total: number) => void }, + ) => { + state.events.push("extract-all"); + for (const video of extracted) delete video.deferredRange; + input.onProgress?.(1200, 1200); + }, + // Stands in for ffmpeg: writes the frames the given output frames show, as the real one names them. + extractDeferredFrames: async (input: Parameters[0]) => { + const frames = [...input.frames]; + state.extractions.push({ frames, outputRoot: input.outputRoot }); + const shown = actual.shownDeferredFrameIndexes(input, frames, input.fps); + return input.extracted + .filter((video) => video.deferredRange) + .map((video) => { + const outputDir = join(input.outputRoot, video.videoId); + const framePaths = new Map(); + for (const index of shown.get(video.videoId) ?? []) { + mkdirSync(outputDir, { recursive: true }); + const path = join(outputDir, `frame_${String(index + 1).padStart(5, "0")}.jpg`); + writeFileSync(path, "x"); + framePaths.set(index, path); + } + return { ...video, outputDir, framePaths, ownedByLookup: false }; + }); + }, + }; +}); +vi.mock("./render/stages/audioStage.js", () => ({ + runAudioStage: async () => ({ audioOutputPath: undefined, hasAudio: false, audioProcessMs: 0 }), +})); +vi.mock("./render/captureCost.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveRenderWorkerCount: () => 1, + runCaptureCalibration: async (input: { + forceScreenshot: boolean; + probeSession: unknown; + createRenderVideoFrameInjector: () => ((...args: never[]) => Promise) | null; + }) => { + const root = state.extractions.find((e) => e.outputRoot.endsWith("calibration"))?.outputRoot; + const hook = input.createRenderVideoFrameInjector(); + const table = state.tables.at(-1); + const shown = hook && root ? table?.getActiveFramePayloads(600 / 30).get("long") : undefined; + state.calibrations.push({ + folderExisted: root !== undefined && existsSync(root), + frameIndex: shown?.frameIndex, + frameOnDisk: shown !== undefined && existsSync(shown.framePath), + extractionsBefore: state.extractions.length, + }); + return { + calibration: undefined, + forceScreenshot: input.forceScreenshot, + probeSession: input.probeSession, + lastBrowserConsole: [], + }; + }, +})); +vi.mock("./render/stages/captureSegmentedStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureSegmentedStage: async (input: { segmentVideoFrames?: unknown }) => { + state.events.push("segmented"); + const calibrationRoot = state.extractions.find((e) => e.outputRoot.endsWith("calibration")); + if (calibrationRoot) state.folderAtNextStage.push(existsSync(calibrationRoot.outputRoot)); + state.segmentedWindow.push(input.segmentVideoFrames !== undefined); + const outcome = state.segmentedOutcomes.shift(); + if (outcome === "spawn_failed") return { success: false }; + if (outcome === "drawelement_failure") { + throw new CaptureStageError({ + cause: new DrawElementCaptureError(640, "suspect small frame"), + browserConsole: [], + }); + } + throw new Error(REACHED); + }, +})); +vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureStreamingStage: async () => { + state.events.push("streaming"); + throw new Error(REACHED); + }, +})); +vi.mock("./render/stages/captureHdrStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureHdrStage: async () => { + state.events.push("hdr"); + throw new Error(REACHED); + }, +})); + +const dirs: string[] = []; +beforeEach(() => { + state.events = []; + state.deferRequests = []; + state.segmentedWindow = []; + state.segmentedOutcomes = []; + state.offerDeferredFrames = true; + state.nativeHdr = false; + state.extractions = []; + state.calibrations = []; + state.tables = []; + state.folderAtNextStage = []; + state.logs = []; + state.stages = []; + vi.stubEnv("HF_SEGMENTED_CAPTURE", "true"); + vi.stubEnv("HF_VIDEO_FRAME_WINDOW", "true"); + vi.stubEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", ""); + vi.stubEnv("HF_DE_PARALLEL_STREAM", ""); + vi.stubEnv("HF_DE_PARALLEL_ROUTER", "false"); + vi.stubEnv("HF_DE_PARALLEL_MIN_MEM_MB", "0"); + vi.stubEnv("HF_DE_SINGLE_MIN_FRAMES", "900"); +}); +afterEach(() => { + vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function render(config: Partial[0]> = {}): Promise { + const root = mkdtempSync(join(tmpdir(), "hf-video-frame-window-")); + dirs.push(root); + writeFileSync(join(root, "index.html"), "
"); + const job = createRenderJob({ + fps: 30, + quality: "standard", + hdrMode: "auto", + producerConfig: resolveConfig({ useDrawElement: true, browserGpuMode: "software" }), + logger: { + info: (message: string) => void state.logs.push(message), + warn() {}, + error() {}, + debug() {}, + }, + ...config, + }); + return executeRenderJob(job, root, join(root, `output.${config.format ?? "mp4"}`), (j) => { + if (j.currentStage && state.stages.at(-1) !== j.currentStage) state.stages.push(j.currentStage); + }); +} + +it("a segmented render with the flag on defers the long video and windows its frames", async () => { + await expect(render()).rejects.toThrow(REACHED); + expect(state.deferRequests).toEqual([true]); + expect(state.segmentedWindow).toEqual([true]); + expect(state.events).toEqual(["segmented"]); +}); + +it.each(["false", "0", "off", "no", ""])( + "with the flag %j, deferRangeExtraction stays unset and the render is as it was", + async (value) => { + vi.stubEnv("HF_VIDEO_FRAME_WINDOW", value); + await expect(render()).rejects.toThrow(REACHED); + expect(state.deferRequests).toEqual([undefined]); + expect(state.segmentedWindow).toEqual([false]); + expect(state.events).toEqual(["segmented"]); + }, +); + +it("with the flag unset, deferRangeExtraction stays unset", async () => { + vi.stubEnv("HF_VIDEO_FRAME_WINDOW", undefined); + await expect(render()).rejects.toThrow(REACHED); + expect(state.deferRequests).toEqual([undefined]); + expect(state.segmentedWindow).toEqual([false]); +}); + +it("a render that would not segment never defers", async () => { + vi.stubEnv("HF_SEGMENTED_CAPTURE", "false"); + await expect(render()).rejects.toThrow(REACHED); + expect(state.deferRequests).toEqual([undefined]); + expect(state.events).toEqual(["streaming"]); +}); + +it.each([ + ["a webm render", { format: "webm" as const }, "streaming"], + ["a motion-blur render", { motionBlur: { samplesPerFrame: 4 } }, "segmented"], +])("%s never defers", async (_label, config, stage) => { + await expect(render(config)).rejects.toThrow(REACHED); + expect(state.deferRequests).toEqual([undefined]); + expect(state.segmentedWindow).not.toContain(true); + expect(state.events).toEqual([stage]); +}); + +it("flag on with no long video to defer is the same render as flag off", async () => { + state.offerDeferredFrames = false; + await expect(render()).rejects.toThrow(REACHED); + expect(state.segmentedWindow).toEqual([false]); + expect(state.events).toEqual(["segmented"]); +}); + +it("a render that leaves the segmented route has every frame before capture starts", async () => { + state.segmentedOutcomes = ["spawn_failed"]; + await expect(render()).rejects.toThrow(REACHED); + expect(state.events).toEqual(["segmented", "extract-all", "streaming"]); +}); + +it("a render whose final plan is not segmented extracts everything before its first capture", async () => { + state.nativeHdr = true; + await expect(render({ hdrMode: "force-hdr" })).rejects.toThrow(REACHED); + expect(state.events).toEqual(["extract-all", "hdr"]); +}); + +it("a drawElement retry stays on the segmented route and keeps windowing", async () => { + state.segmentedOutcomes = ["drawelement_failure"]; + await expect(render()).rejects.toThrow(REACHED); + expect(state.events).toEqual(["segmented", "segmented"]); + expect(state.segmentedWindow).toEqual([true, true]); +}); + +it("calibration on the windowed route injects real frames for its samples and removes them afterwards", async () => { + await expect(render()).rejects.toThrow(REACHED); + const calibration = state.extractions.find((e) => e.outputRoot.endsWith("calibration")); + expect(calibration?.frames).toEqual(selectCaptureCalibrationFrames(1200)); + expect(state.calibrations).toEqual([ + { folderExisted: true, frameIndex: 600, frameOnDisk: true, extractionsBefore: 1 }, + ]); + expect(state.folderAtNextStage).toEqual([false]); +}); + +it("extracts the calibration frames before the first timed sample", async () => { + await expect(render()).rejects.toThrow(REACHED); + // The scripted calibration is the first thing that would be timed; by then the extraction is done. + expect(state.calibrations.map((c) => c.extractionsBefore)).toEqual([1]); +}); + +it("with the flag off, calibration extracts nothing", async () => { + vi.stubEnv("HF_VIDEO_FRAME_WINDOW", "false"); + await expect(render()).rejects.toThrow(REACHED); + expect(state.extractions).toEqual([]); + expect(state.calibrations).toHaveLength(1); +}); + +it("the late full extraction is logged and moves the job's progress", async () => { + state.segmentedOutcomes = ["spawn_failed"]; + await expect(render()).rejects.toThrow(REACHED); + expect(state.logs).toEqual( + expect.arrayContaining([ + "[Render] Extracting the frames of the videos a window would have held back", + "[Render] Extracted the deferred video frames", + ]), + ); + expect(state.stages).toContain("Extracting video frames"); +}); + +const WINDOW_LOGS = [ + "[Render] HF_VIDEO_FRAME_WINDOW is set but this render extracts every frame", + "[Render] Video frame window", +]; + +it("a windowed render says how many videos it windowed and how many stayed fully extracted", async () => { + await expect(render()).rejects.toThrow(REACHED); + expect(state.logs).toContain("[Render] Video frame window"); + expect(state.logs).not.toContain(WINDOW_LOGS[0]); +}); + +it.each([ + ["a webm render", { format: "webm" as const }], + ["a motion-blur render", { motionBlur: { samplesPerFrame: 4 } }], +])("%s with the flag on says it extracts every frame", async (_label, config) => { + await expect(render(config)).rejects.toThrow(REACHED); + expect(state.logs).toContain(WINDOW_LOGS[0]); + expect(state.logs).not.toContain(WINDOW_LOGS[1]); +}); + +it("a render that would not segment says so when the flag is on", async () => { + vi.stubEnv("HF_SEGMENTED_CAPTURE", "false"); + await expect(render()).rejects.toThrow(REACHED); + expect(state.logs).toContain(WINDOW_LOGS[0]); +}); + +it("with the flag off, nothing about windowing is logged", async () => { + vi.stubEnv("HF_VIDEO_FRAME_WINDOW", "false"); + await expect(render()).rejects.toThrow(REACHED); + expect(state.logs.filter((message) => WINDOW_LOGS.includes(message))).toEqual([]); +}); diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index e6dfac2ae7..6672242ff4 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -36,7 +36,7 @@ "unpacked": 541601 }, "@hyperframes/producer": { - "packed": 845000, + "packed": 857000, "unpacked": 3081613 }, "@hyperframes/sdk": {