diff --git a/packages/producer/src/services/render/segmentManifest.test.ts b/packages/producer/src/services/render/segmentManifest.test.ts index 95ed19fa40..81a25af0fa 100644 --- a/packages/producer/src/services/render/segmentManifest.test.ts +++ b/packages/producer/src/services/render/segmentManifest.test.ts @@ -190,6 +190,33 @@ describe("validateCompletedSegments", () => { expect((await validateCompletedSegments(manifest, "h", async () => 3)).size).toBe(0); }); + it("a resume whose segment files were removed after a join captures them again rather than failing", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + const joined = [0, 1].map((index) => join(dir, `segment_0000${index}.mp4`)); + const manifest: SegmentManifest = { + version: 1, + planHash: "h", + totalFrames: 6, + segmentFrames: 3, + completed: joined.map((path, index) => ({ + index, + startFrame: index * 3, + endFrame: index * 3 + 3, + path, + bytes: 2, + completedAt: "", + })), + }; + // The manifest still lists them, but the join deleted the files. + let probed = 0; + const probe = async () => { + probed += 1; + return 3; + }; + expect((await validateCompletedSegments(manifest, "h", probe)).size).toBe(0); + expect(probed).toBe(0); + }); + it("skips nothing when the plan hash differs", async () => { // The entry is otherwise perfectly reusable — present, right size, right // frame count — so only the hash check can reject it. With an empty diff --git a/packages/producer/src/services/render/segmentVideoFrames.test.ts b/packages/producer/src/services/render/segmentVideoFrames.test.ts new file mode 100644 index 0000000000..1dc657db4b --- /dev/null +++ b/packages/producer/src/services/render/segmentVideoFrames.test.ts @@ -0,0 +1,304 @@ +import { mkdirSync, mkdtempSync, existsSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "bun:test"; +import type { EngineConfig, ExtractedFrames, VideoElement } from "@hyperframes/engine"; +import { type extractDeferredFrames, shownDeferredFrameIndexes } from "./deferredVideoFrames.js"; +import { createSegmentVideoFrames } from "./segmentVideoFrames.js"; + +const root = mkdtempSync(join(tmpdir(), "hf-segment-video-frames-")); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +const metadata = { + durationSeconds: 4, + videoStreamDurationSeconds: 4, + videoStreamStartSeconds: 0, + width: 64, + height: 36, + fps: 30, + videoCodec: "h264", + hasAudio: false, + isVFR: false, + hasAlpha: false, + colorSpace: null, +}; + +const base = (videoId: string, totalFrames: number): ExtractedFrames => ({ + videoId, + srcPath: `/source/${videoId}.mp4`, + outputDir: "", + framePattern: "frame_%05d.jpg", + fps: 30, + totalFrames, + metadata, + framePaths: new Map(), +}); + +const deferredVideo = (videoId: string): ExtractedFrames => ({ + ...base(videoId, 75), + deferredRange: { startTime: 1.5, durationSeconds: 2.5, format: "jpg" }, +}); + +const wholeVideo = (videoId: string): ExtractedFrames => ({ + ...base(videoId, 75), + outputDir: "/full-extraction/whole", + framePaths: new Map( + Array.from({ length: 75 }, (_, i) => [i, `/full-extraction/whole/frame_${i + 1}.jpg`]), + ), +}); + +const element = (id: string, start: number, end: number): VideoElement => ({ + id, + src: `${id}.mp4`, + start, + end, + mediaStart: 1.5, + loop: false, + hasAudio: false, +}); + +const fps = { num: 30, den: 1 }; +const cfg = { ffmpegProcessTimeout: 1000 } as EngineConfig; +const slice = (index: number, startFrame: number, endFrame: number) => ({ + index, + startFrame, + endFrame, +}); +const span = (start: number, end: number) => + Array.from({ length: end - start }, (_, i) => start + i); + +/** Stands in for the extractor: writes the frames the slice shows, as the real one names them. */ +function fakeExtractor() { + const calls: Array<{ outputRoot: string; frames: number[]; sourceOf: string[] }> = []; + const extractFrames = async (input: Parameters[0]) => { + const frames = [...input.frames]; + const shown = shownDeferredFrameIndexes(input, frames, input.fps); + calls.push({ + outputRoot: input.outputRoot, + frames, + sourceOf: input.extracted.filter((v) => v.deferredRange).map((v) => input.sourceOf(v)), + }); + return input.extracted + .filter((video) => video.deferredRange) + .map((video) => { + const outputDir = join(input.outputRoot, video.videoId); + const framePaths = new Map(); + for (const index of shown.get(video.videoId) ?? []) { + mkdirSync(outputDir, { recursive: true }); + const path = join(outputDir, `frame_${String(index + 1).padStart(5, "0")}.jpg`); + writeFileSync(path, "x"); + framePaths.set(index, path); + } + return { ...video, outputDir, framePaths, ownedByLookup: false }; + }); + }; + return { calls, extractFrames }; +} + +function setup(extra: Partial[0]> = {}) { + const served: string[] = []; + const dir = mkdtempSync(join(root, "work-")); + const extractor = fakeExtractor(); + const segmentVideoFrames = createSegmentVideoFrames({ + videos: [element("clip", 0, 4), element("whole", 0, 4)], + extracted: [deferredVideo("clip"), wholeVideo("whole")], + fps, + rootDir: dir, + cfg, + injectorOptions: { + frameSrcResolver: (path) => { + served.push(path); + return `http://frames/${path}`; + }, + }, + extractFrames: extractor.extractFrames, + ...extra, + }); + const page = { evaluate: async () => [] } as never; + return { + dir, + served, + extractor, + segmentVideoFrames, + workerFrames: segmentVideoFrames.forWorker(), + page, + }; +} + +describe("createSegmentVideoFrames", () => { + it("extracts only the frames a slice shows, under that slice's own folder", async () => { + const { dir, served, extractor, workerFrames, page } = setup(); + await workerFrames.prepare(slice(1, 30, 45)); + + const folder = join(dir, "segment-frames", "segment_00001"); + expect(extractor.calls).toHaveLength(1); + expect(extractor.calls[0]!.outputRoot).toBe(folder); + expect(extractor.calls[0]!.frames).toEqual(span(30, 45)); + expect(readdirSync(join(folder, "clip")).sort()).toEqual( + span(30, 45).map((i) => `frame_${String(i + 1).padStart(5, "0")}.jpg`), + ); + + await workerFrames.hook(page, 31 / 30); + expect(served).toContain(join(folder, "clip", "frame_00032.jpg")); + }); + + it("release removes the folder and a second release does nothing", async () => { + const { dir, workerFrames } = setup(); + await workerFrames.prepare(slice(2, 60, 75)); + const folder = join(dir, "segment-frames", "segment_00002"); + expect(existsSync(folder)).toBe(true); + + workerFrames.release(); + expect(existsSync(folder)).toBe(false); + expect(() => workerFrames.release()).not.toThrow(); + }); + + it("a video that was fully extracted is served from its existing frames and never extracted again", async () => { + const { served, extractor, workerFrames, page } = setup(); + await workerFrames.prepare(slice(0, 0, 15)); + + await workerFrames.hook(page, 3 / 30); + expect(served).toContain("/full-extraction/whole/frame_4.jpg"); + // The extractor is handed the whole list and picks out the deferred ones; the full one has no source read. + expect(extractor.calls[0]!.sourceOf).toEqual(["/source/clip.mp4"]); + expect(existsSync(join(extractor.calls[0]!.outputRoot, "whole"))).toBe(false); + }); + + it("a slice showing no deferred video extracts nothing", async () => { + const dir = mkdtempSync(join(root, "work-")); + const segmentVideoFrames = createSegmentVideoFrames({ + videos: [element("later", 10, 12)], + extracted: [deferredVideo("later")], + fps, + rootDir: dir, + cfg, + injectorOptions: {}, + sourceOf: () => { + throw new Error("a slice that shows no video must not read a source"); + }, + }); + const workerFrames = segmentVideoFrames.forWorker(); + + await workerFrames.prepare(slice(0, 0, 15)); + + const folder = join(dir, "segment-frames", "segment_00000"); + expect(readdirSync(folder).flatMap((entry) => readdirSync(join(folder, entry)))).toEqual([]); + }); + + it("a prepare that fails mid-extraction leaves no folder behind", async () => { + const { dir, workerFrames } = setup({ + extractFrames: async (input) => { + mkdirSync(join(input.outputRoot, "clip"), { recursive: true }); + writeFileSync(join(input.outputRoot, "clip", "frame_00001.jpg"), "x"); + throw new Error("aborted"); + }, + }); + + await expect(workerFrames.prepare(slice(3, 90, 105))).rejects.toThrow("aborted"); + + expect(existsSync(join(dir, "segment-frames", "segment_00003"))).toBe(false); + }); + describe("outside a prepared window", () => { + const onDemandFolders = (dir: string) => { + const root = join(dir, "segment-frames"); + return existsSync(root) ? readdirSync(root).filter((n) => n.startsWith("on-demand_")) : []; + }; + + /** Records, at injection time, whether the folder the served frame lives in still existed. */ + function setupWatching() { + const seen: Array<{ path: string; existed: boolean }> = []; + const made = setup({ + injectorOptions: { + frameSrcResolver: (path) => { + seen.push({ path, existed: existsSync(path) }); + return `http://frames/${path}`; + }, + }, + }); + return { ...made, seen }; + } + + it("before any prepare, the hook extracts and injects the frame asked for, then removes its folder", async () => { + const { dir, seen, extractor, workerFrames, page } = setupWatching(); + + await workerFrames.hook(page, 31 / 30); + + expect(extractor.calls).toHaveLength(1); + expect(extractor.calls[0]!.frames).toEqual([31]); + expect( + extractor.calls[0]!.outputRoot.startsWith(join(dir, "segment-frames", "on-demand_")), + ).toBe(true); + const clip = seen.find((s) => s.path.includes("/clip/")); + expect(clip).toEqual({ path: expect.stringContaining("frame_00032.jpg"), existed: true }); + // The fully extracted video is injected too, from its own frames. + expect(seen.some((s) => s.path === "/full-extraction/whole/frame_32.jpg")).toBe(true); + expect(onDemandFolders(dir)).toEqual([]); + }); + + it("asks for the held frame too when the capture holds footage for a motion-blur sample", async () => { + const { extractor, workerFrames, page } = setupWatching(); + + await workerFrames.hook(page, 32 / 30, 30 / 30); + + expect(extractor.calls[0]!.frames).toEqual([32, 30]); + }); + + it("after release, the hook extracts on demand again", async () => { + const { dir, seen, extractor, workerFrames, page } = setupWatching(); + await workerFrames.prepare(slice(1, 30, 45)); + await workerFrames.hook(page, 31 / 30); + expect(extractor.calls).toHaveLength(1); + + workerFrames.release(); + await workerFrames.hook(page, 61 / 30); + + expect(extractor.calls).toHaveLength(2); + expect(extractor.calls[1]!.frames).toEqual([61]); + expect( + seen.some((s) => s.path.includes("on-demand_") && s.path.includes("frame_00062.jpg")), + ).toBe(true); + expect(onDemandFolders(dir)).toEqual([]); + }); + + it("fresh returns an independent hook that also extracts on demand", async () => { + const { dir, seen, extractor, workerFrames, page } = setupWatching(); + await workerFrames.prepare(slice(1, 30, 45)); + + const fresh = workerFrames.hook.fresh!(); + expect(fresh).not.toBe(workerFrames.hook); + // The worker's window is prepared, yet the fresh hook still reads its own frames. + await fresh(page, 40 / 30); + + expect(extractor.calls).toHaveLength(2); + expect(extractor.calls[1]!.frames).toEqual([40]); + expect( + seen.some((s) => s.path.includes("on-demand_") && s.path.includes("frame_00041.jpg")), + ).toBe(true); + expect(onDemandFolders(dir)).toEqual([]); + expect(typeof fresh.fresh).toBe("function"); + }); + + it("an on-demand extraction that fails removes its folder and rethrows", async () => { + const { dir, workerFrames, page } = setup({ + extractFrames: async (input) => { + mkdirSync(join(input.outputRoot, "clip"), { recursive: true }); + writeFileSync(join(input.outputRoot, "clip", "frame_00001.jpg"), "x"); + throw new Error("aborted"); + }, + }); + + await expect(workerFrames.hook(page, 31 / 30)).rejects.toThrow("aborted"); + + expect(onDemandFolders(dir)).toEqual([]); + }); + + it("gives two workers different folders", async () => { + const { extractor, segmentVideoFrames, workerFrames, page } = setup(); + const other = segmentVideoFrames.forWorker(); + + await Promise.all([workerFrames.hook(page, 31 / 30), other.hook(page, 31 / 30)]); + + expect(new Set(extractor.calls.map((call) => call.outputRoot)).size).toBe(2); + }); + }); +}); diff --git a/packages/producer/src/services/render/segmentVideoFrames.ts b/packages/producer/src/services/render/segmentVideoFrames.ts new file mode 100644 index 0000000000..4117b98e29 --- /dev/null +++ b/packages/producer/src/services/render/segmentVideoFrames.ts @@ -0,0 +1,149 @@ +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import type { Page } from "puppeteer-core"; +import { + createFrameLookupTable, + createVideoFrameInjector, + type BeforeCaptureHook, + type EngineConfig, + type ExtractedFrames, + type VideoElement, +} from "@hyperframes/engine"; +import { resolveRenderFpsConfig } from "../fileServer.js"; +import { extractDeferredFrames } from "./deferredVideoFrames.js"; +import type { SegmentSlice } from "./segmentPlan.js"; + +export interface SegmentWorkerFrames { + /** + * One hook for the worker's whole life, so a recycled session keeps it. Inside a + * prepared window it injects from the window; outside one it extracts the frames + * it is asked for and removes them again. + */ + hook: BeforeCaptureHook; + /** Extracts the frames `segment` shows and points the hook at them. */ + prepare(segment: SegmentSlice): Promise; + /** Deletes the segment's frames. Safe to call again. */ + release(): void; +} + +export interface SegmentVideoFrames { + forWorker(): SegmentWorkerFrames; +} + +type FrameExtractor = typeof extractDeferredFrames; + +function segmentFramesDir(rootDir: string, index: number): string { + return join(rootDir, "segment-frames", `segment_${String(index).padStart(5, "0")}`); +} + +/** Each worker reads only the video frames its current segment shows, and deletes them when the segment closes. */ +export function createSegmentVideoFrames(input: { + videos: VideoElement[]; + extracted: ExtractedFrames[]; + fps: { num: number; den: number }; + /** The render's work dir; frames go under `segment-frames/segment_NNNNN/`. */ + rootDir: string; + cfg: EngineConfig; + injectorOptions: Parameters[1]; + abortSignal?: AbortSignal; + sourceOf?: (video: ExtractedFrames) => string; + extractFrames?: FrameExtractor; +}): SegmentVideoFrames { + const extractFrames = input.extractFrames ?? extractDeferredFrames; + const sourceOf = input.sourceOf ?? ((video: ExtractedFrames) => video.srcPath); + const fullyExtracted = input.extracted.filter((video) => !video.deferredRange); + + /** Extracts the given output frames into `dir` and returns an injector over them plus the fully extracted videos. */ + const injectorFor = async (frames: number[], dir: string): Promise => { + const sliced = await extractFrames({ + videos: input.videos, + extracted: input.extracted, + frames, + fps: input.fps, + outputRoot: dir, + cfg: input.cfg, + signal: input.abortSignal, + sourceOf, + onMissingFrame: (video, index) => + new Error( + `[Render] video ${JSON.stringify(video.videoId)} has no frame ${index}; ` + + `it was expected to have ${video.totalFrames} frames.`, + ), + }); + const table = createFrameLookupTable( + input.videos, + [...fullyExtracted, ...sliced], + resolveRenderFpsConfig(input.fps).value, + ); + return createVideoFrameInjector(table, input.injectorOptions); + }; + + /** + * The drawElement self-verify runs the hook at session open and after every + * recycle, outside any prepared window, to photograph the video frames the + * full route would show. Each call reads just the frames it is asked for. + */ + let onDemandCalls = 0; + const onDemandHook = (): BeforeCaptureHook => + Object.assign( + async (page: Page, time: number, heldVideoTime?: number) => { + const dir = join( + input.rootDir, + "segment-frames", + `on-demand_${String(onDemandCalls++).padStart(5, "0")}`, + ); + try { + const frames = [time, heldVideoTime] + .filter((t): t is number => t !== undefined) + .map((t) => Math.round((t * input.fps.num) / input.fps.den)); + const injector = await injectorFor(frames, dir); + await injector?.(page, time, heldVideoTime); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + { fresh: onDemandHook }, + ); + + return { + forWorker() { + let injector: BeforeCaptureHook | null = null; + let folder: string | null = null; + const onDemand = onDemandHook(); + + const release = () => { + injector = null; + if (folder) rmSync(folder, { recursive: true, force: true }); + folder = null; + }; + + return { + hook: Object.assign( + (page: Page, time: number, heldVideoTime?: number) => + (injector ?? onDemand)(page, time, heldVideoTime), + { fresh: onDemandHook }, + ), + release, + async prepare(segment) { + release(); + const dir = segmentFramesDir(input.rootDir, segment.index); + folder = dir; + try { + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + injector = await injectorFor( + Array.from( + { length: segment.endFrame - segment.startFrame }, + (_, i) => segment.startFrame + i, + ), + dir, + ); + } catch (error) { + release(); + throw error; + } + }, + }; + }, + }; +} diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts index f570068e5b..6bc6bf9ab2 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.test.ts @@ -1,6 +1,6 @@ // fallow-ignore-file code-duplication import { afterAll, describe, expect, it, mock } from "bun:test"; -import { mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -29,6 +29,8 @@ let initializeSessionErrorMessage = "initialize failed"; const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"]; const closeCaptureSession = mock(async () => {}); const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = []; +const createdSessionHooks: unknown[] = []; +const reusedSessionHooks: unknown[] = []; class DrawElementVerificationError extends Error {} mock.module("@hyperframes/engine", () => ({ @@ -60,6 +62,7 @@ mock.module("@hyperframes/engine", () => ({ completeDeferredDrawElementInit: async () => {}, createCaptureSession: async (...args: unknown[]) => { createdSessionConfigs.push(args[4] as { enableBrowserPool?: boolean }); + createdSessionHooks.push(args[3]); return { isInitialized: false, browserConsoleBuffer, @@ -128,7 +131,8 @@ mock.module("@hyperframes/engine", () => ({ pixelFormat: "yuv420p", }), initTransparentBackground: async () => {}, - prepareCaptureSessionForReuse: () => { + prepareCaptureSessionForReuse: (_session: unknown, _framesDir: string, hook: unknown) => { + reusedSessionHooks.push(hook); if (failPrepareCaptureSessionForReuse) { throw new Error("prepare reuse failed: ENOSPC"); } @@ -928,3 +932,357 @@ describe("runCaptureSegmentedStage", () => { expect(capturedAfterClose).toBe(0); }); }); + +describe("runCaptureSegmentedStage with segmentVideoFrames", () => { + type Slice = { index: number; startFrame: number; endFrame: number }; + + /** A stand-in for the windowed frames: every call is logged, and prepare makes a real folder so release is observable. */ + function fakeVideoFrames( + events: string[], + options: { prepare?: (segment: Slice) => Promise; folderRoot?: string } = {}, + ) { + const hooks: Array<(...args: unknown[]) => Promise> = []; + const prepared: number[] = []; + let current: string | null = null; + return { + hooks, + prepared, + segmentVideoFrames: { + forWorker: () => { + const hook = async () => {}; + hooks.push(hook); + return { + hook, + prepare: async (segment: Slice) => { + events.push(`prepare ${segment.index}`); + prepared.push(segment.index); + if (options.folderRoot) { + current = join(options.folderRoot, `segment_${segment.index}`); + mkdirSync(current, { recursive: true }); + } + await options.prepare?.(segment); + }, + release: () => { + events.push("release"); + if (current) rmSync(current, { recursive: true, force: true }); + }, + }; + }, + }, + }; + } + + function loggingEncoder(events: string[]) { + let segment = -1; + return mock(async (outputPath: string) => { + segment = Number(/segment_(\d+)/.exec(outputPath)![1]); + events.push(`spawn ${segment}`); + const own = segment; + return { + writeFrame: async (buffer: Buffer) => { + events.push(`frame ${buffer.readUInt32BE(0)}`); + return true; + }, + close: async () => { + events.push(`close ${own}`); + return { success: true, durationMs: 1, fileSize: 1 }; + }, + getExitStatus: () => "success" as const, + getExitError: () => undefined, + }; + }); + } + + const frameBuffer = async (_s: unknown, frameIndex: number) => { + const buffer = Buffer.alloc(4); + buffer.writeUInt32BE(frameIndex, 0); + return { buffer }; + }; + + function stageDeps(overrides: Record = {}) { + return { + captureFrame: mock(frameBuffer), + concat: mock(async () => ({ success: true as const })), + closeSession: mock(async () => {}), + removeFile: () => {}, + ...overrides, + }; + } + + it("prepares before the encoder's first frame and releases after the segment closes, in order, for each segment", async () => { + const events: string[] = []; + const input = fakeStageInput({ totalFrames: 4 }); + const { segmentVideoFrames } = fakeVideoFrames(events, { + prepare: async () => { + events.push(`stage "${input.job.currentStage}"`); + }, + }); + + await runCaptureSegmentedStage({ + ...input, + segmentFrames: 2, + segmentDir: join(fixtureRoot, "windowed-order"), + segmentVideoFrames, + deps: stageDeps({ spawnEncoder: loggingEncoder(events) }), + }); + + expect(events).toEqual([ + "prepare 0", + 'stage "Reading video for segment 1/2"', + "spawn 0", + "frame 0", + "frame 1", + "close 0", + "release", + "prepare 1", + 'stage "Reading video for segment 2/2"', + "spawn 1", + "frame 2", + "frame 3", + "close 1", + "release", + ]); + }); + + it("a retried segment prepares again from a released state", async () => { + const events: string[] = []; + let next = 0; + let failed = false; + const { segmentVideoFrames } = fakeVideoFrames(events); + + await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 4 }), + segmentFrames: 2, + segmentDir: join(fixtureRoot, "windowed-retry"), + sessionFactory: { + create: async () => ({ ...fakeStageInput({ totalFrames: 1 }).probeSession, id: next++ }), + }, + segmentVideoFrames, + deps: stageDeps({ + spawnEncoder: loggingEncoder(events), + captureFrame: mock(async (session: unknown, i: number) => { + if (i === 2 && !failed) { + failed = true; + throw new Error("Protocol error (Page.captureScreenshot): Target closed"); + } + return frameBuffer(session, i); + }), + }), + }); + + expect(events.filter((e) => e.startsWith("prepare") || e === "release")).toEqual([ + "prepare 0", + "release", + "prepare 1", + "release", + "prepare 1", + "release", + ]); + }); + + it("a segment skipped on resume never prepares", async () => { + const events: string[] = []; + const frames = fakeVideoFrames(events); + + await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 6 }), + segmentFrames: 2, + segmentDir: join(fixtureRoot, "windowed-resume"), + completedSegments: new Set([0]), + segmentVideoFrames: frames.segmentVideoFrames, + deps: stageDeps({ spawnEncoder: loggingEncoder(events) }), + }); + + expect(frames.prepared).toEqual([1, 2]); + }); + + it("an abort mid-segment releases its folder", async () => { + const folderRoot = join(fixtureRoot, "windowed-abort-folders"); + const folderOf = (index: number) => join(folderRoot, `segment_${index}`); + + const midCapture = new AbortController(); + await expect( + runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 4 }), + segmentFrames: 4, + segmentDir: join(fixtureRoot, "windowed-abort-capture"), + abortSignal: midCapture.signal, + assertNotAborted: () => { + if (midCapture.signal.aborted) throw new Error("render cancelled"); + }, + segmentVideoFrames: fakeVideoFrames([], { folderRoot }).segmentVideoFrames, + deps: stageDeps({ + spawnEncoder: loggingEncoder([]), + captureFrame: mock(async (session: unknown, i: number) => { + if (i === 1) midCapture.abort(); + return frameBuffer(session, i); + }), + }), + }), + ).rejects.toThrow("render cancelled"); + expect(existsSync(folderOf(0))).toBe(false); + + const midExtraction = new AbortController(); + const extraction = runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 4 }), + segmentFrames: 4, + segmentDir: join(fixtureRoot, "windowed-abort-extraction"), + abortSignal: midExtraction.signal, + assertNotAborted: () => { + if (midExtraction.signal.aborted) throw new Error("render cancelled"); + }, + segmentVideoFrames: fakeVideoFrames([], { + folderRoot, + prepare: () => + new Promise((_resolve, reject) => { + expect(existsSync(folderOf(0))).toBe(true); + midExtraction.signal.addEventListener("abort", () => reject(new Error("aborted"))); + setTimeout(() => midExtraction.abort(), 5); + }), + }).segmentVideoFrames, + deps: stageDeps({ spawnEncoder: loggingEncoder([]) }), + }); + await expect(extraction).rejects.toThrow("aborted"); + expect(existsSync(folderOf(0))).toBe(false); + }); + + it("a recycled session keeps the worker's hook", async () => { + createdSessionHooks.length = 0; + const frames = fakeVideoFrames([]); + + await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 6 }), + probeSession: null, + segmentFrames: 2, + segmentDir: join(fixtureRoot, "windowed-recycle"), + browserRecycleEverySegments: 1, + segmentVideoFrames: frames.segmentVideoFrames, + deps: stageDeps({ spawnEncoder: loggingEncoder([]) }), + }); + + expect(frames.hooks).toHaveLength(1); + expect(createdSessionHooks).toEqual([frames.hooks[0], frames.hooks[0], frames.hooks[0]]); + }); + + it("gives each worker its own hook, on the reused probe session and on a fresh one", async () => { + createdSessionHooks.length = 0; + reusedSessionHooks.length = 0; + const frames = fakeVideoFrames([]); + + await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 6 }), + segmentFrames: 3, + segmentDir: join(fixtureRoot, "windowed-two-workers"), + workerCount: 2, + segmentVideoFrames: frames.segmentVideoFrames, + deps: stageDeps({ spawnEncoder: loggingEncoder([]) }), + }); + + expect(frames.hooks).toHaveLength(2); + expect(reusedSessionHooks).toHaveLength(1); + expect(createdSessionHooks).toHaveLength(1); + expect(new Set([...reusedSessionHooks, ...createdSessionHooks])).toEqual(new Set(frames.hooks)); + }); + + it("time spent preparing does not count toward the capture stall timeout", async () => { + // The stall clock runs inside the frame loop, which starts once the encoder is + // open. Preparing happens before it, so a slow read of the source video is not a stall. + const previous = process.env.HF_DE_STALL_MS; + process.env.HF_DE_STALL_MS = "60"; + try { + const frames = fakeVideoFrames([], { + prepare: () => new Promise((resolve) => setTimeout(resolve, 250)), + }); + const result = await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 2 }), + segmentFrames: 2, + segmentDir: join(fixtureRoot, "windowed-stall"), + segmentVideoFrames: frames.segmentVideoFrames, + deps: stageDeps({ spawnEncoder: loggingEncoder([]) }), + }); + expect(result.success).toBe(true); + expect(frames.prepared).toEqual([0]); + } finally { + if (previous === undefined) delete process.env.HF_DE_STALL_MS; + else process.env.HF_DE_STALL_MS = previous; + } + }); + + it("with segmentVideoFrames, segment files are removed after the join succeeds", async () => { + const events: string[] = []; + const segmentDir = join(fixtureRoot, "windowed-join"); + + await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 6 }), + segmentFrames: 2, + segmentDir, + segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames, + deps: stageDeps({ + spawnEncoder: loggingEncoder([]), + concat: mock(async () => { + events.push("concat"); + return { success: true as const }; + }), + removeFile: (path: string) => events.push(`remove ${path}`), + }), + }); + + expect(events).toEqual([ + "concat", + ...[0, 1, 2].map((i) => `remove ${segmentOutputPath(segmentDir, i)}`), + ]); + }); + + it("with segmentVideoFrames, a resume that only joins removes the segment files too", async () => { + const removed: string[] = []; + const segmentDir = join(fixtureRoot, "windowed-resume-join"); + + await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 4 }), + segmentFrames: 2, + segmentDir, + completedSegments: new Set([0, 1]), + segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames, + deps: stageDeps({ removeFile: (path: string) => removed.push(path) }), + }); + + expect(removed).toEqual([0, 1].map((i) => segmentOutputPath(segmentDir, i))); + }); + + it("a failed join removes no segment files", async () => { + const removed: string[] = []; + + await expect( + runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 4 }), + segmentFrames: 2, + segmentDir: join(fixtureRoot, "windowed-join-fails"), + segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames, + deps: stageDeps({ + spawnEncoder: loggingEncoder([]), + concat: mock(async () => ({ success: false as const, error: "concat failed" })), + removeFile: (path: string) => removed.push(path), + }), + }), + ).rejects.toThrow(/concat failed/); + + expect(removed).toEqual([]); + }); + + it("without segmentVideoFrames, segment files are kept", async () => { + const removed: string[] = []; + + const result = await runCaptureSegmentedStage({ + ...fakeStageInput({ totalFrames: 4 }), + segmentFrames: 2, + segmentDir: join(fixtureRoot, "plain-keeps-files"), + deps: stageDeps({ + spawnEncoder: loggingEncoder([]), + removeFile: (path: string) => removed.push(path), + }), + }); + + expect(result.success).toBe(true); + expect(removed).toEqual([]); + }); +}); diff --git a/packages/producer/src/services/render/stages/captureSegmentedStage.ts b/packages/producer/src/services/render/stages/captureSegmentedStage.ts index 7b5f69f8d3..5c4d62f540 100644 --- a/packages/producer/src/services/render/stages/captureSegmentedStage.ts +++ b/packages/producer/src/services/render/stages/captureSegmentedStage.ts @@ -42,6 +42,7 @@ import { reportFrameProgress } from "../shared.js"; import { encoderFailureError } from "../encoderInterruption.js"; import type { SdrSegmentedCapturePlan } from "../capturePlan.js"; import { planSegments, type SegmentSlice } from "../segmentPlan.js"; +import type { SegmentVideoFrames, SegmentWorkerFrames } from "../segmentVideoFrames.js"; import { createSegmentQueue } from "../segmentQueue.js"; import { isTargetLossError } from "../segmentRecycle.js"; import { @@ -59,9 +60,9 @@ export interface SegmentedStageDeps { removeFile: (path: string) => void; } -/** Opens a fresh, initialized capture session with the video injector attached. */ +/** Opens a fresh, initialized capture session with the video injector attached; a custom factory may ignore the override. */ export interface SessionFactory { - create: () => Promise; + create: (injectorOverride?: () => BeforeCaptureHook | null) => Promise; } export interface CaptureSegmentedStageInput { @@ -80,6 +81,8 @@ export interface CaptureSegmentedStageInput { streamingEncoderOptions: StreamingEncoderOptions; buildCaptureOptions: () => CaptureOptions; createRenderVideoFrameInjector: () => BeforeCaptureHook | null; + /** Windowed frames: extracted per segment, deleted when it closes; segment files are deleted once joined. */ + segmentVideoFrames?: SegmentVideoFrames; abortSignal: AbortSignal | undefined; assertNotAborted: () => void; onProgress?: ProgressCallback; @@ -174,6 +177,7 @@ class SegmentEncoderSpawnError extends Error { interface SegmentCaptureContext { session: CaptureSession; + frames?: SegmentWorkerFrames; job: RenderJob; cfg: EngineConfig; totalFrames: number; @@ -228,10 +232,34 @@ async function captureSegmentFrames( } } -/** Capture one segment into its own encoder. Returns the encoder's result. */ +/** Capture one segment; when windowed, its video frames are read first and released however it ends. */ async function captureSegment( ctx: SegmentCaptureContext, segment: SegmentSlice, +): Promise { + const { frames } = ctx; + if (!frames) return captureSegmentToEncoder(ctx, segment); + try { + reportFrameProgress( + ctx.job, + `Reading video for segment ${segment.index + 1}/${ctx.segmentCount}`, + Math.round(25 + (segment.startFrame / ctx.totalFrames) * 55), + ctx.onProgress, + segment.startFrame, + ctx.totalFrames, + ); + await frames.prepare(segment); + ctx.assertNotAborted(); + return await captureSegmentToEncoder(ctx, segment); + } finally { + frames.release(); + } +} + +/** Capture one segment into its own encoder. Returns the encoder's result. */ +async function captureSegmentToEncoder( + ctx: SegmentCaptureContext, + segment: SegmentSlice, ): Promise { const segmentPath = segmentOutputPath(ctx.segmentDir, segment.index); let encoder: StreamingEncoder; @@ -295,15 +323,15 @@ const DEFAULT_DEPS: SegmentedStageDeps = { function defaultSessionFactory(input: CaptureSegmentedStageInput): SessionFactory { let probeSession = input.probeSession; return { - create: async () => { + create: async (injector = input.createRenderVideoFrameInjector) => { const probeCannotCaptureEveryFrame = Boolean(probeSession?.deRiskFrames?.size); if (probeSession && probeCannotCaptureEveryFrame) { await closeCaptureSession(probeSession).catch(() => {}); probeSession = null; } - const session = probeSession ?? (await openSegmentedSession(input)); + const session = probeSession ?? (await openSegmentedSession(input, injector)); if (probeSession) { - openSegmentedSessionReuse(input, probeSession); + openSegmentedSessionReuse(input, probeSession, injector); probeSession = null; } try { @@ -321,18 +349,22 @@ function defaultSessionFactory(input: CaptureSegmentedStageInput): SessionFactor function openSegmentedSessionReuse( input: CaptureSegmentedStageInput, session: CaptureSession, + injector: () => BeforeCaptureHook | null, ): void { - prepareCaptureSessionForReuse(session, input.framesDir, input.createRenderVideoFrameInjector()); + prepareCaptureSessionForReuse(session, input.framesDir, injector()); } -async function openSegmentedSession(input: CaptureSegmentedStageInput): Promise { +async function openSegmentedSession( + input: CaptureSegmentedStageInput, + injector: () => BeforeCaptureHook | null, +): Promise { // Same reasoning as the streaming stage: the resolved forceScreenshot comes // from the immutable plan, not from the caller-owned cfg. const captureCfg: EngineConfig = input.cfg.forceScreenshot === input.plan.forceScreenshot ? input.cfg : { ...input.cfg, forceScreenshot: input.plan.forceScreenshot }; - const videoInjector = input.createRenderVideoFrameInjector(); + const videoInjector = injector(); const captureOptions = { ...input.buildCaptureOptions(), deFrameFallback: false }; const ownBrowser = shouldDisableBrowserPoolForParallelWorker({ parallel: (input.workerCount ?? 1) > 1, @@ -346,6 +378,12 @@ async function openSegmentedSession(input: CaptureSegmentedStageInput): Promise< ); } +/** Once joined, the windowed route has no use for the segments; keeping them would hold the whole video twice. */ +function removeJoinedSegments(run: SegmentRun, segmentPaths: readonly string[]): void { + if (!run.input.segmentVideoFrames) return; + for (const path of segmentPaths) run.deps.removeFile(path); +} + /** Stream-copy the finished segments into the video-only file, or throw. */ async function concatSegments( concat: SegmentedStageDeps["concat"], @@ -395,12 +433,14 @@ interface SegmentRun { async function createSegmentWorker(run: SegmentRun, id: number): Promise { const { input, deps } = run; const factory = input.sessionFactoryForWorker?.(id) ?? run.sessionFactory; + const frames = input.segmentVideoFrames?.forWorker(); return { id, factory, sessionSegments: 0, ctx: { - session: await factory.create(), + session: await factory.create(frames && (() => frames.hook)), + frames, job: input.job, cfg: input.cfg, totalFrames: input.totalFrames, @@ -441,7 +481,8 @@ async function recycleWorkerSession( run.input.dedupPerfs.push(getCapturePerfSummary(worker.ctx.session)); const memory = worker.ctx.session.chromeMemory?.stats(); await run.deps.closeSession(worker.ctx.session); - worker.ctx.session = await worker.factory.create(); + const { frames } = worker.ctx; + worker.ctx.session = await worker.factory.create(frames && (() => frames.hook)); worker.sessionSegments = 0; if (why === "cadence") run.totals.browserRecycles += 1; run.input.log.info(`[Render] segment browser recycled (${why})`, { @@ -549,6 +590,7 @@ async function concatResumedRun( input.abortSignal, input.cfg, ); + removeJoinedSegments(run, segmentPaths); return { success: true, encodeMs: 0, @@ -639,6 +681,7 @@ export async function runCaptureSegmentedStage( input.abortSignal, input.cfg, ); + removeJoinedSegments(run, segmentPaths); return { success: true,