diff --git a/packages/producer/src/services/render/segmentManifest.test.ts b/packages/producer/src/services/render/segmentManifest.test.ts index 95ed19fa40..5de0db5660 100644 --- a/packages/producer/src/services/render/segmentManifest.test.ts +++ b/packages/producer/src/services/render/segmentManifest.test.ts @@ -97,6 +97,7 @@ describe("manifest io", () => { const manifest: SegmentManifest = { version: 1, planHash: "0123456789abcdef", + forceScreenshot: false, totalFrames: 7, segmentFrames: 3, completed: [ @@ -133,6 +134,21 @@ describe("manifest io", () => { expect(readSegmentManifest(dir)).toBeNull(); }); + it("does not resume from a manifest that does not record its capture mode", () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + writeFileSync( + join(dir, "segments.json"), + JSON.stringify({ + version: 1, + planHash: "h", + totalFrames: 3, + segmentFrames: 3, + completed: [], + }), + ); + expect(readSegmentManifest(dir)).toBeNull(); + }); + it("segmentDirFor nests under renders/.hf-segments", () => { dir = mkdtempSync(join(tmpdir(), "hf-seg-")); expect(segmentDirFor("/p/renders", "abc")).toBe("/p/renders/.hf-segments/abc"); @@ -140,6 +156,7 @@ describe("manifest io", () => { }); describe("validateCompletedSegments", () => { + const unforced = (planHash: string) => ({ planHash, forceScreenshot: false }); let dir = ""; afterEach(() => rmSync(dir, { recursive: true, force: true })); @@ -153,6 +170,7 @@ describe("validateCompletedSegments", () => { const manifest: SegmentManifest = { version: 1, planHash: "h", + forceScreenshot: false, totalFrames: 9, segmentFrames: 3, completed: [ @@ -169,7 +187,7 @@ describe("validateCompletedSegments", () => { ], }; const probe = async (p: string) => (p === ok ? 3 : p === short ? 2 : null); - expect([...(await validateCompletedSegments(manifest, "h", probe))]).toEqual([0]); + expect([...(await validateCompletedSegments(manifest, unforced("h"), probe))]).toEqual([0]); }); it("rejects a segment whose size no longer matches the manifest", async () => { @@ -181,13 +199,14 @@ describe("validateCompletedSegments", () => { const manifest: SegmentManifest = { version: 1, planHash: "h", + forceScreenshot: false, totalFrames: 3, segmentFrames: 3, completed: [ { index: 0, startFrame: 0, endFrame: 3, path: truncated, bytes: 999, completedAt: "" }, ], }; - expect((await validateCompletedSegments(manifest, "h", async () => 3)).size).toBe(0); + expect((await validateCompletedSegments(manifest, unforced("h"), async () => 3)).size).toBe(0); }); it("skips nothing when the plan hash differs", async () => { @@ -200,12 +219,39 @@ describe("validateCompletedSegments", () => { const manifest: SegmentManifest = { version: 1, planHash: "old", + forceScreenshot: false, totalFrames: 3, segmentFrames: 3, completed: [{ index: 0, startFrame: 0, endFrame: 3, path: good, bytes: 2, completedAt: "" }], }; - expect((await validateCompletedSegments(manifest, "old", async () => 3)).size).toBe(1); - expect((await validateCompletedSegments(manifest, "new", async () => 3)).size).toBe(0); + expect((await validateCompletedSegments(manifest, unforced("old"), async () => 3)).size).toBe( + 1, + ); + expect((await validateCompletedSegments(manifest, unforced("new"), async () => 3)).size).toBe( + 0, + ); + }); + + it("skips nothing when the segments were captured in another capture mode", async () => { + dir = mkdtempSync(join(tmpdir(), "hf-seg-")); + const good = join(dir, "segment_00000.mp4"); + writeFileSync(good, "xx"); + const screenshotManifest: SegmentManifest = { + version: 1, + planHash: "h", + forceScreenshot: true, + totalFrames: 3, + segmentFrames: 3, + completed: [{ index: 0, startFrame: 0, endFrame: 3, path: good, bytes: 2, completedAt: "" }], + }; + const resume = (forceScreenshot: boolean) => + validateCompletedSegments( + screenshotManifest, + { planHash: "h", forceScreenshot }, + async () => 3, + ); + expect((await resume(true)).size).toBe(1); + expect((await resume(false)).size).toBe(0); }); }); diff --git a/packages/producer/src/services/render/segmentManifest.ts b/packages/producer/src/services/render/segmentManifest.ts index a484aa00be..f13dfe3526 100644 --- a/packages/producer/src/services/render/segmentManifest.ts +++ b/packages/producer/src/services/render/segmentManifest.ts @@ -23,6 +23,7 @@ export interface SegmentManifestEntry { export interface SegmentManifest { version: 1; planHash: string; + forceScreenshot: boolean; totalFrames: number; segmentFrames: number; completed: SegmentManifestEntry[]; @@ -109,6 +110,7 @@ function isManifest(value: unknown): value is SegmentManifest { return ( Reflect.get(value, "version") === 1 && hasString(value, "planHash") && + typeof Reflect.get(value, "forceScreenshot") === "boolean" && hasNumber(value, "totalFrames") && hasNumber(value, "segmentFrames") && Array.isArray(completed) && @@ -162,18 +164,19 @@ export async function probeSegmentFrameCount( } /** - * Indices safe to skip on resume. A segment counts only if the hash matches, - * the file exists at the recorded size, and its frame count equals the slice - * length. Anything else re-captures — the cost of re-capturing a good segment - * is minutes; the cost of reusing a bad one is a silently wrong video. + * Indices safe to skip on resume. A segment counts only if the hash and mode + * match, the file exists at the recorded size, and its frame count equals the + * slice length. Anything else re-captures — the cost of re-capturing a good + * segment is minutes; the cost of reusing a bad one is a silently wrong video. */ export async function validateCompletedSegments( manifest: SegmentManifest, - expectedHash: string, + expected: Pick, probeFrames: (path: string) => Promise, ): Promise> { const ok = new Set(); - if (manifest.planHash !== expectedHash) return ok; + if (manifest.planHash !== expected.planHash) return ok; + if (manifest.forceScreenshot !== expected.forceScreenshot) return ok; for (const entry of manifest.completed) { if (!existsSync(entry.path)) continue; const size = statSync(entry.path).size; diff --git a/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts b/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts index ccc611b218..f737d133db 100644 --- a/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts +++ b/packages/producer/src/services/renderOrchestrator.segmentedDeRetry.test.ts @@ -1,7 +1,15 @@ import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { DrawElementCaptureError, DrawElementVerificationError, @@ -19,10 +27,12 @@ const { state, RETRY_REACHED } = vi.hoisted(() => ({ kind: string; probeSession: unknown; completedSegments: number; + segmentDir: string; segmentDirEntries: string[]; }>, failures: [] as unknown[], resumable: false, + storedForceScreenshot: undefined as boolean | undefined, sessions: [] as object[], closed: [] as object[], }, @@ -113,15 +123,43 @@ vi.mock("./render/captureCost.js", async (importOriginal) => ({ }), })); // A previous run left segment 0 complete; validating it for real needs ffprobe on a real file. +// With storedForceScreenshot set, that segment is real and only the probe is faked. vi.mock("./render/segmentManifest.js", async (importOriginal) => { const actual = await importOriginal(); + const storedSegmentManifest = (dir: string, forceScreenshot: boolean) => { + mkdirSync(dir, { recursive: true }); + const path = join(dir, "segment_00000.mp4"); + writeFileSync(path, "stored segment"); + return { + version: 1 as const, + planHash: basename(dir), + forceScreenshot, + totalFrames: 1200, + segmentFrames: 600, + completed: [{ index: 0, startFrame: 0, endFrame: 600, path, bytes: 14, completedAt: "" }], + }; + }; return { ...actual, readSegmentManifest: (dir: string) => - state.resumable - ? { version: 1, planHash: "", totalFrames: 1200, segmentFrames: 600, completed: [] } - : actual.readSegmentManifest(dir), - validateCompletedSegments: async () => new Set(state.resumable ? [0] : []), + state.storedForceScreenshot !== undefined + ? storedSegmentManifest(dir, state.storedForceScreenshot) + : state.resumable + ? { + version: 1, + planHash: "", + forceScreenshot: false, + totalFrames: 1200, + segmentFrames: 600, + completed: [], + } + : actual.readSegmentManifest(dir), + validateCompletedSegments: async ( + ...args: Parameters + ) => + state.storedForceScreenshot !== undefined + ? actual.validateCompletedSegments(args[0], args[1], async () => 600) + : new Set(state.resumable ? [0] : []), }; }); vi.mock("./render/stages/captureSegmentedStage.js", async (importOriginal) => ({ @@ -144,15 +182,16 @@ vi.mock("./render/stages/captureSegmentedStage.js", async (importOriginal) => ({ kind: input.plan.kind, probeSession: input.probeSession, completedSegments: input.completedSegments?.size ?? 0, + segmentDir: input.segmentDir, segmentDirEntries: existsSync(input.segmentDir) ? readdirSync(input.segmentDir) : [], }); const failure = state.failures.shift(); - if (failure === undefined) throw new Error(RETRY_REACHED); // A segment finished before the failing frame, as it would mid-render. mkdirSync(input.segmentDir, { recursive: true }); const path = join(input.segmentDir, "segment_00000.mp4"); writeFileSync(path, "drawElement segment"); input.onSegmentComplete?.({ index: 0, startFrame: 0, endFrame: 600, path, bytes: 19 }); + if (failure === undefined) throw new Error(RETRY_REACHED); throw new CaptureStageError({ cause: failure, browserConsole: [] }); }, })); @@ -164,6 +203,7 @@ beforeEach(() => { state.sessions = []; state.closed = []; state.resumable = false; + state.storedForceScreenshot = undefined; vi.stubEnv("HF_SEGMENTED_CAPTURE", "true"); vi.stubEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", ""); vi.stubEnv("HF_DE_PARALLEL_STREAM", ""); @@ -234,6 +274,27 @@ it("re-captures every segment, including resumed ones, after a drawElement failu expect(state.attempts[1]?.segmentDirEntries).toEqual([]); }); +it("records the screenshot capture mode in the manifest the retry writes", async () => { + state.failures = [new DrawElementCaptureError(640, "suspect small frame")]; + await expect(render()).rejects.toThrow(RETRY_REACHED); + const retryDir = state.attempts[1]?.segmentDir ?? ""; + const manifest = JSON.parse(readFileSync(join(retryDir, "segments.json"), "utf-8")); + expect(manifest.forceScreenshot).toBe(true); +}); + +it.each([ + ["reuses", false, 1], + ["does not reuse", true, 0], +])( + "%s segments a drawElement resume finds captured with forceScreenshot %s", + async (_label, storedForceScreenshot, reused) => { + state.storedForceScreenshot = storedForceScreenshot; + await expect(render(true)).rejects.toThrow(RETRY_REACHED); + expect(state.attempts[0]?.forceScreenshot).toBe(false); + expect(state.attempts[0]?.completedSegments).toBe(reused); + }, +); + it("does not retry a second drawElement failure", async () => { state.failures = [ new DrawElementCaptureError(640, "suspect small frame"), diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 21f5ebbbd8..6bece3b600 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -4651,8 +4651,10 @@ async function executeRenderPipeline(input: { if (job.config.resumeSegments === true) { const existing = readSegmentManifest(segmentDir); if (existing) { - completedSegments = await validateCompletedSegments(existing, segmentPlanHash, (path) => - probeSegmentFrameCount(path, extractMediaMetadata), + completedSegments = await validateCompletedSegments( + existing, + { planHash: segmentPlanHash, forceScreenshot: capturePlan.forceScreenshot }, + (path) => probeSegmentFrameCount(path, extractMediaMetadata), ); log.info(`[Render] resuming: ${completedSegments.size} segments complete`, { segmentDir, @@ -4669,6 +4671,7 @@ async function executeRenderPipeline(input: { const emptySegmentManifest = (): SegmentManifest => ({ version: 1, planHash: segmentPlanHash, + forceScreenshot: capturePlan.forceScreenshot, totalFrames, segmentFrames, completed: [], @@ -4725,6 +4728,7 @@ async function executeRenderPipeline(input: { browserRecycleEverySegments: resolveSegmentBrowserRecycle(process.env), completedSegments, onSegmentComplete: (entry) => { + segmentManifest.forceScreenshot = segmentedPlan.forceScreenshot; segmentManifest.completed = [ ...segmentManifest.completed.filter((e) => e.index !== entry.index), { ...entry, completedAt: new Date().toISOString() },