mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
fix(producer): do not resume segments captured in another capture mode
The plan hash does not cover forceScreenshot, so a killed screenshot retry left screenshot segments under the same hash, and the next --resume run planned drawElement again and spliced them in. Toggling --force-screenshot between runs had the same gap. Record the plan's forceScreenshot in the segment manifest, stamped from the plan each segment was captured under, so the retry's manifest says screenshot. validateCompletedSegments returns no completed segments when the recorded mode differs from the plan's. computeSegmentPlanHash is unchanged. A manifest without the field is rejected when read, so resume starts over. Such a manifest only exists for an interrupted render from a build before this one, and its capture mode is unknowable: reusing it risks a silently mixed video, re-capturing costs minutes once. — Via Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
30e2987d52
commit
187cddbd2d
@@ -97,6 +97,7 @@ describe("manifest io", () => {
|
||||
const manifest: SegmentManifest = {
|
||||
version: 1,
|
||||
planHash: "0123456789abcdef",
|
||||
forceScreenshot: false,
|
||||
totalFrames: 7,
|
||||
segmentFrames: 3,
|
||||
completed: [
|
||||
@@ -133,6 +134,21 @@ describe("manifest io", () => {
|
||||
expect(readSegmentManifest(dir)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not resume from a manifest that does not record its capture mode", () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
writeFileSync(
|
||||
join(dir, "segments.json"),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
planHash: "h",
|
||||
totalFrames: 3,
|
||||
segmentFrames: 3,
|
||||
completed: [],
|
||||
}),
|
||||
);
|
||||
expect(readSegmentManifest(dir)).toBeNull();
|
||||
});
|
||||
|
||||
it("segmentDirFor nests under renders/.hf-segments", () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
expect(segmentDirFor("/p/renders", "abc")).toBe("/p/renders/.hf-segments/abc");
|
||||
@@ -140,6 +156,7 @@ describe("manifest io", () => {
|
||||
});
|
||||
|
||||
describe("validateCompletedSegments", () => {
|
||||
const unforced = (planHash: string) => ({ planHash, forceScreenshot: false });
|
||||
let dir = "";
|
||||
afterEach(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
@@ -153,6 +170,7 @@ describe("validateCompletedSegments", () => {
|
||||
const manifest: SegmentManifest = {
|
||||
version: 1,
|
||||
planHash: "h",
|
||||
forceScreenshot: false,
|
||||
totalFrames: 9,
|
||||
segmentFrames: 3,
|
||||
completed: [
|
||||
@@ -169,7 +187,7 @@ describe("validateCompletedSegments", () => {
|
||||
],
|
||||
};
|
||||
const probe = async (p: string) => (p === ok ? 3 : p === short ? 2 : null);
|
||||
expect([...(await validateCompletedSegments(manifest, "h", probe))]).toEqual([0]);
|
||||
expect([...(await validateCompletedSegments(manifest, unforced("h"), probe))]).toEqual([0]);
|
||||
});
|
||||
|
||||
it("rejects a segment whose size no longer matches the manifest", async () => {
|
||||
@@ -181,13 +199,14 @@ describe("validateCompletedSegments", () => {
|
||||
const manifest: SegmentManifest = {
|
||||
version: 1,
|
||||
planHash: "h",
|
||||
forceScreenshot: false,
|
||||
totalFrames: 3,
|
||||
segmentFrames: 3,
|
||||
completed: [
|
||||
{ index: 0, startFrame: 0, endFrame: 3, path: truncated, bytes: 999, completedAt: "" },
|
||||
],
|
||||
};
|
||||
expect((await validateCompletedSegments(manifest, "h", async () => 3)).size).toBe(0);
|
||||
expect((await validateCompletedSegments(manifest, unforced("h"), async () => 3)).size).toBe(0);
|
||||
});
|
||||
|
||||
it("skips nothing when the plan hash differs", async () => {
|
||||
@@ -200,12 +219,39 @@ describe("validateCompletedSegments", () => {
|
||||
const manifest: SegmentManifest = {
|
||||
version: 1,
|
||||
planHash: "old",
|
||||
forceScreenshot: false,
|
||||
totalFrames: 3,
|
||||
segmentFrames: 3,
|
||||
completed: [{ index: 0, startFrame: 0, endFrame: 3, path: good, bytes: 2, completedAt: "" }],
|
||||
};
|
||||
expect((await validateCompletedSegments(manifest, "old", async () => 3)).size).toBe(1);
|
||||
expect((await validateCompletedSegments(manifest, "new", async () => 3)).size).toBe(0);
|
||||
expect((await validateCompletedSegments(manifest, unforced("old"), async () => 3)).size).toBe(
|
||||
1,
|
||||
);
|
||||
expect((await validateCompletedSegments(manifest, unforced("new"), async () => 3)).size).toBe(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it("skips nothing when the segments were captured in another capture mode", async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
|
||||
const good = join(dir, "segment_00000.mp4");
|
||||
writeFileSync(good, "xx");
|
||||
const screenshotManifest: SegmentManifest = {
|
||||
version: 1,
|
||||
planHash: "h",
|
||||
forceScreenshot: true,
|
||||
totalFrames: 3,
|
||||
segmentFrames: 3,
|
||||
completed: [{ index: 0, startFrame: 0, endFrame: 3, path: good, bytes: 2, completedAt: "" }],
|
||||
};
|
||||
const resume = (forceScreenshot: boolean) =>
|
||||
validateCompletedSegments(
|
||||
screenshotManifest,
|
||||
{ planHash: "h", forceScreenshot },
|
||||
async () => 3,
|
||||
);
|
||||
expect((await resume(true)).size).toBe(1);
|
||||
expect((await resume(false)).size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ export interface SegmentManifestEntry {
|
||||
export interface SegmentManifest {
|
||||
version: 1;
|
||||
planHash: string;
|
||||
forceScreenshot: boolean;
|
||||
totalFrames: number;
|
||||
segmentFrames: number;
|
||||
completed: SegmentManifestEntry[];
|
||||
@@ -109,6 +110,7 @@ function isManifest(value: unknown): value is SegmentManifest {
|
||||
return (
|
||||
Reflect.get(value, "version") === 1 &&
|
||||
hasString(value, "planHash") &&
|
||||
typeof Reflect.get(value, "forceScreenshot") === "boolean" &&
|
||||
hasNumber(value, "totalFrames") &&
|
||||
hasNumber(value, "segmentFrames") &&
|
||||
Array.isArray(completed) &&
|
||||
@@ -162,18 +164,19 @@ export async function probeSegmentFrameCount(
|
||||
}
|
||||
|
||||
/**
|
||||
* Indices safe to skip on resume. A segment counts only if the hash matches,
|
||||
* the file exists at the recorded size, and its frame count equals the slice
|
||||
* length. Anything else re-captures — the cost of re-capturing a good segment
|
||||
* is minutes; the cost of reusing a bad one is a silently wrong video.
|
||||
* Indices safe to skip on resume. A segment counts only if the hash and mode
|
||||
* match, the file exists at the recorded size, and its frame count equals the
|
||||
* slice length. Anything else re-captures — the cost of re-capturing a good
|
||||
* segment is minutes; the cost of reusing a bad one is a silently wrong video.
|
||||
*/
|
||||
export async function validateCompletedSegments(
|
||||
manifest: SegmentManifest,
|
||||
expectedHash: string,
|
||||
expected: Pick<SegmentManifest, "planHash" | "forceScreenshot">,
|
||||
probeFrames: (path: string) => Promise<number | null>,
|
||||
): Promise<Set<number>> {
|
||||
const ok = new Set<number>();
|
||||
if (manifest.planHash !== expectedHash) return ok;
|
||||
if (manifest.planHash !== expected.planHash) return ok;
|
||||
if (manifest.forceScreenshot !== expected.forceScreenshot) return ok;
|
||||
for (const entry of manifest.completed) {
|
||||
if (!existsSync(entry.path)) continue;
|
||||
const size = statSync(entry.path).size;
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { basename, join } from "node:path";
|
||||
import {
|
||||
DrawElementCaptureError,
|
||||
DrawElementVerificationError,
|
||||
@@ -19,10 +27,12 @@ const { state, RETRY_REACHED } = vi.hoisted(() => ({
|
||||
kind: string;
|
||||
probeSession: unknown;
|
||||
completedSegments: number;
|
||||
segmentDir: string;
|
||||
segmentDirEntries: string[];
|
||||
}>,
|
||||
failures: [] as unknown[],
|
||||
resumable: false,
|
||||
storedForceScreenshot: undefined as boolean | undefined,
|
||||
sessions: [] as object[],
|
||||
closed: [] as object[],
|
||||
},
|
||||
@@ -113,15 +123,43 @@ vi.mock("./render/captureCost.js", async (importOriginal) => ({
|
||||
}),
|
||||
}));
|
||||
// A previous run left segment 0 complete; validating it for real needs ffprobe on a real file.
|
||||
// With storedForceScreenshot set, that segment is real and only the probe is faked.
|
||||
vi.mock("./render/segmentManifest.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("./render/segmentManifest.js")>();
|
||||
const storedSegmentManifest = (dir: string, forceScreenshot: boolean) => {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const path = join(dir, "segment_00000.mp4");
|
||||
writeFileSync(path, "stored segment");
|
||||
return {
|
||||
version: 1 as const,
|
||||
planHash: basename(dir),
|
||||
forceScreenshot,
|
||||
totalFrames: 1200,
|
||||
segmentFrames: 600,
|
||||
completed: [{ index: 0, startFrame: 0, endFrame: 600, path, bytes: 14, completedAt: "" }],
|
||||
};
|
||||
};
|
||||
return {
|
||||
...actual,
|
||||
readSegmentManifest: (dir: string) =>
|
||||
state.resumable
|
||||
? { version: 1, planHash: "", totalFrames: 1200, segmentFrames: 600, completed: [] }
|
||||
: actual.readSegmentManifest(dir),
|
||||
validateCompletedSegments: async () => new Set(state.resumable ? [0] : []),
|
||||
state.storedForceScreenshot !== undefined
|
||||
? storedSegmentManifest(dir, state.storedForceScreenshot)
|
||||
: state.resumable
|
||||
? {
|
||||
version: 1,
|
||||
planHash: "",
|
||||
forceScreenshot: false,
|
||||
totalFrames: 1200,
|
||||
segmentFrames: 600,
|
||||
completed: [],
|
||||
}
|
||||
: actual.readSegmentManifest(dir),
|
||||
validateCompletedSegments: async (
|
||||
...args: Parameters<typeof actual.validateCompletedSegments>
|
||||
) =>
|
||||
state.storedForceScreenshot !== undefined
|
||||
? actual.validateCompletedSegments(args[0], args[1], async () => 600)
|
||||
: new Set(state.resumable ? [0] : []),
|
||||
};
|
||||
});
|
||||
vi.mock("./render/stages/captureSegmentedStage.js", async (importOriginal) => ({
|
||||
@@ -144,15 +182,16 @@ vi.mock("./render/stages/captureSegmentedStage.js", async (importOriginal) => ({
|
||||
kind: input.plan.kind,
|
||||
probeSession: input.probeSession,
|
||||
completedSegments: input.completedSegments?.size ?? 0,
|
||||
segmentDir: input.segmentDir,
|
||||
segmentDirEntries: existsSync(input.segmentDir) ? readdirSync(input.segmentDir) : [],
|
||||
});
|
||||
const failure = state.failures.shift();
|
||||
if (failure === undefined) throw new Error(RETRY_REACHED);
|
||||
// A segment finished before the failing frame, as it would mid-render.
|
||||
mkdirSync(input.segmentDir, { recursive: true });
|
||||
const path = join(input.segmentDir, "segment_00000.mp4");
|
||||
writeFileSync(path, "drawElement segment");
|
||||
input.onSegmentComplete?.({ index: 0, startFrame: 0, endFrame: 600, path, bytes: 19 });
|
||||
if (failure === undefined) throw new Error(RETRY_REACHED);
|
||||
throw new CaptureStageError({ cause: failure, browserConsole: [] });
|
||||
},
|
||||
}));
|
||||
@@ -164,6 +203,7 @@ beforeEach(() => {
|
||||
state.sessions = [];
|
||||
state.closed = [];
|
||||
state.resumable = false;
|
||||
state.storedForceScreenshot = undefined;
|
||||
vi.stubEnv("HF_SEGMENTED_CAPTURE", "true");
|
||||
vi.stubEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE", "");
|
||||
vi.stubEnv("HF_DE_PARALLEL_STREAM", "");
|
||||
@@ -234,6 +274,27 @@ it("re-captures every segment, including resumed ones, after a drawElement failu
|
||||
expect(state.attempts[1]?.segmentDirEntries).toEqual([]);
|
||||
});
|
||||
|
||||
it("records the screenshot capture mode in the manifest the retry writes", async () => {
|
||||
state.failures = [new DrawElementCaptureError(640, "suspect small frame")];
|
||||
await expect(render()).rejects.toThrow(RETRY_REACHED);
|
||||
const retryDir = state.attempts[1]?.segmentDir ?? "";
|
||||
const manifest = JSON.parse(readFileSync(join(retryDir, "segments.json"), "utf-8"));
|
||||
expect(manifest.forceScreenshot).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["reuses", false, 1],
|
||||
["does not reuse", true, 0],
|
||||
])(
|
||||
"%s segments a drawElement resume finds captured with forceScreenshot %s",
|
||||
async (_label, storedForceScreenshot, reused) => {
|
||||
state.storedForceScreenshot = storedForceScreenshot;
|
||||
await expect(render(true)).rejects.toThrow(RETRY_REACHED);
|
||||
expect(state.attempts[0]?.forceScreenshot).toBe(false);
|
||||
expect(state.attempts[0]?.completedSegments).toBe(reused);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not retry a second drawElement failure", async () => {
|
||||
state.failures = [
|
||||
new DrawElementCaptureError(640, "suspect small frame"),
|
||||
|
||||
@@ -4651,8 +4651,10 @@ async function executeRenderPipeline(input: {
|
||||
if (job.config.resumeSegments === true) {
|
||||
const existing = readSegmentManifest(segmentDir);
|
||||
if (existing) {
|
||||
completedSegments = await validateCompletedSegments(existing, segmentPlanHash, (path) =>
|
||||
probeSegmentFrameCount(path, extractMediaMetadata),
|
||||
completedSegments = await validateCompletedSegments(
|
||||
existing,
|
||||
{ planHash: segmentPlanHash, forceScreenshot: capturePlan.forceScreenshot },
|
||||
(path) => probeSegmentFrameCount(path, extractMediaMetadata),
|
||||
);
|
||||
log.info(`[Render] resuming: ${completedSegments.size} segments complete`, {
|
||||
segmentDir,
|
||||
@@ -4669,6 +4671,7 @@ async function executeRenderPipeline(input: {
|
||||
const emptySegmentManifest = (): SegmentManifest => ({
|
||||
version: 1,
|
||||
planHash: segmentPlanHash,
|
||||
forceScreenshot: capturePlan.forceScreenshot,
|
||||
totalFrames,
|
||||
segmentFrames,
|
||||
completed: [],
|
||||
@@ -4725,6 +4728,7 @@ async function executeRenderPipeline(input: {
|
||||
browserRecycleEverySegments: resolveSegmentBrowserRecycle(process.env),
|
||||
completedSegments,
|
||||
onSegmentComplete: (entry) => {
|
||||
segmentManifest.forceScreenshot = segmentedPlan.forceScreenshot;
|
||||
segmentManifest.completed = [
|
||||
...segmentManifest.completed.filter((e) => e.index !== entry.index),
|
||||
{ ...entry, completedAt: new Date().toISOString() },
|
||||
|
||||
Reference in New Issue
Block a user