feat(producer): a capture segment extracts the frames it shows and deletes them (#7241)

* feat(producer): a capture segment extracts the frames it shows and deletes them when it closes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(producer): the windowed video hook reads its own frames outside a prepared window

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(producer): trim the comments the segment video frames hook added

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vance Ingalls
2026-11-05 10:53:17 -08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 35cbca9927
commit 599eaad2ba
5 changed files with 894 additions and 13 deletions
@@ -190,6 +190,33 @@ describe("validateCompletedSegments", () => {
expect((await validateCompletedSegments(manifest, "h", async () => 3)).size).toBe(0);
});
it("a resume whose segment files were removed after a join captures them again rather than failing", async () => {
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
const joined = [0, 1].map((index) => join(dir, `segment_0000${index}.mp4`));
const manifest: SegmentManifest = {
version: 1,
planHash: "h",
totalFrames: 6,
segmentFrames: 3,
completed: joined.map((path, index) => ({
index,
startFrame: index * 3,
endFrame: index * 3 + 3,
path,
bytes: 2,
completedAt: "",
})),
};
// The manifest still lists them, but the join deleted the files.
let probed = 0;
const probe = async () => {
probed += 1;
return 3;
};
expect((await validateCompletedSegments(manifest, "h", probe)).size).toBe(0);
expect(probed).toBe(0);
});
it("skips nothing when the plan hash differs", async () => {
// The entry is otherwise perfectly reusable — present, right size, right
// frame count — so only the hash check can reject it. With an empty
@@ -0,0 +1,304 @@
import { mkdirSync, mkdtempSync, existsSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "bun:test";
import type { EngineConfig, ExtractedFrames, VideoElement } from "@hyperframes/engine";
import { type extractDeferredFrames, shownDeferredFrameIndexes } from "./deferredVideoFrames.js";
import { createSegmentVideoFrames } from "./segmentVideoFrames.js";
const root = mkdtempSync(join(tmpdir(), "hf-segment-video-frames-"));
afterAll(() => rmSync(root, { recursive: true, force: true }));
const metadata = {
durationSeconds: 4,
videoStreamDurationSeconds: 4,
videoStreamStartSeconds: 0,
width: 64,
height: 36,
fps: 30,
videoCodec: "h264",
hasAudio: false,
isVFR: false,
hasAlpha: false,
colorSpace: null,
};
const base = (videoId: string, totalFrames: number): ExtractedFrames => ({
videoId,
srcPath: `/source/${videoId}.mp4`,
outputDir: "",
framePattern: "frame_%05d.jpg",
fps: 30,
totalFrames,
metadata,
framePaths: new Map(),
});
const deferredVideo = (videoId: string): ExtractedFrames => ({
...base(videoId, 75),
deferredRange: { startTime: 1.5, durationSeconds: 2.5, format: "jpg" },
});
const wholeVideo = (videoId: string): ExtractedFrames => ({
...base(videoId, 75),
outputDir: "/full-extraction/whole",
framePaths: new Map(
Array.from({ length: 75 }, (_, i) => [i, `/full-extraction/whole/frame_${i + 1}.jpg`]),
),
});
const element = (id: string, start: number, end: number): VideoElement => ({
id,
src: `${id}.mp4`,
start,
end,
mediaStart: 1.5,
loop: false,
hasAudio: false,
});
const fps = { num: 30, den: 1 };
const cfg = { ffmpegProcessTimeout: 1000 } as EngineConfig;
const slice = (index: number, startFrame: number, endFrame: number) => ({
index,
startFrame,
endFrame,
});
const span = (start: number, end: number) =>
Array.from({ length: end - start }, (_, i) => start + i);
/** Stands in for the extractor: writes the frames the slice shows, as the real one names them. */
function fakeExtractor() {
const calls: Array<{ outputRoot: string; frames: number[]; sourceOf: string[] }> = [];
const extractFrames = async (input: Parameters<typeof extractDeferredFrames>[0]) => {
const frames = [...input.frames];
const shown = shownDeferredFrameIndexes(input, frames, input.fps);
calls.push({
outputRoot: input.outputRoot,
frames,
sourceOf: input.extracted.filter((v) => v.deferredRange).map((v) => input.sourceOf(v)),
});
return input.extracted
.filter((video) => video.deferredRange)
.map((video) => {
const outputDir = join(input.outputRoot, video.videoId);
const framePaths = new Map<number, string>();
for (const index of shown.get(video.videoId) ?? []) {
mkdirSync(outputDir, { recursive: true });
const path = join(outputDir, `frame_${String(index + 1).padStart(5, "0")}.jpg`);
writeFileSync(path, "x");
framePaths.set(index, path);
}
return { ...video, outputDir, framePaths, ownedByLookup: false };
});
};
return { calls, extractFrames };
}
function setup(extra: Partial<Parameters<typeof createSegmentVideoFrames>[0]> = {}) {
const served: string[] = [];
const dir = mkdtempSync(join(root, "work-"));
const extractor = fakeExtractor();
const segmentVideoFrames = createSegmentVideoFrames({
videos: [element("clip", 0, 4), element("whole", 0, 4)],
extracted: [deferredVideo("clip"), wholeVideo("whole")],
fps,
rootDir: dir,
cfg,
injectorOptions: {
frameSrcResolver: (path) => {
served.push(path);
return `http://frames/${path}`;
},
},
extractFrames: extractor.extractFrames,
...extra,
});
const page = { evaluate: async () => [] } as never;
return {
dir,
served,
extractor,
segmentVideoFrames,
workerFrames: segmentVideoFrames.forWorker(),
page,
};
}
describe("createSegmentVideoFrames", () => {
it("extracts only the frames a slice shows, under that slice's own folder", async () => {
const { dir, served, extractor, workerFrames, page } = setup();
await workerFrames.prepare(slice(1, 30, 45));
const folder = join(dir, "segment-frames", "segment_00001");
expect(extractor.calls).toHaveLength(1);
expect(extractor.calls[0]!.outputRoot).toBe(folder);
expect(extractor.calls[0]!.frames).toEqual(span(30, 45));
expect(readdirSync(join(folder, "clip")).sort()).toEqual(
span(30, 45).map((i) => `frame_${String(i + 1).padStart(5, "0")}.jpg`),
);
await workerFrames.hook(page, 31 / 30);
expect(served).toContain(join(folder, "clip", "frame_00032.jpg"));
});
it("release removes the folder and a second release does nothing", async () => {
const { dir, workerFrames } = setup();
await workerFrames.prepare(slice(2, 60, 75));
const folder = join(dir, "segment-frames", "segment_00002");
expect(existsSync(folder)).toBe(true);
workerFrames.release();
expect(existsSync(folder)).toBe(false);
expect(() => workerFrames.release()).not.toThrow();
});
it("a video that was fully extracted is served from its existing frames and never extracted again", async () => {
const { served, extractor, workerFrames, page } = setup();
await workerFrames.prepare(slice(0, 0, 15));
await workerFrames.hook(page, 3 / 30);
expect(served).toContain("/full-extraction/whole/frame_4.jpg");
// The extractor is handed the whole list and picks out the deferred ones; the full one has no source read.
expect(extractor.calls[0]!.sourceOf).toEqual(["/source/clip.mp4"]);
expect(existsSync(join(extractor.calls[0]!.outputRoot, "whole"))).toBe(false);
});
it("a slice showing no deferred video extracts nothing", async () => {
const dir = mkdtempSync(join(root, "work-"));
const segmentVideoFrames = createSegmentVideoFrames({
videos: [element("later", 10, 12)],
extracted: [deferredVideo("later")],
fps,
rootDir: dir,
cfg,
injectorOptions: {},
sourceOf: () => {
throw new Error("a slice that shows no video must not read a source");
},
});
const workerFrames = segmentVideoFrames.forWorker();
await workerFrames.prepare(slice(0, 0, 15));
const folder = join(dir, "segment-frames", "segment_00000");
expect(readdirSync(folder).flatMap((entry) => readdirSync(join(folder, entry)))).toEqual([]);
});
it("a prepare that fails mid-extraction leaves no folder behind", async () => {
const { dir, workerFrames } = setup({
extractFrames: async (input) => {
mkdirSync(join(input.outputRoot, "clip"), { recursive: true });
writeFileSync(join(input.outputRoot, "clip", "frame_00001.jpg"), "x");
throw new Error("aborted");
},
});
await expect(workerFrames.prepare(slice(3, 90, 105))).rejects.toThrow("aborted");
expect(existsSync(join(dir, "segment-frames", "segment_00003"))).toBe(false);
});
describe("outside a prepared window", () => {
const onDemandFolders = (dir: string) => {
const root = join(dir, "segment-frames");
return existsSync(root) ? readdirSync(root).filter((n) => n.startsWith("on-demand_")) : [];
};
/** Records, at injection time, whether the folder the served frame lives in still existed. */
function setupWatching() {
const seen: Array<{ path: string; existed: boolean }> = [];
const made = setup({
injectorOptions: {
frameSrcResolver: (path) => {
seen.push({ path, existed: existsSync(path) });
return `http://frames/${path}`;
},
},
});
return { ...made, seen };
}
it("before any prepare, the hook extracts and injects the frame asked for, then removes its folder", async () => {
const { dir, seen, extractor, workerFrames, page } = setupWatching();
await workerFrames.hook(page, 31 / 30);
expect(extractor.calls).toHaveLength(1);
expect(extractor.calls[0]!.frames).toEqual([31]);
expect(
extractor.calls[0]!.outputRoot.startsWith(join(dir, "segment-frames", "on-demand_")),
).toBe(true);
const clip = seen.find((s) => s.path.includes("/clip/"));
expect(clip).toEqual({ path: expect.stringContaining("frame_00032.jpg"), existed: true });
// The fully extracted video is injected too, from its own frames.
expect(seen.some((s) => s.path === "/full-extraction/whole/frame_32.jpg")).toBe(true);
expect(onDemandFolders(dir)).toEqual([]);
});
it("asks for the held frame too when the capture holds footage for a motion-blur sample", async () => {
const { extractor, workerFrames, page } = setupWatching();
await workerFrames.hook(page, 32 / 30, 30 / 30);
expect(extractor.calls[0]!.frames).toEqual([32, 30]);
});
it("after release, the hook extracts on demand again", async () => {
const { dir, seen, extractor, workerFrames, page } = setupWatching();
await workerFrames.prepare(slice(1, 30, 45));
await workerFrames.hook(page, 31 / 30);
expect(extractor.calls).toHaveLength(1);
workerFrames.release();
await workerFrames.hook(page, 61 / 30);
expect(extractor.calls).toHaveLength(2);
expect(extractor.calls[1]!.frames).toEqual([61]);
expect(
seen.some((s) => s.path.includes("on-demand_") && s.path.includes("frame_00062.jpg")),
).toBe(true);
expect(onDemandFolders(dir)).toEqual([]);
});
it("fresh returns an independent hook that also extracts on demand", async () => {
const { dir, seen, extractor, workerFrames, page } = setupWatching();
await workerFrames.prepare(slice(1, 30, 45));
const fresh = workerFrames.hook.fresh!();
expect(fresh).not.toBe(workerFrames.hook);
// The worker's window is prepared, yet the fresh hook still reads its own frames.
await fresh(page, 40 / 30);
expect(extractor.calls).toHaveLength(2);
expect(extractor.calls[1]!.frames).toEqual([40]);
expect(
seen.some((s) => s.path.includes("on-demand_") && s.path.includes("frame_00041.jpg")),
).toBe(true);
expect(onDemandFolders(dir)).toEqual([]);
expect(typeof fresh.fresh).toBe("function");
});
it("an on-demand extraction that fails removes its folder and rethrows", async () => {
const { dir, workerFrames, page } = setup({
extractFrames: async (input) => {
mkdirSync(join(input.outputRoot, "clip"), { recursive: true });
writeFileSync(join(input.outputRoot, "clip", "frame_00001.jpg"), "x");
throw new Error("aborted");
},
});
await expect(workerFrames.hook(page, 31 / 30)).rejects.toThrow("aborted");
expect(onDemandFolders(dir)).toEqual([]);
});
it("gives two workers different folders", async () => {
const { extractor, segmentVideoFrames, workerFrames, page } = setup();
const other = segmentVideoFrames.forWorker();
await Promise.all([workerFrames.hook(page, 31 / 30), other.hook(page, 31 / 30)]);
expect(new Set(extractor.calls.map((call) => call.outputRoot)).size).toBe(2);
});
});
});
@@ -0,0 +1,149 @@
import { mkdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import type { Page } from "puppeteer-core";
import {
createFrameLookupTable,
createVideoFrameInjector,
type BeforeCaptureHook,
type EngineConfig,
type ExtractedFrames,
type VideoElement,
} from "@hyperframes/engine";
import { resolveRenderFpsConfig } from "../fileServer.js";
import { extractDeferredFrames } from "./deferredVideoFrames.js";
import type { SegmentSlice } from "./segmentPlan.js";
export interface SegmentWorkerFrames {
/**
* One hook for the worker's whole life, so a recycled session keeps it. Inside a
* prepared window it injects from the window; outside one it extracts the frames
* it is asked for and removes them again.
*/
hook: BeforeCaptureHook;
/** Extracts the frames `segment` shows and points the hook at them. */
prepare(segment: SegmentSlice): Promise<void>;
/** Deletes the segment's frames. Safe to call again. */
release(): void;
}
export interface SegmentVideoFrames {
forWorker(): SegmentWorkerFrames;
}
type FrameExtractor = typeof extractDeferredFrames;
function segmentFramesDir(rootDir: string, index: number): string {
return join(rootDir, "segment-frames", `segment_${String(index).padStart(5, "0")}`);
}
/** Each worker reads only the video frames its current segment shows, and deletes them when the segment closes. */
export function createSegmentVideoFrames(input: {
videos: VideoElement[];
extracted: ExtractedFrames[];
fps: { num: number; den: number };
/** The render's work dir; frames go under `segment-frames/segment_NNNNN/`. */
rootDir: string;
cfg: EngineConfig;
injectorOptions: Parameters<typeof createVideoFrameInjector>[1];
abortSignal?: AbortSignal;
sourceOf?: (video: ExtractedFrames) => string;
extractFrames?: FrameExtractor;
}): SegmentVideoFrames {
const extractFrames = input.extractFrames ?? extractDeferredFrames;
const sourceOf = input.sourceOf ?? ((video: ExtractedFrames) => video.srcPath);
const fullyExtracted = input.extracted.filter((video) => !video.deferredRange);
/** Extracts the given output frames into `dir` and returns an injector over them plus the fully extracted videos. */
const injectorFor = async (frames: number[], dir: string): Promise<BeforeCaptureHook | null> => {
const sliced = await extractFrames({
videos: input.videos,
extracted: input.extracted,
frames,
fps: input.fps,
outputRoot: dir,
cfg: input.cfg,
signal: input.abortSignal,
sourceOf,
onMissingFrame: (video, index) =>
new Error(
`[Render] video ${JSON.stringify(video.videoId)} has no frame ${index}; ` +
`it was expected to have ${video.totalFrames} frames.`,
),
});
const table = createFrameLookupTable(
input.videos,
[...fullyExtracted, ...sliced],
resolveRenderFpsConfig(input.fps).value,
);
return createVideoFrameInjector(table, input.injectorOptions);
};
/**
* The drawElement self-verify runs the hook at session open and after every
* recycle, outside any prepared window, to photograph the video frames the
* full route would show. Each call reads just the frames it is asked for.
*/
let onDemandCalls = 0;
const onDemandHook = (): BeforeCaptureHook =>
Object.assign(
async (page: Page, time: number, heldVideoTime?: number) => {
const dir = join(
input.rootDir,
"segment-frames",
`on-demand_${String(onDemandCalls++).padStart(5, "0")}`,
);
try {
const frames = [time, heldVideoTime]
.filter((t): t is number => t !== undefined)
.map((t) => Math.round((t * input.fps.num) / input.fps.den));
const injector = await injectorFor(frames, dir);
await injector?.(page, time, heldVideoTime);
} finally {
rmSync(dir, { recursive: true, force: true });
}
},
{ fresh: onDemandHook },
);
return {
forWorker() {
let injector: BeforeCaptureHook | null = null;
let folder: string | null = null;
const onDemand = onDemandHook();
const release = () => {
injector = null;
if (folder) rmSync(folder, { recursive: true, force: true });
folder = null;
};
return {
hook: Object.assign(
(page: Page, time: number, heldVideoTime?: number) =>
(injector ?? onDemand)(page, time, heldVideoTime),
{ fresh: onDemandHook },
),
release,
async prepare(segment) {
release();
const dir = segmentFramesDir(input.rootDir, segment.index);
folder = dir;
try {
rmSync(dir, { recursive: true, force: true });
mkdirSync(dir, { recursive: true });
injector = await injectorFor(
Array.from(
{ length: segment.endFrame - segment.startFrame },
(_, i) => segment.startFrame + i,
),
dir,
);
} catch (error) {
release();
throw error;
}
},
};
},
};
}
@@ -1,6 +1,6 @@
// fallow-ignore-file code-duplication
import { afterAll, describe, expect, it, mock } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -29,6 +29,8 @@ let initializeSessionErrorMessage = "initialize failed";
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
const closeCaptureSession = mock(async () => {});
const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = [];
const createdSessionHooks: unknown[] = [];
const reusedSessionHooks: unknown[] = [];
class DrawElementVerificationError extends Error {}
mock.module("@hyperframes/engine", () => ({
@@ -60,6 +62,7 @@ mock.module("@hyperframes/engine", () => ({
completeDeferredDrawElementInit: async () => {},
createCaptureSession: async (...args: unknown[]) => {
createdSessionConfigs.push(args[4] as { enableBrowserPool?: boolean });
createdSessionHooks.push(args[3]);
return {
isInitialized: false,
browserConsoleBuffer,
@@ -128,7 +131,8 @@ mock.module("@hyperframes/engine", () => ({
pixelFormat: "yuv420p",
}),
initTransparentBackground: async () => {},
prepareCaptureSessionForReuse: () => {
prepareCaptureSessionForReuse: (_session: unknown, _framesDir: string, hook: unknown) => {
reusedSessionHooks.push(hook);
if (failPrepareCaptureSessionForReuse) {
throw new Error("prepare reuse failed: ENOSPC");
}
@@ -928,3 +932,357 @@ describe("runCaptureSegmentedStage", () => {
expect(capturedAfterClose).toBe(0);
});
});
describe("runCaptureSegmentedStage with segmentVideoFrames", () => {
type Slice = { index: number; startFrame: number; endFrame: number };
/** A stand-in for the windowed frames: every call is logged, and prepare makes a real folder so release is observable. */
function fakeVideoFrames(
events: string[],
options: { prepare?: (segment: Slice) => Promise<void>; folderRoot?: string } = {},
) {
const hooks: Array<(...args: unknown[]) => Promise<void>> = [];
const prepared: number[] = [];
let current: string | null = null;
return {
hooks,
prepared,
segmentVideoFrames: {
forWorker: () => {
const hook = async () => {};
hooks.push(hook);
return {
hook,
prepare: async (segment: Slice) => {
events.push(`prepare ${segment.index}`);
prepared.push(segment.index);
if (options.folderRoot) {
current = join(options.folderRoot, `segment_${segment.index}`);
mkdirSync(current, { recursive: true });
}
await options.prepare?.(segment);
},
release: () => {
events.push("release");
if (current) rmSync(current, { recursive: true, force: true });
},
};
},
},
};
}
function loggingEncoder(events: string[]) {
let segment = -1;
return mock(async (outputPath: string) => {
segment = Number(/segment_(\d+)/.exec(outputPath)![1]);
events.push(`spawn ${segment}`);
const own = segment;
return {
writeFrame: async (buffer: Buffer) => {
events.push(`frame ${buffer.readUInt32BE(0)}`);
return true;
},
close: async () => {
events.push(`close ${own}`);
return { success: true, durationMs: 1, fileSize: 1 };
},
getExitStatus: () => "success" as const,
getExitError: () => undefined,
};
});
}
const frameBuffer = async (_s: unknown, frameIndex: number) => {
const buffer = Buffer.alloc(4);
buffer.writeUInt32BE(frameIndex, 0);
return { buffer };
};
function stageDeps(overrides: Record<string, unknown> = {}) {
return {
captureFrame: mock(frameBuffer),
concat: mock(async () => ({ success: true as const })),
closeSession: mock(async () => {}),
removeFile: () => {},
...overrides,
};
}
it("prepares before the encoder's first frame and releases after the segment closes, in order, for each segment", async () => {
const events: string[] = [];
const input = fakeStageInput({ totalFrames: 4 });
const { segmentVideoFrames } = fakeVideoFrames(events, {
prepare: async () => {
events.push(`stage "${input.job.currentStage}"`);
},
});
await runCaptureSegmentedStage({
...input,
segmentFrames: 2,
segmentDir: join(fixtureRoot, "windowed-order"),
segmentVideoFrames,
deps: stageDeps({ spawnEncoder: loggingEncoder(events) }),
});
expect(events).toEqual([
"prepare 0",
'stage "Reading video for segment 1/2"',
"spawn 0",
"frame 0",
"frame 1",
"close 0",
"release",
"prepare 1",
'stage "Reading video for segment 2/2"',
"spawn 1",
"frame 2",
"frame 3",
"close 1",
"release",
]);
});
it("a retried segment prepares again from a released state", async () => {
const events: string[] = [];
let next = 0;
let failed = false;
const { segmentVideoFrames } = fakeVideoFrames(events);
await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 4 }),
segmentFrames: 2,
segmentDir: join(fixtureRoot, "windowed-retry"),
sessionFactory: {
create: async () => ({ ...fakeStageInput({ totalFrames: 1 }).probeSession, id: next++ }),
},
segmentVideoFrames,
deps: stageDeps({
spawnEncoder: loggingEncoder(events),
captureFrame: mock(async (session: unknown, i: number) => {
if (i === 2 && !failed) {
failed = true;
throw new Error("Protocol error (Page.captureScreenshot): Target closed");
}
return frameBuffer(session, i);
}),
}),
});
expect(events.filter((e) => e.startsWith("prepare") || e === "release")).toEqual([
"prepare 0",
"release",
"prepare 1",
"release",
"prepare 1",
"release",
]);
});
it("a segment skipped on resume never prepares", async () => {
const events: string[] = [];
const frames = fakeVideoFrames(events);
await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 6 }),
segmentFrames: 2,
segmentDir: join(fixtureRoot, "windowed-resume"),
completedSegments: new Set([0]),
segmentVideoFrames: frames.segmentVideoFrames,
deps: stageDeps({ spawnEncoder: loggingEncoder(events) }),
});
expect(frames.prepared).toEqual([1, 2]);
});
it("an abort mid-segment releases its folder", async () => {
const folderRoot = join(fixtureRoot, "windowed-abort-folders");
const folderOf = (index: number) => join(folderRoot, `segment_${index}`);
const midCapture = new AbortController();
await expect(
runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 4 }),
segmentFrames: 4,
segmentDir: join(fixtureRoot, "windowed-abort-capture"),
abortSignal: midCapture.signal,
assertNotAborted: () => {
if (midCapture.signal.aborted) throw new Error("render cancelled");
},
segmentVideoFrames: fakeVideoFrames([], { folderRoot }).segmentVideoFrames,
deps: stageDeps({
spawnEncoder: loggingEncoder([]),
captureFrame: mock(async (session: unknown, i: number) => {
if (i === 1) midCapture.abort();
return frameBuffer(session, i);
}),
}),
}),
).rejects.toThrow("render cancelled");
expect(existsSync(folderOf(0))).toBe(false);
const midExtraction = new AbortController();
const extraction = runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 4 }),
segmentFrames: 4,
segmentDir: join(fixtureRoot, "windowed-abort-extraction"),
abortSignal: midExtraction.signal,
assertNotAborted: () => {
if (midExtraction.signal.aborted) throw new Error("render cancelled");
},
segmentVideoFrames: fakeVideoFrames([], {
folderRoot,
prepare: () =>
new Promise((_resolve, reject) => {
expect(existsSync(folderOf(0))).toBe(true);
midExtraction.signal.addEventListener("abort", () => reject(new Error("aborted")));
setTimeout(() => midExtraction.abort(), 5);
}),
}).segmentVideoFrames,
deps: stageDeps({ spawnEncoder: loggingEncoder([]) }),
});
await expect(extraction).rejects.toThrow("aborted");
expect(existsSync(folderOf(0))).toBe(false);
});
it("a recycled session keeps the worker's hook", async () => {
createdSessionHooks.length = 0;
const frames = fakeVideoFrames([]);
await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 6 }),
probeSession: null,
segmentFrames: 2,
segmentDir: join(fixtureRoot, "windowed-recycle"),
browserRecycleEverySegments: 1,
segmentVideoFrames: frames.segmentVideoFrames,
deps: stageDeps({ spawnEncoder: loggingEncoder([]) }),
});
expect(frames.hooks).toHaveLength(1);
expect(createdSessionHooks).toEqual([frames.hooks[0], frames.hooks[0], frames.hooks[0]]);
});
it("gives each worker its own hook, on the reused probe session and on a fresh one", async () => {
createdSessionHooks.length = 0;
reusedSessionHooks.length = 0;
const frames = fakeVideoFrames([]);
await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 6 }),
segmentFrames: 3,
segmentDir: join(fixtureRoot, "windowed-two-workers"),
workerCount: 2,
segmentVideoFrames: frames.segmentVideoFrames,
deps: stageDeps({ spawnEncoder: loggingEncoder([]) }),
});
expect(frames.hooks).toHaveLength(2);
expect(reusedSessionHooks).toHaveLength(1);
expect(createdSessionHooks).toHaveLength(1);
expect(new Set([...reusedSessionHooks, ...createdSessionHooks])).toEqual(new Set(frames.hooks));
});
it("time spent preparing does not count toward the capture stall timeout", async () => {
// The stall clock runs inside the frame loop, which starts once the encoder is
// open. Preparing happens before it, so a slow read of the source video is not a stall.
const previous = process.env.HF_DE_STALL_MS;
process.env.HF_DE_STALL_MS = "60";
try {
const frames = fakeVideoFrames([], {
prepare: () => new Promise((resolve) => setTimeout(resolve, 250)),
});
const result = await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 2 }),
segmentFrames: 2,
segmentDir: join(fixtureRoot, "windowed-stall"),
segmentVideoFrames: frames.segmentVideoFrames,
deps: stageDeps({ spawnEncoder: loggingEncoder([]) }),
});
expect(result.success).toBe(true);
expect(frames.prepared).toEqual([0]);
} finally {
if (previous === undefined) delete process.env.HF_DE_STALL_MS;
else process.env.HF_DE_STALL_MS = previous;
}
});
it("with segmentVideoFrames, segment files are removed after the join succeeds", async () => {
const events: string[] = [];
const segmentDir = join(fixtureRoot, "windowed-join");
await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 6 }),
segmentFrames: 2,
segmentDir,
segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames,
deps: stageDeps({
spawnEncoder: loggingEncoder([]),
concat: mock(async () => {
events.push("concat");
return { success: true as const };
}),
removeFile: (path: string) => events.push(`remove ${path}`),
}),
});
expect(events).toEqual([
"concat",
...[0, 1, 2].map((i) => `remove ${segmentOutputPath(segmentDir, i)}`),
]);
});
it("with segmentVideoFrames, a resume that only joins removes the segment files too", async () => {
const removed: string[] = [];
const segmentDir = join(fixtureRoot, "windowed-resume-join");
await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 4 }),
segmentFrames: 2,
segmentDir,
completedSegments: new Set([0, 1]),
segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames,
deps: stageDeps({ removeFile: (path: string) => removed.push(path) }),
});
expect(removed).toEqual([0, 1].map((i) => segmentOutputPath(segmentDir, i)));
});
it("a failed join removes no segment files", async () => {
const removed: string[] = [];
await expect(
runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 4 }),
segmentFrames: 2,
segmentDir: join(fixtureRoot, "windowed-join-fails"),
segmentVideoFrames: fakeVideoFrames([]).segmentVideoFrames,
deps: stageDeps({
spawnEncoder: loggingEncoder([]),
concat: mock(async () => ({ success: false as const, error: "concat failed" })),
removeFile: (path: string) => removed.push(path),
}),
}),
).rejects.toThrow(/concat failed/);
expect(removed).toEqual([]);
});
it("without segmentVideoFrames, segment files are kept", async () => {
const removed: string[] = [];
const result = await runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 4 }),
segmentFrames: 2,
segmentDir: join(fixtureRoot, "plain-keeps-files"),
deps: stageDeps({
spawnEncoder: loggingEncoder([]),
removeFile: (path: string) => removed.push(path),
}),
});
expect(result.success).toBe(true);
expect(removed).toEqual([]);
});
});
@@ -42,6 +42,7 @@ import { reportFrameProgress } from "../shared.js";
import { encoderFailureError } from "../encoderInterruption.js";
import type { SdrSegmentedCapturePlan } from "../capturePlan.js";
import { planSegments, type SegmentSlice } from "../segmentPlan.js";
import type { SegmentVideoFrames, SegmentWorkerFrames } from "../segmentVideoFrames.js";
import { createSegmentQueue } from "../segmentQueue.js";
import { isTargetLossError } from "../segmentRecycle.js";
import {
@@ -59,9 +60,9 @@ export interface SegmentedStageDeps {
removeFile: (path: string) => void;
}
/** Opens a fresh, initialized capture session with the video injector attached. */
/** Opens a fresh, initialized capture session with the video injector attached; a custom factory may ignore the override. */
export interface SessionFactory {
create: () => Promise<CaptureSession>;
create: (injectorOverride?: () => BeforeCaptureHook | null) => Promise<CaptureSession>;
}
export interface CaptureSegmentedStageInput {
@@ -80,6 +81,8 @@ export interface CaptureSegmentedStageInput {
streamingEncoderOptions: StreamingEncoderOptions;
buildCaptureOptions: () => CaptureOptions;
createRenderVideoFrameInjector: () => BeforeCaptureHook | null;
/** Windowed frames: extracted per segment, deleted when it closes; segment files are deleted once joined. */
segmentVideoFrames?: SegmentVideoFrames;
abortSignal: AbortSignal | undefined;
assertNotAborted: () => void;
onProgress?: ProgressCallback;
@@ -174,6 +177,7 @@ class SegmentEncoderSpawnError extends Error {
interface SegmentCaptureContext {
session: CaptureSession;
frames?: SegmentWorkerFrames;
job: RenderJob;
cfg: EngineConfig;
totalFrames: number;
@@ -228,10 +232,34 @@ async function captureSegmentFrames(
}
}
/** Capture one segment into its own encoder. Returns the encoder's result. */
/** Capture one segment; when windowed, its video frames are read first and released however it ends. */
async function captureSegment(
ctx: SegmentCaptureContext,
segment: SegmentSlice,
): Promise<StreamingEncoderResult> {
const { frames } = ctx;
if (!frames) return captureSegmentToEncoder(ctx, segment);
try {
reportFrameProgress(
ctx.job,
`Reading video for segment ${segment.index + 1}/${ctx.segmentCount}`,
Math.round(25 + (segment.startFrame / ctx.totalFrames) * 55),
ctx.onProgress,
segment.startFrame,
ctx.totalFrames,
);
await frames.prepare(segment);
ctx.assertNotAborted();
return await captureSegmentToEncoder(ctx, segment);
} finally {
frames.release();
}
}
/** Capture one segment into its own encoder. Returns the encoder's result. */
async function captureSegmentToEncoder(
ctx: SegmentCaptureContext,
segment: SegmentSlice,
): Promise<StreamingEncoderResult> {
const segmentPath = segmentOutputPath(ctx.segmentDir, segment.index);
let encoder: StreamingEncoder;
@@ -295,15 +323,15 @@ const DEFAULT_DEPS: SegmentedStageDeps = {
function defaultSessionFactory(input: CaptureSegmentedStageInput): SessionFactory {
let probeSession = input.probeSession;
return {
create: async () => {
create: async (injector = input.createRenderVideoFrameInjector) => {
const probeCannotCaptureEveryFrame = Boolean(probeSession?.deRiskFrames?.size);
if (probeSession && probeCannotCaptureEveryFrame) {
await closeCaptureSession(probeSession).catch(() => {});
probeSession = null;
}
const session = probeSession ?? (await openSegmentedSession(input));
const session = probeSession ?? (await openSegmentedSession(input, injector));
if (probeSession) {
openSegmentedSessionReuse(input, probeSession);
openSegmentedSessionReuse(input, probeSession, injector);
probeSession = null;
}
try {
@@ -321,18 +349,22 @@ function defaultSessionFactory(input: CaptureSegmentedStageInput): SessionFactor
function openSegmentedSessionReuse(
input: CaptureSegmentedStageInput,
session: CaptureSession,
injector: () => BeforeCaptureHook | null,
): void {
prepareCaptureSessionForReuse(session, input.framesDir, input.createRenderVideoFrameInjector());
prepareCaptureSessionForReuse(session, input.framesDir, injector());
}
async function openSegmentedSession(input: CaptureSegmentedStageInput): Promise<CaptureSession> {
async function openSegmentedSession(
input: CaptureSegmentedStageInput,
injector: () => BeforeCaptureHook | null,
): Promise<CaptureSession> {
// Same reasoning as the streaming stage: the resolved forceScreenshot comes
// from the immutable plan, not from the caller-owned cfg.
const captureCfg: EngineConfig =
input.cfg.forceScreenshot === input.plan.forceScreenshot
? input.cfg
: { ...input.cfg, forceScreenshot: input.plan.forceScreenshot };
const videoInjector = input.createRenderVideoFrameInjector();
const videoInjector = injector();
const captureOptions = { ...input.buildCaptureOptions(), deFrameFallback: false };
const ownBrowser = shouldDisableBrowserPoolForParallelWorker({
parallel: (input.workerCount ?? 1) > 1,
@@ -346,6 +378,12 @@ async function openSegmentedSession(input: CaptureSegmentedStageInput): Promise<
);
}
/** Once joined, the windowed route has no use for the segments; keeping them would hold the whole video twice. */
function removeJoinedSegments(run: SegmentRun, segmentPaths: readonly string[]): void {
if (!run.input.segmentVideoFrames) return;
for (const path of segmentPaths) run.deps.removeFile(path);
}
/** Stream-copy the finished segments into the video-only file, or throw. */
async function concatSegments(
concat: SegmentedStageDeps["concat"],
@@ -395,12 +433,14 @@ interface SegmentRun {
async function createSegmentWorker(run: SegmentRun, id: number): Promise<SegmentWorker> {
const { input, deps } = run;
const factory = input.sessionFactoryForWorker?.(id) ?? run.sessionFactory;
const frames = input.segmentVideoFrames?.forWorker();
return {
id,
factory,
sessionSegments: 0,
ctx: {
session: await factory.create(),
session: await factory.create(frames && (() => frames.hook)),
frames,
job: input.job,
cfg: input.cfg,
totalFrames: input.totalFrames,
@@ -441,7 +481,8 @@ async function recycleWorkerSession(
run.input.dedupPerfs.push(getCapturePerfSummary(worker.ctx.session));
const memory = worker.ctx.session.chromeMemory?.stats();
await run.deps.closeSession(worker.ctx.session);
worker.ctx.session = await worker.factory.create();
const { frames } = worker.ctx;
worker.ctx.session = await worker.factory.create(frames && (() => frames.hook));
worker.sessionSegments = 0;
if (why === "cadence") run.totals.browserRecycles += 1;
run.input.log.info(`[Render] segment browser recycled (${why})`, {
@@ -549,6 +590,7 @@ async function concatResumedRun(
input.abortSignal,
input.cfg,
);
removeJoinedSegments(run, segmentPaths);
return {
success: true,
encodeMs: 0,
@@ -639,6 +681,7 @@ export async function runCaptureSegmentedStage(
input.abortSignal,
input.cfg,
);
removeJoinedSegments(run, segmentPaths);
return {
success: true,