refactor(producer): make capture plans immutable

This commit is contained in:
James
2026-07-17 19:14:04 -04:00
parent 4a2c9eeedb
commit 6a84e95c68
8 changed files with 458 additions and 155 deletions
@@ -59,6 +59,7 @@ import { defaultLogger } from "../../logger.js";
import { runEncodeStage } from "../render/stages/encodeStage.js"; import { runEncodeStage } from "../render/stages/encodeStage.js";
import { runCaptureStage } from "../render/stages/captureStage.js"; import { runCaptureStage } from "../render/stages/captureStage.js";
import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js"; import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
import { createCapturePlan } from "../render/capturePlan.js";
import { import {
type ChunkSliceJson, type ChunkSliceJson,
type LockedRenderConfig, type LockedRenderConfig,
@@ -589,6 +590,18 @@ export async function renderChunk(
// one Chrome session per worker, so captureStageMs includes those // one Chrome session per worker, so captureStageMs includes those
// boots; sessionBootMs stays 0 there. // boots; sessionBootMs stays 0 there.
const captureStarted = Date.now(); const captureStarted = Date.now();
const capturePlan = createCapturePlan({
workerCount: chunkWorkerCount,
forceScreenshot: encoder.forceScreenshot,
useStreamingEncode: false,
useLayeredComposite: false,
usePageSideCompositing: false,
hasHdrContent: false,
needsAlpha: plan.dimensions.format !== "mp4",
});
if (capturePlan.kind !== "sdr_disk") {
throw new Error(`Distributed chunk requires sdr_disk plan; got ${capturePlan.kind}`);
}
await runCaptureStage({ await runCaptureStage({
fileServer, fileServer,
workDir, workDir,
@@ -596,11 +609,9 @@ export async function renderChunk(
job, job,
totalFrames: framesInChunk, totalFrames: framesInChunk,
cfg, cfg,
forceScreenshot: encoder.forceScreenshot, plan: capturePlan,
log, log,
workerCount: chunkWorkerCount,
probeSession: session, probeSession: session,
needsAlpha: plan.dimensions.format !== "mp4",
captureAttempts: [], captureAttempts: [],
// Distributed chunks run on Linux (beginframe) where dedup never arms; // Distributed chunks run on Linux (beginframe) where dedup never arms;
// a throwaway sink satisfies the type without per-chunk dedup reporting. // a throwaway sink satisfies the type without per-chunk dedup reporting.
@@ -0,0 +1,123 @@
import { describe, expect, it } from "vitest";
import { createCapturePlan, replanAfterFailure, type CaptureRouting } from "./capturePlan.js";
function streaming(routing?: CaptureRouting) {
return createCapturePlan({
workerCount: 1,
forceScreenshot: false,
forceParallelStream: false,
useStreamingEncode: true,
useLayeredComposite: false,
usePageSideCompositing: false,
hasHdrContent: false,
needsAlpha: false,
routing,
});
}
describe("CapturePlan", () => {
it("makes layered capture dominant and enforces its screenshot invariant", () => {
const plan = createCapturePlan({
workerCount: 3,
forceScreenshot: false,
forceParallelStream: true,
useStreamingEncode: true,
useLayeredComposite: true,
usePageSideCompositing: false,
hasHdrContent: true,
needsAlpha: false,
});
expect(plan).toMatchObject({
kind: "hdr_layered",
workerCount: 3,
forceScreenshot: true,
forceParallelStream: false,
});
expect(Object.isFrozen(plan)).toBe(true);
expect(Object.isFrozen(plan.routing)).toBe(true);
});
it("falls back from an unavailable streaming encoder to the same disk route", () => {
const initial = streaming();
const next = replanAfterFailure(initial, { kind: "streaming_unavailable" });
expect(next).toMatchObject({ kind: "sdr_disk", workerCount: 1, forceScreenshot: false });
expect(initial.kind).toBe("sdr_streaming");
});
it("makes page-side compositing force screenshot capture", () => {
const plan = createCapturePlan({
workerCount: 1,
forceScreenshot: false,
forceParallelStream: false,
useStreamingEncode: true,
useLayeredComposite: false,
usePageSideCompositing: true,
hasHdrContent: false,
needsAlpha: false,
});
expect(plan).toMatchObject({ kind: "sdr_streaming", forceScreenshot: true });
});
it("retries an ordinary verification failure in streaming screenshot mode", () => {
expect(replanAfterFailure(streaming(), { kind: "draw_element_verification" })).toMatchObject({
kind: "sdr_streaming",
workerCount: 1,
forceScreenshot: true,
routing: { kind: "default" },
});
});
it("atomically restores the pre-inversion disk route after verification failure", () => {
const initial = streaming({
kind: "worker_inversion",
state: "active",
fallback: { kind: "sdr_disk", workerCount: 5, forceParallelStream: false },
});
const next = replanAfterFailure(initial, { kind: "draw_element_verification" });
expect(next).toMatchObject({
kind: "sdr_disk",
workerCount: 5,
forceScreenshot: true,
routing: { kind: "worker_inversion", state: "reverted" },
});
expect(initial).toMatchObject({ workerCount: 1, routing: { state: "active" } });
expect(Object.isFrozen(next.routing)).toBe(true);
});
it("reduces an OOM fallback to one worker without losing immutable routing state", () => {
const initial = streaming({
kind: "parallel_router",
state: "active",
fallback: { kind: "sdr_disk", workerCount: 5, forceParallelStream: false },
});
const next = replanAfterFailure(initial, {
kind: "capture_failure",
memoryExhaustion: true,
});
expect(next).toMatchObject({
kind: "sdr_disk",
workerCount: 1,
forceScreenshot: true,
routing: { kind: "parallel_router", state: "reverted" },
});
});
it("rejects a streaming transition from a non-streaming plan", () => {
const disk = createCapturePlan({
workerCount: 2,
forceScreenshot: true,
useStreamingEncode: false,
useLayeredComposite: false,
usePageSideCompositing: false,
hasHdrContent: false,
needsAlpha: false,
});
expect(() => replanAfterFailure(disk, { kind: "streaming_unavailable" })).toThrow(
"Cannot apply streaming_unavailable to sdr_disk",
);
});
});
@@ -0,0 +1,150 @@
/**
* Immutable routing decision for the capture phase.
*
* The orchestrator used to carry the same decision in several mutable booleans
* (`useStreamingEncode`, `useLayeredComposite`, `forceScreenshot`) plus worker
* routing state. Keeping those values independently mutable made invalid
* combinations representable during fallback. A CapturePlan is the single
* value consumed by capture stages, and `replanAfterFailure` is the only
* transition between variants.
*/
export type CapturePlanTarget = Readonly<{
kind: "sdr_streaming" | "sdr_disk";
workerCount: number;
forceParallelStream: boolean;
}>;
export type CaptureRouting =
| Readonly<{ kind: "default" }>
| Readonly<{
kind: "worker_inversion" | "parallel_router";
state: "active" | "reverted";
fallback: CapturePlanTarget;
}>;
interface CapturePlanBase {
readonly workerCount: number;
readonly forceScreenshot: boolean;
readonly forceParallelStream: boolean;
readonly usePageSideCompositing: boolean;
readonly hasHdrContent: boolean;
readonly needsAlpha: boolean;
readonly routing: CaptureRouting;
}
export interface SdrStreamingCapturePlan extends CapturePlanBase {
readonly kind: "sdr_streaming";
}
export interface SdrDiskCapturePlan extends CapturePlanBase {
readonly kind: "sdr_disk";
readonly forceParallelStream: false;
}
export interface HdrLayeredCapturePlan extends CapturePlanBase {
readonly kind: "hdr_layered";
readonly forceScreenshot: true;
readonly forceParallelStream: false;
}
export type CapturePlan = SdrStreamingCapturePlan | SdrDiskCapturePlan | HdrLayeredCapturePlan;
export interface CreateCapturePlanInput {
workerCount: number;
forceScreenshot: boolean;
forceParallelStream?: boolean;
useStreamingEncode: boolean;
useLayeredComposite: boolean;
usePageSideCompositing: boolean;
hasHdrContent: boolean;
needsAlpha: boolean;
routing?: CaptureRouting;
}
export type CapturePlanFailure =
| Readonly<{ kind: "streaming_unavailable" }>
| Readonly<{ kind: "draw_element_verification" }>
| Readonly<{ kind: "capture_failure"; memoryExhaustion: boolean }>;
function assertWorkerCount(workerCount: number): void {
if (!Number.isInteger(workerCount) || workerCount < 1) {
throw new Error(`CapturePlan workerCount must be a positive integer; got ${workerCount}`);
}
}
function freezeTarget(target: CapturePlanTarget): CapturePlanTarget {
assertWorkerCount(target.workerCount);
if (target.kind === "sdr_disk" && target.forceParallelStream) {
throw new Error("CapturePlan disk fallback cannot force parallel streaming");
}
return Object.freeze({ ...target });
}
function freezeRouting(routing: CaptureRouting | undefined): CaptureRouting {
if (!routing || routing.kind === "default") return Object.freeze({ kind: "default" });
return Object.freeze({ ...routing, fallback: freezeTarget(routing.fallback) });
}
export function createCapturePlan(input: CreateCapturePlanInput): CapturePlan {
assertWorkerCount(input.workerCount);
const base = {
workerCount: input.workerCount,
forceScreenshot: input.forceScreenshot || input.usePageSideCompositing,
forceParallelStream: input.useStreamingEncode ? (input.forceParallelStream ?? false) : false,
usePageSideCompositing: input.usePageSideCompositing,
hasHdrContent: input.hasHdrContent,
needsAlpha: input.needsAlpha,
routing: freezeRouting(input.routing),
};
if (input.useLayeredComposite) {
return Object.freeze({
...base,
kind: "hdr_layered",
forceScreenshot: true,
forceParallelStream: false,
});
}
if (input.useStreamingEncode) {
return Object.freeze({ ...base, kind: "sdr_streaming" });
}
return Object.freeze({ ...base, kind: "sdr_disk", forceParallelStream: false });
}
function revertedRouting(routing: CaptureRouting): CaptureRouting {
if (routing.kind === "default") return routing;
return freezeRouting({ ...routing, state: "reverted" });
}
/** Pure, exhaustive capture fallback transition. The input plan is never mutated. */
export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan {
if (plan.kind !== "sdr_streaming") {
throw new Error(`Cannot apply ${failure.kind} to ${plan.kind} capture plan`);
}
if (failure.kind === "streaming_unavailable") {
return createCapturePlan({
...plan,
useStreamingEncode: false,
useLayeredComposite: false,
forceParallelStream: false,
});
}
const fallback =
plan.routing.kind === "default"
? { kind: plan.kind, workerCount: plan.workerCount, forceParallelStream: false }
: plan.routing.fallback;
const workerCount =
failure.kind === "capture_failure" && failure.memoryExhaustion ? 1 : fallback.workerCount;
return createCapturePlan({
...plan,
workerCount,
forceScreenshot: true,
forceParallelStream: fallback.forceParallelStream,
useStreamingEncode: fallback.kind === "sdr_streaming",
useLayeredComposite: false,
routing: revertedRouting(plan.routing),
});
}
@@ -25,9 +25,8 @@
* because `captureAlphaPng` hangs under `--enable-begin-frame-control`. * because `captureAlphaPng` hangs under `--enable-begin-frame-control`.
* Previously the stage mutated `cfg.forceScreenshot = true` directly; * Previously the stage mutated `cfg.forceScreenshot = true` directly;
* the value is now derived into a local `hdrCfg` so the caller-owned * the value is now derived into a local `hdrCfg` so the caller-owned
* `cfg` survives the stage unchanged. The sequencer is expected to * `cfg` survives the stage unchanged. The sequencer passes an immutable
* pass `forceScreenshot: true` for the layered branch as a contract * `hdr_layered` plan whose construction guarantees screenshot mode.
* check.
* *
* Resource setup (HDR video extraction, image decode, dim probing) lives * Resource setup (HDR video extraction, image decode, dim probing) lives
* in `captureHdrResources.ts`; per-frame work lives in * in `captureHdrResources.ts`; per-frame work lives in
@@ -44,7 +43,6 @@ import {
type EngineConfig, type EngineConfig,
type HdrTransfer, type HdrTransfer,
type StreamingEncoder, type StreamingEncoder,
calculateOptimalWorkers,
closeCaptureSession, closeCaptureSession,
createCaptureSession, createCaptureSession,
getEncoderPreset, getEncoderPreset,
@@ -77,18 +75,13 @@ import { partitionTransitionFrames, shouldUseHybridLayeredPath } from "./capture
import { runSequentialLayeredFrameLoop } from "./captureHdrSequentialLoop.js"; import { runSequentialLayeredFrameLoop } from "./captureHdrSequentialLoop.js";
import { runHybridLayeredFrameLoop } from "./captureHdrHybridLoop.js"; import { runHybridLayeredFrameLoop } from "./captureHdrHybridLoop.js";
import { wrapCaptureStageError } from "../captureStageError.js"; import { wrapCaptureStageError } from "../captureStageError.js";
import type { HdrLayeredCapturePlan } from "../capturePlan.js";
export interface CaptureHdrStageInput { export interface CaptureHdrStageInput {
job: RenderJob; job: RenderJob;
cfg: EngineConfig; cfg: EngineConfig;
/** /** Immutable layered route selected by the sequencer. */
* Capture-mode flag threaded from `compileStage`. The HDR layered plan: HdrLayeredCapturePlan;
* branch requires `true` (see file header for the
* `captureAlphaPng` / `--enable-begin-frame-control` constraint);
* the stage throws if called with `false`. Stored locally as
* `hdrCfg.forceScreenshot` so the caller-owned `cfg` is not mutated.
*/
forceScreenshot: boolean;
log: ProducerLogger; log: ProducerLogger;
projectDir: string; projectDir: string;
@@ -120,13 +113,6 @@ export interface CaptureHdrStageInput {
/** Mutated in place (counters incremented). */ /** Mutated in place (counters incremented). */
hdrDiagnostics: HdrDiagnostics; hdrDiagnostics: HdrDiagnostics;
/**
* Worker budget for the hybrid layered path. Only consulted when the
* gating predicate (`shouldUseHybridLayeredPath`) returns true. The
* sequential loop always runs on a single DOM session.
*/
workerCount?: number;
abortSignal: AbortSignal | undefined; abortSignal: AbortSignal | undefined;
assertNotAborted: () => void; assertNotAborted: () => void;
onProgress?: ProgressCallback; onProgress?: ProgressCallback;
@@ -160,7 +146,7 @@ export async function runCaptureHdrStage(
const { const {
job, job,
cfg, cfg,
forceScreenshot, plan,
log, log,
projectDir, projectDir,
compiledDir, compiledDir,
@@ -184,17 +170,11 @@ export async function runCaptureHdrStage(
buildCaptureOptions, buildCaptureOptions,
createRenderVideoFrameInjector, createRenderVideoFrameInjector,
hdrDiagnostics, hdrDiagnostics,
workerCount,
abortSignal, abortSignal,
assertNotAborted, assertNotAborted,
onProgress, onProgress,
} = input; } = input;
const { workerCount } = plan;
if (!forceScreenshot) {
throw new Error(
"captureHdrStage requires forceScreenshot=true; the layered composite path uses captureAlphaPng which hangs under --enable-begin-frame-control.",
);
}
const stageStart = Date.now(); const stageStart = Date.now();
let lastBrowserConsole: string[] = []; let lastBrowserConsole: string[] = [];
@@ -378,16 +358,7 @@ export async function runCaptureHdrStage(
}; };
// ── Dispatch to sequential or hybrid frame loop ──────────────────── // ── Dispatch to sequential or hybrid frame loop ────────────────────
// Resolve the worker budget here rather than threading it through the const effectiveWorkerCount = Math.max(1, workerCount);
// renderOrchestrator call: keeps the renderOrchestrator diff zero
// (hf#732 PR 4 is intentionally a producer-stage-local change), at the
// cost of recomputing the same number the orchestrator already knows.
// The cost is negligible (one cpus() call) and the two values stay in
// lockstep because `calculateOptimalWorkers` is pure.
const effectiveWorkerCount =
workerCount !== undefined
? Math.max(1, workerCount)
: calculateOptimalWorkers(totalFrames, job.config.workers, hdrCfg);
const transitionFrameCount = partitionTransitionFrames(transitionRanges, totalFrames).size; const transitionFrameCount = partitionTransitionFrames(transitionRanges, totalFrames).size;
const useHybrid = shouldUseHybridLayeredPath({ const useHybrid = shouldUseHybridLayeredPath({
hasHdrContent, hasHdrContent,
@@ -62,6 +62,7 @@ import {
} from "../../renderOrchestrator.js"; } from "../../renderOrchestrator.js";
import { wrapCaptureStageError } from "../captureStageError.js"; import { wrapCaptureStageError } from "../captureStageError.js";
import { updateJobStatus } from "../shared.js"; import { updateJobStatus } from "../shared.js";
import type { SdrDiskCapturePlan } from "../capturePlan.js";
export interface CaptureStageInput { export interface CaptureStageInput {
fileServer: FileServerHandle; fileServer: FileServerHandle;
@@ -76,23 +77,11 @@ export interface CaptureStageInput {
*/ */
totalFrames: number; totalFrames: number;
cfg: EngineConfig; cfg: EngineConfig;
/** /** Immutable route selected by the sequencer. */
* Capture-mode flag threaded from `compileStage`. The stage derives a plan: SdrDiskCapturePlan;
* local copy of `cfg` with this value applied to `forceScreenshot`
* before any engine call, so the caller-owned `cfg` is never mutated.
* The sequencer may override `compileResult.forceScreenshot` after a
* BeginFrame calibration timeout — passing the override through this
* parameter keeps the decision visible at the call site instead of
* hiding it inside a shared mutable config.
*/
forceScreenshot: boolean;
log: ProducerLogger; log: ProducerLogger;
/** Initial worker count from `resolveRenderWorkerCount`; adaptive retry may reduce it. */
workerCount: number;
/** Reused for the sequential path's first session if non-null. */ /** Reused for the sequential path's first session if non-null. */
probeSession: CaptureSession | null; probeSession: CaptureSession | null;
/** True for webm / mov / png-sequence (controls capture format + extension). */
needsAlpha: boolean;
/** Mutated in place — each parallel retry attempt is appended. */ /** Mutated in place — each parallel retry attempt is appended. */
captureAttempts: CaptureAttemptSummary[]; captureAttempts: CaptureAttemptSummary[];
/** /**
@@ -154,7 +143,7 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
job, job,
totalFrames, totalFrames,
cfg, cfg,
forceScreenshot, plan,
log, log,
captureAttempts, captureAttempts,
buildCaptureOptions, buildCaptureOptions,
@@ -162,17 +151,18 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
abortSignal, abortSignal,
assertNotAborted, assertNotAborted,
onProgress, onProgress,
needsAlpha,
frameRange, frameRange,
dedupPerfs, dedupPerfs,
} = input; } = input;
let { workerCount, probeSession } = input; let { probeSession } = input;
let { workerCount } = plan;
const { forceScreenshot, needsAlpha } = plan;
let lastBrowserConsole: string[] = []; let lastBrowserConsole: string[] = [];
let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport; let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport;
// Derive a local cfg view rather than reading `forceScreenshot` from the // Derive a local cfg view rather than reading `forceScreenshot` from the
// caller-owned `cfg`. The sequencer threads the resolved value via the // caller-owned `cfg`. The sequencer threads the resolved value via the
// explicit parameter; this keeps the engine-facing config a pure // immutable plan; this keeps the engine-facing config a pure
// pass-through. // pass-through.
const captureCfg: EngineConfig = const captureCfg: EngineConfig =
cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot }; cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot };
@@ -1,5 +1,6 @@
import { describe, expect, it, mock } from "bun:test"; import { describe, expect, it, mock } from "bun:test";
import { getCaptureStageBrowserConsole } from "../captureStageError.js"; import { getCaptureStageBrowserConsole } from "../captureStageError.js";
import { createCapturePlan } from "../capturePlan.js";
type MinimalEngineConfig = { type MinimalEngineConfig = {
forceScreenshot: boolean; forceScreenshot: boolean;
@@ -171,14 +172,21 @@ function createInput(cfg: MinimalEngineConfig) {
}, },
totalFrames: 0, totalFrames: 0,
cfg, cfg,
forceScreenshot: false, plan: createCapturePlan({
workerCount: 1,
forceScreenshot: false,
useStreamingEncode: true,
useLayeredComposite: false,
usePageSideCompositing: false,
hasHdrContent: false,
needsAlpha: false,
}),
log: { log: {
error: () => {}, error: () => {},
warn: () => {}, warn: () => {},
info: () => {}, info: () => {},
debug: () => {}, debug: () => {},
}, },
workerCount: 1,
probeSession: null, probeSession: null,
outputFormat: "mp4", outputFormat: "mp4",
streamingEncoderOptions: { fps: { num: 30, den: 1 }, width: 1920, height: 1080 }, streamingEncoderOptions: { fps: { num: 30, den: 1 }, width: 1920, height: 1080 },
@@ -408,10 +416,18 @@ describe("runCaptureStage", () => {
try { try {
await runCaptureStage({ await runCaptureStage({
...createInput(cfg), ...createInput(cfg),
plan: createCapturePlan({
workerCount: 1,
forceScreenshot: false,
useStreamingEncode: false,
useLayeredComposite: false,
usePageSideCompositing: false,
hasHdrContent: false,
needsAlpha: false,
}),
videoOnlyPath: undefined, videoOnlyPath: undefined,
outputFormat: undefined, outputFormat: undefined,
streamingEncoderOptions: undefined, streamingEncoderOptions: undefined,
needsAlpha: false,
captureAttempts: [], captureAttempts: [],
}); });
} catch (error) { } catch (error) {
@@ -447,7 +463,15 @@ describe("runCaptureHdrStage", () => {
duration: 1, duration: 1,
}, },
cfg: { forceScreenshot: true }, cfg: { forceScreenshot: true },
forceScreenshot: true, plan: createCapturePlan({
workerCount: 1,
forceScreenshot: true,
useStreamingEncode: false,
useLayeredComposite: true,
usePageSideCompositing: false,
hasHdrContent: false,
needsAlpha: false,
}),
log: { log: {
error: () => {}, error: () => {},
warn: () => {}, warn: () => {},
@@ -496,7 +520,6 @@ describe("runCaptureHdrStage", () => {
videoExtractionFailures: 0, videoExtractionFailures: 0,
imageDecodeFailures: 0, imageDecodeFailures: 0,
}, },
workerCount: 1,
abortSignal: undefined, abortSignal: undefined,
assertNotAborted: () => {}, assertNotAborted: () => {},
}); });
@@ -78,6 +78,7 @@ import { wrapCaptureStageError } from "../captureStageError.js";
import { pushWorkerDedupPerfs } from "../perfSummary.js"; import { pushWorkerDedupPerfs } from "../perfSummary.js";
import { ensureFrameWritten } from "./captureHdrFrameShared.js"; import { ensureFrameWritten } from "./captureHdrFrameShared.js";
import { updateJobStatus } from "../shared.js"; import { updateJobStatus } from "../shared.js";
import type { SdrStreamingCapturePlan } from "../capturePlan.js";
/** /**
* No-frame-progress watchdog for DE streaming capture. A worker (parallel * No-frame-progress watchdog for DE streaming capture. A worker (parallel
@@ -173,27 +174,10 @@ export interface CaptureStreamingStageInput {
*/ */
totalFrames: number; totalFrames: number;
cfg: EngineConfig; cfg: EngineConfig;
/** /** Immutable route selected by the sequencer. */
* Capture-mode flag threaded from `compileStage`. The stage derives a plan: SdrStreamingCapturePlan;
* local copy of `cfg` with this value applied to `forceScreenshot`
* before any engine call, so the caller-owned `cfg` is never mutated.
* The sequencer may override `compileResult.forceScreenshot` after a
* BeginFrame calibration timeout — passing the override through this
* parameter keeps the decision visible at the call site instead of
* hiding it inside a shared mutable config.
*/
forceScreenshot: boolean;
log: ProducerLogger; log: ProducerLogger;
workerCount: number;
probeSession: CaptureSession | null; probeSession: CaptureSession | null;
/**
* Per-render override from the DE parallel router — see
* deParallelStreamForced's declaration in renderOrchestrator.ts. Distinct
* from the `HF_DE_PARALLEL_STREAM` manual opt-in (still read directly by
* this stage) because the router's decision must not leak across
* concurrently-running renders sharing this process via a global env var.
*/
forceParallelStream?: boolean;
/** For the spawn-failure log message context only. */ /** For the spawn-failure log message context only. */
outputFormat: string; outputFormat: string;
/** Pre-built encoder options; passed straight to `spawnStreamingEncoder`. */ /** Pre-built encoder options; passed straight to `spawnStreamingEncoder`. */
@@ -552,7 +536,7 @@ export async function runCaptureStreamingStage(
job, job,
totalFrames, totalFrames,
cfg, cfg,
forceScreenshot, plan,
log, log,
outputFormat, outputFormat,
streamingEncoderOptions, streamingEncoderOptions,
@@ -562,16 +546,17 @@ export async function runCaptureStreamingStage(
assertNotAborted, assertNotAborted,
onProgress, onProgress,
dedupPerfs, dedupPerfs,
forceParallelStream,
} = input; } = input;
let { workerCount, probeSession } = input; let { probeSession } = input;
let { workerCount } = plan;
const { forceScreenshot, forceParallelStream } = plan;
let lastBrowserConsole: string[] = []; let lastBrowserConsole: string[] = [];
let deDrainStats: DeDrainStats | undefined; let deDrainStats: DeDrainStats | undefined;
let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport; let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport;
// Derive a local cfg view rather than reading `forceScreenshot` from the // Derive a local cfg view rather than reading `forceScreenshot` from the
// caller-owned `cfg`. The sequencer threads the resolved value via the // caller-owned `cfg`. The sequencer threads the resolved value via the
// explicit parameter; this keeps the engine-facing config a pure // immutable plan; this keeps the engine-facing config a pure
// pass-through. // pass-through.
const captureCfg: EngineConfig = const captureCfg: EngineConfig =
cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot }; cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot };
@@ -104,6 +104,12 @@ import { buildRenderErrorDetails } from "./render/cleanup.js";
import { publishRenderFailure } from "./render/renderEventPublisher.js"; import { publishRenderFailure } from "./render/renderEventPublisher.js";
import { RenderExecutionContext } from "./render/renderExecutionContext.js"; import { RenderExecutionContext } from "./render/renderExecutionContext.js";
import { ArtifactTransaction } from "./render/artifactTransaction.js"; import { ArtifactTransaction } from "./render/artifactTransaction.js";
import {
createCapturePlan,
replanAfterFailure,
type CapturePlan,
type CaptureRouting,
} from "./render/capturePlan.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { formatCaptureFrameName } from "../utils/paths.js"; import { formatCaptureFrameName } from "../utils/paths.js";
import { resolveEffectiveHdrMode } from "./render/hdrMode.js"; import { resolveEffectiveHdrMode } from "./render/hdrMode.js";
@@ -2693,22 +2699,85 @@ async function executeRenderPipeline(input: {
hasShaderTransitions: compiled.hasShaderTransitions && !isGif, hasShaderTransitions: compiled.hasShaderTransitions && !isGif,
isPngSequence, isPngSequence,
}); });
updateCaptureObservability({ const inversionFallback = resolveInversionRetryPlan({
deWorkerInversion,
preInversionWorkerCount: preRoutingWorkerCount,
cfg,
outputFormat,
durationSeconds: job.duration,
isMemoryExhaustion: false,
});
const parallelRouterFallback = resolveParallelRouterRetryPlan({
deParallelRouter,
preRouterWorkerCount: preRoutingWorkerCount,
cfg,
outputFormat,
durationSeconds: job.duration,
isMemoryExhaustion: false,
});
const captureRouting: CaptureRouting = inversionFallback
? {
kind: "worker_inversion",
state: "active",
fallback: {
kind: inversionFallback.useStreamingEncode ? "sdr_streaming" : "sdr_disk",
workerCount: inversionFallback.workerCount,
forceParallelStream: false,
},
}
: parallelRouterFallback
? {
kind: "parallel_router",
state: "active",
fallback: {
kind: parallelRouterFallback.useStreamingEncode ? "sdr_streaming" : "sdr_disk",
workerCount: parallelRouterFallback.workerCount,
forceParallelStream: false,
},
}
: { kind: "default" };
let capturePlan: CapturePlan = createCapturePlan({
workerCount, workerCount,
forceScreenshot: captureForceScreenshot,
forceParallelStream: deParallelStreamForced || captureParallelStreamForced,
useStreamingEncode, useStreamingEncode,
useLayeredComposite, useLayeredComposite,
usePageSideCompositing: usePageSideCompositingForTransitions, usePageSideCompositing: usePageSideCompositingForTransitions,
hasHdrContent, hasHdrContent,
forceScreenshot: captureForceScreenshot, needsAlpha,
routing: captureRouting,
});
const syncCapturePlan = (): void => {
workerCount = capturePlan.workerCount;
captureForceScreenshot = capturePlan.forceScreenshot;
useStreamingEncode = capturePlan.kind === "sdr_streaming";
deParallelStreamForced =
capturePlan.kind === "sdr_streaming" && capturePlan.forceParallelStream;
if (capturePlan.routing.kind === "worker_inversion") {
deWorkerInversion = capturePlan.routing.state === "active" ? "inverted" : "reverted";
}
if (capturePlan.routing.kind === "parallel_router") {
deParallelRouter = capturePlan.routing.state === "active" ? "routed" : "reverted";
}
};
syncCapturePlan();
updateCaptureObservability({
workerCount: capturePlan.workerCount,
useStreamingEncode: capturePlan.kind === "sdr_streaming",
useLayeredComposite: capturePlan.kind === "hdr_layered",
usePageSideCompositing: capturePlan.usePageSideCompositing,
hasHdrContent: capturePlan.hasHdrContent,
forceScreenshot: capturePlan.forceScreenshot,
}); });
observability.checkpoint("capture_strategy", "resolved", { observability.checkpoint("capture_strategy", "resolved", {
workerCount, plan: capturePlan.kind,
forceScreenshot: captureForceScreenshot, workerCount: capturePlan.workerCount,
forceScreenshot: capturePlan.forceScreenshot,
captureBeyondViewport: resolvedCaptureBeyondViewport ?? null, captureBeyondViewport: resolvedCaptureBeyondViewport ?? null,
useStreamingEncode, useStreamingEncode: capturePlan.kind === "sdr_streaming",
useLayeredComposite, useLayeredComposite: capturePlan.kind === "hdr_layered",
usePageSideCompositing: usePageSideCompositingForTransitions, usePageSideCompositing: capturePlan.usePageSideCompositing,
hasHdrContent, hasHdrContent: capturePlan.hasHdrContent,
hasShaderTransitions: compiled.hasShaderTransitions, hasShaderTransitions: compiled.hasShaderTransitions,
isPngSequence, isPngSequence,
}); });
@@ -2750,13 +2819,13 @@ async function executeRenderPipeline(input: {
// into the active rgb48le signal space. Shader transitions use this same // into the active rgb48le signal space. Shader transitions use this same
// path for SDR compositions so the engine can apply transition math to // path for SDR compositions so the engine can apply transition math to
// isolated scene buffers instead of recording plain DOM screenshots. // isolated scene buffers instead of recording plain DOM screenshots.
if (useLayeredComposite) { if (capturePlan.kind === "hdr_layered") {
const layeredPlan = capturePlan;
// Layered composite always runs in screenshot mode — keep // Layered composite always runs in screenshot mode — keep
// `captureForceScreenshot` in sync so the perf summary and any // `captureForceScreenshot` in sync so the perf summary and any
// post-HDR diagnostic that reads the boolean see the same value // post-HDR diagnostic that reads the boolean see the same value
// the stage uses internally. // the stage uses internally.
captureForceScreenshot = true; updateCaptureObservability({ forceScreenshot: layeredPlan.forceScreenshot });
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
const hdrRes = await observeRenderStage( const hdrRes = await observeRenderStage(
observability, observability,
"capture_hdr_layered", "capture_hdr_layered",
@@ -2765,7 +2834,7 @@ async function executeRenderPipeline(input: {
runCaptureHdrStage({ runCaptureHdrStage({
job, job,
cfg, cfg,
forceScreenshot: captureForceScreenshot, plan: layeredPlan,
log, log,
projectDir, projectDir,
compiledDir, compiledDir,
@@ -2808,9 +2877,13 @@ async function executeRenderPipeline(input: {
// streaming spawn fails (non-abort) the stage returns { success: false } // streaming spawn fails (non-abort) the stage returns { success: false }
// and we fall back to the disk path below. // and we fall back to the disk path below.
let streamingHandled = false; let streamingHandled = false;
if (useStreamingEncode) { if (capturePlan.kind === "sdr_streaming") {
const captureFrameStart = Date.now(); const captureFrameStart = Date.now();
const invokeStreaming = () => { const invokeStreaming = () => {
if (capturePlan.kind !== "sdr_streaming") {
throw new Error(`Cannot invoke streaming stage with ${capturePlan.kind} plan`);
}
const streamingPlan = capturePlan;
resetCaptureAttemptProgress(job); resetCaptureAttemptProgress(job);
return observeRenderStage( return observeRenderStage(
observability, observability,
@@ -2825,12 +2898,10 @@ async function executeRenderPipeline(input: {
job, job,
totalFrames, totalFrames,
cfg, cfg,
forceScreenshot: captureForceScreenshot, plan: streamingPlan,
log, log,
workerCount,
probeSession, probeSession,
outputFormat, outputFormat,
forceParallelStream: deParallelStreamForced || captureParallelStreamForced,
streamingEncoderOptions: { streamingEncoderOptions: {
fps: job.config.fps, fps: job.config.fps,
width, width,
@@ -2909,74 +2980,48 @@ async function executeRenderPipeline(input: {
? "drawElement self-verify failed; retrying with forceScreenshot" ? "drawElement self-verify failed; retrying with forceScreenshot"
: "capture failed on pinned worker count; retrying with forceScreenshot", : "capture failed on pinned worker count; retrying with forceScreenshot",
); );
captureForceScreenshot = true; const failedRouting = capturePlan.routing.kind;
capturePlan = replanAfterFailure(
capturePlan,
isVerifyError
? { kind: "draw_element_verification" }
: { kind: "capture_failure", memoryExhaustion: isMemoryExhaustion },
);
syncCapturePlan();
updateCaptureObservability({ updateCaptureObservability({
forceScreenshot: true, forceScreenshot: capturePlan.forceScreenshot,
deSelfVerifyFallback, deSelfVerifyFallback,
deFallbackReason, deFallbackReason,
deFallbackFailedDb, deFallbackFailedDb,
deFallbackFrameIndex, deFallbackFrameIndex,
deFallbackThresholdDb, deFallbackThresholdDb,
workerCount: capturePlan.workerCount,
useStreamingEncode: capturePlan.kind === "sdr_streaming",
deWorkerInversion,
deParallelRouter,
}); });
probeSession = null; probeSession = null;
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes if (failedRouting === "worker_inversion") {
// useStreamingEncode, or shouldUseStreamingEncode would keep
// resolving to the parallel-streaming shape on the retry instead
// of the well-tested parallel-disk fallback.
if (deParallelRouter === "routed") deParallelStreamForced = false;
const inversionRetryPlan = resolveInversionRetryPlan({
deWorkerInversion,
preInversionWorkerCount: preRoutingWorkerCount,
cfg,
outputFormat,
durationSeconds: job.duration,
isMemoryExhaustion,
});
const parallelRouterRetryPlan = resolveParallelRouterRetryPlan({
deParallelRouter,
preRouterWorkerCount: preRoutingWorkerCount,
cfg,
outputFormat,
durationSeconds: job.duration,
isMemoryExhaustion,
});
if (inversionRetryPlan) {
// The inversion bet on drawElement and lost — re-render on the // The inversion bet on drawElement and lost — re-render on the
// pre-inversion parallel screenshot path instead of single-worker // pre-inversion parallel screenshot path instead of single-worker
// screenshot streaming (the slowest capture shape for this size). // screenshot streaming (the slowest capture shape for this size).
// "reverted" (not cleared) so telemetry keeps the lost-inversion // "reverted" (not cleared) so telemetry keeps the lost-inversion
// cohort distinguishable from renders that never inverted. // cohort distinguishable from renders that never inverted.
deWorkerInversion = inversionRetryPlan.deWorkerInversion;
workerCount = inversionRetryPlan.workerCount;
useStreamingEncode = inversionRetryPlan.useStreamingEncode;
updateCaptureObservability({
workerCount,
useStreamingEncode,
deWorkerInversion,
});
log.info( log.info(
`[Render] Reverting worker inversion for the retry: ${workerCount} workers, ` + `[Render] Reverting worker inversion for the retry: ${capturePlan.workerCount} workers, ` +
`streaming=${useStreamingEncode}.`, `plan=${capturePlan.kind}.`,
); );
} else if (parallelRouterRetryPlan) { } else if (failedRouting === "parallel_router") {
// The router's bet on verified parallel streaming lost — re-render // The router's bet on verified parallel streaming lost — re-render
// on the ordinary (non-DE) parallel path at the pre-router worker // on the ordinary (non-DE) parallel path at the pre-router worker
// count, same "reverted, not cleared" telemetry contract as the // count, same "reverted, not cleared" telemetry contract as the
// inversion above. // inversion above.
deParallelRouter = parallelRouterRetryPlan.deParallelRouter;
workerCount = parallelRouterRetryPlan.workerCount;
useStreamingEncode = parallelRouterRetryPlan.useStreamingEncode;
updateCaptureObservability({
workerCount,
useStreamingEncode,
deParallelRouter,
});
log.info( log.info(
`[Render] Reverting parallel router for the retry: ${workerCount} workers, ` + `[Render] Reverting parallel router for the retry: ${capturePlan.workerCount} workers, ` +
`streaming=${useStreamingEncode}.`, `plan=${capturePlan.kind}.`,
); );
} }
if (useStreamingEncode) { if (capturePlan.kind === "sdr_streaming") {
streamingRes = await invokeStreaming(); streamingRes = await invokeStreaming();
} else { } else {
// Parallel retry goes through the disk path below. // Parallel retry goes through the disk path below.
@@ -3005,7 +3050,10 @@ async function executeRenderPipeline(input: {
perfStages.captureSetupMs = Math.max(0, perfStages.captureMs - captureFrameMs); perfStages.captureSetupMs = Math.max(0, perfStages.captureMs - captureFrameMs);
perfStages.encodeMs = streamingRes.encodeMs; // Overlapped with capture perfStages.encodeMs = streamingRes.encodeMs; // Overlapped with capture
} else { } else {
useStreamingEncode = false; if (capturePlan.kind === "sdr_streaming") {
capturePlan = replanAfterFailure(capturePlan, { kind: "streaming_unavailable" });
syncCapturePlan();
}
// The disk path has no drain-time self-verification — clamp // The disk path has no drain-time self-verification — clamp
// default-on drawElement here exactly like the pre-capture clamp // default-on drawElement here exactly like the pre-capture clamp
// (verified-path confinement). Skipped when screenshots are already // (verified-path confinement). Skipped when screenshots are already
@@ -3013,7 +3061,7 @@ async function executeRenderPipeline(input: {
// opt-in, mirroring the clamp above. // opt-in, mirroring the clamp above.
if ( if (
cfg.useDrawElement && cfg.useDrawElement &&
!captureForceScreenshot && !capturePlan.forceScreenshot &&
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE !== "true" process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE !== "true"
) { ) {
cfg.useDrawElement = false; cfg.useDrawElement = false;
@@ -3029,19 +3077,23 @@ async function executeRenderPipeline(input: {
probeSession = null; probeSession = null;
} }
} }
updateCaptureObservability({ useStreamingEncode }); updateCaptureObservability({ useStreamingEncode: false });
observability.checkpoint("capture_streaming", "spawn failed; falling back to disk"); observability.checkpoint("capture_streaming", "spawn failed; falling back to disk");
} }
} }
if (!streamingHandled) { if (!streamingHandled) {
if (capturePlan.kind !== "sdr_disk") {
throw new Error(`Disk capture requires sdr_disk plan; got ${capturePlan.kind}`);
}
const diskPlan = capturePlan;
// ── Disk-based capture (original flow) ──────────────────────────── // ── Disk-based capture (original flow) ────────────────────────────
resetCaptureAttemptProgress(job); resetCaptureAttemptProgress(job);
const captureFrameStart = Date.now(); const captureFrameStart = Date.now();
const captureRes = await observeRenderStage( const captureRes = await observeRenderStage(
observability, observability,
"capture_disk", "capture_disk",
captureStageObservationData({ needsAlpha }), captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
() => () =>
runCaptureStage({ runCaptureStage({
fileServer: activeFileServer, fileServer: activeFileServer,
@@ -3050,11 +3102,9 @@ async function executeRenderPipeline(input: {
job, job,
totalFrames, totalFrames,
cfg, cfg,
forceScreenshot: captureForceScreenshot, plan: diskPlan,
log, log,
workerCount,
probeSession, probeSession,
needsAlpha,
captureAttempts, captureAttempts,
dedupPerfs, dedupPerfs,
buildCaptureOptions, buildCaptureOptions,