mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-10 22:20:14 +00:00
refactor(producer): make capture plans immutable
This commit is contained in:
@@ -59,6 +59,7 @@ import { defaultLogger } from "../../logger.js";
|
|||||||
import { runEncodeStage } from "../render/stages/encodeStage.js";
|
import { runEncodeStage } from "../render/stages/encodeStage.js";
|
||||||
import { runCaptureStage } from "../render/stages/captureStage.js";
|
import { runCaptureStage } from "../render/stages/captureStage.js";
|
||||||
import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
|
import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
|
||||||
|
import { createCapturePlan } from "../render/capturePlan.js";
|
||||||
import {
|
import {
|
||||||
type ChunkSliceJson,
|
type ChunkSliceJson,
|
||||||
type LockedRenderConfig,
|
type LockedRenderConfig,
|
||||||
@@ -589,6 +590,18 @@ export async function renderChunk(
|
|||||||
// one Chrome session per worker, so captureStageMs includes those
|
// one Chrome session per worker, so captureStageMs includes those
|
||||||
// boots; sessionBootMs stays 0 there.
|
// boots; sessionBootMs stays 0 there.
|
||||||
const captureStarted = Date.now();
|
const captureStarted = Date.now();
|
||||||
|
const capturePlan = createCapturePlan({
|
||||||
|
workerCount: chunkWorkerCount,
|
||||||
|
forceScreenshot: encoder.forceScreenshot,
|
||||||
|
useStreamingEncode: false,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
usePageSideCompositing: false,
|
||||||
|
hasHdrContent: false,
|
||||||
|
needsAlpha: plan.dimensions.format !== "mp4",
|
||||||
|
});
|
||||||
|
if (capturePlan.kind !== "sdr_disk") {
|
||||||
|
throw new Error(`Distributed chunk requires sdr_disk plan; got ${capturePlan.kind}`);
|
||||||
|
}
|
||||||
await runCaptureStage({
|
await runCaptureStage({
|
||||||
fileServer,
|
fileServer,
|
||||||
workDir,
|
workDir,
|
||||||
@@ -596,11 +609,9 @@ export async function renderChunk(
|
|||||||
job,
|
job,
|
||||||
totalFrames: framesInChunk,
|
totalFrames: framesInChunk,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot: encoder.forceScreenshot,
|
plan: capturePlan,
|
||||||
log,
|
log,
|
||||||
workerCount: chunkWorkerCount,
|
|
||||||
probeSession: session,
|
probeSession: session,
|
||||||
needsAlpha: plan.dimensions.format !== "mp4",
|
|
||||||
captureAttempts: [],
|
captureAttempts: [],
|
||||||
// Distributed chunks run on Linux (beginframe) where dedup never arms;
|
// Distributed chunks run on Linux (beginframe) where dedup never arms;
|
||||||
// a throwaway sink satisfies the type without per-chunk dedup reporting.
|
// a throwaway sink satisfies the type without per-chunk dedup reporting.
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { createCapturePlan, replanAfterFailure, type CaptureRouting } from "./capturePlan.js";
|
||||||
|
|
||||||
|
function streaming(routing?: CaptureRouting) {
|
||||||
|
return createCapturePlan({
|
||||||
|
workerCount: 1,
|
||||||
|
forceScreenshot: false,
|
||||||
|
forceParallelStream: false,
|
||||||
|
useStreamingEncode: true,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
usePageSideCompositing: false,
|
||||||
|
hasHdrContent: false,
|
||||||
|
needsAlpha: false,
|
||||||
|
routing,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("CapturePlan", () => {
|
||||||
|
it("makes layered capture dominant and enforces its screenshot invariant", () => {
|
||||||
|
const plan = createCapturePlan({
|
||||||
|
workerCount: 3,
|
||||||
|
forceScreenshot: false,
|
||||||
|
forceParallelStream: true,
|
||||||
|
useStreamingEncode: true,
|
||||||
|
useLayeredComposite: true,
|
||||||
|
usePageSideCompositing: false,
|
||||||
|
hasHdrContent: true,
|
||||||
|
needsAlpha: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(plan).toMatchObject({
|
||||||
|
kind: "hdr_layered",
|
||||||
|
workerCount: 3,
|
||||||
|
forceScreenshot: true,
|
||||||
|
forceParallelStream: false,
|
||||||
|
});
|
||||||
|
expect(Object.isFrozen(plan)).toBe(true);
|
||||||
|
expect(Object.isFrozen(plan.routing)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back from an unavailable streaming encoder to the same disk route", () => {
|
||||||
|
const initial = streaming();
|
||||||
|
const next = replanAfterFailure(initial, { kind: "streaming_unavailable" });
|
||||||
|
|
||||||
|
expect(next).toMatchObject({ kind: "sdr_disk", workerCount: 1, forceScreenshot: false });
|
||||||
|
expect(initial.kind).toBe("sdr_streaming");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("makes page-side compositing force screenshot capture", () => {
|
||||||
|
const plan = createCapturePlan({
|
||||||
|
workerCount: 1,
|
||||||
|
forceScreenshot: false,
|
||||||
|
forceParallelStream: false,
|
||||||
|
useStreamingEncode: true,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
usePageSideCompositing: true,
|
||||||
|
hasHdrContent: false,
|
||||||
|
needsAlpha: false,
|
||||||
|
});
|
||||||
|
expect(plan).toMatchObject({ kind: "sdr_streaming", forceScreenshot: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries an ordinary verification failure in streaming screenshot mode", () => {
|
||||||
|
expect(replanAfterFailure(streaming(), { kind: "draw_element_verification" })).toMatchObject({
|
||||||
|
kind: "sdr_streaming",
|
||||||
|
workerCount: 1,
|
||||||
|
forceScreenshot: true,
|
||||||
|
routing: { kind: "default" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("atomically restores the pre-inversion disk route after verification failure", () => {
|
||||||
|
const initial = streaming({
|
||||||
|
kind: "worker_inversion",
|
||||||
|
state: "active",
|
||||||
|
fallback: { kind: "sdr_disk", workerCount: 5, forceParallelStream: false },
|
||||||
|
});
|
||||||
|
const next = replanAfterFailure(initial, { kind: "draw_element_verification" });
|
||||||
|
|
||||||
|
expect(next).toMatchObject({
|
||||||
|
kind: "sdr_disk",
|
||||||
|
workerCount: 5,
|
||||||
|
forceScreenshot: true,
|
||||||
|
routing: { kind: "worker_inversion", state: "reverted" },
|
||||||
|
});
|
||||||
|
expect(initial).toMatchObject({ workerCount: 1, routing: { state: "active" } });
|
||||||
|
expect(Object.isFrozen(next.routing)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reduces an OOM fallback to one worker without losing immutable routing state", () => {
|
||||||
|
const initial = streaming({
|
||||||
|
kind: "parallel_router",
|
||||||
|
state: "active",
|
||||||
|
fallback: { kind: "sdr_disk", workerCount: 5, forceParallelStream: false },
|
||||||
|
});
|
||||||
|
const next = replanAfterFailure(initial, {
|
||||||
|
kind: "capture_failure",
|
||||||
|
memoryExhaustion: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(next).toMatchObject({
|
||||||
|
kind: "sdr_disk",
|
||||||
|
workerCount: 1,
|
||||||
|
forceScreenshot: true,
|
||||||
|
routing: { kind: "parallel_router", state: "reverted" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a streaming transition from a non-streaming plan", () => {
|
||||||
|
const disk = createCapturePlan({
|
||||||
|
workerCount: 2,
|
||||||
|
forceScreenshot: true,
|
||||||
|
useStreamingEncode: false,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
usePageSideCompositing: false,
|
||||||
|
hasHdrContent: false,
|
||||||
|
needsAlpha: false,
|
||||||
|
});
|
||||||
|
expect(() => replanAfterFailure(disk, { kind: "streaming_unavailable" })).toThrow(
|
||||||
|
"Cannot apply streaming_unavailable to sdr_disk",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
/**
|
||||||
|
* Immutable routing decision for the capture phase.
|
||||||
|
*
|
||||||
|
* The orchestrator used to carry the same decision in several mutable booleans
|
||||||
|
* (`useStreamingEncode`, `useLayeredComposite`, `forceScreenshot`) plus worker
|
||||||
|
* routing state. Keeping those values independently mutable made invalid
|
||||||
|
* combinations representable during fallback. A CapturePlan is the single
|
||||||
|
* value consumed by capture stages, and `replanAfterFailure` is the only
|
||||||
|
* transition between variants.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type CapturePlanTarget = Readonly<{
|
||||||
|
kind: "sdr_streaming" | "sdr_disk";
|
||||||
|
workerCount: number;
|
||||||
|
forceParallelStream: boolean;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
export type CaptureRouting =
|
||||||
|
| Readonly<{ kind: "default" }>
|
||||||
|
| Readonly<{
|
||||||
|
kind: "worker_inversion" | "parallel_router";
|
||||||
|
state: "active" | "reverted";
|
||||||
|
fallback: CapturePlanTarget;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
interface CapturePlanBase {
|
||||||
|
readonly workerCount: number;
|
||||||
|
readonly forceScreenshot: boolean;
|
||||||
|
readonly forceParallelStream: boolean;
|
||||||
|
readonly usePageSideCompositing: boolean;
|
||||||
|
readonly hasHdrContent: boolean;
|
||||||
|
readonly needsAlpha: boolean;
|
||||||
|
readonly routing: CaptureRouting;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SdrStreamingCapturePlan extends CapturePlanBase {
|
||||||
|
readonly kind: "sdr_streaming";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SdrDiskCapturePlan extends CapturePlanBase {
|
||||||
|
readonly kind: "sdr_disk";
|
||||||
|
readonly forceParallelStream: false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HdrLayeredCapturePlan extends CapturePlanBase {
|
||||||
|
readonly kind: "hdr_layered";
|
||||||
|
readonly forceScreenshot: true;
|
||||||
|
readonly forceParallelStream: false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CapturePlan = SdrStreamingCapturePlan | SdrDiskCapturePlan | HdrLayeredCapturePlan;
|
||||||
|
|
||||||
|
export interface CreateCapturePlanInput {
|
||||||
|
workerCount: number;
|
||||||
|
forceScreenshot: boolean;
|
||||||
|
forceParallelStream?: boolean;
|
||||||
|
useStreamingEncode: boolean;
|
||||||
|
useLayeredComposite: boolean;
|
||||||
|
usePageSideCompositing: boolean;
|
||||||
|
hasHdrContent: boolean;
|
||||||
|
needsAlpha: boolean;
|
||||||
|
routing?: CaptureRouting;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CapturePlanFailure =
|
||||||
|
| Readonly<{ kind: "streaming_unavailable" }>
|
||||||
|
| Readonly<{ kind: "draw_element_verification" }>
|
||||||
|
| Readonly<{ kind: "capture_failure"; memoryExhaustion: boolean }>;
|
||||||
|
|
||||||
|
function assertWorkerCount(workerCount: number): void {
|
||||||
|
if (!Number.isInteger(workerCount) || workerCount < 1) {
|
||||||
|
throw new Error(`CapturePlan workerCount must be a positive integer; got ${workerCount}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function freezeTarget(target: CapturePlanTarget): CapturePlanTarget {
|
||||||
|
assertWorkerCount(target.workerCount);
|
||||||
|
if (target.kind === "sdr_disk" && target.forceParallelStream) {
|
||||||
|
throw new Error("CapturePlan disk fallback cannot force parallel streaming");
|
||||||
|
}
|
||||||
|
return Object.freeze({ ...target });
|
||||||
|
}
|
||||||
|
|
||||||
|
function freezeRouting(routing: CaptureRouting | undefined): CaptureRouting {
|
||||||
|
if (!routing || routing.kind === "default") return Object.freeze({ kind: "default" });
|
||||||
|
return Object.freeze({ ...routing, fallback: freezeTarget(routing.fallback) });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createCapturePlan(input: CreateCapturePlanInput): CapturePlan {
|
||||||
|
assertWorkerCount(input.workerCount);
|
||||||
|
const base = {
|
||||||
|
workerCount: input.workerCount,
|
||||||
|
forceScreenshot: input.forceScreenshot || input.usePageSideCompositing,
|
||||||
|
forceParallelStream: input.useStreamingEncode ? (input.forceParallelStream ?? false) : false,
|
||||||
|
usePageSideCompositing: input.usePageSideCompositing,
|
||||||
|
hasHdrContent: input.hasHdrContent,
|
||||||
|
needsAlpha: input.needsAlpha,
|
||||||
|
routing: freezeRouting(input.routing),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (input.useLayeredComposite) {
|
||||||
|
return Object.freeze({
|
||||||
|
...base,
|
||||||
|
kind: "hdr_layered",
|
||||||
|
forceScreenshot: true,
|
||||||
|
forceParallelStream: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (input.useStreamingEncode) {
|
||||||
|
return Object.freeze({ ...base, kind: "sdr_streaming" });
|
||||||
|
}
|
||||||
|
return Object.freeze({ ...base, kind: "sdr_disk", forceParallelStream: false });
|
||||||
|
}
|
||||||
|
|
||||||
|
function revertedRouting(routing: CaptureRouting): CaptureRouting {
|
||||||
|
if (routing.kind === "default") return routing;
|
||||||
|
return freezeRouting({ ...routing, state: "reverted" });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pure, exhaustive capture fallback transition. The input plan is never mutated. */
|
||||||
|
export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan {
|
||||||
|
if (plan.kind !== "sdr_streaming") {
|
||||||
|
throw new Error(`Cannot apply ${failure.kind} to ${plan.kind} capture plan`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failure.kind === "streaming_unavailable") {
|
||||||
|
return createCapturePlan({
|
||||||
|
...plan,
|
||||||
|
useStreamingEncode: false,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
forceParallelStream: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const fallback =
|
||||||
|
plan.routing.kind === "default"
|
||||||
|
? { kind: plan.kind, workerCount: plan.workerCount, forceParallelStream: false }
|
||||||
|
: plan.routing.fallback;
|
||||||
|
const workerCount =
|
||||||
|
failure.kind === "capture_failure" && failure.memoryExhaustion ? 1 : fallback.workerCount;
|
||||||
|
return createCapturePlan({
|
||||||
|
...plan,
|
||||||
|
workerCount,
|
||||||
|
forceScreenshot: true,
|
||||||
|
forceParallelStream: fallback.forceParallelStream,
|
||||||
|
useStreamingEncode: fallback.kind === "sdr_streaming",
|
||||||
|
useLayeredComposite: false,
|
||||||
|
routing: revertedRouting(plan.routing),
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -25,9 +25,8 @@
|
|||||||
* because `captureAlphaPng` hangs under `--enable-begin-frame-control`.
|
* because `captureAlphaPng` hangs under `--enable-begin-frame-control`.
|
||||||
* Previously the stage mutated `cfg.forceScreenshot = true` directly;
|
* Previously the stage mutated `cfg.forceScreenshot = true` directly;
|
||||||
* the value is now derived into a local `hdrCfg` so the caller-owned
|
* the value is now derived into a local `hdrCfg` so the caller-owned
|
||||||
* `cfg` survives the stage unchanged. The sequencer is expected to
|
* `cfg` survives the stage unchanged. The sequencer passes an immutable
|
||||||
* pass `forceScreenshot: true` for the layered branch as a contract
|
* `hdr_layered` plan whose construction guarantees screenshot mode.
|
||||||
* check.
|
|
||||||
*
|
*
|
||||||
* Resource setup (HDR video extraction, image decode, dim probing) lives
|
* Resource setup (HDR video extraction, image decode, dim probing) lives
|
||||||
* in `captureHdrResources.ts`; per-frame work lives in
|
* in `captureHdrResources.ts`; per-frame work lives in
|
||||||
@@ -44,7 +43,6 @@ import {
|
|||||||
type EngineConfig,
|
type EngineConfig,
|
||||||
type HdrTransfer,
|
type HdrTransfer,
|
||||||
type StreamingEncoder,
|
type StreamingEncoder,
|
||||||
calculateOptimalWorkers,
|
|
||||||
closeCaptureSession,
|
closeCaptureSession,
|
||||||
createCaptureSession,
|
createCaptureSession,
|
||||||
getEncoderPreset,
|
getEncoderPreset,
|
||||||
@@ -77,18 +75,13 @@ import { partitionTransitionFrames, shouldUseHybridLayeredPath } from "./capture
|
|||||||
import { runSequentialLayeredFrameLoop } from "./captureHdrSequentialLoop.js";
|
import { runSequentialLayeredFrameLoop } from "./captureHdrSequentialLoop.js";
|
||||||
import { runHybridLayeredFrameLoop } from "./captureHdrHybridLoop.js";
|
import { runHybridLayeredFrameLoop } from "./captureHdrHybridLoop.js";
|
||||||
import { wrapCaptureStageError } from "../captureStageError.js";
|
import { wrapCaptureStageError } from "../captureStageError.js";
|
||||||
|
import type { HdrLayeredCapturePlan } from "../capturePlan.js";
|
||||||
|
|
||||||
export interface CaptureHdrStageInput {
|
export interface CaptureHdrStageInput {
|
||||||
job: RenderJob;
|
job: RenderJob;
|
||||||
cfg: EngineConfig;
|
cfg: EngineConfig;
|
||||||
/**
|
/** Immutable layered route selected by the sequencer. */
|
||||||
* Capture-mode flag threaded from `compileStage`. The HDR layered
|
plan: HdrLayeredCapturePlan;
|
||||||
* branch requires `true` (see file header for the
|
|
||||||
* `captureAlphaPng` / `--enable-begin-frame-control` constraint);
|
|
||||||
* the stage throws if called with `false`. Stored locally as
|
|
||||||
* `hdrCfg.forceScreenshot` so the caller-owned `cfg` is not mutated.
|
|
||||||
*/
|
|
||||||
forceScreenshot: boolean;
|
|
||||||
log: ProducerLogger;
|
log: ProducerLogger;
|
||||||
|
|
||||||
projectDir: string;
|
projectDir: string;
|
||||||
@@ -120,13 +113,6 @@ export interface CaptureHdrStageInput {
|
|||||||
/** Mutated in place (counters incremented). */
|
/** Mutated in place (counters incremented). */
|
||||||
hdrDiagnostics: HdrDiagnostics;
|
hdrDiagnostics: HdrDiagnostics;
|
||||||
|
|
||||||
/**
|
|
||||||
* Worker budget for the hybrid layered path. Only consulted when the
|
|
||||||
* gating predicate (`shouldUseHybridLayeredPath`) returns true. The
|
|
||||||
* sequential loop always runs on a single DOM session.
|
|
||||||
*/
|
|
||||||
workerCount?: number;
|
|
||||||
|
|
||||||
abortSignal: AbortSignal | undefined;
|
abortSignal: AbortSignal | undefined;
|
||||||
assertNotAborted: () => void;
|
assertNotAborted: () => void;
|
||||||
onProgress?: ProgressCallback;
|
onProgress?: ProgressCallback;
|
||||||
@@ -160,7 +146,7 @@ export async function runCaptureHdrStage(
|
|||||||
const {
|
const {
|
||||||
job,
|
job,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot,
|
plan,
|
||||||
log,
|
log,
|
||||||
projectDir,
|
projectDir,
|
||||||
compiledDir,
|
compiledDir,
|
||||||
@@ -184,17 +170,11 @@ export async function runCaptureHdrStage(
|
|||||||
buildCaptureOptions,
|
buildCaptureOptions,
|
||||||
createRenderVideoFrameInjector,
|
createRenderVideoFrameInjector,
|
||||||
hdrDiagnostics,
|
hdrDiagnostics,
|
||||||
workerCount,
|
|
||||||
abortSignal,
|
abortSignal,
|
||||||
assertNotAborted,
|
assertNotAborted,
|
||||||
onProgress,
|
onProgress,
|
||||||
} = input;
|
} = input;
|
||||||
|
const { workerCount } = plan;
|
||||||
if (!forceScreenshot) {
|
|
||||||
throw new Error(
|
|
||||||
"captureHdrStage requires forceScreenshot=true; the layered composite path uses captureAlphaPng which hangs under --enable-begin-frame-control.",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const stageStart = Date.now();
|
const stageStart = Date.now();
|
||||||
let lastBrowserConsole: string[] = [];
|
let lastBrowserConsole: string[] = [];
|
||||||
@@ -378,16 +358,7 @@ export async function runCaptureHdrStage(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// ── Dispatch to sequential or hybrid frame loop ────────────────────
|
// ── Dispatch to sequential or hybrid frame loop ────────────────────
|
||||||
// Resolve the worker budget here rather than threading it through the
|
const effectiveWorkerCount = Math.max(1, workerCount);
|
||||||
// renderOrchestrator call: keeps the renderOrchestrator diff zero
|
|
||||||
// (hf#732 PR 4 is intentionally a producer-stage-local change), at the
|
|
||||||
// cost of recomputing the same number the orchestrator already knows.
|
|
||||||
// The cost is negligible (one cpus() call) and the two values stay in
|
|
||||||
// lockstep because `calculateOptimalWorkers` is pure.
|
|
||||||
const effectiveWorkerCount =
|
|
||||||
workerCount !== undefined
|
|
||||||
? Math.max(1, workerCount)
|
|
||||||
: calculateOptimalWorkers(totalFrames, job.config.workers, hdrCfg);
|
|
||||||
const transitionFrameCount = partitionTransitionFrames(transitionRanges, totalFrames).size;
|
const transitionFrameCount = partitionTransitionFrames(transitionRanges, totalFrames).size;
|
||||||
const useHybrid = shouldUseHybridLayeredPath({
|
const useHybrid = shouldUseHybridLayeredPath({
|
||||||
hasHdrContent,
|
hasHdrContent,
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ import {
|
|||||||
} from "../../renderOrchestrator.js";
|
} from "../../renderOrchestrator.js";
|
||||||
import { wrapCaptureStageError } from "../captureStageError.js";
|
import { wrapCaptureStageError } from "../captureStageError.js";
|
||||||
import { updateJobStatus } from "../shared.js";
|
import { updateJobStatus } from "../shared.js";
|
||||||
|
import type { SdrDiskCapturePlan } from "../capturePlan.js";
|
||||||
|
|
||||||
export interface CaptureStageInput {
|
export interface CaptureStageInput {
|
||||||
fileServer: FileServerHandle;
|
fileServer: FileServerHandle;
|
||||||
@@ -76,23 +77,11 @@ export interface CaptureStageInput {
|
|||||||
*/
|
*/
|
||||||
totalFrames: number;
|
totalFrames: number;
|
||||||
cfg: EngineConfig;
|
cfg: EngineConfig;
|
||||||
/**
|
/** Immutable route selected by the sequencer. */
|
||||||
* Capture-mode flag threaded from `compileStage`. The stage derives a
|
plan: SdrDiskCapturePlan;
|
||||||
* local copy of `cfg` with this value applied to `forceScreenshot`
|
|
||||||
* before any engine call, so the caller-owned `cfg` is never mutated.
|
|
||||||
* The sequencer may override `compileResult.forceScreenshot` after a
|
|
||||||
* BeginFrame calibration timeout — passing the override through this
|
|
||||||
* parameter keeps the decision visible at the call site instead of
|
|
||||||
* hiding it inside a shared mutable config.
|
|
||||||
*/
|
|
||||||
forceScreenshot: boolean;
|
|
||||||
log: ProducerLogger;
|
log: ProducerLogger;
|
||||||
/** Initial worker count from `resolveRenderWorkerCount`; adaptive retry may reduce it. */
|
|
||||||
workerCount: number;
|
|
||||||
/** Reused for the sequential path's first session if non-null. */
|
/** Reused for the sequential path's first session if non-null. */
|
||||||
probeSession: CaptureSession | null;
|
probeSession: CaptureSession | null;
|
||||||
/** True for webm / mov / png-sequence (controls capture format + extension). */
|
|
||||||
needsAlpha: boolean;
|
|
||||||
/** Mutated in place — each parallel retry attempt is appended. */
|
/** Mutated in place — each parallel retry attempt is appended. */
|
||||||
captureAttempts: CaptureAttemptSummary[];
|
captureAttempts: CaptureAttemptSummary[];
|
||||||
/**
|
/**
|
||||||
@@ -154,7 +143,7 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
|
|||||||
job,
|
job,
|
||||||
totalFrames,
|
totalFrames,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot,
|
plan,
|
||||||
log,
|
log,
|
||||||
captureAttempts,
|
captureAttempts,
|
||||||
buildCaptureOptions,
|
buildCaptureOptions,
|
||||||
@@ -162,17 +151,18 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
|
|||||||
abortSignal,
|
abortSignal,
|
||||||
assertNotAborted,
|
assertNotAborted,
|
||||||
onProgress,
|
onProgress,
|
||||||
needsAlpha,
|
|
||||||
frameRange,
|
frameRange,
|
||||||
dedupPerfs,
|
dedupPerfs,
|
||||||
} = input;
|
} = input;
|
||||||
let { workerCount, probeSession } = input;
|
let { probeSession } = input;
|
||||||
|
let { workerCount } = plan;
|
||||||
|
const { forceScreenshot, needsAlpha } = plan;
|
||||||
let lastBrowserConsole: string[] = [];
|
let lastBrowserConsole: string[] = [];
|
||||||
let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport;
|
let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport;
|
||||||
|
|
||||||
// Derive a local cfg view rather than reading `forceScreenshot` from the
|
// Derive a local cfg view rather than reading `forceScreenshot` from the
|
||||||
// caller-owned `cfg`. The sequencer threads the resolved value via the
|
// caller-owned `cfg`. The sequencer threads the resolved value via the
|
||||||
// explicit parameter; this keeps the engine-facing config a pure
|
// immutable plan; this keeps the engine-facing config a pure
|
||||||
// pass-through.
|
// pass-through.
|
||||||
const captureCfg: EngineConfig =
|
const captureCfg: EngineConfig =
|
||||||
cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot };
|
cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot };
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { describe, expect, it, mock } from "bun:test";
|
import { describe, expect, it, mock } from "bun:test";
|
||||||
import { getCaptureStageBrowserConsole } from "../captureStageError.js";
|
import { getCaptureStageBrowserConsole } from "../captureStageError.js";
|
||||||
|
import { createCapturePlan } from "../capturePlan.js";
|
||||||
|
|
||||||
type MinimalEngineConfig = {
|
type MinimalEngineConfig = {
|
||||||
forceScreenshot: boolean;
|
forceScreenshot: boolean;
|
||||||
@@ -171,14 +172,21 @@ function createInput(cfg: MinimalEngineConfig) {
|
|||||||
},
|
},
|
||||||
totalFrames: 0,
|
totalFrames: 0,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot: false,
|
plan: createCapturePlan({
|
||||||
|
workerCount: 1,
|
||||||
|
forceScreenshot: false,
|
||||||
|
useStreamingEncode: true,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
usePageSideCompositing: false,
|
||||||
|
hasHdrContent: false,
|
||||||
|
needsAlpha: false,
|
||||||
|
}),
|
||||||
log: {
|
log: {
|
||||||
error: () => {},
|
error: () => {},
|
||||||
warn: () => {},
|
warn: () => {},
|
||||||
info: () => {},
|
info: () => {},
|
||||||
debug: () => {},
|
debug: () => {},
|
||||||
},
|
},
|
||||||
workerCount: 1,
|
|
||||||
probeSession: null,
|
probeSession: null,
|
||||||
outputFormat: "mp4",
|
outputFormat: "mp4",
|
||||||
streamingEncoderOptions: { fps: { num: 30, den: 1 }, width: 1920, height: 1080 },
|
streamingEncoderOptions: { fps: { num: 30, den: 1 }, width: 1920, height: 1080 },
|
||||||
@@ -408,10 +416,18 @@ describe("runCaptureStage", () => {
|
|||||||
try {
|
try {
|
||||||
await runCaptureStage({
|
await runCaptureStage({
|
||||||
...createInput(cfg),
|
...createInput(cfg),
|
||||||
|
plan: createCapturePlan({
|
||||||
|
workerCount: 1,
|
||||||
|
forceScreenshot: false,
|
||||||
|
useStreamingEncode: false,
|
||||||
|
useLayeredComposite: false,
|
||||||
|
usePageSideCompositing: false,
|
||||||
|
hasHdrContent: false,
|
||||||
|
needsAlpha: false,
|
||||||
|
}),
|
||||||
videoOnlyPath: undefined,
|
videoOnlyPath: undefined,
|
||||||
outputFormat: undefined,
|
outputFormat: undefined,
|
||||||
streamingEncoderOptions: undefined,
|
streamingEncoderOptions: undefined,
|
||||||
needsAlpha: false,
|
|
||||||
captureAttempts: [],
|
captureAttempts: [],
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -447,7 +463,15 @@ describe("runCaptureHdrStage", () => {
|
|||||||
duration: 1,
|
duration: 1,
|
||||||
},
|
},
|
||||||
cfg: { forceScreenshot: true },
|
cfg: { forceScreenshot: true },
|
||||||
forceScreenshot: true,
|
plan: createCapturePlan({
|
||||||
|
workerCount: 1,
|
||||||
|
forceScreenshot: true,
|
||||||
|
useStreamingEncode: false,
|
||||||
|
useLayeredComposite: true,
|
||||||
|
usePageSideCompositing: false,
|
||||||
|
hasHdrContent: false,
|
||||||
|
needsAlpha: false,
|
||||||
|
}),
|
||||||
log: {
|
log: {
|
||||||
error: () => {},
|
error: () => {},
|
||||||
warn: () => {},
|
warn: () => {},
|
||||||
@@ -496,7 +520,6 @@ describe("runCaptureHdrStage", () => {
|
|||||||
videoExtractionFailures: 0,
|
videoExtractionFailures: 0,
|
||||||
imageDecodeFailures: 0,
|
imageDecodeFailures: 0,
|
||||||
},
|
},
|
||||||
workerCount: 1,
|
|
||||||
abortSignal: undefined,
|
abortSignal: undefined,
|
||||||
assertNotAborted: () => {},
|
assertNotAborted: () => {},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -78,6 +78,7 @@ import { wrapCaptureStageError } from "../captureStageError.js";
|
|||||||
import { pushWorkerDedupPerfs } from "../perfSummary.js";
|
import { pushWorkerDedupPerfs } from "../perfSummary.js";
|
||||||
import { ensureFrameWritten } from "./captureHdrFrameShared.js";
|
import { ensureFrameWritten } from "./captureHdrFrameShared.js";
|
||||||
import { updateJobStatus } from "../shared.js";
|
import { updateJobStatus } from "../shared.js";
|
||||||
|
import type { SdrStreamingCapturePlan } from "../capturePlan.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* No-frame-progress watchdog for DE streaming capture. A worker (parallel
|
* No-frame-progress watchdog for DE streaming capture. A worker (parallel
|
||||||
@@ -173,27 +174,10 @@ export interface CaptureStreamingStageInput {
|
|||||||
*/
|
*/
|
||||||
totalFrames: number;
|
totalFrames: number;
|
||||||
cfg: EngineConfig;
|
cfg: EngineConfig;
|
||||||
/**
|
/** Immutable route selected by the sequencer. */
|
||||||
* Capture-mode flag threaded from `compileStage`. The stage derives a
|
plan: SdrStreamingCapturePlan;
|
||||||
* local copy of `cfg` with this value applied to `forceScreenshot`
|
|
||||||
* before any engine call, so the caller-owned `cfg` is never mutated.
|
|
||||||
* The sequencer may override `compileResult.forceScreenshot` after a
|
|
||||||
* BeginFrame calibration timeout — passing the override through this
|
|
||||||
* parameter keeps the decision visible at the call site instead of
|
|
||||||
* hiding it inside a shared mutable config.
|
|
||||||
*/
|
|
||||||
forceScreenshot: boolean;
|
|
||||||
log: ProducerLogger;
|
log: ProducerLogger;
|
||||||
workerCount: number;
|
|
||||||
probeSession: CaptureSession | null;
|
probeSession: CaptureSession | null;
|
||||||
/**
|
|
||||||
* Per-render override from the DE parallel router — see
|
|
||||||
* deParallelStreamForced's declaration in renderOrchestrator.ts. Distinct
|
|
||||||
* from the `HF_DE_PARALLEL_STREAM` manual opt-in (still read directly by
|
|
||||||
* this stage) because the router's decision must not leak across
|
|
||||||
* concurrently-running renders sharing this process via a global env var.
|
|
||||||
*/
|
|
||||||
forceParallelStream?: boolean;
|
|
||||||
/** For the spawn-failure log message context only. */
|
/** For the spawn-failure log message context only. */
|
||||||
outputFormat: string;
|
outputFormat: string;
|
||||||
/** Pre-built encoder options; passed straight to `spawnStreamingEncoder`. */
|
/** Pre-built encoder options; passed straight to `spawnStreamingEncoder`. */
|
||||||
@@ -552,7 +536,7 @@ export async function runCaptureStreamingStage(
|
|||||||
job,
|
job,
|
||||||
totalFrames,
|
totalFrames,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot,
|
plan,
|
||||||
log,
|
log,
|
||||||
outputFormat,
|
outputFormat,
|
||||||
streamingEncoderOptions,
|
streamingEncoderOptions,
|
||||||
@@ -562,16 +546,17 @@ export async function runCaptureStreamingStage(
|
|||||||
assertNotAborted,
|
assertNotAborted,
|
||||||
onProgress,
|
onProgress,
|
||||||
dedupPerfs,
|
dedupPerfs,
|
||||||
forceParallelStream,
|
|
||||||
} = input;
|
} = input;
|
||||||
let { workerCount, probeSession } = input;
|
let { probeSession } = input;
|
||||||
|
let { workerCount } = plan;
|
||||||
|
const { forceScreenshot, forceParallelStream } = plan;
|
||||||
let lastBrowserConsole: string[] = [];
|
let lastBrowserConsole: string[] = [];
|
||||||
let deDrainStats: DeDrainStats | undefined;
|
let deDrainStats: DeDrainStats | undefined;
|
||||||
let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport;
|
let captureBeyondViewport: boolean | undefined = probeSession?.options.captureBeyondViewport;
|
||||||
|
|
||||||
// Derive a local cfg view rather than reading `forceScreenshot` from the
|
// Derive a local cfg view rather than reading `forceScreenshot` from the
|
||||||
// caller-owned `cfg`. The sequencer threads the resolved value via the
|
// caller-owned `cfg`. The sequencer threads the resolved value via the
|
||||||
// explicit parameter; this keeps the engine-facing config a pure
|
// immutable plan; this keeps the engine-facing config a pure
|
||||||
// pass-through.
|
// pass-through.
|
||||||
const captureCfg: EngineConfig =
|
const captureCfg: EngineConfig =
|
||||||
cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot };
|
cfg.forceScreenshot === forceScreenshot ? cfg : { ...cfg, forceScreenshot };
|
||||||
|
|||||||
@@ -104,6 +104,12 @@ import { buildRenderErrorDetails } from "./render/cleanup.js";
|
|||||||
import { publishRenderFailure } from "./render/renderEventPublisher.js";
|
import { publishRenderFailure } from "./render/renderEventPublisher.js";
|
||||||
import { RenderExecutionContext } from "./render/renderExecutionContext.js";
|
import { RenderExecutionContext } from "./render/renderExecutionContext.js";
|
||||||
import { ArtifactTransaction } from "./render/artifactTransaction.js";
|
import { ArtifactTransaction } from "./render/artifactTransaction.js";
|
||||||
|
import {
|
||||||
|
createCapturePlan,
|
||||||
|
replanAfterFailure,
|
||||||
|
type CapturePlan,
|
||||||
|
type CaptureRouting,
|
||||||
|
} from "./render/capturePlan.js";
|
||||||
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
||||||
import { formatCaptureFrameName } from "../utils/paths.js";
|
import { formatCaptureFrameName } from "../utils/paths.js";
|
||||||
import { resolveEffectiveHdrMode } from "./render/hdrMode.js";
|
import { resolveEffectiveHdrMode } from "./render/hdrMode.js";
|
||||||
@@ -2693,22 +2699,85 @@ async function executeRenderPipeline(input: {
|
|||||||
hasShaderTransitions: compiled.hasShaderTransitions && !isGif,
|
hasShaderTransitions: compiled.hasShaderTransitions && !isGif,
|
||||||
isPngSequence,
|
isPngSequence,
|
||||||
});
|
});
|
||||||
updateCaptureObservability({
|
const inversionFallback = resolveInversionRetryPlan({
|
||||||
|
deWorkerInversion,
|
||||||
|
preInversionWorkerCount: preRoutingWorkerCount,
|
||||||
|
cfg,
|
||||||
|
outputFormat,
|
||||||
|
durationSeconds: job.duration,
|
||||||
|
isMemoryExhaustion: false,
|
||||||
|
});
|
||||||
|
const parallelRouterFallback = resolveParallelRouterRetryPlan({
|
||||||
|
deParallelRouter,
|
||||||
|
preRouterWorkerCount: preRoutingWorkerCount,
|
||||||
|
cfg,
|
||||||
|
outputFormat,
|
||||||
|
durationSeconds: job.duration,
|
||||||
|
isMemoryExhaustion: false,
|
||||||
|
});
|
||||||
|
const captureRouting: CaptureRouting = inversionFallback
|
||||||
|
? {
|
||||||
|
kind: "worker_inversion",
|
||||||
|
state: "active",
|
||||||
|
fallback: {
|
||||||
|
kind: inversionFallback.useStreamingEncode ? "sdr_streaming" : "sdr_disk",
|
||||||
|
workerCount: inversionFallback.workerCount,
|
||||||
|
forceParallelStream: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: parallelRouterFallback
|
||||||
|
? {
|
||||||
|
kind: "parallel_router",
|
||||||
|
state: "active",
|
||||||
|
fallback: {
|
||||||
|
kind: parallelRouterFallback.useStreamingEncode ? "sdr_streaming" : "sdr_disk",
|
||||||
|
workerCount: parallelRouterFallback.workerCount,
|
||||||
|
forceParallelStream: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: { kind: "default" };
|
||||||
|
let capturePlan: CapturePlan = createCapturePlan({
|
||||||
workerCount,
|
workerCount,
|
||||||
|
forceScreenshot: captureForceScreenshot,
|
||||||
|
forceParallelStream: deParallelStreamForced || captureParallelStreamForced,
|
||||||
useStreamingEncode,
|
useStreamingEncode,
|
||||||
useLayeredComposite,
|
useLayeredComposite,
|
||||||
usePageSideCompositing: usePageSideCompositingForTransitions,
|
usePageSideCompositing: usePageSideCompositingForTransitions,
|
||||||
hasHdrContent,
|
hasHdrContent,
|
||||||
forceScreenshot: captureForceScreenshot,
|
needsAlpha,
|
||||||
|
routing: captureRouting,
|
||||||
|
});
|
||||||
|
const syncCapturePlan = (): void => {
|
||||||
|
workerCount = capturePlan.workerCount;
|
||||||
|
captureForceScreenshot = capturePlan.forceScreenshot;
|
||||||
|
useStreamingEncode = capturePlan.kind === "sdr_streaming";
|
||||||
|
deParallelStreamForced =
|
||||||
|
capturePlan.kind === "sdr_streaming" && capturePlan.forceParallelStream;
|
||||||
|
if (capturePlan.routing.kind === "worker_inversion") {
|
||||||
|
deWorkerInversion = capturePlan.routing.state === "active" ? "inverted" : "reverted";
|
||||||
|
}
|
||||||
|
if (capturePlan.routing.kind === "parallel_router") {
|
||||||
|
deParallelRouter = capturePlan.routing.state === "active" ? "routed" : "reverted";
|
||||||
|
}
|
||||||
|
};
|
||||||
|
syncCapturePlan();
|
||||||
|
updateCaptureObservability({
|
||||||
|
workerCount: capturePlan.workerCount,
|
||||||
|
useStreamingEncode: capturePlan.kind === "sdr_streaming",
|
||||||
|
useLayeredComposite: capturePlan.kind === "hdr_layered",
|
||||||
|
usePageSideCompositing: capturePlan.usePageSideCompositing,
|
||||||
|
hasHdrContent: capturePlan.hasHdrContent,
|
||||||
|
forceScreenshot: capturePlan.forceScreenshot,
|
||||||
});
|
});
|
||||||
observability.checkpoint("capture_strategy", "resolved", {
|
observability.checkpoint("capture_strategy", "resolved", {
|
||||||
workerCount,
|
plan: capturePlan.kind,
|
||||||
forceScreenshot: captureForceScreenshot,
|
workerCount: capturePlan.workerCount,
|
||||||
|
forceScreenshot: capturePlan.forceScreenshot,
|
||||||
captureBeyondViewport: resolvedCaptureBeyondViewport ?? null,
|
captureBeyondViewport: resolvedCaptureBeyondViewport ?? null,
|
||||||
useStreamingEncode,
|
useStreamingEncode: capturePlan.kind === "sdr_streaming",
|
||||||
useLayeredComposite,
|
useLayeredComposite: capturePlan.kind === "hdr_layered",
|
||||||
usePageSideCompositing: usePageSideCompositingForTransitions,
|
usePageSideCompositing: capturePlan.usePageSideCompositing,
|
||||||
hasHdrContent,
|
hasHdrContent: capturePlan.hasHdrContent,
|
||||||
hasShaderTransitions: compiled.hasShaderTransitions,
|
hasShaderTransitions: compiled.hasShaderTransitions,
|
||||||
isPngSequence,
|
isPngSequence,
|
||||||
});
|
});
|
||||||
@@ -2750,13 +2819,13 @@ async function executeRenderPipeline(input: {
|
|||||||
// into the active rgb48le signal space. Shader transitions use this same
|
// into the active rgb48le signal space. Shader transitions use this same
|
||||||
// path for SDR compositions so the engine can apply transition math to
|
// path for SDR compositions so the engine can apply transition math to
|
||||||
// isolated scene buffers instead of recording plain DOM screenshots.
|
// isolated scene buffers instead of recording plain DOM screenshots.
|
||||||
if (useLayeredComposite) {
|
if (capturePlan.kind === "hdr_layered") {
|
||||||
|
const layeredPlan = capturePlan;
|
||||||
// Layered composite always runs in screenshot mode — keep
|
// Layered composite always runs in screenshot mode — keep
|
||||||
// `captureForceScreenshot` in sync so the perf summary and any
|
// `captureForceScreenshot` in sync so the perf summary and any
|
||||||
// post-HDR diagnostic that reads the boolean see the same value
|
// post-HDR diagnostic that reads the boolean see the same value
|
||||||
// the stage uses internally.
|
// the stage uses internally.
|
||||||
captureForceScreenshot = true;
|
updateCaptureObservability({ forceScreenshot: layeredPlan.forceScreenshot });
|
||||||
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
|
|
||||||
const hdrRes = await observeRenderStage(
|
const hdrRes = await observeRenderStage(
|
||||||
observability,
|
observability,
|
||||||
"capture_hdr_layered",
|
"capture_hdr_layered",
|
||||||
@@ -2765,7 +2834,7 @@ async function executeRenderPipeline(input: {
|
|||||||
runCaptureHdrStage({
|
runCaptureHdrStage({
|
||||||
job,
|
job,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot: captureForceScreenshot,
|
plan: layeredPlan,
|
||||||
log,
|
log,
|
||||||
projectDir,
|
projectDir,
|
||||||
compiledDir,
|
compiledDir,
|
||||||
@@ -2808,9 +2877,13 @@ async function executeRenderPipeline(input: {
|
|||||||
// streaming spawn fails (non-abort) the stage returns { success: false }
|
// streaming spawn fails (non-abort) the stage returns { success: false }
|
||||||
// and we fall back to the disk path below.
|
// and we fall back to the disk path below.
|
||||||
let streamingHandled = false;
|
let streamingHandled = false;
|
||||||
if (useStreamingEncode) {
|
if (capturePlan.kind === "sdr_streaming") {
|
||||||
const captureFrameStart = Date.now();
|
const captureFrameStart = Date.now();
|
||||||
const invokeStreaming = () => {
|
const invokeStreaming = () => {
|
||||||
|
if (capturePlan.kind !== "sdr_streaming") {
|
||||||
|
throw new Error(`Cannot invoke streaming stage with ${capturePlan.kind} plan`);
|
||||||
|
}
|
||||||
|
const streamingPlan = capturePlan;
|
||||||
resetCaptureAttemptProgress(job);
|
resetCaptureAttemptProgress(job);
|
||||||
return observeRenderStage(
|
return observeRenderStage(
|
||||||
observability,
|
observability,
|
||||||
@@ -2825,12 +2898,10 @@ async function executeRenderPipeline(input: {
|
|||||||
job,
|
job,
|
||||||
totalFrames,
|
totalFrames,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot: captureForceScreenshot,
|
plan: streamingPlan,
|
||||||
log,
|
log,
|
||||||
workerCount,
|
|
||||||
probeSession,
|
probeSession,
|
||||||
outputFormat,
|
outputFormat,
|
||||||
forceParallelStream: deParallelStreamForced || captureParallelStreamForced,
|
|
||||||
streamingEncoderOptions: {
|
streamingEncoderOptions: {
|
||||||
fps: job.config.fps,
|
fps: job.config.fps,
|
||||||
width,
|
width,
|
||||||
@@ -2909,74 +2980,48 @@ async function executeRenderPipeline(input: {
|
|||||||
? "drawElement self-verify failed; retrying with forceScreenshot"
|
? "drawElement self-verify failed; retrying with forceScreenshot"
|
||||||
: "capture failed on pinned worker count; retrying with forceScreenshot",
|
: "capture failed on pinned worker count; retrying with forceScreenshot",
|
||||||
);
|
);
|
||||||
captureForceScreenshot = true;
|
const failedRouting = capturePlan.routing.kind;
|
||||||
|
capturePlan = replanAfterFailure(
|
||||||
|
capturePlan,
|
||||||
|
isVerifyError
|
||||||
|
? { kind: "draw_element_verification" }
|
||||||
|
: { kind: "capture_failure", memoryExhaustion: isMemoryExhaustion },
|
||||||
|
);
|
||||||
|
syncCapturePlan();
|
||||||
updateCaptureObservability({
|
updateCaptureObservability({
|
||||||
forceScreenshot: true,
|
forceScreenshot: capturePlan.forceScreenshot,
|
||||||
deSelfVerifyFallback,
|
deSelfVerifyFallback,
|
||||||
deFallbackReason,
|
deFallbackReason,
|
||||||
deFallbackFailedDb,
|
deFallbackFailedDb,
|
||||||
deFallbackFrameIndex,
|
deFallbackFrameIndex,
|
||||||
deFallbackThresholdDb,
|
deFallbackThresholdDb,
|
||||||
|
workerCount: capturePlan.workerCount,
|
||||||
|
useStreamingEncode: capturePlan.kind === "sdr_streaming",
|
||||||
|
deWorkerInversion,
|
||||||
|
deParallelRouter,
|
||||||
});
|
});
|
||||||
probeSession = null;
|
probeSession = null;
|
||||||
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes
|
if (failedRouting === "worker_inversion") {
|
||||||
// useStreamingEncode, or shouldUseStreamingEncode would keep
|
|
||||||
// resolving to the parallel-streaming shape on the retry instead
|
|
||||||
// of the well-tested parallel-disk fallback.
|
|
||||||
if (deParallelRouter === "routed") deParallelStreamForced = false;
|
|
||||||
const inversionRetryPlan = resolveInversionRetryPlan({
|
|
||||||
deWorkerInversion,
|
|
||||||
preInversionWorkerCount: preRoutingWorkerCount,
|
|
||||||
cfg,
|
|
||||||
outputFormat,
|
|
||||||
durationSeconds: job.duration,
|
|
||||||
isMemoryExhaustion,
|
|
||||||
});
|
|
||||||
const parallelRouterRetryPlan = resolveParallelRouterRetryPlan({
|
|
||||||
deParallelRouter,
|
|
||||||
preRouterWorkerCount: preRoutingWorkerCount,
|
|
||||||
cfg,
|
|
||||||
outputFormat,
|
|
||||||
durationSeconds: job.duration,
|
|
||||||
isMemoryExhaustion,
|
|
||||||
});
|
|
||||||
if (inversionRetryPlan) {
|
|
||||||
// The inversion bet on drawElement and lost — re-render on the
|
// The inversion bet on drawElement and lost — re-render on the
|
||||||
// pre-inversion parallel screenshot path instead of single-worker
|
// pre-inversion parallel screenshot path instead of single-worker
|
||||||
// screenshot streaming (the slowest capture shape for this size).
|
// screenshot streaming (the slowest capture shape for this size).
|
||||||
// "reverted" (not cleared) so telemetry keeps the lost-inversion
|
// "reverted" (not cleared) so telemetry keeps the lost-inversion
|
||||||
// cohort distinguishable from renders that never inverted.
|
// cohort distinguishable from renders that never inverted.
|
||||||
deWorkerInversion = inversionRetryPlan.deWorkerInversion;
|
|
||||||
workerCount = inversionRetryPlan.workerCount;
|
|
||||||
useStreamingEncode = inversionRetryPlan.useStreamingEncode;
|
|
||||||
updateCaptureObservability({
|
|
||||||
workerCount,
|
|
||||||
useStreamingEncode,
|
|
||||||
deWorkerInversion,
|
|
||||||
});
|
|
||||||
log.info(
|
log.info(
|
||||||
`[Render] Reverting worker inversion for the retry: ${workerCount} workers, ` +
|
`[Render] Reverting worker inversion for the retry: ${capturePlan.workerCount} workers, ` +
|
||||||
`streaming=${useStreamingEncode}.`,
|
`plan=${capturePlan.kind}.`,
|
||||||
);
|
);
|
||||||
} else if (parallelRouterRetryPlan) {
|
} else if (failedRouting === "parallel_router") {
|
||||||
// The router's bet on verified parallel streaming lost — re-render
|
// The router's bet on verified parallel streaming lost — re-render
|
||||||
// on the ordinary (non-DE) parallel path at the pre-router worker
|
// on the ordinary (non-DE) parallel path at the pre-router worker
|
||||||
// count, same "reverted, not cleared" telemetry contract as the
|
// count, same "reverted, not cleared" telemetry contract as the
|
||||||
// inversion above.
|
// inversion above.
|
||||||
deParallelRouter = parallelRouterRetryPlan.deParallelRouter;
|
|
||||||
workerCount = parallelRouterRetryPlan.workerCount;
|
|
||||||
useStreamingEncode = parallelRouterRetryPlan.useStreamingEncode;
|
|
||||||
updateCaptureObservability({
|
|
||||||
workerCount,
|
|
||||||
useStreamingEncode,
|
|
||||||
deParallelRouter,
|
|
||||||
});
|
|
||||||
log.info(
|
log.info(
|
||||||
`[Render] Reverting parallel router for the retry: ${workerCount} workers, ` +
|
`[Render] Reverting parallel router for the retry: ${capturePlan.workerCount} workers, ` +
|
||||||
`streaming=${useStreamingEncode}.`,
|
`plan=${capturePlan.kind}.`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (useStreamingEncode) {
|
if (capturePlan.kind === "sdr_streaming") {
|
||||||
streamingRes = await invokeStreaming();
|
streamingRes = await invokeStreaming();
|
||||||
} else {
|
} else {
|
||||||
// Parallel retry goes through the disk path below.
|
// Parallel retry goes through the disk path below.
|
||||||
@@ -3005,7 +3050,10 @@ async function executeRenderPipeline(input: {
|
|||||||
perfStages.captureSetupMs = Math.max(0, perfStages.captureMs - captureFrameMs);
|
perfStages.captureSetupMs = Math.max(0, perfStages.captureMs - captureFrameMs);
|
||||||
perfStages.encodeMs = streamingRes.encodeMs; // Overlapped with capture
|
perfStages.encodeMs = streamingRes.encodeMs; // Overlapped with capture
|
||||||
} else {
|
} else {
|
||||||
useStreamingEncode = false;
|
if (capturePlan.kind === "sdr_streaming") {
|
||||||
|
capturePlan = replanAfterFailure(capturePlan, { kind: "streaming_unavailable" });
|
||||||
|
syncCapturePlan();
|
||||||
|
}
|
||||||
// The disk path has no drain-time self-verification — clamp
|
// The disk path has no drain-time self-verification — clamp
|
||||||
// default-on drawElement here exactly like the pre-capture clamp
|
// default-on drawElement here exactly like the pre-capture clamp
|
||||||
// (verified-path confinement). Skipped when screenshots are already
|
// (verified-path confinement). Skipped when screenshots are already
|
||||||
@@ -3013,7 +3061,7 @@ async function executeRenderPipeline(input: {
|
|||||||
// opt-in, mirroring the clamp above.
|
// opt-in, mirroring the clamp above.
|
||||||
if (
|
if (
|
||||||
cfg.useDrawElement &&
|
cfg.useDrawElement &&
|
||||||
!captureForceScreenshot &&
|
!capturePlan.forceScreenshot &&
|
||||||
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE !== "true"
|
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE !== "true"
|
||||||
) {
|
) {
|
||||||
cfg.useDrawElement = false;
|
cfg.useDrawElement = false;
|
||||||
@@ -3029,19 +3077,23 @@ async function executeRenderPipeline(input: {
|
|||||||
probeSession = null;
|
probeSession = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
updateCaptureObservability({ useStreamingEncode });
|
updateCaptureObservability({ useStreamingEncode: false });
|
||||||
observability.checkpoint("capture_streaming", "spawn failed; falling back to disk");
|
observability.checkpoint("capture_streaming", "spawn failed; falling back to disk");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!streamingHandled) {
|
if (!streamingHandled) {
|
||||||
|
if (capturePlan.kind !== "sdr_disk") {
|
||||||
|
throw new Error(`Disk capture requires sdr_disk plan; got ${capturePlan.kind}`);
|
||||||
|
}
|
||||||
|
const diskPlan = capturePlan;
|
||||||
// ── Disk-based capture (original flow) ────────────────────────────
|
// ── Disk-based capture (original flow) ────────────────────────────
|
||||||
resetCaptureAttemptProgress(job);
|
resetCaptureAttemptProgress(job);
|
||||||
const captureFrameStart = Date.now();
|
const captureFrameStart = Date.now();
|
||||||
const captureRes = await observeRenderStage(
|
const captureRes = await observeRenderStage(
|
||||||
observability,
|
observability,
|
||||||
"capture_disk",
|
"capture_disk",
|
||||||
captureStageObservationData({ needsAlpha }),
|
captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
|
||||||
() =>
|
() =>
|
||||||
runCaptureStage({
|
runCaptureStage({
|
||||||
fileServer: activeFileServer,
|
fileServer: activeFileServer,
|
||||||
@@ -3050,11 +3102,9 @@ async function executeRenderPipeline(input: {
|
|||||||
job,
|
job,
|
||||||
totalFrames,
|
totalFrames,
|
||||||
cfg,
|
cfg,
|
||||||
forceScreenshot: captureForceScreenshot,
|
plan: diskPlan,
|
||||||
log,
|
log,
|
||||||
workerCount,
|
|
||||||
probeSession,
|
probeSession,
|
||||||
needsAlpha,
|
|
||||||
captureAttempts,
|
captureAttempts,
|
||||||
dedupPerfs,
|
dedupPerfs,
|
||||||
buildCaptureOptions,
|
buildCaptureOptions,
|
||||||
|
|||||||
Reference in New Issue
Block a user