Merge pull request #2095 from heygen-com/feat/de-parallel-router

feat(producer): default-off router for verified parallel drawElement
This commit is contained in:
Vance Ingalls
2026-07-09 01:03:53 -07:00
committed by GitHub
7 changed files with 393 additions and 24 deletions
+2
View File
@@ -1688,6 +1688,8 @@ function trackRenderMetrics(
deClampReason: perf?.drawElement?.clampReason,
deWorkerInversion: perf?.drawElement?.workerInversion,
dePreInversionWorkers: perf?.drawElement?.preInversionWorkers,
deParallelRouter: perf?.drawElement?.parallelRouter,
dePreRouterWorkers: perf?.drawElement?.preRouterWorkers,
deGateReason: perf?.drawElement?.gateReason,
deWorkerEncode: perf?.drawElement?.workerEncode,
deVerifyArmed: perf?.drawElement?.verifyArmed,
+4
View File
@@ -126,6 +126,8 @@ export function trackRenderComplete(
deClampReason?: string;
deWorkerInversion?: string;
dePreInversionWorkers?: number;
deParallelRouter?: string;
dePreRouterWorkers?: number;
deGateReason?: string;
deWorkerEncode?: boolean;
deVerifyArmed?: number;
@@ -204,6 +206,8 @@ export function trackRenderComplete(
de_clamp_reason: props.deClampReason,
de_worker_inversion: props.deWorkerInversion,
de_pre_inversion_workers: props.dePreInversionWorkers,
de_parallel_router: props.deParallelRouter,
de_pre_router_workers: props.dePreRouterWorkers,
de_gate_reason: props.deGateReason,
de_worker_encode: props.deWorkerEncode,
de_verify_armed: props.deVerifyArmed,
@@ -44,6 +44,8 @@ export interface RenderCaptureObservability {
deSelfVerifyFallback?: boolean;
/** Auto-parallel inversion outcome: "inverted" (fired, held) | "reverted" (fired, self-verify retry rolled back). */
deWorkerInversion?: "inverted" | "reverted";
/** DE parallel-router outcome: "routed" (fired, held) | "reverted" (fired, self-verify retry rolled back). */
deParallelRouter?: "routed" | "reverted";
protocolTimeoutMs?: number;
pageNavigationTimeoutMs?: number;
playerReadyTimeoutMs?: number;
@@ -67,6 +67,9 @@ export interface DrawElementPerfInput {
workerInversion?: "inverted" | "reverted";
/** Auto-resolved worker count before the inversion pinned it to 1 (set only when the inversion fired). */
preInversionWorkers?: number;
parallelRouter?: "routed" | "reverted";
/** Auto-resolved worker count before the router pinned it to 3 (set only when the router fired). */
preRouterWorkers?: number;
selfVerifyFallback: boolean;
fallbackReason?: string;
drainStats?: {
@@ -96,6 +99,8 @@ function aggregateDrawElement(
clampReason: de.clampReason,
workerInversion: de.workerInversion ?? "none",
preInversionWorkers: de.preInversionWorkers,
parallelRouter: de.parallelRouter ?? "none",
preRouterWorkers: de.preRouterWorkers,
gateReason: gateReasons.length > 0 ? gateReasons.join("|") : undefined,
workerEncode: perfs.some((p) => p.deWorkerEncode),
verifyArmed: perfs.reduce((sum, p) => sum + (p.deVerifyArmed ?? 0), 0),
@@ -113,6 +113,14 @@ export interface CaptureStreamingStageInput {
log: ProducerLogger;
workerCount: number;
probeSession: CaptureSession | null;
/**
* Per-render override from the DE parallel router — see
* deParallelStreamForced's declaration in renderOrchestrator.ts. Distinct
* from the `HF_DE_PARALLEL_STREAM` manual opt-in (still read directly by
* this stage) because the router's decision must not leak across
* concurrently-running renders sharing this process via a global env var.
*/
forceParallelStream?: boolean;
/** For the spawn-failure log message context only. */
outputFormat: string;
/** Pre-built encoder options; passed straight to `spawnStreamingEncoder`. */
@@ -456,6 +464,7 @@ export async function runCaptureStreamingStage(
assertNotAborted,
onProgress,
dedupPerfs,
forceParallelStream,
} = input;
let { workerCount, probeSession } = input;
let lastBrowserConsole: string[] = [];
@@ -507,13 +516,14 @@ export async function runCaptureStreamingStage(
if (workerCount > 1) {
// Parallel capture → streaming encode
// HF_DE_PARALLEL_STREAM (opt-in): interleaved distribution — worker i
// takes frames i, i+N, i+2N… so the ordered writer's reorder window is
// N frames and workers run in lockstep instead of serializing behind
// contiguous ranges (see distributeFramesInterleaved). Each worker runs
// the depth-2 pipelined drawElement produce when its session initialized
// in drawelement mode.
const deParallelStream = process.env.HF_DE_PARALLEL_STREAM === "true";
// HF_DE_PARALLEL_STREAM (manual opt-in) / forceParallelStream (router):
// interleaved distribution — worker i takes frames i, i+N, i+2N… so the
// ordered writer's reorder window is N frames and workers run in
// lockstep instead of serializing behind contiguous ranges (see
// distributeFramesInterleaved). Each worker runs the depth-2 pipelined
// drawElement produce when its session initialized in drawelement mode.
const deParallelStream =
forceParallelStream === true || process.env.HF_DE_PARALLEL_STREAM === "true";
const tasks = deParallelStream
? distributeFramesInterleaved(totalFrames, workerCount, workDir)
: distributeFrames(totalFrames, workerCount, workDir);
@@ -29,6 +29,8 @@ import {
resolveCaptureForceScreenshotForPageSideCompositing,
shouldDiscardProbeSessionForPageSideCompositing,
resolveInversionRetryPlan,
resolveParallelRouterRetryPlan,
shouldPreferParallelDrawElement,
shouldPreferSingleWorkerDrawElement,
shouldUseStreamingEncode,
} from "./renderOrchestrator.js";
@@ -402,6 +404,14 @@ describe("shouldUseStreamingEncode", () => {
expect(shouldUseStreamingEncode(streamingEnabledConfig, "mp4", 2, 240)).toBe(false);
});
it("forceParallelStream overrides the parallel-capture clamp for verified multi-worker streaming", () => {
expect(shouldUseStreamingEncode(streamingEnabledConfig, "mp4", 3, 240, true)).toBe(true);
expect(shouldUseStreamingEncode(streamingEnabledConfig, "mp4", 3, 240, false)).toBe(false);
expect(shouldUseStreamingEncode(streamingEnabledConfig, "png-sequence", 3, 240, true)).toBe(
false,
);
});
it("keeps renders over the configured max duration on normal encoding", () => {
expect(shouldUseStreamingEncode(streamingEnabledConfig, "mp4", 1, 240)).toBe(true);
expect(shouldUseStreamingEncode(streamingEnabledConfig, "mp4", 1, 240.001)).toBe(false);
@@ -1702,3 +1712,123 @@ describe("resolveInversionRetryPlan (self-verify retry rollback)", () => {
});
});
});
describe("shouldPreferParallelDrawElement (DE parallel router)", () => {
const eligible = {
workerCount: 5,
requestedWorkers: "auto" as const,
useDrawElement: true,
deCompileGate: undefined,
forceScreenshot: false,
outputFormat: "mp4" as const,
totalFrames: 2381,
minFrames: 2000,
layeredOrEffectRoute: false,
supersampling: false,
probeDeGated: false,
experimentalParallelDeOptIn: false,
routerEnabled: true,
};
it("routes an auto-resolved multi-worker render for an eligible long comp", () => {
expect(shouldPreferParallelDrawElement(eligible)).toBe(true);
});
it("is disabled by default (routerEnabled: false is the shipped default)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, routerEnabled: false })).toBe(false);
});
it("honors explicitly requested workers", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, requestedWorkers: 3 })).toBe(false);
});
it("routes for requestedWorkers undefined — the value production actually passes for auto", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, requestedWorkers: undefined })).toBe(
true,
);
});
it("skips comps routed to layered/HDR/shader paths (drawElement never runs there)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, layeredOrEffectRoute: true })).toBe(
false,
);
});
it("skips supersampled renders (engine init-time DE gate)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, supersampling: true })).toBe(false);
});
it("skips when the probe session already shows DE gated out", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, probeDeGated: true })).toBe(false);
});
it("honors the explicit experimental parallel-DE opt-in (already parallel, router is a no-op)", () => {
expect(
shouldPreferParallelDrawElement({ ...eligible, experimentalParallelDeOptIn: true }),
).toBe(false);
});
it("skips below the amortization threshold (benchmark: real-work comps clear 1.25x at ~2,000+ frames)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, totalFrames: 915 })).toBe(false);
expect(shouldPreferParallelDrawElement({ ...eligible, totalFrames: 2000 })).toBe(true);
});
it("is disabled by minFrames <= 0 (HF_DE_PARALLEL_MIN_FRAMES=0 kill switch)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, minFrames: 0 })).toBe(false);
expect(shouldPreferParallelDrawElement({ ...eligible, minFrames: -1 })).toBe(false);
});
it("requires drawElement to be enabled and ungated", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, useDrawElement: false })).toBe(false);
expect(shouldPreferParallelDrawElement({ ...eligible, deCompileGate: "3d" })).toBe(false);
expect(shouldPreferParallelDrawElement({ ...eligible, forceScreenshot: true })).toBe(false);
});
it("only applies to mp4 (the benchmarked configuration)", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, outputFormat: "webm" })).toBe(false);
});
it("is a no-op when workers already resolved to 1", () => {
expect(shouldPreferParallelDrawElement({ ...eligible, workerCount: 1 })).toBe(false);
});
});
describe("resolveParallelRouterRetryPlan (self-verify retry rollback)", () => {
const cfg = { enableStreamingEncode: true, streamingEncodeMaxDurationSeconds: 240 };
it("returns null when the render was never router-routed", () => {
expect(
resolveParallelRouterRetryPlan({
deParallelRouter: undefined,
preRouterWorkerCount: 5,
cfg,
outputFormat: "mp4",
durationSeconds: 80,
}),
).toBe(null);
expect(
resolveParallelRouterRetryPlan({
deParallelRouter: "reverted",
preRouterWorkerCount: 5,
cfg,
outputFormat: "mp4",
durationSeconds: 80,
}),
).toBe(null);
});
it("restores the pre-router worker count and routes multi-worker retries to disk", () => {
const plan = resolveParallelRouterRetryPlan({
deParallelRouter: "routed",
preRouterWorkerCount: 5,
cfg,
outputFormat: "mp4",
durationSeconds: 80,
});
expect(plan).toEqual({
workerCount: 5,
useStreamingEncode: false,
deParallelRouter: "reverted",
});
});
});
@@ -387,6 +387,10 @@ export interface RenderPerfSummary {
workerInversion?: string;
/** Worker count the auto-resolution chose BEFORE the inversion pinned it to 1 — the parallel counterfactual for speedup math. Only set when the inversion fired. */
preInversionWorkers?: number;
/** DE parallel-router outcome: "routed" (fired, held), "reverted" (fired, self-verify retry rolled back), "none". Mutually exclusive with workerInversion. */
parallelRouter?: string;
/** Worker count the auto-resolution chose BEFORE the router pinned it to 3 — the single-worker-inversion counterfactual. Only set when the router fired. */
preRouterWorkers?: number;
/** Engine init-time gate: swiftshader | css_effect:* | at_risk_timeline | 3d_init_failed | supersampling | render_mode_hint. */
gateReason?: string;
/** Worker-encode drain (the verified path) was active. */
@@ -957,19 +961,23 @@ export function shouldUseStreamingEncode(
workerCount: number,
// Composition timeline duration in seconds.
durationSeconds: number,
// Per-render override (set by the DE parallel router) — see
// deParallelStreamForced's declaration in executeRenderJob for why this is
// a parameter instead of an env-var read.
forceParallelStream = false,
): boolean {
if (!cfg.enableStreamingEncode) return false;
if (outputFormat === "png-sequence") return false;
if (outputFormat === "gif") return false;
if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) return false;
if (durationSeconds > cfg.streamingEncodeMaxDurationSeconds) return false;
// SPIKE (HF_DE_PARALLEL_STREAM): allow multi-worker streaming for the
// interleaved drawElement produce experiment. Contiguous-chunk parallel
// streaming stalls (worker k+1's first frame waits for ALL of worker k's),
// so this only makes sense with the interleaved distribution the capture
// stage selects under the same flag. Explicit opt-in, unverified — do not
// ship default-on without threading guardFrame through onFrameBuffer.
if (process.env.HF_DE_PARALLEL_STREAM === "true") return true;
// HF_DE_PARALLEL_STREAM (manual opt-in) / forceParallelStream (router):
// allow multi-worker streaming for the interleaved drawElement produce
// path. Contiguous-chunk parallel streaming stalls (worker k+1's first
// frame waits for ALL of worker k's), so this only makes sense with the
// interleaved distribution the capture stage selects under the same
// condition.
if (forceParallelStream || process.env.HF_DE_PARALLEL_STREAM === "true") return true;
return workerCount === 1;
}
@@ -1065,6 +1073,95 @@ export function resolveInversionRetryPlan(args: {
};
}
/**
* DE parallel-router predicate: should an AUTO-resolved multi-worker render
* use VERIFIED PARALLEL drawElement streaming (HF_DE_PARALLEL_STREAM) instead
* of the #2026 single-worker inversion?
*
* Benchmarked 2026-07-08 (clean, quiet-machine re-run): par3/single 1.161.36x
* on real-work comps 2,000 frames (2,381f GSAP graphics 1.36x, 3,245f rAF
* high-variance 1.29x, 915f crossover probe 1.27x); the one comp that didn't
* clear 1.25x (3,600f, 39% static/dedup-heavy) still didn't LOSE to single-
* worker (1.16x) dedup already skips the capture work parallelism would
* split, so there's mechanically less headroom, not a regression. No comp
* anywhere showed par3 < single. Default-off (HF_DE_PARALLEL_ROUTER): this
* promotes the opt-in mechanism from #2056 into the auto-routing decision,
* but the decision itself stays gated behind its own flag pending the
* telemetry soak (revert rate, de_verify_min_db distribution) on real wild
* traffic there is currently none, since nothing routes here by default.
* Takes priority over the single-worker inversion when both would fire (a
* higher minFrames than HF_DE_SINGLE_MIN_FRAMES is the intended shape: this
* only picks up the long tail the inversion's own benchmark didn't cover).
*/
export function shouldPreferParallelDrawElement(args: {
workerCount: number;
/** job.config.workers — a number means the user explicitly chose. */
requestedWorkers: number | "auto" | undefined;
useDrawElement: boolean;
deCompileGate: string | undefined;
forceScreenshot: boolean;
outputFormat: NonNullable<RenderConfig["format"]>;
totalFrames: number;
/** Amortization threshold; <=0 disables the router. */
minFrames: number;
layeredOrEffectRoute: boolean;
supersampling: boolean;
probeDeGated: boolean;
experimentalParallelDeOptIn: boolean;
/** HF_DE_PARALLEL_ROUTER === "true" — the router's own kill switch, default off. */
routerEnabled: boolean;
}): boolean {
return (
args.routerEnabled &&
args.workerCount > 1 &&
typeof args.requestedWorkers !== "number" &&
args.useDrawElement &&
!args.deCompileGate &&
!args.forceScreenshot &&
args.outputFormat === "mp4" &&
args.minFrames > 0 &&
args.totalFrames >= args.minFrames &&
!args.layeredOrEffectRoute &&
!args.supersampling &&
!args.probeDeGated &&
!args.experimentalParallelDeOptIn
);
}
/**
* Plan the self-verify retry for a router-routed render: the bet on verified
* parallel drawElement streaming lost, so the re-render falls back to the
* pre-router worker count on the ordinary (non-DE) parallel path. Unlike
* `resolveInversionRetryPlan`, the caller must also clear the router's
* `deParallelStreamForced` local BEFORE calling this `shouldUseStreamingEncode`
* takes it as a direct argument, so a stale `true` would keep resolving to
* the parallel-streaming shape on the retry instead of the well-tested
* parallel-disk fallback. Returns null when the render was not router-routed.
*/
export function resolveParallelRouterRetryPlan(args: {
deParallelRouter: "routed" | "reverted" | undefined;
preRouterWorkerCount: number;
cfg: Pick<EngineConfig, "enableStreamingEncode" | "streamingEncodeMaxDurationSeconds">;
outputFormat: NonNullable<RenderConfig["format"]>;
durationSeconds: number;
}): {
workerCount: number;
useStreamingEncode: boolean;
deParallelRouter: "reverted";
} | null {
if (args.deParallelRouter !== "routed") return null;
return {
workerCount: args.preRouterWorkerCount,
useStreamingEncode: shouldUseStreamingEncode(
args.cfg,
args.outputFormat,
args.preRouterWorkerCount,
args.durationSeconds,
),
deParallelRouter: "reverted",
};
}
export function resolveCaptureForceScreenshotForPageSideCompositing(args: {
forceScreenshot: boolean;
usePageSideCompositing: boolean;
@@ -1217,6 +1314,9 @@ export async function executeRenderJob(
// between declaration and the try-block (currently impossible, but
// defensible if more setup ever lands here) can't leak the interval.
let memSampler: MemorySampler | null = null;
// "routed" = the parallel router fired and held; "reverted" = fired but
// the self-verify retry rolled back; undefined = never fired.
let deParallelRouter: "routed" | "reverted" | undefined;
try {
memSampler = createMemorySampler();
@@ -1341,6 +1441,20 @@ export async function executeRenderJob(
// "inverted" = fired and held; "reverted" = fired but the self-verify
// retry rolled back to the parallel path; undefined = never fired.
let deWorkerInversion: "inverted" | "reverted" | undefined;
// deParallelRouter is mutually exclusive with deWorkerInversion — the
// router takes priority when both would be eligible (see
// shouldPreferParallelDrawElement).
//
// Per-render (not process-global) signal that the router wants parallel
// drawElement streaming. `HF_DE_PARALLEL_STREAM` env var stays as the
// manual opt-in for local testing (read directly by
// shouldUseStreamingEncode / the capture stage), but the router itself
// must NOT mutate process.env: the producer server runs concurrent
// renders in one process (PRODUCER_MAX_CONCURRENT_RENDERS), and a global
// flag set by one render's router decision would leak into an unrelated
// render already executing in the same process. Threading this as a
// local instead closes that cross-talk, not just the sequential leak.
let deParallelStreamForced = false;
let deSelfVerifyFallback = false;
let deFallbackReason: string | undefined;
let deDrainStats: import("./render/stages/captureStreamingStage.js").DeDrainStats | undefined;
@@ -1718,12 +1832,46 @@ export async function executeRenderJob(
// Verified parallel DE streaming (opt-in) wants its parallelism kept.
process.env.HF_DE_PARALLEL_STREAM === "true",
});
// DE parallel-router eligibility — see shouldPreferParallelDrawElement.
// Default-off (HF_DE_PARALLEL_ROUTER); HF_DE_PARALLEL_MIN_FRAMES defaults
// higher than the single-worker inversion's threshold since it targets
// the long tail the inversion's own benchmark didn't cover.
const deParallelRouterEnabled = process.env.HF_DE_PARALLEL_ROUTER === "true";
const deParallelMinFramesRaw = process.env.HF_DE_PARALLEL_MIN_FRAMES;
const deParallelMinFramesNum =
deParallelMinFramesRaw === undefined || deParallelMinFramesRaw.trim() === ""
? 2000
: Number(deParallelMinFramesRaw);
const deParallelMinFrames = Number.isFinite(deParallelMinFramesNum)
? deParallelMinFramesNum
: 2000;
const deParallelRouterEligible = shouldPreferParallelDrawElement({
workerCount: WOULD_RESOLVE_MULTI_WORKER,
requestedWorkers: job.config.workers,
useDrawElement: cfg.useDrawElement,
deCompileGate,
forceScreenshot: captureForceScreenshot,
outputFormat,
totalFrames,
minFrames: deParallelMinFrames,
layeredOrEffectRoute: hasHdrContent || compiled.hasShaderTransitions,
supersampling: deviceScaleFactor > 1,
probeDeGated:
probeSession !== null &&
probeSession.captureMode !== "drawelement" &&
!probeSession.deInitDeferred,
experimentalParallelDeOptIn:
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE === "true" ||
process.env.HF_DE_PARALLEL_STREAM === "true",
routerEnabled: deParallelRouterEnabled,
});
if (
job.config.workers === undefined &&
totalFrames >= 60 &&
!htmlInCanvasDetected &&
!cfg.lowMemoryMode &&
!deInversionEligible
!deInversionEligible &&
!deParallelRouterEligible
) {
const outcome = await observeRenderStage(
observability,
@@ -1763,6 +1911,7 @@ export async function executeRenderJob(
htmlInCanvasDetected,
lowMemoryMode: Boolean(cfg.lowMemoryMode),
deInversionEligible,
deParallelRouterEligible,
});
}
@@ -1782,10 +1931,36 @@ export async function executeRenderJob(
// INIT-time gate at capture (css-effects / at-risk, ~1.5% of local
// renders) render single-worker screenshot streaming — slower than
// parallel would have been, accepted for the routing win everywhere else.
// `preInversionWorkerCount` lets the self-verify retry return to the
// parallel path when the drawElement bet loses.
const preInversionWorkerCount = workerCount;
if (deInversionEligible && workerCount > 1) {
// `preRoutingWorkerCount` lets the self-verify retry return to the
// parallel path when the drawElement bet loses — shared by both the
// inversion and the router below, whichever fires (mutually exclusive).
const preRoutingWorkerCount = workerCount;
// Router takes priority over the single-worker inversion when both would
// fire — its higher frame threshold means this only ever picks up long-
// tail comps the inversion's own benchmark didn't cover (see
// shouldPreferParallelDrawElement). Pins to a fixed worker count exactly
// like the inversion pins to 1 — calibration is skipped for both (see
// the capture_calibration gate above), so this deliberately overrides
// whatever a calibrated resolution would have chosen (e.g. 2, on a
// resource-constrained host): the benchmark validated par3 specifically,
// not "whatever calibration picks above 1", and the self-verify retry is
// the safety net if 3 workers tips a given host over.
if (deParallelRouterEligible && workerCount > 1) {
deParallelRouter = "routed";
// Fixed at 3, not calibration-derived: the benchmark validated exactly
// this worker count (par3 beat par2 consistently; W4/W5 unmeasured for
// this path), same shape as the single-worker inversion pinning to a
// fixed 1 rather than a calibrated count.
const ROUTER_WORKER_COUNT = 3;
log.info(
"[Render] Fast capture: verified parallel drawElement streaming preferred over " +
`single-worker inversion (${totalFrames} frames >= ${deParallelMinFrames}; ` +
"benchmark-validated at 3 workers, pinned regardless of calibration). " +
"Set HF_DE_PARALLEL_ROUTER=false or --workers N to override.",
);
workerCount = ROUTER_WORKER_COUNT;
deParallelStreamForced = true;
} else if (deInversionEligible && workerCount > 1) {
deWorkerInversion = "inverted";
log.info(
"[Render] Fast capture: single-worker drawElement streaming preferred over " +
@@ -1795,10 +1970,11 @@ export async function executeRenderJob(
);
workerCount = 1;
}
updateCaptureObservability({ workerCount, deWorkerInversion });
updateCaptureObservability({ workerCount, deWorkerInversion, deParallelRouter });
observability.checkpoint("worker_resolution", "resolved", {
workerCount,
deWorkerInversion: deWorkerInversion ?? "none",
deParallelRouter: deParallelRouter ?? "none",
});
if (workerCount > 1 && probeSession) {
@@ -1812,7 +1988,13 @@ export async function executeRenderJob(
// let auto-parallel renders use disk frames: the current ordered streaming
// writer would otherwise stall later workers behind earlier frame ranges.
// png-sequence has no encoded video output, so streaming is always bypassed.
let useStreamingEncode = shouldUseStreamingEncode(cfg, outputFormat, workerCount, job.duration);
let useStreamingEncode = shouldUseStreamingEncode(
cfg,
outputFormat,
workerCount,
job.duration,
deParallelStreamForced,
);
log.info("streaming-encode gate", {
enabled: useStreamingEncode,
configFlag: cfg.enableStreamingEncode,
@@ -1831,7 +2013,9 @@ export async function executeRenderJob(
// drain guard), so the confinement rule is satisfied and the parallel
// clamp does not apply. The disk path stays clamped.
const deParallelStreamVerified =
process.env.HF_DE_PARALLEL_STREAM === "true" && useStreamingEncode && workerCount > 1;
(deParallelStreamForced || process.env.HF_DE_PARALLEL_STREAM === "true") &&
useStreamingEncode &&
workerCount > 1;
if (
cfg.useDrawElement &&
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE !== "true" &&
@@ -2052,6 +2236,7 @@ export async function executeRenderJob(
workerCount,
probeSession,
outputFormat,
forceParallelStream: deParallelStreamForced,
streamingEncoderOptions: {
fps: job.config.fps,
width,
@@ -2101,9 +2286,21 @@ export async function executeRenderJob(
deSelfVerifyFallback: true,
});
probeSession = null;
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes
// useStreamingEncode, or shouldUseStreamingEncode would keep
// resolving to the parallel-streaming shape on the retry instead
// of the well-tested parallel-disk fallback.
if (deParallelRouter === "routed") deParallelStreamForced = false;
const inversionRetryPlan = resolveInversionRetryPlan({
deWorkerInversion,
preInversionWorkerCount,
preInversionWorkerCount: preRoutingWorkerCount,
cfg,
outputFormat,
durationSeconds: job.duration,
});
const parallelRouterRetryPlan = resolveParallelRouterRetryPlan({
deParallelRouter,
preRouterWorkerCount: preRoutingWorkerCount,
cfg,
outputFormat,
durationSeconds: job.duration,
@@ -2126,6 +2323,23 @@ export async function executeRenderJob(
`[Render] Reverting worker inversion for the retry: ${workerCount} workers, ` +
`streaming=${useStreamingEncode}.`,
);
} else if (parallelRouterRetryPlan) {
// The router's bet on verified parallel streaming lost — re-render
// on the ordinary (non-DE) parallel path at the pre-router worker
// count, same "reverted, not cleared" telemetry contract as the
// inversion above.
deParallelRouter = parallelRouterRetryPlan.deParallelRouter;
workerCount = parallelRouterRetryPlan.workerCount;
useStreamingEncode = parallelRouterRetryPlan.useStreamingEncode;
updateCaptureObservability({
workerCount,
useStreamingEncode,
deParallelRouter,
});
log.info(
`[Render] Reverting parallel router for the retry: ${workerCount} workers, ` +
`streaming=${useStreamingEncode}.`,
);
}
if (useStreamingEncode) {
streamingRes = await invokeStreaming();
@@ -2337,7 +2551,9 @@ export async function executeRenderJob(
compileGate: deCompileGate,
clampReason: deClampReason,
workerInversion: deWorkerInversion,
preInversionWorkers: deWorkerInversion ? preInversionWorkerCount : undefined,
preInversionWorkers: deWorkerInversion ? preRoutingWorkerCount : undefined,
parallelRouter: deParallelRouter,
preRouterWorkers: deParallelRouter ? preRoutingWorkerCount : undefined,
selfVerifyFallback: deSelfVerifyFallback,
fallbackReason: deFallbackReason,
drainStats: deDrainStats,