fix(distributed): gate per-worker SwiftShader probe to worker 0 only (#956)

After #916 moved `assertSwiftShader` from `renderChunk()`'s eager probe
session into `executeWorkerTask`, every parallel worker began running its
own `chrome://gpu` / canvas-WebGL probe. At `chunkWorkerCount=6` (texture
launch at chunks=3) that's 6 concurrent CDP page-loads per chunk × 3
chunks = 18 simultaneous probes. Bench data on dev (12 producer pods × 22
vCPU) showed c=3 worst-case wall-clock at 67.3s, 24.7s above c=6 worst
(42.6s) — pod_total inflates 100s → 147s uniformly across all three
chunks per slow iter, the signature of cluster-level CDP contention
rather than within-pod contention.

Workers within a chunk share the same Chrome binary, flags, and OS/driver
state on a single pod, so worker 0's success is representative for the
rest. Gate the probe via `shouldVerifyWorkerGpu(workerId, config)` so
only worker 0 navigates to the probe page; workers 1..N-1 skip it. The
fail-fast contract still holds at the chunk level (worker 0 still aborts
the chunk if SwiftShader didn't load) — just without the concurrent CDP
traffic.

Expected wall-clock impact: c=3 worst drops from ~67s to in line with
c=6 worst (~42-44s). c=6 (3 workers/pod) and c=8 (2 workers/pod) should
see smaller wins; c=12 (1 worker/pod, sequential branch) is unaffected.

Closes #955.
This commit is contained in:
James Russo
2026-05-18 22:30:13 -04:00
committed by GitHub
parent 7354d61371
commit 17f47f30dd
3 changed files with 63 additions and 15 deletions
@@ -1,5 +1,10 @@
import { describe, it, expect } from "vitest";
import { calculateOptimalWorkers, distributeFrames } from "./parallelCoordinator.js";
import {
calculateOptimalWorkers,
distributeFrames,
shouldVerifyWorkerGpu,
} from "./parallelCoordinator.js";
import type { EngineConfig } from "../config.js";
describe("distributeFrames", () => {
it("distributes frames evenly across workers", () => {
@@ -68,3 +73,29 @@ describe("calculateOptimalWorkers", () => {
expect(workers).toBe(4);
});
});
describe("shouldVerifyWorkerGpu", () => {
const softwareConfig: Partial<EngineConfig> = { browserGpuMode: "software" };
it("returns true for worker 0 when GPU mode is software", () => {
expect(shouldVerifyWorkerGpu(0, softwareConfig)).toBe(true);
});
it("returns false for non-zero workers when GPU mode is software", () => {
expect(shouldVerifyWorkerGpu(1, softwareConfig)).toBe(false);
expect(shouldVerifyWorkerGpu(5, softwareConfig)).toBe(false);
expect(shouldVerifyWorkerGpu(17, softwareConfig)).toBe(false);
});
it("returns false for any worker when GPU mode is not software", () => {
expect(shouldVerifyWorkerGpu(0, { browserGpuMode: "hardware" } as Partial<EngineConfig>)).toBe(
false,
);
expect(shouldVerifyWorkerGpu(0, {})).toBe(false);
});
it("returns false when config is undefined", () => {
expect(shouldVerifyWorkerGpu(0, undefined)).toBe(false);
expect(shouldVerifyWorkerGpu(3, undefined)).toBe(false);
});
});
@@ -181,6 +181,16 @@ export function distributeFrames(
return tasks;
}
/**
* Decide whether a parallel worker should run the per-worker SwiftShader
* assertion. Gated to worker 0 only: workers within a chunk share the same
* Chrome binary, flags, and OS/driver state, so one verification per chunk
* is sufficient. See `heygen-com/hyperframes#955`.
*/
export function shouldVerifyWorkerGpu(workerId: number, config?: Partial<EngineConfig>): boolean {
return config?.browserGpuMode === "software" && workerId === 0;
}
async function executeWorkerTask(
task: WorkerTask,
serverUrl: string,
@@ -207,17 +217,22 @@ async function executeWorkerTask(
createBeforeCaptureHook(),
config,
);
// Per-worker SwiftShader assertion: when the caller declares
// `browserGpuMode: "software"`, every worker session must verify Chrome's
// WebGL backend is actually SwiftShader before the first frame. Hosts
// that fall back to a hardware GL backend (or silently fail to load
// Per-worker SwiftShader assertion, gated to worker 0 only.
// When `browserGpuMode: "software"` is declared, the chunk's GL backend
// must be verified as SwiftShader before the first frame — a host that
// falls back to a hardware GL backend (or silently fails to load
// SwiftShader) would otherwise produce non-deterministic pixels and
// break the distributed byte-identical-retry contract — the parallel
// branch wouldn't catch it via the pre-warmup probe (renderChunk now
// skips that when chunkWorkerCount > 1). The canvas-based reader works
// on both regular Chrome and chrome-headless-shell (which serves
// `chrome://gpu` as an empty document).
if (config?.browserGpuMode === "software") {
// break the distributed byte-identical-retry contract. Running this
// probe on every worker means N concurrent navigations to a WebGL
// probe page per chunk; with `chunkWorkerCount=6` × 3 chunks, that's
// 18 simultaneous CDP page-loads, which inflated c=3 worst-case wall
// by ~24s vs c=6/c=8 on the texture-launch bench. Workers in the same
// chunk share the same Chrome binary, flags, and OS/driver state, so
// worker 0's success is representative — gate it there and skip the
// rest. See `heygen-com/hyperframes#955` for the bench data and the
// pre-warmup probe interaction (which `renderChunk` already skips
// when `chunkWorkerCount > 1`).
if (shouldVerifyWorkerGpu(task.workerId, config)) {
await assertSwiftShader(session.page, readWebGlVendorInfoFromCanvas);
}
await initializeSession(session);