mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-05 00:56:23 +00:00
fix(distributed): gate per-worker SwiftShader probe to worker 0 only (#956)
After #916 moved `assertSwiftShader` from `renderChunk()`'s eager probe session into `executeWorkerTask`, every parallel worker began running its own `chrome://gpu` / canvas-WebGL probe. At `chunkWorkerCount=6` (texture launch at chunks=3) that's 6 concurrent CDP page-loads per chunk × 3 chunks = 18 simultaneous probes. Bench data on dev (12 producer pods × 22 vCPU) showed c=3 worst-case wall-clock at 67.3s, 24.7s above c=6 worst (42.6s) — pod_total inflates 100s → 147s uniformly across all three chunks per slow iter, the signature of cluster-level CDP contention rather than within-pod contention. Workers within a chunk share the same Chrome binary, flags, and OS/driver state on a single pod, so worker 0's success is representative for the rest. Gate the probe via `shouldVerifyWorkerGpu(workerId, config)` so only worker 0 navigates to the probe page; workers 1..N-1 skip it. The fail-fast contract still holds at the chunk level (worker 0 still aborts the chunk if SwiftShader didn't load) — just without the concurrent CDP traffic. Expected wall-clock impact: c=3 worst drops from ~67s to in line with c=6 worst (~42-44s). c=6 (3 workers/pod) and c=8 (2 workers/pod) should see smaller wins; c=12 (1 worker/pod, sequential branch) is unaffected. Closes #955.
This commit is contained in:
@@ -1,5 +1,10 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { calculateOptimalWorkers, distributeFrames } from "./parallelCoordinator.js";
|
||||
import {
|
||||
calculateOptimalWorkers,
|
||||
distributeFrames,
|
||||
shouldVerifyWorkerGpu,
|
||||
} from "./parallelCoordinator.js";
|
||||
import type { EngineConfig } from "../config.js";
|
||||
|
||||
describe("distributeFrames", () => {
|
||||
it("distributes frames evenly across workers", () => {
|
||||
@@ -68,3 +73,29 @@ describe("calculateOptimalWorkers", () => {
|
||||
expect(workers).toBe(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldVerifyWorkerGpu", () => {
|
||||
const softwareConfig: Partial<EngineConfig> = { browserGpuMode: "software" };
|
||||
|
||||
it("returns true for worker 0 when GPU mode is software", () => {
|
||||
expect(shouldVerifyWorkerGpu(0, softwareConfig)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for non-zero workers when GPU mode is software", () => {
|
||||
expect(shouldVerifyWorkerGpu(1, softwareConfig)).toBe(false);
|
||||
expect(shouldVerifyWorkerGpu(5, softwareConfig)).toBe(false);
|
||||
expect(shouldVerifyWorkerGpu(17, softwareConfig)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for any worker when GPU mode is not software", () => {
|
||||
expect(shouldVerifyWorkerGpu(0, { browserGpuMode: "hardware" } as Partial<EngineConfig>)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(shouldVerifyWorkerGpu(0, {})).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when config is undefined", () => {
|
||||
expect(shouldVerifyWorkerGpu(0, undefined)).toBe(false);
|
||||
expect(shouldVerifyWorkerGpu(3, undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -181,6 +181,16 @@ export function distributeFrames(
|
||||
return tasks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a parallel worker should run the per-worker SwiftShader
|
||||
* assertion. Gated to worker 0 only: workers within a chunk share the same
|
||||
* Chrome binary, flags, and OS/driver state, so one verification per chunk
|
||||
* is sufficient. See `heygen-com/hyperframes#955`.
|
||||
*/
|
||||
export function shouldVerifyWorkerGpu(workerId: number, config?: Partial<EngineConfig>): boolean {
|
||||
return config?.browserGpuMode === "software" && workerId === 0;
|
||||
}
|
||||
|
||||
async function executeWorkerTask(
|
||||
task: WorkerTask,
|
||||
serverUrl: string,
|
||||
@@ -207,17 +217,22 @@ async function executeWorkerTask(
|
||||
createBeforeCaptureHook(),
|
||||
config,
|
||||
);
|
||||
// Per-worker SwiftShader assertion: when the caller declares
|
||||
// `browserGpuMode: "software"`, every worker session must verify Chrome's
|
||||
// WebGL backend is actually SwiftShader before the first frame. Hosts
|
||||
// that fall back to a hardware GL backend (or silently fail to load
|
||||
// Per-worker SwiftShader assertion, gated to worker 0 only.
|
||||
// When `browserGpuMode: "software"` is declared, the chunk's GL backend
|
||||
// must be verified as SwiftShader before the first frame — a host that
|
||||
// falls back to a hardware GL backend (or silently fails to load
|
||||
// SwiftShader) would otherwise produce non-deterministic pixels and
|
||||
// break the distributed byte-identical-retry contract — the parallel
|
||||
// branch wouldn't catch it via the pre-warmup probe (renderChunk now
|
||||
// skips that when chunkWorkerCount > 1). The canvas-based reader works
|
||||
// on both regular Chrome and chrome-headless-shell (which serves
|
||||
// `chrome://gpu` as an empty document).
|
||||
if (config?.browserGpuMode === "software") {
|
||||
// break the distributed byte-identical-retry contract. Running this
|
||||
// probe on every worker means N concurrent navigations to a WebGL
|
||||
// probe page per chunk; with `chunkWorkerCount=6` × 3 chunks, that's
|
||||
// 18 simultaneous CDP page-loads, which inflated c=3 worst-case wall
|
||||
// by ~24s vs c=6/c=8 on the texture-launch bench. Workers in the same
|
||||
// chunk share the same Chrome binary, flags, and OS/driver state, so
|
||||
// worker 0's success is representative — gate it there and skip the
|
||||
// rest. See `heygen-com/hyperframes#955` for the bench data and the
|
||||
// pre-warmup probe interaction (which `renderChunk` already skips
|
||||
// when `chunkWorkerCount > 1`).
|
||||
if (shouldVerifyWorkerGpu(task.workerId, config)) {
|
||||
await assertSwiftShader(session.page, readWebGlVendorInfoFromCanvas);
|
||||
}
|
||||
await initializeSession(session);
|
||||
|
||||
Reference in New Issue
Block a user