feat(producer): calibration-aware heartbeat + worker-death terminal-error contract

Field signals ts=1784019503 (heartbeat reports 0 frames during 64s
browser calibration — reads as broken but is healthy) and ts=1784042064
(1292s Windows render hard-exited during video frame extraction with
no final error string — silent worker crash).

Add calibrating/capturing state to heartbeat labels; surface synthetic
terminal error on unexpected worker exit when no explicit error was
emitted.

Stack: PR #6 of 9 (base via/overlay-count-lint).
Signed-off-by: Via <vance@heygen.com>
This commit is contained in:
Via
2026-07-15 23:14:49 +00:00
parent 75de927fb5
commit 971bcf39ae
5 changed files with 287 additions and 4 deletions
@@ -2,10 +2,12 @@ import { describe, it, expect } from "vitest";
import {
calculateOptimalWorkers,
distributeFrames,
expectedFramesForTask,
formatWorkerFailure,
selectWorkerDiagnostics,
shouldDisableBrowserPoolForParallelWorker,
shouldVerifyWorkerGpu,
synthesizeSilentWorkerExitError,
resolveParallelDeVerifySamples,
} from "./parallelCoordinator.js";
import type { EngineConfig } from "../config.js";
@@ -161,6 +163,91 @@ describe("worker failure diagnostics", () => {
"Worker 1: Navigation timeout of 60000 ms exceeded; diagnostics: [FrameCapture:ERROR] page.goto failed mode=screenshot timeoutMs=60000",
);
});
// Field signal ts=1784042064: a Windows render hard-exited during video
// frame extraction without emitting a terminal error string; the parent
// treated the result as success because `error` was falsy. The synthesized
// string surfaces the shortfall so downstream telemetry can classify the
// failure instead of it disappearing silently.
it("synthesizes a terminal error when a worker exits with no error string", () => {
const message = formatWorkerFailure({
workerId: 3,
framesCaptured: 12,
startFrame: 0,
endFrame: 60,
durationMs: 180_000,
});
expect(message).toContain("Worker 3: worker 3 exited without terminal error string");
expect(message).toContain("framesCaptured=12");
expect(message).toContain("expected=60");
expect(message).toContain("range=[0, 60)");
expect(message).toContain("ts=1784042064");
expect(message).toContain("--workers=1");
});
it("prefers an explicit error string over the synthesized silent-exit message", () => {
const message = formatWorkerFailure({
workerId: 3,
framesCaptured: 12,
startFrame: 0,
endFrame: 60,
durationMs: 180_000,
error: "Target closed",
});
expect(message).toBe("Worker 3: Target closed");
expect(message).not.toContain("ts=1784042064");
expect(message).not.toContain("exited without terminal error string");
});
it("treats an empty error string as a silent exit for the message contract", () => {
const message = formatWorkerFailure({
workerId: 3,
framesCaptured: 12,
startFrame: 0,
endFrame: 60,
durationMs: 180_000,
error: "",
});
expect(message).toContain("exited without terminal error string");
expect(message).toContain("ts=1784042064");
});
});
describe("expectedFramesForTask", () => {
it("returns the range length for contiguous tasks", () => {
expect(expectedFramesForTask({ startFrame: 0, endFrame: 30 })).toBe(30);
expect(expectedFramesForTask({ startFrame: 10, endFrame: 25 })).toBe(15);
});
it("divides by stride for interleaved tasks", () => {
// Worker 0 in a 3-way interleave over 30 frames captures 0, 3, 6, ..., 27 → 10 frames.
expect(expectedFramesForTask({ startFrame: 0, endFrame: 30, frameStride: 3 })).toBe(10);
});
it("rounds up so the last-stride-remainder frame is expected", () => {
// Worker 0 in a 3-way interleave over 31 frames captures 0, 3, ..., 30 → 11 frames.
expect(expectedFramesForTask({ startFrame: 0, endFrame: 31, frameStride: 3 })).toBe(11);
});
it("returns zero for empty ranges", () => {
expect(expectedFramesForTask({ startFrame: 10, endFrame: 10 })).toBe(0);
expect(expectedFramesForTask({ startFrame: 30, endFrame: 10 })).toBe(0);
});
});
describe("synthesizeSilentWorkerExitError", () => {
it("names the field signal and reruns hint so operators can classify the failure", () => {
const message = synthesizeSilentWorkerExitError(
{ workerId: 7, framesCaptured: 40, startFrame: 60, endFrame: 120 },
60,
);
expect(message).toContain("worker 7 exited without terminal error string");
expect(message).toContain("framesCaptured=40");
expect(message).toContain("expected=60");
expect(message).toContain("range=[60, 120)");
expect(message).toContain("Field signal ts=1784042064");
expect(message).toContain("--workers=1");
});
});
describe("shouldVerifyWorkerGpu", () => {