feat(producer): calibration-aware heartbeat + worker-death terminal-error contract

Field signals ts=1784019503 (heartbeat reports 0 frames during 64s
browser calibration — reads as broken but is healthy) and ts=1784042064
(1292s Windows render hard-exited during video frame extraction with
no final error string — silent worker crash).

Add calibrating/capturing state to heartbeat labels; surface synthetic
terminal error on unexpected worker exit when no explicit error was
emitted.

Stack: PR #6 of 9 (base via/overlay-count-lint).
Signed-off-by: Via <vance@heygen.com>
This commit is contained in:
Via
2026-07-15 23:14:49 +00:00
parent 75de927fb5
commit 971bcf39ae
5 changed files with 287 additions and 4 deletions
@@ -147,8 +147,49 @@ function compactDiagnosticLine(line: string): string {
return line.replace(/\s+/g, " ").trim();
}
/**
* Expected frame count for a worker task, honoring its stride. Contiguous
* tasks (stride 1) expect `endFrame - startFrame`; interleaved tasks
* (stride > 1) expect `ceil((endFrame - startFrame) / stride)`, matching
* the loop shape in `captureFrameRange`.
*/
export function expectedFramesForTask(task: {
startFrame: number;
endFrame: number;
frameStride?: number;
}): number {
const stride = task.frameStride ?? 1;
return Math.max(0, Math.ceil((task.endFrame - task.startFrame) / stride));
}
/**
* Synthetic terminal-error message for a worker whose exit didn't produce
* an explicit error string but under-captured its expected frame range.
* Field signal ts=1784042064: a 1292s Windows render hard-exited during
* capture with no final error string, leaving the operator with no
* actionable trace. This message surfaces the shortfall + reruns hint
* so downstream telemetry (and operators grepping logs) can classify the
* failure instead of it disappearing silently.
*/
export function synthesizeSilentWorkerExitError(
result: Pick<WorkerResult, "workerId" | "framesCaptured" | "startFrame" | "endFrame">,
expectedFrames: number,
): string {
return (
`worker ${result.workerId} exited without terminal error string ` +
`(framesCaptured=${result.framesCaptured}, expected=${expectedFrames}, ` +
`range=[${result.startFrame}, ${result.endFrame})). ` +
`Field signal ts=1784042064 — this class of failure has been reported; ` +
`consider re-run with --workers=1 to isolate.`
);
}
export function formatWorkerFailure(result: WorkerResult): string {
const base = `Worker ${result.workerId}: ${result.error ?? "unknown error"}`;
const errorText =
result.error && result.error.length > 0
? result.error
: synthesizeSilentWorkerExitError(result, expectedFramesForTask(result));
const base = `Worker ${result.workerId}: ${errorText}`;
if (!result.diagnostics || result.diagnostics.length === 0) return base;
const diagnostics = result.diagnostics.map(compactDiagnosticLine).join(" | ");
@@ -541,6 +582,17 @@ export async function executeParallelCapture(
),
);
// A worker may return without an error string yet with framesCaptured
// below the task's expected count — that's the silent-exit shape field
// signal ts=1784042064 called out. Synthesize a terminal error string
// in-place so the filter below treats it as a failure (and so the
// caller's failure message actually names what went wrong).
for (const r of results) {
if (!r.error && r.framesCaptured < expectedFramesForTask(r)) {
r.error = synthesizeSilentWorkerExitError(r, expectedFramesForTask(r));
}
}
const errors = results.filter((r) => r.error);
if (errors.length > 0) {
const errorMessages = errors.map(formatWorkerFailure).join("; ");