mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-04 16:42:27 +00:00
feat(producer): calibration-aware heartbeat + worker-death terminal-error contract
Field signals ts=1784019503 (heartbeat reports 0 frames during 64s browser calibration — reads as broken but is healthy) and ts=1784042064 (1292s Windows render hard-exited during video frame extraction with no final error string — silent worker crash). Add calibrating/capturing state to heartbeat labels; surface synthetic terminal error on unexpected worker exit when no explicit error was emitted. Stack: PR #6 of 9 (base via/overlay-count-lint). Signed-off-by: Via <vance@heygen.com>
This commit is contained in:
@@ -2,10 +2,12 @@ import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
calculateOptimalWorkers,
|
||||
distributeFrames,
|
||||
expectedFramesForTask,
|
||||
formatWorkerFailure,
|
||||
selectWorkerDiagnostics,
|
||||
shouldDisableBrowserPoolForParallelWorker,
|
||||
shouldVerifyWorkerGpu,
|
||||
synthesizeSilentWorkerExitError,
|
||||
resolveParallelDeVerifySamples,
|
||||
} from "./parallelCoordinator.js";
|
||||
import type { EngineConfig } from "../config.js";
|
||||
@@ -161,6 +163,91 @@ describe("worker failure diagnostics", () => {
|
||||
"Worker 1: Navigation timeout of 60000 ms exceeded; diagnostics: [FrameCapture:ERROR] page.goto failed mode=screenshot timeoutMs=60000",
|
||||
);
|
||||
});
|
||||
|
||||
// Field signal ts=1784042064: a Windows render hard-exited during video
|
||||
// frame extraction without emitting a terminal error string; the parent
|
||||
// treated the result as success because `error` was falsy. The synthesized
|
||||
// string surfaces the shortfall so downstream telemetry can classify the
|
||||
// failure instead of it disappearing silently.
|
||||
it("synthesizes a terminal error when a worker exits with no error string", () => {
|
||||
const message = formatWorkerFailure({
|
||||
workerId: 3,
|
||||
framesCaptured: 12,
|
||||
startFrame: 0,
|
||||
endFrame: 60,
|
||||
durationMs: 180_000,
|
||||
});
|
||||
expect(message).toContain("Worker 3: worker 3 exited without terminal error string");
|
||||
expect(message).toContain("framesCaptured=12");
|
||||
expect(message).toContain("expected=60");
|
||||
expect(message).toContain("range=[0, 60)");
|
||||
expect(message).toContain("ts=1784042064");
|
||||
expect(message).toContain("--workers=1");
|
||||
});
|
||||
|
||||
it("prefers an explicit error string over the synthesized silent-exit message", () => {
|
||||
const message = formatWorkerFailure({
|
||||
workerId: 3,
|
||||
framesCaptured: 12,
|
||||
startFrame: 0,
|
||||
endFrame: 60,
|
||||
durationMs: 180_000,
|
||||
error: "Target closed",
|
||||
});
|
||||
expect(message).toBe("Worker 3: Target closed");
|
||||
expect(message).not.toContain("ts=1784042064");
|
||||
expect(message).not.toContain("exited without terminal error string");
|
||||
});
|
||||
|
||||
it("treats an empty error string as a silent exit for the message contract", () => {
|
||||
const message = formatWorkerFailure({
|
||||
workerId: 3,
|
||||
framesCaptured: 12,
|
||||
startFrame: 0,
|
||||
endFrame: 60,
|
||||
durationMs: 180_000,
|
||||
error: "",
|
||||
});
|
||||
expect(message).toContain("exited without terminal error string");
|
||||
expect(message).toContain("ts=1784042064");
|
||||
});
|
||||
});
|
||||
|
||||
describe("expectedFramesForTask", () => {
|
||||
it("returns the range length for contiguous tasks", () => {
|
||||
expect(expectedFramesForTask({ startFrame: 0, endFrame: 30 })).toBe(30);
|
||||
expect(expectedFramesForTask({ startFrame: 10, endFrame: 25 })).toBe(15);
|
||||
});
|
||||
|
||||
it("divides by stride for interleaved tasks", () => {
|
||||
// Worker 0 in a 3-way interleave over 30 frames captures 0, 3, 6, ..., 27 → 10 frames.
|
||||
expect(expectedFramesForTask({ startFrame: 0, endFrame: 30, frameStride: 3 })).toBe(10);
|
||||
});
|
||||
|
||||
it("rounds up so the last-stride-remainder frame is expected", () => {
|
||||
// Worker 0 in a 3-way interleave over 31 frames captures 0, 3, ..., 30 → 11 frames.
|
||||
expect(expectedFramesForTask({ startFrame: 0, endFrame: 31, frameStride: 3 })).toBe(11);
|
||||
});
|
||||
|
||||
it("returns zero for empty ranges", () => {
|
||||
expect(expectedFramesForTask({ startFrame: 10, endFrame: 10 })).toBe(0);
|
||||
expect(expectedFramesForTask({ startFrame: 30, endFrame: 10 })).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("synthesizeSilentWorkerExitError", () => {
|
||||
it("names the field signal and reruns hint so operators can classify the failure", () => {
|
||||
const message = synthesizeSilentWorkerExitError(
|
||||
{ workerId: 7, framesCaptured: 40, startFrame: 60, endFrame: 120 },
|
||||
60,
|
||||
);
|
||||
expect(message).toContain("worker 7 exited without terminal error string");
|
||||
expect(message).toContain("framesCaptured=40");
|
||||
expect(message).toContain("expected=60");
|
||||
expect(message).toContain("range=[60, 120)");
|
||||
expect(message).toContain("Field signal ts=1784042064");
|
||||
expect(message).toContain("--workers=1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldVerifyWorkerGpu", () => {
|
||||
|
||||
@@ -147,8 +147,49 @@ function compactDiagnosticLine(line: string): string {
|
||||
return line.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Expected frame count for a worker task, honoring its stride. Contiguous
|
||||
* tasks (stride 1) expect `endFrame - startFrame`; interleaved tasks
|
||||
* (stride > 1) expect `ceil((endFrame - startFrame) / stride)`, matching
|
||||
* the loop shape in `captureFrameRange`.
|
||||
*/
|
||||
export function expectedFramesForTask(task: {
|
||||
startFrame: number;
|
||||
endFrame: number;
|
||||
frameStride?: number;
|
||||
}): number {
|
||||
const stride = task.frameStride ?? 1;
|
||||
return Math.max(0, Math.ceil((task.endFrame - task.startFrame) / stride));
|
||||
}
|
||||
|
||||
/**
|
||||
* Synthetic terminal-error message for a worker whose exit didn't produce
|
||||
* an explicit error string but under-captured its expected frame range.
|
||||
* Field signal ts=1784042064: a 1292s Windows render hard-exited during
|
||||
* capture with no final error string, leaving the operator with no
|
||||
* actionable trace. This message surfaces the shortfall + reruns hint
|
||||
* so downstream telemetry (and operators grepping logs) can classify the
|
||||
* failure instead of it disappearing silently.
|
||||
*/
|
||||
export function synthesizeSilentWorkerExitError(
|
||||
result: Pick<WorkerResult, "workerId" | "framesCaptured" | "startFrame" | "endFrame">,
|
||||
expectedFrames: number,
|
||||
): string {
|
||||
return (
|
||||
`worker ${result.workerId} exited without terminal error string ` +
|
||||
`(framesCaptured=${result.framesCaptured}, expected=${expectedFrames}, ` +
|
||||
`range=[${result.startFrame}, ${result.endFrame})). ` +
|
||||
`Field signal ts=1784042064 — this class of failure has been reported; ` +
|
||||
`consider re-run with --workers=1 to isolate.`
|
||||
);
|
||||
}
|
||||
|
||||
export function formatWorkerFailure(result: WorkerResult): string {
|
||||
const base = `Worker ${result.workerId}: ${result.error ?? "unknown error"}`;
|
||||
const errorText =
|
||||
result.error && result.error.length > 0
|
||||
? result.error
|
||||
: synthesizeSilentWorkerExitError(result, expectedFramesForTask(result));
|
||||
const base = `Worker ${result.workerId}: ${errorText}`;
|
||||
if (!result.diagnostics || result.diagnostics.length === 0) return base;
|
||||
|
||||
const diagnostics = result.diagnostics.map(compactDiagnosticLine).join(" | ");
|
||||
@@ -541,6 +582,17 @@ export async function executeParallelCapture(
|
||||
),
|
||||
);
|
||||
|
||||
// A worker may return without an error string yet with framesCaptured
|
||||
// below the task's expected count — that's the silent-exit shape field
|
||||
// signal ts=1784042064 called out. Synthesize a terminal error string
|
||||
// in-place so the filter below treats it as a failure (and so the
|
||||
// caller's failure message actually names what went wrong).
|
||||
for (const r of results) {
|
||||
if (!r.error && r.framesCaptured < expectedFramesForTask(r)) {
|
||||
r.error = synthesizeSilentWorkerExitError(r, expectedFramesForTask(r));
|
||||
}
|
||||
}
|
||||
|
||||
const errors = results.filter((r) => r.error);
|
||||
if (errors.length > 0) {
|
||||
const errorMessages = errors.map(formatWorkerFailure).join("; ");
|
||||
|
||||
Reference in New Issue
Block a user