feat(producer): calibration-aware heartbeat + worker-death terminal-error contract

Field signals ts=1784019503 (heartbeat reports 0 frames during 64s
browser calibration — reads as broken but is healthy) and ts=1784042064
(1292s Windows render hard-exited during video frame extraction with
no final error string — silent worker crash).

Add calibrating/capturing state to heartbeat labels; surface synthetic
terminal error on unexpected worker exit when no explicit error was
emitted.

Stack: PR #6 of 9 (base via/overlay-count-lint).
Signed-off-by: Via <vance@heygen.com>
This commit is contained in:
Via
2026-07-15 23:14:49 +00:00
parent 75de927fb5
commit 971bcf39ae
5 changed files with 287 additions and 4 deletions
@@ -164,6 +164,12 @@ const ALLOWED_STRING_DATA_KEYS = new Set([
"renderJobId",
"requestedHdrMode",
"requestedWorkers",
// "calibrating" during pre-capture browser warm-up / calibration stages;
// "capturing" during capture_disk / capture_streaming / capture_hdr_*
// stages. Distinguishes healthy 0-frame heartbeats (browser starting up)
// from actual zero-frame stalls once capture is meant to be underway.
// Field signal ts=1784019503.
"stagePhase",
]);
const RESERVED_LOG_KEYS = new Set([
"data",
@@ -416,12 +422,40 @@ function heartbeatTargetMs(index: number): number {
return HEARTBEAT_RAMP_END_MS + overflow * HEARTBEAT_REPEAT_MS;
}
/**
* Options for `observeRenderStage` heartbeat behavior.
*
* The default heartbeat message is "stage still running", chosen for the
* capture stages where a live frame count in the observation data already
* communicates progress. Stages that run BEFORE any frame count is
* meaningful — the ~64s browser calibration path in particular — inherit
* that default and confusingly report "stage still running / framesCompleted:
* 0" to downstream consumers. Field signal ts=1784019503 captured exactly
* that read-as-broken shape on a healthy 64s calibration.
*
* `heartbeatMessage` lets the calibration call sites override the message
* to "browser calibrating" so operator-facing logs and downstream metrics
* can distinguish healthy pre-capture waits from actual zero-frame stalls
* mid-capture. The `data` payload also flows through (callers pass a
* `stagePhase: "calibrating" | "capturing"` field) so structured consumers
* don't have to string-match on the message.
*/
export interface ObserveRenderStageOptions {
/**
* Message to attach to each heartbeat checkpoint for this stage.
* Defaults to "stage still running".
*/
heartbeatMessage?: string;
}
export async function observeRenderStage<T>(
recorder: RenderObservabilityRecorder,
phase: string,
data: RenderObservationData | undefined,
fn: () => Promise<T>,
options: ObserveRenderStageOptions = {},
): Promise<T> {
const heartbeatMessage = options.heartbeatMessage ?? "stage still running";
const startedAt = recorder.stageStart(phase, data);
let heartbeatCount = 0;
let lastFiredAtMs = 0;
@@ -431,7 +465,7 @@ export async function observeRenderStage<T>(
heartbeatTimer = setTimeout(() => {
lastFiredAtMs = targetMs;
heartbeatCount += 1;
recorder.checkpoint(phase, "stage still running", {
recorder.checkpoint(phase, heartbeatMessage, {
...data,
heartbeatIndex: heartbeatCount,
stageElapsedMs: Date.now() - startedAt,