mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-10 22:20:14 +00:00
feat(producer): calibration-aware heartbeat + worker-death terminal-error contract
Field signals ts=1784019503 (heartbeat reports 0 frames during 64s browser calibration — reads as broken but is healthy) and ts=1784042064 (1292s Windows render hard-exited during video frame extraction with no final error string — silent worker crash). Add calibrating/capturing state to heartbeat labels; surface synthetic terminal error on unexpected worker exit when no explicit error was emitted. Stack: PR #6 of 9 (base via/overlay-count-lint). Signed-off-by: Via <vance@heygen.com>
This commit is contained in:
@@ -164,6 +164,12 @@ const ALLOWED_STRING_DATA_KEYS = new Set([
|
||||
"renderJobId",
|
||||
"requestedHdrMode",
|
||||
"requestedWorkers",
|
||||
// "calibrating" during pre-capture browser warm-up / calibration stages;
|
||||
// "capturing" during capture_disk / capture_streaming / capture_hdr_*
|
||||
// stages. Distinguishes healthy 0-frame heartbeats (browser starting up)
|
||||
// from actual zero-frame stalls once capture is meant to be underway.
|
||||
// Field signal ts=1784019503.
|
||||
"stagePhase",
|
||||
]);
|
||||
const RESERVED_LOG_KEYS = new Set([
|
||||
"data",
|
||||
@@ -416,12 +422,40 @@ function heartbeatTargetMs(index: number): number {
|
||||
return HEARTBEAT_RAMP_END_MS + overflow * HEARTBEAT_REPEAT_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for `observeRenderStage` heartbeat behavior.
|
||||
*
|
||||
* The default heartbeat message is "stage still running", chosen for the
|
||||
* capture stages where a live frame count in the observation data already
|
||||
* communicates progress. Stages that run BEFORE any frame count is
|
||||
* meaningful — the ~64s browser calibration path in particular — inherit
|
||||
* that default and confusingly report "stage still running / framesCompleted:
|
||||
* 0" to downstream consumers. Field signal ts=1784019503 captured exactly
|
||||
* that read-as-broken shape on a healthy 64s calibration.
|
||||
*
|
||||
* `heartbeatMessage` lets the calibration call sites override the message
|
||||
* to "browser calibrating" so operator-facing logs and downstream metrics
|
||||
* can distinguish healthy pre-capture waits from actual zero-frame stalls
|
||||
* mid-capture. The `data` payload also flows through (callers pass a
|
||||
* `stagePhase: "calibrating" | "capturing"` field) so structured consumers
|
||||
* don't have to string-match on the message.
|
||||
*/
|
||||
export interface ObserveRenderStageOptions {
|
||||
/**
|
||||
* Message to attach to each heartbeat checkpoint for this stage.
|
||||
* Defaults to "stage still running".
|
||||
*/
|
||||
heartbeatMessage?: string;
|
||||
}
|
||||
|
||||
export async function observeRenderStage<T>(
|
||||
recorder: RenderObservabilityRecorder,
|
||||
phase: string,
|
||||
data: RenderObservationData | undefined,
|
||||
fn: () => Promise<T>,
|
||||
options: ObserveRenderStageOptions = {},
|
||||
): Promise<T> {
|
||||
const heartbeatMessage = options.heartbeatMessage ?? "stage still running";
|
||||
const startedAt = recorder.stageStart(phase, data);
|
||||
let heartbeatCount = 0;
|
||||
let lastFiredAtMs = 0;
|
||||
@@ -431,7 +465,7 @@ export async function observeRenderStage<T>(
|
||||
heartbeatTimer = setTimeout(() => {
|
||||
lastFiredAtMs = targetMs;
|
||||
heartbeatCount += 1;
|
||||
recorder.checkpoint(phase, "stage still running", {
|
||||
recorder.checkpoint(phase, heartbeatMessage, {
|
||||
...data,
|
||||
heartbeatIndex: heartbeatCount,
|
||||
stageElapsedMs: Date.now() - startedAt,
|
||||
|
||||
Reference in New Issue
Block a user