fix(render): gate pre-frame retry on loopback connection loss

Address review findings on the streaming interruption recovery.

Classifier (engine): walk the error cause chain (depth 5, including
AggregateError members) so undici's "fetch failed" wrapper and a
dual-stack localhost AggregateError classify by the errno they carry.
Recognise the loopback spellings Node, Bun and Chrome emit: unbracketed
"::1", "[::1]", "127.0.0.1" and "localhost" with ETIMEDOUT, ECONNREFUSED
or ECONNRESET, and Chrome's ERR_TIMED_OUT, ERR_CONNECTION_TIMED_OUT,
ERR_CONNECTION_REFUSED, ERR_CONNECTION_RESET, ERR_CONNECTION_CLOSED and
ERR_EMPTY_RESPONSE. Normalise the IPv6 host to "::1". Export
isLoopbackConnectionLoss, LoopbackConnectionLoss and
CaptureEndpointDiagnostic.

Endpoint provenance: CaptureStageError now forwards the classified
endpoint, so the orchestrator's re-classification (which early-returns
the stage error) sees the real host and port instead of always logging
an unknown owner.

Retry gate: replace `transient_browser && workerCount === 1` with
resolvePreFrameLoopbackLoss in the new render/preFrameRecovery module.
Only a failure that names a loopback endpoint, on exactly one worker,
with zero frames written, qualifies. Target closed, Page crashed, a
browser that cannot launch or a runtime that never becomes ready fail
immediately as before the widening; a SIGTERM-killed Chrome cannot hide
behind same-host retry work. recoverPreFrameFileServer owns the health
log and the restart-on-unhealthy decision so both are unit-testable.

File server: one createRenderFileServer factory for probe discovery,
frame capture and the pre-frame restart. A rejected body.cancel() no
longer marks a healthy server unhealthy; the probe error carries the
errno (ECONNREFUSED on Node, ConnectionRefused on Bun).

Audio: share canRetryMixLocally and resolveAutomationRerun between the
grouped and ungrouped mixes so retryable failures (external
interruption, managed deadline/inactivity, missing FFmpeg) never trigger
a compatibility or automation-degradation rerun, an interruption landing
on the rerun is reported instead of discarded, and a cancelled group
submix uses the canonical cancelled/user classification.

Tests pin each guard: dropping endpoint forwarding, deleting the restart
block, workerCount >= 1, dropping the zero-frame gate, forcing the
identity-header check true, removing the bare ::1 pattern, removing the
cause walk, reverting the ungrouped audio guard and discarding the rerun
interruption each turn a test red.

Co-Authored-By: Miguel Ángel <miguel.sierra@heygen.com>
This commit is contained in:
miga-heygen
2026-09-14 17:51:28 +00:00
co-authored by Miguel Ángel
parent 1c20a52f78
commit 6fcf2308b6
14 changed files with 976 additions and 162 deletions
+3
View File
@@ -133,8 +133,11 @@ export {
CaptureFailure,
classifyCaptureFailure,
isFatalCaptureFailure,
isLoopbackConnectionLoss,
type CaptureEndpointDiagnostic,
type CaptureFailureKind,
type CaptureWorkerDiagnostic,
type LoopbackConnectionLoss,
} from "./services/captureFailure.js";
// ── Screenshot (BeginFrame) ─────────────────────────────────────────────────────
@@ -933,6 +933,212 @@ describe("processCompositionAudio", () => {
expect(existsSync(workDir)).toBe(false);
});
it("reports a cancelled group sub-mix with the canonical cancellation classification", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice-a.wav"), "stub");
writeFileSync(join(baseDir, "voice-b.wav"), "stub");
const controller = new AbortController();
runFfmpegMock.mockImplementation(async (args: string[]) => {
if (String(args.at(-1)).includes("group-voiceover")) {
// The user cancels while the sub-mix is running.
controller.abort();
return {
success: false,
durationMs: 1,
stderr: "",
exitCode: null,
terminationReason: "abort" as const,
};
}
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
});
const result = await processCompositionAudio(
[
{
id: "voice-a",
src: "voice-a.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
groupId: "voiceover",
type: "audio",
},
{
id: "voice-b",
src: "voice-b.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 1,
volume: 1,
groupId: "voiceover",
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
controller.signal,
);
expect(result.success).toBe(false);
expect(result.failures).toContainEqual(
expect.objectContaining({
stage: "cancelled",
reason: "cancelled",
owner: "user",
retryable: false,
elementId: "voiceover",
}),
);
expect(result.failures.map((failure) => failure.reason)).not.toContain("ffmpeg_failed");
expect(result.failures.map((failure) => failure.owner)).not.toContain("system");
});
it("surfaces an external interruption that lands on the group automation rerun", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice-a.wav"), "stub");
writeFileSync(join(baseDir, "voice-b.wav"), "stub");
let groupMixCalls = 0;
runFfmpegMock.mockImplementation(async (args: string[]) => {
if (String(args.at(-1)).includes("group-voiceover")) {
groupMixCalls += 1;
// First run: the automation expression is rejected (a local fallback
// is allowed). Second run, without automation: SIGTERM arrives.
return groupMixCalls === 1
? {
success: false,
durationMs: 1,
stderr: "Error initializing filters",
exitCode: 234,
terminationReason: "exit" as const,
}
: {
success: false,
durationMs: 1,
stderr: "ffmpeg exited after signal 15",
exitCode: null,
terminationReason: "signal" as const,
failureReason: "external_interruption" as const,
};
}
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
});
const result = await processCompositionAudio(
[
{
id: "voice-a",
src: "voice-a.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
volumeKeyframes: [
{ time: 0, volume: 1 },
{ time: 2, volume: 0.5 },
],
groupId: "voiceover",
type: "audio",
},
{
id: "voice-b",
src: "voice-b.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 1,
volume: 1,
groupId: "voiceover",
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(groupMixCalls).toBe(2);
expect(result.success).toBe(false);
// The interruption that ended the rerun is what the caller retries on —
// not the automation error the rerun was trying to work around.
expect(result.failures).toEqual([
expect.objectContaining({
stage: "mix",
reason: "external_interruption",
owner: "system",
retryable: true,
elementId: "voiceover",
}),
]);
});
it("does not rerun an ungrouped automated mix after a managed deadline", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "bgm.wav"), "stub");
// prepare = call 0, automated mix = call 1 (hits its deadline). A
// static-volume rerun would spend a second full ffmpegProcessTimeout on a
// failure the caller is going to retry anyway.
runFfmpegMock
.mockImplementationOnce(async () => ({
success: true,
durationMs: 1,
stderr: "",
exitCode: 0,
}))
.mockImplementationOnce(async () => ({
success: false,
durationMs: 300_000,
stderr: "managed deadline reached",
exitCode: null,
terminationReason: "deadline" as const,
}));
const result = await processCompositionAudio(
[
{
id: "bgm",
src: "bgm.wav",
start: 0,
end: 5,
mediaStart: 0,
layer: 0,
volume: 0.8,
volumeKeyframes: [
{ time: 0, volume: 0.8 },
{ time: 5, volume: 0 },
],
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
5,
);
expect(result.success).toBe(false);
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
expect(result.failures).toEqual([
expect.objectContaining({ stage: "mix", reason: "ffmpeg_timeout", retryable: true }),
]);
});
it("preserves a managed deadline from a group sub-mix without degradation retries", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+57 -38
View File
@@ -373,6 +373,32 @@ function downloadFailure(error: unknown, elementId: string): AudioProcessingFail
};
}
/**
* Whether a failed mix may try a local fallback (drop volume automation, drop
* `normalize=0`). A retryable failure — external interruption, managed
* deadline/inactivity, missing FFmpeg — is the caller's to retry; rerunning it
* here would burn a second full timeout and report the fallback's failure in
* place of the first one. Cancellation is never retried locally.
*/
function canRetryMixLocally(result: RunFfmpegResult, signal: AbortSignal | undefined): boolean {
return !result.success && !signal?.aborted && !ffmpegFailure("mix", result).retryable;
}
/**
* Which of a failed mix and its automation-dropping rerun the caller reports.
* A successful rerun degrades the automation away; a rerun that was itself
* interrupted or hit its deadline is the failure the caller must see (and
* retry), not the automation error it was trying to work around.
*/
function resolveAutomationRerun(
original: RunFfmpegResult,
rerun: RunFfmpegResult,
): { result: RunFfmpegResult; degradedAutomation: boolean } {
if (rerun.success) return { result: rerun, degradedAutomation: true };
const keepRerun = ffmpegFailure("mix", rerun).retryable;
return { result: keepRerun ? rerun : original, degradedAutomation: false };
}
function ffmpegFailure(
stage: Extract<AudioFailureStage, "extract" | "prepare" | "mix" | "silence">,
result: RunFfmpegResult,
@@ -865,17 +891,10 @@ async function mixAudioTracks(
// dropped from the output entirely — a missing fade beats missing audio.
let degradedAutomation = false;
const hasAutomation = tracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
if (
!result.success &&
result.failureReason !== "external_interruption" &&
!signal?.aborted &&
hasAutomation
) {
const retry = await runMix(true);
if (retry.success) {
result = retry;
degradedAutomation = true;
}
if (canRetryMixLocally(result, signal) && hasAutomation) {
const rerun = resolveAutomationRerun(result, await runMix(true));
result = rerun.result;
degradedAutomation = rerun.degradedAutomation;
}
if (signal?.aborted) {
@@ -955,12 +974,10 @@ async function mixGroupMembers(
totalDuration: number,
signal?: AbortSignal,
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
): Promise<{
success: boolean;
error?: string;
degradedAutomation?: boolean;
failure?: AudioProcessingFailure;
}> {
): Promise<
| { success: true; degradedAutomation: boolean }
| { success: false; error: string; failure: AudioProcessingFailure }
> {
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
@@ -1039,9 +1056,7 @@ async function mixGroupMembers(
let useNormalize = true;
let result = await runOnce(useNormalize);
const canRetrySubmix = () =>
!result.success && !signal?.aborted && !ffmpegFailure("mix", result).retryable;
if (canRetrySubmix() && groupNormalizeOptionUnsupported(result.stderr)) {
if (canRetryMixLocally(result, signal) && groupNormalizeOptionUnsupported(result.stderr)) {
useNormalize = false;
result = await runOnce(useNormalize);
}
@@ -1053,15 +1068,28 @@ async function mixGroupMembers(
// grouped, it took the entire composition's audio down with it.
let degradedAutomation = false;
const hasAutomation = memberTracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
if (canRetrySubmix() && hasAutomation) {
const retry = await runOnce(useNormalize, true);
if (retry.success) {
result = retry;
degradedAutomation = true;
}
if (canRetryMixLocally(result, signal) && hasAutomation) {
const rerun = resolveAutomationRerun(result, await runOnce(useNormalize, true));
result = rerun.result;
degradedAutomation = rerun.degradedAutomation;
}
if (signal?.aborted) return { success: false, error: "Group sub-mix cancelled" };
if (signal?.aborted) {
// Same canonical shape as every other cancelled audio stage: a user
// cancel is not a system FFmpeg fault.
const cancelledDetail = "Group sub-mix cancelled";
return {
success: false,
error: cancelledDetail,
failure: {
stage: "cancelled",
reason: "cancelled",
owner: "user",
retryable: false,
detail: cancelledDetail,
},
};
}
if (!result.success)
return {
success: false,
@@ -1446,19 +1474,10 @@ export async function processCompositionAudio(
config,
);
if (!subMix.success) {
const failure = subMix.failure ?? {
stage: "mix",
reason: "ffmpeg_failed",
owner: "system",
retryable: false,
detail: subMix.error ?? "unknown",
};
failures.push({
...failure,
...subMix.failure,
elementId: groupId,
detail: boundedDetail(
`Group sub-mix failed for group ${groupId}: ${subMix.error ?? "unknown"}`,
),
detail: boundedDetail(`Group sub-mix failed for group ${groupId}: ${subMix.error}`),
});
continue;
}
@@ -1,11 +1,24 @@
import { describe, expect, it } from "vitest";
import { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure } from "./captureFailure.js";
import {
CaptureFailure,
classifyCaptureFailure,
isFatalCaptureFailure,
isLoopbackConnectionLoss,
} from "./captureFailure.js";
describe("classifyCaptureFailure", () => {
it.each([
["Target closed", "transient_browser"],
["connect ETIMEDOUT 127.0.0.1:49152", "transient_browser"],
["connect ETIMEDOUT ::1:49152", "transient_browser"],
["connect ETIMEDOUT [::1]:49152", "transient_browser"],
["connect ECONNREFUSED ::1:49152", "transient_browser"],
["net::ERR_TIMED_OUT at http://localhost:49152/index.html", "transient_browser"],
["net::ERR_CONNECTION_TIMED_OUT at http://localhost:49152/index.html", "transient_browser"],
["net::ERR_CONNECTION_REFUSED at http://localhost:49152/index.html", "transient_browser"],
["net::ERR_CONNECTION_REFUSED at http://[::1]:49152/index.html", "transient_browser"],
["net::ERR_CONNECTION_CLOSED at http://localhost:49152/index.html", "transient_browser"],
["net::ERR_EMPTY_RESPONSE at http://127.0.0.1:49152/index.html", "transient_browser"],
["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"],
["Runtime.evaluate timed out", "protocol_timeout"],
["Network.enable timed out. Increase the protocolTimeout setting.", "protocol_timeout"],
@@ -72,14 +85,95 @@ describe("classifyCaptureFailure", () => {
expect(Object.isFrozen(failure.workerDiagnostics[0]?.lines)).toBe(true);
});
it("retains loopback endpoint provenance for capture diagnostics", () => {
expect(classifyCaptureFailure(new Error("connect ETIMEDOUT 127.0.0.1:49152"))).toMatchObject({
kind: "transient_browser",
endpoint: { host: "127.0.0.1", port: 49152 },
it.each([
["connect ETIMEDOUT 127.0.0.1:49152", { host: "127.0.0.1", port: 49152 }],
["connect ETIMEDOUT ::1:49152", { host: "::1", port: 49152 }],
["connect ETIMEDOUT [::1]:49152", { host: "::1", port: 49152 }],
["connect ECONNREFUSED 127.0.0.1:49152", { host: "127.0.0.1", port: 49152 }],
["net::ERR_TIMED_OUT at http://localhost:4173/index.html", { host: "localhost", port: 4173 }],
[
"net::ERR_CONNECTION_TIMED_OUT at http://localhost:4173/index.html",
{ host: "localhost", port: 4173 },
],
[
"net::ERR_CONNECTION_REFUSED at http://127.0.0.1:4173/index.html",
{ host: "127.0.0.1", port: 4173 },
],
["net::ERR_CONNECTION_REFUSED at http://[::1]:4173/", { host: "::1", port: 4173 }],
// Chrome's spelling when the file server destroys live sockets mid-request.
[
"net::ERR_CONNECTION_CLOSED at http://localhost:4173/index.html",
{ host: "localhost", port: 4173 },
],
[
"net::ERR_EMPTY_RESPONSE at http://localhost:4173/index.html",
{ host: "localhost", port: 4173 },
],
] as const)("retains loopback endpoint provenance for %s", (message, endpoint) => {
const failure = classifyCaptureFailure(new Error(message));
expect(failure).toMatchObject({ kind: "transient_browser", endpoint });
expect(isLoopbackConnectionLoss(failure)).toBe(true);
});
it("reads the loopback endpoint through a Node fetch wrapper's cause chain", () => {
const wrapped = new TypeError("fetch failed", {
cause: Object.assign(new Error("connect ETIMEDOUT ::1:49152"), {
code: "ETIMEDOUT",
address: "::1",
port: 49152,
}),
});
const failure = classifyCaptureFailure(wrapped);
expect(failure.kind).toBe("transient_browser");
expect(failure.endpoint).toEqual({ host: "::1", port: 49152 });
expect(failure.message).toBe("fetch failed");
expect(failure.cause).toBe(wrapped);
});
it("reads the loopback endpoint out of a dual-stack AggregateError with an empty message", () => {
const aggregate = new AggregateError(
[
new Error("connect ECONNREFUSED ::1:49152"),
new Error("connect ECONNREFUSED 127.0.0.1:49152"),
],
"",
);
const wrapped = new TypeError("fetch failed", { cause: aggregate });
expect(classifyCaptureFailure(wrapped)).toMatchObject({
kind: "transient_browser",
endpoint: { host: "::1", port: 49152 },
});
});
it("stops walking a cause chain deeper than five levels", () => {
let error: Error = new Error("connect ETIMEDOUT ::1:49152");
for (let depth = 0; depth < 5; depth++) error = new Error(`wrapper ${depth}`, { cause: error });
expect(classifyCaptureFailure(error).endpoint).toBeUndefined();
});
it("keeps remote connection losses and bare target losses outside loopback recovery", () => {
for (const message of [
"connect ETIMEDOUT 203.0.113.10:443",
"net::ERR_CONNECTION_REFUSED at https://example.com/",
"Target closed",
"Failed to launch the browser process! spawn ENOENT",
]) {
const failure = classifyCaptureFailure(new Error(message));
expect(failure.endpoint).toBeUndefined();
expect(isLoopbackConnectionLoss(failure)).toBe(false);
}
});
it("does not let a loopback host inside a protocol timeout override the timeout kind", () => {
expect(
classifyCaptureFailure(new Error("net::ERR_TIMED_OUT at http://localhost:4173/index.html")),
).toMatchObject({ endpoint: { host: "localhost", port: 4173 } });
classifyCaptureFailure(
new Error("Page.captureScreenshot timed out\nconnect ETIMEDOUT 127.0.0.1:49152"),
).kind,
).toBe("protocol_timeout");
});
it("classifies repeated operation text in linear time", () => {
+129 -33
View File
@@ -115,15 +115,63 @@ function messageOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function loopbackTimeoutEndpoint(message: string): CaptureEndpointDiagnostic | undefined {
const match =
/connect ETIMEDOUT (127\.0\.0\.1|localhost|\[::1\]):(\d+)/i.exec(message) ??
/net::ERR_TIMED_OUT at https?:\/\/(127\.0\.0\.1|localhost|\[::1\]):(\d+)/i.exec(message);
if (!match?.[1] || !match[2]) return undefined;
const port = Number(match[2]);
return Number.isInteger(port) && port > 0 && port <= 65_535
? { host: match[1], port }
: undefined;
const CAUSE_CHAIN_DEPTH = 5;
/**
* Every message reachable from `error` through `.cause` (and the members of
* an `AggregateError`), top-level first. undici flattens a failed connect to
* a top-level `"fetch failed"` whose `.cause` carries the errno message, and a
* `localhost` connect that tried both address families surfaces as an
* `AggregateError` with an empty own message, so classification must read the
* whole chain — the same walk `isDrawElementVerificationError` performs.
*/
function collectMessages(error: unknown): string[] {
const messages: string[] = [];
let current: unknown = error;
for (
let depth = 0;
depth < CAUSE_CHAIN_DEPTH && current !== undefined && current !== null;
depth++
) {
const message = messageOf(current);
if (message) messages.push(message);
if (current instanceof AggregateError) {
for (const member of current.errors) {
const memberMessage = messageOf(member);
if (memberMessage) messages.push(memberMessage);
}
}
current = typeof current === "object" ? (current as { cause?: unknown }).cause : undefined;
}
return messages;
}
// Node/Bun errno text is unbracketed (`connect ETIMEDOUT ::1:49152`); Chrome
// URLs bracket IPv6 (`http://[::1]:49152/`). Both name the same loopback.
const LOOPBACK_HOST = String.raw`(127\.0\.0\.1|localhost|\[::1\]|::1)`;
const LOOPBACK_CONNECTION_LOSS_PATTERNS = [
new RegExp(String.raw`connect (?:ETIMEDOUT|ECONNREFUSED|ECONNRESET) ${LOOPBACK_HOST}:(\d+)`, "i"),
new RegExp(
String.raw`net::ERR_(?:TIMED_OUT|CONNECTION_TIMED_OUT|CONNECTION_REFUSED|CONNECTION_RESET|CONNECTION_CLOSED|EMPTY_RESPONSE) at https?://${LOOPBACK_HOST}:(\d+)`,
"i",
),
];
/**
* The loopback endpoint a connection-loss error names, when it names one.
* Only loopback hosts qualify: the render's file server and Chrome's DevTools
* endpoint both live there, so a lost loopback connection is the signal the
* pre-frame recovery acts on. A remote host that times out stays fatal.
*/
function loopbackConnectionLossEndpoint(text: string): CaptureEndpointDiagnostic | undefined {
for (const pattern of LOOPBACK_CONNECTION_LOSS_PATTERNS) {
const match = pattern.exec(text);
if (!match?.[1] || !match[2]) continue;
const port = Number(match[2]);
if (!Number.isInteger(port) || port <= 0 || port > 65_535) continue;
return { host: match[1] === "[::1]" ? "::1" : match[1], port };
}
return undefined;
}
function matchesAny(message: string, patterns: readonly RegExp[]): boolean {
@@ -152,6 +200,47 @@ function ioError(error: unknown, message: string): boolean {
);
}
function isMemoryExhaustionText(message: string, chainText: string): boolean {
return (
BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim()) ||
BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(chainText) ||
matchesAny(chainText, MEMORY_EXHAUSTION_ERROR_PATTERNS)
);
}
interface CaptureFailureKindInput {
error: unknown;
message: string;
chainText: string;
aborted: boolean;
loopbackLoss: boolean;
}
const CANCELLATION_PATTERN = /(?:render|capture)?_?cancelled|AbortError/i;
// Order matters: cancellation and memory exhaustion outrank everything, a
// protocol timeout outranks a loopback host mentioned in the same text, and
// authoring/io are the fall-through.
const ORDERED_KIND_MATCHERS: ReadonlyArray<
readonly [CaptureFailureKind, (input: CaptureFailureKindInput) => boolean]
> = [
["cancelled", (input) => input.aborted || CANCELLATION_PATTERN.test(input.chainText)],
["memory_exhaustion", (input) => isMemoryExhaustionText(input.message, input.chainText)],
["verification", (input) => matchesAny(input.chainText, VERIFICATION_ERROR_PATTERNS)],
["protocol_timeout", (input) => matchesAny(input.chainText, PROTOCOL_TIMEOUT_PATTERNS)],
[
"transient_browser",
(input) => input.loopbackLoss || matchesAny(input.chainText, TRANSIENT_BROWSER_ERROR_PATTERNS),
],
["authoring", (input) => matchesAny(input.chainText, AUTHORING_ERROR_PATTERNS)],
];
function resolveCaptureFailureKind(input: CaptureFailureKindInput): CaptureFailureKind {
const matched = ORDERED_KIND_MATCHERS.find(([, matches]) => matches(input));
if (matched) return matched[0];
return ioError(input.error, input.chainText) ? "io" : "authoring";
}
export function classifyCaptureFailure(
error: unknown,
options: {
@@ -162,31 +251,21 @@ export function classifyCaptureFailure(
if (error instanceof CaptureFailure && !options.workerDiagnostics && !options.signal?.aborted) {
return error;
}
const message = messageOf(error);
const timeoutEndpoint = loopbackTimeoutEndpoint(message);
const endpoint = error instanceof CaptureFailure ? error.endpoint : timeoutEndpoint;
let kind: CaptureFailureKind;
if (options.signal?.aborted || /(?:render|capture)?_?cancelled|AbortError/i.test(message)) {
kind = "cancelled";
} else if (
BUN_MEMORY_EXHAUSTION_EXACT_MESSAGE.test(message.trim()) ||
BUN_MEMORY_EXHAUSTION_WRAPPED_WORKER_MESSAGE.test(message) ||
matchesAny(message, MEMORY_EXHAUSTION_ERROR_PATTERNS)
) {
kind = "memory_exhaustion";
} else if (matchesAny(message, VERIFICATION_ERROR_PATTERNS)) {
kind = "verification";
} else if (matchesAny(message, PROTOCOL_TIMEOUT_PATTERNS)) {
kind = "protocol_timeout";
} else if (timeoutEndpoint || matchesAny(message, TRANSIENT_BROWSER_ERROR_PATTERNS)) {
kind = "transient_browser";
} else if (matchesAny(message, AUTHORING_ERROR_PATTERNS)) {
kind = "authoring";
} else {
kind = ioError(error, message) ? "io" : "authoring";
}
const messages = collectMessages(error);
const message = messages[0] ?? messageOf(error);
// Patterns match against the whole cause chain so a wrapped connect failure
// classifies by the errno it carries, not by the wrapper's generic text.
const chainText = messages.join("\n");
const lossEndpoint = loopbackConnectionLossEndpoint(chainText);
const endpoint = error instanceof CaptureFailure ? error.endpoint : lossEndpoint;
return new CaptureFailure({
kind,
kind: resolveCaptureFailureKind({
error,
message,
chainText,
aborted: options.signal?.aborted === true,
loopbackLoss: lossEndpoint !== undefined,
}),
message,
cause: error,
workerDiagnostics:
@@ -200,6 +279,23 @@ export function isTransientBrowserError(error: unknown): boolean {
return classifyCaptureFailure(error).kind === "transient_browser";
}
/**
* A transient failure that names the loopback endpoint whose connection was
* lost — the file server or Chrome's DevTools port. This is the only
* transient shape the producer's pre-frame recovery retries on an ordinary
* one-worker stream; a bare `Target closed` (Chrome killed by SIGTERM) carries
* no endpoint and stays outside it.
*/
export type LoopbackConnectionLoss = CaptureFailure & {
endpoint: Readonly<CaptureEndpointDiagnostic>;
};
export function isLoopbackConnectionLoss(
failure: CaptureFailure,
): failure is LoopbackConnectionLoss {
return failure.kind === "transient_browser" && failure.endpoint !== undefined;
}
export function isMemoryExhaustionError(error: unknown): boolean {
return classifyCaptureFailure(error).kind === "memory_exhaustion";
}
@@ -1,5 +1,6 @@
import { describe, expect, it } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { createServer } from "node:http";
import path, { join } from "node:path";
import { tmpdir } from "node:os";
import {
@@ -104,6 +105,38 @@ describe("file server health", () => {
expect(health.durationMs).toBeLessThan(1_000);
});
it("rejects a foreign listener that answers 200 without the identity header", async () => {
// A stale port can be re-bound by an unrelated process between the probe
// and the restart decision; a bare 200 must not read as "our server".
const foreign = createServer((_request, response) => response.end("ok"));
await new Promise<void>((resolve) => foreign.listen(0, "127.0.0.1", resolve));
const address = foreign.address();
if (!address || typeof address === "string") throw new Error("foreign server has no port");
try {
const health = await probeFileServerHealth({ url: `http://127.0.0.1:${address.port}` }, 500);
expect(health).toMatchObject({ healthy: false, status: 200 });
expect(health.error).toBeUndefined();
} finally {
await new Promise<void>((resolve) => foreign.close(() => resolve()));
}
});
it("names the connection-refused code of a probe against a closed port", async () => {
const vacated = createServer();
await new Promise<void>((resolve) => vacated.listen(0, "127.0.0.1", resolve));
const address = vacated.address();
if (!address || typeof address === "string") throw new Error("server has no port");
await new Promise<void>((resolve) => vacated.close(() => resolve()));
const health = await probeFileServerHealth({ url: `http://127.0.0.1:${address.port}` }, 500);
expect(health.healthy).toBe(false);
// Bun reports `ConnectionRefused` on the error itself; Node buries
// `ECONNREFUSED` in `.cause`. Either must surface in the diagnostic.
expect(health.error).toMatch(/ConnectionRefused|ECONNREFUSED/);
});
it("bounds a health endpoint that never responds", async () => {
const server = Bun.serve({
hostname: "127.0.0.1",
+47 -2
View File
@@ -718,8 +718,12 @@ export async function probeFileServerHealth(
signal: controller.signal,
cache: "no-store",
});
// A coincidental foreign listener on the same port can answer 200; only
// the identity header proves the responder is this render's file server.
const healthy = response.ok && response.headers.get(FILE_SERVER_HEALTH_HEADER) === "healthy";
await response.body?.cancel();
// Release the connection, but never let a rejected cancel() turn a
// healthy server into a restart.
await response.body?.cancel().catch(() => undefined);
return {
healthy,
status: response.status,
@@ -729,13 +733,54 @@ export async function probeFileServerHealth(
return {
healthy: false,
durationMs: Date.now() - startedAt,
error: error instanceof Error ? error.message : String(error),
error: describeHealthProbeError(error),
};
} finally {
clearTimeout(timer);
}
}
/**
* The probe's failure text with the code that names it. Node's `fetch` reports
* every connect failure as "fetch failed" and buries `ECONNREFUSED` /
* `ETIMEDOUT` in `.cause`; Bun's carries `ConnectionRefused` on the error
* itself. Either way the code is what a restart decision gets read against.
*/
function describeHealthProbeError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
let current: unknown = error;
for (let depth = 0; depth < 5 && current instanceof Error; depth++) {
const code = (current as NodeJS.ErrnoException).code;
if (typeof code === "string" && code.length > 0) {
return message.includes(code) ? message : `${message} (${code})`;
}
current = current.cause;
}
return message;
}
/**
* The one shape every render-path file server takes: the compiled tree under
* `workDir`, an ephemeral port, the virtual-time shim first in `<head>`, and
* the job's fps. Probe discovery, frame capture, and the pre-frame capture
* retry all construct through here so the three sites cannot drift.
*/
export function createRenderFileServer(input: {
projectDir: string;
workDir: string;
fps: Fps;
/** Injected after the virtual-time shim (e.g. the page-side compositing stub). */
preHeadScripts?: readonly string[];
}): Promise<FileServerHandle> {
return createFileServer({
projectDir: input.projectDir,
compiledDir: join(input.workDir, "compiled"),
port: 0,
preHeadScripts: [VIRTUAL_TIME_SHIM, ...(input.preHeadScripts ?? [])],
fps: input.fps,
});
}
/**
* Set before the Hyperframes runtime executes so render/probe pages can avoid
* preview-only initialization work that mutates the live visual timeline.
@@ -0,0 +1,46 @@
import { describe, expect, it } from "vitest";
import { classifyCaptureFailure, isLoopbackConnectionLoss } from "@hyperframes/engine";
import { CaptureStageError, wrapCaptureStageError } from "./captureStageError.js";
import { EncoderInterruptedError } from "./encoderInterruption.js";
describe("CaptureStageError", () => {
it("carries the classified loopback endpoint through the streaming-stage wrapper", () => {
const wrapped = wrapCaptureStageError(new Error("connect ETIMEDOUT ::1:49152"), ["console"]);
expect(wrapped).toBeInstanceOf(CaptureStageError);
expect(wrapped).toMatchObject({
kind: "transient_browser",
endpoint: { host: "::1", port: 49152 },
browserConsole: ["console"],
});
});
it("keeps the endpoint when the orchestrator re-classifies the stage error", () => {
// Production path: captureStreamingStage throws wrapCaptureStageError(err)
// and the orchestrator's catch calls classifyCaptureFailure on that result,
// which early-returns the CaptureFailure it was handed.
const wrapped = wrapCaptureStageError(
new TypeError("fetch failed", { cause: new Error("connect ECONNREFUSED 127.0.0.1:4173") }),
[],
);
const controller = new AbortController();
const reclassified = classifyCaptureFailure(wrapped, { signal: controller.signal });
expect(reclassified).toBe(wrapped);
expect(isLoopbackConnectionLoss(reclassified)).toBe(true);
expect(reclassified.endpoint).toEqual({ host: "127.0.0.1", port: 4173 });
});
it("leaves a bare target loss without an endpoint", () => {
const wrapped = wrapCaptureStageError(new Error("Target closed"), []);
expect(wrapped).toMatchObject({ kind: "transient_browser" });
expect((wrapped as CaptureStageError).endpoint).toBeUndefined();
});
it("passes encoder interruptions through unchanged", () => {
const interrupted = new EncoderInterruptedError("Encoder interrupted", "SIGTERM");
expect(wrapCaptureStageError(interrupted, [])).toBe(interrupted);
});
});
@@ -12,6 +12,7 @@ export class CaptureStageError extends CaptureFailure {
message: normalizeErrorMessage(input.cause),
cause: input.cause,
workerDiagnostics: classified.workerDiagnostics,
endpoint: classified.endpoint,
});
this.name = "CaptureStageError";
this.browserConsole = input.browserConsole.slice();
@@ -0,0 +1,183 @@
import { describe, expect, it, vi } from "vitest";
import { classifyCaptureFailure, CaptureFailure } from "@hyperframes/engine";
import {
type LoopbackConnectionLoss,
resolvePreFrameLoopbackLoss,
recoverPreFrameFileServer,
} from "./preFrameRecovery.js";
import { wrapCaptureStageError } from "./captureStageError.js";
const lossFrom = (message: string) =>
classifyCaptureFailure(new Error(message)) as LoopbackConnectionLoss;
const loopbackLoss = () => lossFrom("connect ETIMEDOUT ::1:49152");
describe("resolvePreFrameLoopbackLoss", () => {
it("qualifies a loopback connection loss on a one-worker stream before the first frame", () => {
expect(
resolvePreFrameLoopbackLoss({
failure: loopbackLoss(),
workerCount: 1,
framesRendered: 0,
}),
).toBeDefined();
});
it("qualifies a stage-wrapped Node fetch failure with the errno in its cause", () => {
const failure = wrapCaptureStageError(
new TypeError("fetch failed", { cause: new Error("connect ECONNREFUSED 127.0.0.1:49152") }),
[],
);
expect(
resolvePreFrameLoopbackLoss({
failure: failure as CaptureFailure,
workerCount: 1,
framesRendered: 0,
}),
).toBeDefined();
});
it.each([
["Target closed", "a SIGTERM-killed Chrome target"],
["Failed to launch the browser process! spawn ENOENT", "a browser that cannot launch"],
["Page crashed!", "a page crash"],
["Composition has zero duration. Runtime ready: false", "a runtime that never became ready"],
["Navigation timeout of 30000 ms exceeded", "a navigation timeout"],
["connect ETIMEDOUT 203.0.113.10:443", "a remote connect timeout"],
])("does not qualify %s (%s)", (message) => {
const failure = classifyCaptureFailure(new Error(message));
expect(
resolvePreFrameLoopbackLoss({ failure, workerCount: 1, framesRendered: 0 }),
).toBeUndefined();
});
it("requires exactly one worker", () => {
for (const workerCount of [0, 2, 4]) {
expect(
resolvePreFrameLoopbackLoss({
failure: loopbackLoss(),
workerCount,
framesRendered: 0,
}),
).toBeUndefined();
}
});
it("requires that no frame has been written yet", () => {
expect(
resolvePreFrameLoopbackLoss({
failure: loopbackLoss(),
workerCount: 1,
framesRendered: 1,
}),
).toBeUndefined();
expect(
resolvePreFrameLoopbackLoss({
failure: loopbackLoss(),
workerCount: 1,
framesRendered: 9_500,
}),
).toBeUndefined();
});
it("does not qualify a cancelled failure even when it names a loopback endpoint", () => {
const controller = new AbortController();
controller.abort();
const cancelled = classifyCaptureFailure(new Error("connect ETIMEDOUT ::1:49152"), {
signal: controller.signal,
});
expect(cancelled.kind).toBe("cancelled");
expect(
resolvePreFrameLoopbackLoss({ failure: cancelled, workerCount: 1, framesRendered: 0 }),
).toBeUndefined();
});
});
describe("recoverPreFrameFileServer", () => {
const server = { url: "http://localhost:49152", port: 49152 };
const replacement = { url: "http://localhost:50001", port: 50001 };
it("restarts the file server when the probe reports it unhealthy", async () => {
const restart = vi.fn(async () => replacement);
const warn = vi.fn();
await recoverPreFrameFileServer({
failure: loopbackLoss(),
fileServer: server,
health: Promise.resolve({
healthy: false,
durationMs: 12,
error: "fetch failed (ECONNREFUSED)",
}),
restart,
log: { warn },
});
expect(restart).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledWith(
"[Render] Pre-frame capture endpoint health",
expect.objectContaining({
reportedEndpoint: "::1:49152",
endpointOwner: "file_server",
fileServerEndpoint: server.url,
fileServerHealthy: false,
healthProbeError: "fetch failed (ECONNREFUSED)",
}),
);
expect(warn).toHaveBeenCalledWith(
"[Render] Recreated unhealthy file server before bounded capture retry",
{ fileServerEndpoint: replacement.url },
);
});
it("keeps a healthy file server and never calls restart", async () => {
const restart = vi.fn(async () => replacement);
await recoverPreFrameFileServer({
failure: loopbackLoss(),
fileServer: server,
health: Promise.resolve({ healthy: true, status: 200, durationMs: 3 }),
restart,
log: { warn: vi.fn() },
});
expect(restart).not.toHaveBeenCalled();
});
it("reports a foreign port as browser_or_unknown while still restarting on an unhealthy probe", async () => {
const failure = lossFrom("net::ERR_CONNECTION_TIMED_OUT at http://localhost:9222/json/version");
const restart = vi.fn(async () => replacement);
const warn = vi.fn();
await recoverPreFrameFileServer({
failure,
fileServer: server,
health: Promise.resolve({
healthy: false,
durationMs: 1_008,
error: "This operation was aborted",
}),
restart,
log: { warn },
});
expect(warn).toHaveBeenCalledWith(
"[Render] Pre-frame capture endpoint health",
expect.objectContaining({ endpointOwner: "browser_or_unknown" }),
);
expect(restart).toHaveBeenCalledTimes(1);
});
it("propagates a failed restart instead of retrying against a dead server", async () => {
await expect(
recoverPreFrameFileServer({
failure: loopbackLoss(),
fileServer: server,
health: Promise.resolve({ healthy: false, durationMs: 1 }),
restart: async () => {
throw new Error("EADDRINUSE");
},
log: { warn: vi.fn() },
}),
).rejects.toThrow("EADDRINUSE");
});
});
@@ -0,0 +1,65 @@
import {
type CaptureFailure,
isLoopbackConnectionLoss,
type LoopbackConnectionLoss,
} from "@hyperframes/engine";
import type { ProducerLogger } from "../../logger.js";
import type { FileServerHandle, FileServerHealth } from "../fileServer.js";
/**
* Pre-frame loopback recovery for an ordinary one-worker screenshot stream.
*
* The streaming stage can lose its loopback connection — to the file server or
* to Chrome's DevTools port — before the first frame is captured. That is the
* only transient shape this path retries, and this function is its single
* owner: the failure must name a loopback endpoint (`isLoopbackConnectionLoss`),
* the stream must be single-worker (a multi-worker stream has its own adaptive
* retry), and no frame may have been written yet (a later loss is a different
* failure and would restart the whole render from frame 0 on the slower
* path). A bare `Target closed` — Chrome killed by SIGTERM on a host that is
* going away — carries no endpoint and is deliberately excluded, matching
* the encoder-interruption rule in `shouldRetryViaPinnedFallback`.
*/
export function resolvePreFrameLoopbackLoss(input: {
failure: CaptureFailure;
workerCount: number;
framesRendered: number;
}): LoopbackConnectionLoss | undefined {
if (input.workerCount !== 1 || input.framesRendered !== 0) return undefined;
return isLoopbackConnectionLoss(input.failure) ? input.failure : undefined;
}
/**
* Decide whether the retry needs a fresh file server and perform the restart.
* The restart rebinds the caller's active server; nothing is returned.
*
* The probe result is passed in (not taken here) so the caller can start it
* while the failed browser session is still closing. The restart keys on the
* probe, not on the reported endpoint: a bare loopback timeout cannot prove
* which listener was gone, so the file server's own health is the evidence.
*/
export async function recoverPreFrameFileServer(input: {
failure: LoopbackConnectionLoss;
fileServer: Pick<FileServerHandle, "url" | "port">;
health: Promise<FileServerHealth>;
restart: () => Promise<Pick<FileServerHandle, "url" | "port">>;
log: Pick<ProducerLogger, "warn">;
}): Promise<void> {
const health = await input.health;
const endpointOwner =
input.failure.endpoint.port === input.fileServer.port ? "file_server" : "browser_or_unknown";
input.log.warn("[Render] Pre-frame capture endpoint health", {
reportedEndpoint: `${input.failure.endpoint.host}:${input.failure.endpoint.port}`,
endpointOwner,
fileServerEndpoint: input.fileServer.url,
fileServerHealthy: health.healthy,
fileServerStatus: health.status,
healthProbeMs: health.durationMs,
healthProbeError: health.error,
});
if (health.healthy) return;
const fileServer = await input.restart();
input.log.warn("[Render] Recreated unhealthy file server before bounded capture retry", {
fileServerEndpoint: fileServer.url,
});
}
@@ -54,9 +54,8 @@ import {
} from "../../htmlCompiler.js";
import {
closeFileServerSafely,
createFileServer,
createRenderFileServer,
type FileServerHandle,
VIRTUAL_TIME_SHIM,
} from "../../fileServer.js";
import type { ProducerLogger } from "../../../logger.js";
import {
@@ -319,13 +318,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
reasons,
});
fileServer = await createFileServer({
projectDir,
compiledDir: join(workDir, "compiled"),
port: 0,
preHeadScripts: [VIRTUAL_TIME_SHIM],
fps: job.config.fps,
});
fileServer = await createRenderFileServer({ projectDir, workDir, fps: job.config.fps });
assertNotAborted();
const captureOpts: CaptureOptions = {
@@ -6,6 +6,7 @@ import type { CaptureOptions, EngineConfig, ExtractedFrames } from "@hyperframes
import {
DEFAULT_CONFIG,
DrawElementCaptureError,
classifyCaptureFailure,
executeParallelCapture,
mergeWorkerFrames,
} from "@hyperframes/engine";
@@ -56,6 +57,8 @@ import {
createCaptureObservabilityUpdater,
} from "./renderOrchestrator.js";
import { probeRequiresBrowser } from "./render/stages/probeStage.js";
import { wrapCaptureStageError } from "./render/captureStageError.js";
import { resolvePreFrameLoopbackLoss } from "./render/preFrameRecovery.js";
import { ensureFrameWritten } from "./render/stages/captureHdrFrameShared.js";
import { resolveCompositeTransfer, shouldUseLayeredComposite } from "./hdrCompositor.js";
import {
@@ -2690,30 +2693,70 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
).toBe(false);
});
it("retries a transient browser death on an ordinary one-worker streaming route", () => {
it("retries a pre-frame loopback connection loss on an ordinary one-worker streaming route", () => {
expect(
shouldRetryViaPinnedFallback({
isVerifyError: false,
isCancellation: false,
isTransientSingleWorkerFailure: true,
isPreFrameLoopbackConnectionLoss: true,
deWorkerInversion: undefined,
deParallelRouter: undefined,
}),
).toBe(true);
});
it("does not widen the transient streaming retry to an unpinned multi-worker route", () => {
it("does not widen the loopback retry to an unpinned route that did not qualify", () => {
expect(
shouldRetryViaPinnedFallback({
isVerifyError: false,
isCancellation: false,
isTransientSingleWorkerFailure: false,
isPreFrameLoopbackConnectionLoss: false,
deWorkerInversion: undefined,
deParallelRouter: undefined,
}),
).toBe(false);
});
it("never qualifies a SIGTERM-killed Chrome (`Target closed`) for the loopback recovery path", () => {
// Host shutdown: Chrome dies before the encoder notices, so the error is a
// plain `Target closed`, not an EncoderInterruptedError. It names no
// loopback endpoint, so the pre-frame gate alone never retries it; only
// isTransientCaptureError (routing-independent transient retry) can.
const failure = classifyCaptureFailure(wrapCaptureStageError(new Error("Target closed"), []));
const preFrameLoopbackLoss = resolvePreFrameLoopbackLoss({
failure,
workerCount: 1,
framesRendered: 0,
});
expect(failure.kind).toBe("transient_browser");
expect(preFrameLoopbackLoss).toBeUndefined();
expect(
shouldRetryViaPinnedFallback({
isVerifyError: false,
isCancellation: false,
isEncoderInterrupted: false,
isPreFrameLoopbackConnectionLoss: preFrameLoopbackLoss !== undefined,
deWorkerInversion: undefined,
deParallelRouter: undefined,
}),
).toBe(false);
});
it("keeps a pre-frame loopback loss non-retryable once cancellation or an encoder interruption is present", () => {
for (const overrides of [{ isCancellation: true }, { isEncoderInterrupted: true }]) {
expect(
shouldRetryViaPinnedFallback({
isVerifyError: false,
isCancellation: false,
isPreFrameLoopbackConnectionLoss: true,
deWorkerInversion: undefined,
deParallelRouter: undefined,
...overrides,
}),
).toBe(false);
}
});
it("retries OOM too when the router pinned the worker count (fallback's Chrome processes are already dead by the time this runs, and the fallback is pooled/lighter than the pinned path)", () => {
expect(
shouldRetryViaPinnedFallback({
@@ -2779,7 +2822,7 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
shouldRetryViaPinnedFallback({
isVerifyError: false,
isCancellation: true,
isTransientSingleWorkerFailure: true,
isPreFrameLoopbackConnectionLoss: true,
deWorkerInversion: undefined,
deParallelRouter: undefined,
}),
@@ -100,11 +100,10 @@ import { randomUUID } from "crypto";
import { fileURLToPath } from "url";
import {
closeFileServerSafely,
createFileServer,
createRenderFileServer,
probeFileServerHealth,
type FileServerHandle,
HF_PAGE_SIDE_COMPOSITING_STUB,
VIRTUAL_TIME_SHIM,
} from "./fileServer.js";
import { defaultLogger, type ProducerLogger } from "../logger.js";
import {
@@ -136,6 +135,10 @@ import {
worstSubTimelineWaitOutcome,
} from "./render/perfSummary.js";
import { getCaptureStageBrowserConsole } from "./render/captureStageError.js";
import {
recoverPreFrameFileServer,
resolvePreFrameLoopbackLoss,
} from "./render/preFrameRecovery.js";
import { resolveVideoCaptureBeyondViewport } from "./render/captureBeyondViewport.js";
import {
type CaptureCalibrationSample,
@@ -1856,11 +1859,16 @@ export function resolveParallelRouterRetryPlan(args: {
* of calibration, so a generic capture failure on that pinned count is
* exactly the scenario the pin itself introduced risk for.
*
* An ordinary one-worker stream also gets one retry when Chrome itself dies.
* There is no worker count to reduce, but the failed stage has already closed
* its session and encoder; replanning forces screenshot capture and the second
* invoke creates fresh resources. The surrounding catch performs this at most
* once, so a deterministically dying composition still fails.
* An ordinary one-worker stream also gets one retry when it loses its loopback
* connection (file server or DevTools port) before the first frame — see
* resolvePreFrameLoopbackLoss for the exact shape. There is no worker
* count to reduce, but the failed stage has already closed its session and
* encoder; replanning forces screenshot capture and the second invoke creates
* fresh resources, recreating the file server first when its health probe
* fails. The loopback gate is deliberately narrow — a Chrome killed by a host
* shutdown looks exactly like `Target closed`, so it never qualifies here;
* whether such a shape retries at all is decided by `isTransientCaptureError`
* below, which covers every transient browser failure routing-independently.
*
* Includes OOM (previously excluded — see PR history): every worker's
* `executeWorkerTask` closes its capture session in a `finally` that awaits
@@ -1895,7 +1903,8 @@ export function shouldRetryViaPinnedFallback(args: {
isDeCaptureError?: boolean;
isCancellation: boolean;
isEncoderInterrupted?: boolean;
isTransientSingleWorkerFailure?: boolean;
/** Pre-frame loopback connection loss on an ordinary one-worker stream. */
isPreFrameLoopbackConnectionLoss?: boolean;
deWorkerInversion: "inverted" | "reverted" | undefined;
deParallelRouter: "routed" | "reverted" | undefined;
/**
@@ -1924,7 +1933,7 @@ export function shouldRetryViaPinnedFallback(args: {
if (args.isVerifyError || args.isDeCaptureError) return true;
if (args.isDeRendererStall === true || args.isSequentialCaptureStall === true) return true;
if (args.isTransientCaptureError === true) return true;
if (args.isTransientSingleWorkerFailure) return true;
if (args.isPreFrameLoopbackConnectionLoss) return true;
return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed";
}
@@ -2746,13 +2755,7 @@ async function executeRenderPipeline(input: {
if (!fileServer) {
const fileServerStart = observability.stageStart("file_server", { reused: false });
try {
fileServer = await createFileServer({
projectDir,
compiledDir: join(workDir, "compiled"),
port: 0,
preHeadScripts: [VIRTUAL_TIME_SHIM],
fps: job.config.fps,
});
fileServer = await createRenderFileServer({ projectDir, workDir, fps: job.config.fps });
assertNotAborted();
observability.stageEnd("file_server", fileServerStart);
} catch (error) {
@@ -3459,20 +3462,20 @@ async function executeRenderPipeline(input: {
"screenshot per output frame.",
);
}
const restartCaptureFileServer = async (): Promise<void> => {
// `invokeStreaming` reads `activeFileServer` when called, so rebinding it
// here is what the bounded retry picks up. The same factory as the first
// construction, plus the compositing stub that addPreHeadScript added.
const restartCaptureFileServer = async (): Promise<FileServerHandle> => {
closeFileServerSafely(activeFileServer, "capture retry", log);
fileServer = null;
activeFileServer = await createFileServer({
activeFileServer = await createRenderFileServer({
projectDir,
compiledDir: join(workDir, "compiled"),
port: 0,
preHeadScripts: [
VIRTUAL_TIME_SHIM,
...(usePageSideCompositingForTransitions ? [HF_PAGE_SIDE_COMPOSITING_STUB] : []),
],
workDir,
fps: job.config.fps,
preHeadScripts: usePageSideCompositingForTransitions ? [HF_PAGE_SIDE_COMPOSITING_STUB] : [],
});
fileServer = activeFileServer;
return activeFileServer;
};
const useLayeredComposite =
!usePageSideCompositingForTransitions &&
@@ -3752,30 +3755,31 @@ async function executeRenderPipeline(input: {
streamingRes = await invokeStreaming();
} catch (err) {
// drawElement self-verification, a sequential no-progress deadline,
// or an ordinary single-worker stream losing its browser restarts
// the whole render from a fresh screenshot session. When an
// inversion/router pinned the worker count, other capture-stage
// failures (host timeout, worker crash, OOM) can use that same tested
// baseline. The stage closes the failed session before throwing;
// probeSession (if any) was consumed by it. See
// shouldRetryViaPinnedFallback for exactly which errors qualify.
// or an ordinary single-worker stream losing its loopback connection
// before the first frame restarts the whole render from a fresh
// screenshot session. When an inversion/router pinned the worker
// count, other capture-stage failures (host timeout, worker crash,
// OOM) can use that same tested baseline. The stage closes the failed
// session before throwing; probeSession (if any) was consumed by it.
// See shouldRetryViaPinnedFallback for exactly which errors qualify.
const isVerifyError = isDrawElementVerificationError(err);
const isDeCaptureError = isDrawElementCaptureError(err);
const isDeStall = isDeRendererStallError(err);
const isSequentialStall = isSequentialCaptureStallError(err);
const isCancellation =
err instanceof RenderCancelledError || executionSignal?.aborted === true;
const captureFailure = classifyCaptureFailure(err, { signal: executionSignal });
const isTransientBrowserFailure = captureFailure.kind === "transient_browser";
const isTransientSingleWorkerFailure =
isTransientBrowserFailure && capturePlan.workerCount === 1;
const preFrameLoopbackLoss = resolvePreFrameLoopbackLoss({
failure: classifyCaptureFailure(err, { signal: executionSignal }),
workerCount: capturePlan.workerCount,
framesRendered: job.framesRendered ?? 0,
});
if (
!shouldRetryViaPinnedFallback({
isVerifyError,
isDeCaptureError,
isCancellation,
isEncoderInterrupted: err instanceof EncoderInterruptedError,
isTransientSingleWorkerFailure,
isPreFrameLoopbackConnectionLoss: preFrameLoopbackLoss !== undefined,
deWorkerInversion,
deParallelRouter,
isDeRendererStall: isDeStall,
@@ -3806,8 +3810,8 @@ async function executeRenderPipeline(input: {
? "[Render] drawElement renderer stalled; re-rendering via screenshot"
: isSequentialStall
? "[Render] sequential capture stalled; retrying on a fresh screenshot session"
: isTransientSingleWorkerFailure
? "[Render] transient single-worker browser failure; retrying with a fresh screenshot session"
: preFrameLoopbackLoss
? "[Render] pre-frame loopback connection loss; retrying with a fresh screenshot session"
: "[Render] capture failed; re-rendering via a fresh screenshot session",
{ error: err instanceof Error ? err.message : String(err) },
);
@@ -3819,8 +3823,8 @@ async function executeRenderPipeline(input: {
? "drawElement renderer stalled; retrying with forceScreenshot"
: isSequentialStall
? "sequential capture stalled; retrying with a fresh screenshot session"
: isTransientSingleWorkerFailure
? "transient single-worker browser failure; retrying with a fresh screenshot session"
: preFrameLoopbackLoss
? "pre-frame loopback connection loss; retrying with a fresh screenshot session"
: "capture failed; retrying with a fresh screenshot session",
);
const failedRouting = capturePlan.routing.kind;
@@ -3844,10 +3848,11 @@ async function executeRenderPipeline(input: {
deWorkerInversion,
deParallelRouter,
});
const preFrameHealthPromise =
isTransientSingleWorkerFailure && job.framesRendered === 0
? probeFileServerHealth(activeFileServer)
: null;
// Probe the file server while the failed session closes below; the
// recovery decision awaits the result once the session is gone.
const preFrameRecovery = preFrameLoopbackLoss
? { failure: preFrameLoopbackLoss, health: probeFileServerHealth(activeFileServer) }
: null;
// Streaming stage aims to close the probe in its own finally; if it
// threw before doing so, the Chrome process would orphan through the
// pinned-fallback retry. Close defensively before we release the
@@ -3858,31 +3863,13 @@ async function executeRenderPipeline(input: {
probeSession = null;
await closeOrphanedProbeForRetry(orphaned, closeCaptureSession, log, "streaming");
}
if (preFrameHealthPromise) {
const health = await preFrameHealthPromise;
const endpointOwner =
captureFailure.endpoint?.port === activeFileServer.port
? "file_server"
: captureFailure.endpoint
? "browser_or_unknown"
: "unknown";
log.warn("[Render] Pre-frame capture endpoint health", {
reportedEndpoint: captureFailure.endpoint
? `${captureFailure.endpoint.host}:${captureFailure.endpoint.port}`
: undefined,
endpointOwner,
fileServerEndpoint: activeFileServer.url,
fileServerHealthy: health.healthy,
fileServerStatus: health.status,
healthProbeMs: health.durationMs,
healthProbeError: health.error,
if (preFrameRecovery) {
await recoverPreFrameFileServer({
...preFrameRecovery,
fileServer: activeFileServer,
restart: restartCaptureFileServer,
log,
});
if (!health.healthy) {
await restartCaptureFileServer();
log.warn("[Render] Recreated unhealthy file server before bounded capture retry", {
fileServerEndpoint: activeFileServer.url,
});
}
}
if (failedRouting !== "default") {
// The inversion's / router's bet on the pinned path lost. Prefer