mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-04 16:42:27 +00:00
fix(encoder): signal host interruptions for retry (#3578)
* fix(encoder): signal host interruptions for retry * fix(encoder): cover all render interruption paths * fix(encoder): classify HDR pre-extraction drains
This commit is contained in:
@@ -309,6 +309,7 @@ export {
|
||||
export {
|
||||
runFfmpeg,
|
||||
formatFfmpegError,
|
||||
isExternalFfmpegInterruption,
|
||||
type RunFfmpegOptions,
|
||||
type RunFfmpegResult,
|
||||
} from "./utils/runFfmpeg.js";
|
||||
|
||||
@@ -383,7 +383,10 @@ function ffmpegFailure(
|
||||
let owner: AudioProcessingFailure["owner"] = "system";
|
||||
let retryable = false;
|
||||
|
||||
if (result.terminationReason === "abort") {
|
||||
if (result.failureReason === "external_interruption") {
|
||||
reason = "external_interruption";
|
||||
retryable = true;
|
||||
} else if (result.terminationReason === "abort") {
|
||||
reason = "cancelled";
|
||||
owner = "user";
|
||||
} else if (result.terminationReason === "deadline" || result.terminationReason === "inactivity") {
|
||||
@@ -862,7 +865,12 @@ async function mixAudioTracks(
|
||||
// dropped from the output entirely — a missing fade beats missing audio.
|
||||
let degradedAutomation = false;
|
||||
const hasAutomation = tracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
|
||||
if (!result.success && !signal?.aborted && hasAutomation) {
|
||||
if (
|
||||
!result.success &&
|
||||
result.failureReason !== "external_interruption" &&
|
||||
!signal?.aborted &&
|
||||
hasAutomation
|
||||
) {
|
||||
const retry = await runMix(true);
|
||||
if (retry.success) {
|
||||
result = retry;
|
||||
|
||||
@@ -69,6 +69,7 @@ export type AudioFailureReason =
|
||||
| "ffmpeg_unsupported"
|
||||
| "ffmpeg_timeout"
|
||||
| "ffmpeg_unavailable"
|
||||
| "external_interruption"
|
||||
| "ffmpeg_failed"
|
||||
| "cancelled"
|
||||
| "internal";
|
||||
|
||||
@@ -374,6 +374,45 @@ describe("encodeFramesChunkedConcat ffmpegEncodeTimeout", () => {
|
||||
});
|
||||
|
||||
describe("muxVideoWithAudio audio codec handling", () => {
|
||||
it("preserves an external interruption from mux", async () => {
|
||||
const { spawn, calls } = createSpawnSpy();
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
|
||||
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
|
||||
const muxPromise = muxVideoWithAudio(
|
||||
"/tmp/video-only.mp4",
|
||||
"/tmp/audio.aac",
|
||||
"/tmp/output.mp4",
|
||||
);
|
||||
|
||||
await flushMuxCodecResolution();
|
||||
calls[0]!.proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
|
||||
emitClose(calls[0]!.proc, 255);
|
||||
|
||||
await expect(muxPromise).resolves.toMatchObject({
|
||||
success: false,
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves an external interruption from faststart", async () => {
|
||||
const { spawn, calls } = createSpawnSpy();
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
|
||||
const { applyFaststart } = await import("./chunkEncoder.js");
|
||||
const faststartPromise = applyFaststart("/tmp/video-only.mp4", "/tmp/output.mp4");
|
||||
|
||||
calls[0]!.proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
|
||||
emitClose(calls[0]!.proc, 255);
|
||||
|
||||
await expect(faststartPromise).resolves.toMatchObject({
|
||||
success: false,
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
});
|
||||
|
||||
it("copies HyperFrames AAC sidecars into MP4 instead of re-encoding", async () => {
|
||||
const { spawn, calls } = createSpawnSpy();
|
||||
vi.resetModules();
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
} from "../utils/gpuEncoder.js";
|
||||
import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
|
||||
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
|
||||
import { formatFfmpegError, runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
import { formatFfmpegError, isExternalFfmpegInterruption, runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
import { extractAudioMetadata } from "../utils/ffprobe.js";
|
||||
import { type Fps, fpsToFfmpegArg } from "@hyperframes/core";
|
||||
import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
|
||||
@@ -518,6 +518,7 @@ export async function encodeFramesFromDir(
|
||||
result.terminationReason === "deadline",
|
||||
encodeTimeout,
|
||||
),
|
||||
failureReason: isExternalFfmpegInterruption(result) ? "external_interruption" : undefined,
|
||||
};
|
||||
}
|
||||
const fileSize = existsSync(outputPath) ? statSync(outputPath).size : 0;
|
||||
@@ -612,6 +613,9 @@ export async function encodeFramesChunkedConcat(
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: chunkResult.error,
|
||||
failureReason: isExternalFfmpegInterruption(processResult)
|
||||
? "external_interruption"
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
chunkPaths.push(chunkPath);
|
||||
@@ -650,6 +654,9 @@ export async function encodeFramesChunkedConcat(
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: concatResult.error,
|
||||
failureReason: isExternalFfmpegInterruption(concatProcessResult)
|
||||
? "external_interruption"
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -739,6 +746,7 @@ export async function muxVideoWithAudio(
|
||||
outputPath,
|
||||
durationMs: result.durationMs,
|
||||
error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
|
||||
failureReason: result.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -781,5 +789,6 @@ export async function applyFaststart(
|
||||
outputPath,
|
||||
durationMs: result.durationMs,
|
||||
error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
|
||||
failureReason: result.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -50,6 +50,8 @@ export interface EncodeResult {
|
||||
framesEncoded: number;
|
||||
fileSize: number;
|
||||
error?: string;
|
||||
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
|
||||
failureReason?: "external_interruption";
|
||||
}
|
||||
|
||||
export interface MuxResult {
|
||||
@@ -57,4 +59,6 @@ export interface MuxResult {
|
||||
outputPath: string;
|
||||
durationMs: number;
|
||||
error?: string;
|
||||
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
|
||||
failureReason?: "external_interruption";
|
||||
}
|
||||
|
||||
@@ -548,6 +548,24 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
|
||||
expect(result.error).toContain("Encoder error");
|
||||
});
|
||||
|
||||
it("classifies ffmpeg's handled SIGTERM as an external interruption", async () => {
|
||||
const { spawn, calls } = createSpawnSpy();
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
|
||||
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
|
||||
const dir = mkdtempSync(join(tmpdir(), "se-interrupted-"));
|
||||
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
|
||||
const proc = calls[0]!.proc;
|
||||
proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
|
||||
process.nextTick(() => proc.emit("close", 255));
|
||||
|
||||
const result = await encoder.close();
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.failureReason).toBe("external_interruption");
|
||||
expect(encoder.getExitFailureReason?.()).toBe("external_interruption");
|
||||
});
|
||||
|
||||
it("getExitError surfaces the ffmpeg failure reason after a non-zero exit", async () => {
|
||||
const { spawn, calls } = createSpawnSpy();
|
||||
vi.resetModules();
|
||||
@@ -696,6 +714,27 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
|
||||
expect(await encoder.writeFrame(Buffer.from([0]))).toBe(false);
|
||||
});
|
||||
|
||||
it("waits for child close when stdin dies first so the interruption reason is observable", async () => {
|
||||
const { spawn, calls } = createSpawnSpy();
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({ spawn }));
|
||||
|
||||
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
|
||||
const dir = mkdtempSync(join(tmpdir(), "se-epipe-before-close-"));
|
||||
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
|
||||
const proc = calls[0]!.proc;
|
||||
proc.stdin.destroyed = true;
|
||||
|
||||
const writePromise = encoder.writeFrame(Buffer.from([0]));
|
||||
await expect(resolveWithin(writePromise, 10)).resolves.toBe("timeout");
|
||||
|
||||
proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
|
||||
proc.emit("close", 255);
|
||||
|
||||
await expect(writePromise).resolves.toBe(false);
|
||||
expect(encoder.getExitFailureReason?.()).toBe("external_interruption");
|
||||
});
|
||||
|
||||
it("writeFrame waits for stdin drain when FFmpeg applies back-pressure", async () => {
|
||||
const { spawn, calls } = createSpawnSpy();
|
||||
vi.resetModules();
|
||||
|
||||
@@ -29,7 +29,7 @@ import {
|
||||
getGpuEncoderName,
|
||||
mapPresetForGpuEncoder,
|
||||
} from "../utils/gpuEncoder.js";
|
||||
import { formatFfmpegError } from "../utils/runFfmpeg.js";
|
||||
import { formatFfmpegError, isExternalFfmpegInterruption } from "../utils/runFfmpeg.js";
|
||||
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
|
||||
import { getHdrEncoderColorParams } from "../utils/hdr.js";
|
||||
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
|
||||
@@ -159,6 +159,8 @@ export interface StreamingEncoderResult {
|
||||
durationMs: number;
|
||||
fileSize: number;
|
||||
error?: string;
|
||||
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
|
||||
failureReason?: "external_interruption";
|
||||
}
|
||||
|
||||
export interface StreamingEncoder {
|
||||
@@ -179,6 +181,8 @@ export interface StreamingEncoder {
|
||||
* unsupported codec, disk full) instead of a bare "encoder exited" message.
|
||||
*/
|
||||
getExitError: () => string | undefined;
|
||||
/** Machine-readable cause available after FFmpeg exits unexpectedly mid-write. */
|
||||
getExitFailureReason?: () => "external_interruption" | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -465,6 +469,7 @@ export async function spawnStreamingEncoder(
|
||||
let exitStatus: "running" | "success" | "error" = "running";
|
||||
let stderr = "";
|
||||
let exitCode: number | null = null;
|
||||
let exitSignal: NodeJS.Signals | null = null;
|
||||
let terminationReason: ManagedProcessTerminationReason = "exit";
|
||||
|
||||
ffmpeg.stdin?.on("error", () => {});
|
||||
@@ -486,6 +491,7 @@ export async function spawnStreamingEncoder(
|
||||
});
|
||||
const exitPromise = managed.wait().then((outcome) => {
|
||||
exitCode = outcome.exitCode;
|
||||
exitSignal = outcome.signal;
|
||||
stderr = outcome.stderr;
|
||||
terminationReason = outcome.reason;
|
||||
exitStatus = outcome.reason === "exit" && outcome.exitCode === 0 ? "success" : "error";
|
||||
@@ -530,7 +536,15 @@ export async function spawnStreamingEncoder(
|
||||
const encoder: StreamingEncoder = {
|
||||
writeFrame: async (buffer: Buffer): Promise<boolean> => {
|
||||
const stdin = ffmpeg.stdin;
|
||||
if (exitStatus !== "running" || !stdin || stdin.destroyed) {
|
||||
if (exitStatus !== "running") {
|
||||
return false;
|
||||
}
|
||||
if (!stdin || stdin.destroyed) {
|
||||
// The OS can close the pipe (EPIPE) before Node delivers the child
|
||||
// process `close` event. Wait for the shared exit settlement so the
|
||||
// caller can synchronously inspect getExitFailureReason() instead of
|
||||
// losing a host-interruption signal in this narrow race.
|
||||
await exitPromise;
|
||||
return false;
|
||||
}
|
||||
// Copy the buffer before writing — Node streams hold a reference to the
|
||||
@@ -604,6 +618,14 @@ export async function spawnStreamingEncoder(
|
||||
durationMs,
|
||||
fileSize: 0,
|
||||
error: `${formatFfmpegError(exitCode, stderr)}${inactivitySuffix}`,
|
||||
failureReason: isExternalFfmpegInterruption({
|
||||
exitCode,
|
||||
signal: exitSignal,
|
||||
stderr,
|
||||
terminationReason,
|
||||
})
|
||||
? "external_interruption"
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -618,6 +640,18 @@ export async function spawnStreamingEncoder(
|
||||
if (exitStatus !== "error") return undefined;
|
||||
return formatFfmpegError(exitCode, stderr);
|
||||
},
|
||||
|
||||
getExitFailureReason: () => {
|
||||
if (exitStatus !== "error") return undefined;
|
||||
return isExternalFfmpegInterruption({
|
||||
exitCode,
|
||||
signal: exitSignal,
|
||||
stderr,
|
||||
terminationReason,
|
||||
})
|
||||
? "external_interruption"
|
||||
: undefined;
|
||||
},
|
||||
};
|
||||
|
||||
return encoder;
|
||||
|
||||
@@ -260,6 +260,7 @@ export interface ExtractionPhaseBreakdown {
|
||||
|
||||
export type VideoExtractionFailureKind =
|
||||
| "cancelled"
|
||||
| "external_interruption"
|
||||
| "source_missing"
|
||||
| "source_rejected"
|
||||
| "download_not_found"
|
||||
@@ -785,6 +786,14 @@ export async function extractVideoFramesRange(
|
||||
args.push("-y", outputPattern);
|
||||
|
||||
const processResult = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
|
||||
if (processResult.failureReason === "external_interruption") {
|
||||
throw new VideoSourceExtractionError(
|
||||
"external_interruption",
|
||||
true,
|
||||
"Video frame extraction interrupted by host lifecycle",
|
||||
`FFmpeg exited with code ${processResult.exitCode}: ${processResult.stderr.slice(-500)}`,
|
||||
);
|
||||
}
|
||||
if (processResult.terminationReason === "abort") {
|
||||
throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
|
||||
}
|
||||
|
||||
@@ -2,7 +2,65 @@ import { EventEmitter } from "node:events";
|
||||
import { resolve } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { formatFfmpegError } from "./runFfmpeg.js";
|
||||
import { formatFfmpegError, isExternalFfmpegInterruption } from "./runFfmpeg.js";
|
||||
|
||||
describe("isExternalFfmpegInterruption", () => {
|
||||
const base = {
|
||||
exitCode: 255,
|
||||
signal: null,
|
||||
stderr: "",
|
||||
terminationReason: "exit" as const,
|
||||
};
|
||||
|
||||
it("recognizes a direct external termination signal", () => {
|
||||
expect(isExternalFfmpegInterruption({ ...base, exitCode: null, signal: "SIGTERM" })).toBe(true);
|
||||
});
|
||||
|
||||
it("does not broaden the retry signal to SIGKILL", () => {
|
||||
expect(isExternalFfmpegInterruption({ ...base, exitCode: null, signal: "SIGKILL" })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("recognizes ffmpeg's handled SIGTERM exit-255 signature", () => {
|
||||
expect(
|
||||
isExternalFfmpegInterruption({
|
||||
...base,
|
||||
stderr: "frame= 42\nExiting normally, received signal 15.\n",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not classify an ordinary exit 255", () => {
|
||||
expect(isExternalFfmpegInterruption({ ...base, stderr: "Encoder initialization failed" })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("requires exit 255 when classification relies on ffmpeg stderr", () => {
|
||||
expect(
|
||||
isExternalFfmpegInterruption({
|
||||
...base,
|
||||
exitCode: 1,
|
||||
stderr: "Exiting normally, received signal 15.",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["abort", "deadline", "inactivity"] as const)(
|
||||
"keeps a managed %s termination non-retryable",
|
||||
(terminationReason) => {
|
||||
expect(
|
||||
isExternalFfmpegInterruption({
|
||||
...base,
|
||||
signal: "SIGTERM",
|
||||
stderr: "Exiting normally, received signal 15.",
|
||||
terminationReason,
|
||||
}),
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("formatFfmpegError", () => {
|
||||
const originalPlatform = process.platform;
|
||||
|
||||
@@ -23,12 +23,32 @@ export interface RunFfmpegOptions {
|
||||
export interface RunFfmpegResult {
|
||||
success: boolean;
|
||||
exitCode: number | null;
|
||||
signal?: NodeJS.Signals | null;
|
||||
stderr: string;
|
||||
durationMs: number;
|
||||
terminationReason: ManagedProcessTerminationReason;
|
||||
failureReason?: "external_interruption";
|
||||
error?: Error;
|
||||
}
|
||||
|
||||
const FFMPEG_SIGTERM_EXIT_LINE = /^Exiting normally, received signal 15\.?\r?$/m;
|
||||
|
||||
/**
|
||||
* Return true only when ffmpeg was terminated from outside this managed call.
|
||||
*
|
||||
* FFmpeg handles SIGTERM itself and can therefore report exit code 255 with a
|
||||
* null Node signal. The exact terminal stderr line covers that case. Managed
|
||||
* abort/deadline/inactivity reasons always take precedence so our own SIGTERM
|
||||
* requests never become retryable lifecycle interruptions.
|
||||
*/
|
||||
export function isExternalFfmpegInterruption(
|
||||
result: Pick<RunFfmpegResult, "exitCode" | "signal" | "stderr" | "terminationReason">,
|
||||
): boolean {
|
||||
if (result.terminationReason !== "exit" || result.exitCode === 0) return false;
|
||||
if (result.signal === "SIGTERM") return true;
|
||||
return result.exitCode === 255 && FFMPEG_SIGTERM_EXIT_LINE.test(result.stderr);
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT = 300_000;
|
||||
|
||||
const DEFAULT_STDERR_TAIL_LINES = 15;
|
||||
@@ -104,12 +124,17 @@ export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promis
|
||||
onStderr: opts?.onStderr,
|
||||
});
|
||||
const outcome = await managed.wait();
|
||||
return {
|
||||
const result: RunFfmpegResult = {
|
||||
success: outcome.reason === "exit" && outcome.exitCode === 0,
|
||||
exitCode: outcome.exitCode,
|
||||
signal: outcome.signal,
|
||||
stderr: outcome.stderr,
|
||||
durationMs: outcome.durationMs,
|
||||
terminationReason: outcome.reason,
|
||||
error: outcome.error,
|
||||
};
|
||||
if (isExternalFfmpegInterruption(result)) {
|
||||
result.failureReason = "external_interruption";
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { extractSafeRenderErrorCode, extractSafeRenderErrorMetadata } from "./server.js";
|
||||
import { VideoExtractionStageError } from "./services/render/stages/extractVideosStage.js";
|
||||
import { AssetMediaTypeMismatchError } from "./services/assetMediaType.js";
|
||||
import { EncoderInterruptedError } from "./services/render/encoderInterruption.js";
|
||||
|
||||
describe("extractSafeRenderErrorCode", () => {
|
||||
it("preserves allowlisted typed extraction codes", () => {
|
||||
@@ -40,6 +41,16 @@ describe("extractSafeRenderErrorCode", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("transports the bounded encoder interruption contract", () => {
|
||||
const error = new EncoderInterruptedError("Encoding failed", "private ffmpeg stderr");
|
||||
expect(extractSafeRenderErrorMetadata(error)).toEqual({
|
||||
errorCode: "ENCODER_INTERRUPTED",
|
||||
errorOwner: "system",
|
||||
retryable: true,
|
||||
});
|
||||
expect(error.message).not.toContain("private ffmpeg stderr");
|
||||
});
|
||||
|
||||
it("does not forward arbitrary codes or parse message text", () => {
|
||||
expect(extractSafeRenderErrorCode({ code: "INTERNAL_ERROR" })).toBeUndefined();
|
||||
expect(
|
||||
|
||||
@@ -125,6 +125,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
|
||||
"INVALID_VIDEO_METADATA",
|
||||
"VIDEO_SOURCE_UNRENDERABLE",
|
||||
"VIDEO_EXTRACTION_FAILED",
|
||||
"ENCODER_INTERRUPTED",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -14,6 +14,7 @@ import { parseHTML } from "linkedom";
|
||||
import { parseAnimatedGifMetadata, type AnimatedGifMetadata } from "@hyperframes/core";
|
||||
import { DEFAULT_VP9_CPU_USED, runFfmpeg } from "@hyperframes/engine";
|
||||
import { isHttpUrl } from "../utils/urlDownloader.js";
|
||||
import { encoderFailureError } from "./render/encoderInterruption.js";
|
||||
|
||||
const PREPARED_GIF_SUBDIR = "_animated_gif";
|
||||
const CACHE_SCHEMA = "hfgif-v1";
|
||||
@@ -249,6 +250,12 @@ async function runAnimatedGifTranscode(request: AnimatedGifTranscodeRequest): Pr
|
||||
const timeout = request.timeoutMs ?? 300_000;
|
||||
const result = await runFfmpeg(request.args, { timeout });
|
||||
if (result.success) return;
|
||||
if (result.failureReason === "external_interruption") {
|
||||
throw encoderFailureError("Animated GIF transcode failed", {
|
||||
error: `exit ${result.exitCode}: ${result.stderr.slice(-500)}`,
|
||||
failureReason: result.failureReason,
|
||||
});
|
||||
}
|
||||
if (result.terminationReason === "deadline") {
|
||||
throw new Error(`Animated GIF transcode timed out after ${timeout}ms`);
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ import { fpsToFfmpegArg } from "@hyperframes/core";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
import { formatExportFrameName } from "../../utils/paths.js";
|
||||
import { padOrTrimAudioToVideoFrameCount } from "../render/audioPadTrim.js";
|
||||
import { encoderFailureError } from "../render/encoderInterruption.js";
|
||||
import type { ChunkSliceJson } from "../render/stages/freezePlan.js";
|
||||
import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";
|
||||
import { validatePlanV2MaterializedTarget } from "./planV2.js";
|
||||
@@ -183,10 +184,10 @@ export async function assemble(
|
||||
remuxArgs.push("-y", concatOutputPath);
|
||||
const remuxResult = await runFfmpeg(remuxArgs, { signal: abortSignal });
|
||||
if (!remuxResult.success) {
|
||||
throw new Error(
|
||||
`[assemble] ffmpeg single-chunk remux failed (exit ${remuxResult.exitCode}): ` +
|
||||
`${remuxResult.stderr.slice(-400)}`,
|
||||
);
|
||||
throw encoderFailureError("[assemble] ffmpeg single-chunk remux failed", {
|
||||
error: `exit ${remuxResult.exitCode}: ${remuxResult.stderr.slice(-400)}`,
|
||||
failureReason: remuxResult.failureReason,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Concat list file — one `file '<path>'` per chunk, in order. ffmpeg's
|
||||
@@ -218,10 +219,10 @@ export async function assemble(
|
||||
concatArgs.push("-y", concatOutputPath);
|
||||
const concatResult = await runFfmpeg(concatArgs, { signal: abortSignal });
|
||||
if (!concatResult.success) {
|
||||
throw new Error(
|
||||
`[assemble] ffmpeg concat-copy failed (exit ${concatResult.exitCode}): ` +
|
||||
`${concatResult.stderr.slice(-400)}`,
|
||||
);
|
||||
throw encoderFailureError("[assemble] ffmpeg concat-copy failed", {
|
||||
error: `exit ${concatResult.exitCode}: ${concatResult.stderr.slice(-400)}`,
|
||||
failureReason: concatResult.failureReason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,10 +289,10 @@ export async function assemble(
|
||||
cfrArgs.push("-y", cfrOutputPath);
|
||||
const cfrResult = await runFfmpeg(cfrArgs, { signal: abortSignal });
|
||||
if (!cfrResult.success) {
|
||||
throw new Error(
|
||||
`[assemble] ffmpeg cfr re-encode failed (exit ${cfrResult.exitCode}): ` +
|
||||
`${cfrResult.stderr.slice(-400)}`,
|
||||
);
|
||||
throw encoderFailureError("[assemble] ffmpeg cfr re-encode failed", {
|
||||
error: `exit ${cfrResult.exitCode}: ${cfrResult.stderr.slice(-400)}`,
|
||||
failureReason: cfrResult.failureReason,
|
||||
});
|
||||
}
|
||||
postConcatPath = cfrOutputPath;
|
||||
log.info("[assemble] cfr re-encode applied", {
|
||||
@@ -312,7 +313,7 @@ export async function assemble(
|
||||
signal: abortSignal,
|
||||
});
|
||||
if (!padTrimResult.success) {
|
||||
throw new Error(`[assemble] audio pad/trim failed: ${padTrimResult.error}`);
|
||||
throw encoderFailureError("[assemble] audio pad/trim failed", padTrimResult);
|
||||
}
|
||||
normalizedAudioPath = paddedAudioPath;
|
||||
log.info("[assemble] audio normalized for mux", {
|
||||
@@ -342,7 +343,7 @@ export async function assemble(
|
||||
{ num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
|
||||
);
|
||||
if (!muxResult.success) {
|
||||
throw new Error(`[assemble] audio mux failed: ${muxResult.error}`);
|
||||
throw encoderFailureError("[assemble] audio mux failed", muxResult);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -359,7 +360,7 @@ export async function assemble(
|
||||
},
|
||||
);
|
||||
if (!faststartResult.success) {
|
||||
throw new Error(`[assemble] faststart failed: ${faststartResult.error}`);
|
||||
throw encoderFailureError("[assemble] faststart failed", faststartResult);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
|
||||
@@ -115,7 +115,11 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
|
||||
function harness(opts: {
|
||||
video: ProbeVideoFrameInfo | "throw";
|
||||
audio: AudioProbeInfo | "throw";
|
||||
ffmpeg?: (args: string[]) => Promise<{ success: boolean; error?: string }>;
|
||||
ffmpeg?: (args: string[]) => Promise<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}>;
|
||||
}): { input: PadTrimAudioInput; captured: { args: string[][] } } {
|
||||
const captured = { args: [] as string[][] };
|
||||
const input: PadTrimAudioInput = {
|
||||
@@ -269,6 +273,25 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
|
||||
expect(result.operation).toBe("pad");
|
||||
expect(result.targetDurationSeconds).toBe(6);
|
||||
});
|
||||
|
||||
it("preserves an external interruption from the audio pad/trim ffmpeg pass", async () => {
|
||||
const { input } = harness({
|
||||
video: { frameCount: 180, fpsNum: 30, fpsDen: 1 },
|
||||
audio: { durationSeconds: 5.0 },
|
||||
ffmpeg: async () => ({
|
||||
success: false,
|
||||
error: "synthetic interruption diagnostics",
|
||||
failureReason: "external_interruption",
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await padOrTrimAudioToVideoFrameCount(input);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── Public-path path redaction ────────────────────────────────────────────
|
||||
|
||||
@@ -73,7 +73,11 @@ export interface PadTrimAudioInput {
|
||||
*/
|
||||
probeVideoFrameInfo?: (videoPath: string) => Promise<ProbeVideoFrameInfo>;
|
||||
probeAudioInfo?: (audioPath: string, signal?: AbortSignal) => Promise<AudioProbeInfo>;
|
||||
runFfmpeg?: (args: string[]) => Promise<{ success: boolean; error?: string }>;
|
||||
runFfmpeg?: (args: string[]) => Promise<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}>;
|
||||
}
|
||||
|
||||
export type PadTrimOperation = "pad" | "trim" | "copy";
|
||||
@@ -89,6 +93,8 @@ export interface PadTrimAudioResult {
|
||||
operation: PadTrimOperation;
|
||||
/** Populated only when `success === false`. */
|
||||
error?: string;
|
||||
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
|
||||
failureReason?: "external_interruption";
|
||||
}
|
||||
|
||||
export type PadTrimAudioStepKind = "copy" | "trim" | "normalize";
|
||||
@@ -322,6 +328,7 @@ export async function padOrTrimAudioToVideoFrameCount(
|
||||
sourceDurationSeconds: audioInfo.durationSeconds,
|
||||
operation: plan.operation,
|
||||
error: ffmpegResult.error,
|
||||
failureReason: ffmpegResult.failureReason,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -457,12 +464,17 @@ async function defaultProbeAudioInfo(
|
||||
async function defaultRunFfmpeg(
|
||||
args: string[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ success: boolean; error?: string }> {
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}> {
|
||||
const result = await runFfmpeg(args, { signal });
|
||||
if (result.success) return { success: true };
|
||||
return {
|
||||
success: false,
|
||||
error: `[audioPadTrim] ${formatFfmpegError(result.exitCode, result.stderr)}`,
|
||||
failureReason: result.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { normalizeErrorMessage } from "../../utils/errorMessage.js";
|
||||
import { CaptureFailure, classifyCaptureFailure } from "@hyperframes/engine";
|
||||
import { EncoderInterruptedError } from "./encoderInterruption.js";
|
||||
|
||||
export class CaptureStageError extends CaptureFailure {
|
||||
readonly browserConsole: string[];
|
||||
@@ -20,8 +21,11 @@ export class CaptureStageError extends CaptureFailure {
|
||||
}
|
||||
}
|
||||
|
||||
export function wrapCaptureStageError(error: unknown, browserConsole: string[]): CaptureStageError {
|
||||
if (error instanceof CaptureStageError) return error;
|
||||
export function wrapCaptureStageError(
|
||||
error: unknown,
|
||||
browserConsole: string[],
|
||||
): CaptureStageError | EncoderInterruptedError {
|
||||
if (error instanceof CaptureStageError || error instanceof EncoderInterruptedError) return error;
|
||||
return new CaptureStageError({ cause: error, browserConsole });
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
const ENCODER_INTERRUPTED = "ENCODER_INTERRUPTED" as const;
|
||||
|
||||
interface EncoderFailureResult {
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}
|
||||
|
||||
/** A host/process lifecycle interruption that is safe to retry on a fresh producer. */
|
||||
export class EncoderInterruptedError extends Error {
|
||||
readonly code = ENCODER_INTERRUPTED;
|
||||
readonly owner = "system" as const;
|
||||
readonly retryable = true as const;
|
||||
|
||||
/** Retained inside the producer for diagnostics; never copied to the wire envelope. */
|
||||
readonly diagnosticMessage: string;
|
||||
|
||||
constructor(prefix: string, diagnosticMessage: string) {
|
||||
super(`${prefix}: encoder process was interrupted by the host lifecycle`);
|
||||
this.name = "EncoderInterruptedError";
|
||||
this.diagnosticMessage = diagnosticMessage;
|
||||
}
|
||||
}
|
||||
|
||||
export function encoderFailureError(prefix: string, result: EncoderFailureResult): Error {
|
||||
const message = `${prefix}: ${result.error ?? "unknown encoder failure"}`;
|
||||
return result.failureReason === "external_interruption"
|
||||
? new EncoderInterruptedError(prefix, message)
|
||||
: new Error(message);
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { CaptureStageError, getCaptureStageBrowserConsole } from "./captureStageError.js";
|
||||
import {
|
||||
CaptureStageError,
|
||||
getCaptureStageBrowserConsole,
|
||||
wrapCaptureStageError,
|
||||
} from "./captureStageError.js";
|
||||
import { EncoderInterruptedError } from "./encoderInterruption.js";
|
||||
import {
|
||||
computeCompositionObservabilityHash,
|
||||
observeRenderStage,
|
||||
@@ -82,6 +87,11 @@ describe("sanitizeObservationMessage", () => {
|
||||
});
|
||||
|
||||
describe("CaptureStageError", () => {
|
||||
it("preserves a typed encoder interruption through capture wrapping", () => {
|
||||
const cause = new EncoderInterruptedError("Streaming encode failed", "private stderr");
|
||||
expect(wrapCaptureStageError(cause, ["console output"])).toBe(cause);
|
||||
});
|
||||
|
||||
it("preserves the original message and browser console diagnostics", () => {
|
||||
const cause = new Error("Navigation timeout of 60000 ms exceeded");
|
||||
const browserConsole = ["[FrameCapture:ERROR] page.goto failed"];
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AssembleStageInput } from "./assembleStage.js";
|
||||
|
||||
const { muxVideoWithAudioMock, padOrTrimAudioMock } = vi.hoisted(() => ({
|
||||
const { applyFaststartMock, muxVideoWithAudioMock, padOrTrimAudioMock } = vi.hoisted(() => ({
|
||||
applyFaststartMock: vi.fn(),
|
||||
muxVideoWithAudioMock: vi.fn(),
|
||||
padOrTrimAudioMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@hyperframes/engine", () => ({
|
||||
applyFaststart: vi.fn(),
|
||||
applyFaststart: applyFaststartMock,
|
||||
muxVideoWithAudio: muxVideoWithAudioMock,
|
||||
}));
|
||||
|
||||
@@ -20,6 +21,7 @@ vi.mock("../shared.js", () => ({
|
||||
}));
|
||||
|
||||
import { runAssembleStage } from "./assembleStage.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
function makeInput(overrides: Partial<AssembleStageInput> = {}): AssembleStageInput {
|
||||
return {
|
||||
@@ -44,8 +46,10 @@ function makeInput(overrides: Partial<AssembleStageInput> = {}): AssembleStageIn
|
||||
|
||||
describe("runAssembleStage audio duration parity", () => {
|
||||
beforeEach(() => {
|
||||
applyFaststartMock.mockReset();
|
||||
muxVideoWithAudioMock.mockReset();
|
||||
padOrTrimAudioMock.mockReset();
|
||||
applyFaststartMock.mockResolvedValue({ success: true });
|
||||
muxVideoWithAudioMock.mockResolvedValue({ success: true });
|
||||
padOrTrimAudioMock.mockResolvedValue({
|
||||
success: true,
|
||||
@@ -63,6 +67,7 @@ describe("runAssembleStage audio duration parity", () => {
|
||||
videoPath: "/tmp/video-only.mp4",
|
||||
audioPath: "/tmp/audio.m4a",
|
||||
outputPath: "/tmp/audio.duration-normalized.m4a",
|
||||
signal: undefined,
|
||||
});
|
||||
expect(muxVideoWithAudioMock).toHaveBeenCalledWith(
|
||||
"/tmp/video-only.mp4",
|
||||
@@ -81,6 +86,7 @@ describe("runAssembleStage audio duration parity", () => {
|
||||
videoPath: "/tmp/video-only.mp4",
|
||||
audioPath: "/tmp/audio.m4a",
|
||||
outputPath: "/tmp/audio.duration-normalized.m4a",
|
||||
signal: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -99,4 +105,36 @@ describe("runAssembleStage audio duration parity", () => {
|
||||
);
|
||||
expect(muxVideoWithAudioMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves an external interruption from final audio mux", async () => {
|
||||
muxVideoWithAudioMock.mockResolvedValue({
|
||||
success: false,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
|
||||
await expect(runAssembleStage(makeInput())).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("preserves an external interruption from audio duration normalization", async () => {
|
||||
padOrTrimAudioMock.mockResolvedValue({
|
||||
success: false,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
|
||||
await expect(runAssembleStage(makeInput())).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("preserves an external interruption from MP4 faststart", async () => {
|
||||
applyFaststartMock.mockResolvedValue({
|
||||
success: false,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
|
||||
await expect(runAssembleStage(makeInput({ hasAudio: false }))).rejects.toBeInstanceOf(
|
||||
EncoderInterruptedError,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@ import { applyFaststart, muxVideoWithAudio } from "@hyperframes/engine";
|
||||
import { extname } from "node:path";
|
||||
import type { ProgressCallback, RenderJob } from "../../renderOrchestrator.js";
|
||||
import { padOrTrimAudioToVideoFrameCount } from "../audioPadTrim.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
import { updateJobStatus } from "../shared.js";
|
||||
|
||||
export interface AssembleStageInput {
|
||||
@@ -66,10 +67,11 @@ export async function runAssembleStage(input: AssembleStageInput): Promise<Assem
|
||||
videoPath: videoOnlyPath,
|
||||
audioPath: audioOutputPath,
|
||||
outputPath: normalizedAudioPath,
|
||||
signal: abortSignal,
|
||||
});
|
||||
assertNotAborted();
|
||||
if (!normalizeResult.success) {
|
||||
throw new Error(`Audio duration normalization failed: ${normalizeResult.error}`);
|
||||
throw encoderFailureError("Audio duration normalization failed", normalizeResult);
|
||||
}
|
||||
const muxResult = await muxVideoWithAudio(
|
||||
videoOnlyPath,
|
||||
@@ -83,7 +85,7 @@ export async function runAssembleStage(input: AssembleStageInput): Promise<Assem
|
||||
);
|
||||
assertNotAborted();
|
||||
if (!muxResult.success) {
|
||||
throw new Error(`Audio muxing failed: ${muxResult.error}`);
|
||||
throw encoderFailureError("Audio muxing failed", muxResult);
|
||||
}
|
||||
} else {
|
||||
const faststartResult = await applyFaststart(
|
||||
@@ -95,7 +97,7 @@ export async function runAssembleStage(input: AssembleStageInput): Promise<Assem
|
||||
);
|
||||
assertNotAborted();
|
||||
if (!faststartResult.success) {
|
||||
throw new Error(`Faststart failed: ${faststartResult.error}`);
|
||||
throw encoderFailureError("Faststart failed", faststartResult);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
|
||||
});
|
||||
|
||||
import { runAudioStage } from "./audioStage.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
// Regression: hasAudio flipping to false used to be indistinguishable from
|
||||
// "no audio was authored" — processCompositionAudio's error (per-element
|
||||
@@ -114,6 +115,26 @@ describe("runAudioStage", () => {
|
||||
expect(result.audioError).toBeUndefined();
|
||||
});
|
||||
|
||||
it("throws a structured retry signal when audio ffmpeg is externally interrupted", async () => {
|
||||
processCompositionAudioMock.mockResolvedValue({
|
||||
success: false,
|
||||
outputPath: "audio.m4a",
|
||||
durationMs: 1,
|
||||
tracksProcessed: 0,
|
||||
failures: [
|
||||
{
|
||||
stage: "mix",
|
||||
reason: "external_interruption",
|
||||
owner: "system",
|
||||
retryable: true,
|
||||
detail: "ffmpeg handled signal 15",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(runAudioStage(makeInput())).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("does not set audioError when there is no audio to mix", async () => {
|
||||
const result = await runAudioStage(makeInput({ audios: [] }));
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
} from "@hyperframes/engine";
|
||||
import type { CompositionMetadata } from "../shared.js";
|
||||
import type { ProducerLogger } from "../../../logger.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
export interface AudioStageInput {
|
||||
projectDir: string;
|
||||
@@ -130,6 +131,16 @@ export async function runAudioStage(input: AudioStageInput): Promise<AudioStageR
|
||||
}
|
||||
assertNotAborted();
|
||||
|
||||
const interrupted = audioResult.failures?.find(
|
||||
(failure) => failure.reason === "external_interruption",
|
||||
);
|
||||
if (interrupted) {
|
||||
throw encoderFailureError("Audio processing failed", {
|
||||
error: interrupted.detail,
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
}
|
||||
|
||||
hasAudio = audioResult.success;
|
||||
audioFailures = audioResult.failures;
|
||||
// processCompositionAudio's error (per-element failures or the mix's own
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
timeHdrPhase,
|
||||
timeHdrPhaseAsync,
|
||||
} from "../hdrPerf.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
// ─── Hybrid path gating + partitioning ─────────────────────────────────────
|
||||
|
||||
@@ -363,12 +364,18 @@ export async function captureTransitionFrameOnWorker(
|
||||
export function ensureFrameWritten(
|
||||
frameWritten: boolean,
|
||||
frameIndex: number,
|
||||
encoder?: { getExitError: () => string | undefined },
|
||||
encoder?: {
|
||||
getExitError: () => string | undefined;
|
||||
getExitFailureReason?: () => "external_interruption" | undefined;
|
||||
},
|
||||
): void {
|
||||
if (frameWritten) return;
|
||||
const reason = encoder?.getExitError();
|
||||
const base = `Streaming encoder exited before frame ${frameIndex} was written`;
|
||||
throw new Error(reason ? `${base}: ${reason}` : base);
|
||||
throw encoderFailureError(base, {
|
||||
error: reason,
|
||||
failureReason: encoder?.getExitFailureReason?.(),
|
||||
});
|
||||
}
|
||||
|
||||
// ─── HDR video raw-frame cleanup (sequential path only) ────────────────────
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
resolveHdrExtractionWindow,
|
||||
} from "./captureHdrResources.js";
|
||||
import type { CompositionMetadata } from "../shared.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
@@ -488,6 +489,26 @@ describe("reserveHdrExtractionBytes", () => {
|
||||
});
|
||||
|
||||
describe("extractHdrVideoFrames", () => {
|
||||
it("preserves an external FFmpeg interruption as the structured retry signal", async () => {
|
||||
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-interrupted-"));
|
||||
const fixture = hdrExtractionFixture([hdrVideo("interrupted")], framesDir);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
extractHdrVideoFrames({
|
||||
...fixture,
|
||||
runFfmpegImpl: async () => ({
|
||||
...ffmpegResult(false),
|
||||
exitCode: 255,
|
||||
failureReason: "external_interruption",
|
||||
}),
|
||||
}),
|
||||
).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
} finally {
|
||||
rmSync(framesDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("pins FFmpeg seek/duration, raw frame count, and reservation lifetime", async () => {
|
||||
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-extract-"));
|
||||
const video = hdrVideo("preroll", { start: -60, end: 120, mediaStart: 0 });
|
||||
|
||||
@@ -52,6 +52,7 @@ import {
|
||||
} from "../../hdrCompositor.js";
|
||||
import type { HdrDiagnostics, RenderJob } from "../../renderOrchestrator.js";
|
||||
import type { CompositionMetadata } from "../shared.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
const NO_FOLLOW_FLAG = constants.O_NOFOLLOW ?? 0;
|
||||
|
||||
@@ -496,10 +497,10 @@ export async function extractHdrVideoFrames(args: {
|
||||
srcPath,
|
||||
stderr: result.stderr.slice(-400),
|
||||
});
|
||||
throw new Error(
|
||||
`HDR frame extraction failed for video "${videoId}". ` +
|
||||
`Aborting render to avoid shipping black HDR layers.`,
|
||||
);
|
||||
throw encoderFailureError(`HDR frame extraction failed for video "${videoId}"`, {
|
||||
error: `Aborting render to avoid shipping black HDR layers: ${result.stderr.slice(-400)}`,
|
||||
failureReason: result.failureReason,
|
||||
});
|
||||
}
|
||||
const frameSize = dims.width * dims.height * 6;
|
||||
const fd = openSync(rawPath, constants.O_RDONLY | NO_FOLLOW_FLAG);
|
||||
|
||||
@@ -76,6 +76,7 @@ import { partitionTransitionFrames, shouldUseHybridLayeredPath } from "./capture
|
||||
import { runSequentialLayeredFrameLoop } from "./captureHdrSequentialLoop.js";
|
||||
import { runHybridLayeredFrameLoop } from "./captureHdrHybridLoop.js";
|
||||
import { wrapCaptureStageError } from "../captureStageError.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
import type { HdrLayeredCapturePlan } from "../capturePlan.js";
|
||||
|
||||
export interface CaptureHdrStageInput {
|
||||
@@ -431,7 +432,7 @@ export async function runCaptureHdrStage(
|
||||
hdrEncoderClosed = true;
|
||||
assertNotAborted();
|
||||
if (!hdrEncodeResult.success) {
|
||||
throw new Error(`HDR encode failed: ${hdrEncodeResult.error}`);
|
||||
throw encoderFailureError("HDR encode failed", hdrEncodeResult);
|
||||
}
|
||||
captureDurationMs = Date.now() - stageStart;
|
||||
encodeMs = hdrEncodeResult.durationMs;
|
||||
|
||||
@@ -77,6 +77,7 @@ import { wrapCaptureStageError } from "../captureStageError.js";
|
||||
import { pushWorkerDedupPerfs } from "../perfSummary.js";
|
||||
import { ensureFrameWritten } from "./captureHdrFrameShared.js";
|
||||
import { updateJobStatus } from "../shared.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
import type { SdrStreamingCapturePlan } from "../capturePlan.js";
|
||||
|
||||
/**
|
||||
@@ -867,7 +868,7 @@ export async function runCaptureStreamingStage(
|
||||
assertNotAborted();
|
||||
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Streaming encode failed: ${encodeResult.error}`);
|
||||
throw encoderFailureError("Streaming encode failed", encodeResult);
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -171,6 +171,76 @@ describe("gif encode args", () => {
|
||||
});
|
||||
|
||||
describe("runEncodeStage config plumbing", () => {
|
||||
it("throws a typed retryable error when the GIF encoder is externally interrupted", async () => {
|
||||
const { EncoderInterruptedError } = await import("../encoderInterruption.js");
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
runFfmpegMock.mockImplementationOnce(async () => ({
|
||||
success: false,
|
||||
exitCode: 255,
|
||||
stderr: "Exiting normally, received signal 15.\nprivate stderr",
|
||||
durationMs: 1,
|
||||
failureReason: "external_interruption" as const,
|
||||
}));
|
||||
const paths = createFramesDir("jpg");
|
||||
|
||||
try {
|
||||
await runEncodeStage(
|
||||
makeInput({
|
||||
framesDir: paths.framesDir,
|
||||
outputPath: join(paths.root, "out.gif"),
|
||||
videoOnlyPath: join(paths.root, "video-only.mp4"),
|
||||
isGif: true,
|
||||
}),
|
||||
);
|
||||
throw new Error("expected runEncodeStage to reject");
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(EncoderInterruptedError);
|
||||
expect(String(error)).not.toContain("private stderr");
|
||||
}
|
||||
});
|
||||
|
||||
it("throws a typed retryable error for an external encoder interruption", async () => {
|
||||
const { EncoderInterruptedError } = await import("../encoderInterruption.js");
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
encodeFramesFromDirMock.mockImplementationOnce(async (_framesDir, _pattern, outputPath) => ({
|
||||
success: false,
|
||||
outputPath,
|
||||
durationMs: 12,
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption" as const,
|
||||
}));
|
||||
|
||||
try {
|
||||
await runEncodeStage(makeInput());
|
||||
throw new Error("expected runEncodeStage to reject");
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(EncoderInterruptedError);
|
||||
expect(error).toMatchObject({
|
||||
code: "ENCODER_INTERRUPTED",
|
||||
owner: "system",
|
||||
retryable: true,
|
||||
});
|
||||
expect(String(error)).not.toContain("private stderr");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps a generic exit 255 untyped", async () => {
|
||||
const { EncoderInterruptedError } = await import("../encoderInterruption.js");
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
encodeFramesFromDirMock.mockImplementationOnce(async (_framesDir, _pattern, outputPath) => ({
|
||||
success: false,
|
||||
outputPath,
|
||||
durationMs: 12,
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: "FFmpeg exited with code 255: invalid encoder settings",
|
||||
}));
|
||||
|
||||
await expect(runEncodeStage(makeInput())).rejects.not.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("scales the encode timeout for long compositions", async () => {
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
type GifEncodeArgsInput,
|
||||
} from "./gifEncodeArgs.js";
|
||||
import { updateJobStatus } from "../shared.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
export interface EncodeStageInput {
|
||||
job: RenderJob;
|
||||
@@ -165,6 +166,7 @@ async function encodeGifFromDir(
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: formatFfmpegError(paletteResult.exitCode, paletteResult.stderr),
|
||||
failureReason: paletteResult.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -180,6 +182,7 @@ async function encodeGifFromDir(
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: formatFfmpegError(gifResult.exitCode, gifResult.stderr),
|
||||
failureReason: gifResult.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -276,7 +279,7 @@ export async function runEncodeStage(input: EncodeStageInput): Promise<EncodeSta
|
||||
});
|
||||
assertNotAborted();
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Encoding failed: ${encodeResult.error}`);
|
||||
throw encoderFailureError("Encoding failed", encodeResult);
|
||||
}
|
||||
return { encodeMs: Date.now() - stage5Start };
|
||||
}
|
||||
@@ -335,7 +338,7 @@ export async function runEncodeStage(input: EncodeStageInput): Promise<EncodeSta
|
||||
assertNotAborted();
|
||||
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Encoding failed: ${encodeResult.error}`);
|
||||
throw encoderFailureError("Encoding failed", encodeResult);
|
||||
}
|
||||
|
||||
return { encodeMs: Date.now() - stage5Start };
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
shouldCopyExtractedFrames,
|
||||
VideoExtractionStageError,
|
||||
} from "./extractVideosStage.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
function makeVideo(overrides: Partial<VideoElement> = {}): VideoElement {
|
||||
return {
|
||||
@@ -78,6 +79,21 @@ function extractionResult(errors: VideoExtractionFailure[]): ExtractionResult {
|
||||
};
|
||||
}
|
||||
|
||||
describe("encoder interruption classification", () => {
|
||||
it("preserves an external ffmpeg interruption as the structured retry signal", () => {
|
||||
const result = extractionResult([
|
||||
{
|
||||
videoId: "v1",
|
||||
kind: "external_interruption",
|
||||
retryable: true,
|
||||
error: "ffmpeg handled signal 15",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(() => assertVideoExtractionSucceeded(result)).toThrow(EncoderInterruptedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendAutoDetectedVideoAudio", () => {
|
||||
it("adds audio for an audible video whose file has an audio track", () => {
|
||||
const composition = { videos: [makeVideo()], audios: [] as never[] };
|
||||
|
||||
@@ -55,6 +55,7 @@ import {
|
||||
} from "../../renderOrchestrator.js";
|
||||
import { materializeExtractedFramesForCompiledDir, type CompositionMetadata } from "../shared.js";
|
||||
import type { ProducerLogger } from "../../../logger.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
export interface ExtractVideosStageInput {
|
||||
projectDir: string;
|
||||
@@ -169,10 +170,20 @@ export class VideoExtractionStageError extends Error {
|
||||
}
|
||||
|
||||
export function assertVideoExtractionSucceeded(result: ExtractionResult): void {
|
||||
throwIfEncoderInterrupted(result);
|
||||
const error = buildVideoExtractionStageError(result);
|
||||
if (error) throw error;
|
||||
}
|
||||
|
||||
function throwIfEncoderInterrupted(result: ExtractionResult): void {
|
||||
const interrupted = result.errors.find((failure) => failure.kind === "external_interruption");
|
||||
if (!interrupted) return;
|
||||
throw encoderFailureError("Video frame extraction failed", {
|
||||
error: String(interrupted.error),
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
}
|
||||
|
||||
function buildVideoExtractionStageError(
|
||||
result: ExtractionResult,
|
||||
): VideoExtractionStageError | null {
|
||||
@@ -226,6 +237,16 @@ function throwHdrProbeFailures(
|
||||
mode: VideoExtractionFailureMode,
|
||||
): void {
|
||||
if (failures.length === 0) return;
|
||||
const interrupted = failures.find(
|
||||
(failure) => failure.classified.kind === "external_interruption",
|
||||
);
|
||||
if (interrupted) {
|
||||
throw encoderFailureError("Video HDR probe failed", {
|
||||
error:
|
||||
interrupted.error instanceof Error ? interrupted.error.message : String(interrupted.error),
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
}
|
||||
if (mode === "enforce") {
|
||||
throw buildHdrProbeStageError(failures.map((failure) => failure.classified));
|
||||
}
|
||||
@@ -413,6 +434,7 @@ export async function runExtractVideosStage(
|
||||
extractionResult.phaseBreakdown.transientRetries =
|
||||
(extractionResult.phaseBreakdown.transientRetries ?? 0) + hdrProbeTransientRetries;
|
||||
assertNotAborted();
|
||||
throwIfEncoderInterrupted(extractionResult);
|
||||
failureToEnforce = applyVideoExtractionFailurePolicy(extractionResult, extractionPolicy, log);
|
||||
|
||||
materializeExtractedFramesForCompiledDir(extractionResult.extracted, compiledDir, {
|
||||
|
||||
@@ -2563,6 +2563,18 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("never hides an encoder host interruption behind the same-host pinned fallback", () => {
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
isEncoderInterrupted: true,
|
||||
deWorkerInversion: "inverted",
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldStreamParallelCapture (non-DE parallel streaming router)", () => {
|
||||
|
||||
@@ -111,6 +111,7 @@ import {
|
||||
import { createMemorySampler, type MemorySampler, updateJobStatus } from "./render/shared.js";
|
||||
import { buildRenderErrorDetails } from "./render/cleanup.js";
|
||||
import { publishRenderFailure } from "./render/renderEventPublisher.js";
|
||||
import { EncoderInterruptedError } from "./render/encoderInterruption.js";
|
||||
import { RenderExecutionContext } from "./render/renderExecutionContext.js";
|
||||
import { ArtifactTransaction } from "./render/artifactTransaction.js";
|
||||
import {
|
||||
@@ -1842,10 +1843,11 @@ export function resolveParallelRouterRetryPlan(args: {
|
||||
export function shouldRetryViaPinnedFallback(args: {
|
||||
isVerifyError: boolean;
|
||||
isCancellation: boolean;
|
||||
isEncoderInterrupted?: boolean;
|
||||
deWorkerInversion: "inverted" | "reverted" | undefined;
|
||||
deParallelRouter: "routed" | "reverted" | undefined;
|
||||
}): boolean {
|
||||
if (args.isCancellation) return false;
|
||||
if (args.isCancellation || args.isEncoderInterrupted) return false;
|
||||
if (args.isVerifyError) return true;
|
||||
return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed";
|
||||
}
|
||||
@@ -3601,6 +3603,7 @@ async function executeRenderPipeline(input: {
|
||||
!shouldRetryViaPinnedFallback({
|
||||
isVerifyError,
|
||||
isCancellation,
|
||||
isEncoderInterrupted: err instanceof EncoderInterruptedError,
|
||||
deWorkerInversion,
|
||||
deParallelRouter,
|
||||
})
|
||||
@@ -4053,6 +4056,12 @@ async function executeRenderPipeline(input: {
|
||||
? error
|
||||
: new RenderCancelledError("render_cancelled");
|
||||
}
|
||||
if (error instanceof EncoderInterruptedError) {
|
||||
log.warn("[Render] encoder process interrupted by host lifecycle", {
|
||||
code: error.code,
|
||||
diagnostic: error.diagnosticMessage.slice(-2_000),
|
||||
});
|
||||
}
|
||||
const memoryGuidance = describeMemoryExhaustion(error, {
|
||||
width: captureCompositionWidth,
|
||||
height: captureCompositionHeight,
|
||||
|
||||
Reference in New Issue
Block a user