mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-04 16:42:27 +00:00
fix(encoder): signal host interruptions for retry (#3578)
* fix(encoder): signal host interruptions for retry * fix(encoder): cover all render interruption paths * fix(encoder): classify HDR pre-extraction drains
This commit is contained in:
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { extractSafeRenderErrorCode, extractSafeRenderErrorMetadata } from "./server.js";
|
||||
import { VideoExtractionStageError } from "./services/render/stages/extractVideosStage.js";
|
||||
import { AssetMediaTypeMismatchError } from "./services/assetMediaType.js";
|
||||
import { EncoderInterruptedError } from "./services/render/encoderInterruption.js";
|
||||
|
||||
describe("extractSafeRenderErrorCode", () => {
|
||||
it("preserves allowlisted typed extraction codes", () => {
|
||||
@@ -40,6 +41,16 @@ describe("extractSafeRenderErrorCode", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("transports the bounded encoder interruption contract", () => {
|
||||
const error = new EncoderInterruptedError("Encoding failed", "private ffmpeg stderr");
|
||||
expect(extractSafeRenderErrorMetadata(error)).toEqual({
|
||||
errorCode: "ENCODER_INTERRUPTED",
|
||||
errorOwner: "system",
|
||||
retryable: true,
|
||||
});
|
||||
expect(error.message).not.toContain("private ffmpeg stderr");
|
||||
});
|
||||
|
||||
it("does not forward arbitrary codes or parse message text", () => {
|
||||
expect(extractSafeRenderErrorCode({ code: "INTERNAL_ERROR" })).toBeUndefined();
|
||||
expect(
|
||||
|
||||
@@ -125,6 +125,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
|
||||
"INVALID_VIDEO_METADATA",
|
||||
"VIDEO_SOURCE_UNRENDERABLE",
|
||||
"VIDEO_EXTRACTION_FAILED",
|
||||
"ENCODER_INTERRUPTED",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -14,6 +14,7 @@ import { parseHTML } from "linkedom";
|
||||
import { parseAnimatedGifMetadata, type AnimatedGifMetadata } from "@hyperframes/core";
|
||||
import { DEFAULT_VP9_CPU_USED, runFfmpeg } from "@hyperframes/engine";
|
||||
import { isHttpUrl } from "../utils/urlDownloader.js";
|
||||
import { encoderFailureError } from "./render/encoderInterruption.js";
|
||||
|
||||
const PREPARED_GIF_SUBDIR = "_animated_gif";
|
||||
const CACHE_SCHEMA = "hfgif-v1";
|
||||
@@ -249,6 +250,12 @@ async function runAnimatedGifTranscode(request: AnimatedGifTranscodeRequest): Pr
|
||||
const timeout = request.timeoutMs ?? 300_000;
|
||||
const result = await runFfmpeg(request.args, { timeout });
|
||||
if (result.success) return;
|
||||
if (result.failureReason === "external_interruption") {
|
||||
throw encoderFailureError("Animated GIF transcode failed", {
|
||||
error: `exit ${result.exitCode}: ${result.stderr.slice(-500)}`,
|
||||
failureReason: result.failureReason,
|
||||
});
|
||||
}
|
||||
if (result.terminationReason === "deadline") {
|
||||
throw new Error(`Animated GIF transcode timed out after ${timeout}ms`);
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ import { fpsToFfmpegArg } from "@hyperframes/core";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
import { formatExportFrameName } from "../../utils/paths.js";
|
||||
import { padOrTrimAudioToVideoFrameCount } from "../render/audioPadTrim.js";
|
||||
import { encoderFailureError } from "../render/encoderInterruption.js";
|
||||
import type { ChunkSliceJson } from "../render/stages/freezePlan.js";
|
||||
import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";
|
||||
import { validatePlanV2MaterializedTarget } from "./planV2.js";
|
||||
@@ -183,10 +184,10 @@ export async function assemble(
|
||||
remuxArgs.push("-y", concatOutputPath);
|
||||
const remuxResult = await runFfmpeg(remuxArgs, { signal: abortSignal });
|
||||
if (!remuxResult.success) {
|
||||
throw new Error(
|
||||
`[assemble] ffmpeg single-chunk remux failed (exit ${remuxResult.exitCode}): ` +
|
||||
`${remuxResult.stderr.slice(-400)}`,
|
||||
);
|
||||
throw encoderFailureError("[assemble] ffmpeg single-chunk remux failed", {
|
||||
error: `exit ${remuxResult.exitCode}: ${remuxResult.stderr.slice(-400)}`,
|
||||
failureReason: remuxResult.failureReason,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Concat list file — one `file '<path>'` per chunk, in order. ffmpeg's
|
||||
@@ -218,10 +219,10 @@ export async function assemble(
|
||||
concatArgs.push("-y", concatOutputPath);
|
||||
const concatResult = await runFfmpeg(concatArgs, { signal: abortSignal });
|
||||
if (!concatResult.success) {
|
||||
throw new Error(
|
||||
`[assemble] ffmpeg concat-copy failed (exit ${concatResult.exitCode}): ` +
|
||||
`${concatResult.stderr.slice(-400)}`,
|
||||
);
|
||||
throw encoderFailureError("[assemble] ffmpeg concat-copy failed", {
|
||||
error: `exit ${concatResult.exitCode}: ${concatResult.stderr.slice(-400)}`,
|
||||
failureReason: concatResult.failureReason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,10 +289,10 @@ export async function assemble(
|
||||
cfrArgs.push("-y", cfrOutputPath);
|
||||
const cfrResult = await runFfmpeg(cfrArgs, { signal: abortSignal });
|
||||
if (!cfrResult.success) {
|
||||
throw new Error(
|
||||
`[assemble] ffmpeg cfr re-encode failed (exit ${cfrResult.exitCode}): ` +
|
||||
`${cfrResult.stderr.slice(-400)}`,
|
||||
);
|
||||
throw encoderFailureError("[assemble] ffmpeg cfr re-encode failed", {
|
||||
error: `exit ${cfrResult.exitCode}: ${cfrResult.stderr.slice(-400)}`,
|
||||
failureReason: cfrResult.failureReason,
|
||||
});
|
||||
}
|
||||
postConcatPath = cfrOutputPath;
|
||||
log.info("[assemble] cfr re-encode applied", {
|
||||
@@ -312,7 +313,7 @@ export async function assemble(
|
||||
signal: abortSignal,
|
||||
});
|
||||
if (!padTrimResult.success) {
|
||||
throw new Error(`[assemble] audio pad/trim failed: ${padTrimResult.error}`);
|
||||
throw encoderFailureError("[assemble] audio pad/trim failed", padTrimResult);
|
||||
}
|
||||
normalizedAudioPath = paddedAudioPath;
|
||||
log.info("[assemble] audio normalized for mux", {
|
||||
@@ -342,7 +343,7 @@ export async function assemble(
|
||||
{ num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
|
||||
);
|
||||
if (!muxResult.success) {
|
||||
throw new Error(`[assemble] audio mux failed: ${muxResult.error}`);
|
||||
throw encoderFailureError("[assemble] audio mux failed", muxResult);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -359,7 +360,7 @@ export async function assemble(
|
||||
},
|
||||
);
|
||||
if (!faststartResult.success) {
|
||||
throw new Error(`[assemble] faststart failed: ${faststartResult.error}`);
|
||||
throw encoderFailureError("[assemble] faststart failed", faststartResult);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
|
||||
@@ -115,7 +115,11 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
|
||||
function harness(opts: {
|
||||
video: ProbeVideoFrameInfo | "throw";
|
||||
audio: AudioProbeInfo | "throw";
|
||||
ffmpeg?: (args: string[]) => Promise<{ success: boolean; error?: string }>;
|
||||
ffmpeg?: (args: string[]) => Promise<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}>;
|
||||
}): { input: PadTrimAudioInput; captured: { args: string[][] } } {
|
||||
const captured = { args: [] as string[][] };
|
||||
const input: PadTrimAudioInput = {
|
||||
@@ -269,6 +273,25 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
|
||||
expect(result.operation).toBe("pad");
|
||||
expect(result.targetDurationSeconds).toBe(6);
|
||||
});
|
||||
|
||||
it("preserves an external interruption from the audio pad/trim ffmpeg pass", async () => {
|
||||
const { input } = harness({
|
||||
video: { frameCount: 180, fpsNum: 30, fpsDen: 1 },
|
||||
audio: { durationSeconds: 5.0 },
|
||||
ffmpeg: async () => ({
|
||||
success: false,
|
||||
error: "synthetic interruption diagnostics",
|
||||
failureReason: "external_interruption",
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await padOrTrimAudioToVideoFrameCount(input);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── Public-path path redaction ────────────────────────────────────────────
|
||||
|
||||
@@ -73,7 +73,11 @@ export interface PadTrimAudioInput {
|
||||
*/
|
||||
probeVideoFrameInfo?: (videoPath: string) => Promise<ProbeVideoFrameInfo>;
|
||||
probeAudioInfo?: (audioPath: string, signal?: AbortSignal) => Promise<AudioProbeInfo>;
|
||||
runFfmpeg?: (args: string[]) => Promise<{ success: boolean; error?: string }>;
|
||||
runFfmpeg?: (args: string[]) => Promise<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}>;
|
||||
}
|
||||
|
||||
export type PadTrimOperation = "pad" | "trim" | "copy";
|
||||
@@ -89,6 +93,8 @@ export interface PadTrimAudioResult {
|
||||
operation: PadTrimOperation;
|
||||
/** Populated only when `success === false`. */
|
||||
error?: string;
|
||||
/** Stable machine-readable cause for failures safe to retry on a fresh host. */
|
||||
failureReason?: "external_interruption";
|
||||
}
|
||||
|
||||
export type PadTrimAudioStepKind = "copy" | "trim" | "normalize";
|
||||
@@ -322,6 +328,7 @@ export async function padOrTrimAudioToVideoFrameCount(
|
||||
sourceDurationSeconds: audioInfo.durationSeconds,
|
||||
operation: plan.operation,
|
||||
error: ffmpegResult.error,
|
||||
failureReason: ffmpegResult.failureReason,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -457,12 +464,17 @@ async function defaultProbeAudioInfo(
|
||||
async function defaultRunFfmpeg(
|
||||
args: string[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ success: boolean; error?: string }> {
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}> {
|
||||
const result = await runFfmpeg(args, { signal });
|
||||
if (result.success) return { success: true };
|
||||
return {
|
||||
success: false,
|
||||
error: `[audioPadTrim] ${formatFfmpegError(result.exitCode, result.stderr)}`,
|
||||
failureReason: result.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { normalizeErrorMessage } from "../../utils/errorMessage.js";
|
||||
import { CaptureFailure, classifyCaptureFailure } from "@hyperframes/engine";
|
||||
import { EncoderInterruptedError } from "./encoderInterruption.js";
|
||||
|
||||
export class CaptureStageError extends CaptureFailure {
|
||||
readonly browserConsole: string[];
|
||||
@@ -20,8 +21,11 @@ export class CaptureStageError extends CaptureFailure {
|
||||
}
|
||||
}
|
||||
|
||||
export function wrapCaptureStageError(error: unknown, browserConsole: string[]): CaptureStageError {
|
||||
if (error instanceof CaptureStageError) return error;
|
||||
export function wrapCaptureStageError(
|
||||
error: unknown,
|
||||
browserConsole: string[],
|
||||
): CaptureStageError | EncoderInterruptedError {
|
||||
if (error instanceof CaptureStageError || error instanceof EncoderInterruptedError) return error;
|
||||
return new CaptureStageError({ cause: error, browserConsole });
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
const ENCODER_INTERRUPTED = "ENCODER_INTERRUPTED" as const;
|
||||
|
||||
interface EncoderFailureResult {
|
||||
error?: string;
|
||||
failureReason?: "external_interruption";
|
||||
}
|
||||
|
||||
/** A host/process lifecycle interruption that is safe to retry on a fresh producer. */
|
||||
export class EncoderInterruptedError extends Error {
|
||||
readonly code = ENCODER_INTERRUPTED;
|
||||
readonly owner = "system" as const;
|
||||
readonly retryable = true as const;
|
||||
|
||||
/** Retained inside the producer for diagnostics; never copied to the wire envelope. */
|
||||
readonly diagnosticMessage: string;
|
||||
|
||||
constructor(prefix: string, diagnosticMessage: string) {
|
||||
super(`${prefix}: encoder process was interrupted by the host lifecycle`);
|
||||
this.name = "EncoderInterruptedError";
|
||||
this.diagnosticMessage = diagnosticMessage;
|
||||
}
|
||||
}
|
||||
|
||||
export function encoderFailureError(prefix: string, result: EncoderFailureResult): Error {
|
||||
const message = `${prefix}: ${result.error ?? "unknown encoder failure"}`;
|
||||
return result.failureReason === "external_interruption"
|
||||
? new EncoderInterruptedError(prefix, message)
|
||||
: new Error(message);
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { CaptureStageError, getCaptureStageBrowserConsole } from "./captureStageError.js";
|
||||
import {
|
||||
CaptureStageError,
|
||||
getCaptureStageBrowserConsole,
|
||||
wrapCaptureStageError,
|
||||
} from "./captureStageError.js";
|
||||
import { EncoderInterruptedError } from "./encoderInterruption.js";
|
||||
import {
|
||||
computeCompositionObservabilityHash,
|
||||
observeRenderStage,
|
||||
@@ -82,6 +87,11 @@ describe("sanitizeObservationMessage", () => {
|
||||
});
|
||||
|
||||
describe("CaptureStageError", () => {
|
||||
it("preserves a typed encoder interruption through capture wrapping", () => {
|
||||
const cause = new EncoderInterruptedError("Streaming encode failed", "private stderr");
|
||||
expect(wrapCaptureStageError(cause, ["console output"])).toBe(cause);
|
||||
});
|
||||
|
||||
it("preserves the original message and browser console diagnostics", () => {
|
||||
const cause = new Error("Navigation timeout of 60000 ms exceeded");
|
||||
const browserConsole = ["[FrameCapture:ERROR] page.goto failed"];
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AssembleStageInput } from "./assembleStage.js";
|
||||
|
||||
const { muxVideoWithAudioMock, padOrTrimAudioMock } = vi.hoisted(() => ({
|
||||
const { applyFaststartMock, muxVideoWithAudioMock, padOrTrimAudioMock } = vi.hoisted(() => ({
|
||||
applyFaststartMock: vi.fn(),
|
||||
muxVideoWithAudioMock: vi.fn(),
|
||||
padOrTrimAudioMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@hyperframes/engine", () => ({
|
||||
applyFaststart: vi.fn(),
|
||||
applyFaststart: applyFaststartMock,
|
||||
muxVideoWithAudio: muxVideoWithAudioMock,
|
||||
}));
|
||||
|
||||
@@ -20,6 +21,7 @@ vi.mock("../shared.js", () => ({
|
||||
}));
|
||||
|
||||
import { runAssembleStage } from "./assembleStage.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
function makeInput(overrides: Partial<AssembleStageInput> = {}): AssembleStageInput {
|
||||
return {
|
||||
@@ -44,8 +46,10 @@ function makeInput(overrides: Partial<AssembleStageInput> = {}): AssembleStageIn
|
||||
|
||||
describe("runAssembleStage audio duration parity", () => {
|
||||
beforeEach(() => {
|
||||
applyFaststartMock.mockReset();
|
||||
muxVideoWithAudioMock.mockReset();
|
||||
padOrTrimAudioMock.mockReset();
|
||||
applyFaststartMock.mockResolvedValue({ success: true });
|
||||
muxVideoWithAudioMock.mockResolvedValue({ success: true });
|
||||
padOrTrimAudioMock.mockResolvedValue({
|
||||
success: true,
|
||||
@@ -63,6 +67,7 @@ describe("runAssembleStage audio duration parity", () => {
|
||||
videoPath: "/tmp/video-only.mp4",
|
||||
audioPath: "/tmp/audio.m4a",
|
||||
outputPath: "/tmp/audio.duration-normalized.m4a",
|
||||
signal: undefined,
|
||||
});
|
||||
expect(muxVideoWithAudioMock).toHaveBeenCalledWith(
|
||||
"/tmp/video-only.mp4",
|
||||
@@ -81,6 +86,7 @@ describe("runAssembleStage audio duration parity", () => {
|
||||
videoPath: "/tmp/video-only.mp4",
|
||||
audioPath: "/tmp/audio.m4a",
|
||||
outputPath: "/tmp/audio.duration-normalized.m4a",
|
||||
signal: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -99,4 +105,36 @@ describe("runAssembleStage audio duration parity", () => {
|
||||
);
|
||||
expect(muxVideoWithAudioMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves an external interruption from final audio mux", async () => {
|
||||
muxVideoWithAudioMock.mockResolvedValue({
|
||||
success: false,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
|
||||
await expect(runAssembleStage(makeInput())).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("preserves an external interruption from audio duration normalization", async () => {
|
||||
padOrTrimAudioMock.mockResolvedValue({
|
||||
success: false,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
|
||||
await expect(runAssembleStage(makeInput())).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("preserves an external interruption from MP4 faststart", async () => {
|
||||
applyFaststartMock.mockResolvedValue({
|
||||
success: false,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
|
||||
await expect(runAssembleStage(makeInput({ hasAudio: false }))).rejects.toBeInstanceOf(
|
||||
EncoderInterruptedError,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@ import { applyFaststart, muxVideoWithAudio } from "@hyperframes/engine";
|
||||
import { extname } from "node:path";
|
||||
import type { ProgressCallback, RenderJob } from "../../renderOrchestrator.js";
|
||||
import { padOrTrimAudioToVideoFrameCount } from "../audioPadTrim.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
import { updateJobStatus } from "../shared.js";
|
||||
|
||||
export interface AssembleStageInput {
|
||||
@@ -66,10 +67,11 @@ export async function runAssembleStage(input: AssembleStageInput): Promise<Assem
|
||||
videoPath: videoOnlyPath,
|
||||
audioPath: audioOutputPath,
|
||||
outputPath: normalizedAudioPath,
|
||||
signal: abortSignal,
|
||||
});
|
||||
assertNotAborted();
|
||||
if (!normalizeResult.success) {
|
||||
throw new Error(`Audio duration normalization failed: ${normalizeResult.error}`);
|
||||
throw encoderFailureError("Audio duration normalization failed", normalizeResult);
|
||||
}
|
||||
const muxResult = await muxVideoWithAudio(
|
||||
videoOnlyPath,
|
||||
@@ -83,7 +85,7 @@ export async function runAssembleStage(input: AssembleStageInput): Promise<Assem
|
||||
);
|
||||
assertNotAborted();
|
||||
if (!muxResult.success) {
|
||||
throw new Error(`Audio muxing failed: ${muxResult.error}`);
|
||||
throw encoderFailureError("Audio muxing failed", muxResult);
|
||||
}
|
||||
} else {
|
||||
const faststartResult = await applyFaststart(
|
||||
@@ -95,7 +97,7 @@ export async function runAssembleStage(input: AssembleStageInput): Promise<Assem
|
||||
);
|
||||
assertNotAborted();
|
||||
if (!faststartResult.success) {
|
||||
throw new Error(`Faststart failed: ${faststartResult.error}`);
|
||||
throw encoderFailureError("Faststart failed", faststartResult);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
|
||||
});
|
||||
|
||||
import { runAudioStage } from "./audioStage.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
// Regression: hasAudio flipping to false used to be indistinguishable from
|
||||
// "no audio was authored" — processCompositionAudio's error (per-element
|
||||
@@ -114,6 +115,26 @@ describe("runAudioStage", () => {
|
||||
expect(result.audioError).toBeUndefined();
|
||||
});
|
||||
|
||||
it("throws a structured retry signal when audio ffmpeg is externally interrupted", async () => {
|
||||
processCompositionAudioMock.mockResolvedValue({
|
||||
success: false,
|
||||
outputPath: "audio.m4a",
|
||||
durationMs: 1,
|
||||
tracksProcessed: 0,
|
||||
failures: [
|
||||
{
|
||||
stage: "mix",
|
||||
reason: "external_interruption",
|
||||
owner: "system",
|
||||
retryable: true,
|
||||
detail: "ffmpeg handled signal 15",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(runAudioStage(makeInput())).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("does not set audioError when there is no audio to mix", async () => {
|
||||
const result = await runAudioStage(makeInput({ audios: [] }));
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
} from "@hyperframes/engine";
|
||||
import type { CompositionMetadata } from "../shared.js";
|
||||
import type { ProducerLogger } from "../../../logger.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
export interface AudioStageInput {
|
||||
projectDir: string;
|
||||
@@ -130,6 +131,16 @@ export async function runAudioStage(input: AudioStageInput): Promise<AudioStageR
|
||||
}
|
||||
assertNotAborted();
|
||||
|
||||
const interrupted = audioResult.failures?.find(
|
||||
(failure) => failure.reason === "external_interruption",
|
||||
);
|
||||
if (interrupted) {
|
||||
throw encoderFailureError("Audio processing failed", {
|
||||
error: interrupted.detail,
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
}
|
||||
|
||||
hasAudio = audioResult.success;
|
||||
audioFailures = audioResult.failures;
|
||||
// processCompositionAudio's error (per-element failures or the mix's own
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
timeHdrPhase,
|
||||
timeHdrPhaseAsync,
|
||||
} from "../hdrPerf.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
// ─── Hybrid path gating + partitioning ─────────────────────────────────────
|
||||
|
||||
@@ -363,12 +364,18 @@ export async function captureTransitionFrameOnWorker(
|
||||
export function ensureFrameWritten(
|
||||
frameWritten: boolean,
|
||||
frameIndex: number,
|
||||
encoder?: { getExitError: () => string | undefined },
|
||||
encoder?: {
|
||||
getExitError: () => string | undefined;
|
||||
getExitFailureReason?: () => "external_interruption" | undefined;
|
||||
},
|
||||
): void {
|
||||
if (frameWritten) return;
|
||||
const reason = encoder?.getExitError();
|
||||
const base = `Streaming encoder exited before frame ${frameIndex} was written`;
|
||||
throw new Error(reason ? `${base}: ${reason}` : base);
|
||||
throw encoderFailureError(base, {
|
||||
error: reason,
|
||||
failureReason: encoder?.getExitFailureReason?.(),
|
||||
});
|
||||
}
|
||||
|
||||
// ─── HDR video raw-frame cleanup (sequential path only) ────────────────────
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
resolveHdrExtractionWindow,
|
||||
} from "./captureHdrResources.js";
|
||||
import type { CompositionMetadata } from "../shared.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
@@ -488,6 +489,26 @@ describe("reserveHdrExtractionBytes", () => {
|
||||
});
|
||||
|
||||
describe("extractHdrVideoFrames", () => {
|
||||
it("preserves an external FFmpeg interruption as the structured retry signal", async () => {
|
||||
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-interrupted-"));
|
||||
const fixture = hdrExtractionFixture([hdrVideo("interrupted")], framesDir);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
extractHdrVideoFrames({
|
||||
...fixture,
|
||||
runFfmpegImpl: async () => ({
|
||||
...ffmpegResult(false),
|
||||
exitCode: 255,
|
||||
failureReason: "external_interruption",
|
||||
}),
|
||||
}),
|
||||
).rejects.toBeInstanceOf(EncoderInterruptedError);
|
||||
} finally {
|
||||
rmSync(framesDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("pins FFmpeg seek/duration, raw frame count, and reservation lifetime", async () => {
|
||||
const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-extract-"));
|
||||
const video = hdrVideo("preroll", { start: -60, end: 120, mediaStart: 0 });
|
||||
|
||||
@@ -52,6 +52,7 @@ import {
|
||||
} from "../../hdrCompositor.js";
|
||||
import type { HdrDiagnostics, RenderJob } from "../../renderOrchestrator.js";
|
||||
import type { CompositionMetadata } from "../shared.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
const NO_FOLLOW_FLAG = constants.O_NOFOLLOW ?? 0;
|
||||
|
||||
@@ -496,10 +497,10 @@ export async function extractHdrVideoFrames(args: {
|
||||
srcPath,
|
||||
stderr: result.stderr.slice(-400),
|
||||
});
|
||||
throw new Error(
|
||||
`HDR frame extraction failed for video "${videoId}". ` +
|
||||
`Aborting render to avoid shipping black HDR layers.`,
|
||||
);
|
||||
throw encoderFailureError(`HDR frame extraction failed for video "${videoId}"`, {
|
||||
error: `Aborting render to avoid shipping black HDR layers: ${result.stderr.slice(-400)}`,
|
||||
failureReason: result.failureReason,
|
||||
});
|
||||
}
|
||||
const frameSize = dims.width * dims.height * 6;
|
||||
const fd = openSync(rawPath, constants.O_RDONLY | NO_FOLLOW_FLAG);
|
||||
|
||||
@@ -76,6 +76,7 @@ import { partitionTransitionFrames, shouldUseHybridLayeredPath } from "./capture
|
||||
import { runSequentialLayeredFrameLoop } from "./captureHdrSequentialLoop.js";
|
||||
import { runHybridLayeredFrameLoop } from "./captureHdrHybridLoop.js";
|
||||
import { wrapCaptureStageError } from "../captureStageError.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
import type { HdrLayeredCapturePlan } from "../capturePlan.js";
|
||||
|
||||
export interface CaptureHdrStageInput {
|
||||
@@ -431,7 +432,7 @@ export async function runCaptureHdrStage(
|
||||
hdrEncoderClosed = true;
|
||||
assertNotAborted();
|
||||
if (!hdrEncodeResult.success) {
|
||||
throw new Error(`HDR encode failed: ${hdrEncodeResult.error}`);
|
||||
throw encoderFailureError("HDR encode failed", hdrEncodeResult);
|
||||
}
|
||||
captureDurationMs = Date.now() - stageStart;
|
||||
encodeMs = hdrEncodeResult.durationMs;
|
||||
|
||||
@@ -77,6 +77,7 @@ import { wrapCaptureStageError } from "../captureStageError.js";
|
||||
import { pushWorkerDedupPerfs } from "../perfSummary.js";
|
||||
import { ensureFrameWritten } from "./captureHdrFrameShared.js";
|
||||
import { updateJobStatus } from "../shared.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
import type { SdrStreamingCapturePlan } from "../capturePlan.js";
|
||||
|
||||
/**
|
||||
@@ -867,7 +868,7 @@ export async function runCaptureStreamingStage(
|
||||
assertNotAborted();
|
||||
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Streaming encode failed: ${encodeResult.error}`);
|
||||
throw encoderFailureError("Streaming encode failed", encodeResult);
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -171,6 +171,76 @@ describe("gif encode args", () => {
|
||||
});
|
||||
|
||||
describe("runEncodeStage config plumbing", () => {
|
||||
it("throws a typed retryable error when the GIF encoder is externally interrupted", async () => {
|
||||
const { EncoderInterruptedError } = await import("../encoderInterruption.js");
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
runFfmpegMock.mockImplementationOnce(async () => ({
|
||||
success: false,
|
||||
exitCode: 255,
|
||||
stderr: "Exiting normally, received signal 15.\nprivate stderr",
|
||||
durationMs: 1,
|
||||
failureReason: "external_interruption" as const,
|
||||
}));
|
||||
const paths = createFramesDir("jpg");
|
||||
|
||||
try {
|
||||
await runEncodeStage(
|
||||
makeInput({
|
||||
framesDir: paths.framesDir,
|
||||
outputPath: join(paths.root, "out.gif"),
|
||||
videoOnlyPath: join(paths.root, "video-only.mp4"),
|
||||
isGif: true,
|
||||
}),
|
||||
);
|
||||
throw new Error("expected runEncodeStage to reject");
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(EncoderInterruptedError);
|
||||
expect(String(error)).not.toContain("private stderr");
|
||||
}
|
||||
});
|
||||
|
||||
it("throws a typed retryable error for an external encoder interruption", async () => {
|
||||
const { EncoderInterruptedError } = await import("../encoderInterruption.js");
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
encodeFramesFromDirMock.mockImplementationOnce(async (_framesDir, _pattern, outputPath) => ({
|
||||
success: false,
|
||||
outputPath,
|
||||
durationMs: 12,
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: "FFmpeg exited with code 255\nprivate stderr",
|
||||
failureReason: "external_interruption" as const,
|
||||
}));
|
||||
|
||||
try {
|
||||
await runEncodeStage(makeInput());
|
||||
throw new Error("expected runEncodeStage to reject");
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(EncoderInterruptedError);
|
||||
expect(error).toMatchObject({
|
||||
code: "ENCODER_INTERRUPTED",
|
||||
owner: "system",
|
||||
retryable: true,
|
||||
});
|
||||
expect(String(error)).not.toContain("private stderr");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps a generic exit 255 untyped", async () => {
|
||||
const { EncoderInterruptedError } = await import("../encoderInterruption.js");
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
encodeFramesFromDirMock.mockImplementationOnce(async (_framesDir, _pattern, outputPath) => ({
|
||||
success: false,
|
||||
outputPath,
|
||||
durationMs: 12,
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: "FFmpeg exited with code 255: invalid encoder settings",
|
||||
}));
|
||||
|
||||
await expect(runEncodeStage(makeInput())).rejects.not.toBeInstanceOf(EncoderInterruptedError);
|
||||
});
|
||||
|
||||
it("scales the encode timeout for long compositions", async () => {
|
||||
const { runEncodeStage } = await import("./encodeStage.js");
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
type GifEncodeArgsInput,
|
||||
} from "./gifEncodeArgs.js";
|
||||
import { updateJobStatus } from "../shared.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
export interface EncodeStageInput {
|
||||
job: RenderJob;
|
||||
@@ -165,6 +166,7 @@ async function encodeGifFromDir(
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: formatFfmpegError(paletteResult.exitCode, paletteResult.stderr),
|
||||
failureReason: paletteResult.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -180,6 +182,7 @@ async function encodeGifFromDir(
|
||||
framesEncoded: 0,
|
||||
fileSize: 0,
|
||||
error: formatFfmpegError(gifResult.exitCode, gifResult.stderr),
|
||||
failureReason: gifResult.failureReason,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -276,7 +279,7 @@ export async function runEncodeStage(input: EncodeStageInput): Promise<EncodeSta
|
||||
});
|
||||
assertNotAborted();
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Encoding failed: ${encodeResult.error}`);
|
||||
throw encoderFailureError("Encoding failed", encodeResult);
|
||||
}
|
||||
return { encodeMs: Date.now() - stage5Start };
|
||||
}
|
||||
@@ -335,7 +338,7 @@ export async function runEncodeStage(input: EncodeStageInput): Promise<EncodeSta
|
||||
assertNotAborted();
|
||||
|
||||
if (!encodeResult.success) {
|
||||
throw new Error(`Encoding failed: ${encodeResult.error}`);
|
||||
throw encoderFailureError("Encoding failed", encodeResult);
|
||||
}
|
||||
|
||||
return { encodeMs: Date.now() - stage5Start };
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
shouldCopyExtractedFrames,
|
||||
VideoExtractionStageError,
|
||||
} from "./extractVideosStage.js";
|
||||
import { EncoderInterruptedError } from "../encoderInterruption.js";
|
||||
|
||||
function makeVideo(overrides: Partial<VideoElement> = {}): VideoElement {
|
||||
return {
|
||||
@@ -78,6 +79,21 @@ function extractionResult(errors: VideoExtractionFailure[]): ExtractionResult {
|
||||
};
|
||||
}
|
||||
|
||||
describe("encoder interruption classification", () => {
|
||||
it("preserves an external ffmpeg interruption as the structured retry signal", () => {
|
||||
const result = extractionResult([
|
||||
{
|
||||
videoId: "v1",
|
||||
kind: "external_interruption",
|
||||
retryable: true,
|
||||
error: "ffmpeg handled signal 15",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(() => assertVideoExtractionSucceeded(result)).toThrow(EncoderInterruptedError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendAutoDetectedVideoAudio", () => {
|
||||
it("adds audio for an audible video whose file has an audio track", () => {
|
||||
const composition = { videos: [makeVideo()], audios: [] as never[] };
|
||||
|
||||
@@ -55,6 +55,7 @@ import {
|
||||
} from "../../renderOrchestrator.js";
|
||||
import { materializeExtractedFramesForCompiledDir, type CompositionMetadata } from "../shared.js";
|
||||
import type { ProducerLogger } from "../../../logger.js";
|
||||
import { encoderFailureError } from "../encoderInterruption.js";
|
||||
|
||||
export interface ExtractVideosStageInput {
|
||||
projectDir: string;
|
||||
@@ -169,10 +170,20 @@ export class VideoExtractionStageError extends Error {
|
||||
}
|
||||
|
||||
export function assertVideoExtractionSucceeded(result: ExtractionResult): void {
|
||||
throwIfEncoderInterrupted(result);
|
||||
const error = buildVideoExtractionStageError(result);
|
||||
if (error) throw error;
|
||||
}
|
||||
|
||||
function throwIfEncoderInterrupted(result: ExtractionResult): void {
|
||||
const interrupted = result.errors.find((failure) => failure.kind === "external_interruption");
|
||||
if (!interrupted) return;
|
||||
throw encoderFailureError("Video frame extraction failed", {
|
||||
error: String(interrupted.error),
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
}
|
||||
|
||||
function buildVideoExtractionStageError(
|
||||
result: ExtractionResult,
|
||||
): VideoExtractionStageError | null {
|
||||
@@ -226,6 +237,16 @@ function throwHdrProbeFailures(
|
||||
mode: VideoExtractionFailureMode,
|
||||
): void {
|
||||
if (failures.length === 0) return;
|
||||
const interrupted = failures.find(
|
||||
(failure) => failure.classified.kind === "external_interruption",
|
||||
);
|
||||
if (interrupted) {
|
||||
throw encoderFailureError("Video HDR probe failed", {
|
||||
error:
|
||||
interrupted.error instanceof Error ? interrupted.error.message : String(interrupted.error),
|
||||
failureReason: "external_interruption",
|
||||
});
|
||||
}
|
||||
if (mode === "enforce") {
|
||||
throw buildHdrProbeStageError(failures.map((failure) => failure.classified));
|
||||
}
|
||||
@@ -413,6 +434,7 @@ export async function runExtractVideosStage(
|
||||
extractionResult.phaseBreakdown.transientRetries =
|
||||
(extractionResult.phaseBreakdown.transientRetries ?? 0) + hdrProbeTransientRetries;
|
||||
assertNotAborted();
|
||||
throwIfEncoderInterrupted(extractionResult);
|
||||
failureToEnforce = applyVideoExtractionFailurePolicy(extractionResult, extractionPolicy, log);
|
||||
|
||||
materializeExtractedFramesForCompiledDir(extractionResult.extracted, compiledDir, {
|
||||
|
||||
@@ -2563,6 +2563,18 @@ describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic c
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("never hides an encoder host interruption behind the same-host pinned fallback", () => {
|
||||
expect(
|
||||
shouldRetryViaPinnedFallback({
|
||||
isVerifyError: false,
|
||||
isCancellation: false,
|
||||
isEncoderInterrupted: true,
|
||||
deWorkerInversion: "inverted",
|
||||
deParallelRouter: undefined,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldStreamParallelCapture (non-DE parallel streaming router)", () => {
|
||||
|
||||
@@ -111,6 +111,7 @@ import {
|
||||
import { createMemorySampler, type MemorySampler, updateJobStatus } from "./render/shared.js";
|
||||
import { buildRenderErrorDetails } from "./render/cleanup.js";
|
||||
import { publishRenderFailure } from "./render/renderEventPublisher.js";
|
||||
import { EncoderInterruptedError } from "./render/encoderInterruption.js";
|
||||
import { RenderExecutionContext } from "./render/renderExecutionContext.js";
|
||||
import { ArtifactTransaction } from "./render/artifactTransaction.js";
|
||||
import {
|
||||
@@ -1842,10 +1843,11 @@ export function resolveParallelRouterRetryPlan(args: {
|
||||
export function shouldRetryViaPinnedFallback(args: {
|
||||
isVerifyError: boolean;
|
||||
isCancellation: boolean;
|
||||
isEncoderInterrupted?: boolean;
|
||||
deWorkerInversion: "inverted" | "reverted" | undefined;
|
||||
deParallelRouter: "routed" | "reverted" | undefined;
|
||||
}): boolean {
|
||||
if (args.isCancellation) return false;
|
||||
if (args.isCancellation || args.isEncoderInterrupted) return false;
|
||||
if (args.isVerifyError) return true;
|
||||
return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed";
|
||||
}
|
||||
@@ -3601,6 +3603,7 @@ async function executeRenderPipeline(input: {
|
||||
!shouldRetryViaPinnedFallback({
|
||||
isVerifyError,
|
||||
isCancellation,
|
||||
isEncoderInterrupted: err instanceof EncoderInterruptedError,
|
||||
deWorkerInversion,
|
||||
deParallelRouter,
|
||||
})
|
||||
@@ -4053,6 +4056,12 @@ async function executeRenderPipeline(input: {
|
||||
? error
|
||||
: new RenderCancelledError("render_cancelled");
|
||||
}
|
||||
if (error instanceof EncoderInterruptedError) {
|
||||
log.warn("[Render] encoder process interrupted by host lifecycle", {
|
||||
code: error.code,
|
||||
diagnostic: error.diagnosticMessage.slice(-2_000),
|
||||
});
|
||||
}
|
||||
const memoryGuidance = describeMemoryExhaustion(error, {
|
||||
width: captureCompositionWidth,
|
||||
height: captureCompositionHeight,
|
||||
|
||||
Reference in New Issue
Block a user