fix(encoder): signal host interruptions for retry (#3578)

* fix(encoder): signal host interruptions for retry

* fix(encoder): cover all render interruption paths

* fix(encoder): classify HDR pre-extraction drains

packages/engine/src/index.ts

@@ -309,6 +309,7 @@ export {
 export {
   runFfmpeg,
   formatFfmpegError,
+  isExternalFfmpegInterruption,
   type RunFfmpegOptions,
   type RunFfmpegResult,
 } from "./utils/runFfmpeg.js";

packages/engine/src/services/audioMixer.ts

@@ -383,7 +383,10 @@ function ffmpegFailure(
   let owner: AudioProcessingFailure["owner"] = "system";
   let retryable = false;
 
-  if (result.terminationReason === "abort") {
+  if (result.failureReason === "external_interruption") {
+    reason = "external_interruption";
+    retryable = true;
+  } else if (result.terminationReason === "abort") {
     reason = "cancelled";
     owner = "user";
   } else if (result.terminationReason === "deadline" || result.terminationReason === "inactivity") {
@@ -862,7 +865,12 @@ async function mixAudioTracks(
   // dropped from the output entirely — a missing fade beats missing audio.
   let degradedAutomation = false;
   const hasAutomation = tracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
-  if (!result.success && !signal?.aborted && hasAutomation) {
+  if (
+    !result.success &&
+    result.failureReason !== "external_interruption" &&
+    !signal?.aborted &&
+    hasAutomation
+  ) {
     const retry = await runMix(true);
     if (retry.success) {
       result = retry;

packages/engine/src/services/audioMixer.types.ts

@@ -69,6 +69,7 @@ export type AudioFailureReason =
   | "ffmpeg_unsupported"
   | "ffmpeg_timeout"
   | "ffmpeg_unavailable"
+  | "external_interruption"
   | "ffmpeg_failed"
   | "cancelled"
   | "internal";

packages/engine/src/services/chunkEncoder.test.ts

@@ -374,6 +374,45 @@ describe("encodeFramesChunkedConcat ffmpegEncodeTimeout", () => {
 });
 
 describe("muxVideoWithAudio audio codec handling", () => {
+  it("preserves an external interruption from mux", async () => {
+    const { spawn, calls } = createSpawnSpy();
+    vi.resetModules();
+    vi.doMock("child_process", () => ({ spawn }));
+
+    const { muxVideoWithAudio } = await import("./chunkEncoder.js");
+    const muxPromise = muxVideoWithAudio(
+      "/tmp/video-only.mp4",
+      "/tmp/audio.aac",
+      "/tmp/output.mp4",
+    );
+
+    await flushMuxCodecResolution();
+    calls[0]!.proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
+    emitClose(calls[0]!.proc, 255);
+
+    await expect(muxPromise).resolves.toMatchObject({
+      success: false,
+      failureReason: "external_interruption",
+    });
+  });
+
+  it("preserves an external interruption from faststart", async () => {
+    const { spawn, calls } = createSpawnSpy();
+    vi.resetModules();
+    vi.doMock("child_process", () => ({ spawn }));
+
+    const { applyFaststart } = await import("./chunkEncoder.js");
+    const faststartPromise = applyFaststart("/tmp/video-only.mp4", "/tmp/output.mp4");
+
+    calls[0]!.proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
+    emitClose(calls[0]!.proc, 255);
+
+    await expect(faststartPromise).resolves.toMatchObject({
+      success: false,
+      failureReason: "external_interruption",
+    });
+  });
+
   it("copies HyperFrames AAC sidecars into MP4 instead of re-encoding", async () => {
     const { spawn, calls } = createSpawnSpy();
     vi.resetModules();

packages/engine/src/services/chunkEncoder.ts

@@ -17,7 +17,7 @@ import {
 } from "../utils/gpuEncoder.js";
 import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
 import { withEvenDimensionPad } from "../utils/evenDimensions.js";
-import { formatFfmpegError, runFfmpeg } from "../utils/runFfmpeg.js";
+import { formatFfmpegError, isExternalFfmpegInterruption, runFfmpeg } from "../utils/runFfmpeg.js";
 import { extractAudioMetadata } from "../utils/ffprobe.js";
 import { type Fps, fpsToFfmpegArg } from "@hyperframes/core";
 import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
@@ -518,6 +518,7 @@ export async function encodeFramesFromDir(
         result.terminationReason === "deadline",
         encodeTimeout,
       ),
+      failureReason: isExternalFfmpegInterruption(result) ? "external_interruption" : undefined,
     };
   }
   const fileSize = existsSync(outputPath) ? statSync(outputPath).size : 0;
@@ -612,6 +613,9 @@ export async function encodeFramesChunkedConcat(
         framesEncoded: 0,
         fileSize: 0,
         error: chunkResult.error,
+        failureReason: isExternalFfmpegInterruption(processResult)
+          ? "external_interruption"
+          : undefined,
       };
     }
     chunkPaths.push(chunkPath);
@@ -650,6 +654,9 @@ export async function encodeFramesChunkedConcat(
       framesEncoded: 0,
       fileSize: 0,
       error: concatResult.error,
+      failureReason: isExternalFfmpegInterruption(concatProcessResult)
+        ? "external_interruption"
+        : undefined,
     };
   }
 
@@ -739,6 +746,7 @@ export async function muxVideoWithAudio(
     outputPath,
     durationMs: result.durationMs,
     error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
+    failureReason: result.failureReason,
   };
 }
 
@@ -781,5 +789,6 @@ export async function applyFaststart(
     outputPath,
     durationMs: result.durationMs,
     error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,
+    failureReason: result.failureReason,
   };
 }

packages/engine/src/services/chunkEncoder.types.ts

@@ -50,6 +50,8 @@ export interface EncodeResult {
   framesEncoded: number;
   fileSize: number;
   error?: string;
+  /** Stable machine-readable cause for failures safe to retry on a fresh host. */
+  failureReason?: "external_interruption";
 }
 
 export interface MuxResult {
@@ -57,4 +59,6 @@ export interface MuxResult {
   outputPath: string;
   durationMs: number;
   error?: string;
+  /** Stable machine-readable cause for failures safe to retry on a fresh host. */
+  failureReason?: "external_interruption";
 }

packages/engine/src/services/streamingEncoder.test.ts

@@ -548,6 +548,24 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
     expect(result.error).toContain("Encoder error");
   });
 
+  it("classifies ffmpeg's handled SIGTERM as an external interruption", async () => {
+    const { spawn, calls } = createSpawnSpy();
+    vi.resetModules();
+    vi.doMock("child_process", () => ({ spawn }));
+
+    const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
+    const dir = mkdtempSync(join(tmpdir(), "se-interrupted-"));
+    const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
+    const proc = calls[0]!.proc;
+    proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
+    process.nextTick(() => proc.emit("close", 255));
+
+    const result = await encoder.close();
+    expect(result.success).toBe(false);
+    expect(result.failureReason).toBe("external_interruption");
+    expect(encoder.getExitFailureReason?.()).toBe("external_interruption");
+  });
+
   it("getExitError surfaces the ffmpeg failure reason after a non-zero exit", async () => {
     const { spawn, calls } = createSpawnSpy();
     vi.resetModules();
@@ -696,6 +714,27 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
     expect(await encoder.writeFrame(Buffer.from([0]))).toBe(false);
   });
 
+  it("waits for child close when stdin dies first so the interruption reason is observable", async () => {
+    const { spawn, calls } = createSpawnSpy();
+    vi.resetModules();
+    vi.doMock("child_process", () => ({ spawn }));
+
+    const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
+    const dir = mkdtempSync(join(tmpdir(), "se-epipe-before-close-"));
+    const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
+    const proc = calls[0]!.proc;
+    proc.stdin.destroyed = true;
+
+    const writePromise = encoder.writeFrame(Buffer.from([0]));
+    await expect(resolveWithin(writePromise, 10)).resolves.toBe("timeout");
+
+    proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
+    proc.emit("close", 255);
+
+    await expect(writePromise).resolves.toBe(false);
+    expect(encoder.getExitFailureReason?.()).toBe("external_interruption");
+  });
+
   it("writeFrame waits for stdin drain when FFmpeg applies back-pressure", async () => {
     const { spawn, calls } = createSpawnSpy();
     vi.resetModules();

packages/engine/src/services/streamingEncoder.ts

@@ -29,7 +29,7 @@ import {
   getGpuEncoderName,
   mapPresetForGpuEncoder,
 } from "../utils/gpuEncoder.js";
-import { formatFfmpegError } from "../utils/runFfmpeg.js";
+import { formatFfmpegError, isExternalFfmpegInterruption } from "../utils/runFfmpeg.js";
 import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
 import { getHdrEncoderColorParams } from "../utils/hdr.js";
 import { withEvenDimensionPad } from "../utils/evenDimensions.js";
@@ -159,6 +159,8 @@ export interface StreamingEncoderResult {
   durationMs: number;
   fileSize: number;
   error?: string;
+  /** Stable machine-readable cause for failures safe to retry on a fresh host. */
+  failureReason?: "external_interruption";
 }
 
 export interface StreamingEncoder {
@@ -179,6 +181,8 @@ export interface StreamingEncoder {
    * unsupported codec, disk full) instead of a bare "encoder exited" message.
    */
   getExitError: () => string | undefined;
+  /** Machine-readable cause available after FFmpeg exits unexpectedly mid-write. */
+  getExitFailureReason?: () => "external_interruption" | undefined;
 }
 
 /**
@@ -465,6 +469,7 @@ export async function spawnStreamingEncoder(
   let exitStatus: "running" | "success" | "error" = "running";
   let stderr = "";
   let exitCode: number | null = null;
+  let exitSignal: NodeJS.Signals | null = null;
   let terminationReason: ManagedProcessTerminationReason = "exit";
 
   ffmpeg.stdin?.on("error", () => {});
@@ -486,6 +491,7 @@ export async function spawnStreamingEncoder(
   });
   const exitPromise = managed.wait().then((outcome) => {
     exitCode = outcome.exitCode;
+    exitSignal = outcome.signal;
     stderr = outcome.stderr;
     terminationReason = outcome.reason;
     exitStatus = outcome.reason === "exit" && outcome.exitCode === 0 ? "success" : "error";
@@ -530,7 +536,15 @@ export async function spawnStreamingEncoder(
   const encoder: StreamingEncoder = {
     writeFrame: async (buffer: Buffer): Promise<boolean> => {
       const stdin = ffmpeg.stdin;
-      if (exitStatus !== "running" || !stdin || stdin.destroyed) {
+      if (exitStatus !== "running") {
+        return false;
+      }
+      if (!stdin || stdin.destroyed) {
+        // The OS can close the pipe (EPIPE) before Node delivers the child
+        // process `close` event. Wait for the shared exit settlement so the
+        // caller can synchronously inspect getExitFailureReason() instead of
+        // losing a host-interruption signal in this narrow race.
+        await exitPromise;
         return false;
       }
       // Copy the buffer before writing — Node streams hold a reference to the
@@ -604,6 +618,14 @@ export async function spawnStreamingEncoder(
           durationMs,
           fileSize: 0,
           error: `${formatFfmpegError(exitCode, stderr)}${inactivitySuffix}`,
+          failureReason: isExternalFfmpegInterruption({
+            exitCode,
+            signal: exitSignal,
+            stderr,
+            terminationReason,
+          })
+            ? "external_interruption"
+            : undefined,
         };
       }
 
@@ -618,6 +640,18 @@ export async function spawnStreamingEncoder(
       if (exitStatus !== "error") return undefined;
       return formatFfmpegError(exitCode, stderr);
     },
+
+    getExitFailureReason: () => {
+      if (exitStatus !== "error") return undefined;
+      return isExternalFfmpegInterruption({
+        exitCode,
+        signal: exitSignal,
+        stderr,
+        terminationReason,
+      })
+        ? "external_interruption"
+        : undefined;
+    },
   };
 
   return encoder;

packages/engine/src/services/videoFrameExtractor.ts

@@ -260,6 +260,7 @@ export interface ExtractionPhaseBreakdown {
 
 export type VideoExtractionFailureKind =
   | "cancelled"
+  | "external_interruption"
   | "source_missing"
   | "source_rejected"
   | "download_not_found"
@@ -785,6 +786,14 @@ export async function extractVideoFramesRange(
   args.push("-y", outputPattern);
 
   const processResult = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
+  if (processResult.failureReason === "external_interruption") {
+    throw new VideoSourceExtractionError(
+      "external_interruption",
+      true,
+      "Video frame extraction interrupted by host lifecycle",
+      `FFmpeg exited with code ${processResult.exitCode}: ${processResult.stderr.slice(-500)}`,
+    );
+  }
   if (processResult.terminationReason === "abort") {
     throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
   }

packages/engine/src/utils/runFfmpeg.test.ts

@@ -2,7 +2,65 @@ import { EventEmitter } from "node:events";
 import { resolve } from "node:path";
 import { afterEach, describe, expect, it, vi } from "vitest";
 
-import { formatFfmpegError } from "./runFfmpeg.js";
+import { formatFfmpegError, isExternalFfmpegInterruption } from "./runFfmpeg.js";
+
+describe("isExternalFfmpegInterruption", () => {
+  const base = {
+    exitCode: 255,
+    signal: null,
+    stderr: "",
+    terminationReason: "exit" as const,
+  };
+
+  it("recognizes a direct external termination signal", () => {
+    expect(isExternalFfmpegInterruption({ ...base, exitCode: null, signal: "SIGTERM" })).toBe(true);
+  });
+
+  it("does not broaden the retry signal to SIGKILL", () => {
+    expect(isExternalFfmpegInterruption({ ...base, exitCode: null, signal: "SIGKILL" })).toBe(
+      false,
+    );
+  });
+
+  it("recognizes ffmpeg's handled SIGTERM exit-255 signature", () => {
+    expect(
+      isExternalFfmpegInterruption({
+        ...base,
+        stderr: "frame= 42\nExiting normally, received signal 15.\n",
+      }),
+    ).toBe(true);
+  });
+
+  it("does not classify an ordinary exit 255", () => {
+    expect(isExternalFfmpegInterruption({ ...base, stderr: "Encoder initialization failed" })).toBe(
+      false,
+    );
+  });
+
+  it("requires exit 255 when classification relies on ffmpeg stderr", () => {
+    expect(
+      isExternalFfmpegInterruption({
+        ...base,
+        exitCode: 1,
+        stderr: "Exiting normally, received signal 15.",
+      }),
+    ).toBe(false);
+  });
+
+  it.each(["abort", "deadline", "inactivity"] as const)(
+    "keeps a managed %s termination non-retryable",
+    (terminationReason) => {
+      expect(
+        isExternalFfmpegInterruption({
+          ...base,
+          signal: "SIGTERM",
+          stderr: "Exiting normally, received signal 15.",
+          terminationReason,
+        }),
+      ).toBe(false);
+    },
+  );
+});
 
 describe("formatFfmpegError", () => {
   const originalPlatform = process.platform;

packages/engine/src/utils/runFfmpeg.ts

@@ -23,12 +23,32 @@ export interface RunFfmpegOptions {
 export interface RunFfmpegResult {
   success: boolean;
   exitCode: number | null;
+  signal?: NodeJS.Signals | null;
   stderr: string;
   durationMs: number;
   terminationReason: ManagedProcessTerminationReason;
+  failureReason?: "external_interruption";
   error?: Error;
 }
 
+const FFMPEG_SIGTERM_EXIT_LINE = /^Exiting normally, received signal 15\.?\r?$/m;
+
+/**
+ * Return true only when ffmpeg was terminated from outside this managed call.
+ *
+ * FFmpeg handles SIGTERM itself and can therefore report exit code 255 with a
+ * null Node signal. The exact terminal stderr line covers that case. Managed
+ * abort/deadline/inactivity reasons always take precedence so our own SIGTERM
+ * requests never become retryable lifecycle interruptions.
+ */
+export function isExternalFfmpegInterruption(
+  result: Pick<RunFfmpegResult, "exitCode" | "signal" | "stderr" | "terminationReason">,
+): boolean {
+  if (result.terminationReason !== "exit" || result.exitCode === 0) return false;
+  if (result.signal === "SIGTERM") return true;
+  return result.exitCode === 255 && FFMPEG_SIGTERM_EXIT_LINE.test(result.stderr);
+}
+
 const DEFAULT_TIMEOUT = 300_000;
 
 const DEFAULT_STDERR_TAIL_LINES = 15;
@@ -104,12 +124,17 @@ export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promis
     onStderr: opts?.onStderr,
   });
   const outcome = await managed.wait();
-  return {
+  const result: RunFfmpegResult = {
     success: outcome.reason === "exit" && outcome.exitCode === 0,
     exitCode: outcome.exitCode,
+    signal: outcome.signal,
     stderr: outcome.stderr,
     durationMs: outcome.durationMs,
     terminationReason: outcome.reason,
     error: outcome.error,
   };
+  if (isExternalFfmpegInterruption(result)) {
+    result.failureReason = "external_interruption";
+  }
+  return result;
 }