fix(producer): transport safe extraction failure metadata (#3592)

* fix(producer): transport safe extraction failure metadata

* refactor(producer): generalize public error metadata

* test(producer): use vendor-neutral media hosts

packages/producer/src/server.errorCode.test.ts

@@ -30,10 +30,6 @@ describe("extractSafeRenderErrorCode", () => {
     const error = new AssetMediaTypeMismatchError([
       { expected: "video", detected: "image", elementFingerprint: "0123456789abcdef" },
     ]);
-    expect(error.code).toBe("ASSET_MEDIA_TYPE_MISMATCH");
-    expect(error.owner).toBe("user");
-    expect(error.retryable).toBe(false);
-    expect(extractSafeRenderErrorCode(error)).toBe("ASSET_MEDIA_TYPE_MISMATCH");
     expect(extractSafeRenderErrorMetadata(error)).toEqual({
       errorCode: "ASSET_MEDIA_TYPE_MISMATCH",
       errorOwner: "user",
@@ -51,6 +47,51 @@ describe("extractSafeRenderErrorCode", () => {
     expect(error.message).not.toContain("private ffmpeg stderr");
   });
 
+  it("transports producer-authored public metadata without interpreting its schema", () => {
+    const error = new VideoExtractionStageError(
+      "VIDEO_EXTRACTION_FAILED",
+      true,
+      [{ kind: "download_transient", count: 1 }],
+      {
+        schemaVersion: 1,
+        kindCounts: [{ kind: "download_transient", affectedElementCount: 1 }],
+        groups: [
+          {
+            kind: "download_transient",
+            affectedElementCount: 1,
+            sourceFingerprint: `sha256:${"0".repeat(64)}`,
+            host: "media.customer-cdn.example",
+            statusClass: "http_5xx",
+            retry: { phase: "download", used: 1, budget: 1 },
+          },
+        ],
+        omittedGroupCount: 0,
+      },
+    );
+
+    expect(extractSafeRenderErrorMetadata(error)).toEqual({
+      errorCode: "VIDEO_EXTRACTION_FAILED",
+      errorOwner: undefined,
+      retryable: true,
+      errorMetadata: error.publicMetadata,
+    });
+  });
+
+  it("does not transport arbitrary private fields or non-object public metadata", () => {
+    expect(
+      extractSafeRenderErrorMetadata({
+        code: "VIDEO_EXTRACTION_FAILED",
+        retryable: true,
+        publicMetadata: "https://media.example/private.mp4?signature=secret",
+        localPath: "/tmp/private.mp4",
+      }),
+    ).toEqual({
+      errorCode: "VIDEO_EXTRACTION_FAILED",
+      errorOwner: undefined,
+      retryable: true,
+    });
+  });
+
   it("does not forward arbitrary codes or parse message text", () => {
     expect(extractSafeRenderErrorCode({ code: "INTERNAL_ERROR" })).toBeUndefined();
     expect(

packages/producer/src/server.extractionFailureMetadata.test.ts

@@ -0,0 +1,115 @@
+import { Hono } from "hono";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const safeExtractionFailure = vi.hoisted(() => ({
+  schemaVersion: 1,
+  kindCounts: [{ kind: "download_transient", affectedElementCount: 1 }],
+  groups: [
+    {
+      kind: "download_transient",
+      affectedElementCount: 1,
+      sourceFingerprint: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+      host: "media.customer-cdn.example",
+      statusClass: "http_5xx",
+      retry: { phase: "download", used: 1, budget: 1 },
+    },
+  ],
+  omittedGroupCount: 0,
+}));
+vi.mock("./services/renderOrchestrator.js", () => {
+  class RenderCancelledError extends Error {}
+  class MockVideoExtractionStageError extends Error {
+    readonly code = "VIDEO_EXTRACTION_FAILED";
+    readonly retryable = true;
+    readonly publicMetadata = { extractionFailure: safeExtractionFailure };
+    readonly source =
+      "https://media.customer-cdn.example/private/clip.mp4?X-Amz-Signature=must-not-reach-wire";
+    readonly videoId = "private-video-id";
+    readonly statusText = "upstream private status text";
+    readonly localPath = "/tmp/private-render/clip.mp4";
+  }
+
+  return {
+    RenderCancelledError,
+    createRenderJob: (config: Record<string, unknown>) => ({
+      config,
+      progress: 0,
+      currentStage: "video_extract",
+      framesRendered: 0,
+      totalFrames: 0,
+      warnings: [],
+    }),
+    executeRenderJob: async () => {
+      throw new MockVideoExtractionStageError("Video extraction failed");
+    },
+  };
+});
+
+import { createRenderHandlers } from "./server.js";
+
+function createApp(): Hono {
+  const app = new Hono();
+  const handlers = createRenderHandlers({
+    getRequestId: () => "extraction-failure-test",
+    maxConcurrentRenders: 1,
+  });
+  app.post("/v1/render", handlers.render);
+  app.post("/v1/render-stream", handlers.renderStream);
+  return app;
+}
+
+function request(path: string): Promise<Response> {
+  return createApp().request(path, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ html: "<html><body></body></html>" }),
+  });
+}
+
+function expectPrivateDiagnosticsAbsent(body: string): void {
+  expect(body).not.toContain("must-not-reach-wire");
+  expect(body).not.toContain("/private/");
+  expect(body).not.toContain("private-video-id");
+  expect(body).not.toContain("private status text");
+  expect(body).not.toContain("/tmp/private-render");
+}
+
+describe("server extraction failure metadata", () => {
+  beforeEach(() => {
+    safeExtractionFailure.groups[0]!.host = "media.customer-cdn.example";
+  });
+
+  it("emits the bounded top-level contract in blocking JSON", async () => {
+    const response = await request("/v1/render");
+    const body = await response.text();
+
+    expect(response.status).toBe(500);
+    expect(JSON.parse(body)).toMatchObject({
+      success: false,
+      errorCode: "VIDEO_EXTRACTION_FAILED",
+      retryable: true,
+      errorMetadata: { extractionFailure: safeExtractionFailure },
+    });
+    expectPrivateDiagnosticsAbsent(body);
+  });
+
+  it("emits the same bounded top-level contract in SSE", async () => {
+    const response = await request("/v1/render-stream");
+    const body = await response.text();
+
+    expect(response.status).toBe(200);
+    expect(body).toContain('"errorCode":"VIDEO_EXTRACTION_FAILED"');
+    expect(body).toContain('"retryable":true');
+    expect(body).toContain(
+      `"errorMetadata":{"extractionFailure":${JSON.stringify(safeExtractionFailure)}}`,
+    );
+    expectPrivateDiagnosticsAbsent(body);
+  });
+
+  it("does not impose a caller-specific schema on public metadata", async () => {
+    Object.assign(safeExtractionFailure, { callerDefinedField: "v2" });
+    const body = await (await request("/v1/render")).text();
+    expect(body).toContain('"callerDefinedField":"v2"');
+    delete (safeExtractionFailure as Record<string, unknown>).callerDefinedField;
+  });
+});

packages/producer/src/server.ts

@@ -142,9 +142,14 @@ export interface SafeRenderErrorMetadata {
   errorCode: string;
   errorOwner?: "system" | "user";
   retryable?: boolean;
+  /** Public, producer-authored data whose schema and policy belong to callers. */
+  errorMetadata?: Readonly<Record<string, unknown>>;
 }
 
-/** Additive bounded metadata for typed producer failures. */
+/**
+ * Additive public metadata for typed producer failures. The producer server
+ * only transports it; callers own schema validation and policy decisions.
+ */
 export function extractSafeRenderErrorMetadata(
   error: unknown,
 ): SafeRenderErrorMetadata | undefined {
@@ -152,10 +157,12 @@ export function extractSafeRenderErrorMetadata(
   if (!errorCode || typeof error !== "object" || error === null) return undefined;
   const owner = "owner" in error ? error.owner : undefined;
   const retryable = "retryable" in error ? error.retryable : undefined;
+  const publicMetadata = "publicMetadata" in error ? error.publicMetadata : undefined;
   return {
     errorCode,
     errorOwner: owner === "user" || owner === "system" ? owner : undefined,
     retryable: typeof retryable === "boolean" ? retryable : undefined,
+    ...(isPlainObject(publicMetadata) ? { errorMetadata: publicMetadata } : {}),
   };
 }
 
@@ -625,6 +632,7 @@ async function writeRenderStreamFailure(input: {
       errorCode: safeError?.errorCode,
       errorOwner: safeError?.errorOwner,
       retryable: safeError?.retryable,
+      errorMetadata: safeError?.errorMetadata,
       stage: job.currentStage,
       elapsedMs,
       errorDetails: job.errorDetails ?? null,
@@ -788,6 +796,7 @@ export function createRenderHandlers(options: HandlerOptions = {}): RenderHandle
           errorCode: safeError?.errorCode,
           errorOwner: safeError?.errorOwner,
           retryable: safeError?.retryable,
+          errorMetadata: safeError?.errorMetadata,
           stage: job.currentStage,
           durationMs,
           errorDetails: job.errorDetails ?? null,

packages/producer/src/services/render/extractionFailureMetadata.ts

@@ -0,0 +1,80 @@
+import type {
+  VideoExtractionFailureKind,
+  VideoExtractionFailureRetry,
+  VideoExtractionFailureStatusClass,
+} from "@hyperframes/engine";
+
+export interface ExtractionFailureKindCount {
+  kind: VideoExtractionFailureKind;
+  affectedElementCount: number;
+}
+
+export interface ExtractionFailureGroup {
+  kind: VideoExtractionFailureKind;
+  affectedElementCount: number;
+  sourceFingerprint?: string;
+  host?: string;
+  statusClass?: VideoExtractionFailureStatusClass;
+  retry?: VideoExtractionFailureRetry;
+}
+
+export interface ExtractionFailureMetadataV1 {
+  schemaVersion: 1;
+  kindCounts: ExtractionFailureKindCount[];
+  groups: ExtractionFailureGroup[];
+  omittedGroupCount: number;
+}
+
+type ExtractionFailureGroupSortTuple = readonly [
+  string,
+  string,
+  string,
+  string,
+  string,
+  number,
+  number,
+];
+
+function stringOrEmpty(value: string | undefined): string {
+  return value === undefined ? "" : value;
+}
+
+function retrySortTuple(
+  retry: VideoExtractionFailureRetry | undefined,
+): readonly [string, number, number] {
+  return retry === undefined ? ["", -1, -1] : [retry.phase, retry.used, retry.budget];
+}
+
+function extractionFailureGroupSortTuple(
+  group: ExtractionFailureGroup,
+): ExtractionFailureGroupSortTuple {
+  const retry = retrySortTuple(group.retry);
+  return [
+    group.kind,
+    stringOrEmpty(group.sourceFingerprint),
+    stringOrEmpty(group.host),
+    stringOrEmpty(group.statusClass),
+    retry[0],
+    retry[1],
+    retry[2],
+  ];
+}
+
+export function compareExtractionFailureGroups(
+  left: ExtractionFailureGroup,
+  right: ExtractionFailureGroup,
+): number {
+  const leftTuple = extractionFailureGroupSortTuple(left);
+  const rightTuple = extractionFailureGroupSortTuple(right);
+  for (let index = 0; index < leftTuple.length; index += 1) {
+    const leftValue = leftTuple[index]!;
+    const rightValue = rightTuple[index]!;
+    if (leftValue < rightValue) return -1;
+    if (leftValue > rightValue) return 1;
+  }
+  return 0;
+}
+
+export function extractionFailureGroupIdentityKey(group: ExtractionFailureGroup): string {
+  return JSON.stringify(extractionFailureGroupSortTuple(group));
+}

packages/producer/src/services/render/stages/extractVideosStage.test.ts

@@ -14,6 +14,7 @@ import {
   assertVideoExtractionSucceeded,
   buildHdrProbeStageError,
   resolveVideoExtractionPolicy,
+  safeVideoExtractionSourceLogMetadata,
   shouldCopyExtractedFrames,
   VideoExtractionStageError,
 } from "./extractVideosStage.js";
@@ -249,6 +250,28 @@ describe("resolveVideoExtractionPolicy", () => {
   });
 });
 
+describe("safeVideoExtractionSourceLogMetadata", () => {
+  it("logs only a query-free fingerprint and normalized host for a signed remote source", () => {
+    const source =
+      "https://MEDIA.CUSTOMER-CDN.EXAMPLE/private/clip.mp4?X-Amz-Signature=super-secret#fragment";
+    const metadata = safeVideoExtractionSourceLogMetadata(source);
+
+    expect(metadata).toMatchObject({
+      sourceType: "remote",
+      sourceFingerprint: expect.stringMatching(/^sha256:[0-9a-f]{64}$/),
+      host: "media.customer-cdn.example",
+    });
+    expect(JSON.stringify(metadata)).not.toContain("super-secret");
+    expect(JSON.stringify(metadata)).not.toContain("/private/clip.mp4");
+  });
+
+  it("uses a non-sensitive marker for local sources", () => {
+    expect(safeVideoExtractionSourceLogMetadata("/private/render/clip.mp4")).toEqual({
+      sourceType: "local",
+    });
+  });
+});
+
 describe("assertVideoExtractionSucceeded", () => {
   it("accepts a complete extraction", () => {
     expect(() => assertVideoExtractionSucceeded(extractionResult([]))).not.toThrow();
@@ -319,6 +342,57 @@ describe("assertVideoExtractionSucceeded", () => {
     );
   });
 
+  it("attaches bounded, deterministic v1 groups while keeping kind counts exhaustive", () => {
+    const errors: VideoExtractionFailure[] = Array.from({ length: 9 }, (_, index) => ({
+      videoId: `private-${index}`,
+      kind: "download_transient",
+      retryable: true,
+      group: {
+        sourceFingerprint: `sha256:${index.toString(16).padStart(64, "0")}`,
+        host: index % 2 === 0 ? "media.customer-cdn.example" : "other",
+        statusClass: "http_5xx",
+        retry: { phase: "download", used: 1, budget: 1 },
+      },
+      error: `https://media.customer-cdn.example/private/${index}.mp4?signature=secret`,
+    }));
+    errors.push(
+      {
+        videoId: "bad-a",
+        kind: "invalid_media",
+        retryable: false,
+        error: "/tmp/private-a.mp4",
+      },
+      {
+        videoId: "bad-b",
+        kind: "invalid_media",
+        retryable: false,
+        error: "/tmp/private-b.mp4",
+      },
+    );
+
+    let caught: unknown;
+    try {
+      assertVideoExtractionSucceeded(extractionResult(errors));
+    } catch (error) {
+      caught = error;
+    }
+    expect(caught).toBeInstanceOf(VideoExtractionStageError);
+    if (!(caught instanceof VideoExtractionStageError)) return;
+
+    expect(caught.extractionFailure.schemaVersion).toBe(1);
+    expect(caught.extractionFailure.kindCounts).toEqual([
+      { kind: "download_transient", affectedElementCount: 9 },
+      { kind: "invalid_media", affectedElementCount: 2 },
+    ]);
+    expect(caught.extractionFailure.groups).toHaveLength(8);
+    expect(caught.extractionFailure.omittedGroupCount).toBe(2);
+    expect(caught.extractionFailure.groups.map((group) => group.sourceFingerprint)).toEqual(
+      Array.from({ length: 8 }, (_, index) => `sha256:${index.toString(16).padStart(64, "0")}`),
+    );
+    expect(JSON.stringify(caught.extractionFailure)).not.toContain("signature=secret");
+    expect(JSON.stringify(caught.extractionFailure)).not.toContain("private-");
+  });
+
   it("fails closed for legacy failures without a kind or retryability", () => {
     expect(() =>
       assertVideoExtractionSucceeded(

packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts

@@ -1,5 +1,11 @@
-import { resolveConfig, type ExtractionResult, type VideoElement } from "@hyperframes/engine";
+import {
+  resolveConfig,
+  safeDownloadUrlIdentity,
+  type ExtractionResult,
+  type VideoElement,
+} from "@hyperframes/engine";
 import { describe, expect, it, vi } from "vitest";
+import type { ProducerLogger } from "../../../logger.js";
 
 const extractionCalls = vi.hoisted(
   () => new Array<{ timelineEnd: number | undefined; durationSeconds: number }>(),
@@ -61,13 +67,17 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
 import { createRenderJob } from "../../renderOrchestrator.js";
 import { runExtractVideosStage } from "./extractVideosStage.js";
 
-async function runStage(compositionDuration: number, materializeSymlinks: boolean): Promise<void> {
+async function runStage(
+  compositionDuration: number,
+  materializeSymlinks: boolean,
+  options: { source?: string; log?: ProducerLogger } = {},
+): Promise<void> {
   const composition = {
     duration: compositionDuration,
     videos: [
       {
         id: "root-video",
-        src: "long.mp4",
+        src: options.source ?? "long.mp4",
         start: 0,
         end: Number.POSITIVE_INFINITY,
         mediaStart: 0,
@@ -89,6 +99,7 @@ async function runStage(compositionDuration: number, materializeSymlinks: boolea
       hdrMode: "force-sdr",
     }),
     cfg: resolveConfig(),
+    log: options.log,
     composition,
     abortSignal: undefined,
     assertNotAborted: () => {},
@@ -118,3 +129,34 @@ describe.each([
     expect(extractionCalls).toEqual([{ timelineEnd: 10, durationSeconds: 2 }]);
   });
 });
+
+describe("video extraction source logging", () => {
+  it("keeps the actual logger message source-free and emits only safe remote metadata", async () => {
+    const source =
+      "https://media.customer-cdn.example/private/clip.mp4?X-Amz-Signature=must-not-log#fragment";
+    const log = {
+      error: vi.fn(),
+      warn: vi.fn(),
+      info: vi.fn(),
+      debug: vi.fn(),
+    } satisfies ProducerLogger;
+
+    await runStage(2, false, { source, log });
+
+    const extractionCall = log.info.mock.calls.find(([message]) =>
+      message.startsWith("Extracting frames from video"),
+    );
+    expect(extractionCall).toEqual([
+      "Extracting frames from video 1/1",
+      {
+        sourceType: "remote",
+        sourceFingerprint: `sha256:${safeDownloadUrlIdentity(source).urlFingerprint}`,
+        host: "media.customer-cdn.example",
+      },
+    ]);
+    const serializedCall = JSON.stringify(extractionCall);
+    expect(serializedCall).not.toContain(source);
+    expect(serializedCall).not.toContain("must-not-log");
+    expect(serializedCall).not.toContain("/private/clip.mp4");
+  });
+});

packages/producer/src/services/render/stages/extractVideosStage.ts

@@ -38,6 +38,7 @@ import {
   type FrameLookupTable,
   type HdrTransfer,
   type VideoExtractionFailureKind,
+  type VideoExtractionFailureGroupDetails,
   type VideoColorSpace,
   classifyVideoExtractionError,
   createFrameLookupTable,
@@ -47,6 +48,7 @@ import {
   isHdrColorSpace,
   resolveProjectRelativeSrc,
   runVideoExtractionWithRetry,
+  safeVideoExtractionSourceIdentity,
 } from "@hyperframes/engine";
 import {
   collectVideoMetadataHints,
@@ -56,6 +58,11 @@ import {
 import { materializeExtractedFramesForCompiledDir, type CompositionMetadata } from "../shared.js";
 import type { ProducerLogger } from "../../../logger.js";
 import { encoderFailureError } from "../encoderInterruption.js";
+import {
+  compareExtractionFailureGroups,
+  extractionFailureGroupIdentityKey,
+  type ExtractionFailureMetadataV1,
+} from "../extractionFailureMetadata.js";
 
 export interface ExtractVideosStageInput {
   projectDir: string;
@@ -124,6 +131,60 @@ export interface VideoExtractionStageFailureSummary {
   count: number;
 }
 
+const MAX_EXTRACTION_FAILURE_GROUPS = 8;
+
+interface ExtractionFailureAggregateInput extends VideoExtractionFailureGroupDetails {
+  kind: VideoExtractionFailureKind;
+  affectedElementCount: number;
+}
+
+function buildExtractionFailureMetadata(
+  inputs: readonly ExtractionFailureAggregateInput[],
+): ExtractionFailureMetadataV1 {
+  const kindCounts = new Map<VideoExtractionFailureKind, number>();
+  const grouped = new Map<string, ExtractionFailureAggregateInput>();
+  for (const input of inputs) {
+    kindCounts.set(input.kind, (kindCounts.get(input.kind) ?? 0) + input.affectedElementCount);
+    const key = extractionFailureGroupIdentityKey(input);
+    const existing = grouped.get(key);
+    if (existing) {
+      existing.affectedElementCount += input.affectedElementCount;
+    } else {
+      grouped.set(key, { ...input });
+    }
+  }
+
+  const allGroups = [...grouped.values()].sort(compareExtractionFailureGroups);
+  return {
+    schemaVersion: 1,
+    kindCounts: [...kindCounts]
+      .map(([kind, affectedElementCount]) => ({
+        kind,
+        affectedElementCount,
+      }))
+      .sort((a, b) => (a.kind < b.kind ? -1 : a.kind > b.kind ? 1 : 0)),
+    groups: allGroups.slice(0, MAX_EXTRACTION_FAILURE_GROUPS),
+    omittedGroupCount: Math.max(0, allGroups.length - MAX_EXTRACTION_FAILURE_GROUPS),
+  };
+}
+
+function extractionFailureMetadataFromResult(
+  result: ExtractionResult,
+): ExtractionFailureMetadataV1 {
+  return buildExtractionFailureMetadata(
+    result.errors.map((failure) => ({
+      kind: failure.kind ?? "internal",
+      affectedElementCount: 1,
+      ...failure.group,
+    })),
+  );
+}
+
+export function safeVideoExtractionSourceLogMetadata(source: string): Record<string, unknown> {
+  const identity = safeVideoExtractionSourceIdentity(source);
+  return identity ? { sourceType: "remote", ...identity } : { sourceType: "local" };
+}
+
 export type VideoExtractionFailureMode = "off" | "observe" | "enforce";
 
 export interface VideoExtractionPolicy {
@@ -157,15 +218,29 @@ export function resolveVideoExtractionPolicy(
  * the cause without leaking those values.
  */
 export class VideoExtractionStageError extends Error {
+  /**
+   * Producer servers may expose this explicitly public, JSON-compatible data
+   * without knowing its schema. Boundary adapters remain responsible for
+   * validating and applying policy to it.
+   */
+  readonly publicMetadata: Readonly<Record<string, unknown>>;
+
   constructor(
     readonly code: VideoExtractionStageErrorCode,
     readonly retryable: boolean,
     readonly failures: readonly VideoExtractionStageFailureSummary[],
+    readonly extractionFailure: ExtractionFailureMetadataV1 = buildExtractionFailureMetadata(
+      failures.map((failure) => ({
+        kind: failure.kind,
+        affectedElementCount: failure.count,
+      })),
+    ),
   ) {
     const total = failures.reduce((sum, failure) => sum + failure.count, 0);
     const breakdown = failures.map((failure) => `${failure.kind}=${failure.count}`).join(",");
     super(`Video extraction failed for ${total} source(s) [${code}; ${breakdown}]`);
     this.name = "VideoExtractionStageError";
+    this.publicMetadata = { extractionFailure };
   }
 }
 
@@ -202,6 +277,7 @@ function buildVideoExtractionStageError(
     retryable ? "VIDEO_EXTRACTION_FAILED" : "VIDEO_SOURCE_UNRENDERABLE",
     retryable,
     failures,
+    extractionFailureMetadataFromResult(result),
   );
 }
 
@@ -220,6 +296,12 @@ export function buildHdrProbeStageError(
     retryable ? "VIDEO_EXTRACTION_FAILED" : "VIDEO_SOURCE_UNRENDERABLE",
     retryable,
     summary,
+    buildExtractionFailureMetadata(
+      failures.map((failure) => ({
+        kind: failure.kind,
+        affectedElementCount: 1,
+      })),
+    ),
   );
 }
 
@@ -411,7 +493,10 @@ export async function runExtractVideosStage(
     const totalVideos = composition.videos.length;
     for (let i = 0; i < totalVideos; i++) {
       const v = composition.videos[i]!;
-      log?.info(`Extracting frames from video ${i + 1}/${totalVideos}: ${v.src}`);
+      log?.info(
+        `Extracting frames from video ${i + 1}/${totalVideos}`,
+        safeVideoExtractionSourceLogMetadata(v.src),
+      );
     }
     extractionResult = await extractAllVideoFrames(
       composition.videos,