From 557d82b6a9547e403598713e2840c41736d35544 Mon Sep 17 00:00:00 2001 From: James Russo Date: Tue, 28 Jul 2026 00:42:36 -0700 Subject: [PATCH] fix(producer): validate distributed video metadata (#2839) ## What - enforce a finite, validated `meta/videos.json` contract shared by Plan v1 and Plan v2 - preserve authored finite ends and source-derived trim-aware ends; bound any still-open end at the validated composition end - fail distributed planning when any declared video source did not extract instead of publishing a blank-capable plan - make the v1 chunk reader reject malformed/null video timing before frame injection - route deterministic video-source/metadata failures as non-retryable in AWS and GCP while retaining retries for transient extraction failures ## Why An open-ended video whose remote source could not be resolved retained `Infinity` through planning. Plan v2 correctly rejected that value, while Plan v1 serialized it as `null`; the v1 frame lookup could then suppress injected frames and silently produce incorrect output. The invariant belongs at the shared metadata boundary. Both protocols must receive identical finite timing, and unavailable sources must fail closed before plan publication. ## Test plan - [x] producer distributed planning, metadata, v1 chunk boundary, Plan v2 conversion/materialization, and public exports - [x] core runtime media semantics (authored slots, natural duration, looping, non-looping hold) - [x] engine video extraction and frame lookup - [x] AWS Lambda/CDK/SAM and GCP Cloud Run error normalization/retry classification - [x] producer, core, engine, AWS, and GCP typechecks/builds - [x] formatting, oxlint, tracked-artifact, fallow, and commit hooks - [x] exact incident composition replayed through the AWS Lambda handler's Lambda-local path in a Lambda-like container; Plan v1 and Plan v2 both fail closed as `VIDEO_SOURCE_UNRENDERABLE` during planning, before plan publication - [x] full PR CI, including all nine regression shards and Windows render/tests No production flags or deployment/release workflows are changed. --- examples/aws-lambda/template.yaml | 6 + .../HyperframesRenderStack.snapshot.test.ts | 36 +++ .../src/cdk/HyperframesRenderStack.ts | 3 + packages/aws-lambda/src/handler.test.ts | 17 +- packages/aws-lambda/src/handler.ts | 7 +- packages/gcp-cloud-run/src/server.test.ts | 61 +++++ packages/gcp-cloud-run/src/server.ts | 7 +- packages/producer/src/distributed.ts | 3 +- .../producer/src/server.errorCode.test.ts | 3 + packages/producer/src/server.ts | 1 + .../src/services/distributed/plan.test.ts | 77 +++++++ .../producer/src/services/distributed/plan.ts | 19 +- .../src/services/distributed/planV2.test.ts | 25 +++ .../src/services/distributed/planV2.ts | 99 +-------- .../distributed/publicExports.test.ts | 1 + .../src/services/distributed/renderChunk.ts | 21 +- .../renderChunkVideoMetadata.test.ts | 34 +++ .../src/services/distributed/shared.ts | 209 ++++++++++++++++++ .../distributed/videoMetadata.test.ts | 147 ++++++++++++ 19 files changed, 672 insertions(+), 104 deletions(-) create mode 100644 packages/producer/src/services/distributed/renderChunkVideoMetadata.test.ts create mode 100644 packages/producer/src/services/distributed/videoMetadata.test.ts diff --git a/examples/aws-lambda/template.yaml b/examples/aws-lambda/template.yaml index fb2a15973..4aaad5b38 100644 --- a/examples/aws-lambda/template.yaml +++ b/examples/aws-lambda/template.yaml @@ -263,6 +263,8 @@ Resources: - PlanTooLargeError - PLAN_PROTOCOL_UNSUPPORTED - PlanProtocolUnsupportedError + - VIDEO_SOURCE_UNRENDERABLE + - INVALID_VIDEO_METADATA - PLAN_ARTIFACT_DIGEST_MISMATCH - FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED MaxAttempts: 0 @@ -305,6 +307,8 @@ Resources: - PLAN_PROTOCOL_UNSUPPORTED - PlanProtocolUnsupportedError - PLAN_V2_INTEGRITY_UNRECOVERABLE + - VIDEO_SOURCE_UNRENDERABLE + - INVALID_VIDEO_METADATA - PlanV2IntegrityError - PLAN_ARTIFACT_DIGEST_MISMATCH - FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED @@ -401,6 +405,7 @@ Resources: - PlanTooLargeError - PLAN_PROTOCOL_UNSUPPORTED - PlanProtocolUnsupportedError + - INVALID_VIDEO_METADATA - PLAN_ARTIFACT_DIGEST_MISMATCH MaxAttempts: 0 - ErrorEquals: [States.ALL] @@ -497,6 +502,7 @@ Resources: - PLAN_PROTOCOL_UNSUPPORTED - PlanProtocolUnsupportedError - PLAN_V2_INTEGRITY_UNRECOVERABLE + - INVALID_VIDEO_METADATA - PlanV2IntegrityError - PLAN_ARTIFACT_DIGEST_MISMATCH - ChromeBinaryUnavailableError diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts index e5ef15d74..4a32e94fa 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts @@ -76,6 +76,8 @@ const EXPECTED_NON_RETRYABLE_ERRORS = new Set([ "PLAN_PROTOCOL_UNSUPPORTED", "PlanProtocolUnsupportedError", "PLAN_V2_INTEGRITY_UNRECOVERABLE", + "VIDEO_SOURCE_UNRENDERABLE", + "INVALID_VIDEO_METADATA", "PlanV2IntegrityError", "PLAN_ARTIFACT_DIGEST_MISMATCH", "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED", @@ -209,6 +211,25 @@ describe("HyperframesRenderStack — snapshot", () => { } }); + it("routes video failures consistently across SAM/CDK and both plan protocols", () => { + for (const definition of [SYNTHED.definition, readSamDefinition()]) { + const v1 = getV1TaskStates(definition); + const v2 = getV2TaskStates(definition); + for (const planState of [v1.Plan, v2.PlanV2]) { + const errors = new Set(); + collectNonRetryableErrors(planState, errors); + expect(errors.has("VIDEO_SOURCE_UNRENDERABLE")).toBe(true); + expect(errors.has("INVALID_VIDEO_METADATA")).toBe(true); + expect(errors.has("VIDEO_EXTRACTION_FAILED")).toBe(false); + } + for (const chunkState of [v1.RenderChunk, v2.RenderChunkV2]) { + const errors = new Set(); + collectNonRetryableErrors(chunkState, errors); + expect(errors.has("INVALID_VIDEO_METADATA")).toBe(true); + } + } + }); + it("keeps v1 and v2 locators disjoint across orchestration branches", () => { const { definition } = SYNTHED; const v1 = JSON.stringify({ @@ -271,6 +292,21 @@ function getV2TaskStates(definition: { }; } +function getV1TaskStates(definition: { + States: Record; +}): Record<"Plan" | "RenderChunk" | "Assemble", unknown> { + const renderChunks = requireRecord(definition.States.RenderChunks, "RenderChunks state"); + const processor = isRecord(renderChunks.Iterator) + ? renderChunks.Iterator + : requireRecord(renderChunks.ItemProcessor, "RenderChunks processor"); + const innerStates = requireRecord(processor.States, "RenderChunks processor states"); + return { + Plan: definition.States.Plan, + RenderChunk: innerStates.RenderChunk, + Assemble: definition.States.Assemble, + }; +} + function readSamDefinition(): { States: Record } { const source = readFileSync( new URL("../../../../examples/aws-lambda/template.yaml", import.meta.url), diff --git a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts index 9cbbb1a2f..1c371de24 100644 --- a/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts +++ b/packages/aws-lambda/src/cdk/HyperframesRenderStack.ts @@ -204,6 +204,8 @@ export class HyperframesRenderStack extends Construct { "PLAN_PROTOCOL_UNSUPPORTED", "PlanProtocolUnsupportedError", "PLAN_V2_INTEGRITY_UNRECOVERABLE", + "VIDEO_SOURCE_UNRENDERABLE", + "INVALID_VIDEO_METADATA", "PlanV2IntegrityError", "PLAN_ARTIFACT_DIGEST_MISMATCH", "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED", @@ -219,6 +221,7 @@ export class HyperframesRenderStack extends Construct { "PLAN_PROTOCOL_UNSUPPORTED", "PlanProtocolUnsupportedError", "PLAN_V2_INTEGRITY_UNRECOVERABLE", + "INVALID_VIDEO_METADATA", "PlanV2IntegrityError", "PLAN_ARTIFACT_DIGEST_MISMATCH", "ChromeBinaryUnavailableError", diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.ts index 5aa1adb94..3c4a4e656 100644 --- a/packages/aws-lambda/src/handler.test.ts +++ b/packages/aws-lambda/src/handler.test.ts @@ -22,6 +22,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { CURRENT_PLAN_PROTOCOL, + PlanVideosMetadataError, type AssembleResult, type ChunkResult, type PlanResult, @@ -243,19 +244,25 @@ describe("handler dispatch", () => { ).toBe(true); }); - it("normalizes producer terminal codes to Step Functions error names", async () => { + it("normalizes producer workflow codes to Step Functions error names", async () => { for (const code of [ "PLAN_TOO_LARGE", "PLAN_PROTOCOL_UNSUPPORTED", "PLAN_V2_INTEGRITY_UNRECOVERABLE", + "VIDEO_SOURCE_UNRENDERABLE", + "VIDEO_EXTRACTION_FAILED", + "INVALID_VIDEO_METADATA", ] as const) { const tmpRoot = makeTmpRoot(); const s3 = new FakeS3Client(); s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar()); - const terminal = Object.assign(new Error(`terminal: ${code}`), { - code, - name: "ProducerError", - }); + const terminal = + code === "INVALID_VIDEO_METADATA" + ? new PlanVideosMetadataError("test invalid plan video metadata") + : Object.assign(new Error(`terminal: ${code}`), { + code, + name: "ProducerError", + }); await expect( handler( diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.ts index 92018f4af..8cb4c8c1b 100644 --- a/packages/aws-lambda/src/handler.ts +++ b/packages/aws-lambda/src/handler.ts @@ -138,7 +138,7 @@ export async function handler(event: LambdaEvent, deps?: HandlerDeps): Promise { } }); + it.each([ + ["VIDEO_SOURCE_UNRENDERABLE", 400], + ["VIDEO_EXTRACTION_FAILED", 500], + ] as const)("routes producer video code %s with HTTP %s", async (code, expectedStatus) => { + const gcs = new FakeGcs(); + await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH); + const app = createApp( + depsWith(gcs, { + renderChunk: async () => { + throw Object.assign(new Error(`test ${code}`), { + name: "ProducerError", + code, + }); + }, + }), + ); + const res = await app.request("/", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + Action: "renderChunk", + PlanGcsUri: "gs://b/renders/r1/plan.tar.gz", + PlanHash: PLAN_HASH, + ChunkIndex: 0, + ChunkOutputGcsPrefix: "gs://b/renders/r1/", + Format: "mp4", + }), + }); + + expect(res.status).toBe(expectedStatus); + const body = (await res.json()) as { error: string }; + expect(body.error).toBe(code); + }); + + it("routes the real plan metadata error as non-retryable", async () => { + const gcs = new FakeGcs(); + await seedProjectTar(gcs, "gs://b/sites/invalid-video-metadata/project.tar.gz"); + const app = createApp( + depsWith(gcs, { + plan: async () => { + throw new PlanVideosMetadataError("test invalid plan video metadata"); + }, + }), + ); + const res = await app.request("/", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + Action: "plan", + ProjectGcsUri: "gs://b/sites/invalid-video-metadata/project.tar.gz", + PlanOutputGcsPrefix: "gs://b/renders/invalid-video-metadata/", + Config: { fps: 30, width: 640, height: 360, format: "mp4" }, + }), + }); + + expect(res.status).toBe(400); + const body = (await res.json()) as { error: string }; + expect(body.error).toBe("INVALID_VIDEO_METADATA"); + }); + it("returns 500 for a retryable/unknown error", async () => { const gcs = new FakeGcs(); // plan tar NOT seeded → download fails (retryable) const app = createApp(depsWith(gcs)); diff --git a/packages/gcp-cloud-run/src/server.ts b/packages/gcp-cloud-run/src/server.ts index 1c142dfd0..1b14a0685 100644 --- a/packages/gcp-cloud-run/src/server.ts +++ b/packages/gcp-cloud-run/src/server.ts @@ -182,7 +182,10 @@ function normalizeTerminalErrorName(error: unknown): void { if ( candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || - candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" + candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" || + candidate.code === "VIDEO_SOURCE_UNRENDERABLE" || + candidate.code === "VIDEO_EXTRACTION_FAILED" || + candidate.code === "INVALID_VIDEO_METADATA" ) { candidate.name = candidate.code; } @@ -899,6 +902,8 @@ const NON_RETRYABLE_ERROR_NAMES = new Set([ "PLAN_ARTIFACT_DIGEST_MISMATCH", "PLAN_PROTOCOL_UNSUPPORTED", "PLAN_V2_INTEGRITY_UNRECOVERABLE", + "VIDEO_SOURCE_UNRENDERABLE", + "INVALID_VIDEO_METADATA", // Producer error class names (`.name`) + their string code aliases — the // class sets `.name` to the class name but wraps a `code`; cover both so a // raw-code throw is caught too. Mirrors the AWS state machine's diff --git a/packages/producer/src/distributed.ts b/packages/producer/src/distributed.ts index 324cbe465..3000892fa 100644 --- a/packages/producer/src/distributed.ts +++ b/packages/producer/src/distributed.ts @@ -94,6 +94,7 @@ export { type EffectiveChunkResult, // Error codes + classes FFMPEG_VERSION_MISMATCH, + INVALID_VIDEO_METADATA, PLAN_HASH_MISMATCH, RenderChunkValidationError, } from "./services/distributed/renderChunk.js"; @@ -142,7 +143,7 @@ export { // ── Format union ──────────────────────────────────────────────────────────── // Canonical output-format type. The aws-lambda package re-exports it so // CLI / adopter SDKs can derive runtime allowlists from one source. -export type { DistributedFormat } from "./services/distributed/shared.js"; +export { PlanVideosMetadataError, type DistributedFormat } from "./services/distributed/shared.js"; // ── Plan-time shared types from `freezePlan` ─────────────────────────────── // Re-exported so adopters that deserialize a planDir's `meta/encoder.json` diff --git a/packages/producer/src/server.errorCode.test.ts b/packages/producer/src/server.errorCode.test.ts index b372a2048..1479a1b3a 100644 --- a/packages/producer/src/server.errorCode.test.ts +++ b/packages/producer/src/server.errorCode.test.ts @@ -19,6 +19,9 @@ describe("extractSafeRenderErrorCode", () => { expect(extractSafeRenderErrorCode({ code: "VIDEO_SOURCE_UNRENDERABLE" })).toBe( "VIDEO_SOURCE_UNRENDERABLE", ); + expect(extractSafeRenderErrorCode({ code: "INVALID_VIDEO_METADATA" })).toBe( + "INVALID_VIDEO_METADATA", + ); }); it("does not forward arbitrary codes or parse message text", () => { diff --git a/packages/producer/src/server.ts b/packages/producer/src/server.ts index 9a3a2f88c..75611b262 100644 --- a/packages/producer/src/server.ts +++ b/packages/producer/src/server.ts @@ -119,6 +119,7 @@ interface PreparedRenderInput { const DEFAULT_SERVER_FPS = { num: 30, den: 1 } as const; const SAFE_RENDER_ERROR_CODES = new Set([ + "INVALID_VIDEO_METADATA", "VIDEO_SOURCE_UNRENDERABLE", "VIDEO_EXTRACTION_FAILED", ]); diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.ts index e8dd5ecb8..027bcc120 100644 --- a/packages/producer/src/services/distributed/plan.test.ts +++ b/packages/producer/src/services/distributed/plan.test.ts @@ -396,6 +396,83 @@ describe("plan() — golden planDir + planHash determinism", () => { // runtime resolution variance on the CI host. const TIMEOUT_MS = 30_000; + it( + "fails closed when an open-ended distributed video source cannot be extracted", + async () => { + const brokenProjectDir = join(runRoot, "broken-video-project"); + const brokenPlanDir = join(runRoot, "broken-video-plan"); + mkdirSync(brokenProjectDir, { recursive: true }); + mkdirSync(brokenPlanDir, { recursive: true }); + writeFileSync( + join(brokenProjectDir, "index.html"), + ` +
+ +
`, + ); + + let caught: unknown; + try { + await plan( + brokenProjectDir, + { fps: 30, width: 320, height: 240, format: "mp4", chunkSize: 240 }, + brokenPlanDir, + ); + } catch (err) { + caught = err; + } + + expect(caught).toHaveProperty("name", "VideoExtractionStageError"); + expect(caught).toHaveProperty("code", "VIDEO_SOURCE_UNRENDERABLE"); + expect(caught).toHaveProperty("retryable", false); + expect(existsSync(join(brokenPlanDir, "meta", "videos.json"))).toBe(false); + }, + TIMEOUT_MS, + ); + + it( + "maps an open-ended remote video HTTP 404 to a terminal source error", + async () => { + const brokenProjectDir = join(runRoot, "remote-404-video-project"); + const brokenPlanDir = join(runRoot, "remote-404-video-plan"); + mkdirSync(brokenProjectDir, { recursive: true }); + mkdirSync(brokenPlanDir, { recursive: true }); + writeFileSync( + join(brokenProjectDir, "index.html"), + ` +
+ +
`, + ); + const originalFetch = globalThis.fetch; + let fetchCalls = 0; + globalThis.fetch = (async () => { + fetchCalls += 1; + return new Response(null, { status: 404, statusText: "Not Found" }); + }) as typeof fetch; + + let caught: unknown; + try { + await plan( + brokenProjectDir, + { fps: 30, width: 320, height: 240, format: "mp4", chunkSize: 240 }, + brokenPlanDir, + ); + } catch (err) { + caught = err; + } finally { + globalThis.fetch = originalFetch; + } + + expect(fetchCalls).toBeGreaterThan(0); + expect(caught).toHaveProperty("name", "VideoExtractionStageError"); + expect(caught).toHaveProperty("code", "VIDEO_SOURCE_UNRENDERABLE"); + expect(caught).toHaveProperty("retryable", false); + expect(existsSync(join(brokenPlanDir, "meta", "videos.json"))).toBe(false); + }, + TIMEOUT_MS, + ); + it( "produces the documented planDir layout", async () => { diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 2960251dc..12e021930 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -44,7 +44,10 @@ import { import { closeFileServerSafely } from "../fileServer.js"; import { runAudioStage } from "../render/stages/audioStage.js"; import { runCompileStage } from "../render/stages/compileStage.js"; -import { runExtractVideosStage } from "../render/stages/extractVideosStage.js"; +import { + assertVideoExtractionSucceeded, + runExtractVideosStage, +} from "../render/stages/extractVideosStage.js"; import { runProbeStage } from "../render/stages/probeStage.js"; import { type ChunkSliceJson, @@ -65,6 +68,7 @@ import { import { snapshotRuntimeEnv } from "../render/runtimeEnvSnapshot.js"; import { buildSyntheticRenderJob, + buildPlanVideosJson, type DistributedFormat, PLAN_AUDIO_RELATIVE_PATH, PLAN_VIDEOS_META_RELATIVE_PATH, @@ -1007,6 +1011,14 @@ export async function plan( materializeSymlinks: true, }); if (extractResult.failureToEnforce) throw extractResult.failureToEnforce; + if (extractResult.extractionResult) { + // Distributed chunks cannot safely fall back to native remote decoding: + // the planner-local source may be unavailable on another worker, and a + // missing frame set otherwise renders as a silent blank video. Unlike the + // separately canaried in-process policy, distributed planning always + // requires every declared source to extract successfully. + assertVideoExtractionSucceeded(extractResult.extractionResult); + } // Skip `extractResult.frameLookup.cleanup()`: it would rm-rf each // video's outputDir, but in `plan()` those directories ARE the source // material the renames below move into `planDir/video-frames/`. @@ -1049,8 +1061,9 @@ export async function plan( // page's native `