diff --git a/bun.lock b/bun.lock index 4cd5e7ae35..0303bfcd1a 100644 --- a/bun.lock +++ b/bun.lock @@ -1322,7 +1322,6 @@ "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], - "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="], "brotli": ["brotli@1.3.3", "", { "dependencies": { "base64-js": "^1.1.2" } }, "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg=="], diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.ts index af9143414b..2405e1f01f 100644 --- a/packages/aws-lambda/src/handler.test.ts +++ b/packages/aws-lambda/src/handler.test.ts @@ -23,6 +23,7 @@ import { dirname, join } from "node:path"; import { S3Client } from "@aws-sdk/client-s3"; import { CURRENT_PLAN_PROTOCOL, + PLAN_V2_MATERIALIZATION_MARKER, PlanVideosMetadataError, type AssembleResult, type ChunkResult, @@ -35,6 +36,7 @@ import { import { recomputePlanHashFromPlanDir } from "../../producer/src/services/render/stages/freezePlan.js"; import type { AssembleEvent, LambdaEvent, PlanEvent, RenderChunkEvent } from "./events.js"; import { handler, unwrapEvent } from "./handler.js"; +import { S3PlanV2ArtifactPublisher } from "./s3PlanV2Publisher.js"; interface FakeS3Op { kind: "download" | "upload"; @@ -52,6 +54,7 @@ class FakeS3Client { // Map S3 URIs → byte buffers the fake serves. objects = new Map(); metadata = new Map>(); + checksums = new Map(); // Methods called by the real S3 transport — minimal surface so the // handler's call sites don't need rewriting under test. @@ -85,6 +88,7 @@ class FakeS3Client { return { ContentLength: bytes.length, Metadata: this.metadata.get(uri), + ChecksumSHA256: this.checksums.get(uri), }; } if (cmdName === "PutObjectCommand") { @@ -104,12 +108,24 @@ class FakeS3Client { this.objects.set(uri, bytes); const metadata = (command as { input: { Metadata?: Record } }).input.Metadata; if (metadata) this.metadata.set(uri, metadata); + const checksum = (command as { input: { ChecksumSHA256?: string } }).input.ChecksumSHA256; + if (checksum) this.checksums.set(uri, checksum); return {}; } return {}; } } +/** A real client (so it can pre-sign offline) whose requests go to the fake. */ +function presigningClient(fake: FakeS3Client): S3Client { + const client = new S3Client({ + region: "us-east-2", + credentials: { accessKeyId: "AKIDTEST", secretAccessKey: "test-secret" }, + }); + client.send = fake.send.bind(fake) as S3Client["send"]; + return client; +} + const tmpDirs: string[] = []; beforeEach(() => { @@ -684,6 +700,85 @@ describe("handler dispatch", () => { ).toBe(true); }); + it("hands a chunk its deferred video as a pre-signed URL and downloads the rest", async () => { + const tmpRoot = makeTmpRoot(); + const fake = new FakeS3Client(); + const s3 = presigningClient(fake); + const v1Dir = join(tmpRoot, "v1"); + makeMinimalV1PlanDir(v1Dir, true, true); + const publisher = new S3PlanV2ArtifactPublisher({ + s3, + planOutputS3Prefix: "s3://bucket/renders/deferred/", + temporaryRoot: tmpRoot, + }); + const manifest = await publishPlanV2FromExecutionPlan(v1Dir, publisher); + const video = manifest.artifacts.find((artifact) => artifact.path === "compiled/clip.mp4")!; + expect(video.chunks).toEqual([0]); + const blobUri = (digest: string) => + `${publisher.artifactPrefix}/${digest.slice(0, 2)}/${digest}`; + + let marker: { remoteSources?: Record } = {}; + const renderChunkMock = mock( + async (planDir: string, _chunkIndex: number, outputPath: string): Promise => { + expect(existsSync(join(planDir, "compiled", "clip.mp4"))).toBe(false); + marker = JSON.parse(readFileSync(join(planDir, PLAN_V2_MATERIALIZATION_MARKER), "utf-8")); + writeFileSync(outputPath, "CHUNK"); + return { + outputPath, + outputKind: "file", + framesEncoded: 30, + sha256: "c".repeat(64), + durationMs: 1, + planHashMs: 0, + sessionBootMs: 0, + captureStageMs: 1, + encodeStageMs: 0, + workers: 1, + captureMode: "beginframe", + perfPath: `${outputPath}.perf.json`, + }; + }, + ); + const beforeChunk = fake.ops.length; + await handler( + { + Action: "renderChunk", + PlanV2ManifestS3Uri: publisher.manifestUri, + PlanV2ArtifactS3Prefix: publisher.artifactPrefix, + PlanHash: manifest.planHash, + ChunkIndex: 0, + ChunkOutputS3Prefix: "s3://bucket/renders/deferred/", + Format: "mp4", + }, + { + s3, + primitives: { + plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan, + renderChunk: + renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk, + assemble: + mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble, + }, + tmpRoot, + skipChromeResolution: true, + }, + ); + + const downloads = fake.ops + .slice(beforeChunk) + .filter((operation) => operation.kind === "download") + .map((operation) => operation.uri); + const expectedDownloads = manifest.artifacts + .filter((artifact) => artifact.chunks === "all") + .map((artifact) => blobUri(artifact.sha256)); + expect(new Set(downloads)).toEqual(new Set([publisher.manifestUri, ...expectedDownloads])); + const url = new URL(marker.remoteSources?.["compiled/clip.mp4"] ?? ""); + expect(url.protocol).toBe("https:"); + expect(url.pathname).toEndWith(`/${video.sha256}`); + expect(url.searchParams.get("X-Amz-Expires")).toBe("3600"); + expect(url.searchParams.get("X-Amz-Signature")).toMatch(/^[a-f0-9]{64}$/); + }); + it("rejects unknown plan protocol values", async () => { const tmpRoot = makeTmpRoot(); const s3 = new FakeS3Client(); @@ -983,7 +1078,7 @@ async function makeMinimalPlanTar(): Promise { return rf(tarPath); } -function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void { +function makeMinimalV1PlanDir(dir: string, withAudio: boolean, withDeferredVideo = false): void { mkdirSync(join(dir, "meta"), { recursive: true }); mkdirSync(join(dir, "compiled"), { recursive: true }); writeFileSync(join(dir, "compiled", "index.html"), "aws v2 fixture"); @@ -1003,6 +1098,54 @@ function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void { ); writeFileSync(join(dir, "meta", "encoder.json"), "{}"); if (withAudio) writeFileSync(join(dir, "audio.m4a"), "AAC"); + if (withDeferredVideo) writeDeferredVideo(dir); planJson.planHash = recomputePlanHashFromPlanDir(dir); writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson)); } + +/** One clip the chunk extracts from `compiled/clip.mp4` itself. */ +function writeDeferredVideo(dir: string): void { + writeFileSync(join(dir, "compiled", "clip.mp4"), "VIDEO-BYTES"); + const metadata = { + durationSeconds: 1, + videoStreamDurationSeconds: 1, + width: 640, + height: 360, + fps: 30, + videoCodec: "h264", + hasAudio: false, + isVFR: false, + hasAlpha: false, + colorSpace: null, + }; + const videos = { + videos: [ + { + id: "clip", + src: "clip.mp4", + start: 0, + end: 1, + mediaStart: 0, + loop: false, + hasAudio: false, + }, + ], + extracted: [ + { + videoId: "clip", + srcPath: "/planner/clip.mp4", + framePattern: "frame_%05d.jpg", + fps: 30, + totalFrames: 30, + metadata, + deferredRange: { + sourcePath: "compiled/clip.mp4", + startTime: 0, + durationSeconds: 1, + format: "jpg", + }, + }, + ], + }; + writeFileSync(join(dir, "meta", "videos.json"), JSON.stringify(videos)); +} diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.ts index 79a191b1e8..4b911ef949 100644 --- a/packages/aws-lambda/src/handler.ts +++ b/packages/aws-lambda/src/handler.ts @@ -29,6 +29,7 @@ import { resolvePlanAudioPath, planV2WithPublisher, type PlanResult, + isPlanV2DeferredSourceArtifact, type PlanV2Artifact, type PlanV2Manifest, type PlanV2MaterializationTarget, @@ -57,6 +58,7 @@ import { downloadS3ObjectToFileVerified, mapConcurrent, parseS3Uri, + presignVerifiedS3GetUrl, tarDirectory, untarDirectory, uploadFileToS3, @@ -723,17 +725,54 @@ async function downloadAndMaterializePlanV2( throwPlanHashMismatch(event.PlanHash, manifest.planHash); } const artifacts = listPlanV2ArtifactsForTarget(manifest, target); + const remoteSources = await presignDeferredSources(s3, event.PlanV2ArtifactS3Prefix, artifacts); const uniqueArtifacts = [ - ...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values(), + ...new Map( + artifacts + .filter((artifact) => !Object.hasOwn(remoteSources, artifact.path)) + .map((artifact) => [artifact.sha256, artifact]), + ).values(), ]; await mapConcurrent(uniqueArtifacts, 16, async (artifact) => { await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact); }); const planDir = join(work, "plan"); - materializePlanV2Target(transportDir, target, planDir); + materializePlanV2Target(transportDir, target, planDir, { remoteSources }); return planDir; } +/** Outlives the longest chunk (Lambda caps an invocation at 900 s) with margin. */ +const DEFERRED_SOURCE_URL_TTL_SECONDS = 3600; + +/** + * A chunk's ffmpeg reads a deferred video by byte range from a pre-signed URL + * instead of downloading it whole. The key is the content hash, written once. + */ +async function presignDeferredSources( + s3: S3Client, + artifactPrefix: string, + artifacts: readonly Readonly[], +): Promise> { + const entries = await Promise.all( + artifacts + .filter(isPlanV2DeferredSourceArtifact) + .map( + async (artifact) => + [ + artifact.path, + await presignVerifiedS3GetUrl( + s3, + planV2BlobUri(artifactPrefix, artifact.sha256), + artifact.sha256, + artifact.sizeBytes, + DEFERRED_SOURCE_URL_TTL_SECONDS, + ), + ] as const, + ), + ); + return Object.fromEntries(entries); +} + async function downloadPlanV2Artifact( s3: S3Client, artifactPrefix: string, diff --git a/packages/aws-lambda/src/s3Transport.test.ts b/packages/aws-lambda/src/s3Transport.test.ts index ca4e798e27..d9097a1e48 100644 --- a/packages/aws-lambda/src/s3Transport.test.ts +++ b/packages/aws-lambda/src/s3Transport.test.ts @@ -10,10 +10,12 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js"; +import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3"; import { downloadS3ObjectToFileVerified, formatS3Uri, parseS3Uri, + presignVerifiedS3GetUrl, sha256File, tarDirectory, untarDirectory, @@ -212,6 +214,54 @@ describe("content-addressed v2 artifacts", () => { }); }); +describe("presignVerifiedS3GetUrl", () => { + const digest = "ab".repeat(32); + const uri = `s3://bucket/v2/artifacts/sha256/ab/${digest}`; + function clientHolding(head: { ChecksumSHA256?: string; ContentLength: number }): S3Client { + const client = new S3Client({ + region: "us-east-2", + credentials: { accessKeyId: "AKIDTEST", secretAccessKey: "test-secret" }, + }); + client.send = (async (command: unknown) => { + expect(command).toBeInstanceOf(HeadObjectCommand); + expect((command as HeadObjectCommand).input.ChecksumMode).toBe("ENABLED"); + return head; + }) as S3Client["send"]; + return client; + } + + it.each([ + { uri, key: `/v2/artifacts/sha256/ab/${digest}` }, + { uri: contentStoreUri("bucket", digest), key: `/cas/sha256/ab/${digest}` }, + ])("signs an attested legacy or shared-content source at $key", async (source) => { + const checksum = Buffer.from(digest, "hex").toString("base64"); + const url = new URL( + await presignVerifiedS3GetUrl( + clientHolding({ ChecksumSHA256: checksum, ContentLength: 42 }), + source.uri, + digest, + 42, + 600, + ), + ); + expect(url.pathname).toBe(source.key); + expect(url.searchParams.get("X-Amz-Expires")).toBe("600"); + }); + + it("refuses an object whose stored checksum is not the pinned digest", async () => { + const other = Buffer.from("cd".repeat(32), "hex").toString("base64"); + await expect( + presignVerifiedS3GetUrl( + clientHolding({ ChecksumSHA256: other, ContentLength: 42 }), + uri, + digest, + 42, + 600, + ), + ).rejects.toMatchObject({ name: "PLAN_ARTIFACT_DIGEST_MISMATCH" }); + }); +}); + class ContentAddressedFakeS3 { readonly objects = new Map(); readonly copies: Record[] = []; diff --git a/packages/aws-lambda/src/s3Transport.ts b/packages/aws-lambda/src/s3Transport.ts index a089f84aa5..61cddce6ef 100644 --- a/packages/aws-lambda/src/s3Transport.ts +++ b/packages/aws-lambda/src/s3Transport.ts @@ -104,6 +104,35 @@ export async function downloadS3ObjectToFileVerified( } } +/** + * Pre-sign a GET for a plan-v2 artifact, after S3 attests the object's stored + * SHA-256 and size are the ones the manifest pins. + */ +export async function presignVerifiedS3GetUrl( + client: S3Client, + uri: string, + expectedSha256: string, + expectedSize: number, + expiresInSeconds: number, +): Promise { + assertSha256(expectedSha256); + const { bucket, key } = parseS3Uri(uri); + const head = await client.send( + new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" }), + ); + const expectedChecksum = Buffer.from(expectedSha256, "hex").toString("base64"); + if (head.ChecksumSHA256 !== expectedChecksum || head.ContentLength !== expectedSize) { + const error = new Error( + `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256} (${expectedSize} bytes), S3 holds checksum ${String(head.ChecksumSHA256)} (${String(head.ContentLength)} bytes)`, + ); + error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH"; + throw error; + } + return getSignedUrl(client, new GetObjectCommand({ Bucket: bucket, Key: key }), { + expiresIn: expiresInSeconds, + }); +} + /** * Upload a local file's contents to an S3 URI using a streaming * `PutObjectCommand`. PutObject's 5 GB cap comfortably exceeds the diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 717d38f98c..eb90b6b760 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -297,7 +297,11 @@ export { quantizeTimeToFrame, type MediaVisualStyleProperty, } from "./inline-scripts/parityContract"; -export { redactKnownPaths, redactTelemetryString } from "./telemetryRedaction"; +export { + redactKnownPaths, + redactTelemetryString, + redactUrlQueryStrings, +} from "./telemetryRedaction"; export { folderGone, isProjectRootMissing, diff --git a/packages/core/src/telemetryRedaction.ts b/packages/core/src/telemetryRedaction.ts index 423a9d29e9..cc93adfd68 100644 --- a/packages/core/src/telemetryRedaction.ts +++ b/packages/core/src/telemetryRedaction.ts @@ -5,7 +5,7 @@ function truncateTelemetryString(value: string, maxLength: number): string { return `${value.slice(0, maxLength - 1)}…`; } -function redactUrlQueryStrings(value: string): string { +export function redactUrlQueryStrings(value: string): string { return value.replace(/\b(https?:\/\/[^\s?]+)\?[^\s]*/g, "$1?…"); } diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index b1573134f5..798cfded1b 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -45,6 +45,7 @@ import { type VideoMetadata, } from "../utils/ffprobe.js"; import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js"; +import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, @@ -816,7 +817,9 @@ export function parseImageElements(html: string): ImageElement[] { /** The first shown frame's colour tags; see {@link firstFrameColourArgs}. */ async function readFirstFrameColour(videoPath: string, signal?: AbortSignal) { - const result = await runFfmpeg(firstFrameColourArgs(videoPath), { signal }); + const result = await runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)], { + signal, + }); return parseFirstFrameColour(result.stderr); } @@ -928,6 +931,16 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise +): Promise { + try { + return await extractVideoFramesRangeUnredacted(...args); + } catch (error) { + throw redactUrlQueriesInError(error); + } +} + +async function extractVideoFramesRangeUnredacted( videoPath: string, videoId: string, startTime: number, @@ -987,7 +1000,7 @@ export async function extractVideoFramesRange( throw new Error("frameRanges extraction needs a constant-frame-rate source"); } - const args: string[] = []; + const args: string[] = urlInputArgs(videoPath); if (decodeWithVideoToolbox) { args.push("-hwaccel", "videotoolbox"); } diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index fe8d526aeb..7659580761 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -5,6 +5,7 @@ import { chunkCrc32 } from "./crc32.js"; import { StringDecoder } from "node:string_decoder"; import { basename } from "path"; import { redactTelemetryString } from "@hyperframes/core"; +import { urlInputArgs } from "./urlInput.js"; import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js"; import { ManagedChildProcess } from "./managedChildProcess.js"; import { trackChildProcess } from "./processTracker.js"; @@ -65,7 +66,15 @@ async function runFfprobe( } const command = getFfprobeBinary(); - const proc = spawn(command, ["-v", "error", ...argsWithoutInput, "--", filePath], { + const args = [ + "-v", + "error", + ...urlInputArgs(filePath, { tool: "ffprobe" }), + ...argsWithoutInput, + "--", + filePath, + ]; + const proc = spawn(command, args, { // Nothing is ever written to the child's stdin; leaving it as a pipe is // what lets a stdin-reading invocation block indefinitely. stdio: ["ignore", "pipe", "pipe"], diff --git a/packages/engine/src/utils/urlInput.test.ts b/packages/engine/src/utils/urlInput.test.ts new file mode 100644 index 0000000000..c51711420b --- /dev/null +++ b/packages/engine/src/utils/urlInput.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { VideoSourceExtractionError } from "../services/videoFrameExtractor.js"; +import { redactUrlQueriesInError, urlInputArgs } from "./urlInput.js"; + +const signed = + "https://bucket.s3.us-east-2.amazonaws.com/a/b?X-Amz-Credential=AKID&X-Amz-Signature=deadbeef"; + +describe("urlInputArgs", () => { + const caBundles = ["/nonexistent.crt", import.meta.filename]; + const verifyAndReconnect = [ + "-tls_verify", + "1", + "-ca_file", + import.meta.filename, + "-rw_timeout", + "30000000", + "-reconnect", + "1", + "-reconnect_delay_max", + "2", + ]; + + it("makes ffmpeg verify the certificate of an https input and reconnect on network errors", () => { + expect(urlInputArgs(signed, { caBundles })).toEqual([ + ...verifyAndReconnect, + "-reconnect_on_network_error", + "1", + ]); + }); + + it("gives ffprobe only the options its 4.0 build accepts", () => { + expect(urlInputArgs(signed, { tool: "ffprobe", caBundles })).toEqual(verifyAndReconnect); + }); + + it("adds nothing for local paths and plain http", () => { + expect(urlInputArgs("/plan/compiled/bg.mp4", { caBundles: [] })).toEqual([]); + expect(urlInputArgs("http://127.0.0.1:8080/bg.mp4", { caBundles: [] })).toEqual([]); + }); + + it("refuses an https input when no CA bundle exists", () => { + expect(() => urlInputArgs(signed, { caBundles: ["/nonexistent.crt"] })).toThrow(/no CA bundle/); + }); +}); + +describe("redactUrlQueriesInError", () => { + it("strips a signed URL's query from the message, diagnostic and stack", () => { + const error = redactUrlQueriesInError( + new VideoSourceExtractionError("ffmpeg_failed", false, `read ${signed}`, `${signed}: 403`), + ); + for (const text of [error.message, error.diagnostic, error.stack]) { + expect(text).toContain("https://bucket.s3.us-east-2.amazonaws.com/a/b?"); + expect(text).not.toContain("deadbeef"); + } + }); + + it("also strips it from a wrapped cause", () => { + const error = redactUrlQueriesInError( + new Error("render failed", { cause: new Error(`read ${signed}`) }), + ); + expect(String((error.cause as Error).message)).not.toContain("deadbeef"); + }); +}); diff --git a/packages/engine/src/utils/urlInput.ts b/packages/engine/src/utils/urlInput.ts new file mode 100644 index 0000000000..32cff701da --- /dev/null +++ b/packages/engine/src/utils/urlInput.ts @@ -0,0 +1,42 @@ +import { existsSync } from "node:fs"; +import { redactUrlQueryStrings } from "@hyperframes/core"; + +// The Lambda nodejs image (Amazon Linux 2023) ships the first; Debian/Ubuntu the second; macOS the third. +const CA_BUNDLES = [ + "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/certs/ca-certificates.crt", + "/etc/ssl/cert.pem", +]; + +// Lambda's ffprobe 4.0 accepts these but not -reconnect_on_network_error; a stalled read fails after 30 s. +const RECONNECT_ARGS = ["-rw_timeout", "30000000", "-reconnect", "1", "-reconnect_delay_max", "2"]; + +/** ffmpeg and ffprobe skip TLS certificate checks unless told to; an https input must be verified. */ +export function urlInputArgs( + input: string, + { + tool = "ffmpeg", + caBundles = CA_BUNDLES, + }: { tool?: "ffmpeg" | "ffprobe"; caBundles?: readonly string[] } = {}, +): string[] { + if (!/^https:\/\//i.test(input)) return []; + const caFile = caBundles.find((path) => existsSync(path)); + if (!caFile) { + throw new Error( + `[FFmpeg] cannot verify TLS for an https input: no CA bundle at ${caBundles.join(", ")}`, + ); + } + const networkRetry = tool === "ffmpeg" ? ["-reconnect_on_network_error", "1"] : []; + return ["-tls_verify", "1", "-ca_file", caFile, ...RECONNECT_ARGS, ...networkRetry]; +} + +/** A signed URL grants access until it expires: strip query strings from every string field of an error and its causes. */ +export function redactUrlQueriesInError(error: T): T { + if (!(error instanceof Error)) return error; + const fields = error as unknown as Record; + for (const key of ["message", "stack", ...Object.keys(error)]) { + if (typeof fields[key] === "string") fields[key] = redactUrlQueryStrings(fields[key]); + } + if (error.cause !== error) redactUrlQueriesInError(error.cause); + return error; +} diff --git a/packages/producer/src/distributed.ts b/packages/producer/src/distributed.ts index 42d278423b..3e4bcca0a2 100644 --- a/packages/producer/src/distributed.ts +++ b/packages/producer/src/distributed.ts @@ -60,6 +60,7 @@ export { createPlanV2FromExecutionPlan, createPlanV2FromV1, getPlanV2ExecutionPlanHash, + isPlanV2DeferredSourceArtifact, listPlanV2ArtifactsForTarget, materializePlanV2Target, planV2, @@ -74,8 +75,10 @@ export { type PlanV2Artifact, type PlanV2Limitations, type PlanV2Manifest, + type PlanV2MaterializeOptions, type PlanV2MaterializationResult, type PlanV2MaterializationTarget, + type PlanV2RemoteSources, type PlanV2Result, type PlanV2WithPublisherOptions, } from "./services/distributed/planV2.js"; diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.ts index 9c1be46b88..de5e957c8e 100644 --- a/packages/producer/src/services/distributed/planV2.test.ts +++ b/packages/producer/src/services/distributed/planV2.test.ts @@ -27,8 +27,10 @@ import { publishPlanV2FromV1, publishPlanV2FromExecutionPlan, readPlanV2Manifest, + readValidatedPlanV2Materialization, validatePlanV2MaterializedTarget, } from "./planV2.js"; +import { planV2BlobPath } from "./planV2Layout.js"; import { LocalPlanV2ArtifactPublisher, type PlanV2ArtifactPublisher } from "./planV2Publisher.js"; import { buildPlanVideosJson, type PlanVideosJson } from "./shared.js"; @@ -450,20 +452,60 @@ describe("Plan v2 manifest", () => { ); }); - it("keeps a deferred source the page itself loads in every chunk", () => { + it("ships each compiled/ video source only to the chunks that show it", () => { const root = tempPath("hf-plan-v2-deferred-compiled-source-"); const v1 = createV1Plan(root, { video: true, videoStart: 1 / 30, deferredSource: "compiled/hero.mp4", }); + // A second clip shown in both chunks, also shipped from compiled/. + const videosPath = join(v1, "meta", "videos.json"); + const videos = JSON.parse(readFileSync(videosPath, "utf-8")) as PlanVideosJson; + const hero = videos.extracted[0]!; + videos.videos.push({ ...videos.videos[0]!, id: "long", src: "long.mp4", start: 0 }); + videos.extracted.push({ + ...hero, + videoId: "long", + deferredRange: { ...hero.deferredRange!, sourcePath: "compiled/long.mp4" }, + }); + writeFileSync(videosPath, JSON.stringify(videos)); + writeFileSync(join(v1, "compiled", "long.mp4"), "long source video"); + refreshV1PlanHash(v1); const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir); + const chunksOf = (path: string) => + manifest.artifacts.find((artifact) => artifact.path === path)?.chunks; expect(manifest.limitations.videoDependencyMode).toBe("source-extract"); - for (const chunkIndex of [0, 1]) { - const artifacts = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex }); - expect(artifacts.some((artifact) => artifact.path === "compiled/hero.mp4")).toBe(true); - } + expect(chunksOf("compiled/hero.mp4")).toEqual([1]); + expect(chunksOf("compiled/long.mp4")).toEqual([0, 1]); + expect(chunksOf("compiled/index.html")).toBe("all"); + expect(chunksOf("compiled/asset.txt")).toBe("all"); + + const chunk0Dir = join(root, "chunk-0"); + materializePlanV2Target(join(root, "v2"), { role: "chunk", chunkIndex: 0 }, chunk0Dir); + expect(existsSync(join(chunk0Dir, "compiled", "hero.mp4"))).toBe(false); + expect(existsSync(join(chunk0Dir, "compiled", "long.mp4"))).toBe(true); + expect(validatePlanV2MaterializedTarget(chunk0Dir, { role: "chunk", chunkIndex: 0 })).not.toBe( + null, + ); + + // A source read by URL is neither verified locally nor copied, and is recorded for the chunk. + const longDigest = manifest.artifacts.find((a) => a.path === "compiled/long.mp4")!.sha256; + rmSync(planV2BlobPath(join(root, "v2"), longDigest)); + const remoteSources = { "compiled/long.mp4": "https://example.test/long.mp4" }; + const byUrlDir = join(root, "chunk-0-by-url"); + const chunk0 = { role: "chunk", chunkIndex: 0 } as const; + materializePlanV2Target(join(root, "v2"), chunk0, byUrlDir, { remoteSources }); + expect(existsSync(join(byUrlDir, "compiled", "long.mp4"))).toBe(false); + expect(readValidatedPlanV2Materialization(byUrlDir, chunk0)?.remoteSources).toEqual( + remoteSources, + ); + expect(() => + materializePlanV2Target(join(root, "v2"), chunk0, join(root, "chunk-0-stray"), { + remoteSources: { "compiled/index.html": "https://example.test/index.html" }, + }), + ).toThrow(/not a selected deferred video/); }); it("rejects a deferred video source outside the plan directory", () => { diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.ts index 50230652ee..a3a00f35c8 100644 --- a/packages/producer/src/services/distributed/planV2.ts +++ b/packages/producer/src/services/distributed/planV2.ts @@ -161,6 +161,13 @@ export interface PlanV2MaterializationResult { readonly audioPath: string | null; } +export type PlanV2RemoteSources = Readonly>; + +export interface PlanV2MaterializeOptions { + /** Deferred video sources the chunk reads by URL; the caller vouches for their bytes. */ + readonly remoteSources?: PlanV2RemoteSources; +} + function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } @@ -275,8 +282,18 @@ function artifactTargets( if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") { return { chunks: "all", assembler: true }; } - // Each chunk's page loads every