diff --git a/packages/engine/src/services/videoFrameExtractor.downloadPayload.test.ts b/packages/engine/src/services/videoFrameExtractor.downloadPayload.test.ts new file mode 100644 index 0000000000..3bfad62b78 --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.downloadPayload.test.ts @@ -0,0 +1,196 @@ +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import type { VideoElement } from "../types.js"; +import { runFfmpeg } from "../utils/runFfmpeg.js"; + +// The download hands back `served.main`, with `served.beside` saved next to it. +const served = vi.hoisted(() => ({ main: "", beside: [] as string[] })); +vi.mock("../utils/urlDownloader.js", async (importOriginal) => ({ + ...(await importOriginal()), + downloadToTemp: async (_url: string, destDir: string) => { + for (const file of served.beside) copyFileSync(file, join(destDir, basename(file))); + const local = join(destDir, "download.mp4"); + copyFileSync(served.main, local); + return local; + }, +})); + +const probed = vi.hoisted(() => ({ + paths: [] as string[], + signals: [] as (AbortSignal | undefined)[], +})); +vi.mock("../utils/ffprobe.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + runFfprobe: (path: string, args: string[], signal?: AbortSignal) => { + probed.signals.push(signal); + return actual.runFfprobe(path, args, signal); + }, + extractMediaMetadata: (path: string) => { + probed.paths.push(path); + return actual.extractMediaMetadata(path); + }, + }; +}); + +const { extractAllVideoFrames } = await import("./videoFrameExtractor.js"); + +describe.skipIf(process.platform === "win32")("extractAllVideoFrames downloaded payloads", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-extract-download-payload-")); + let renders = 0; + let mp4 = ""; + let mov = ""; + let webm = ""; + let segment = ""; + let wideFtyp = ""; + let leadingFree = ""; + let uuidAfterFtyp = ""; + + async function synth(name: string, codec: string[]): Promise { + const clip = join(dir, name); + const made = await runFfmpeg([ + ...["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc2=s=32x16:d=1:r=30"], + ...codec, + clip, + ]); + expect(made.success, made.stderr).toBe(true); + return clip; + } + + function box(type: string, size: number, fill = 0): Buffer { + const bytes = Buffer.alloc(size, fill); + bytes.writeUInt32BE(size, 0); + bytes.write(type, 4, "latin1"); + return bytes; + } + + // A fragmented mp4 addresses its samples from each moof, so boxes added ahead of them shift nothing they name. + async function fragmentedWith(name: string, h264: string[], edit: (ftyp: Buffer) => Buffer[]) { + const fragmented = await synth(`${name}-source.mp4`, [ + ...h264, + ...["-movflags", "frag_keyframe+empty_moov+default_base_moof"], + ]); + const bytes = readFileSync(fragmented); + expect(bytes.toString("latin1", 4, 8)).toBe("ftyp"); + const ftypSize = bytes.readUInt32BE(0); + const path = join(dir, `${name}.mp4`); + const rest = bytes.subarray(ftypSize); + writeFileSync(path, Buffer.concat([...edit(bytes.subarray(0, ftypSize)), rest])); + return path; + } + + function sixtyByteFtyp(): Buffer { + const ftyp = box("ftyp", 60); + ftyp.write("isom", 8, "latin1"); + ftyp.writeUInt32BE(0x200, 12); + ftyp.write("isomiso2avc1mp41iso5iso6dashmsdhmsixcmfcmp42", 16, "latin1"); + return ftyp; + } + + function write(name: string, text: string): string { + const path = join(dir, name); + writeFileSync(path, text); + return path; + } + + function render(main: string, beside: string[] = [], signal?: AbortSignal) { + served.main = main; + served.beside = beside; + probed.paths = []; + probed.signals = []; + const outputDir = join(dir, `render-${++renders}`); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "clip", + src: "https://cdn.example.com/clip.mp4", + ...{ start: 0, end: 1, mediaStart: 0, loop: false, hasAudio: false }, + }; + return extractAllVideoFrames( + [video], + dir, + { fps: 30, outputDir, collectProbeFailures: true }, + signal, + ); + } + + function framesOf(result: Awaited>): Buffer[] { + const extracted = result.extracted[0]; + if (!extracted) throw new Error("nothing extracted"); + return [...extracted.framePaths.values()].map((path) => readFileSync(path)); + } + + beforeAll(async () => { + const h264 = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]; + mp4 = await synth("real.mp4", h264); + mov = await synth("real.mov", h264); + webm = await synth("real.webm", ["-c:v", "libvpx-vp9", "-deadline", "realtime"]); + segment = await synth("segment.ts", h264); + wideFtyp = await fragmentedWith("wide-ftyp", h264, () => [sixtyByteFtyp()]); + leadingFree = await fragmentedWith("leading-free", h264, (ftyp) => [box("free", 60), ftyp]); + uuidAfterFtyp = await fragmentedWith("uuid", h264, () => [ + sixtyByteFtyp(), + box("uuid", 24, 0x5a), + ]); + }); + + afterEach(() => vi.unstubAllEnvs()); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it("refuses a downloaded manifest as not media before any probe reads it", async () => { + const manifest = write( + "manifest.mpd", + '\n' + + `${mp4}` + + "\n", + ); + const result = await render(manifest); + expect(result.extracted).toEqual([]); + expect(result.errors).toEqual([ + expect.objectContaining({ videoId: "clip", kind: "invalid_media", retryable: false }), + ]); + expect(result.errors[0]?.error).toContain("text documents"); + expect(probed.paths).toEqual([]); + }); + + it("on a render service refuses a downloaded playlist naming a file beside it", async () => { + vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true"); + const playlist = write( + "playlist.m3u8", + `#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXTINF:1,\n${basename(segment)}\n#EXT-X-ENDLIST\n`, + ); + const result = await render(playlist, [segment]); + expect(result.extracted).toEqual([]); + expect(result.errors).toEqual([ + expect.objectContaining({ videoId: "clip", kind: "invalid_media", retryable: false }), + ]); + expect(probed.paths).toEqual([]); + }); + + it("on a render service the download's format check runs under the render's cancel signal", async () => { + vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true"); + const cancel = new AbortController(); + const result = await render(mp4, [], cancel.signal); + expect(result.errors).toEqual([]); + expect(probed.signals[0]).toBe(cancel.signal); + }); + + it.each([ + ["mp4", () => mp4], + ["mov", () => mov], + ["webm", () => webm], + ["fragmented mp4 with a 60-byte ftyp", () => wideFtyp], + ["fragmented mp4 opening with a 60-byte free box", () => leadingFree], + ["fragmented mp4 with a uuid box after a 60-byte ftyp", () => uuidAfterFtyp], + ])("extracts a real %s download the same on a render service", async (_kind, clip) => { + const local = await render(clip()); + vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true"); + const restricted = await render(clip()); + expect(local.errors).toEqual([]); + expect(restricted.errors).toEqual([]); + expect(restricted.extracted[0]?.totalFrames).toBe(30); + expect(framesOf(restricted)).toEqual(framesOf(local)); + }); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 4bcf2a4530..1758fe1a87 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -58,13 +58,19 @@ import { readMetadataCache, readStoredFinalFrame, resolvePlayableVideoDuration, + runFfprobe, videoStartOnMediaTimeline, writeMetadataCache, type MediaProbeCacheEntry, type VideoMetadata, } from "../utils/ffprobe.js"; import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js"; -import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js"; +import { + redactUrlQueriesInError, + selfContainedFormatArgs, + urlInputArgs, +} from "../utils/urlInput.js"; +import { assertMediaPayload, NotMediaPayloadError } from "../utils/notMediaPayload.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, @@ -320,6 +326,18 @@ export interface ExtractionOptions { const DOWNLOAD_SUBDIR = "_downloads"; +/** A downloaded video passes the not-media sniff, and on a render service opens only as a self-contained container, before anything reads it as media. */ +async function assertDownloadedVideoIsMedia( + path: string, + videoId: string, + signal?: AbortSignal, +): Promise { + await assertMediaPayload(path, videoId); + const formatLimit = selfContainedFormatArgs(); + if (formatLimit.length === 0) return; + await runFfprobe(path, [...formatLimit, "-show_entries", "format=format_name"], signal); +} + export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000; const GC_STALENESS_MS = 24 * 60 * 60 * 1000; const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"; @@ -649,6 +667,14 @@ export function classifyVideoExtractionError(error: unknown): VideoSourceExtract diagnostic, ); } + if (error instanceof NotMediaPayloadError) { + return new VideoSourceExtractionError( + "invalid_media", + false, + "Video source is a text document, not media", + diagnostic, + ); + } if (lowered.includes("cancelled") || lowered.includes("aborted")) { return new VideoSourceExtractionError( "cancelled", @@ -2267,6 +2293,7 @@ export async function extractAllVideoFrames( () => recordTransientRetries(1), { onTelemetry: writeUrlDownloadTelemetry }, ); + await assertDownloadedVideoIsMedia(videoPath, video.id, signal); } if (!existsSync(videoPath)) { diff --git a/packages/engine/src/utils/notMediaPayload.test.ts b/packages/engine/src/utils/notMediaPayload.test.ts index 5342bdbb84..83a584054a 100644 --- a/packages/engine/src/utils/notMediaPayload.test.ts +++ b/packages/engine/src/utils/notMediaPayload.test.ts @@ -104,6 +104,65 @@ describe("isNotMediaPayload", () => { expect(await isNotMediaPayload(writeFixture("clip.bin", Buffer.from(bytes)))).toBe(false); }); + function box(type: string, size: number, body = Buffer.alloc(Math.max(size - 8, 0))): Buffer { + const header = Buffer.alloc(8); + header.writeUInt32BE(size, 0); + header.write(type, 4, "latin1"); + return Buffer.concat([header, body]); + } + + function largeBox(type: string, size: bigint, body: Buffer): Buffer { + const header = Buffer.alloc(16); + header.writeUInt32BE(1, 0); + header.write(type, 4, "latin1"); + header.writeBigUInt64BE(size, 8); + return Buffer.concat([header, body]); + } + + it.each([ + ["<", 0x3c, "ftyp"], + ["#", 0x23, "ftyp"], + ["{", 0x7b, "styp"], + ["[", 0x5b, "ftyp"], + ["space", 0x20, "ftyp"], + ["newline", 0x0a, "styp"], + ])("does not flag an mp4 whose box size reads as %s", async (_label, size, type) => { + const mp4 = Buffer.concat([box(type, size), box("moov", 16)]); + expect(await isNotMediaPayload(writeFixture("clip.mp4", mp4))).toBe(false); + }); + + it.each([ + ["leading padding", [box("free", 0x3c), box("ftyp", 0x3c), box("moov", 16)]], + ["an extension box after ftyp", [box("ftyp", 0x3c), box("uuid", 24), box("moov", 16)]], + ["a QuickTime movie opening with wide", [box("wide", 8), box("mdat", 0x3c), box("moov", 16)]], + ["a QuickTime metadata box", [box("ftyp", 0x3c), box("\xa9nam", 12), box("moov", 16)]], + ])("does not flag an mp4 with %s", async (_label, boxes) => { + expect(await isNotMediaPayload(writeFixture("clip.mp4", Buffer.concat(boxes)))).toBe(false); + }); + + it("does not flag an mp4 whose second box runs to the end or has a 64-bit size", async () => { + const toEnd = Buffer.concat([box("ftyp", 0x3c), box("mdat", 0), Buffer.from("")]); + const large = Buffer.concat([box("ftyp", 0x3c), largeBox("mdat", 24n, Buffer.alloc(8))]); + expect(await isNotMediaPayload(writeFixture("to-end.mp4", toEnd))).toBe(false); + expect(await isNotMediaPayload(writeFixture("large.mp4", large))).toBe(false); + }); + + it.each([ + ["a box larger than the file", box("ftyp", 0x3c, Buffer.from("hi"))], + [ + "a box followed by text", + Buffer.concat([box("ftyp", 0x3c), Buffer.from("")]), + ], + [ + "a second box whose 64-bit size runs past the file", + Buffer.concat([box("ftyp", 0x3c), largeBox("mdat", 1n << 40n, Buffer.from(""))]), + ], + ["a box type no mp4 opens with", box("html", 0x3c, Buffer.from("".padEnd(52)))], + ["an XML manifest behind NULs", Buffer.from('\0\0\0', "latin1")], + ])("still detects a document shaped like %s", async (_label, bytes) => { + expect(await isNotMediaPayload(writeFixture("fake.mp4", bytes))).toBe(true); + }); + it("does not flag a container that merely contains a document byte further in", async () => { const filePath = writeFixture( "not-html.bin", diff --git a/packages/engine/src/utils/notMediaPayload.ts b/packages/engine/src/utils/notMediaPayload.ts index 4e58fbb722..906401a043 100644 --- a/packages/engine/src/utils/notMediaPayload.ts +++ b/packages/engine/src/utils/notMediaPayload.ts @@ -66,9 +66,8 @@ const SNIFF_BYTES = 512; // error body, which some gateways return with a 200. const DOCUMENT_OPENING_BYTES = new Set([0x3c, 0x7b, 0x5b]); // NUL is skippable so UTF-16-encoded text (`3C 00 68 00 …`) is caught, and so -// is a payload padded with NULs. No supported container is defeated by this: -// mp4/mov open with a box size whose first non-NUL byte is the size itself, and -// MPEG-PS `00 00 01 BA` stops at 0x01. +// is a payload padded with NULs. An mp4/mov box header is recognised before +// skipping, since its size byte can read as `<`; MPEG-PS `00 00 01 BA` stops at 0x01. const SKIPPABLE_LEADING_BYTES = new Set([0x00, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20]); const BYTE_ORDER_MARKS = [ Buffer.from([0xef, 0xbb, 0xbf]), // UTF-8 @@ -116,18 +115,63 @@ async function openRegular(path: string | Buffer): Promise { return null; } -/** Read up to {@link SNIFF_BYTES} from the front of the file; null when it is not a regular file. */ -async function readHead(filePath: string): Promise { +/** Read up to {@link SNIFF_BYTES} from the front of the file, with its length; null when it is not a regular file. */ +async function readHead(filePath: string): Promise<{ head: Buffer; fileSize: number } | null> { const fh = await openRegular(filePath); if (!fh) return null; try { - return await readAt(fh, SNIFF_BYTES, 0); + const { size } = await fh.stat(); + return { head: await readAt(fh, SNIFF_BYTES, 0), fileSize: size }; } finally { await fh.close().catch(() => {}); } } -function startsWithDocumentByte(head: Buffer): boolean { +const FIRST_BOX_TYPES = new Set([ + "ftyp", + "styp", + "free", + "skip", + "wide", + "uuid", + "pdin", + "moov", + "mdat", + "moof", +]); +const BOX_TYPE = /^[\x21-\x7e\xa9][\x21-\x7e]{3}$/; + +/** A box's declared length, from its 64-bit size when its size is 1; -1 when shorter than its own header. */ +function boxLength(head: Buffer, offset: number): number { + const size = head.readUInt32BE(offset); + if (size !== 1) return size >= 8 ? size : -1; + if (offset + 16 > head.length) return -1; + const largeSize = Number(head.readBigUInt64BE(offset + 8)); + return largeSize >= 16 ? largeSize : -1; +} + +/** Where the box at `offset` ends, or -1 when no box header with a printable type that fits the file is there. */ +function boxEnd(head: Buffer, offset: number, fileSize: number): number { + if (offset + 8 > head.length || !BOX_TYPE.test(head.toString("latin1", offset + 4, offset + 8))) { + return -1; + } + if (head.readUInt32BE(offset) === 0) return fileSize; + const length = boxLength(head, offset); + return length >= 0 && offset + length <= fileSize ? offset + length : -1; +} + +/** An mp4/mov opening: a box an mp4 or QuickTime file can start with, then boxes that each fit the file, as far as the sniff window holds their headers. */ +function opensWithMediaBox(head: Buffer, fileSize: number): boolean { + if (!FIRST_BOX_TYPES.has(head.toString("latin1", 4, 8))) return false; + let offset = boxEnd(head, 0, fileSize); + while (offset >= 0 && offset < fileSize && offset + 16 <= head.length) { + offset = boxEnd(head, offset, fileSize); + } + return offset >= 0; +} + +function startsWithDocumentByte(head: Buffer, fileSize: number): boolean { + if (opensWithMediaBox(head, fileSize)) return false; for (let index = byteOrderMarkLength(head); index < head.length; index++) { const byte = head[index]; if (byte === undefined) break; @@ -157,8 +201,12 @@ function startsWithDocumentByte(head: Buffer): boolean { */ export async function isNotMediaPayload(filePath: string): Promise { try { - const head = await readHead(filePath); - return !head || startsWithDocumentByte(head) || (await playlistLeavesItsFolder(filePath)); + const sniffed = await readHead(filePath); + return ( + !sniffed || + startsWithDocumentByte(sniffed.head, sniffed.fileSize) || + (await playlistLeavesItsFolder(filePath)) + ); } catch { return false; } diff --git a/packages/engine/src/utils/urlInput.ts b/packages/engine/src/utils/urlInput.ts index 47d237fdb0..b2c26dde12 100644 --- a/packages/engine/src/utils/urlInput.ts +++ b/packages/engine/src/utils/urlInput.ts @@ -15,6 +15,10 @@ const RECONNECT_ARGS = ["-rw_timeout", "30000000", "-reconnect", "1", "-reconnec const SELF_CONTAINED_FORMATS = "mov,matroska,mxf,avi,mpeg,mpegts,mpegvideo,flv,asf,ogg,wav,mp3,aac,flac,aiff,gif,image2,png_pipe,jpeg_pipe,webp_pipe,bmp_pipe,tiff_pipe"; +export function selfContainedFormatArgs(): string[] { + return resolveRestrictPageNetwork() ? ["-format_whitelist", SELF_CONTAINED_FORMATS] : []; +} + /** ffmpeg and ffprobe skip TLS certificate checks unless told to; an https input must be verified, and on a render service be a self-contained container that names only HTTPS. */ export function urlInputArgs( input: string, @@ -32,7 +36,7 @@ export function urlInputArgs( } const networkRetry = tool === "ffmpeg" ? ["-reconnect_on_network_error", "1"] : []; const renderServiceLimits = resolveRestrictPageNetwork() - ? ["-protocol_whitelist", "https,tls,tcp", "-format_whitelist", SELF_CONTAINED_FORMATS] + ? ["-protocol_whitelist", "https,tls,tcp", ...selfContainedFormatArgs()] : []; return [ "-tls_verify",