From f0fb7aa5a89713382d162eb6ce530704898475a7 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Sun, 15 Nov 2026 19:11:17 +0000 Subject: [PATCH] fix(engine): extraction-time video downloads get the not-media and format checks (#7838) * fix(engine): extraction-time video downloads get the not-media check A video src that is still remote at frame extraction is downloaded again and then probed. That copy now gets the checks other media downloads get before anything reads it as media: - the not-media sniff, so an HTML, XML or JSON body (including a DASH manifest) is refused with the existing NotMediaPayloadError, classified as invalid_media; - on a render service (PRODUCER_RESTRICT_PAGE_NETWORK), the same -format_whitelist an https input gets, so a playlist, manifest or concat list fails to open whatever its bytes say, and no file it names is read. Real mp4, mov and webm downloads extract the same frames as before. * fix(engine): the download format check stops when the render is cancelled * fix(engine): the not-media sniff keeps mp4 files whose box size reads as text An ISO BMFF file opens with a 4-byte box size and a box type. A size of 60 is the byte `<`, so after the leading NULs were skipped a valid mp4 with a 60-byte ftyp was refused as a text document. The sniff now recognises an mp4/mov box header (size 0, 1 or at least 8, then ftyp, styp, moov, moof, mdat, free, skip, wide, sidx or pdin) before it skips NULs. HTML, XML, JSON and playlist refusals are unchanged. * fix(engine): the mp4 exception in the not-media sniff needs a box that fits the file The sniff only skips its document check for an mp4/mov opening when the first box is ftyp or styp, its declared size (including a 64-bit size) fits the file length, and the next box header, when the sniff window holds it, is a known media box that also fits. A document that only spells a box header falls through to the document and playlist checks as before. * fix(engine): the not-media sniff keeps mp4 files with leading padding or extension boxes An mp4 or QuickTime file may open with free, skip, wide, uuid, pdin, moov, mdat or moof as well as ftyp or styp, and may carry any box type after it. The sniff now accepts any of those first boxes, then walks every box header the sniff window holds: each needs a printable four-character type and a size that fits the file. A document that only spells a box header still falls through to the document and playlist checks. --- ...ideoFrameExtractor.downloadPayload.test.ts | 196 ++++++++++++++++++ .../src/services/videoFrameExtractor.ts | 29 ++- .../engine/src/utils/notMediaPayload.test.ts | 59 ++++++ packages/engine/src/utils/notMediaPayload.ts | 66 +++++- packages/engine/src/utils/urlInput.ts | 6 +- 5 files changed, 345 insertions(+), 11 deletions(-) create mode 100644 packages/engine/src/services/videoFrameExtractor.downloadPayload.test.ts diff --git a/packages/engine/src/services/videoFrameExtractor.downloadPayload.test.ts b/packages/engine/src/services/videoFrameExtractor.downloadPayload.test.ts new file mode 100644 index 0000000000..3bfad62b78 --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.downloadPayload.test.ts @@ -0,0 +1,196 @@ +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import type { VideoElement } from "../types.js"; +import { runFfmpeg } from "../utils/runFfmpeg.js"; + +// The download hands back `served.main`, with `served.beside` saved next to it. +const served = vi.hoisted(() => ({ main: "", beside: [] as string[] })); +vi.mock("../utils/urlDownloader.js", async (importOriginal) => ({ + ...(await importOriginal()), + downloadToTemp: async (_url: string, destDir: string) => { + for (const file of served.beside) copyFileSync(file, join(destDir, basename(file))); + const local = join(destDir, "download.mp4"); + copyFileSync(served.main, local); + return local; + }, +})); + +const probed = vi.hoisted(() => ({ + paths: [] as string[], + signals: [] as (AbortSignal | undefined)[], +})); +vi.mock("../utils/ffprobe.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + runFfprobe: (path: string, args: string[], signal?: AbortSignal) => { + probed.signals.push(signal); + return actual.runFfprobe(path, args, signal); + }, + extractMediaMetadata: (path: string) => { + probed.paths.push(path); + return actual.extractMediaMetadata(path); + }, + }; +}); + +const { extractAllVideoFrames } = await import("./videoFrameExtractor.js"); + +describe.skipIf(process.platform === "win32")("extractAllVideoFrames downloaded payloads", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-extract-download-payload-")); + let renders = 0; + let mp4 = ""; + let mov = ""; + let webm = ""; + let segment = ""; + let wideFtyp = ""; + let leadingFree = ""; + let uuidAfterFtyp = ""; + + async function synth(name: string, codec: string[]): Promise { + const clip = join(dir, name); + const made = await runFfmpeg([ + ...["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc2=s=32x16:d=1:r=30"], + ...codec, + clip, + ]); + expect(made.success, made.stderr).toBe(true); + return clip; + } + + function box(type: string, size: number, fill = 0): Buffer { + const bytes = Buffer.alloc(size, fill); + bytes.writeUInt32BE(size, 0); + bytes.write(type, 4, "latin1"); + return bytes; + } + + // A fragmented mp4 addresses its samples from each moof, so boxes added ahead of them shift nothing they name. + async function fragmentedWith(name: string, h264: string[], edit: (ftyp: Buffer) => Buffer[]) { + const fragmented = await synth(`${name}-source.mp4`, [ + ...h264, + ...["-movflags", "frag_keyframe+empty_moov+default_base_moof"], + ]); + const bytes = readFileSync(fragmented); + expect(bytes.toString("latin1", 4, 8)).toBe("ftyp"); + const ftypSize = bytes.readUInt32BE(0); + const path = join(dir, `${name}.mp4`); + const rest = bytes.subarray(ftypSize); + writeFileSync(path, Buffer.concat([...edit(bytes.subarray(0, ftypSize)), rest])); + return path; + } + + function sixtyByteFtyp(): Buffer { + const ftyp = box("ftyp", 60); + ftyp.write("isom", 8, "latin1"); + ftyp.writeUInt32BE(0x200, 12); + ftyp.write("isomiso2avc1mp41iso5iso6dashmsdhmsixcmfcmp42", 16, "latin1"); + return ftyp; + } + + function write(name: string, text: string): string { + const path = join(dir, name); + writeFileSync(path, text); + return path; + } + + function render(main: string, beside: string[] = [], signal?: AbortSignal) { + served.main = main; + served.beside = beside; + probed.paths = []; + probed.signals = []; + const outputDir = join(dir, `render-${++renders}`); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "clip", + src: "https://cdn.example.com/clip.mp4", + ...{ start: 0, end: 1, mediaStart: 0, loop: false, hasAudio: false }, + }; + return extractAllVideoFrames( + [video], + dir, + { fps: 30, outputDir, collectProbeFailures: true }, + signal, + ); + } + + function framesOf(result: Awaited>): Buffer[] { + const extracted = result.extracted[0]; + if (!extracted) throw new Error("nothing extracted"); + return [...extracted.framePaths.values()].map((path) => readFileSync(path)); + } + + beforeAll(async () => { + const h264 = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]; + mp4 = await synth("real.mp4", h264); + mov = await synth("real.mov", h264); + webm = await synth("real.webm", ["-c:v", "libvpx-vp9", "-deadline", "realtime"]); + segment = await synth("segment.ts", h264); + wideFtyp = await fragmentedWith("wide-ftyp", h264, () => [sixtyByteFtyp()]); + leadingFree = await fragmentedWith("leading-free", h264, (ftyp) => [box("free", 60), ftyp]); + uuidAfterFtyp = await fragmentedWith("uuid", h264, () => [ + sixtyByteFtyp(), + box("uuid", 24, 0x5a), + ]); + }); + + afterEach(() => vi.unstubAllEnvs()); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it("refuses a downloaded manifest as not media before any probe reads it", async () => { + const manifest = write( + "manifest.mpd", + '\n' + + `${mp4}` + + "\n", + ); + const result = await render(manifest); + expect(result.extracted).toEqual([]); + expect(result.errors).toEqual([ + expect.objectContaining({ videoId: "clip", kind: "invalid_media", retryable: false }), + ]); + expect(result.errors[0]?.error).toContain("text documents"); + expect(probed.paths).toEqual([]); + }); + + it("on a render service refuses a downloaded playlist naming a file beside it", async () => { + vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true"); + const playlist = write( + "playlist.m3u8", + `#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXTINF:1,\n${basename(segment)}\n#EXT-X-ENDLIST\n`, + ); + const result = await render(playlist, [segment]); + expect(result.extracted).toEqual([]); + expect(result.errors).toEqual([ + expect.objectContaining({ videoId: "clip", kind: "invalid_media", retryable: false }), + ]); + expect(probed.paths).toEqual([]); + }); + + it("on a render service the download's format check runs under the render's cancel signal", async () => { + vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true"); + const cancel = new AbortController(); + const result = await render(mp4, [], cancel.signal); + expect(result.errors).toEqual([]); + expect(probed.signals[0]).toBe(cancel.signal); + }); + + it.each([ + ["mp4", () => mp4], + ["mov", () => mov], + ["webm", () => webm], + ["fragmented mp4 with a 60-byte ftyp", () => wideFtyp], + ["fragmented mp4 opening with a 60-byte free box", () => leadingFree], + ["fragmented mp4 with a uuid box after a 60-byte ftyp", () => uuidAfterFtyp], + ])("extracts a real %s download the same on a render service", async (_kind, clip) => { + const local = await render(clip()); + vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true"); + const restricted = await render(clip()); + expect(local.errors).toEqual([]); + expect(restricted.errors).toEqual([]); + expect(restricted.extracted[0]?.totalFrames).toBe(30); + expect(framesOf(restricted)).toEqual(framesOf(local)); + }); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 4bcf2a4530..1758fe1a87 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -58,13 +58,19 @@ import { readMetadataCache, readStoredFinalFrame, resolvePlayableVideoDuration, + runFfprobe, videoStartOnMediaTimeline, writeMetadataCache, type MediaProbeCacheEntry, type VideoMetadata, } from "../utils/ffprobe.js"; import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js"; -import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js"; +import { + redactUrlQueriesInError, + selfContainedFormatArgs, + urlInputArgs, +} from "../utils/urlInput.js"; +import { assertMediaPayload, NotMediaPayloadError } from "../utils/notMediaPayload.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, @@ -320,6 +326,18 @@ export interface ExtractionOptions { const DOWNLOAD_SUBDIR = "_downloads"; +/** A downloaded video passes the not-media sniff, and on a render service opens only as a self-contained container, before anything reads it as media. */ +async function assertDownloadedVideoIsMedia( + path: string, + videoId: string, + signal?: AbortSignal, +): Promise { + await assertMediaPayload(path, videoId); + const formatLimit = selfContainedFormatArgs(); + if (formatLimit.length === 0) return; + await runFfprobe(path, [...formatLimit, "-show_entries", "format=format_name"], signal); +} + export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000; const GC_STALENESS_MS = 24 * 60 * 60 * 1000; const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"; @@ -649,6 +667,14 @@ export function classifyVideoExtractionError(error: unknown): VideoSourceExtract diagnostic, ); } + if (error instanceof NotMediaPayloadError) { + return new VideoSourceExtractionError( + "invalid_media", + false, + "Video source is a text document, not media", + diagnostic, + ); + } if (lowered.includes("cancelled") || lowered.includes("aborted")) { return new VideoSourceExtractionError( "cancelled", @@ -2267,6 +2293,7 @@ export async function extractAllVideoFrames( () => recordTransientRetries(1), { onTelemetry: writeUrlDownloadTelemetry }, ); + await assertDownloadedVideoIsMedia(videoPath, video.id, signal); } if (!existsSync(videoPath)) { diff --git a/packages/engine/src/utils/notMediaPayload.test.ts b/packages/engine/src/utils/notMediaPayload.test.ts index 5342bdbb84..83a584054a 100644 --- a/packages/engine/src/utils/notMediaPayload.test.ts +++ b/packages/engine/src/utils/notMediaPayload.test.ts @@ -104,6 +104,65 @@ describe("isNotMediaPayload", () => { expect(await isNotMediaPayload(writeFixture("clip.bin", Buffer.from(bytes)))).toBe(false); }); + function box(type: string, size: number, body = Buffer.alloc(Math.max(size - 8, 0))): Buffer { + const header = Buffer.alloc(8); + header.writeUInt32BE(size, 0); + header.write(type, 4, "latin1"); + return Buffer.concat([header, body]); + } + + function largeBox(type: string, size: bigint, body: Buffer): Buffer { + const header = Buffer.alloc(16); + header.writeUInt32BE(1, 0); + header.write(type, 4, "latin1"); + header.writeBigUInt64BE(size, 8); + return Buffer.concat([header, body]); + } + + it.each([ + ["<", 0x3c, "ftyp"], + ["#", 0x23, "ftyp"], + ["{", 0x7b, "styp"], + ["[", 0x5b, "ftyp"], + ["space", 0x20, "ftyp"], + ["newline", 0x0a, "styp"], + ])("does not flag an mp4 whose box size reads as %s", async (_label, size, type) => { + const mp4 = Buffer.concat([box(type, size), box("moov", 16)]); + expect(await isNotMediaPayload(writeFixture("clip.mp4", mp4))).toBe(false); + }); + + it.each([ + ["leading padding", [box("free", 0x3c), box("ftyp", 0x3c), box("moov", 16)]], + ["an extension box after ftyp", [box("ftyp", 0x3c), box("uuid", 24), box("moov", 16)]], + ["a QuickTime movie opening with wide", [box("wide", 8), box("mdat", 0x3c), box("moov", 16)]], + ["a QuickTime metadata box", [box("ftyp", 0x3c), box("\xa9nam", 12), box("moov", 16)]], + ])("does not flag an mp4 with %s", async (_label, boxes) => { + expect(await isNotMediaPayload(writeFixture("clip.mp4", Buffer.concat(boxes)))).toBe(false); + }); + + it("does not flag an mp4 whose second box runs to the end or has a 64-bit size", async () => { + const toEnd = Buffer.concat([box("ftyp", 0x3c), box("mdat", 0), Buffer.from("")]); + const large = Buffer.concat([box("ftyp", 0x3c), largeBox("mdat", 24n, Buffer.alloc(8))]); + expect(await isNotMediaPayload(writeFixture("to-end.mp4", toEnd))).toBe(false); + expect(await isNotMediaPayload(writeFixture("large.mp4", large))).toBe(false); + }); + + it.each([ + ["a box larger than the file", box("ftyp", 0x3c, Buffer.from("hi"))], + [ + "a box followed by text", + Buffer.concat([box("ftyp", 0x3c), Buffer.from("")]), + ], + [ + "a second box whose 64-bit size runs past the file", + Buffer.concat([box("ftyp", 0x3c), largeBox("mdat", 1n << 40n, Buffer.from(""))]), + ], + ["a box type no mp4 opens with", box("html", 0x3c, Buffer.from("".padEnd(52)))], + ["an XML manifest behind NULs", Buffer.from('\0\0\0', "latin1")], + ])("still detects a document shaped like %s", async (_label, bytes) => { + expect(await isNotMediaPayload(writeFixture("fake.mp4", bytes))).toBe(true); + }); + it("does not flag a container that merely contains a document byte further in", async () => { const filePath = writeFixture( "not-html.bin", diff --git a/packages/engine/src/utils/notMediaPayload.ts b/packages/engine/src/utils/notMediaPayload.ts index 4e58fbb722..906401a043 100644 --- a/packages/engine/src/utils/notMediaPayload.ts +++ b/packages/engine/src/utils/notMediaPayload.ts @@ -66,9 +66,8 @@ const SNIFF_BYTES = 512; // error body, which some gateways return with a 200. const DOCUMENT_OPENING_BYTES = new Set([0x3c, 0x7b, 0x5b]); // NUL is skippable so UTF-16-encoded text (`3C 00 68 00 …`) is caught, and so -// is a payload padded with NULs. No supported container is defeated by this: -// mp4/mov open with a box size whose first non-NUL byte is the size itself, and -// MPEG-PS `00 00 01 BA` stops at 0x01. +// is a payload padded with NULs. An mp4/mov box header is recognised before +// skipping, since its size byte can read as `<`; MPEG-PS `00 00 01 BA` stops at 0x01. const SKIPPABLE_LEADING_BYTES = new Set([0x00, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20]); const BYTE_ORDER_MARKS = [ Buffer.from([0xef, 0xbb, 0xbf]), // UTF-8 @@ -116,18 +115,63 @@ async function openRegular(path: string | Buffer): Promise { return null; } -/** Read up to {@link SNIFF_BYTES} from the front of the file; null when it is not a regular file. */ -async function readHead(filePath: string): Promise { +/** Read up to {@link SNIFF_BYTES} from the front of the file, with its length; null when it is not a regular file. */ +async function readHead(filePath: string): Promise<{ head: Buffer; fileSize: number } | null> { const fh = await openRegular(filePath); if (!fh) return null; try { - return await readAt(fh, SNIFF_BYTES, 0); + const { size } = await fh.stat(); + return { head: await readAt(fh, SNIFF_BYTES, 0), fileSize: size }; } finally { await fh.close().catch(() => {}); } } -function startsWithDocumentByte(head: Buffer): boolean { +const FIRST_BOX_TYPES = new Set([ + "ftyp", + "styp", + "free", + "skip", + "wide", + "uuid", + "pdin", + "moov", + "mdat", + "moof", +]); +const BOX_TYPE = /^[\x21-\x7e\xa9][\x21-\x7e]{3}$/; + +/** A box's declared length, from its 64-bit size when its size is 1; -1 when shorter than its own header. */ +function boxLength(head: Buffer, offset: number): number { + const size = head.readUInt32BE(offset); + if (size !== 1) return size >= 8 ? size : -1; + if (offset + 16 > head.length) return -1; + const largeSize = Number(head.readBigUInt64BE(offset + 8)); + return largeSize >= 16 ? largeSize : -1; +} + +/** Where the box at `offset` ends, or -1 when no box header with a printable type that fits the file is there. */ +function boxEnd(head: Buffer, offset: number, fileSize: number): number { + if (offset + 8 > head.length || !BOX_TYPE.test(head.toString("latin1", offset + 4, offset + 8))) { + return -1; + } + if (head.readUInt32BE(offset) === 0) return fileSize; + const length = boxLength(head, offset); + return length >= 0 && offset + length <= fileSize ? offset + length : -1; +} + +/** An mp4/mov opening: a box an mp4 or QuickTime file can start with, then boxes that each fit the file, as far as the sniff window holds their headers. */ +function opensWithMediaBox(head: Buffer, fileSize: number): boolean { + if (!FIRST_BOX_TYPES.has(head.toString("latin1", 4, 8))) return false; + let offset = boxEnd(head, 0, fileSize); + while (offset >= 0 && offset < fileSize && offset + 16 <= head.length) { + offset = boxEnd(head, offset, fileSize); + } + return offset >= 0; +} + +function startsWithDocumentByte(head: Buffer, fileSize: number): boolean { + if (opensWithMediaBox(head, fileSize)) return false; for (let index = byteOrderMarkLength(head); index < head.length; index++) { const byte = head[index]; if (byte === undefined) break; @@ -157,8 +201,12 @@ function startsWithDocumentByte(head: Buffer): boolean { */ export async function isNotMediaPayload(filePath: string): Promise { try { - const head = await readHead(filePath); - return !head || startsWithDocumentByte(head) || (await playlistLeavesItsFolder(filePath)); + const sniffed = await readHead(filePath); + return ( + !sniffed || + startsWithDocumentByte(sniffed.head, sniffed.fileSize) || + (await playlistLeavesItsFolder(filePath)) + ); } catch { return false; } diff --git a/packages/engine/src/utils/urlInput.ts b/packages/engine/src/utils/urlInput.ts index 47d237fdb0..b2c26dde12 100644 --- a/packages/engine/src/utils/urlInput.ts +++ b/packages/engine/src/utils/urlInput.ts @@ -15,6 +15,10 @@ const RECONNECT_ARGS = ["-rw_timeout", "30000000", "-reconnect", "1", "-reconnec const SELF_CONTAINED_FORMATS = "mov,matroska,mxf,avi,mpeg,mpegts,mpegvideo,flv,asf,ogg,wav,mp3,aac,flac,aiff,gif,image2,png_pipe,jpeg_pipe,webp_pipe,bmp_pipe,tiff_pipe"; +export function selfContainedFormatArgs(): string[] { + return resolveRestrictPageNetwork() ? ["-format_whitelist", SELF_CONTAINED_FORMATS] : []; +} + /** ffmpeg and ffprobe skip TLS certificate checks unless told to; an https input must be verified, and on a render service be a self-contained container that names only HTTPS. */ export function urlInputArgs( input: string, @@ -32,7 +36,7 @@ export function urlInputArgs( } const networkRetry = tool === "ffmpeg" ? ["-reconnect_on_network_error", "1"] : []; const renderServiceLimits = resolveRestrictPageNetwork() - ? ["-protocol_whitelist", "https,tls,tcp", "-format_whitelist", SELF_CONTAINED_FORMATS] + ? ["-protocol_whitelist", "https,tls,tcp", ...selfContainedFormatArgs()] : []; return [ "-tls_verify",