fix(engine): extraction-time video downloads get the not-media and format checks (#7838)

* fix(engine): extraction-time video downloads get the not-media check

A video src that is still remote at frame extraction is downloaded again and
then probed. That copy now gets the checks other media downloads get before
anything reads it as media:

- the not-media sniff, so an HTML, XML or JSON body (including a DASH
  manifest) is refused with the existing NotMediaPayloadError, classified as
  invalid_media;
- on a render service (PRODUCER_RESTRICT_PAGE_NETWORK), the same
  -format_whitelist an https input gets, so a playlist, manifest or concat
  list fails to open whatever its bytes say, and no file it names is read.

Real mp4, mov and webm downloads extract the same frames as before.

* fix(engine): the download format check stops when the render is cancelled

* fix(engine): the not-media sniff keeps mp4 files whose box size reads as text

An ISO BMFF file opens with a 4-byte box size and a box type. A size of 60
is the byte `<`, so after the leading NULs were skipped a valid mp4 with a
60-byte ftyp was refused as a text document. The sniff now recognises an
mp4/mov box header (size 0, 1 or at least 8, then ftyp, styp, moov, moof,
mdat, free, skip, wide, sidx or pdin) before it skips NULs. HTML, XML,
JSON and playlist refusals are unchanged.

* fix(engine): the mp4 exception in the not-media sniff needs a box that fits the file

The sniff only skips its document check for an mp4/mov opening when the
first box is ftyp or styp, its declared size (including a 64-bit size) fits
the file length, and the next box header, when the sniff window holds it, is
a known media box that also fits. A document that only spells a box header
falls through to the document and playlist checks as before.

* fix(engine): the not-media sniff keeps mp4 files with leading padding or extension boxes

An mp4 or QuickTime file may open with free, skip, wide, uuid, pdin, moov,
mdat or moof as well as ftyp or styp, and may carry any box type after it.
The sniff now accepts any of those first boxes, then walks every box header
the sniff window holds: each needs a printable four-character type and a
size that fits the file. A document that only spells a box header still
falls through to the document and playlist checks.
This commit is contained in:
Michael Wang
2026-11-15 19:11:17 +00:00
committed by GitHub
parent c46143dd79
commit f0fb7aa5a8
5 changed files with 345 additions and 11 deletions
@@ -0,0 +1,196 @@
import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import type { VideoElement } from "../types.js";
import { runFfmpeg } from "../utils/runFfmpeg.js";
// The download hands back `served.main`, with `served.beside` saved next to it.
const served = vi.hoisted(() => ({ main: "", beside: [] as string[] }));
vi.mock("../utils/urlDownloader.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../utils/urlDownloader.js")>()),
downloadToTemp: async (_url: string, destDir: string) => {
for (const file of served.beside) copyFileSync(file, join(destDir, basename(file)));
const local = join(destDir, "download.mp4");
copyFileSync(served.main, local);
return local;
},
}));
const probed = vi.hoisted(() => ({
paths: [] as string[],
signals: [] as (AbortSignal | undefined)[],
}));
vi.mock("../utils/ffprobe.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../utils/ffprobe.js")>();
return {
...actual,
runFfprobe: (path: string, args: string[], signal?: AbortSignal) => {
probed.signals.push(signal);
return actual.runFfprobe(path, args, signal);
},
extractMediaMetadata: (path: string) => {
probed.paths.push(path);
return actual.extractMediaMetadata(path);
},
};
});
const { extractAllVideoFrames } = await import("./videoFrameExtractor.js");
describe.skipIf(process.platform === "win32")("extractAllVideoFrames downloaded payloads", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-extract-download-payload-"));
let renders = 0;
let mp4 = "";
let mov = "";
let webm = "";
let segment = "";
let wideFtyp = "";
let leadingFree = "";
let uuidAfterFtyp = "";
async function synth(name: string, codec: string[]): Promise<string> {
const clip = join(dir, name);
const made = await runFfmpeg([
...["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc2=s=32x16:d=1:r=30"],
...codec,
clip,
]);
expect(made.success, made.stderr).toBe(true);
return clip;
}
function box(type: string, size: number, fill = 0): Buffer {
const bytes = Buffer.alloc(size, fill);
bytes.writeUInt32BE(size, 0);
bytes.write(type, 4, "latin1");
return bytes;
}
// A fragmented mp4 addresses its samples from each moof, so boxes added ahead of them shift nothing they name.
async function fragmentedWith(name: string, h264: string[], edit: (ftyp: Buffer) => Buffer[]) {
const fragmented = await synth(`${name}-source.mp4`, [
...h264,
...["-movflags", "frag_keyframe+empty_moov+default_base_moof"],
]);
const bytes = readFileSync(fragmented);
expect(bytes.toString("latin1", 4, 8)).toBe("ftyp");
const ftypSize = bytes.readUInt32BE(0);
const path = join(dir, `${name}.mp4`);
const rest = bytes.subarray(ftypSize);
writeFileSync(path, Buffer.concat([...edit(bytes.subarray(0, ftypSize)), rest]));
return path;
}
function sixtyByteFtyp(): Buffer {
const ftyp = box("ftyp", 60);
ftyp.write("isom", 8, "latin1");
ftyp.writeUInt32BE(0x200, 12);
ftyp.write("isomiso2avc1mp41iso5iso6dashmsdhmsixcmfcmp42", 16, "latin1");
return ftyp;
}
function write(name: string, text: string): string {
const path = join(dir, name);
writeFileSync(path, text);
return path;
}
function render(main: string, beside: string[] = [], signal?: AbortSignal) {
served.main = main;
served.beside = beside;
probed.paths = [];
probed.signals = [];
const outputDir = join(dir, `render-${++renders}`);
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "clip",
src: "https://cdn.example.com/clip.mp4",
...{ start: 0, end: 1, mediaStart: 0, loop: false, hasAudio: false },
};
return extractAllVideoFrames(
[video],
dir,
{ fps: 30, outputDir, collectProbeFailures: true },
signal,
);
}
function framesOf(result: Awaited<ReturnType<typeof render>>): Buffer[] {
const extracted = result.extracted[0];
if (!extracted) throw new Error("nothing extracted");
return [...extracted.framePaths.values()].map((path) => readFileSync(path));
}
beforeAll(async () => {
const h264 = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"];
mp4 = await synth("real.mp4", h264);
mov = await synth("real.mov", h264);
webm = await synth("real.webm", ["-c:v", "libvpx-vp9", "-deadline", "realtime"]);
segment = await synth("segment.ts", h264);
wideFtyp = await fragmentedWith("wide-ftyp", h264, () => [sixtyByteFtyp()]);
leadingFree = await fragmentedWith("leading-free", h264, (ftyp) => [box("free", 60), ftyp]);
uuidAfterFtyp = await fragmentedWith("uuid", h264, () => [
sixtyByteFtyp(),
box("uuid", 24, 0x5a),
]);
});
afterEach(() => vi.unstubAllEnvs());
afterAll(() => rmSync(dir, { recursive: true, force: true }));
it("refuses a downloaded manifest as not media before any probe reads it", async () => {
const manifest = write(
"manifest.mpd",
'<?xml version="1.0"?>\n<MPD profiles="urn:mpeg:dash:profile:isoff-on-demand:2011">' +
`<Period><AdaptationSet><Representation id="1"><BaseURL>${mp4}</BaseURL>` +
"</Representation></AdaptationSet></Period></MPD>\n",
);
const result = await render(manifest);
expect(result.extracted).toEqual([]);
expect(result.errors).toEqual([
expect.objectContaining({ videoId: "clip", kind: "invalid_media", retryable: false }),
]);
expect(result.errors[0]?.error).toContain("text documents");
expect(probed.paths).toEqual([]);
});
it("on a render service refuses a downloaded playlist naming a file beside it", async () => {
vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true");
const playlist = write(
"playlist.m3u8",
`#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXTINF:1,\n${basename(segment)}\n#EXT-X-ENDLIST\n`,
);
const result = await render(playlist, [segment]);
expect(result.extracted).toEqual([]);
expect(result.errors).toEqual([
expect.objectContaining({ videoId: "clip", kind: "invalid_media", retryable: false }),
]);
expect(probed.paths).toEqual([]);
});
it("on a render service the download's format check runs under the render's cancel signal", async () => {
vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true");
const cancel = new AbortController();
const result = await render(mp4, [], cancel.signal);
expect(result.errors).toEqual([]);
expect(probed.signals[0]).toBe(cancel.signal);
});
it.each([
["mp4", () => mp4],
["mov", () => mov],
["webm", () => webm],
["fragmented mp4 with a 60-byte ftyp", () => wideFtyp],
["fragmented mp4 opening with a 60-byte free box", () => leadingFree],
["fragmented mp4 with a uuid box after a 60-byte ftyp", () => uuidAfterFtyp],
])("extracts a real %s download the same on a render service", async (_kind, clip) => {
const local = await render(clip());
vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true");
const restricted = await render(clip());
expect(local.errors).toEqual([]);
expect(restricted.errors).toEqual([]);
expect(restricted.extracted[0]?.totalFrames).toBe(30);
expect(framesOf(restricted)).toEqual(framesOf(local));
});
});
@@ -58,13 +58,19 @@ import {
readMetadataCache,
readStoredFinalFrame,
resolvePlayableVideoDuration,
runFfprobe,
videoStartOnMediaTimeline,
writeMetadataCache,
type MediaProbeCacheEntry,
type VideoMetadata,
} from "../utils/ffprobe.js";
import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js";
import {
redactUrlQueriesInError,
selfContainedFormatArgs,
urlInputArgs,
} from "../utils/urlInput.js";
import { assertMediaPayload, NotMediaPayloadError } from "../utils/notMediaPayload.js";
import {
analyzeCompositionHdr,
isHdrColorSpace as isHdrColorSpaceUtil,
@@ -320,6 +326,18 @@ export interface ExtractionOptions {
const DOWNLOAD_SUBDIR = "_downloads";
/** A downloaded video passes the not-media sniff, and on a render service opens only as a self-contained container, before anything reads it as media. */
async function assertDownloadedVideoIsMedia(
path: string,
videoId: string,
signal?: AbortSignal,
): Promise<void> {
await assertMediaPayload(path, videoId);
const formatLimit = selfContainedFormatArgs();
if (formatLimit.length === 0) return;
await runFfprobe(path, [...formatLimit, "-show_entries", "format=format_name"], signal);
}
export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
@@ -649,6 +667,14 @@ export function classifyVideoExtractionError(error: unknown): VideoSourceExtract
diagnostic,
);
}
if (error instanceof NotMediaPayloadError) {
return new VideoSourceExtractionError(
"invalid_media",
false,
"Video source is a text document, not media",
diagnostic,
);
}
if (lowered.includes("cancelled") || lowered.includes("aborted")) {
return new VideoSourceExtractionError(
"cancelled",
@@ -2267,6 +2293,7 @@ export async function extractAllVideoFrames(
() => recordTransientRetries(1),
{ onTelemetry: writeUrlDownloadTelemetry },
);
await assertDownloadedVideoIsMedia(videoPath, video.id, signal);
}
if (!existsSync(videoPath)) {
@@ -104,6 +104,65 @@ describe("isNotMediaPayload", () => {
expect(await isNotMediaPayload(writeFixture("clip.bin", Buffer.from(bytes)))).toBe(false);
});
function box(type: string, size: number, body = Buffer.alloc(Math.max(size - 8, 0))): Buffer {
const header = Buffer.alloc(8);
header.writeUInt32BE(size, 0);
header.write(type, 4, "latin1");
return Buffer.concat([header, body]);
}
function largeBox(type: string, size: bigint, body: Buffer): Buffer {
const header = Buffer.alloc(16);
header.writeUInt32BE(1, 0);
header.write(type, 4, "latin1");
header.writeBigUInt64BE(size, 8);
return Buffer.concat([header, body]);
}
it.each([
["<", 0x3c, "ftyp"],
["#", 0x23, "ftyp"],
["{", 0x7b, "styp"],
["[", 0x5b, "ftyp"],
["space", 0x20, "ftyp"],
["newline", 0x0a, "styp"],
])("does not flag an mp4 whose box size reads as %s", async (_label, size, type) => {
const mp4 = Buffer.concat([box(type, size), box("moov", 16)]);
expect(await isNotMediaPayload(writeFixture("clip.mp4", mp4))).toBe(false);
});
it.each([
["leading padding", [box("free", 0x3c), box("ftyp", 0x3c), box("moov", 16)]],
["an extension box after ftyp", [box("ftyp", 0x3c), box("uuid", 24), box("moov", 16)]],
["a QuickTime movie opening with wide", [box("wide", 8), box("mdat", 0x3c), box("moov", 16)]],
["a QuickTime metadata box", [box("ftyp", 0x3c), box("\xa9nam", 12), box("moov", 16)]],
])("does not flag an mp4 with %s", async (_label, boxes) => {
expect(await isNotMediaPayload(writeFixture("clip.mp4", Buffer.concat(boxes)))).toBe(false);
});
it("does not flag an mp4 whose second box runs to the end or has a 64-bit size", async () => {
const toEnd = Buffer.concat([box("ftyp", 0x3c), box("mdat", 0), Buffer.from("<html>")]);
const large = Buffer.concat([box("ftyp", 0x3c), largeBox("mdat", 24n, Buffer.alloc(8))]);
expect(await isNotMediaPayload(writeFixture("to-end.mp4", toEnd))).toBe(false);
expect(await isNotMediaPayload(writeFixture("large.mp4", large))).toBe(false);
});
it.each([
["a box larger than the file", box("ftyp", 0x3c, Buffer.from("<html><body>hi</body></html>"))],
[
"a box followed by text",
Buffer.concat([box("ftyp", 0x3c), Buffer.from("<!doctype html><html></html>")]),
],
[
"a second box whose 64-bit size runs past the file",
Buffer.concat([box("ftyp", 0x3c), largeBox("mdat", 1n << 40n, Buffer.from("<html>"))]),
],
["a box type no mp4 opens with", box("html", 0x3c, Buffer.from("<?xml?>".padEnd(52)))],
["an XML manifest behind NULs", Buffer.from('\0\0\0<?xml version="1.0"?><MPD/>', "latin1")],
])("still detects a document shaped like %s", async (_label, bytes) => {
expect(await isNotMediaPayload(writeFixture("fake.mp4", bytes))).toBe(true);
});
it("does not flag a container that merely contains a document byte further in", async () => {
const filePath = writeFixture(
"not-html.bin",
+57 -9
View File
@@ -66,9 +66,8 @@ const SNIFF_BYTES = 512;
// error body, which some gateways return with a 200.
const DOCUMENT_OPENING_BYTES = new Set([0x3c, 0x7b, 0x5b]);
// NUL is skippable so UTF-16-encoded text (`3C 00 68 00 …`) is caught, and so
// is a payload padded with NULs. No supported container is defeated by this:
// mp4/mov open with a box size whose first non-NUL byte is the size itself, and
// MPEG-PS `00 00 01 BA` stops at 0x01.
// is a payload padded with NULs. An mp4/mov box header is recognised before
// skipping, since its size byte can read as `<`; MPEG-PS `00 00 01 BA` stops at 0x01.
const SKIPPABLE_LEADING_BYTES = new Set([0x00, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20]);
const BYTE_ORDER_MARKS = [
Buffer.from([0xef, 0xbb, 0xbf]), // UTF-8
@@ -116,18 +115,63 @@ async function openRegular(path: string | Buffer): Promise<FileHandle | null> {
return null;
}
/** Read up to {@link SNIFF_BYTES} from the front of the file; null when it is not a regular file. */
async function readHead(filePath: string): Promise<Buffer | null> {
/** Read up to {@link SNIFF_BYTES} from the front of the file, with its length; null when it is not a regular file. */
async function readHead(filePath: string): Promise<{ head: Buffer; fileSize: number } | null> {
const fh = await openRegular(filePath);
if (!fh) return null;
try {
return await readAt(fh, SNIFF_BYTES, 0);
const { size } = await fh.stat();
return { head: await readAt(fh, SNIFF_BYTES, 0), fileSize: size };
} finally {
await fh.close().catch(() => {});
}
}
function startsWithDocumentByte(head: Buffer): boolean {
const FIRST_BOX_TYPES = new Set([
"ftyp",
"styp",
"free",
"skip",
"wide",
"uuid",
"pdin",
"moov",
"mdat",
"moof",
]);
const BOX_TYPE = /^[\x21-\x7e\xa9][\x21-\x7e]{3}$/;
/** A box's declared length, from its 64-bit size when its size is 1; -1 when shorter than its own header. */
function boxLength(head: Buffer, offset: number): number {
const size = head.readUInt32BE(offset);
if (size !== 1) return size >= 8 ? size : -1;
if (offset + 16 > head.length) return -1;
const largeSize = Number(head.readBigUInt64BE(offset + 8));
return largeSize >= 16 ? largeSize : -1;
}
/** Where the box at `offset` ends, or -1 when no box header with a printable type that fits the file is there. */
function boxEnd(head: Buffer, offset: number, fileSize: number): number {
if (offset + 8 > head.length || !BOX_TYPE.test(head.toString("latin1", offset + 4, offset + 8))) {
return -1;
}
if (head.readUInt32BE(offset) === 0) return fileSize;
const length = boxLength(head, offset);
return length >= 0 && offset + length <= fileSize ? offset + length : -1;
}
/** An mp4/mov opening: a box an mp4 or QuickTime file can start with, then boxes that each fit the file, as far as the sniff window holds their headers. */
function opensWithMediaBox(head: Buffer, fileSize: number): boolean {
if (!FIRST_BOX_TYPES.has(head.toString("latin1", 4, 8))) return false;
let offset = boxEnd(head, 0, fileSize);
while (offset >= 0 && offset < fileSize && offset + 16 <= head.length) {
offset = boxEnd(head, offset, fileSize);
}
return offset >= 0;
}
function startsWithDocumentByte(head: Buffer, fileSize: number): boolean {
if (opensWithMediaBox(head, fileSize)) return false;
for (let index = byteOrderMarkLength(head); index < head.length; index++) {
const byte = head[index];
if (byte === undefined) break;
@@ -157,8 +201,12 @@ function startsWithDocumentByte(head: Buffer): boolean {
*/
export async function isNotMediaPayload(filePath: string): Promise<boolean> {
try {
const head = await readHead(filePath);
return !head || startsWithDocumentByte(head) || (await playlistLeavesItsFolder(filePath));
const sniffed = await readHead(filePath);
return (
!sniffed ||
startsWithDocumentByte(sniffed.head, sniffed.fileSize) ||
(await playlistLeavesItsFolder(filePath))
);
} catch {
return false;
}
+5 -1
View File
@@ -15,6 +15,10 @@ const RECONNECT_ARGS = ["-rw_timeout", "30000000", "-reconnect", "1", "-reconnec
const SELF_CONTAINED_FORMATS =
"mov,matroska,mxf,avi,mpeg,mpegts,mpegvideo,flv,asf,ogg,wav,mp3,aac,flac,aiff,gif,image2,png_pipe,jpeg_pipe,webp_pipe,bmp_pipe,tiff_pipe";
export function selfContainedFormatArgs(): string[] {
return resolveRestrictPageNetwork() ? ["-format_whitelist", SELF_CONTAINED_FORMATS] : [];
}
/** ffmpeg and ffprobe skip TLS certificate checks unless told to; an https input must be verified, and on a render service be a self-contained container that names only HTTPS. */
export function urlInputArgs(
input: string,
@@ -32,7 +36,7 @@ export function urlInputArgs(
}
const networkRetry = tool === "ffmpeg" ? ["-reconnect_on_network_error", "1"] : [];
const renderServiceLimits = resolveRestrictPageNetwork()
? ["-protocol_whitelist", "https,tls,tcp", "-format_whitelist", SELF_CONTAINED_FORMATS]
? ["-protocol_whitelist", "https,tls,tcp", ...selfContainedFormatArgs()]
: [];
return [
"-tls_verify",