mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(engine,cli): the playlist folder check reads HLS as ffmpeg does; snapshot runs it too (#7810)
* fix(engine): the media playlist check reads HLS lines and attributes as ffmpeg does The folder check now finds every name ffmpeg's hls demuxer would open: - lines end at CR, LF or NUL, and a line is also read cut at ffmpeg's 4095-byte limit; - URI attributes are split like ff_parse_key_value: quoted or bare values, with backslash escapes removed; - leading ID3v2 tags are skipped before the playlist signature, and a file behind more than 16 of them is refused. Each refused shape is checked against ffprobe 8 in the test: ffprobe opens the outside file, and playlistLeavesItsFolder now refuses the list. * fix(cli): snapshot never hands FFmpeg a project video that is a document or an escaping playlist `hyperframes snapshot` decodes held project videos with FFmpeg and FFprobe. It now runs the engine's media check first: a file that is a document, or an HLS or ffconcat list naming files outside its folder (under any extension), is skipped with a "not media" warning and never reaches FFprobe or FFmpeg. Real clips and HLS lists of segments beside them snapshot as before. * fix(engine): playlist check reads lines as bytes and refuses unreadable nested lists - Playlist text is read as latin1, one character per byte, so a line is cut at 4095 bytes exactly where ffmpeg cuts it, even when it holds bytes that are not UTF-8. Nested names are opened by those same bytes. - A nested name that exists but cannot be read is refused; a name that does not exist, or is too long to exist, is still kept. - Each name is checked once, each file is opened once (ID3 header and head in one read), and each file's verdict is reused within a call: 100k segments naming one file drop from seconds to tens of milliseconds. * perf(cli): snapshot checks each project video once and warns once when it is refused * test(engine): the ffprobe playlist proof ends its options with -- before the input * chore(engine): raise the packed size budget to 500000 bytes The byte-exact HLS playlist parser adds about 6 KB to the packed engine (496647 in CI against a 490452 budget). * fix(engine): media inspection never waits on a FIFO or device The document sniff and the playlist walk open with O_RDONLY|O_NONBLOCK where the platform has it, fstat the descriptor, and refuse anything that is not a regular file before reading; a folder still fails as it did. A FIFO with no writer, as a media source or named in a list, is refused at once. * fix(cli): snapshot inspects project videos within the probe deadline and its cleanup scope The held-video hints are gathered inside the capture's cleanup scope, and every media inspection gives up after the same 30 s limit as the FFprobe call it guards, treating the file as not media.
This commit is contained in:
@@ -112,6 +112,7 @@ import {
|
||||
localVideos,
|
||||
probeHeldVideo,
|
||||
projectLocalFile,
|
||||
projectVideoFile,
|
||||
type HeldVideoProbe,
|
||||
} from "./snapshotHeldVideos.js";
|
||||
import type { Example } from "./_examples.js";
|
||||
@@ -1015,12 +1016,12 @@ export async function captureSnapshots(
|
||||
const fps = compositionFps(html);
|
||||
const heldVideoIds = localVideoRenderIds(html);
|
||||
const heldProbes = new Map<string, Promise<HeldVideoProbe>>();
|
||||
const heldHints = await heldVideoHints(projectDir, localVideos(html), heldProbes);
|
||||
|
||||
const savedPaths: string[] = [];
|
||||
let pairSheetFailure: string | null = null;
|
||||
|
||||
try {
|
||||
const heldHints = await heldVideoHints(projectDir, localVideos(html), heldProbes);
|
||||
const {
|
||||
browser: chromeBrowser,
|
||||
page,
|
||||
@@ -1217,6 +1218,21 @@ export async function captureSnapshots(
|
||||
remoteCopies.set(href, copy);
|
||||
return copy;
|
||||
};
|
||||
const projectVideos = new Map<string, ReturnType<typeof projectVideoFile>>();
|
||||
const checkedProjectVideo = (src: string): ReturnType<typeof projectVideoFile> => {
|
||||
const known = projectVideos.get(src);
|
||||
if (known) return known;
|
||||
const checked = projectVideoFile(projectDir, src).then((project) => {
|
||||
if (project?.refused) {
|
||||
const message = `${relative(projectDir, project.file)} is not media: a document, a pipe or device, or a playlist naming files outside its folder`;
|
||||
warnings.push(message);
|
||||
console.warn(` ${c.warn("⚠")} ${message}`);
|
||||
}
|
||||
return project;
|
||||
});
|
||||
projectVideos.set(src, checked);
|
||||
return checked;
|
||||
};
|
||||
const shouldUseVp9AlphaDecoder = (filePath: string): Promise<boolean> => {
|
||||
if (!extractMediaMetadata) return Promise.resolve(false);
|
||||
const cached = alphaDecoderCache.get(filePath);
|
||||
@@ -1335,9 +1351,13 @@ export async function captureSnapshots(
|
||||
// since headless Chrome can't seek them and they would render blank.
|
||||
let ffmpegInput: string | null = null;
|
||||
let inputIsLocal = false;
|
||||
const projectFile = projectLocalFile(projectDir, v.src);
|
||||
if (projectFile) {
|
||||
ffmpegInput = projectFile;
|
||||
const project = await checkedProjectVideo(v.src);
|
||||
if (project?.refused) {
|
||||
unreadVideoFrames += 1;
|
||||
return null;
|
||||
}
|
||||
if (project) {
|
||||
ffmpegInput = project.file;
|
||||
inputIsLocal = true;
|
||||
} else {
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -12,6 +13,7 @@ import {
|
||||
localVideos,
|
||||
probeHeldVideo,
|
||||
projectLocalFile,
|
||||
projectVideoFile,
|
||||
type HeldVideoProbe,
|
||||
} from "./snapshotHeldVideos.js";
|
||||
|
||||
@@ -140,3 +142,76 @@ describe("heldVideoHints", () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("projectVideoFile", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-held-playlist-"));
|
||||
const dir = join(root, "project");
|
||||
mkdirSync(join(root, "outside"));
|
||||
mkdirSync(dir);
|
||||
writeFileSync(join(root, "outside", "secret.mp4"), "outside media");
|
||||
writeFileSync(join(dir, "clip.mp4"), Buffer.from([0, 0, 0, 0x20, 0x66, 0x74, 0x79, 0x70]));
|
||||
const list = (entry: string) =>
|
||||
`#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXTINF:1,\n${entry}\n#EXT-X-ENDLIST\n`;
|
||||
writeFileSync(join(dir, "escape.m3u8"), list("../outside/secret.mp4"));
|
||||
writeFileSync(join(dir, "escape.mp4"), list(join(root, "outside", "secret.mp4")));
|
||||
writeFileSync(join(dir, "stream.m3u8"), list("seg0.ts"));
|
||||
afterAll(() => rmSync(root, { recursive: true, force: true }));
|
||||
beforeEach(() => {
|
||||
run.mockReset();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a playlist naming a file above the project", "escape.m3u8"],
|
||||
["a playlist behind a media extension", "escape.mp4"],
|
||||
])("refuses %s, and FFprobe never reads it", async (_label, src) => {
|
||||
await expect(projectVideoFile(dir, src)).resolves.toEqual({
|
||||
file: join(dir, src),
|
||||
refused: true,
|
||||
});
|
||||
await expect(probeHeldVideo(join(dir, src), new Map(), "ffprobe")).resolves.toEqual({
|
||||
duration: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
});
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a media file", "clip.mp4"],
|
||||
["a playlist of segments beside it", "stream.m3u8"],
|
||||
])("keeps %s", async (_label, src) => {
|
||||
await expect(projectVideoFile(dir, src)).resolves.toEqual({
|
||||
file: join(dir, src),
|
||||
refused: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("gives null for a src outside the project", async () => {
|
||||
await expect(projectVideoFile(dir, "../outside/secret.mp4")).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === "win32")(
|
||||
"a project video that is a FIFO with no writer",
|
||||
() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-held-fifo-"));
|
||||
execFileSync("mkfifo", [join(dir, "pipe.mp4")]);
|
||||
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
||||
beforeEach(() => {
|
||||
run.mockReset();
|
||||
});
|
||||
|
||||
it("is refused for frame extraction without waiting for a writer", async () => {
|
||||
await expect(projectVideoFile(dir, "pipe.mp4")).resolves.toEqual({
|
||||
file: join(dir, "pipe.mp4"),
|
||||
refused: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("gets no held-video hint, and FFprobe never reads it", async () => {
|
||||
const videos = localVideos('<body><video id="pipe" src="pipe.mp4"></video></body>');
|
||||
await expect(heldVideoHints(dir, videos, new Map(), "ffprobe")).resolves.toEqual([]);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
|
||||
import { isAbsolute, relative, resolve } from "node:path";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { MEDIA_RENDER_ID_ATTR } from "@hyperframes/core";
|
||||
import type { CaptureVideoMetadataHint } from "@hyperframes/engine";
|
||||
import { isNotMediaPayload, type CaptureVideoMetadataHint } from "@hyperframes/engine";
|
||||
import { findFFprobe } from "../browser/ffmpeg.js";
|
||||
import { runCancellableProcess } from "../utils/cancellableProcess.js";
|
||||
|
||||
@@ -36,8 +36,29 @@ export function projectLocalFile(projectDir: string, src: string): string | null
|
||||
}
|
||||
}
|
||||
|
||||
export async function projectVideoFile(
|
||||
projectDir: string,
|
||||
src: string,
|
||||
): Promise<{ file: string; refused: boolean } | null> {
|
||||
const file = projectLocalFile(projectDir, src);
|
||||
return file ? { file, refused: !(await readableAsMediaWithinProbeLimit(file)) } : null;
|
||||
}
|
||||
|
||||
const PROBE_TIMEOUT_MS = 30_000;
|
||||
|
||||
async function readableAsMediaWithinProbeLimit(path: string): Promise<boolean> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const deadline = new Promise<boolean>((done) => {
|
||||
timer = setTimeout(() => done(false), PROBE_TIMEOUT_MS);
|
||||
timer.unref?.();
|
||||
});
|
||||
try {
|
||||
return await Promise.race([isNotMediaPayload(path).then((notMedia) => !notMedia), deadline]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HeldVideoProbe {
|
||||
duration: number;
|
||||
/** The size Chrome would lay the video out at: rotated footage swaps its coded width and height. */
|
||||
@@ -84,7 +105,7 @@ export function probeHeldVideo(
|
||||
const known = cache.get(path);
|
||||
if (known) return known;
|
||||
const read = (async () => {
|
||||
if (!ffprobe) return UNKNOWN_VIDEO;
|
||||
if (!ffprobe || !(await readableAsMediaWithinProbeLimit(path))) return UNKNOWN_VIDEO;
|
||||
try {
|
||||
const { stdout } = await runCancellableProcess(
|
||||
ffprobe,
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getFfprobeBinary } from "./ffmpegBinaries.js";
|
||||
import {
|
||||
assertMediaPayload,
|
||||
fingerprintElementId,
|
||||
@@ -17,6 +19,15 @@ function writeFixture(name: string, contents: string | Buffer): string {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function writeTree(files: Record<string, string>): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-playlist-tree-"));
|
||||
for (const [name, contents] of Object.entries(files)) {
|
||||
mkdirSync(join(dir, name, ".."), { recursive: true });
|
||||
writeFileSync(join(dir, name), contents);
|
||||
}
|
||||
return join(dir, Object.keys(files)[0]!);
|
||||
}
|
||||
|
||||
describe("isNotMediaPayload", () => {
|
||||
it.each([
|
||||
["doctype", "<!DOCTYPE html>\n<html><body></body></html>"],
|
||||
@@ -201,15 +212,6 @@ describe("playlistLeavesItsFolder", () => {
|
||||
expect(await isNotMediaPayload(file)).toBe(true);
|
||||
});
|
||||
|
||||
function writeTree(files: Record<string, string>): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-playlist-tree-"));
|
||||
for (const [name, contents] of Object.entries(files)) {
|
||||
mkdirSync(join(dir, name, ".."), { recursive: true });
|
||||
writeFileSync(join(dir, name), contents);
|
||||
}
|
||||
return join(dir, Object.keys(files)[0]!);
|
||||
}
|
||||
|
||||
it("refuses a list whose nested list names a file outside the top folder", async () => {
|
||||
const master = writeTree({
|
||||
"master.m3u8": "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1\nsub/inner.m3u8\n",
|
||||
@@ -252,3 +254,140 @@ describe("playlistLeavesItsFolder", () => {
|
||||
expect(await playlistLeavesItsFolder(writeFixture("clip.m3u8", contents))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("playlistLeavesItsFolder reads a list as ffprobe does", () => {
|
||||
const FFPROBE_8 = /^ffprobe version n?8\./.test(
|
||||
spawnSync(getFfprobeBinary(), ["-version"], { encoding: "utf-8" }).stdout ?? "",
|
||||
);
|
||||
const head = "#EXTM3U\n#EXT-X-TARGETDURATION:1\n";
|
||||
const tail = "#EXTINF:1,\nseg.ts\n#EXT-X-ENDLIST\n";
|
||||
const outside = "../outside/secret.mp4";
|
||||
// Cut at ffmpeg's 4095-byte line limit this names the outside file; whole, it walks back in.
|
||||
const cutLine = "./".repeat((4095 - outside.length) / 2) + outside + "/../../proj/seg.ts";
|
||||
// Each 0xFF is one byte to ffmpeg, so the cut lands after the outside name, not inside it.
|
||||
const tagStart = Buffer.concat([
|
||||
Buffer.from('#EXT-X-MAP:X="'),
|
||||
Buffer.alloc(40, 0xff),
|
||||
Buffer.from('",URI="'),
|
||||
]);
|
||||
const tagPad = 4095 - tagStart.length - outside.length;
|
||||
const notUtf8Line = Buffer.concat([
|
||||
tagStart,
|
||||
Buffer.from(
|
||||
"./".repeat(Math.floor(tagPad / 2)) +
|
||||
(tagPad % 2 ? "/" : "") +
|
||||
outside +
|
||||
'/../../proj/seg.ts"',
|
||||
),
|
||||
]);
|
||||
|
||||
it.each([
|
||||
["a segment after a bare carriage return", `${head}#EXTINF:1,\r${outside}\n#EXT-X-ENDLIST\n`],
|
||||
["a segment after a NUL", `${head}#EXTINF:1,\0${outside}\n#EXT-X-ENDLIST\n`],
|
||||
["an unquoted map URI", `${head}#EXT-X-MAP:URI=${outside}\n${tail}`],
|
||||
["a backslash-escaped map URI", `${head}#EXT-X-MAP:URI=".\\./outside/secret.mp4"\n${tail}`],
|
||||
[
|
||||
"an unquoted rendition URI",
|
||||
`#EXTM3U\n#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=a,NAME=a,URI=${outside}\n` +
|
||||
'#EXT-X-STREAM-INF:BANDWIDTH=1,AUDIO="a"\nv.m3u8\n',
|
||||
],
|
||||
[
|
||||
"a list behind an ID3 tag",
|
||||
Buffer.concat([
|
||||
Buffer.from([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0, 0]),
|
||||
Buffer.from(`${head}#EXTINF:1,\n${outside}\n#EXT-X-ENDLIST\n`),
|
||||
]),
|
||||
],
|
||||
["a segment line longer than ffmpeg reads", `${head}#EXTINF:1,\n${cutLine}\n#EXT-X-ENDLIST\n`],
|
||||
[
|
||||
"a long tag line with bytes that are not UTF-8",
|
||||
Buffer.concat([
|
||||
Buffer.from(head),
|
||||
notUtf8Line,
|
||||
Buffer.from("\n#EXTINF:1,\nseg.ts\n#EXT-X-ENDLIST\n"),
|
||||
]),
|
||||
],
|
||||
])("refuses %s", async (_name, contents) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-playlist-ffprobe-"));
|
||||
mkdirSync(join(root, "proj"));
|
||||
mkdirSync(join(root, "outside"));
|
||||
writeFileSync(join(root, "outside", "secret.mp4"), "outside media");
|
||||
writeFileSync(join(root, "proj", "v.m3u8"), head + tail);
|
||||
writeFileSync(join(root, "proj", "clip.m3u8"), contents);
|
||||
|
||||
expect(await playlistLeavesItsFolder(join(root, "proj", "clip.m3u8"))).toBe(true);
|
||||
if (!FFPROBE_8) return;
|
||||
const probe = spawnSync(
|
||||
getFfprobeBinary(),
|
||||
["-loglevel", "debug", "-show_format", "--", "clip.m3u8"],
|
||||
{
|
||||
cwd: join(root, "proj"),
|
||||
encoding: "utf-8",
|
||||
},
|
||||
);
|
||||
expect(probe.stderr).toMatch(/outside\/secret\.mp4' for reading/);
|
||||
});
|
||||
|
||||
it("refuses a nested name it cannot read, and keeps one that does not exist", async () => {
|
||||
const list = (entry: string) => `${head}#EXTINF:1,\n${entry}\n#EXT-X-ENDLIST\n`;
|
||||
const unreadable = writeTree({ "clip.m3u8": list("sub"), "sub/seg.ts": "segment" });
|
||||
expect(await playlistLeavesItsFolder(unreadable)).toBe(true);
|
||||
const missing = writeTree({ "clip.m3u8": list("gone.ts") });
|
||||
expect(await playlistLeavesItsFolder(missing)).toBe(false);
|
||||
const tooLong = writeTree({ "clip.m3u8": list("a".repeat(5000)) });
|
||||
expect(await playlistLeavesItsFolder(tooLong)).toBe(false);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform !== "linux")(
|
||||
"reads a nested list whose name is not UTF-8 by its bytes",
|
||||
async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-playlist-bytes-"));
|
||||
const name = Buffer.concat([Buffer.from("in"), Buffer.from([0xff]), Buffer.from(".m3u8")]);
|
||||
writeFileSync(Buffer.from(`${root}/${name.toString("latin1")}`, "latin1"), list(outside));
|
||||
const top = Buffer.concat([
|
||||
Buffer.from("#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1\n"),
|
||||
name,
|
||||
Buffer.from("\n"),
|
||||
]);
|
||||
writeFileSync(join(root, "clip.m3u8"), top);
|
||||
expect(await playlistLeavesItsFolder(join(root, "clip.m3u8"))).toBe(true);
|
||||
|
||||
function list(entry: string): string {
|
||||
return `${head}#EXTINF:1,\n${entry}\n#EXT-X-ENDLIST\n`;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a list with the same tags naming files beside it", async () => {
|
||||
const list = `${head}#EXT-X-MAP:URI=init.mp4\r#EXT-X-KEY:METHOD=AES-128,URI="k\\ey.bin"\r${tail}`;
|
||||
expect(await playlistLeavesItsFolder(writeFixture("clip.m3u8", list))).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses a file behind more ID3 tags than it skips", async () => {
|
||||
const tag = Buffer.from([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0, 0]);
|
||||
const file = writeFixture(
|
||||
"clip.m3u8",
|
||||
Buffer.concat([...Array(17).fill(tag), Buffer.from(head)]),
|
||||
);
|
||||
expect(await playlistLeavesItsFolder(file)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === "win32")("a FIFO with no writer", () => {
|
||||
function fifoBeside(list?: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-playlist-fifo-"));
|
||||
execFileSync("mkfifo", [join(dir, "pipe.mp4")]);
|
||||
if (list === undefined) return join(dir, "pipe.mp4");
|
||||
writeFileSync(join(dir, "clip.m3u8"), list);
|
||||
return join(dir, "clip.m3u8");
|
||||
}
|
||||
|
||||
it("is refused as media without waiting for a writer", async () => {
|
||||
expect(await isNotMediaPayload(fifoBeside())).toBe(true);
|
||||
});
|
||||
|
||||
it("is refused when a list names it, without waiting for a writer", async () => {
|
||||
const list = fifoBeside("#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXTINF:1,\npipe.mp4\n");
|
||||
expect(await playlistLeavesItsFolder(list)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import { open as openFile } from "node:fs/promises";
|
||||
import { dirname, isAbsolute, relative, resolve } from "node:path";
|
||||
|
||||
@@ -84,20 +85,43 @@ function byteOrderMarkLength(head: Buffer): number {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Read up to `bytes` (default {@link SNIFF_BYTES}) from the front of the file. */
|
||||
async function readHead(filePath: string, bytes = SNIFF_BYTES): Promise<Buffer> {
|
||||
const fh = await openFile(filePath, "r");
|
||||
type FileHandle = Awaited<ReturnType<typeof openFile>>;
|
||||
|
||||
async function readAt(fh: FileHandle, bytes: number, position: number): Promise<Buffer> {
|
||||
const buf = Buffer.alloc(bytes);
|
||||
let filled = 0;
|
||||
// Looped because a single read can come back short on NFS/FUSE and on
|
||||
// FIFOs, which would truncate the window mid-prefix.
|
||||
while (filled < buf.length) {
|
||||
const { bytesRead } = await fh.read(buf, filled, buf.length - filled, position + filled);
|
||||
if (bytesRead === 0) break;
|
||||
filled += bytesRead;
|
||||
}
|
||||
return buf.subarray(0, filled);
|
||||
}
|
||||
|
||||
const OPEN_WITHOUT_WAITING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
|
||||
|
||||
/** Opens without waiting on a FIFO or device, and gives null for one. A folder opens, and reading it fails as before. */
|
||||
async function openRegular(path: string | Buffer): Promise<FileHandle | null> {
|
||||
const fh = await openFile(path, OPEN_WITHOUT_WAITING);
|
||||
try {
|
||||
const buf = Buffer.alloc(bytes);
|
||||
let filled = 0;
|
||||
// Looped because a single read can come back short on NFS/FUSE and on
|
||||
// FIFOs, which would truncate the window mid-prefix.
|
||||
while (filled < buf.length) {
|
||||
const { bytesRead } = await fh.read(buf, filled, buf.length - filled, filled);
|
||||
if (bytesRead === 0) break;
|
||||
filled += bytesRead;
|
||||
}
|
||||
return buf.subarray(0, filled);
|
||||
const stats = await fh.stat();
|
||||
if (stats.isFile() || stats.isDirectory()) return fh;
|
||||
} catch (error) {
|
||||
await fh.close().catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
await fh.close().catch(() => {});
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Read up to {@link SNIFF_BYTES} from the front of the file; null when it is not a regular file. */
|
||||
async function readHead(filePath: string): Promise<Buffer | null> {
|
||||
const fh = await openRegular(filePath);
|
||||
if (!fh) return null;
|
||||
try {
|
||||
return await readAt(fh, SNIFF_BYTES, 0);
|
||||
} finally {
|
||||
await fh.close().catch(() => {});
|
||||
}
|
||||
@@ -133,9 +157,8 @@ function startsWithDocumentByte(head: Buffer): boolean {
|
||||
*/
|
||||
export async function isNotMediaPayload(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
return (
|
||||
startsWithDocumentByte(await readHead(filePath)) || (await playlistLeavesItsFolder(filePath))
|
||||
);
|
||||
const head = await readHead(filePath);
|
||||
return !head || startsWithDocumentByte(head) || (await playlistLeavesItsFolder(filePath));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
@@ -160,15 +183,65 @@ const HTTP_URL = /^https?:\/\//i;
|
||||
const OTHER_SCHEME = /^[a-z][a-z0-9+.-]+:/i;
|
||||
const DRIVE_PATH = /^[a-z]:[\\/]/i;
|
||||
|
||||
function playlistEntries(text: string): string[] | undefined {
|
||||
const body = text.replace(/^\uFEFF/, "").trimStart();
|
||||
if (body.startsWith("#EXTM3U")) {
|
||||
return body.split(/\r?\n/).flatMap((raw) => {
|
||||
const line = raw.trim();
|
||||
const uris = [...line.matchAll(/URI="([^"]*)"/gi)].map((match) => match[1]!);
|
||||
return line && !line.startsWith("#") ? [line, ...uris] : uris;
|
||||
});
|
||||
// ffmpeg reads a playlist line into a 4096-byte buffer and drops the rest of it.
|
||||
const MAX_HLS_LINE_BYTES = 4095;
|
||||
const ATTRIBUTE_GAP = /[\t\n\v\f\r ,]/;
|
||||
|
||||
/**
|
||||
* Every name an HLS list can open, read as ffmpeg's hls demuxer reads it and also as written.
|
||||
* The text is latin1, one character per byte, so a line is cut where ffmpeg cuts it.
|
||||
*/
|
||||
function hlsEntries(body: string): string[] {
|
||||
const names = new Set<string>();
|
||||
for (const raw of body.replaceAll("\0", "\n").split(/[\r\n]/)) {
|
||||
for (const whole of new Set([raw, raw.slice(0, MAX_HLS_LINE_BYTES)])) {
|
||||
const line = whole.replace(/[\t\v\f ]+$/, "");
|
||||
const found = line.startsWith("#") ? attributeUris(line) : [line, line.trim()];
|
||||
for (const name of found) if (name) names.add(name);
|
||||
}
|
||||
}
|
||||
return [...names];
|
||||
}
|
||||
|
||||
/** `URI=` values of a tag, split as ffmpeg's `ff_parse_key_value` splits them. */
|
||||
function attributeUris(line: string): string[] {
|
||||
const uris: string[] = [];
|
||||
let at = line.indexOf(":") + 1;
|
||||
while (at > 0 && at < line.length) {
|
||||
while (at < line.length && ATTRIBUTE_GAP.test(line[at]!)) at++;
|
||||
const equals = line.indexOf("=", at);
|
||||
if (equals < 0) break;
|
||||
const [value, next] = attributeValue(line, equals + 1);
|
||||
if (line.slice(at, equals).toUpperCase() === "URI") uris.push(value);
|
||||
at = next;
|
||||
}
|
||||
return uris;
|
||||
}
|
||||
|
||||
/** A quoted value drops its quotes and the backslash before any character; a bare one ends at a space or comma. */
|
||||
function attributeValue(line: string, at: number): [string, number] {
|
||||
if (line[at] !== '"') {
|
||||
let end = at;
|
||||
while (end < line.length && !ATTRIBUTE_GAP.test(line[end]!)) end++;
|
||||
return [line.slice(at, end), end];
|
||||
}
|
||||
let value = "";
|
||||
let index = at + 1;
|
||||
for (; index < line.length && line[index] !== '"'; index++) {
|
||||
if (line[index] === "\\") {
|
||||
if (index + 1 >= line.length) break;
|
||||
index++;
|
||||
}
|
||||
value += line[index];
|
||||
}
|
||||
return [value, line[index] === '"' ? index + 1 : index];
|
||||
}
|
||||
|
||||
const UTF8_BOM = "\xEF\xBB\xBF";
|
||||
|
||||
function playlistEntries(text: string): string[] | undefined {
|
||||
const body = (text.startsWith(UTF8_BOM) ? text.slice(UTF8_BOM.length) : text).trimStart();
|
||||
if (body.startsWith("#EXTM3U")) return hlsEntries(body);
|
||||
if (/^ffconcat\s+version/i.test(body)) {
|
||||
return body.split(/\r?\n/).flatMap((raw) => {
|
||||
const match = /^\s*file\s+(.*)$/.exec(raw);
|
||||
@@ -183,7 +256,7 @@ function playlistEntries(text: string): string[] | undefined {
|
||||
function entryLeavesFolder(entry: string, base: string, folder: string): boolean {
|
||||
if (HTTP_URL.test(entry)) return false;
|
||||
if (OTHER_SCHEME.test(entry) && !DRIVE_PATH.test(entry)) return true;
|
||||
return [entry, safeDecode(entry)].some((path) => {
|
||||
return [entry, percentDecoded(entry)].some((path) => {
|
||||
if (isAbsolute(path) || DRIVE_PATH.test(path) || path.startsWith("\\")) return true;
|
||||
const inside = relative(folder, resolve(base, path));
|
||||
return (
|
||||
@@ -192,12 +265,20 @@ function entryLeavesFolder(entry: string, base: string, folder: string): boolean
|
||||
});
|
||||
}
|
||||
|
||||
function safeDecode(value: string): string {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
/** `%XX` escapes decoded to the byte they name, kept as latin1 like every other name here. */
|
||||
function percentDecoded(value: string): string {
|
||||
return value.replace(/%([0-9a-f]{2})/gi, (_escape, hex: string) =>
|
||||
String.fromCharCode(Number.parseInt(hex, 16)),
|
||||
);
|
||||
}
|
||||
|
||||
/** Paths are walked as latin1 strings, one character per byte, and opened by those bytes. */
|
||||
const asBytes = (path: string): string => Buffer.from(path).toString("latin1");
|
||||
|
||||
interface Walk {
|
||||
folder: string;
|
||||
seen: Set<string>;
|
||||
verdicts: Map<string, boolean>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -208,54 +289,105 @@ export async function playlistLeavesItsFolder(
|
||||
filePath: string,
|
||||
placedAt = filePath,
|
||||
): Promise<boolean> {
|
||||
const placed = resolve(placedAt);
|
||||
return listLeaves(resolve(filePath), placed, dirname(placed), new Set(), 0);
|
||||
}
|
||||
|
||||
async function startsAsPlaylist(path: string): Promise<boolean> {
|
||||
const head = (await readHead(path))
|
||||
.toString("utf8")
|
||||
.replace(/^\uFEFF/, "")
|
||||
.trimStart();
|
||||
return head.startsWith("#EXTM3U") || /^ffconcat\s+version/i.test(head);
|
||||
}
|
||||
|
||||
/** Every list a list names is read too, against the top list's folder. */
|
||||
async function listLeaves(
|
||||
readPath: string,
|
||||
path: string,
|
||||
folder: string,
|
||||
seen: Set<string>,
|
||||
depth: number,
|
||||
): Promise<boolean> {
|
||||
const placed = asBytes(resolve(placedAt));
|
||||
const walk: Walk = { folder: dirname(placed), seen: new Set(), verdicts: new Map() };
|
||||
try {
|
||||
if (!(await startsAsPlaylist(readPath))) return false;
|
||||
if (depth >= MAX_PLAYLIST_DEPTH) return true;
|
||||
if (seen.has(path)) return false;
|
||||
seen.add(path);
|
||||
const body = await readHead(readPath, MAX_PLAYLIST_BYTES + 1);
|
||||
if (body.length > MAX_PLAYLIST_BYTES) return true;
|
||||
for (const entry of playlistEntries(body.toString("utf8")) ?? []) {
|
||||
if (await entryLeaves(entry, dirname(path), folder, seen, depth)) return true;
|
||||
}
|
||||
return false;
|
||||
return await listLeaves(asBytes(resolve(filePath)), placed, walk, 0);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function entryLeaves(
|
||||
entry: string,
|
||||
base: string,
|
||||
folder: string,
|
||||
seen: Set<string>,
|
||||
const MAX_ID3_TAGS = 16;
|
||||
|
||||
function isId3Header(head: Buffer): boolean {
|
||||
return (
|
||||
head.length >= 10 &&
|
||||
head.toString("latin1", 0, 3) === "ID3" &&
|
||||
head[3] !== 0xff &&
|
||||
head[4] !== 0xff &&
|
||||
head.subarray(6, 10).every((byte) => byte < 0x80)
|
||||
);
|
||||
}
|
||||
|
||||
/** Where the text starts: ffmpeg skips leading ID3v2 tags before it probes or parses a file. -1 past {@link MAX_ID3_TAGS}. */
|
||||
async function textStart(fh: FileHandle, first: Buffer): Promise<number> {
|
||||
let offset = 0;
|
||||
let head = first;
|
||||
for (let tags = 0; tags <= MAX_ID3_TAGS; tags++) {
|
||||
if (!isId3Header(head)) return offset;
|
||||
const size = (head[6]! << 21) | (head[7]! << 14) | (head[8]! << 7) | head[9]!;
|
||||
offset += 10 + size + (head[5]! & 0x10 ? 10 : 0);
|
||||
head = await readAt(fh, 10, offset);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function startsAsPlaylist(head: Buffer): boolean {
|
||||
const text = head
|
||||
.toString("utf8")
|
||||
.replace(/^\uFEFF/, "")
|
||||
.trimStart();
|
||||
return text.startsWith("#EXTM3U") || /^ffconcat\s+version/i.test(text);
|
||||
}
|
||||
|
||||
/** The list's text as latin1, null when the file is not a list, true when it must be refused unread. One open. */
|
||||
async function readList(path: string): Promise<string | null | true> {
|
||||
const fh = await openRegular(Buffer.from(path, "latin1"));
|
||||
if (!fh) return true;
|
||||
try {
|
||||
const first = await readAt(fh, SNIFF_BYTES, 0);
|
||||
const start = await textStart(fh, first);
|
||||
if (start < 0) return true;
|
||||
if (!startsAsPlaylist(start === 0 ? first : await readAt(fh, SNIFF_BYTES, start))) return null;
|
||||
const body = await readAt(fh, MAX_PLAYLIST_BYTES + 1, start);
|
||||
return body.length > MAX_PLAYLIST_BYTES ? true : body.toString("latin1");
|
||||
} finally {
|
||||
await fh.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
const isMissing = (error: unknown): boolean =>
|
||||
["ENOENT", "ENOTDIR", "ENAMETOOLONG"].includes(
|
||||
(error as NodeJS.ErrnoException | null)?.code ?? "",
|
||||
);
|
||||
|
||||
/** Every list a list names is read too, against the top list's folder. A nested file that exists but cannot be read is refused. */
|
||||
async function listLeaves(
|
||||
readPath: string,
|
||||
path: string,
|
||||
walk: Walk,
|
||||
depth: number,
|
||||
): Promise<boolean> {
|
||||
if (entryLeavesFolder(entry, base, folder)) return true;
|
||||
const text = await readList(readPath).catch((error: unknown) =>
|
||||
depth > 0 && !isMissing(error) ? true : null,
|
||||
);
|
||||
if (text === null) return false;
|
||||
if (text === true || depth >= MAX_PLAYLIST_DEPTH) return true;
|
||||
if (walk.seen.has(path)) return false;
|
||||
walk.seen.add(path);
|
||||
for (const entry of playlistEntries(text) ?? []) {
|
||||
if (await entryLeaves(entry, dirname(path), walk, depth)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function entryLeaves(
|
||||
entry: string,
|
||||
base: string,
|
||||
walk: Walk,
|
||||
depth: number,
|
||||
): Promise<boolean> {
|
||||
if (entryLeavesFolder(entry, base, walk.folder)) return true;
|
||||
if (HTTP_URL.test(entry)) return false;
|
||||
for (const local of new Set([entry, safeDecode(entry)])) {
|
||||
for (const local of new Set([entry, percentDecoded(entry)])) {
|
||||
const nested = resolve(base, local);
|
||||
if (await listLeaves(nested, nested, folder, seen, depth + 1)) return true;
|
||||
let verdict = walk.verdicts.get(nested);
|
||||
if (verdict === undefined) {
|
||||
verdict = await listLeaves(nested, nested, walk, depth + 1);
|
||||
walk.verdicts.set(nested, verdict);
|
||||
}
|
||||
if (verdict) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user