ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-12 15:20:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(studio-server): probe media codec facts for proxy decisions (#2587)

Browse Source 
* feat(studio-server): probe media codec facts for proxy decisions

Adds the codec manifest: one ffprobe-backed answer to what codec an asset
uses, whether a browser can decode it, and whether it carries alpha. Migrates
the existing prober to the shared ff-binaries resolver and to async execFile so
a scan pool runs off the event loop. No consumer yet; the preview weld wires it
up later in the stack.

* fix(studio-server): honor injected ffprobe runners
This commit is contained in:
Miguel Ángel
2026-07-16 20:54:50 -04:00
committed by GitHub
parent d7b38da19d
commit 9ca1e17101
7 changed files with 757 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/studio-server/src/helpers/mediaCodecMap.test.ts
+326
View File
@@ -0,0 +1,326 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import type { FfprobeRunner } from "./mediaMetadata.js";
import {
BROWSER_HOSTILE_CODECS,
decideMediaProxyEligibility,
createMediaCodecProbeCache,
probeAssetCodec,
scanProjectMediaCodecMap,
} from "./mediaCodecMap.js";
// Any real, existing file works as a stand-in ffprobe path — the runner
// passed to probeMediaMetadata is what's actually invoked, mirroring
// packages/lint/src/project.test.ts's hevc_preview_codec pattern (keeps this
// test independent of whether the host actually has ffprobe installed).
const FAKE_FFPROBE_PATH = process.execPath;
function makeRunner(
codecByPath: Record<string, string | { codecName: string; pixFmt?: string }>,
): FfprobeRunner {
return (_command, args) => {
const filePath = args[args.length - 1] ?? "";
const entry = codecByPath[filePath];
const normalized = typeof entry === "string" ? { codecName: entry } : entry;
const streams = normalized
? [
{
codec_type: "video",
codec_name: normalized.codecName,
pix_fmt: normalized.pixFmt,
},
]
: [];
return { status: 0, stdout: JSON.stringify({ streams }), stderr: "" };
};
}
function countingRunner(runner: FfprobeRunner): {
runner: FfprobeRunner;
calls: () => number;
} {
let calls = 0;
return {
runner: (command, args, options) => {
calls++;
return runner(command, args, options);
},
calls: () => calls,
};
}
function videoHtml(...srcs: string[]): string {
const tags = srcs
.map(
(src, i) =>
`<video id="v${i}" class="clip" src="${src}" muted data-start="0" data-duration="5"></video>`,
)
.join("\n");
return `<html><body><div data-composition-id="main" data-width="1920" data-height="1080" data-start="0" data-duration="10">${tags}</div></body></html>`;
}
let dirs: string[] = [];
function tmpProject(): string {
const dir = mkdtempSync(join(tmpdir(), "hf-media-codec-map-test-"));
dirs.push(dir);
return dir;
}
beforeEach(() => {
process.env.HYPERFRAMES_FFPROBE_PATH = FAKE_FFPROBE_PATH;
});
afterEach(() => {
delete process.env.HYPERFRAMES_FFPROBE_PATH;
for (const dir of dirs) rmSync(dir, { recursive: true, force: true });
dirs = [];
});
describe("probeAssetCodec", () => {
it("reports an HEVC asset as browser-hostile with the pinned representative mime", async () => {
const project = tmpProject();
const videoPath = join(project, "clip.mp4");
writeFileSync(videoPath, "fake video bytes");
const facts = await probeAssetCodec(videoPath, makeRunner({ [videoPath]: "hevc" }));
expect(facts).toEqual({
codecName: "hevc",
browserHostile: true,
representativeMime: BROWSER_HOSTILE_CODECS.hevc,
hasAlpha: false,
});
});
it("reports an H.264 asset as not browser-hostile", async () => {
const project = tmpProject();
const videoPath = join(project, "clip.mp4");
writeFileSync(videoPath, "fake video bytes");
const facts = await probeAssetCodec(videoPath, makeRunner({ [videoPath]: "h264" }));
expect(facts).toEqual({
codecName: "h264",
browserHostile: false,
representativeMime: null,
hasAlpha: false,
});
});
it("treats VP9 as conditionally hostile so unsupported browsers can request a proxy", async () => {
const project = tmpProject();
const videoPath = join(project, "clip.webm");
writeFileSync(videoPath, "fake video bytes");
const facts = await probeAssetCodec(videoPath, makeRunner({ [videoPath]: "vp9" }));
expect(facts).toEqual({
codecName: "vp9",
browserHostile: true,
representativeMime: BROWSER_HOSTILE_CODECS.vp9,
hasAlpha: false,
});
});
it("reports ProRes as browser-hostile with no representative mime", async () => {
const project = tmpProject();
const videoPath = join(project, "clip.mov");
writeFileSync(videoPath, "fake video bytes");
const facts = await probeAssetCodec(videoPath, makeRunner({ [videoPath]: "prores" }));
expect(facts).toEqual({
codecName: "prores",
browserHostile: true,
representativeMime: null,
hasAlpha: false,
});
});
it("flags an alpha-bearing pix_fmt (ProRes 4444) with hasAlpha", async () => {
const project = tmpProject();
const videoPath = join(project, "clip.mov");
writeFileSync(videoPath, "fake video bytes");
const facts = await probeAssetCodec(
videoPath,
makeRunner({
[videoPath]: { codecName: "prores", pixFmt: "yuva444p10le" },
}),
);
expect(facts).toEqual({
codecName: "prores",
browserHostile: true,
representativeMime: null,
hasAlpha: true,
});
});
it("returns null (never throws) when ffprobe is unresolvable", async () => {
const project = tmpProject();
const videoPath = join(project, "clip.mp4");
writeFileSync(videoPath, "fake video bytes");
process.env.HYPERFRAMES_FFPROBE_PATH = join(project, "missing-ffprobe");
await expect(probeAssetCodec(videoPath)).resolves.toBeNull();
});
});
describe("decideMediaProxyEligibility", () => {
it("allows only hostile opaque video through the H.264 proxy path", () => {
expect(
decideMediaProxyEligibility({
codecName: "hevc",
browserHostile: true,
representativeMime: null,
hasAlpha: false,
}),
).toEqual({ eligible: true });
});
it("rejects alpha and browser-safe sources before transcoding", () => {
expect(
decideMediaProxyEligibility({
codecName: "prores",
browserHostile: true,
representativeMime: null,
hasAlpha: true,
}),
).toEqual({ eligible: false, reason: "alpha_source" });
expect(
decideMediaProxyEligibility({
codecName: "h264",
browserHostile: false,
representativeMime: null,
hasAlpha: false,
}),
).toEqual({ eligible: false, reason: "browser_safe_codec" });
expect(decideMediaProxyEligibility(null)).toEqual({
eligible: false,
reason: "unknown_codec",
});
});
});
describe("scanProjectMediaCodecMap", () => {
it("omits an asset from the map (no throw) when ffprobe is unresolvable", async () => {
const project = tmpProject();
writeFileSync(join(project, "clip.mp4"), "fake video bytes");
process.env.HYPERFRAMES_FFPROBE_PATH = join(project, "missing-ffprobe");
const map = await scanProjectMediaCodecMap(project, [{ html: videoHtml("clip.mp4") }]);
expect(map).toEqual({});
});
it("keys the map by project-root-relative URL pathnames: decoded, forward-slash, leading-slash", async () => {
const project = tmpProject();
mkdirSync(join(project, "assets", "sub"), { recursive: true });
writeFileSync(join(project, "assets", "sub", "clip.mp4"), "fake video bytes");
writeFileSync(join(project, "assets", "my clip.mp4"), "fake video bytes");
const html = videoHtml("assets/sub/clip.mp4", "assets/my%20clip.mp4");
const map = await scanProjectMediaCodecMap(project, [{ html }], {
runner: makeRunner({
[join(project, "assets", "sub", "clip.mp4")]: "hevc",
[join(project, "assets", "my clip.mp4")]: "h264",
}),
});
expect(Object.keys(map)).toEqual(["/assets/sub/clip.mp4"]);
expect(map["/assets/sub/clip.mp4"]?.browserHostile).toBe(true);
expect(map["/assets/my clip.mp4"]).toBeUndefined();
});
it("rewrites a sub-composition's ../-traversing src via compSrcPath before resolving (rewriteAssetPath)", async () => {
const project = tmpProject();
mkdirSync(join(project, "assets"), { recursive: true });
mkdirSync(join(project, "compositions"), { recursive: true });
writeFileSync(join(project, "assets", "clip.mp4"), "fake video bytes");
const map = await scanProjectMediaCodecMap(
project,
[
{
html: videoHtml("../assets/clip.mp4"),
compSrcPath: "compositions/scene.html",
},
],
{ runner: makeRunner({ [join(project, "assets", "clip.mp4")]: "hevc" }) },
);
// The key is root-relative (what the served DOM resolves to), not the
// sub-composition-relative authored src.
expect(Object.keys(map)).toEqual(["/assets/clip.mp4"]);
expect(map["/assets/clip.mp4"]?.browserHostile).toBe(true);
});
it("caches by path + mtime + size, invalidating same-mtime re-exports and touched files", async () => {
const project = tmpProject();
const videoPath = join(project, "clip.mp4");
writeFileSync(videoPath, "fake video bytes");
const pinnedMtime = new Date(Date.now() - 60_000);
utimesSync(videoPath, pinnedMtime, pinnedMtime);
const html = videoHtml("clip.mp4");
const cache = createMediaCodecProbeCache();
const probe = countingRunner(makeRunner({ [videoPath]: "hevc" }));
const first = await scanProjectMediaCodecMap(project, [{ html }], {
cache,
runner: probe.runner,
});
expect(first["/clip.mp4"]?.codecName).toBe("hevc");
expect(probe.calls()).toBe(1);
const second = await scanProjectMediaCodecMap(project, [{ html }], {
cache,
runner: probe.runner,
});
expect(second["/clip.mp4"]?.codecName).toBe("hevc");
expect(probe.calls()).toBe(1);
writeFileSync(videoPath, "larger fake video bytes");
utimesSync(videoPath, pinnedMtime, pinnedMtime);
const resized = await scanProjectMediaCodecMap(project, [{ html }], {
cache,
runner: probe.runner,
});
expect(resized["/clip.mp4"]?.codecName).toBe("hevc");
expect(probe.calls()).toBe(2);
const future = new Date(Date.now() + 60_000);
utimesSync(videoPath, future, future);
const third = await scanProjectMediaCodecMap(project, [{ html }], {
cache,
runner: probe.runner,
});
expect(third["/clip.mp4"]?.codecName).toBe("hevc");
expect(probe.calls()).toBe(3);
});
it("bounds the long-lived probe cache while retaining the newest assets", async () => {
const project = tmpProject();
const cache = createMediaCodecProbeCache();
const paths = Array.from({ length: 513 }, (_, index) => `clip-${index}.mp4`);
const codecByPath: Record<string, string> = {};
for (const path of paths) {
const absolutePath = join(project, path);
writeFileSync(absolutePath, "fake video bytes");
codecByPath[absolutePath] = "hevc";
}
await scanProjectMediaCodecMap(project, [{ html: videoHtml(...paths) }], {
cache,
runner: makeRunner(codecByPath),
});
expect(cache.size).toBe(512);
expect(paths.slice(0, 8).some((path) => !cache.has(join(project, path)))).toBe(true);
expect(cache.has(join(project, "clip-512.mp4"))).toBe(true);
});
});
packages/studio-server/src/helpers/mediaCodecMap.ts
+262
View File
@@ -0,0 +1,262 @@
import { existsSync, statSync } from "node:fs";
import { relative, resolve, sep } from "node:path";
import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths";
import {
cleanAssetUrl,
isRemoteOrInlineUrl,
maskNonScannableRanges,
resolveLocalAssetCandidates,
} from "@hyperframes/parsers/asset-resolution";
import { pixelFormatHasAlpha, probeMediaMetadata, type FfprobeRunner } from "./mediaMetadata.js";
/**
* One reusable answer to "what codec is this asset, and is it browser-hostile?",
* built on top of `mediaMetadata.ts`'s ffprobe-backed prober so studio-server
* probes each asset once instead of running a second prober.
*/
export interface AssetCodecFacts {
codecName: string;
browserHostile: boolean;
/** Coarse `canPlayType()` input; `null` when not applicable (safe codec) or
* when no representative mime exists (ProRes: browsers never decode it, so
* the runtime always proxies rather than probing `canPlayType`). */
representativeMime: string | null;
/** Source carries an alpha channel (ffprobe pix_fmt). Alpha sources are
* never proxied — an H.264 proxy would destroy the transparency (e.g.
* ProRes 4444 alpha). */
hasAlpha: boolean;
}
/** Server-root-relative URL pathname -> that asset's codec facts. */
export type MediaCodecMap = Record<string, AssetCodecFacts>;
/**
* Browser-hostile codec table v1. One exported constant so extending it is a
* one-line change. `ffprobe` cannot emit exact RFC 6381 codec strings, so
* these `representativeMime` values are deliberately coarse (a false
* positive costs one proxy transcode, never correctness; a false negative is
* rescued by the runtime's reactive zero-videoWidth swap).
*/
export const BROWSER_HOSTILE_CODECS: Record<string, string | null> = {
hevc: 'video/mp4; codecs="hvc1.1.6.L120.B0"',
prores: null,
av1: 'video/mp4; codecs="av01.0.08M.08"',
// VP9 is browser-dependent: Chrome generally decodes it while Safari
// support varies. Treat it as conditional so canPlayType keeps the
// original where supported and transparently proxies it where unsupported.
vp9: 'video/webm; codecs="vp09.00.10.08"',
};
export type MediaProxyIneligibilityReason = "alpha_source" | "browser_safe_codec" | "unknown_codec";
export type MediaProxyEligibility =
| { eligible: true }
| { eligible: false; reason: MediaProxyIneligibilityReason };
/** Single policy gate shared by proactive scans and on-demand proxy routes. */
export function decideMediaProxyEligibility(facts: AssetCodecFacts | null): MediaProxyEligibility {
if (!facts) return { eligible: false, reason: "unknown_codec" };
if (facts.hasAlpha) return { eligible: false, reason: "alpha_source" };
if (!facts.browserHostile) return { eligible: false, reason: "browser_safe_codec" };
return { eligible: true };
}
function codecFactsFor(codecName: string, hasAlpha: boolean): AssetCodecFacts {
const isHostile = Object.hasOwn(BROWSER_HOSTILE_CODECS, codecName);
return {
codecName,
browserHostile: isHostile,
representativeMime: isHostile ? (BROWSER_HOSTILE_CODECS[codecName] ?? null) : null,
hasAlpha,
};
}
/**
* Probe a single video asset. Best-effort: ffprobe missing, erroring, or
* finding no video stream resolves to `null` (asset omitted by the caller),
* never a throw. Async so a pool of probes runs concurrently (the default
* runner is `execFile`-based).
*/
export async function probeAssetCodec(
filePath: string,
runner?: FfprobeRunner,
): Promise<AssetCodecFacts | null> {
const metadata = runner
? await probeMediaMetadata(filePath, runner)
: await probeMediaMetadata(filePath);
if (metadata.kind !== "video" || metadata.probeError) return null;
const codecName = metadata.color.codecName;
if (!codecName) return null;
return codecFactsFor(codecName, pixelFormatHasAlpha(metadata.color.pixelFormat));
}
interface CachedAssetProbe {
mtimeMs: number;
size: number;
facts: AssetCodecFacts | null;
}
/** Per (path, mtime) probe cache. Construct one per project/server lifetime
* and reuse it across scans; a fresh instance defeats the caching benefit. */
export type MediaCodecProbeCache = Map<string, CachedAssetProbe>;
export function createMediaCodecProbeCache(): MediaCodecProbeCache {
return new Map();
}
// Used when a caller doesn't pass its own cache — still correct (probes every
// time a fresh Map would), but callers that want the mtime-cache benefit
// across repeated scans (the studio preview route, etc.) should construct
// and hold their own cache via `createMediaCodecProbeCache`.
const defaultProbeCache: MediaCodecProbeCache = new Map();
const MAX_PROBE_CACHE_ENTRIES = 512;
function rememberProbeResult(
cache: MediaCodecProbeCache,
filePath: string,
result: CachedAssetProbe,
): void {
if (!cache.has(filePath) && cache.size >= MAX_PROBE_CACHE_ENTRIES) {
const oldest = cache.keys().next().value;
if (oldest) cache.delete(oldest);
}
// Refresh insertion order so frequently used assets remain resident.
cache.delete(filePath);
cache.set(filePath, result);
}
async function probeAssetCodecCached(
filePath: string,
cache: MediaCodecProbeCache,
runner?: FfprobeRunner,
): Promise<AssetCodecFacts | null> {
let stat: ReturnType<typeof statSync>;
try {
stat = statSync(filePath);
} catch {
return null;
}
const cached = cache.get(filePath);
if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) {
rememberProbeResult(cache, filePath, cached);
return cached.facts;
}
const facts = await probeAssetCodec(filePath, runner);
rememberProbeResult(cache, filePath, { mtimeMs: stat.mtimeMs, size: stat.size, facts });
return facts;
}
/** Structurally compatible with `packages/lint/src/hevcPreviewLint.ts`'s
* (unexported) `HtmlSourceLike`. */
export interface HtmlSourceLike {
html: string;
compSrcPath?: string;
}
// --- <video src> collection: shared primitives live in
// @hyperframes/parsers/asset-resolution; the <video>-specific regex and the
// pinned key derivation stay here.
const VIDEO_SRC_RE = /<video\b[^>]*\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi;
/**
* Resolve a `<video src>` reference to an existing local file.
*
* `rootRelativePathname` is the map key format PINNED by this plan's Key
* Technical Decisions: project-root-relative URL pathname, percent-decoded,
* query-string-stripped, forward-slash separated, leading-slash prefixed
* (e.g. "/assets/videos/clip.mp4"). This must match what the runtime derives
* via `new URL(el.currentSrc || el.src, document.baseURI).pathname`, because
* server-side scanning resolves filesystem paths while the DOM sees served
* URLs — a documented prior source of this exact class of bug.
*/
function resolveExistingLocalAsset(
projectDir: string,
url: string,
): { resolvedPath: string; rootRelativePathname: string } | null {
const projectRoot = resolve(projectDir);
const resolvedPath = resolveLocalAssetCandidates(projectRoot, url).find((candidate) =>
existsSync(candidate),
);
if (!resolvedPath) return null;
const rootRelative = relative(projectRoot, resolvedPath).split(sep).join("/");
return { resolvedPath, rootRelativePathname: `/${rootRelative}` };
}
/**
* Collects local `<video src>` references, resolved to their absolute path
* and deduped by that path, keyed by the pinned root-relative URL pathname.
*/
// fallow-ignore-next-line complexity
function collectLocalVideoAssets(
projectDir: string,
htmlSources: HtmlSourceLike[],
): Map<string, string> {
const candidates = new Map<string, string>();
for (const { html, compSrcPath } of htmlSources) {
const scannable = maskNonScannableRanges(html);
const re = new RegExp(VIDEO_SRC_RE.source, VIDEO_SRC_RE.flags);
let match: RegExpExecArray | null;
while ((match = re.exec(scannable)) !== null) {
const src = cleanAssetUrl(match[1] ?? "");
if (!src || isRemoteOrInlineUrl(src)) continue;
if (/^__[A-Z_]+__$/.test(src)) continue;
const rootRelativeSrc = compSrcPath ? rewriteAssetPath(compSrcPath, src) : src;
const resolved = resolveExistingLocalAsset(projectDir, rootRelativeSrc);
if (!resolved) continue;
candidates.set(resolved.resolvedPath, resolved.rootRelativePathname);
}
}
return candidates;
}
// Bounds concurrent ffprobe child processes for projects referencing many
// videos, mirroring `PROBE_CONCURRENCY` in `hevcPreviewLint.ts`.
const PROBE_CONCURRENCY = 8;
export interface ScanProjectMediaCodecMapOptions {
/** Persisted across calls by the caller for the mtime-cache benefit;
* defaults to a shared module-level cache when omitted. */
cache?: MediaCodecProbeCache;
runner?: FfprobeRunner;
}
/**
* Scans a project's composition HTML for local `<video src>` references and
* returns the injection map: root-relative URL pathname -> codec facts.
* Best-effort throughout — a video whose codec can't be determined (missing
* ffprobe, probe error, no video stream) is simply omitted, never thrown.
*/
export async function scanProjectMediaCodecMap(
projectDir: string,
htmlSources: HtmlSourceLike[],
options: ScanProjectMediaCodecMapOptions = {},
): Promise<MediaCodecMap> {
const candidates = collectLocalVideoAssets(projectDir, htmlSources);
if (candidates.size === 0) return {};
const cache = options.cache ?? defaultProbeCache;
const entries = [...candidates.entries()]; // [resolvedPath, rootRelativePathname]
const facts = new Array<AssetCodecFacts | null>(entries.length).fill(null);
let nextIndex = 0;
const workerCount = Math.min(PROBE_CONCURRENCY, entries.length);
await Promise.all(
Array.from({ length: workerCount }, async () => {
while (nextIndex < entries.length) {
const index = nextIndex++;
const entry = entries[index];
if (!entry) break;
facts[index] = await probeAssetCodecCached(entry[0], cache, options.runner);
}
}),
);
const map: MediaCodecMap = {};
entries.forEach(([, pathname], index) => {
const entryFacts = facts[index];
if (entryFacts?.browserHostile) map[pathname] = entryFacts;
});
return map;
}
packages/studio-server/src/helpers/mediaMetadata.test.ts
+69 -7
View File
@@ -1,5 +1,7 @@
import { describe, expect, it } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { classifyMediaColor, probeMediaMetadata } from "./mediaMetadata.js"; import { classifyMediaColor, pixelFormatHasAlpha, probeMediaMetadata } from "./mediaMetadata.js";
afterEach(() => vi.unstubAllEnvs());
describe("classifyMediaColor", () => { describe("classifyMediaColor", () => {
it("detects HDR PQ from BT.2020 + smpte2084 metadata", () => { it("detects HDR PQ from BT.2020 + smpte2084 metadata", () => {
@@ -55,8 +57,8 @@ describe("classifyMediaColor", () => {
}); });
describe("probeMediaMetadata", () => { describe("probeMediaMetadata", () => {
it("reads the first video stream from ffprobe JSON", () => { it("reads the first video stream from ffprobe JSON", async () => {
const metadata = probeMediaMetadata("/tmp/clip.mp4", () => ({ const metadata = await probeMediaMetadata("/tmp/clip.mp4", () => ({
status: 0, status: 0,
stdout: JSON.stringify({ stdout: JSON.stringify({
streams: [ streams: [
@@ -80,18 +82,78 @@ describe("probeMediaMetadata", () => {
}); });
}); });
it("returns unknown metadata when ffprobe is unavailable", () => { it("ignores attached cover art and reads the real video stream", async () => {
expect( const metadata = await probeMediaMetadata("/tmp/clip.mp4", () => ({
status: 0,
stdout: JSON.stringify({
streams: [
{
codec_type: "video",
codec_name: "mjpeg",
pix_fmt: "yuvj420p",
disposition: { attached_pic: 1 },
},
{
codec_type: "video",
codec_name: "hevc",
pix_fmt: "yuv420p10le",
disposition: { attached_pic: 0 },
},
],
}),
stderr: "",
}));
expect(metadata).toMatchObject({
kind: "video",
color: { codecName: "hevc", pixelFormat: "yuv420p10le" },
});
});
it("returns unknown metadata when ffprobe is unavailable", async () => {
await expect(
probeMediaMetadata("/tmp/clip.mp4", () => ({ probeMediaMetadata("/tmp/clip.mp4", () => ({
status: null, status: null,
stdout: "", stdout: "",
stderr: "", stderr: "",
error: { code: "ENOENT" } as NodeJS.ErrnoException, error: { code: "ENOENT" } as NodeJS.ErrnoException,
})), })),
).toMatchObject({ ).resolves.toMatchObject({
kind: "video", kind: "video",
color: { dynamicRange: "unknown", isHdr: false }, color: { dynamicRange: "unknown", isHdr: false },
probeError: "ffprobe unavailable", probeError: "ffprobe unavailable",
}); });
}); });
it("supports an injected async runner without requiring local ffprobe", async () => {
vi.stubEnv("HYPERFRAMES_FFPROBE_PATH", "/definitely/missing/ffprobe");
const metadata = await probeMediaMetadata("/tmp/clip.mp4", async () => ({
status: 0,
stdout: JSON.stringify({
streams: [{ codec_type: "video", codec_name: "h264", pix_fmt: "yuv420p" }],
}),
stderr: "",
}));
expect(metadata).toMatchObject({ kind: "video", color: { codecName: "h264" } });
});
});
describe("pixelFormatHasAlpha", () => {
it("detects alpha-bearing pixel formats", () => {
for (const pixFmt of [
"yuva420p",
"yuva444p10le",
"rgba",
"argb",
"bgra",
"abgr",
"gbrap12le",
"ya8",
]) {
expect(pixelFormatHasAlpha(pixFmt)).toBe(true);
}
for (const pixFmt of ["yuv420p", "yuv422p10le", "rgb24", "gbrp", "gray", undefined]) {
expect(pixelFormatHasAlpha(pixFmt)).toBe(false);
}
});
}); });
packages/studio-server/src/helpers/mediaMetadata.ts
+85 -18
View File
@@ -1,16 +1,50 @@
import { spawnSync, type SpawnSyncOptions } from "node:child_process"; import { execFile } from "node:child_process";
import { extname } from "node:path"; import { extname } from "node:path";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
type FfprobeRunner = ( export interface FfprobeRunResult {
command: string,
args: string[],
options?: SpawnSyncOptions,
) => {
status: number | null; status: number | null;
stdout: string | Buffer; stdout: string | Buffer;
stderr: string | Buffer; stderr: string | Buffer;
error?: NodeJS.ErrnoException; /** Spawn-level failure (covers both `NodeJS.ErrnoException` and
}; * `ExecFileException`); only `code === "ENOENT"` is ever inspected. */
error?: { code?: string | number | null | undefined };
}
/** Injectable ffprobe runner. May be synchronous (tests) or async (the
* default `execFile`-based runner below), so cold scans can run many probes
* concurrently off the event loop. */
export type FfprobeRunner = (
command: string,
args: string[],
options?: { timeout?: number; maxBuffer?: number },
) => FfprobeRunResult | Promise<FfprobeRunResult>;
/** Default runner: genuinely async (`execFile`), unlike the previous
* `spawnSync`-based one — a pool of concurrent probes actually parallelizes
* (mirrors `execFileAsync` in packages/lint/src/hevcPreviewLint.ts). */
const execFileRunner: FfprobeRunner = (command, args, options) =>
new Promise<FfprobeRunResult>((resolvePromise) => {
execFile(
command,
args,
{ timeout: options?.timeout, maxBuffer: options?.maxBuffer },
(error, stdout, stderr) => {
if (error && error.code === "ENOENT") {
resolvePromise({ status: null, stdout: "", stderr: "", error });
return;
}
if (error) {
// Nonzero exit / timeout / kill: report a nonzero status; callers
// only distinguish "ok" (0) from "failed" from "ENOENT".
const status = typeof error.code === "number" ? error.code : 1;
resolvePromise({ status, stdout: stdout ?? "", stderr: stderr ?? "" });
return;
}
resolvePromise({ status: 0, stdout: stdout ?? "", stderr: stderr ?? "" });
},
);
});
export type MediaDynamicRange = "hdr" | "sdr" | "unknown"; export type MediaDynamicRange = "hdr" | "sdr" | "unknown";
export type MediaHdrTransfer = "pq" | "hlg" | "unknown"; export type MediaHdrTransfer = "pq" | "hlg" | "unknown";
@@ -44,9 +78,21 @@ interface FfprobeStream {
color_transfer?: string; color_transfer?: string;
color_primaries?: string; color_primaries?: string;
bits_per_raw_sample?: string; bits_per_raw_sample?: string;
disposition?: { attached_pic?: number };
} }
const VIDEO_EXT = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi", ".m4v"]); const VIDEO_EXT = new Set([
".mp4",
".mov",
".webm",
".mkv",
".avi",
".m4v",
".mxf",
".mts",
".m2ts",
".ts",
]);
const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".avif"]); const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".avif"]);
const AUDIO_EXT = new Set([".mp3", ".wav", ".ogg", ".m4a", ".aac"]); const AUDIO_EXT = new Set([".mp3", ".wav", ".ogg", ".m4a", ".aac"]);
@@ -84,6 +130,17 @@ function colorLabel(input: {
return "SDR/unknown"; return "SDR/unknown";
} }
// Conservative alpha-bearing pix_fmt list: yuva* (yuva420p, yuva444p10le...),
// rgba/argb/bgra/abgr (packed RGB+alpha), gbrap* (planar GBR+alpha, ProRes
// 4444 decodes to these), ya* (gray+alpha). Prefix match keeps bit-depth /
// endianness suffixes covered.
const ALPHA_PIX_FMT_RE = /^(?:yuva|rgba|argb|bgra|abgr|gbrap|ya)/;
/** True when an ffprobe `pix_fmt` carries an alpha component. */
export function pixelFormatHasAlpha(pixFmt: string | undefined): boolean {
return pixFmt !== undefined && ALPHA_PIX_FMT_RE.test(pixFmt.toLowerCase());
}
export function classifyMediaColor(stream: FfprobeStream | null | undefined): MediaColorMetadata { export function classifyMediaColor(stream: FfprobeStream | null | undefined): MediaColorMetadata {
const colorPrimaries = lower(stream?.color_primaries); const colorPrimaries = lower(stream?.color_primaries);
const colorSpace = lower(stream?.color_space); const colorSpace = lower(stream?.color_space);
@@ -118,22 +175,31 @@ export function classifyMediaColor(stream: FfprobeStream | null | undefined): Me
}; };
} }
export function probeMediaMetadata( export async function probeMediaMetadata(
filePath: string, filePath: string,
runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner, runner: FfprobeRunner = execFileRunner,
): MediaMetadata { ): Promise<MediaMetadata> {
const kind = inferKindFromPath(filePath); const kind = inferKindFromPath(filePath);
if (kind === "audio" || kind === "unknown") { if (kind === "audio" || kind === "unknown") {
return { kind, color: classifyMediaColor(null) }; return { kind, color: classifyMediaColor(null) };
} }
const result = runner( // The default runner degrades a missing ffprobe to "unavailable" without
"ffprobe", // spawning; injected runners own execution and receive the normal command.
const ffprobePath =
findFfBinary("ffprobe", { configuredMustExist: true }) ??
(runner === execFileRunner ? undefined : "ffprobe");
if (!ffprobePath) {
return { kind, color: classifyMediaColor(null), probeError: "ffprobe unavailable" };
}
const result = await runner(
ffprobePath,
[ [
"-v", "-v",
"error", "error",
"-show_entries", "-show_entries",
"stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample", "stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample:stream_disposition=attached_pic",
"-of", "-of",
"json", "json",
filePath, filePath,
@@ -150,9 +216,10 @@ export function probeMediaMetadata(
try { try {
const parsed = JSON.parse(String(result.stdout || "{}")) as { streams?: FfprobeStream[] }; const parsed = JSON.parse(String(result.stdout || "{}")) as { streams?: FfprobeStream[] };
const stream = parsed.streams?.find((item) => const stream = parsed.streams?.find((item) => {
kind === "image" ? item.codec_type === "video" : item.codec_type === kind, if (kind === "image") return item.codec_type === "video";
); return item.codec_type === kind && item.disposition?.attached_pic !== 1;
});
return { kind, color: classifyMediaColor(stream) }; return { kind, color: classifyMediaColor(stream) };
} catch { } catch {
return { kind, color: classifyMediaColor(null), probeError: "ffprobe returned invalid json" }; return { kind, color: classifyMediaColor(null), probeError: "ffprobe returned invalid json" };
packages/studio-server/src/helpers/mediaValidation.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { basename, join } from "node:path"; import { basename, join } from "node:path";
const VIDEO_EXT = /\.(mp4|webm|mov)$/i; const VIDEO_EXT = /\.(mp4|webm|mov|mkv|avi|m4v|mxf|mts|m2ts|ts)$/i;
const AUDIO_EXT = /\.(mp3|wav|ogg|m4a|aac)$/i; const AUDIO_EXT = /\.(mp3|wav|ogg|m4a|aac)$/i;
type FfprobeRunner = ( type FfprobeRunner = (
packages/studio-server/src/routes/media.test.ts
+1 -1
View File
@@ -64,7 +64,7 @@ function completeJob(opts: Parameters<NonNullable<StudioApiAdapter["startBackgro
describe("registerMediaRoutes", () => { describe("registerMediaRoutes", () => {
it("returns metadata for a project-local media asset", async () => { it("returns metadata for a project-local media asset", async () => {
const probe = vi.fn(() => ({ const probe = vi.fn(async () => ({
kind: "video" as const, kind: "video" as const,
color: { color: {
dynamicRange: "hdr" as const, dynamicRange: "hdr" as const,
packages/studio-server/src/routes/media.ts
+13 -2
View File
@@ -6,7 +6,18 @@ import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
import { resolveWithinProject } from "../helpers/safePath.js"; import { resolveWithinProject } from "../helpers/safePath.js";
import { probeMediaMetadata } from "../helpers/mediaMetadata.js"; import { probeMediaMetadata } from "../helpers/mediaMetadata.js";
const VIDEO_EXTENSIONS = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi"]); const VIDEO_EXTENSIONS = new Set([
".mp4",
".mov",
".webm",
".mkv",
".avi",
".m4v",
".mxf",
".mts",
".m2ts",
".ts",
]);
const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".webp"]); const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".webp"]);
const VIDEO_OUTPUT_EXTENSIONS = new Set([".webm", ".mov"]); const VIDEO_OUTPUT_EXTENSIONS = new Set([".webm", ".mov"]);
const QUALITIES = new Set(["fast", "balanced", "best"]); const QUALITIES = new Set(["fast", "balanced", "best"]);
@@ -126,7 +137,7 @@ export function registerMediaRoutes(
if (!filePath) return c.json({ error: "forbidden" }, 403); if (!filePath) return c.json({ error: "forbidden" }, 403);
if (!existsSync(filePath)) return c.json({ error: "media not found" }, 404); if (!existsSync(filePath)) return c.json({ error: "media not found" }, 404);
return c.json({ path: assetPath, metadata: readMediaMetadata(filePath) }); return c.json({ path: assetPath, metadata: await readMediaMetadata(filePath) });
}); });
api.post( api.post(