perf(engine): on a Mac, video frames for a render decode on the media engine

Frame extraction decoded every source in software, so a long 4K60 HEVC clip
spent much of its CPU on decoding. On macOS, extraction now asks ffmpeg for
VideoToolbox when this ffmpeg has it and the source cannot carry alpha; ffmpeg
falls back to software for a stream VideoToolbox cannot decode, and the frames
written are byte-identical.
This commit is contained in:
Miguel Angel Simon Sierra
2026-10-19 09:34:50 -04:00
parent cea0bad793
commit 1e2b99d796
4 changed files with 165 additions and 12 deletions
@@ -0,0 +1,105 @@
import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
});
const probe = vi.hoisted(() => ({ hasVideoToolbox: true }));
vi.mock("../utils/psnrFilterAvailability.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../utils/psnrFilterAvailability.js")>()),
isFfmpegHwaccelAvailable: async () => probe.hasVideoToolbox,
}));
const { runFfmpeg } = await import("../utils/runFfmpeg.js");
const { extractVideoFramesRange } = await import("./videoFrameExtractor.js");
describe.skipIf(process.platform === "win32")("extraction decode on the Mac's media engine", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-extract-media-engine-"));
const realPlatform = process.platform;
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
const clips = { h264: join(dir, "h264.mp4"), vp9: join(dir, "vp9.webm") };
beforeAll(async () => {
for (const [clip, codec] of [
[clips.h264, ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]],
[clips.vp9, ["-c:v", "libvpx-vp9", "-deadline", "realtime", "-pix_fmt", "yuv420p"]],
] as const) {
const made = await runFfmpeg([
...["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc2=s=64x64:d=1:r=10", ...codec, clip],
]);
expect(made.success, made.stderr).toBe(true);
}
});
beforeEach(() => {
vi.mocked(runFfmpeg).mockClear();
probe.hasVideoToolbox = true;
});
afterAll(() => rmSync(dir, { recursive: true, force: true }));
/** Extracts on `os`, returning the extraction's ffmpeg args and frames. */
async function extractOn(os: NodeJS.Platform, clip: string, id: string) {
Object.defineProperty(process, "platform", { ...platform, value: os });
let frames: string[] = [];
try {
const out = join(dir, id);
await extractVideoFramesRange(clip, id, 0, 1, { fps: 10, outputDir: dir }, undefined, {}, out)
.then(
() =>
(frames = readdirSync(out)
.sort()
.map((name) => join(out, name))),
)
.catch((error: unknown) => {
// Off a Mac, an ffmpeg asked for VideoToolbox fails; the args are what is checked there.
if (realPlatform === "darwin" || os !== "darwin") throw error;
});
} finally {
Object.defineProperty(process, "platform", platform);
}
const args = vi
.mocked(runFfmpeg)
.mock.calls.map(([callArgs]) => callArgs)
.find((callArgs) => callArgs.includes(clip) && callArgs.includes("-vf"))!;
return { args, frames };
}
const hwaccel = (args: string[]) =>
args.includes("-hwaccel") ? args[args.indexOf("-hwaccel") + 1] : null;
it("decodes on the media engine on macOS when this ffmpeg has VideoToolbox", async () => {
const { args } = await extractOn("darwin", clips.h264, "mac");
expect(hwaccel(args)).toBe("videotoolbox");
});
it("decodes in software when this ffmpeg has no VideoToolbox", async () => {
probe.hasVideoToolbox = false;
const { args } = await extractOn("darwin", clips.h264, "mac-no-vt");
expect(hwaccel(args)).toBeNull();
});
it("keeps the software decoder for a codec that may carry alpha", async () => {
const { args } = await extractOn("darwin", clips.vp9, "mac-vp9");
expect(hwaccel(args)).toBeNull();
});
it("never asks for VideoToolbox off macOS", async () => {
const { args } = await extractOn("linux", clips.h264, "linux");
expect(hwaccel(args)).toBeNull();
});
it.runIf(realPlatform === "darwin")("writes the same frames as software decode", async () => {
const engine = await extractOn("darwin", clips.h264, "same-engine");
probe.hasVideoToolbox = false;
const software = await extractOn("darwin", clips.h264, "same-software");
expect(hwaccel(engine.args)).toBe("videotoolbox");
expect(engine.frames).toHaveLength(10);
expect(engine.frames.map((f) => readFileSync(f))).toEqual(
software.frames.map((f) => readFileSync(f)),
);
});
});
@@ -58,7 +58,10 @@ import {
writeUrlDownloadTelemetry,
} from "../utils/urlDownloader.js";
import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import { isFfmpegFilterAvailable } from "../utils/psnrFilterAvailability.js";
import {
isFfmpegFilterAvailable,
isFfmpegHwaccelAvailable,
} from "../utils/psnrFilterAvailability.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
import {
@@ -988,7 +991,15 @@ export async function extractVideoFramesRange(
const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
const isMacOS = process.platform === "darwin";
const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale());
const decodeWithVideoToolbox = isHdr && isMacOS && !toneMappedToSdr;
const keepsHdrForChrome = isHdr && isMacOS && !toneMappedToSdr;
// The Mac's media engine decodes the same pixels at a fraction of the CPU; ffmpeg falls back
// to software for a stream it cannot take.
const decodeWithVideoToolbox =
keepsHdrForChrome ||
(isMacOS &&
!metadata.hasAlpha &&
!codecMayHaveAlpha(metadata.videoCodec) &&
(await isFfmpegHwaccelAvailable("videotoolbox")));
const sampleAtOutputFps = !options.finalFrameOnly;
if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
@@ -1025,7 +1036,7 @@ export async function extractVideoFramesRange(
}
const vfFilters: string[] = [];
if (decodeWithVideoToolbox) {
if (keepsHdrForChrome) {
vfFilters.push("format=nv12");
}
const samplingAt = vfFilters.length;
@@ -189,3 +189,29 @@ describe("isPsnrFilterAvailable", () => {
await expect(isPsnrFilterAvailable()).resolves.toBe(false);
});
});
describe("isFfmpegHwaccelAvailable", () => {
it("reads `ffmpeg -hwaccels`", async () => {
const { execFile, calls } = createExecFileSpy({
kind: "ok",
stdout: "Hardware acceleration methods:\nvideotoolbox\n",
});
vi.doMock("node:child_process", () => ({ execFile }));
const { isFfmpegHwaccelAvailable } = await import("./psnrFilterAvailability.js");
await expect(isFfmpegHwaccelAvailable("videotoolbox")).resolves.toBe(true);
await expect(isFfmpegHwaccelAvailable("cuda")).resolves.toBe(false);
expect(calls.map((call) => call.args)).toEqual([["-hide_banner", "-hwaccels"]]);
});
it("does not answer from the filter list, or the filter question from the hwaccel list", async () => {
const { execFile, calls } = createExecFileSpy({ kind: "ok", stdout: "videotoolbox\n" });
vi.doMock("node:child_process", () => ({ execFile }));
const { isFfmpegFilterAvailable, isFfmpegHwaccelAvailable } =
await import("./psnrFilterAvailability.js");
await isFfmpegFilterAvailable("zscale");
await isFfmpegHwaccelAvailable("videotoolbox");
expect(calls.map((call) => call.args[1])).toEqual(["-filters", "-hwaccels"]);
});
});
@@ -15,7 +15,16 @@ import { getFfmpegBinary } from "./ffmpegBinaries.js";
* not change across renders within the same CLI invocation, and re-probing
* per session would burn ~50-100ms of subprocess spawn per capture worker.
*/
let cachedFilterList: Promise<string | null> | null = null;
const cachedLists = new Map<string, Promise<string | null>>();
function listed(flag: "-filters" | "-hwaccels"): Promise<string | null> {
let list = cachedLists.get(flag);
if (!list) cachedLists.set(flag, (list = listFfmpeg(flag)));
return list;
}
const listsWord = (stdout: string | null, name: string) =>
stdout !== null && new RegExp(`(^|\\s)${name}(\\s|$)`, "m").test(stdout);
/**
* Returns true when the resident ffmpeg exposes the `psnr` filter. False on
@@ -30,26 +39,28 @@ export function isPsnrFilterAvailable(): Promise<boolean> {
return isFfmpegFilterAvailable("psnr");
}
// `-filters` and `-hwaccels` list one name per line; a whole-word match ignores longer names
// and banner prose that mentions it.
export async function isFfmpegFilterAvailable(name: string): Promise<boolean> {
if (cachedFilterList === null) cachedFilterList = listFilters();
const stdout = await cachedFilterList;
// `-filters` lists one filter per line; a whole-word match ignores longer names
// and banner prose that mentions the filter.
return stdout !== null && new RegExp(`(^|\\s)${name}(\\s|$)`, "m").test(stdout);
return listsWord(await listed("-filters"), name);
}
export async function isFfmpegHwaccelAvailable(name: string): Promise<boolean> {
return listsWord(await listed("-hwaccels"), name);
}
/** Test-only: drop the memoized probe result. */
export function resetPsnrFilterAvailabilityCache(): void {
cachedFilterList = null;
cachedLists.clear();
}
async function listFilters(): Promise<string | null> {
async function listFfmpeg(flag: "-filters" | "-hwaccels"): Promise<string | null> {
// Match `psnr.ts`: promisify lazily so a partial `node:child_process` mock
// (test that omits `execFile`) doesn't crash at module load — it fails at
// call time instead, and the try/catch below reports no filters at all.
const execFileP = promisify(execFile);
try {
const { stdout } = await execFileP(getFfmpegBinary(), ["-hide_banner", "-filters"], {
const { stdout } = await execFileP(getFfmpegBinary(), ["-hide_banner", flag], {
maxBuffer: 4 * 1024 * 1024,
timeout: 5_000,
windowsHide: true,