mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
perf(engine): on a Mac, video frames for a render decode on the media engine
Frame extraction decoded every source in software, so a long 4K60 HEVC clip spent much of its CPU on decoding. On macOS, extraction now asks ffmpeg for VideoToolbox when this ffmpeg has it and the source cannot carry alpha; ffmpeg falls back to software for a stream VideoToolbox cannot decode, and the frames written are byte-identical.
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
|
||||
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
|
||||
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
|
||||
});
|
||||
const probe = vi.hoisted(() => ({ hasVideoToolbox: true }));
|
||||
vi.mock("../utils/psnrFilterAvailability.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../utils/psnrFilterAvailability.js")>()),
|
||||
isFfmpegHwaccelAvailable: async () => probe.hasVideoToolbox,
|
||||
}));
|
||||
|
||||
const { runFfmpeg } = await import("../utils/runFfmpeg.js");
|
||||
const { extractVideoFramesRange } = await import("./videoFrameExtractor.js");
|
||||
|
||||
describe.skipIf(process.platform === "win32")("extraction decode on the Mac's media engine", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-extract-media-engine-"));
|
||||
const realPlatform = process.platform;
|
||||
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
|
||||
const clips = { h264: join(dir, "h264.mp4"), vp9: join(dir, "vp9.webm") };
|
||||
|
||||
beforeAll(async () => {
|
||||
for (const [clip, codec] of [
|
||||
[clips.h264, ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]],
|
||||
[clips.vp9, ["-c:v", "libvpx-vp9", "-deadline", "realtime", "-pix_fmt", "yuv420p"]],
|
||||
] as const) {
|
||||
const made = await runFfmpeg([
|
||||
...["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc2=s=64x64:d=1:r=10", ...codec, clip],
|
||||
]);
|
||||
expect(made.success, made.stderr).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(runFfmpeg).mockClear();
|
||||
probe.hasVideoToolbox = true;
|
||||
});
|
||||
|
||||
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
/** Extracts on `os`, returning the extraction's ffmpeg args and frames. */
|
||||
async function extractOn(os: NodeJS.Platform, clip: string, id: string) {
|
||||
Object.defineProperty(process, "platform", { ...platform, value: os });
|
||||
let frames: string[] = [];
|
||||
try {
|
||||
const out = join(dir, id);
|
||||
await extractVideoFramesRange(clip, id, 0, 1, { fps: 10, outputDir: dir }, undefined, {}, out)
|
||||
.then(
|
||||
() =>
|
||||
(frames = readdirSync(out)
|
||||
.sort()
|
||||
.map((name) => join(out, name))),
|
||||
)
|
||||
.catch((error: unknown) => {
|
||||
// Off a Mac, an ffmpeg asked for VideoToolbox fails; the args are what is checked there.
|
||||
if (realPlatform === "darwin" || os !== "darwin") throw error;
|
||||
});
|
||||
} finally {
|
||||
Object.defineProperty(process, "platform", platform);
|
||||
}
|
||||
const args = vi
|
||||
.mocked(runFfmpeg)
|
||||
.mock.calls.map(([callArgs]) => callArgs)
|
||||
.find((callArgs) => callArgs.includes(clip) && callArgs.includes("-vf"))!;
|
||||
return { args, frames };
|
||||
}
|
||||
|
||||
const hwaccel = (args: string[]) =>
|
||||
args.includes("-hwaccel") ? args[args.indexOf("-hwaccel") + 1] : null;
|
||||
|
||||
it("decodes on the media engine on macOS when this ffmpeg has VideoToolbox", async () => {
|
||||
const { args } = await extractOn("darwin", clips.h264, "mac");
|
||||
expect(hwaccel(args)).toBe("videotoolbox");
|
||||
});
|
||||
|
||||
it("decodes in software when this ffmpeg has no VideoToolbox", async () => {
|
||||
probe.hasVideoToolbox = false;
|
||||
const { args } = await extractOn("darwin", clips.h264, "mac-no-vt");
|
||||
expect(hwaccel(args)).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps the software decoder for a codec that may carry alpha", async () => {
|
||||
const { args } = await extractOn("darwin", clips.vp9, "mac-vp9");
|
||||
expect(hwaccel(args)).toBeNull();
|
||||
});
|
||||
|
||||
it("never asks for VideoToolbox off macOS", async () => {
|
||||
const { args } = await extractOn("linux", clips.h264, "linux");
|
||||
expect(hwaccel(args)).toBeNull();
|
||||
});
|
||||
|
||||
it.runIf(realPlatform === "darwin")("writes the same frames as software decode", async () => {
|
||||
const engine = await extractOn("darwin", clips.h264, "same-engine");
|
||||
probe.hasVideoToolbox = false;
|
||||
const software = await extractOn("darwin", clips.h264, "same-software");
|
||||
expect(hwaccel(engine.args)).toBe("videotoolbox");
|
||||
expect(engine.frames).toHaveLength(10);
|
||||
expect(engine.frames.map((f) => readFileSync(f))).toEqual(
|
||||
software.frames.map((f) => readFileSync(f)),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -58,7 +58,10 @@ import {
|
||||
writeUrlDownloadTelemetry,
|
||||
} from "../utils/urlDownloader.js";
|
||||
import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
|
||||
import { isFfmpegFilterAvailable } from "../utils/psnrFilterAvailability.js";
|
||||
import {
|
||||
isFfmpegFilterAvailable,
|
||||
isFfmpegHwaccelAvailable,
|
||||
} from "../utils/psnrFilterAvailability.js";
|
||||
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
||||
import { unwrapTemplate } from "../utils/htmlTemplate.js";
|
||||
import {
|
||||
@@ -988,7 +991,15 @@ export async function extractVideoFramesRange(
|
||||
const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
|
||||
const isMacOS = process.platform === "darwin";
|
||||
const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale());
|
||||
const decodeWithVideoToolbox = isHdr && isMacOS && !toneMappedToSdr;
|
||||
const keepsHdrForChrome = isHdr && isMacOS && !toneMappedToSdr;
|
||||
// The Mac's media engine decodes the same pixels at a fraction of the CPU; ffmpeg falls back
|
||||
// to software for a stream it cannot take.
|
||||
const decodeWithVideoToolbox =
|
||||
keepsHdrForChrome ||
|
||||
(isMacOS &&
|
||||
!metadata.hasAlpha &&
|
||||
!codecMayHaveAlpha(metadata.videoCodec) &&
|
||||
(await isFfmpegHwaccelAvailable("videotoolbox")));
|
||||
|
||||
const sampleAtOutputFps = !options.finalFrameOnly;
|
||||
if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
|
||||
@@ -1025,7 +1036,7 @@ export async function extractVideoFramesRange(
|
||||
}
|
||||
|
||||
const vfFilters: string[] = [];
|
||||
if (decodeWithVideoToolbox) {
|
||||
if (keepsHdrForChrome) {
|
||||
vfFilters.push("format=nv12");
|
||||
}
|
||||
const samplingAt = vfFilters.length;
|
||||
|
||||
@@ -189,3 +189,29 @@ describe("isPsnrFilterAvailable", () => {
|
||||
await expect(isPsnrFilterAvailable()).resolves.toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isFfmpegHwaccelAvailable", () => {
|
||||
it("reads `ffmpeg -hwaccels`", async () => {
|
||||
const { execFile, calls } = createExecFileSpy({
|
||||
kind: "ok",
|
||||
stdout: "Hardware acceleration methods:\nvideotoolbox\n",
|
||||
});
|
||||
vi.doMock("node:child_process", () => ({ execFile }));
|
||||
|
||||
const { isFfmpegHwaccelAvailable } = await import("./psnrFilterAvailability.js");
|
||||
await expect(isFfmpegHwaccelAvailable("videotoolbox")).resolves.toBe(true);
|
||||
await expect(isFfmpegHwaccelAvailable("cuda")).resolves.toBe(false);
|
||||
expect(calls.map((call) => call.args)).toEqual([["-hide_banner", "-hwaccels"]]);
|
||||
});
|
||||
|
||||
it("does not answer from the filter list, or the filter question from the hwaccel list", async () => {
|
||||
const { execFile, calls } = createExecFileSpy({ kind: "ok", stdout: "videotoolbox\n" });
|
||||
vi.doMock("node:child_process", () => ({ execFile }));
|
||||
|
||||
const { isFfmpegFilterAvailable, isFfmpegHwaccelAvailable } =
|
||||
await import("./psnrFilterAvailability.js");
|
||||
await isFfmpegFilterAvailable("zscale");
|
||||
await isFfmpegHwaccelAvailable("videotoolbox");
|
||||
expect(calls.map((call) => call.args[1])).toEqual(["-filters", "-hwaccels"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,7 +15,16 @@ import { getFfmpegBinary } from "./ffmpegBinaries.js";
|
||||
* not change across renders within the same CLI invocation, and re-probing
|
||||
* per session would burn ~50-100ms of subprocess spawn per capture worker.
|
||||
*/
|
||||
let cachedFilterList: Promise<string | null> | null = null;
|
||||
const cachedLists = new Map<string, Promise<string | null>>();
|
||||
|
||||
function listed(flag: "-filters" | "-hwaccels"): Promise<string | null> {
|
||||
let list = cachedLists.get(flag);
|
||||
if (!list) cachedLists.set(flag, (list = listFfmpeg(flag)));
|
||||
return list;
|
||||
}
|
||||
|
||||
const listsWord = (stdout: string | null, name: string) =>
|
||||
stdout !== null && new RegExp(`(^|\\s)${name}(\\s|$)`, "m").test(stdout);
|
||||
|
||||
/**
|
||||
* Returns true when the resident ffmpeg exposes the `psnr` filter. False on
|
||||
@@ -30,26 +39,28 @@ export function isPsnrFilterAvailable(): Promise<boolean> {
|
||||
return isFfmpegFilterAvailable("psnr");
|
||||
}
|
||||
|
||||
// `-filters` and `-hwaccels` list one name per line; a whole-word match ignores longer names
|
||||
// and banner prose that mentions it.
|
||||
export async function isFfmpegFilterAvailable(name: string): Promise<boolean> {
|
||||
if (cachedFilterList === null) cachedFilterList = listFilters();
|
||||
const stdout = await cachedFilterList;
|
||||
// `-filters` lists one filter per line; a whole-word match ignores longer names
|
||||
// and banner prose that mentions the filter.
|
||||
return stdout !== null && new RegExp(`(^|\\s)${name}(\\s|$)`, "m").test(stdout);
|
||||
return listsWord(await listed("-filters"), name);
|
||||
}
|
||||
|
||||
export async function isFfmpegHwaccelAvailable(name: string): Promise<boolean> {
|
||||
return listsWord(await listed("-hwaccels"), name);
|
||||
}
|
||||
|
||||
/** Test-only: drop the memoized probe result. */
|
||||
export function resetPsnrFilterAvailabilityCache(): void {
|
||||
cachedFilterList = null;
|
||||
cachedLists.clear();
|
||||
}
|
||||
|
||||
async function listFilters(): Promise<string | null> {
|
||||
async function listFfmpeg(flag: "-filters" | "-hwaccels"): Promise<string | null> {
|
||||
// Match `psnr.ts`: promisify lazily so a partial `node:child_process` mock
|
||||
// (test that omits `execFile`) doesn't crash at module load — it fails at
|
||||
// call time instead, and the try/catch below reports no filters at all.
|
||||
const execFileP = promisify(execFile);
|
||||
try {
|
||||
const { stdout } = await execFileP(getFfmpegBinary(), ["-hide_banner", "-filters"], {
|
||||
const { stdout } = await execFileP(getFfmpegBinary(), ["-hide_banner", flag], {
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
timeout: 5_000,
|
||||
windowsHide: true,
|
||||
|
||||
Reference in New Issue
Block a user