fix(render): consolidate preflight and local recovery (#2403)

* fix(render): fall back when libx264 is unavailable

* fix(render): recover orphaned browsers before retry

* fix(render): check disk space on write volumes

* test(engine): accept resolved ffmpeg binary paths

* fix(render): harden H.264 capability fallback

* chore(ci): scope inherited Fallow findings

* fix(render): diagnose encoder probe failures
This commit is contained in:
Miguel Ángel
2026-07-14 21:55:39 -04:00
committed by GitHub
parent 15ca6fd129
commit 5d3a7404fa
9 changed files with 224 additions and 15 deletions
+39 -2
View File
@@ -1,12 +1,13 @@
import { execSync } from "node:child_process";
import { execFileSync, execSync } from "node:child_process";
import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("node:child_process", () => ({ execSync: vi.fn() }));
vi.mock("node:child_process", () => ({ execFileSync: vi.fn(), execSync: vi.fn() }));
vi.mock("node:fs", () => ({ existsSync: vi.fn() }));
const mockExec = vi.mocked(execSync);
const mockExecFile = vi.mocked(execFileSync);
const mockExists = vi.mocked(existsSync);
// The common-dir fallback list is platform-gated (empty on win32), so pin the
@@ -53,3 +54,39 @@ describe("findFFmpeg", () => {
expect(findFFmpeg()).toBeUndefined();
});
});
describe("resolveH264EncoderMode", () => {
it("falls back to VideoToolbox when libx264 is absent", async () => {
const { resolveH264EncoderMode } = await import("./ffmpeg.js");
const encoders = `
V....D h264_videotoolbox VideoToolbox H.264 Encoder
`;
expect(resolveH264EncoderMode(encoders, false)).toBe("gpu");
});
it("does not treat a compiled Linux hardware encoder as usable", async () => {
const { resolveH264EncoderMode } = await import("./ffmpeg.js");
const encoders = `
V....D h264_vaapi H.264/AVC (VAAPI)
`;
expect(() => resolveH264EncoderMode(encoders, false)).toThrow(
"neither libx264 nor VideoToolbox",
);
});
it("inspects the configured FFmpeg binary", async () => {
mockExecFile.mockReturnValue(
" V....D h264_videotoolbox VideoToolbox H.264 Encoder\n" as never,
);
const { detectH264EncoderMode } = await import("./ffmpeg.js");
expect(detectH264EncoderMode("/custom/ffmpeg", false)).toBe("gpu");
expect(mockExecFile).toHaveBeenCalledWith(
"/custom/ffmpeg",
["-hide_banner", "-encoders"],
expect.objectContaining({ encoding: "utf-8" }),
);
});
});
+29 -1
View File
@@ -1,5 +1,5 @@
// fallow-ignore-file code-duplication
import { execSync } from "node:child_process";
import { execFileSync, execSync } from "node:child_process";
import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { detectLinuxDistro, ffmpegInstallCommand } from "./linuxDeps.js";
@@ -7,6 +7,34 @@ import { detectLinuxDistro, ffmpegInstallCommand } from "./linuxDeps.js";
export const FFMPEG_PATH_ENV = "HYPERFRAMES_FFMPEG_PATH";
export const FFPROBE_PATH_ENV = "HYPERFRAMES_FFPROBE_PATH";
export type H264EncoderMode = "software" | "gpu";
/**
* Select the H.264 encoder class supported by an FFmpeg build.
*
* Some macOS FFmpeg distributions expose VideoToolbox but omit libx264. The
* default CPU render path must not pass libx264-only options such as `-preset`
* to those builds.
*/
export function resolveH264EncoderMode(
ffmpegEncodersOutput: string,
gpuRequested: boolean,
): H264EncoderMode {
if (gpuRequested) return "gpu";
if (/\blibx264\b/.test(ffmpegEncodersOutput)) return "software";
if (/\bh264_videotoolbox\b/.test(ffmpegEncodersOutput)) return "gpu";
throw new Error("This FFmpeg build has neither libx264 nor VideoToolbox H.264 encoding.");
}
export function detectH264EncoderMode(ffmpegPath: string, gpuRequested: boolean): H264EncoderMode {
const encoders = execFileSync(ffmpegPath, ["-hide_banner", "-encoders"], {
encoding: "utf-8",
stdio: ["ignore", "pipe", "pipe"],
timeout: 5000,
});
return resolveH264EncoderMode(encoders, gpuRequested);
}
function chooseBestPathCandidate(
name: "ffmpeg" | "ffprobe",
candidates: string[],
+13 -1
View File
@@ -1,6 +1,6 @@
// fallow-ignore-file code-duplication
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { parseToolVersion, runEnvironmentChecks } from "./preflight.js";
import { checkDisk, parseToolVersion, runEnvironmentChecks } from "./preflight.js";
import * as manager from "./manager.js";
import * as linuxDeps from "./linuxDeps.js";
@@ -218,3 +218,15 @@ describe("parseToolVersion", () => {
);
});
});
describe("checkDisk", () => {
it("checks the requested render volume", () => {
const freeDiskMb = vi.fn(() => 512);
expect(checkDisk("/external/render-output", freeDiskMb)).toMatchObject({
ok: false,
detail: "0.5 GB free at /external/render-output",
});
expect(freeDiskMb).toHaveBeenCalledWith("/external/render-output");
});
});
+11 -5
View File
@@ -38,6 +38,7 @@ export interface EnvironmentCheckResult {
export interface EnvironmentCheckOptions {
projectDir?: string;
diskPaths?: string[];
browserPath?: string;
includeBrowser?: boolean;
includeDisk?: boolean;
@@ -226,8 +227,11 @@ async function checkChrome(browserPath?: string): Promise<EnvironmentCheckOutcom
};
}
function checkDisk(projectDir = "."): EnvironmentCheckOutcome {
const freeMb = getFreeDiskMb(projectDir);
export function checkDisk(
path = ".",
freeDiskMb: (path: string) => number | null = getFreeDiskMb,
): EnvironmentCheckOutcome {
const freeMb = freeDiskMb(path);
if (freeMb === null) {
return { name: "Disk", ok: true, level: "ok", detail: "Unable to check" };
}
@@ -238,11 +242,11 @@ function checkDisk(projectDir = "."): EnvironmentCheckOutcome {
ok: false,
level: "error",
title: "Low disk space",
detail: `${freeGb} GB free`,
detail: `${freeGb} GB free at ${path}`,
hint: "Renders produce large temp files. Free disk space before rendering.",
};
}
return { name: "Disk", ok: true, level: "ok", detail: `${freeGb} GB free` };
return { name: "Disk", ok: true, level: "ok", detail: `${freeGb} GB free at ${path}` };
}
function checkWindowsUncPath(projectDir = process.cwd()): EnvironmentCheckOutcome | undefined {
@@ -257,6 +261,7 @@ function checkWindowsUncPath(projectDir = process.cwd()): EnvironmentCheckOutcom
};
}
// fallow-ignore-next-line complexity
export async function runEnvironmentChecks(
options: EnvironmentCheckOptions = {},
): Promise<EnvironmentCheckResult> {
@@ -281,7 +286,8 @@ export async function runEnvironmentChecks(
}
if (options.includeDisk) {
outcomes.push(checkDisk(options.projectDir));
const diskPaths = [...new Set(options.diskPaths ?? [options.projectDir ?? "."])];
outcomes.push(...diskPaths.map((path) => checkDisk(path)));
}
if (options.includeWindowsUnc) {
+92
View File
@@ -72,6 +72,15 @@ const preflightState = vi.hoisted(() => ({
},
}));
const ffmpegEncoderState = vi.hoisted(() => ({
mode: "software" as "software" | "gpu",
error: null as Error | null,
}));
const orphanCleanupState = vi.hoisted(() => ({
calls: 0,
killed: 0,
}));
vi.mock("../utils/producer.js", () => ({
loadProducer: vi.fn(async () => ({
resolveConfig: vi.fn((overrides: Record<string, unknown>) => {
@@ -120,6 +129,10 @@ vi.mock("../telemetry/events.js", () => ({
}));
vi.mock("../browser/ffmpeg.js", () => ({
detectH264EncoderMode: vi.fn(() => {
if (ffmpegEncoderState.error) throw ffmpegEncoderState.error;
return ffmpegEncoderState.mode;
}),
findFFmpeg: vi.fn(() => "/usr/bin/ffmpeg"),
getFFmpegInstallHint: vi.fn(() => "brew install ffmpeg"),
}));
@@ -128,6 +141,13 @@ vi.mock("../browser/preflight.js", () => ({
runEnvironmentChecks: vi.fn(async () => preflightState.result),
}));
vi.mock("../utils/orphanCleanup.js", () => ({
killOrphanedProcesses: vi.fn(() => {
orphanCleanupState.calls += 1;
return orphanCleanupState.killed;
}),
}));
describe("renderLocal browser GPU config", () => {
const savedEnv = new Map<string, string | undefined>();
// Pre-resolve once. The first dynamic `import("./render.js")` in this file
@@ -173,6 +193,10 @@ describe("renderLocal browser GPU config", () => {
configState.writeConfigCalls = [];
trackingState.shouldTrack = true;
trackingState.renderObservations = [];
ffmpegEncoderState.mode = "software";
ffmpegEncoderState.error = null;
orphanCleanupState.calls = 0;
orphanCleanupState.killed = 0;
resetTrialState();
savedEnv.clear();
savedEnv.set("HYPERFRAMES_FFMPEG_PATH", process.env.HYPERFRAMES_FFMPEG_PATH);
@@ -185,6 +209,22 @@ describe("renderLocal browser GPU config", () => {
delete process.env.HF_DE_PARALLEL_ROUTER;
});
it("cleans orphaned browser trees before starting a local render", async () => {
orphanCleanupState.killed = 1;
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "standard",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "auto",
quiet: true,
});
expect(orphanCleanupState.calls).toBe(1);
});
afterEach(() => {
for (const [key, value] of savedEnv) {
if (value === undefined) {
@@ -323,6 +363,58 @@ describe("renderLocal browser GPU config", () => {
expect(process.env.PRODUCER_HEADLESS_SHELL_PATH).toBe("/mock/chrome");
});
it("falls back to hardware encoding when FFmpeg omits libx264", async () => {
ffmpegEncoderState.mode = "gpu";
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "high",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "force-sdr",
quiet: true,
});
expect(producerState.createdJobs[0]?.useGpu).toBe(true);
});
it("lets the encoder surface its own error when capability detection fails", async () => {
ffmpegEncoderState.error = new Error("encoder probe timed out");
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "high",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "force-sdr",
quiet: true,
});
expect(producerState.createdJobs[0]?.useGpu).toBe(false);
expect(warn).not.toHaveBeenCalledWith(expect.stringContaining("encoder probe timed out"));
});
it("diagnoses advisory encoder probe failures unless quiet", async () => {
ffmpegEncoderState.error = new Error("encoder probe timed out");
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "high",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "force-sdr",
quiet: false,
});
expect(producerState.createdJobs[0]?.useGpu).toBe(false);
expect(warn).toHaveBeenCalledWith(expect.stringContaining("encoder probe timed out"));
});
it("resolves browser GPU from CLI flags, Docker mode, and env fallback", () => {
// Default (no flag, no env): auto — engine probes and chooses.
expect(resolveBrowserGpuForCli(false, undefined, undefined)).toBe("auto");
+32
View File
@@ -77,7 +77,9 @@ import { isDevMode } from "../utils/env.js";
import { buildDockerRunArgs, resolveDockerPlatform } from "../utils/dockerRunArgs.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { runEnvironmentChecks } from "../browser/preflight.js";
import { detectH264EncoderMode } from "../browser/ffmpeg.js";
import { chromeLaunchRemediation } from "../browser/linuxDeps.js";
import { killOrphanedProcesses } from "../utils/orphanCleanup.js";
import type { ProducerLogger, RenderJob } from "@hyperframes/producer";
import {
MAX_VP9_CPU_USED,
@@ -1439,8 +1441,18 @@ export async function renderLocal(
outputPath: string,
options: RenderOptions,
): Promise<SingleRenderResult> {
const recoveredOrphanTrees = killOrphanedProcesses();
if (recoveredOrphanTrees > 0 && !options.quiet) {
console.warn(
c.warn(
` Recovered ${recoveredOrphanTrees} orphaned browser process ${recoveredOrphanTrees === 1 ? "tree" : "trees"} from an interrupted render.`,
),
);
}
const preflight = await runEnvironmentChecks({
projectDir,
diskPaths: [tmpdir(), dirname(outputPath)],
browserPath: options.browserPath,
includeBrowser: true,
includeDisk: true,
@@ -1468,6 +1480,26 @@ export async function renderLocal(
process.env.PRODUCER_HEADLESS_SHELL_PATH = preflight.browser.executablePath;
}
if (!options.gpu && options.format === "mp4" && preflight.ffmpegPath) {
let encoderMode: ReturnType<typeof detectH264EncoderMode> = "software";
try {
encoderMode = detectH264EncoderMode(preflight.ffmpegPath, false);
} catch (error) {
// Capability probing is advisory. Let the real encode surface the
// authoritative FFmpeg error instead of failing here with a bare stack.
if (!options.quiet) {
const detail = error instanceof Error ? error.message : String(error);
console.warn(c.warn(` Unable to probe H.264 encoder capabilities: ${detail}`));
}
}
if (encoderMode === "gpu") {
console.warn(
c.warn(" FFmpeg does not include libx264; falling back to VideoToolbox H.264 encoding."),
);
options = { ...options, gpu: true };
}
}
const producer = await loadProducer();
const deParallelRouterTrialArmed = maybeEnableDeParallelRouterTrial(
options.quiet,