fix(render): consolidate preflight and local recovery (#2403)

* fix(render): fall back when libx264 is unavailable

* fix(render): recover orphaned browsers before retry

* fix(render): check disk space on write volumes

* test(engine): accept resolved ffmpeg binary paths

* fix(render): harden H.264 capability fallback

* chore(ci): scope inherited Fallow findings

* fix(render): diagnose encoder probe failures
This commit is contained in:
Miguel Ángel
2026-07-14 21:55:39 -04:00
committed by GitHub
parent 15ca6fd129
commit 5d3a7404fa
9 changed files with 224 additions and 15 deletions
+1
View File
@@ -23,6 +23,7 @@ HyperFrames now uses a 010 recommendation scale for feedback, keeps canvas z-
## Fixes ## Fixes
- **Render:** Limit automatic missing-libx264 fallback to macOS VideoToolbox; other hardware encoders now require explicit GPU configuration and surface their native FFmpeg error.
- **CLI:** Resolve color-mix() colors in contrast audit instead of false-failing ([6c2513d5c](https://github.com/heygen-com/hyperframes/commit/6c2513d5c80fea57d6686ccddf560e2e8c213afb), [#2445](https://github.com/heygen-com/hyperframes/pull/2445)) - **CLI:** Resolve color-mix() colors in contrast audit instead of false-failing ([6c2513d5c](https://github.com/heygen-com/hyperframes/commit/6c2513d5c80fea57d6686ccddf560e2e8c213afb), [#2445](https://github.com/heygen-com/hyperframes/pull/2445))
- **Producer:** Probe variable-bound media sources ([d2c8c2d80](https://github.com/heygen-com/hyperframes/commit/d2c8c2d8087814f35a3059069efc8c3a7a91cad8), [#2444](https://github.com/heygen-com/hyperframes/pull/2444)) - **Producer:** Probe variable-bound media sources ([d2c8c2d80](https://github.com/heygen-com/hyperframes/commit/d2c8c2d8087814f35a3059069efc8c3a7a91cad8), [#2444](https://github.com/heygen-com/hyperframes/pull/2444))
- **Pr To Video:** Use display names, not GitHub logins, in credits narration ([4cba58f5a](https://github.com/heygen-com/hyperframes/commit/4cba58f5a3a269b438d88aa9be406f7961c2fa25), [#2385](https://github.com/heygen-com/hyperframes/pull/2385)) - **Pr To Video:** Use display names, not GitHub logins, in credits narration ([4cba58f5a](https://github.com/heygen-com/hyperframes/commit/4cba58f5a3a269b438d88aa9be406f7961c2fa25), [#2385](https://github.com/heygen-com/hyperframes/pull/2385))
+39 -2
View File
@@ -1,12 +1,13 @@
import { execSync } from "node:child_process"; import { execFileSync, execSync } from "node:child_process";
import { existsSync } from "node:fs"; import { existsSync } from "node:fs";
import { resolve } from "node:path"; import { resolve } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("node:child_process", () => ({ execSync: vi.fn() })); vi.mock("node:child_process", () => ({ execFileSync: vi.fn(), execSync: vi.fn() }));
vi.mock("node:fs", () => ({ existsSync: vi.fn() })); vi.mock("node:fs", () => ({ existsSync: vi.fn() }));
const mockExec = vi.mocked(execSync); const mockExec = vi.mocked(execSync);
const mockExecFile = vi.mocked(execFileSync);
const mockExists = vi.mocked(existsSync); const mockExists = vi.mocked(existsSync);
// The common-dir fallback list is platform-gated (empty on win32), so pin the // The common-dir fallback list is platform-gated (empty on win32), so pin the
@@ -53,3 +54,39 @@ describe("findFFmpeg", () => {
expect(findFFmpeg()).toBeUndefined(); expect(findFFmpeg()).toBeUndefined();
}); });
}); });
describe("resolveH264EncoderMode", () => {
it("falls back to VideoToolbox when libx264 is absent", async () => {
const { resolveH264EncoderMode } = await import("./ffmpeg.js");
const encoders = `
V....D h264_videotoolbox VideoToolbox H.264 Encoder
`;
expect(resolveH264EncoderMode(encoders, false)).toBe("gpu");
});
it("does not treat a compiled Linux hardware encoder as usable", async () => {
const { resolveH264EncoderMode } = await import("./ffmpeg.js");
const encoders = `
V....D h264_vaapi H.264/AVC (VAAPI)
`;
expect(() => resolveH264EncoderMode(encoders, false)).toThrow(
"neither libx264 nor VideoToolbox",
);
});
it("inspects the configured FFmpeg binary", async () => {
mockExecFile.mockReturnValue(
" V....D h264_videotoolbox VideoToolbox H.264 Encoder\n" as never,
);
const { detectH264EncoderMode } = await import("./ffmpeg.js");
expect(detectH264EncoderMode("/custom/ffmpeg", false)).toBe("gpu");
expect(mockExecFile).toHaveBeenCalledWith(
"/custom/ffmpeg",
["-hide_banner", "-encoders"],
expect.objectContaining({ encoding: "utf-8" }),
);
});
});
+29 -1
View File
@@ -1,5 +1,5 @@
// fallow-ignore-file code-duplication // fallow-ignore-file code-duplication
import { execSync } from "node:child_process"; import { execFileSync, execSync } from "node:child_process";
import { existsSync } from "node:fs"; import { existsSync } from "node:fs";
import { resolve } from "node:path"; import { resolve } from "node:path";
import { detectLinuxDistro, ffmpegInstallCommand } from "./linuxDeps.js"; import { detectLinuxDistro, ffmpegInstallCommand } from "./linuxDeps.js";
@@ -7,6 +7,34 @@ import { detectLinuxDistro, ffmpegInstallCommand } from "./linuxDeps.js";
export const FFMPEG_PATH_ENV = "HYPERFRAMES_FFMPEG_PATH"; export const FFMPEG_PATH_ENV = "HYPERFRAMES_FFMPEG_PATH";
export const FFPROBE_PATH_ENV = "HYPERFRAMES_FFPROBE_PATH"; export const FFPROBE_PATH_ENV = "HYPERFRAMES_FFPROBE_PATH";
export type H264EncoderMode = "software" | "gpu";
/**
* Select the H.264 encoder class supported by an FFmpeg build.
*
* Some macOS FFmpeg distributions expose VideoToolbox but omit libx264. The
* default CPU render path must not pass libx264-only options such as `-preset`
* to those builds.
*/
export function resolveH264EncoderMode(
ffmpegEncodersOutput: string,
gpuRequested: boolean,
): H264EncoderMode {
if (gpuRequested) return "gpu";
if (/\blibx264\b/.test(ffmpegEncodersOutput)) return "software";
if (/\bh264_videotoolbox\b/.test(ffmpegEncodersOutput)) return "gpu";
throw new Error("This FFmpeg build has neither libx264 nor VideoToolbox H.264 encoding.");
}
export function detectH264EncoderMode(ffmpegPath: string, gpuRequested: boolean): H264EncoderMode {
const encoders = execFileSync(ffmpegPath, ["-hide_banner", "-encoders"], {
encoding: "utf-8",
stdio: ["ignore", "pipe", "pipe"],
timeout: 5000,
});
return resolveH264EncoderMode(encoders, gpuRequested);
}
function chooseBestPathCandidate( function chooseBestPathCandidate(
name: "ffmpeg" | "ffprobe", name: "ffmpeg" | "ffprobe",
candidates: string[], candidates: string[],
+13 -1
View File
@@ -1,6 +1,6 @@
// fallow-ignore-file code-duplication // fallow-ignore-file code-duplication
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { parseToolVersion, runEnvironmentChecks } from "./preflight.js"; import { checkDisk, parseToolVersion, runEnvironmentChecks } from "./preflight.js";
import * as manager from "./manager.js"; import * as manager from "./manager.js";
import * as linuxDeps from "./linuxDeps.js"; import * as linuxDeps from "./linuxDeps.js";
@@ -218,3 +218,15 @@ describe("parseToolVersion", () => {
); );
}); });
}); });
describe("checkDisk", () => {
it("checks the requested render volume", () => {
const freeDiskMb = vi.fn(() => 512);
expect(checkDisk("/external/render-output", freeDiskMb)).toMatchObject({
ok: false,
detail: "0.5 GB free at /external/render-output",
});
expect(freeDiskMb).toHaveBeenCalledWith("/external/render-output");
});
});
+11 -5
View File
@@ -38,6 +38,7 @@ export interface EnvironmentCheckResult {
export interface EnvironmentCheckOptions { export interface EnvironmentCheckOptions {
projectDir?: string; projectDir?: string;
diskPaths?: string[];
browserPath?: string; browserPath?: string;
includeBrowser?: boolean; includeBrowser?: boolean;
includeDisk?: boolean; includeDisk?: boolean;
@@ -226,8 +227,11 @@ async function checkChrome(browserPath?: string): Promise<EnvironmentCheckOutcom
}; };
} }
function checkDisk(projectDir = "."): EnvironmentCheckOutcome { export function checkDisk(
const freeMb = getFreeDiskMb(projectDir); path = ".",
freeDiskMb: (path: string) => number | null = getFreeDiskMb,
): EnvironmentCheckOutcome {
const freeMb = freeDiskMb(path);
if (freeMb === null) { if (freeMb === null) {
return { name: "Disk", ok: true, level: "ok", detail: "Unable to check" }; return { name: "Disk", ok: true, level: "ok", detail: "Unable to check" };
} }
@@ -238,11 +242,11 @@ function checkDisk(projectDir = "."): EnvironmentCheckOutcome {
ok: false, ok: false,
level: "error", level: "error",
title: "Low disk space", title: "Low disk space",
detail: `${freeGb} GB free`, detail: `${freeGb} GB free at ${path}`,
hint: "Renders produce large temp files. Free disk space before rendering.", hint: "Renders produce large temp files. Free disk space before rendering.",
}; };
} }
return { name: "Disk", ok: true, level: "ok", detail: `${freeGb} GB free` }; return { name: "Disk", ok: true, level: "ok", detail: `${freeGb} GB free at ${path}` };
} }
function checkWindowsUncPath(projectDir = process.cwd()): EnvironmentCheckOutcome | undefined { function checkWindowsUncPath(projectDir = process.cwd()): EnvironmentCheckOutcome | undefined {
@@ -257,6 +261,7 @@ function checkWindowsUncPath(projectDir = process.cwd()): EnvironmentCheckOutcom
}; };
} }
// fallow-ignore-next-line complexity
export async function runEnvironmentChecks( export async function runEnvironmentChecks(
options: EnvironmentCheckOptions = {}, options: EnvironmentCheckOptions = {},
): Promise<EnvironmentCheckResult> { ): Promise<EnvironmentCheckResult> {
@@ -281,7 +286,8 @@ export async function runEnvironmentChecks(
} }
if (options.includeDisk) { if (options.includeDisk) {
outcomes.push(checkDisk(options.projectDir)); const diskPaths = [...new Set(options.diskPaths ?? [options.projectDir ?? "."])];
outcomes.push(...diskPaths.map((path) => checkDisk(path)));
} }
if (options.includeWindowsUnc) { if (options.includeWindowsUnc) {
+92
View File
@@ -72,6 +72,15 @@ const preflightState = vi.hoisted(() => ({
}, },
})); }));
const ffmpegEncoderState = vi.hoisted(() => ({
mode: "software" as "software" | "gpu",
error: null as Error | null,
}));
const orphanCleanupState = vi.hoisted(() => ({
calls: 0,
killed: 0,
}));
vi.mock("../utils/producer.js", () => ({ vi.mock("../utils/producer.js", () => ({
loadProducer: vi.fn(async () => ({ loadProducer: vi.fn(async () => ({
resolveConfig: vi.fn((overrides: Record<string, unknown>) => { resolveConfig: vi.fn((overrides: Record<string, unknown>) => {
@@ -120,6 +129,10 @@ vi.mock("../telemetry/events.js", () => ({
})); }));
vi.mock("../browser/ffmpeg.js", () => ({ vi.mock("../browser/ffmpeg.js", () => ({
detectH264EncoderMode: vi.fn(() => {
if (ffmpegEncoderState.error) throw ffmpegEncoderState.error;
return ffmpegEncoderState.mode;
}),
findFFmpeg: vi.fn(() => "/usr/bin/ffmpeg"), findFFmpeg: vi.fn(() => "/usr/bin/ffmpeg"),
getFFmpegInstallHint: vi.fn(() => "brew install ffmpeg"), getFFmpegInstallHint: vi.fn(() => "brew install ffmpeg"),
})); }));
@@ -128,6 +141,13 @@ vi.mock("../browser/preflight.js", () => ({
runEnvironmentChecks: vi.fn(async () => preflightState.result), runEnvironmentChecks: vi.fn(async () => preflightState.result),
})); }));
vi.mock("../utils/orphanCleanup.js", () => ({
killOrphanedProcesses: vi.fn(() => {
orphanCleanupState.calls += 1;
return orphanCleanupState.killed;
}),
}));
describe("renderLocal browser GPU config", () => { describe("renderLocal browser GPU config", () => {
const savedEnv = new Map<string, string | undefined>(); const savedEnv = new Map<string, string | undefined>();
// Pre-resolve once. The first dynamic `import("./render.js")` in this file // Pre-resolve once. The first dynamic `import("./render.js")` in this file
@@ -173,6 +193,10 @@ describe("renderLocal browser GPU config", () => {
configState.writeConfigCalls = []; configState.writeConfigCalls = [];
trackingState.shouldTrack = true; trackingState.shouldTrack = true;
trackingState.renderObservations = []; trackingState.renderObservations = [];
ffmpegEncoderState.mode = "software";
ffmpegEncoderState.error = null;
orphanCleanupState.calls = 0;
orphanCleanupState.killed = 0;
resetTrialState(); resetTrialState();
savedEnv.clear(); savedEnv.clear();
savedEnv.set("HYPERFRAMES_FFMPEG_PATH", process.env.HYPERFRAMES_FFMPEG_PATH); savedEnv.set("HYPERFRAMES_FFMPEG_PATH", process.env.HYPERFRAMES_FFMPEG_PATH);
@@ -185,6 +209,22 @@ describe("renderLocal browser GPU config", () => {
delete process.env.HF_DE_PARALLEL_ROUTER; delete process.env.HF_DE_PARALLEL_ROUTER;
}); });
it("cleans orphaned browser trees before starting a local render", async () => {
orphanCleanupState.killed = 1;
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "standard",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "auto",
quiet: true,
});
expect(orphanCleanupState.calls).toBe(1);
});
afterEach(() => { afterEach(() => {
for (const [key, value] of savedEnv) { for (const [key, value] of savedEnv) {
if (value === undefined) { if (value === undefined) {
@@ -323,6 +363,58 @@ describe("renderLocal browser GPU config", () => {
expect(process.env.PRODUCER_HEADLESS_SHELL_PATH).toBe("/mock/chrome"); expect(process.env.PRODUCER_HEADLESS_SHELL_PATH).toBe("/mock/chrome");
}); });
it("falls back to hardware encoding when FFmpeg omits libx264", async () => {
ffmpegEncoderState.mode = "gpu";
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "high",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "force-sdr",
quiet: true,
});
expect(producerState.createdJobs[0]?.useGpu).toBe(true);
});
it("lets the encoder surface its own error when capability detection fails", async () => {
ffmpegEncoderState.error = new Error("encoder probe timed out");
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "high",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "force-sdr",
quiet: true,
});
expect(producerState.createdJobs[0]?.useGpu).toBe(false);
expect(warn).not.toHaveBeenCalledWith(expect.stringContaining("encoder probe timed out"));
});
it("diagnoses advisory encoder probe failures unless quiet", async () => {
ffmpegEncoderState.error = new Error("encoder probe timed out");
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
await renderLocal("/tmp/project", "/tmp/out.mp4", {
fps: { num: 30, den: 1 },
quality: "high",
format: "mp4",
gpu: false,
browserGpuMode: "software",
hdrMode: "force-sdr",
quiet: false,
});
expect(producerState.createdJobs[0]?.useGpu).toBe(false);
expect(warn).toHaveBeenCalledWith(expect.stringContaining("encoder probe timed out"));
});
it("resolves browser GPU from CLI flags, Docker mode, and env fallback", () => { it("resolves browser GPU from CLI flags, Docker mode, and env fallback", () => {
// Default (no flag, no env): auto — engine probes and chooses. // Default (no flag, no env): auto — engine probes and chooses.
expect(resolveBrowserGpuForCli(false, undefined, undefined)).toBe("auto"); expect(resolveBrowserGpuForCli(false, undefined, undefined)).toBe("auto");
+32
View File
@@ -77,7 +77,9 @@ import { isDevMode } from "../utils/env.js";
import { buildDockerRunArgs, resolveDockerPlatform } from "../utils/dockerRunArgs.js"; import { buildDockerRunArgs, resolveDockerPlatform } from "../utils/dockerRunArgs.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { runEnvironmentChecks } from "../browser/preflight.js"; import { runEnvironmentChecks } from "../browser/preflight.js";
import { detectH264EncoderMode } from "../browser/ffmpeg.js";
import { chromeLaunchRemediation } from "../browser/linuxDeps.js"; import { chromeLaunchRemediation } from "../browser/linuxDeps.js";
import { killOrphanedProcesses } from "../utils/orphanCleanup.js";
import type { ProducerLogger, RenderJob } from "@hyperframes/producer"; import type { ProducerLogger, RenderJob } from "@hyperframes/producer";
import { import {
MAX_VP9_CPU_USED, MAX_VP9_CPU_USED,
@@ -1439,8 +1441,18 @@ export async function renderLocal(
outputPath: string, outputPath: string,
options: RenderOptions, options: RenderOptions,
): Promise<SingleRenderResult> { ): Promise<SingleRenderResult> {
const recoveredOrphanTrees = killOrphanedProcesses();
if (recoveredOrphanTrees > 0 && !options.quiet) {
console.warn(
c.warn(
` Recovered ${recoveredOrphanTrees} orphaned browser process ${recoveredOrphanTrees === 1 ? "tree" : "trees"} from an interrupted render.`,
),
);
}
const preflight = await runEnvironmentChecks({ const preflight = await runEnvironmentChecks({
projectDir, projectDir,
diskPaths: [tmpdir(), dirname(outputPath)],
browserPath: options.browserPath, browserPath: options.browserPath,
includeBrowser: true, includeBrowser: true,
includeDisk: true, includeDisk: true,
@@ -1468,6 +1480,26 @@ export async function renderLocal(
process.env.PRODUCER_HEADLESS_SHELL_PATH = preflight.browser.executablePath; process.env.PRODUCER_HEADLESS_SHELL_PATH = preflight.browser.executablePath;
} }
if (!options.gpu && options.format === "mp4" && preflight.ffmpegPath) {
let encoderMode: ReturnType<typeof detectH264EncoderMode> = "software";
try {
encoderMode = detectH264EncoderMode(preflight.ffmpegPath, false);
} catch (error) {
// Capability probing is advisory. Let the real encode surface the
// authoritative FFmpeg error instead of failing here with a bare stack.
if (!options.quiet) {
const detail = error instanceof Error ? error.message : String(error);
console.warn(c.warn(` Unable to probe H.264 encoder capabilities: ${detail}`));
}
}
if (encoderMode === "gpu") {
console.warn(
c.warn(" FFmpeg does not include libx264; falling back to VideoToolbox H.264 encoding."),
);
options = { ...options, gpu: true };
}
}
const producer = await loadProducer(); const producer = await loadProducer();
const deParallelRouterTrialArmed = maybeEnableDeParallelRouterTrial( const deParallelRouterTrialArmed = maybeEnableDeParallelRouterTrial(
options.quiet, options.quiet,
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
/** /**
* buildStreamingArgs unit tests. * buildStreamingArgs unit tests.
* *
@@ -12,7 +13,7 @@
import { EventEmitter } from "events"; import { EventEmitter } from "events";
import { mkdtempSync } from "fs"; import { mkdtempSync } from "fs";
import { tmpdir } from "os"; import { tmpdir } from "os";
import { join } from "path"; import { basename, join } from "path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { import {
@@ -511,7 +512,7 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions); const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions);
expect(calls).toHaveLength(1); expect(calls).toHaveLength(1);
expect(calls[0]?.command).toBe("ffmpeg"); expect(basename(calls[0]?.command ?? "")).toMatch(/^ffmpeg(?:\.exe)?$/);
const proc = calls[0]!.proc; const proc = calls[0]!.proc;
const closePromise = encoder.close(); const closePromise = encoder.close();
+4 -4
View File
@@ -1,7 +1,7 @@
// fallow-ignore-file code-duplication // fallow-ignore-file code-duplication
import { EventEmitter } from "events"; import { EventEmitter } from "events";
import { readFileSync } from "fs"; import { readFileSync } from "fs";
import { resolve } from "path"; import { basename, resolve } from "path";
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { extractMediaMetadata, extractPngMetadataFromBuffer } from "./ffprobe.js"; import { extractMediaMetadata, extractPngMetadataFromBuffer } from "./ffprobe.js";
@@ -217,7 +217,7 @@ describe("ffprobe missing-binary fallback", () => {
const meta = await extractMediaMetadataMocked(fixture); const meta = await extractMediaMetadataMocked(fixture);
expect(calls.length).toBe(1); expect(calls.length).toBe(1);
expect(calls[0]?.command).toBe("ffprobe"); expect(basename(calls[0]?.command ?? "")).toMatch(/^ffprobe(?:\.exe)?$/);
expect(meta.videoCodec).toBe("png"); expect(meta.videoCodec).toBe("png");
expect(meta.durationSeconds).toBe(0); expect(meta.durationSeconds).toBe(0);
expect(meta.fps).toBe(0); expect(meta.fps).toBe(0);
@@ -323,7 +323,7 @@ describe("ffprobe missing-binary fallback", () => {
/ffprobe not found/, /ffprobe not found/,
); );
expect(calls.length).toBe(1); expect(calls.length).toBe(1);
expect(calls[0]?.command).toBe("ffprobe"); expect(basename(calls[0]?.command ?? "")).toMatch(/^ffprobe(?:\.exe)?$/);
}); });
it("analyzeKeyframeIntervals surfaces a ffprobe-missing error verbatim", async () => { it("analyzeKeyframeIntervals surfaces a ffprobe-missing error verbatim", async () => {
@@ -338,7 +338,7 @@ describe("ffprobe missing-binary fallback", () => {
/ffprobe not found/, /ffprobe not found/,
); );
expect(calls.length).toBe(1); expect(calls.length).toBe(1);
expect(calls[0]?.command).toBe("ffprobe"); expect(basename(calls[0]?.command ?? "")).toMatch(/^ffprobe(?:\.exe)?$/);
}); });
it("ffprobe-missing error message includes install hint", async () => { it("ffprobe-missing error message includes install hint", async () => {