fix(producer): frames start drawing while the audio mix finishes (#7985)

* fix(producer): frames start drawing while the audio mix finishes

* fix(producer): keep the encode stage's comment share

* fix(producer): on machines with under 4 CPUs frames wait for the audio mix

* test(producer): a mix that fails during capture is reported as the audio stage's failure

* fix(producer): a mix that fails during capture names the audio stage as where the render failed
This commit is contained in:
Miguel Ángel
2026-11-17 14:35:07 +00:00
committed by GitHub
parent 72925380cc
commit 0be1a0719c
4 changed files with 327 additions and 64 deletions
@@ -141,6 +141,8 @@ export interface EncodeStageInput {
gopSize?: number;
}
export type VideoEncodeInput = Omit<EncodeStageInput, "hasAudio" | "audioOutputPath">;
export interface EncodeStageResult {
/** Wall-clock ms for the encode (or png-copy) phase. */
encodeMs: number;
@@ -264,7 +266,7 @@ function startEncodeProgress(
return (frames = total) => reportEncodeProgress(job, frames, total, onProgress, 75);
}
function resolveEngineConfig(input: EncodeStageInput) {
function resolveEngineConfig(input: VideoEncodeInput) {
return input.engineConfig ?? input.job.config.producerConfig ?? resolveConfig();
}
@@ -273,7 +275,7 @@ export function overlapEncoderThreads(workerCount: number, cpuCount: number): nu
}
export function videoEncoderOptions(
input: EncodeStageInput,
input: VideoEncodeInput,
engineCfg: Pick<EngineConfig, "vp9CpuUsed">,
softwareEncoderThreads?: number,
) {
@@ -306,7 +308,7 @@ export function videoEncoderOptions(
* arguments. Needs a core the capture workers leave free.
*/
export function canEncodeWhileCapturing(
input: EncodeStageInput,
input: VideoEncodeInput,
workerCount: number,
cpuCount: number,
): boolean {
@@ -350,7 +352,7 @@ async function readLandedFrame(workerPath: string | undefined, mergedPath: strin
* On failure `finish` re-runs `runEncodeStage` on the same frames.
*/
export function encodeWhileCapturing(
input: EncodeStageInput,
input: VideoEncodeInput,
totalFrames: number,
captureBudget?: { workerCount: number; cpuCount: number },
): EncodeWhileCapturing {
@@ -432,7 +434,7 @@ export function encodeWhileCapturing(
error: String(failure ?? result?.error),
},
);
return runEncodeStage(input);
return runEncodeStage({ ...input, hasAudio: false });
},
abort: async () => {
stop.abort();
@@ -5,14 +5,18 @@ import { join } from "node:path";
import { resolveConfig } from "@hyperframes/engine";
import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js";
// Entry: executeRenderJob. The audio mix is held open past capture setup, so capture waits on it; the test drives the
// mix's FFmpeg progress and reads the lines the render reports meanwhile.
// Entry: executeRenderJob. The audio mix is held open past capture, so the steps after it wait on it; the test drives
// the mix's FFmpeg progress and reads the lines the render reports meanwhile.
const trace = vi.hoisted(() => ({
ffmpegProgressed: (): void => {},
failAudio: (_error: Error): void => {},
setUp: false,
}));
vi.mock("@hyperframes/engine", async (importOriginal) => ({
...(await importOriginal<typeof import("@hyperframes/engine")>()),
usableCpuCount: () => 8,
}));
vi.mock("./render/shared.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/shared.js")>()),
createMemorySampler: () => ({
@@ -82,12 +86,21 @@ vi.mock("./render/stages/audioStage.js", () => ({
}));
vi.mock("./render/captureCost.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/captureCost.js")>()),
resolveRenderWorkerCount: () => {
// The mix writes more while capture is still setting up: not a line of its own yet.
trace.ffmpegProgressed();
trace.setUp = true;
return 1;
},
resolveRenderWorkerCount: () => 1,
}));
/** Capture finishes while the mix still runs; the mix writes more meanwhile: not a line of its own yet. */
async function capturedWhileMixing() {
trace.ffmpegProgressed();
trace.setUp = true;
return { success: true, workerCount: 1, probeSession: null, lastBrowserConsole: [], encodeMs: 0 };
}
vi.mock("./render/stages/captureStage.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/stages/captureStage.js")>()),
runCaptureStage: capturedWhileMixing,
}));
vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/stages/captureStreamingStage.js")>()),
runCaptureStreamingStage: capturedWhileMixing,
}));
const dirs: string[] = [];
@@ -98,7 +111,7 @@ afterEach(() => {
const turn = () => new Promise((resolve) => setImmediate(resolve));
it("says it is processing audio while capture waits on a mix whose FFmpeg writes more, at most every 5 s", async () => {
it("says it is processing audio while the render waits on a mix whose FFmpeg writes more, at most every 5 s", async () => {
vi.useFakeTimers({ toFake: ["Date"] });
const root = mkdtempSync(join(tmpdir(), "hf-audio-wait-"));
dirs.push(root);
@@ -123,7 +136,7 @@ it("says it is processing audio while capture waits on a mix whose FFmpeg writes
// Only the stage's own first line.
const before = audioLines().length;
expect(before).toBe(1);
// The mix writes more every 6 s until capture, now waiting on it, says so.
// The mix writes more every 6 s until the render, now waiting on it, says so.
for (let at = 0; at < 200 && audioLines().length === before; at++) {
vi.setSystemTime(Date.now() + 6_000);
trace.ffmpegProgressed();
@@ -0,0 +1,222 @@
import { afterEach, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveConfig } from "@hyperframes/engine";
import { createRenderJob, executeRenderJob, RenderQualityError } from "./renderOrchestrator.js";
import { AUDIO_STAGE } from "./render/shared.js";
// Entry: executeRenderJob. The audio mix is held open; every capture stage records that it started, then waits for the
// test or for the render's signal, so the test sees whether frames are drawn while the mix runs and what stops them.
const trace = vi.hoisted(() => ({
events: [] as string[],
finishAudio: (_result: object): void => {},
failAudio: (_error: Error): void => {},
finishCapture: (): void => {},
cpus: 8,
}));
vi.mock("@hyperframes/engine", async (importOriginal) => ({
...(await importOriginal<typeof import("@hyperframes/engine")>()),
usableCpuCount: () => trace.cpus,
}));
vi.mock("./render/shared.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/shared.js")>()),
createMemorySampler: () => ({
peakRssBytes: () => 0,
peakHeapUsedBytes: () => 0,
stop: () => {},
}),
}));
vi.mock("./fileServer.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./fileServer.js")>()),
createFileServer: async () => ({
url: "http://127.0.0.1:1",
port: 1,
close: () => {},
addPreHeadScript: () => {},
}),
}));
vi.mock("./render/stages/compileStage.js", () => ({
runCompileStage: async () => ({
compiled: {
html: "<div></div>",
hasShaderTransitions: false,
renderModeHints: { reasons: [] },
},
composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] },
deviceScaleFactor: 1,
outputWidth: 16,
outputHeight: 16,
compileOnlyMs: 0,
forceScreenshot: true,
}),
}));
vi.mock("./render/stages/probeStage.js", () => ({
runProbeStage: async ({ compiled }: { compiled: object }) => ({
compiled,
fileServer: null,
probeSession: null,
lastBrowserConsole: [],
duration: 40 / 30,
totalFrames: 40,
browserProbeMs: 0,
beginFrameStalled: false,
}),
}));
vi.mock("./render/stages/extractVideosStage.js", () => ({
runExtractVideosStage: async () => ({
extractionResult: null,
frameLookup: null,
videoReadinessSkipIds: [],
videoMetadataHints: [],
nativeHdrVideoIds: new Set(),
videoTransfers: new Map(),
nativeHdrImageIds: new Set(),
imageTransfers: new Map(),
hdrImageSrcPaths: new Map(),
imageColorSpaces: [],
videoExtractMs: 0,
failureToEnforce: null,
}),
}));
vi.mock("./render/stages/audioStage.js", () => ({
runAudioStage: () =>
new Promise((resolve, reject) => {
trace.events.push("audio started");
trace.finishAudio = (result) => {
trace.events.push("audio done");
resolve(result);
};
trace.failAudio = reject;
}),
}));
vi.mock("./render/captureCost.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/captureCost.js")>()),
resolveRenderWorkerCount: () => 1,
}));
/** Started, then held until the test finishes it (it then stops the render) or the render's signal aborts it. */
function heldCapture({ abortSignal }: { abortSignal?: AbortSignal }): Promise<never> {
trace.events.push("capture started");
return new Promise((_resolve, reject) => {
trace.finishCapture = () => reject(new Error("stop after capture"));
abortSignal?.addEventListener("abort", () => {
trace.events.push("capture aborted");
reject(abortSignal.reason);
});
});
}
vi.mock("./render/stages/captureStage.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/stages/captureStage.js")>()),
runCaptureStage: heldCapture,
}));
vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/stages/captureStreamingStage.js")>()),
runCaptureStreamingStage: heldCapture,
}));
vi.mock("./render/stages/captureSegmentedStage.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/stages/captureSegmentedStage.js")>()),
runCaptureSegmentedStage: heldCapture,
}));
const dirs: string[] = [];
afterEach(() => {
trace.events.length = 0;
trace.cpus = 8;
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
async function until(condition: () => boolean): Promise<boolean> {
for (let turn = 0; turn < 5_000 && !condition(); turn++) {
await new Promise((resolve) => setImmediate(resolve));
}
return condition();
}
function startRender() {
const root = mkdtempSync(join(tmpdir(), "hf-capture-during-audio-"));
dirs.push(root);
writeFileSync(join(root, "index.html"), "<div></div>");
const job = createRenderJob({
fps: 30,
quality: "draft",
format: "mp4",
hdrMode: "force-sdr",
producerConfig: resolveConfig({ enablePageSideCompositing: false }),
logger: { info() {}, warn() {}, error() {}, debug() {} },
});
const failure = executeRenderJob(job, root, join(root, "out.mp4")).then(
() => undefined,
(error: unknown) => error,
);
return { job, root, failure };
}
it("draws frames while the audio still mixes", async () => {
const { root, failure } = startRender();
expect(await until(() => trace.events.includes("capture started"))).toBe(true);
expect(trace.events).toEqual(["audio started", "capture started"]);
trace.finishAudio({
audioOutputPath: join(root, "audio.wav"),
hasAudio: false,
audioProcessMs: 0,
});
trace.finishCapture();
expect(String(await failure)).toContain("stop after capture");
});
it("an audio mix that fails while frames are drawn stops them and fails the render with the mix's message", async () => {
const { job, root, failure } = startRender();
expect(await until(() => trace.events.includes("capture started"))).toBe(true);
trace.finishAudio({
audioOutputPath: join(root, "audio.wav"),
hasAudio: false,
audioProcessMs: 0,
audioError: "the music track could not be read",
audioFailures: [],
});
const error = await failure;
expect(trace.events).toContain("capture aborted");
expect(error).toBeInstanceOf(RenderQualityError);
expect(String(error)).toContain(
"Audio mix failed; output would be video-only: the music track could not be read",
);
expect(job.status).not.toBe("cancelled");
expect(job.failedStage).toBe(AUDIO_STAGE);
});
it("an audio stage that throws while frames are drawn stops them and fails the render with its error", async () => {
const { job, failure } = startRender();
expect(await until(() => trace.events.includes("capture started"))).toBe(true);
trace.failAudio(new Error("the mix's FFmpeg exited with code 1"));
const error = await failure;
expect(trace.events).toContain("capture aborted");
expect(String(error)).toContain("the mix's FFmpeg exited with code 1");
expect(job.status).not.toBe("cancelled");
expect(job.failedStage).toBe(AUDIO_STAGE);
});
it("on fewer than 4 CPUs draws frames only after the mix", async () => {
trace.cpus = 2;
const { root, failure } = startRender();
expect(await until(() => trace.events.includes("audio started"))).toBe(true);
expect(await until(() => trace.events.includes("capture started"))).toBe(false);
trace.finishAudio({
audioOutputPath: join(root, "audio.wav"),
hasAudio: false,
audioProcessMs: 0,
});
expect(await until(() => trace.events.includes("capture started"))).toBe(true);
expect(trace.events).toEqual(["audio started", "audio done", "capture started"]);
trace.finishCapture();
expect(String(await failure)).toContain("stop after capture");
});
@@ -1,5 +1,5 @@
import type { ExpectedOutputDimensions } from "./render/outputDimensions.js";
import { audioFailureDiagnostics } from "@hyperframes/engine";
import { audioFailureDiagnostics, usableCpuCount } from "@hyperframes/engine";
// fallow-ignore-file unused-type circular-dependency code-duplication complexity
/**
* Render Orchestrator Service
@@ -275,7 +275,7 @@ import {
canEncodeWhileCapturing,
encodeWhileCapturing,
runEncodeStage,
type EncodeStageInput,
type VideoEncodeInput,
} from "./render/stages/encodeStage.js";
import { recordLimiterAttenuation, runAssembleStage } from "./render/stages/assembleStage.js";
import { shouldUseLayeredComposite } from "./hdrCompositor.js";
@@ -857,6 +857,8 @@ export class RenderQualityError extends Error {
}
}
const MIN_CPUS_TO_CAPTURE_DURING_MIX = 4;
export function applyRenderWarningPolicy(
job: RenderJob,
captureWarnings: readonly CaptureWarning[],
@@ -3153,10 +3155,15 @@ async function executeRenderPipeline(input: {
const onProgress = execution.onProgress;
const tooSlow = new AbortController();
let tooSlowError: RenderTooSlowError | undefined;
// Frames are drawn while the audio mixes; a failed mix stops them and is the render's error.
const audioFailed = new AbortController();
let audioFailure: unknown;
let calibrationEvaluateTimedOut = false;
const executionSignal = job.config.maxRenderMs
? AbortSignal.any([tooSlow.signal, ...(execution.signal ? [execution.signal] : [])])
: execution.signal;
const executionSignal = AbortSignal.any([
audioFailed.signal,
...(job.config.maxRenderMs ? [tooSlow.signal] : []),
...(execution.signal ? [execution.signal] : []),
]);
let fileServer: FileServerHandle | null = null;
let probeSession: CaptureSession | null = null;
let lastBrowserConsole: string[] = [];
@@ -4940,41 +4947,50 @@ async function executeRenderPipeline(input: {
const effectiveQuality = job.config.crf ?? preset.quality;
const effectiveBitrate = job.config.crf != null ? undefined : job.config.videoBitrate;
const audioResult = await audioWait.wait(audioStage);
const { audioOutputPath, deliverySourcePath, hasAudio } = audioResult;
perfStages.audioProcessMs = audioResult.audioProcessMs;
recordLimiterAttenuation(job, audioResult.audioLoweredDb);
if (audioResult.audioError) {
const audioFailures = audioResult.audioFailures ?? [];
const failureOwner =
audioFailures.length === 0
? undefined
: audioFailures.some((failure) => failure.owner === "system")
? "system"
: "user";
const retryable =
audioFailures.length === 0
? undefined
: audioFailures.every((failure) => failure.retryable);
applyRenderWarningPolicy(
job,
[
{
code: "audio_processing_failed",
message: `Audio mix failed; output would be video-only: ${audioResult.audioError}`,
details: {
mediaType: "audio",
audioFailures: audioFailureDiagnostics(audioFailures),
failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))],
failureStages: [...new Set(audioFailures.map((failure) => failure.stage))],
failureOwner,
retryable,
// Capture never reads the mix: it starts now, and the first step that needs the mix waits for it.
const audioReady = audioStage.then((audioResult) => {
perfStages.audioProcessMs = audioResult.audioProcessMs;
recordLimiterAttenuation(job, audioResult.audioLoweredDb);
if (audioResult.audioError) {
const audioFailures = audioResult.audioFailures ?? [];
const failureOwner =
audioFailures.length === 0
? undefined
: audioFailures.some((failure) => failure.owner === "system")
? "system"
: "user";
const retryable =
audioFailures.length === 0
? undefined
: audioFailures.every((failure) => failure.retryable);
applyRenderWarningPolicy(
job,
[
{
code: "audio_processing_failed",
message: `Audio mix failed; output would be video-only: ${audioResult.audioError}`,
details: {
mediaType: "audio",
audioFailures: audioFailureDiagnostics(audioFailures),
failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))],
failureStages: [...new Set(audioFailures.map((failure) => failure.stage))],
failureOwner,
retryable,
},
},
},
],
log,
);
}
],
log,
);
}
return audioResult;
});
audioReady.catch((error: unknown) => {
if (executionSignal.aborted) return;
audioFailure = error;
audioFailed.abort(error);
});
const mixedAudio = () => audioWait.wait(audioReady);
if (usableCpuCount() < MIN_CPUS_TO_CAPTURE_DURING_MIX) await mixedAudio();
resetCaptureAttemptProgress(job);
@@ -5180,7 +5196,7 @@ async function executeRenderPipeline(input: {
if (
(!isVerifyError && !isDrawElementCaptureError(err) && !isDeStall) ||
err instanceof RenderCancelledError ||
executionSignal?.aborted === true
executionSignal.aborted === true
) {
throw err;
}
@@ -5360,7 +5376,7 @@ async function executeRenderPipeline(input: {
const isParallelStall = isParallelCaptureStallError(err);
const isEncoderDeath = isRetryableEncoderDeath(err);
const isCancellation =
err instanceof RenderCancelledError || executionSignal?.aborted === true;
err instanceof RenderCancelledError || executionSignal.aborted === true;
if (
!shouldRetryViaPinnedFallback({
isVerifyError,
@@ -5549,7 +5565,7 @@ async function executeRenderPipeline(input: {
}
// ── Disk-based capture (original flow) ────────────────────────────
resetCaptureAttemptProgress(job);
const encodeInput: EncodeStageInput = {
const encodeInput: VideoEncodeInput = {
job,
log,
outputPath: stagedOutputPath,
@@ -5559,8 +5575,6 @@ async function executeRenderPipeline(input: {
height,
needsAlpha,
captureImageFormat: captureOptions.format ?? "jpeg",
hasAudio,
audioOutputPath,
isPngSequence,
isGif,
preset,
@@ -5628,7 +5642,7 @@ async function executeRenderPipeline(input: {
if (
(!isDrawElementVerificationError(err) && !isDrawElementCaptureError(err)) ||
err instanceof RenderCancelledError ||
executionSignal?.aborted === true
executionSignal.aborted === true
) {
throw err;
}
@@ -5701,6 +5715,7 @@ async function executeRenderPipeline(input: {
perfStages.captureFrameMs = captureFrameMs;
perfStages.captureSetupMs = Math.max(0, perfStages.captureMs - captureFrameMs);
const { hasAudio, audioOutputPath } = await mixedAudio();
const encodeRes = await observeRenderStage(
observability,
"encode",
@@ -5710,7 +5725,10 @@ async function executeRenderPipeline(input: {
isGif,
chunkedEncode: enableChunkedEncode,
}),
() => (overlap ? overlap.finish() : runEncodeStage(encodeInput)),
() =>
overlap
? overlap.finish()
: runEncodeStage({ ...encodeInput, hasAudio, audioOutputPath }),
);
perfStages.encodeMs = encodeRes.encodeMs;
}
@@ -5751,6 +5769,7 @@ async function executeRenderPipeline(input: {
// directory deliverable, and gif is written directly to outputPath by the
// two-pass palette encoder. hls is a directory deliverable but still runs
// this stage — the HLS packaging IS its assemble step.
const { audioOutputPath, deliverySourcePath, hasAudio } = await mixedAudio();
if (!isPngSequence && !isGif) {
const assembleRes = await observeRenderStage(
observability,
@@ -5900,8 +5919,15 @@ async function executeRenderPipeline(input: {
framesRendered: job.framesRendered ?? 0,
error: caught,
});
const error = surfaceScriptFailure(tooSlowError ?? pageNeverReady ?? caught, job);
if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) {
const error = surfaceScriptFailure(
tooSlowError ?? audioFailure ?? pageNeverReady ?? caught,
job,
);
if (
!tooSlowError &&
!audioFailure &&
(error instanceof RenderCancelledError || executionSignal.aborted)
) {
job.error = error instanceof Error ? error.message : "render_cancelled";
updateJobStatus(job, "cancelled", "Render cancelled", job.progress, onProgress);
await eventPublisher.flush();
@@ -5991,7 +6017,7 @@ async function executeRenderPipeline(input: {
);
}
const failedStage = job.currentStage || "pipeline";
const failedStage = audioFailure === undefined ? job.currentStage || "pipeline" : AUDIO_STAGE;
const observabilitySummary = observability.summary({
lastBrowserConsole,
capture: withCapturedBeyondViewport(captureObservability, dedupPerfs),