ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 17:27:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(engine): hundreds of effects in one audio group mix in time, and a mix timeout is named (#6101)

Browse Source 
* fix(engine): hundreds of effects in one audio group mix in time, and a mix timeout is named

* refactor(engine): name the group mix's single pad instead of commenting it
This commit is contained in:
Miguel Ángel
2026-10-23 01:55:23 +00:00
committed by GitHub
parent f5f884e33d
commit 1accb709ff
2 changed files with 91 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/services/audioMixer.test.ts
+71
View File
@@ -703,6 +703,77 @@ describe("processCompositionAudio", () => {
expect(existsSync(workDir)).toBe(false);
});
const groupedEffects = (baseDir: string) => {
const track = (id: string, start: number) => ({
id,
src: `${id}.wav`,
start,
end: start + 1,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio" as const,
groupId: "sfx",
});
for (const id of ["hit", "whoosh", "pop"]) writeFileSync(join(baseDir, `${id}.wav`), "stub");
return [track("hit", 0), track("whoosh", 1), track("pop", 8)];
};
it("pads a group's members once after summing them, not each member to the film length", async () => {
// Per-member padding made the sub-mix cost members x film length: ~680 effects in a
// 10-minute film ran past the ffmpeg deadline.
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
await processCompositionAudio(
groupedEffects(baseDir),
baseDir,
workDir,
join(baseDir, "out.m4a"),
600,
);
const groupMix = capturedFilterScripts.find((script) => script.includes("normalize=0"));
expect(groupMix).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");
expect(groupMix).not.toMatch(/adelay=[^;]*,apad/);
expect((groupMix?.match(/apad/g) ?? []).length).toBe(1);
expect(groupMix).toContain("apad,asetpts=N/SR/TB,atrim=0:600[out]");
});
it("names the deadline when a group sub-mix is killed for taking too long", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
runFfmpegMock.mockImplementation(async (args: string[]) =>
args.includes("-filter_complex_script")
? {
success: false,
durationMs: 300_000,
stderr: "Stream mapping:\n Stream #0:0 -> #0:0 (pcm_s16le (native))",
exitCode: 255,
terminationReason: "deadline",
}
: { success: true, durationMs: 1, stderr: "", exitCode: 0 },
);
const result = await processCompositionAudio(
groupedEffects(baseDir),
baseDir,
workDir,
join(baseDir, "out.m4a"),
600,
undefined,
{ ffmpegProcessTimeout: 300_000 },
);
expect(result.success).toBe(false);
const failure = result.failures?.find((f) => f.elementId === "sfx");
expect(failure?.reason).toBe("ffmpeg_timeout");
expect(failure?.detail).toContain("FFmpeg timed out after 300000 ms");
expect(failure?.detail).toContain("FFMPEG_PROCESS_TIMEOUT_MS");
});
it("lets an FX tail run past the clip, still bounded by the composition", async () => {
// A reverb is still decaying when the clip's own audio stops. Trimming at
// the clip boundary is what cut every tail short in the render.
packages/engine/src/services/audioMixer.ts
+20 -12
View File
@@ -24,7 +24,7 @@ import {
writeUrlDownloadTelemetry,
} from "../utils/urlDownloader.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import { describeFfmpegFailure, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import { encodeAac } from "./aacEncode.js";
import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
@@ -446,6 +446,7 @@ function downloadFailure(error: unknown, elementId: string): AudioProcessingFail
function ffmpegFailure(
stage: Extract<AudioFailureStage, "extract" | "prepare" | "mix" | "silence">,
result: RunFfmpegResult,
timeoutMs: number,
elementId?: string,
): AudioProcessingFailure {
const stderr = result.stderr ?? "";
@@ -486,8 +487,8 @@ function ffmpegFailure(
subprocess: audioSubprocessEvidence(result),
detail: boundedDetail(
result.error?.message
? `${formatFfmpegError(result.exitCode, stderr)}: ${result.error.message}`
: formatFfmpegError(result.exitCode, stderr),
? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`
: describeFfmpegFailure(result, timeoutMs),
),
};
}
@@ -724,7 +725,7 @@ async function extractAudioSegment(
failure,
};
}
let failure = result.success ? undefined : ffmpegFailure(stage, result);
let failure = result.success ? undefined : ffmpegFailure(stage, result, ffmpegProcessTimeout);
const probedLinkFailedToRead =
readsUrl && (failure?.reason === "invalid_media" || failure?.reason === "ffmpeg_failed");
if (failure && probedLinkFailedToRead) {
@@ -780,7 +781,9 @@ async function generateSilence(
failure,
};
}
const failure = !result.success ? ffmpegFailure("silence", result) : undefined;
const failure = !result.success
? ffmpegFailure("silence", result, ffmpegProcessTimeout)
: undefined;
return {
success: result.success,
outputPath,
@@ -934,7 +937,7 @@ async function mixAudioTracks(
};
}
if (!result.success) {
const failure = ffmpegFailure("mix", result);
const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);
return {
success: false,
outputPath,
@@ -1000,10 +1003,10 @@ async function mixGroupMembers(
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
const buildInputFilters = (ignoreKeyframes: boolean) =>
const buildInputFilters = (ignoreKeyframes: boolean, padMembers: boolean) =>
memberTracks.map((track, i) => {
const volumeFilter = buildVolumeExpression(track, ignoreKeyframes);
return buildTrackInputFilter(track, i, volumeFilter, totalDuration);
return buildTrackInputFilter(track, i, volumeFilter, totalDuration, padMembers);
});
const mixInputs = memberTracks.map((_, i) => `[a${i}]`).join("");
@@ -1011,9 +1014,10 @@ async function mixGroupMembers(
useNormalize: boolean,
ignoreKeyframes = false,
): Promise<RunFfmpegResult> => {
const inputFilters = buildInputFilters(ignoreKeyframes);
const inputFilters = buildInputFilters(ignoreKeyframes, !useNormalize);
const padUnitySumOnceToFilm = `[mixed];[mixed]apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}`;
const mixFilter = useNormalize
? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0[out]`
? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0${padUnitySumOnceToFilm}[out]`
: // Every member is padded and trimmed to totalDuration above, and
// dropout_transition=0 keeps all N inputs active for that whole span.
// amix's default normalize therefore divides by exactly N throughout;
@@ -1089,7 +1093,7 @@ async function mixGroupMembers(
}
if (signal?.aborted || !result.success) {
const failure = ffmpegFailure("mix", result);
const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);
return { success: false, error: failure.detail, failure };
}
return { success: true, degradedAutomation };
@@ -1127,7 +1131,11 @@ async function mixThroughFloatSum(
timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
});
if (!encoded.success) {
const failure = ffmpegFailure("mix", encoded);
const failure = ffmpegFailure(
"mix",
encoded,
config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
);
return {
success: false,
outputPath,