fix(producer): audio drops + blank images on FFmpeg 4.x/CORS-restricted origins (#1140)

* fix(engine): remove amix normalize=0 to fix audio on FFmpeg 4.x/6.x

amix's normalize=0 option is absent from many FFmpeg builds (e.g.
FFmpeg 4.2 on Ubuntu 20.04). When the option is not recognized, FFmpeg
fails the entire filter graph initialization, processCompositionAudio
returns success:false, and the assembled video has no audio stream.

Replace normalize=0 + weights='1...' with the amix default behavior
(normalize=true, divides by track count) and multiply the master output
gain by the track count to restore the original per-track volumes.
The net volume is identical across all FFmpeg versions.

Fixes #1136-adjacent: reported as 'audio doesn't play' in rendered MP4.

* fix(producer): strip img crossorigin + fix audioExtractor normalize=0

Two follow-up fixes:

1. htmlCompiler: strip crossorigin attribute from <img> elements during
   compilation. External images (e.g. S3) with crossorigin='anonymous'
   force CORS-mode requests against the renderer's localhost file server,
   which S3 rejects → images render blank. Matches the existing video
   strip at line 261.

2. audioExtractor: same amix normalize=0 bug as audioMixer.ts. The
   audioExtractor path is used for <video data-has-audio='true'> mixing
   in the CLI's local render pipeline; on FFmpeg 4.x it would also drop
   audio silently. Fix: remove normalize=0, compensate with volume=N.

* test(engine,producer): pin amix normalize contract + img crossorigin strip

- audioMixer.test.ts: assert filter has no normalize=/weights=; add
  3-track test confirming compensatedGain = masterGain × N = 3
- htmlCompiler.test.ts: parallel tests for img and video crossorigin
  strip (covers both elements, not just video)
This commit is contained in:
Miguel Ángel
2026-05-31 09:55:49 -04:00
committed by GitHub
parent f8abff2e1c
commit fc608ad7fc
5 changed files with 121 additions and 5 deletions
@@ -63,6 +63,66 @@ describe("processCompositionAudio", () => {
expect(filter).toContain("volume=0"); expect(filter).toContain("volume=0");
expect(filter).toContain("[mixed]volume=1[out]"); expect(filter).toContain("[mixed]volume=1[out]");
expect(filter).not.toContain("normalize=");
expect(filter).not.toContain("weights=");
});
it("compensates amix normalization so multi-track master gain equals track count", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "a.wav"), "stub");
writeFileSync(join(baseDir, "b.wav"), "stub");
writeFileSync(join(baseDir, "c.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "a",
src: "a.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 0.8,
type: "audio",
},
{
id: "b",
src: "b.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 1,
volume: 1,
type: "audio",
},
{
id: "c",
src: "c.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 2,
volume: 0.5,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
const filter = mixArgs[mixArgs.indexOf("-filter_complex") + 1];
expect(filter).toContain("amix=inputs=3");
expect(filter).not.toContain("normalize=");
// masterOutputGain(1) × tracks(3) = 3
expect(filter).toContain("[mixed]volume=3[out]");
}); });
it("uses frame-evaluated volume automation when keyframes are present", async () => { it("uses frame-evaluated volume automation when keyframes are present", async () => {
+5 -3
View File
@@ -398,9 +398,11 @@ async function mixAudioTracks(
}); });
const mixInputs = tracks.map((_, i) => `[a${i}]`).join(""); const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
const weights = tracks.map(() => "1").join(" "); const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0[mixed]`;
const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`; // amix divides output by inputs count (default normalize=true). Multiply master
const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`; // gain by track count so per-track volumes authored in data-volume are preserved.
const compensatedGain = masterOutputGain * tracks.length;
const postMixGainFilter = `[mixed]volume=${formatFilterNumber(compensatedGain)}[out]`;
const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";"); const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
return [ return [
@@ -211,8 +211,11 @@ async function mixTracks(
}); });
const mixInputs = tracks.map((_, i) => `[a${i}]`).join(""); const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:normalize=0[out]`; // amix divides by track count by default (normalize=true). Compensate with
const fullFilter = [...filterParts, mixFilter].join(";"); // a volume gain to preserve per-track levels across all FFmpeg versions.
const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest[mixed]`;
const postMixGain = `[mixed]volume=${tracks.length}[out]`;
const fullFilter = [...filterParts, mixFilter, postMixGain].join(";");
const args = [ const args = [
...inputs, ...inputs,
@@ -797,6 +797,51 @@ describe("text-rendering rule injection", () => {
}); });
}); });
// ── crossorigin stripping ───────────────────────────────────────────────────
//
// External images/videos with crossorigin="anonymous" force CORS-mode requests
// against the renderer's localhost file server. S3 and similar origins reject
// those requests, so the element renders blank. The strip removes the attribute
// so the browser falls back to no-cors (visual-only) mode.
describe("crossorigin attribute stripping", () => {
it("strips crossorigin from <img> elements", async () => {
const projectDir = mkdtempSync(join(tmpdir(), "hf-crossorigin-img-"));
writeFileSync(
join(projectDir, "index.html"),
`<!DOCTYPE html><html><body>
<div data-composition-id="root" data-width="640" data-height="360" data-duration="1">
<img id="hero" src="https://example.com/photo.jpg" crossorigin="anonymous" alt="" />
<img id="plain" src="local.jpg" alt="" />
</div>
</body></html>`,
);
const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
expect(compiled.html).not.toContain('crossorigin="anonymous"');
expect(compiled.html).toContain('id="hero"');
expect(compiled.html).toContain('id="plain"');
});
it("strips crossorigin from <video> elements", async () => {
const projectDir = mkdtempSync(join(tmpdir(), "hf-crossorigin-video-"));
writeFileSync(
join(projectDir, "index.html"),
`<!DOCTYPE html><html><body>
<div data-composition-id="root" data-width="640" data-height="360" data-duration="1">
<video id="clip" src="https://example.com/clip.mp4" crossorigin="anonymous" data-start="0" data-duration="1"></video>
</div>
</body></html>`,
);
const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
expect(compiled.html).not.toContain("crossorigin");
expect(compiled.html).toContain('id="clip"');
});
});
describe("discoverAudioVolumeAutomationFromTimeline", () => { describe("discoverAudioVolumeAutomationFromTimeline", () => {
it("samples video-derived audio volume without firing GSAP callbacks", async () => { it("samples video-derived audio volume without firing GSAP callbacks", async () => {
class TestAudioElement {} class TestAudioElement {}
@@ -260,6 +260,12 @@ async function compileHtmlFile(
// origins (e.g. S3 without CORS headers) keep readyState=0, blocking page setup. // origins (e.g. S3 without CORS headers) keep readyState=0, blocking page setup.
compiledHtml = compiledHtml.replace(/(<video\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1"); compiledHtml = compiledHtml.replace(/(<video\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
// Strip crossorigin from img elements. The renderer captures DOM frames visually —
// no canvas readback — so CORS compliance is unnecessary. External images from
// CORS-restricted origins (e.g. S3) render blank when crossorigin forces a failed
// CORS request against the renderer's localhost file server.
compiledHtml = compiledHtml.replace(/(<img\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
return { html: compiledHtml, unresolvedCompositions }; return { html: compiledHtml, unresolvedCompositions };
} }