From fc608ad7fce7647c11b986f5f579b5c83704c1da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Sun, 31 May 2026 09:55:49 -0400
Subject: [PATCH] fix(producer): audio drops + blank images on FFmpeg
 4.x/CORS-restricted origins (#1140)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(engine): remove amix normalize=0 to fix audio on FFmpeg 4.x/6.x

amix's normalize=0 option is absent from many FFmpeg builds (e.g.
FFmpeg 4.2 on Ubuntu 20.04). When the option is not recognized, FFmpeg
fails the entire filter graph initialization, processCompositionAudio
returns success:false, and the assembled video has no audio stream.

Replace normalize=0 + weights='1...' with the amix default behavior
(normalize=true, divides by track count) and multiply the master output
gain by the track count to restore the original per-track volumes.
The net volume is identical across all FFmpeg versions.

Fixes #1136-adjacent: reported as 'audio doesn't play' in rendered MP4.

* fix(producer): strip img crossorigin + fix audioExtractor normalize=0

Two follow-up fixes:

1. htmlCompiler: strip crossorigin attribute from <img> elements during
   compilation. External images (e.g. S3) with crossorigin='anonymous'
   force CORS-mode requests against the renderer's localhost file server,
   which S3 rejects → images render blank. Matches the existing video
   strip at line 261.

2. audioExtractor: same amix normalize=0 bug as audioMixer.ts. The
   audioExtractor path is used for <video data-has-audio='true'> mixing
   in the CLI's local render pipeline; on FFmpeg 4.x it would also drop
   audio silently. Fix: remove normalize=0, compensate with volume=N.

* test(engine,producer): pin amix normalize contract + img crossorigin strip

- audioMixer.test.ts: assert filter has no normalize=/weights=; add
  3-track test confirming compensatedGain = masterGain × N = 3
- htmlCompiler.test.ts: parallel tests for img and video crossorigin
  strip (covers both elements, not just video)
---
 .../engine/src/services/audioMixer.test.ts    | 60 +++++++++++++++++++
 packages/engine/src/services/audioMixer.ts    |  8 ++-
 .../producer/src/services/audioExtractor.ts   |  7 ++-
 .../src/services/htmlCompiler.test.ts         | 45 ++++++++++++++
 .../producer/src/services/htmlCompiler.ts     |  6 ++
 5 files changed, 121 insertions(+), 5 deletions(-)

diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index f60fc7a71..8305d6d44 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -63,6 +63,66 @@ describe("processCompositionAudio", () => {
 
     expect(filter).toContain("volume=0");
     expect(filter).toContain("[mixed]volume=1[out]");
+    expect(filter).not.toContain("normalize=");
+    expect(filter).not.toContain("weights=");
+  });
+
+  it("compensates amix normalization so multi-track master gain equals track count", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+
+    writeFileSync(join(baseDir, "a.wav"), "stub");
+    writeFileSync(join(baseDir, "b.wav"), "stub");
+    writeFileSync(join(baseDir, "c.wav"), "stub");
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "a",
+          src: "a.wav",
+          start: 0,
+          end: 2,
+          mediaStart: 0,
+          layer: 0,
+          volume: 0.8,
+          type: "audio",
+        },
+        {
+          id: "b",
+          src: "b.wav",
+          start: 0,
+          end: 2,
+          mediaStart: 0,
+          layer: 1,
+          volume: 1,
+          type: "audio",
+        },
+        {
+          id: "c",
+          src: "c.wav",
+          start: 0,
+          end: 2,
+          mediaStart: 0,
+          layer: 2,
+          volume: 0.5,
+          type: "audio",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      2,
+    );
+
+    expect(result.success).toBe(true);
+    const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
+    const filter = mixArgs[mixArgs.indexOf("-filter_complex") + 1];
+
+    expect(filter).toContain("amix=inputs=3");
+    expect(filter).not.toContain("normalize=");
+    // masterOutputGain(1) × tracks(3) = 3
+    expect(filter).toContain("[mixed]volume=3[out]");
   });
 
   it("uses frame-evaluated volume automation when keyframes are present", async () => {
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index bb31a7099..b47d6171c 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -398,9 +398,11 @@ async function mixAudioTracks(
     });
 
     const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
-    const weights = tracks.map(() => "1").join(" ");
-    const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`;
-    const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`;
+    const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0[mixed]`;
+    // amix divides output by inputs count (default normalize=true). Multiply master
+    // gain by track count so per-track volumes authored in data-volume are preserved.
+    const compensatedGain = masterOutputGain * tracks.length;
+    const postMixGainFilter = `[mixed]volume=${formatFilterNumber(compensatedGain)}[out]`;
     const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
 
     return [
diff --git a/packages/producer/src/services/audioExtractor.ts b/packages/producer/src/services/audioExtractor.ts
index cffff0ddb..ec810d026 100644
--- a/packages/producer/src/services/audioExtractor.ts
+++ b/packages/producer/src/services/audioExtractor.ts
@@ -211,8 +211,11 @@ async function mixTracks(
   });
 
   const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
-  const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:normalize=0[out]`;
-  const fullFilter = [...filterParts, mixFilter].join(";");
+  // amix divides by track count by default (normalize=true). Compensate with
+  // a volume gain to preserve per-track levels across all FFmpeg versions.
+  const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest[mixed]`;
+  const postMixGain = `[mixed]volume=${tracks.length}[out]`;
+  const fullFilter = [...filterParts, mixFilter, postMixGain].join(";");
 
   const args = [
     ...inputs,
diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts
index c83d55048..8372e0c33 100644
--- a/packages/producer/src/services/htmlCompiler.test.ts
+++ b/packages/producer/src/services/htmlCompiler.test.ts
@@ -797,6 +797,51 @@ describe("text-rendering rule injection", () => {
   });
 });
 
+// ── crossorigin stripping ───────────────────────────────────────────────────
+//
+// External images/videos with crossorigin="anonymous" force CORS-mode requests
+// against the renderer's localhost file server. S3 and similar origins reject
+// those requests, so the element renders blank. The strip removes the attribute
+// so the browser falls back to no-cors (visual-only) mode.
+
+describe("crossorigin attribute stripping", () => {
+  it("strips crossorigin from <img> elements", async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), "hf-crossorigin-img-"));
+    writeFileSync(
+      join(projectDir, "index.html"),
+      `<!DOCTYPE html><html><body>
+  <div data-composition-id="root" data-width="640" data-height="360" data-duration="1">
+    <img id="hero" src="https://example.com/photo.jpg" crossorigin="anonymous" alt="" />
+    <img id="plain" src="local.jpg" alt="" />
+  </div>
+</body></html>`,
+    );
+
+    const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
+
+    expect(compiled.html).not.toContain('crossorigin="anonymous"');
+    expect(compiled.html).toContain('id="hero"');
+    expect(compiled.html).toContain('id="plain"');
+  });
+
+  it("strips crossorigin from <video> elements", async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), "hf-crossorigin-video-"));
+    writeFileSync(
+      join(projectDir, "index.html"),
+      `<!DOCTYPE html><html><body>
+  <div data-composition-id="root" data-width="640" data-height="360" data-duration="1">
+    <video id="clip" src="https://example.com/clip.mp4" crossorigin="anonymous" data-start="0" data-duration="1"></video>
+  </div>
+</body></html>`,
+    );
+
+    const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
+
+    expect(compiled.html).not.toContain("crossorigin");
+    expect(compiled.html).toContain('id="clip"');
+  });
+});
+
 describe("discoverAudioVolumeAutomationFromTimeline", () => {
   it("samples video-derived audio volume without firing GSAP callbacks", async () => {
     class TestAudioElement {}
diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts
index dfd77acba..065f468a9 100644
--- a/packages/producer/src/services/htmlCompiler.ts
+++ b/packages/producer/src/services/htmlCompiler.ts
@@ -260,6 +260,12 @@ async function compileHtmlFile(
   // origins (e.g. S3 without CORS headers) keep readyState=0, blocking page setup.
   compiledHtml = compiledHtml.replace(/(<video\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
 
+  // Strip crossorigin from img elements. The renderer captures DOM frames visually —
+  // no canvas readback — so CORS compliance is unnecessary. External images from
+  // CORS-restricted origins (e.g. S3) render blank when crossorigin forces a failed
+  // CORS request against the renderer's localhost file server.
+  compiledHtml = compiledHtml.replace(/(<img\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
+
   return { html: compiledHtml, unresolvedCompositions };
 }