From 8133d9346e312beedd0fdc28faf491d46d46cc87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 10 Jun 2026 16:19:59 -0400
Subject: [PATCH] fix(producer): don't mix audio from muted videos into the
 render (#1322)

* fix(producer): don't mix audio from muted videos into the render

The auto-detect audio block checked ext.metadata.hasAudio (file has
audio track) but not video.hasAudio (element declares itself audible).
A <video muted> whose source file contains audio leaked that audio
into the final render at full volume.

Add video.hasAudio guard so only audible elements contribute audio.

* test(producer): add unit tests for muted video audio guard

* fix: format
---
 .../render/stages/extractVideosStage.test.ts  | 83 +++++++++++++++++++
 .../render/stages/extractVideosStage.ts       | 50 ++++++-----
 2 files changed, 113 insertions(+), 20 deletions(-)
 create mode 100644 packages/producer/src/services/render/stages/extractVideosStage.test.ts

diff --git a/packages/producer/src/services/render/stages/extractVideosStage.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.test.ts
new file mode 100644
index 000000000..72028e41b
--- /dev/null
+++ b/packages/producer/src/services/render/stages/extractVideosStage.test.ts
@@ -0,0 +1,83 @@
+import { describe, expect, it } from "vitest";
+import { appendAutoDetectedVideoAudio } from "./extractVideosStage.js";
+import type { ExtractedFrames, VideoElement } from "@hyperframes/engine";
+
+function makeVideo(overrides: Partial<VideoElement> = {}): VideoElement {
+  return {
+    id: "v1",
+    src: "clip.mp4",
+    start: 0,
+    end: 5,
+    mediaStart: 0,
+    loop: false,
+    hasAudio: true,
+    ...overrides,
+  };
+}
+
+function makeExtracted(videoId: string, fileHasAudio: boolean): ExtractedFrames {
+  return {
+    videoId,
+    srcPath: "/tmp/clip.mp4",
+    outputDir: "/tmp/frames",
+    framePattern: "frame_%05d.jpg",
+    fps: 30,
+    totalFrames: 150,
+    framePaths: new Map(),
+    metadata: {
+      durationSeconds: 5,
+      width: 1920,
+      height: 1080,
+      fps: 30,
+      codec: "h264",
+      hasAudio: fileHasAudio,
+    },
+  } as ExtractedFrames;
+}
+
+describe("appendAutoDetectedVideoAudio", () => {
+  it("adds audio for an audible video whose file has an audio track", () => {
+    const composition = { videos: [makeVideo()], audios: [] as never[] };
+    appendAutoDetectedVideoAudio(composition, [makeExtracted("v1", true)]);
+    expect(composition.audios).toHaveLength(1);
+    expect(composition.audios[0]).toMatchObject({
+      id: "v1-audio",
+      src: "clip.mp4",
+    });
+  });
+
+  it("skips a muted video even when the source file has audio", () => {
+    const composition = {
+      videos: [makeVideo({ hasAudio: false })],
+      audios: [] as never[],
+    };
+    appendAutoDetectedVideoAudio(composition, [makeExtracted("v1", true)]);
+    expect(composition.audios).toHaveLength(0);
+  });
+
+  it("skips when the source file has no audio track", () => {
+    const composition = { videos: [makeVideo()], audios: [] as never[] };
+    appendAutoDetectedVideoAudio(composition, [makeExtracted("v1", false)]);
+    expect(composition.audios).toHaveLength(0);
+  });
+
+  it("does not duplicate audio for a src already in the mix", () => {
+    const composition = {
+      videos: [makeVideo()],
+      audios: [
+        {
+          id: "existing",
+          src: "clip.mp4",
+          start: 0,
+          end: 5,
+          mediaStart: 0,
+          layer: 0,
+          volume: 1,
+          type: "video" as const,
+        },
+      ],
+    };
+    appendAutoDetectedVideoAudio(composition, [makeExtracted("v1", true)]);
+    expect(composition.audios).toHaveLength(1);
+  });
+});
diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.ts
index d36e33a55..4aa3a52bd 100644
--- a/packages/producer/src/services/render/stages/extractVideosStage.ts
+++ b/packages/producer/src/services/render/stages/extractVideosStage.ts
@@ -33,6 +33,7 @@ import { isAbsolute, join } from "node:path";
 import {
   type CaptureVideoMetadataHint,
   type EngineConfig,
+  type ExtractedFrames,
   type FrameLookupTable,
   type HdrTransfer,
   type VideoColorSpace,
@@ -214,26 +215,7 @@ export async function runExtractVideosStage(
     );
     videoMetadataHints = collectVideoMetadataHints(extractionResult.extracted);
 
-    // Auto-detect audio from video files via ffprobe metadata
-    const existingAudioSrcs = new Set(composition.audios.map((a) => a.src));
-    for (const ext of extractionResult.extracted) {
-      if (ext.metadata.hasAudio) {
-        const video = composition.videos.find((v) => v.id === ext.videoId);
-        if (video && !existingAudioSrcs.has(video.src)) {
-          composition.audios.push({
-            id: `${video.id}-audio`,
-            src: video.src,
-            start: video.start,
-            end: video.end,
-            mediaStart: video.mediaStart,
-            layer: 0,
-            volume: 1.0,
-            type: "video",
-          });
-          existingAudioSrcs.add(video.src);
-        }
-      }
-    }
+    appendAutoDetectedVideoAudio(composition, extractionResult.extracted);
   }
   const videoExtractMs = Date.now() - stage2Start;
 
@@ -251,3 +233,31 @@ export async function runExtractVideosStage(
     videoExtractMs,
   };
 }
+
+/**
+ * Auto-detect audio from extracted video files (ffprobe metadata) and append
+ * to composition.audios. Both the file AND the element must declare audio —
+ * a muted <video> whose source contains audio should not leak into the render.
+ */
+export function appendAutoDetectedVideoAudio(
+  composition: Pick<CompositionMetadata, "videos" | "audios">,
+  extracted: ExtractedFrames[],
+): void {
+  const existingAudioSrcs = new Set(composition.audios.map((a) => a.src));
+  for (const ext of extracted) {
+    if (!ext.metadata.hasAudio) continue;
+    const video = composition.videos.find((v) => v.id === ext.videoId);
+    if (!video || !video.hasAudio || existingAudioSrcs.has(video.src)) continue;
+    composition.audios.push({
+      id: `${video.id}-audio`,
+      src: video.src,
+      start: video.start,
+      end: video.end,
+      mediaStart: video.mediaStart,
+      layer: 0,
+      volume: 1.0,
+      type: "video",
+    });
+    existingAudioSrcs.add(video.src);
+  }
+}