mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 06:30:03 +00:00
fix(producer): audio drops + blank images on FFmpeg 4.x/CORS-restricted origins (#1140)
* fix(engine): remove amix normalize=0 to fix audio on FFmpeg 4.x/6.x amix's normalize=0 option is absent from many FFmpeg builds (e.g. FFmpeg 4.2 on Ubuntu 20.04). When the option is not recognized, FFmpeg fails the entire filter graph initialization, processCompositionAudio returns success:false, and the assembled video has no audio stream. Replace normalize=0 + weights='1...' with the amix default behavior (normalize=true, divides by track count) and multiply the master output gain by the track count to restore the original per-track volumes. The net volume is identical across all FFmpeg versions. Fixes #1136-adjacent: reported as 'audio doesn't play' in rendered MP4. * fix(producer): strip img crossorigin + fix audioExtractor normalize=0 Two follow-up fixes: 1. htmlCompiler: strip crossorigin attribute from <img> elements during compilation. External images (e.g. S3) with crossorigin='anonymous' force CORS-mode requests against the renderer's localhost file server, which S3 rejects → images render blank. Matches the existing video strip at line 261. 2. audioExtractor: same amix normalize=0 bug as audioMixer.ts. The audioExtractor path is used for <video data-has-audio='true'> mixing in the CLI's local render pipeline; on FFmpeg 4.x it would also drop audio silently. Fix: remove normalize=0, compensate with volume=N. * test(engine,producer): pin amix normalize contract + img crossorigin strip - audioMixer.test.ts: assert filter has no normalize=/weights=; add 3-track test confirming compensatedGain = masterGain × N = 3 - htmlCompiler.test.ts: parallel tests for img and video crossorigin strip (covers both elements, not just video)
This commit is contained in:
@@ -63,6 +63,66 @@ describe("processCompositionAudio", () => {
|
|||||||
|
|
||||||
expect(filter).toContain("volume=0");
|
expect(filter).toContain("volume=0");
|
||||||
expect(filter).toContain("[mixed]volume=1[out]");
|
expect(filter).toContain("[mixed]volume=1[out]");
|
||||||
|
expect(filter).not.toContain("normalize=");
|
||||||
|
expect(filter).not.toContain("weights=");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("compensates amix normalization so multi-track master gain equals track count", async () => {
|
||||||
|
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
|
||||||
|
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
|
||||||
|
tempDirs.push(baseDir, workDir);
|
||||||
|
|
||||||
|
writeFileSync(join(baseDir, "a.wav"), "stub");
|
||||||
|
writeFileSync(join(baseDir, "b.wav"), "stub");
|
||||||
|
writeFileSync(join(baseDir, "c.wav"), "stub");
|
||||||
|
|
||||||
|
const result = await processCompositionAudio(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
id: "a",
|
||||||
|
src: "a.wav",
|
||||||
|
start: 0,
|
||||||
|
end: 2,
|
||||||
|
mediaStart: 0,
|
||||||
|
layer: 0,
|
||||||
|
volume: 0.8,
|
||||||
|
type: "audio",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "b",
|
||||||
|
src: "b.wav",
|
||||||
|
start: 0,
|
||||||
|
end: 2,
|
||||||
|
mediaStart: 0,
|
||||||
|
layer: 1,
|
||||||
|
volume: 1,
|
||||||
|
type: "audio",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "c",
|
||||||
|
src: "c.wav",
|
||||||
|
start: 0,
|
||||||
|
end: 2,
|
||||||
|
mediaStart: 0,
|
||||||
|
layer: 2,
|
||||||
|
volume: 0.5,
|
||||||
|
type: "audio",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
baseDir,
|
||||||
|
workDir,
|
||||||
|
join(baseDir, "out.m4a"),
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
|
||||||
|
const filter = mixArgs[mixArgs.indexOf("-filter_complex") + 1];
|
||||||
|
|
||||||
|
expect(filter).toContain("amix=inputs=3");
|
||||||
|
expect(filter).not.toContain("normalize=");
|
||||||
|
// masterOutputGain(1) × tracks(3) = 3
|
||||||
|
expect(filter).toContain("[mixed]volume=3[out]");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("uses frame-evaluated volume automation when keyframes are present", async () => {
|
it("uses frame-evaluated volume automation when keyframes are present", async () => {
|
||||||
|
|||||||
@@ -398,9 +398,11 @@ async function mixAudioTracks(
|
|||||||
});
|
});
|
||||||
|
|
||||||
const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
|
const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
|
||||||
const weights = tracks.map(() => "1").join(" ");
|
const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0[mixed]`;
|
||||||
const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`;
|
// amix divides output by inputs count (default normalize=true). Multiply master
|
||||||
const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`;
|
// gain by track count so per-track volumes authored in data-volume are preserved.
|
||||||
|
const compensatedGain = masterOutputGain * tracks.length;
|
||||||
|
const postMixGainFilter = `[mixed]volume=${formatFilterNumber(compensatedGain)}[out]`;
|
||||||
const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
|
const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
|||||||
@@ -211,8 +211,11 @@ async function mixTracks(
|
|||||||
});
|
});
|
||||||
|
|
||||||
const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
|
const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
|
||||||
const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:normalize=0[out]`;
|
// amix divides by track count by default (normalize=true). Compensate with
|
||||||
const fullFilter = [...filterParts, mixFilter].join(";");
|
// a volume gain to preserve per-track levels across all FFmpeg versions.
|
||||||
|
const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest[mixed]`;
|
||||||
|
const postMixGain = `[mixed]volume=${tracks.length}[out]`;
|
||||||
|
const fullFilter = [...filterParts, mixFilter, postMixGain].join(";");
|
||||||
|
|
||||||
const args = [
|
const args = [
|
||||||
...inputs,
|
...inputs,
|
||||||
|
|||||||
@@ -797,6 +797,51 @@ describe("text-rendering rule injection", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── crossorigin stripping ───────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// External images/videos with crossorigin="anonymous" force CORS-mode requests
|
||||||
|
// against the renderer's localhost file server. S3 and similar origins reject
|
||||||
|
// those requests, so the element renders blank. The strip removes the attribute
|
||||||
|
// so the browser falls back to no-cors (visual-only) mode.
|
||||||
|
|
||||||
|
describe("crossorigin attribute stripping", () => {
|
||||||
|
it("strips crossorigin from <img> elements", async () => {
|
||||||
|
const projectDir = mkdtempSync(join(tmpdir(), "hf-crossorigin-img-"));
|
||||||
|
writeFileSync(
|
||||||
|
join(projectDir, "index.html"),
|
||||||
|
`<!DOCTYPE html><html><body>
|
||||||
|
<div data-composition-id="root" data-width="640" data-height="360" data-duration="1">
|
||||||
|
<img id="hero" src="https://example.com/photo.jpg" crossorigin="anonymous" alt="" />
|
||||||
|
<img id="plain" src="local.jpg" alt="" />
|
||||||
|
</div>
|
||||||
|
</body></html>`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
|
||||||
|
|
||||||
|
expect(compiled.html).not.toContain('crossorigin="anonymous"');
|
||||||
|
expect(compiled.html).toContain('id="hero"');
|
||||||
|
expect(compiled.html).toContain('id="plain"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("strips crossorigin from <video> elements", async () => {
|
||||||
|
const projectDir = mkdtempSync(join(tmpdir(), "hf-crossorigin-video-"));
|
||||||
|
writeFileSync(
|
||||||
|
join(projectDir, "index.html"),
|
||||||
|
`<!DOCTYPE html><html><body>
|
||||||
|
<div data-composition-id="root" data-width="640" data-height="360" data-duration="1">
|
||||||
|
<video id="clip" src="https://example.com/clip.mp4" crossorigin="anonymous" data-start="0" data-duration="1"></video>
|
||||||
|
</div>
|
||||||
|
</body></html>`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
|
||||||
|
|
||||||
|
expect(compiled.html).not.toContain("crossorigin");
|
||||||
|
expect(compiled.html).toContain('id="clip"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("discoverAudioVolumeAutomationFromTimeline", () => {
|
describe("discoverAudioVolumeAutomationFromTimeline", () => {
|
||||||
it("samples video-derived audio volume without firing GSAP callbacks", async () => {
|
it("samples video-derived audio volume without firing GSAP callbacks", async () => {
|
||||||
class TestAudioElement {}
|
class TestAudioElement {}
|
||||||
|
|||||||
@@ -260,6 +260,12 @@ async function compileHtmlFile(
|
|||||||
// origins (e.g. S3 without CORS headers) keep readyState=0, blocking page setup.
|
// origins (e.g. S3 without CORS headers) keep readyState=0, blocking page setup.
|
||||||
compiledHtml = compiledHtml.replace(/(<video\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
|
compiledHtml = compiledHtml.replace(/(<video\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
|
||||||
|
|
||||||
|
// Strip crossorigin from img elements. The renderer captures DOM frames visually —
|
||||||
|
// no canvas readback — so CORS compliance is unnecessary. External images from
|
||||||
|
// CORS-restricted origins (e.g. S3) render blank when crossorigin forces a failed
|
||||||
|
// CORS request against the renderer's localhost file server.
|
||||||
|
compiledHtml = compiledHtml.replace(/(<img\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
|
||||||
|
|
||||||
return { html: compiledHtml, unresolvedCompositions };
|
return { html: compiledHtml, unresolvedCompositions };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user