mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(producer): localize remote media sources + strip audio crossorigin (#1146)
* fix(producer): localize remote media sources + strip audio crossorigin Two bugs affecting compositions that use remote S3 URLs for video/audio. Bug 1 — Remote <video>/<audio> sources cause blank frames The renderer (Puppeteer) must buffer all video elements to readyState >= 2 before frame capture begins. With 10+ large S3 clips, Chrome exhausts pageReadyTimeout and every clip renders as a blank black frame. Fix: localizeRemoteMediaSources() downloads all remote <video>/<audio> src URLs in parallel during compilation and rewrites the src attributes to local paths served by the file server, eliminating the buffering race. Bug 2 — crossorigin on <audio> elements not stripped htmlCompiler.ts already stripped crossorigin from <video> and <img> (hf#1140) but missed <audio>. Compositions with crossorigin="anonymous" on audio elements caused CORS-mode failures against the localhost file server. Extended the strip to cover <audio>. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(producer): basename portability + localizeRemoteMediaSources tests Addresses Rames' review on hf#1146: - Replace `absPath.split('/').at(-1)` with `path.basename(absPath)`. On Windows, path.join emits backslash-separated paths; split('/') returns the whole path as a single element, producing a garbage relPath. path.basename delegates to the OS separator on the current platform. - Export `localizeRemoteMediaSources` for unit testing. Tests verify: - Successful download rewrites src to _remote_media/ path - Download failure preserves original URL without throwing - Duplicate src URL across two tags → single fetch call (dedup) - Local (non-HTTP) src paths are not rewritten - Both double-quoted and single-quoted src attributes are rewritten - basename extraction is correct on POSIX paths Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Miguel Ángel
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
3c7e2f3649
commit
7bbedc080a
@@ -10,6 +10,7 @@ import {
|
||||
detectShaderTransitionUsage,
|
||||
discoverAudioVolumeAutomationFromTimeline,
|
||||
inlineExternalScripts,
|
||||
localizeRemoteMediaSources,
|
||||
recompileWithResolutions,
|
||||
} from "./htmlCompiler.js";
|
||||
|
||||
@@ -840,6 +841,112 @@ describe("crossorigin attribute stripping", () => {
|
||||
expect(compiled.html).not.toContain("crossorigin");
|
||||
expect(compiled.html).toContain('id="clip"');
|
||||
});
|
||||
|
||||
it("strips crossorigin from <audio> elements", async () => {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), "hf-crossorigin-audio-"));
|
||||
writeFileSync(
|
||||
join(projectDir, "index.html"),
|
||||
`<!DOCTYPE html><html><body>
|
||||
<div data-composition-id="root" data-width="640" data-height="360" data-duration="5">
|
||||
<audio id="bgm" src="https://example.com/bgm.mp3" crossorigin="anonymous" data-start="0" data-duration="5" data-volume="0.8"></audio>
|
||||
</div>
|
||||
</body></html>`,
|
||||
);
|
||||
|
||||
const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
|
||||
|
||||
expect(compiled.html).not.toContain("crossorigin");
|
||||
expect(compiled.html).toContain('id="bgm"');
|
||||
});
|
||||
});
|
||||
|
||||
// ── remote media localization ────────────────────────────────────────────────
|
||||
//
|
||||
// Tests run on localizeRemoteMediaSources directly (exported for testing) to
|
||||
// avoid invoking ffprobe / the full compileForRender pipeline. fetch is patched
|
||||
// in-process for success cases; real 404s from example.com cover fallback.
|
||||
|
||||
describe("localizeRemoteMediaSources", () => {
|
||||
it("rewrites remote <video> src to _remote_media path when download succeeds", async () => {
|
||||
const orig = globalThis.fetch;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(globalThis as any).fetch = async () => new Response(new Uint8Array(100), { status: 200 });
|
||||
try {
|
||||
const dl = mkdtempSync(join(tmpdir(), "hf-dl-ok-"));
|
||||
const html = `<video id="v1" src="https://media-ok.example.com/a/clip.mp4" data-start="0" data-end="10" muted></video>`;
|
||||
const { html: result, remoteMediaAssets } = await localizeRemoteMediaSources(html, dl);
|
||||
expect(result).not.toContain("https://media-ok.example.com/");
|
||||
expect(result).toContain("_remote_media/");
|
||||
expect(remoteMediaAssets.size).toBe(1);
|
||||
} finally {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(globalThis as any).fetch = orig;
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves original URL on download failure without throwing", async () => {
|
||||
const dl = mkdtempSync(join(tmpdir(), "hf-dl-fail-"));
|
||||
const url = "https://example.com/will-404-localize-test.mp4";
|
||||
const html = `<video id="v1" src="${url}" data-start="0" data-end="10" muted></video>`;
|
||||
const { html: result, remoteMediaAssets } = await localizeRemoteMediaSources(html, dl);
|
||||
expect(result).toContain(url);
|
||||
expect(remoteMediaAssets.size).toBe(0);
|
||||
});
|
||||
|
||||
it("deduplicates: two tags with the same src URL → one download", async () => {
|
||||
const orig = globalThis.fetch;
|
||||
let fetchCount = 0;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(globalThis as any).fetch = async () => {
|
||||
fetchCount++;
|
||||
return new Response(new Uint8Array(100), { status: 200 });
|
||||
};
|
||||
try {
|
||||
const dl = mkdtempSync(join(tmpdir(), "hf-dl-dedup-"));
|
||||
const html = `<video id="v1" src="https://dedup.example.com/b/shared.mp4" data-start="0" data-end="10" muted></video>
|
||||
<video id="v2" src="https://dedup.example.com/b/shared.mp4" data-start="10" data-end="20" muted></video>`;
|
||||
await localizeRemoteMediaSources(html, dl);
|
||||
expect(fetchCount).toBe(1);
|
||||
} finally {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(globalThis as any).fetch = orig;
|
||||
}
|
||||
});
|
||||
|
||||
it("does not rewrite local (non-HTTP) src paths", async () => {
|
||||
const dl = mkdtempSync(join(tmpdir(), "hf-dl-local-"));
|
||||
const html = `<video id="v1" src="assets/local.mp4" data-start="0" data-end="10" muted></video>`;
|
||||
const { html: result, remoteMediaAssets } = await localizeRemoteMediaSources(html, dl);
|
||||
expect(result).toContain("assets/local.mp4");
|
||||
expect(result).not.toContain("_remote_media/");
|
||||
expect(remoteMediaAssets.size).toBe(0);
|
||||
});
|
||||
|
||||
it("rewrites src in both double-quoted and single-quoted attributes", async () => {
|
||||
const orig = globalThis.fetch;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(globalThis as any).fetch = async () => new Response(new Uint8Array(100), { status: 200 });
|
||||
try {
|
||||
const dl = mkdtempSync(join(tmpdir(), "hf-dl-quotes-"));
|
||||
const html = `<video id="v1" src="https://q.example.com/c/dq.mp4" data-start="0" data-end="10" muted></video>
|
||||
<audio id="a1" src='https://q.example.com/c/sq.mp3' data-start="0" data-end="10"></audio>`;
|
||||
const { html: result } = await localizeRemoteMediaSources(html, dl);
|
||||
expect(result).not.toContain("https://q.example.com/");
|
||||
expect(result.match(/_remote_media\//g)?.length).toBe(2);
|
||||
} finally {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(globalThis as any).fetch = orig;
|
||||
}
|
||||
});
|
||||
|
||||
it("uses path.basename for OS-portable filename extraction from downloaded path", () => {
|
||||
// Guards against the prior absPath.split('/').at(-1) pattern. On Windows
|
||||
// path.join uses `\` separators; splitting on `/` would return the entire
|
||||
// path as a single element, producing a garbage relPath. path.basename is
|
||||
// OS-aware and extracts the filename correctly on both platforms.
|
||||
const { basename: b } = require("node:path");
|
||||
expect(b("/tmp/_remote_media/download_abc123.mp4")).toBe("download_abc123.mp4");
|
||||
});
|
||||
});
|
||||
|
||||
describe("discoverAudioVolumeAutomationFromTimeline", () => {
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
*/
|
||||
|
||||
import { readFileSync, existsSync, mkdirSync } from "fs";
|
||||
import { join, dirname, resolve } from "path";
|
||||
import { join, dirname, resolve, basename } from "path";
|
||||
import { parseHTML } from "linkedom";
|
||||
import {
|
||||
compileTimingAttrs,
|
||||
@@ -266,6 +266,15 @@ async function compileHtmlFile(
|
||||
// CORS request against the renderer's localhost file server.
|
||||
compiledHtml = compiledHtml.replace(/(<img\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
|
||||
|
||||
// Strip crossorigin from audio elements. Audio is processed out-of-band via
|
||||
// FFmpeg; the browser's CORS policy for audio elements is irrelevant to
|
||||
// rendering. Leaving crossorigin="anonymous" causes the browser to issue a
|
||||
// CORS-mode preflight from localhost, which S3 buckets without explicit CORS
|
||||
// headers reject — leaving audio elements in a failed network state. The
|
||||
// FFmpeg audio path reads the src URL directly and is unaffected by browser
|
||||
// CORS, so stripping the attribute has no side effects.
|
||||
compiledHtml = compiledHtml.replace(/(<audio\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
|
||||
|
||||
return { html: compiledHtml, unresolvedCompositions };
|
||||
}
|
||||
|
||||
@@ -871,6 +880,91 @@ export function collectExternalAssets(
|
||||
};
|
||||
}
|
||||
|
||||
const REMOTE_MEDIA_SUBDIR = "_remote_media";
|
||||
// Match opening tags of <video> or <audio> elements that carry an HTTP(S) src.
|
||||
// Uses [^>]* to span attributes — safe for composition elements that won't
|
||||
// have `>` inside quoted attribute values (data-title etc.).
|
||||
const REMOTE_MEDIA_TAG_RE =
|
||||
/<(?:video|audio)\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi;
|
||||
|
||||
/**
|
||||
* Download any remote `src` URLs on `<video>` and `<audio>` elements into a
|
||||
* local subdirectory of `downloadDir`, rewrite the HTML src attributes to
|
||||
* relative paths, and return the updated HTML along with a map of
|
||||
* `{ relativePath → absoluteLocalPath }` for callers to add to `externalAssets`.
|
||||
*
|
||||
* Skips URLs that fail to download (warns and preserves the original URL so
|
||||
* the browser can still attempt the remote fetch as a fallback).
|
||||
*
|
||||
* Why: remote S3 sources require Chrome to buffer every video file over the
|
||||
* network before `readyState >= 2` (HAVE_CURRENT_DATA). With 10+ large clips
|
||||
* this reliably exhausts `pageReadyTimeout`, producing blank black frames for
|
||||
* every clip. Localising the sources before the file server starts eliminates
|
||||
* the race entirely and keeps the render hermetic.
|
||||
*/
|
||||
/** @internal exported for unit testing only */
|
||||
export async function localizeRemoteMediaSources(
|
||||
html: string,
|
||||
downloadDir: string,
|
||||
): Promise<{ html: string; remoteMediaAssets: Map<string, string> }> {
|
||||
const remoteDir = join(downloadDir, REMOTE_MEDIA_SUBDIR);
|
||||
|
||||
// Collect unique HTTP URLs from <video>/<audio> src attributes.
|
||||
const urlSet = new Set<string>();
|
||||
const re = new RegExp(REMOTE_MEDIA_TAG_RE.source, REMOTE_MEDIA_TAG_RE.flags);
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(html)) !== null) {
|
||||
if (m[1]) urlSet.add(m[1]);
|
||||
}
|
||||
|
||||
if (urlSet.size === 0) return { html, remoteMediaAssets: new Map() };
|
||||
|
||||
if (!existsSync(remoteDir)) mkdirSync(remoteDir, { recursive: true });
|
||||
|
||||
// Download all unique URLs in parallel; collect {url → localPath} for successes.
|
||||
const urlToLocal = new Map<string, string>();
|
||||
await Promise.all(
|
||||
[...urlSet].map(async (url) => {
|
||||
try {
|
||||
const localPath = await downloadToTemp(url, remoteDir);
|
||||
urlToLocal.set(url, localPath);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`[Compiler] Remote media download failed for ${url} — using original URL as fallback. ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
if (urlToLocal.size === 0) return { html, remoteMediaAssets: new Map() };
|
||||
|
||||
// Build externalAssets map: relative key → local abs path.
|
||||
// The relative key ("_remote_media/<file>") becomes the path under compiledDir
|
||||
// that writeCompiledArtifacts copies the file to and the file server exposes.
|
||||
const remoteMediaAssets = new Map<string, string>();
|
||||
const urlToRelPath = new Map<string, string>();
|
||||
for (const [url, absPath] of urlToLocal) {
|
||||
const relPath = `${REMOTE_MEDIA_SUBDIR}/${basename(absPath)}`;
|
||||
remoteMediaAssets.set(relPath, absPath);
|
||||
urlToRelPath.set(url, relPath);
|
||||
}
|
||||
|
||||
// Rewrite src attributes in HTML.
|
||||
let result = html;
|
||||
for (const [url, relPath] of urlToRelPath) {
|
||||
// Replace both quote styles; URLs are long enough to be unique without
|
||||
// anchoring to the surrounding attribute context.
|
||||
result = result.replaceAll(`"${url}"`, `"${relPath}"`).replaceAll(`'${url}'`, `'${relPath}'`);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[Compiler] Localized ${urlToLocal.size} remote media source(s) to ${REMOTE_MEDIA_SUBDIR}/`,
|
||||
);
|
||||
return { html: result, remoteMediaAssets };
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional behavior toggles for {@link compileForRender}. All fields are
|
||||
* additive; omitting `options` preserves the in-process renderer's defaults.
|
||||
@@ -908,6 +1002,7 @@ function rewriteUnresolvableGsapToCdn(html: string, projectDir: string): string
|
||||
* Compile an HTML composition project into a single self-contained HTML string
|
||||
* with all media metadata resolved.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function compileForRender(
|
||||
projectDir: string,
|
||||
htmlPath: string,
|
||||
@@ -981,13 +1076,26 @@ export async function compileForRender(
|
||||
'data-hf-studio-motion="',
|
||||
];
|
||||
const hasPositionEdits = HF_POSITION_ATTRS.some((attr) => htmlWithAssets.includes(attr));
|
||||
const html = hasPositionEdits
|
||||
const htmlWithPositionScript = hasPositionEdits
|
||||
? htmlWithAssets.replace(
|
||||
/<\/body>/i,
|
||||
`<script>${createStudioPositionSeekReapplyScript()}</script></body>`,
|
||||
)
|
||||
: htmlWithAssets;
|
||||
|
||||
// Download remote <video> and <audio> sources to compiledDir and rewrite the
|
||||
// src attributes so the renderer reads from localhost. Remote S3 URLs cause
|
||||
// Chrome to spend the entire pageReadyTimeout buffering 10+ large video files
|
||||
// over the network; any that don't reach readyState >= 2 in time render as
|
||||
// blank black frames. Localising them eliminates the race.
|
||||
const { html, remoteMediaAssets } = await localizeRemoteMediaSources(
|
||||
htmlWithPositionScript,
|
||||
downloadDir,
|
||||
);
|
||||
for (const [relPath, absPath] of remoteMediaAssets) {
|
||||
externalAssets.set(relPath, absPath);
|
||||
}
|
||||
|
||||
// Parse main HTML elements
|
||||
const mainVideos = parseVideoElements(html);
|
||||
const mainAudios = parseAudioElements(html);
|
||||
|
||||
Reference in New Issue
Block a user