ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 09:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(producer): follow variable audio duration (#3647)

Browse Source
This commit is contained in:
Miguel Ángel
2026-09-04 23:15:33 +00:00
committed by GitHub
parent 78a28b8942
commit 2ca493094e
5 changed files with 182 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/producer/src/services/htmlCompiler.naturalDuration.test.ts
+15
View File
@@ -79,6 +79,21 @@ describe("compileForRender natural media duration parity", () => {
}
});
it("marks only variable-bound media whose natural duration was inferred", async () => {
const { document } = await compile(`
<audio id="inferred" src="ten-seconds.wav" data-var-src="track"></audio>
<audio id="authored" src="ten-seconds.wav" data-var-src="track" data-duration="4"></audio>
<audio id="plain" src="ten-seconds.wav"></audio>`);
expect(document.getElementById("inferred")?.hasAttribute("data-hf-inferred-duration")).toBe(
true,
);
expect(document.getElementById("authored")?.hasAttribute("data-hf-inferred-duration")).toBe(
false,
);
expect(document.getElementById("plain")?.hasAttribute("data-hf-inferred-duration")).toBe(false);
});
it("uses shared playback-start precedence and fallback semantics", async () => {
const cases = [
["precedence", 'data-playback-start="2" data-media-start="7"', 8],
packages/producer/src/services/htmlCompiler.test.ts
+20
View File
@@ -34,12 +34,17 @@ describe("discoverMediaFromBrowser", () => {
html: string,
currentSrcById: Record<string, string>,
serializeCallback = false,
intrinsicDurationById: Record<string, number> = {},
) {
const { document } = parseHTML(html);
for (const [id, currentSrc] of Object.entries(currentSrcById)) {
const element = document.getElementById(id);
if (element) Object.defineProperty(element, "currentSrc", { value: currentSrc });
}
for (const [id, duration] of Object.entries(intrinsicDurationById)) {
const element = document.getElementById(id);
if (element) Object.defineProperty(element, "duration", { value: duration });
}
const previousDocument = Reflect.get(globalThis, "document");
Reflect.set(globalThis, "document", document);
try {
@@ -82,6 +87,21 @@ describe("discoverMediaFromBrowser", () => {
expect(media[0]).toMatchObject({ id: "hf-img-1", tagName: "image" });
});
it("uses intrinsic duration only for variable media with an inferred duration", async () => {
const media = await discover(
`<audio id="inferred" src="fallback.wav" data-start="0" data-duration="3" data-end="3" data-var-src="track" data-hf-inferred-duration></audio>
<audio id="authored" src="fallback.wav" data-start="0" data-duration="3" data-end="3" data-var-src="track"></audio>`,
{ inferred: "selected.wav", authored: "selected.wav" },
false,
{ inferred: 6.530612, authored: 6.530612 },
);
expect(media.find((item) => item.id === "inferred")?.duration).toBe(6.530612);
expect(media.find((item) => item.id === "inferred")?.durationInferred).toBe(true);
expect(media.find((item) => item.id === "authored")?.duration).toBe(3);
expect(media.find((item) => item.id === "authored")?.durationInferred).toBe(false);
});
it("discovers the owning image for a variable-bound picture source", async () => {
const media = await discover(
`<picture>
packages/producer/src/services/htmlCompiler.ts
+47 -12
View File
@@ -104,6 +104,8 @@ export interface CompiledComposition {
hasAncestorBackgroundImage: boolean;
}
const INFERRED_MEDIA_DURATION_ATTR = "data-hf-inferred-duration";
/** Adapts linkedom's `parseHTML` to the `checkSubCompositionUsability` contract. */
function parseSubCompHtmlForValidity(html: string): ParsableDocumentLike {
return parseHTML(html).document as unknown as ParsableDocumentLike;
@@ -534,6 +536,23 @@ async function resolveMediaDuration(
* Compile a single HTML file: static pass + ffprobe for unresolved media.
* Returns compiled HTML and any unresolved composition elements that need browser resolution.
*/
function markInferredVariableMediaDurations(
html: string,
unresolvedMedia: readonly UnresolvedElement[],
): string {
const unresolvedIds = new Set(unresolvedMedia.map((element) => element.id));
if (unresolvedIds.size === 0) return html;
const { document } = parseHTML(html);
let changed = false;
for (const element of document.querySelectorAll("video[data-var-src], audio[data-var-src]")) {
if (!unresolvedIds.has(element.id)) continue;
element.setAttribute(INFERRED_MEDIA_DURATION_ATTR, "");
changed = true;
}
return changed ? document.toString() : html;
}
async function compileHtmlFile(
html: string,
baseDir: string,
@@ -567,8 +586,9 @@ async function compileHtmlFile(
(r): r is ResolvedDuration => r.duration != null && Number.isFinite(r.duration),
);
const markedStaticHtml = markInferredVariableMediaDurations(staticCompiled, mediaUnresolved);
let compiledHtml =
resolutions.length > 0 ? injectDurations(staticCompiled, resolutions) : staticCompiled;
resolutions.length > 0 ? injectDurations(markedStaticHtml, resolutions) : markedStaticHtml;
// Phase 2: Bound authored audio to playable source (parallel ffprobe).
// Explicit video slots may outlive their source and hold the final frame.
@@ -2101,6 +2121,8 @@ export interface BrowserMediaElement {
start: number;
end: number;
duration: number;
/** True when compilation inferred duration from the fallback source. */
durationInferred: boolean;
mediaStart: number;
loop: boolean;
hasAudio: boolean;
@@ -2123,6 +2145,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
start: number;
endRaw: string | null;
durationRaw: string | null;
intrinsicDuration: number;
durationInferred: boolean;
playbackStartRaw: string | null;
mediaStartRaw: string | null;
loop: boolean;
@@ -2169,6 +2193,10 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
const start = parseFloat(htmlEl.getAttribute("data-start") || "0");
const endRaw = htmlEl.getAttribute("data-end");
const durationRaw = htmlEl.getAttribute("data-duration");
const durationInferred = htmlEl.hasAttribute("data-hf-inferred-duration");
const intrinsicDuration = isImage
? 0
: (htmlEl as HTMLVideoElement | HTMLAudioElement).duration;
const playbackStartRaw = htmlEl.getAttribute("data-playback-start");
const mediaStartRaw = htmlEl.getAttribute("data-media-start");
const loop = htmlEl.hasAttribute("loop");
@@ -2185,6 +2213,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
start,
endRaw,
durationRaw,
intrinsicDuration,
durationInferred,
playbackStartRaw,
mediaStartRaw,
loop,
@@ -2197,18 +2227,23 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
return results;
});
return elements.map(({ endRaw, durationRaw, playbackStartRaw, mediaStartRaw, ...element }) => ({
...element,
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
duration: parseStrictFiniteTimingNumber(durationRaw) ?? 0,
mediaStart: readMediaStart({
getAttribute(name: string) {
if (name === "data-playback-start") return playbackStartRaw;
if (name === "data-media-start") return mediaStartRaw;
return null;
},
return elements.map(
({ endRaw, durationRaw, intrinsicDuration, playbackStartRaw, mediaStartRaw, ...element }) => ({
...element,
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
duration:
element.durationInferred && Number.isFinite(intrinsicDuration) && intrinsicDuration > 0
? intrinsicDuration
: (parseStrictFiniteTimingNumber(durationRaw) ?? 0),
mediaStart: readMediaStart({
getAttribute(name: string) {
if (name === "data-playback-start") return playbackStartRaw;
if (name === "data-media-start") return mediaStartRaw;
return null;
},
}),
}),
}));
);
}
export async function discoverAudioVolumeAutomationFromTimeline(
packages/producer/src/services/render/stages/probeStage.test.ts
+73
View File
@@ -444,6 +444,79 @@ describe("runProbeStage — forceScreenshot threading", () => {
expect(mediaPreflightComposition).toBe(input.composition);
});
it("uses selected intrinsic duration only when the variable-bound duration was inferred", async () => {
resetRetryMocks();
const media = (
id: string,
duration: number,
durationInferred: boolean,
): Record<string, unknown> => ({
id,
tagName: "audio",
src: `${id}-selected.wav`,
start: 0,
end: duration,
duration,
durationInferred,
mediaStart: 0,
loop: false,
hasAudio: true,
volume: 1,
muted: false,
});
browserMediaResults = [
media("longer-inferred", 6.530612, true),
media("longer-authored", 6.530612, false),
media("shorter-inferred", 3.836939, true),
];
const { runProbeStage } = await import("./probeStage.js");
const input = makeProbeInput({});
input.composition.duration = 7;
input.composition.audios.push(
{
id: "longer-inferred",
src: "short.wav",
start: 0,
end: 3.836939,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
{
id: "longer-authored",
src: "short.wav",
start: 0,
end: 3.836939,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
{
id: "shorter-inferred",
src: "long.wav",
start: 0,
end: 6.530612,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
);
input.compiled.html = `
<audio id="longer-inferred" src="short.wav" data-var-src="a" data-hf-inferred-duration></audio>
<audio id="longer-authored" src="short.wav" data-var-src="b" data-duration="3.836939"></audio>
<audio id="shorter-inferred" src="long.wav" data-var-src="c" data-hf-inferred-duration></audio>`;
input.job.config.variables = { a: "a.wav", b: "b.wav", c: "c.wav" };
await runProbeStage(input);
expect(input.composition.audios.map((audio) => audio.end)).toEqual([
6.530612, 3.836939, 3.836939,
]);
});
it("passes cancellation through and closes probe-owned resources when preflight rejects", async () => {
resetRetryMocks();
mediaPreflightError = new Error("ASSET_MEDIA_TYPE_MISMATCH");
packages/producer/src/services/render/stages/probeStage.ts
+27 -10
View File
@@ -176,6 +176,17 @@ export function hasVariableBoundMedia(
});
}
function reconcileBrowserMediaEnd(
existingEnd: number,
projectedEnd: number,
sourceChanged: boolean,
durationInferred: boolean,
): number {
if (projectedEnd <= 0) return existingEnd;
if (sourceChanged && durationInferred) return projectedEnd;
return existingEnd <= 0 ? projectedEnd : Math.min(existingEnd, projectedEnd);
}
/**
* Runtime-created media does not exist when the static compiler scans the HTML.
* Launch a browser probe so discoverMediaFromBrowser can reconcile it before
@@ -500,7 +511,8 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
// Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
const existing = composition.videos.find((v) => v.id === el.id);
if (existing) {
if (existing.src !== src) {
const sourceChanged = existing.src !== src;
if (sourceChanged) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
@@ -508,10 +520,12 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
el.start,
resolveBrowserMediaEnd(el.start, el.end, el.duration),
);
if (projectedEnd > 0) {
existing.end =
existing.end <= 0 ? projectedEnd : Math.min(existing.end, projectedEnd);
}
existing.end = reconcileBrowserMediaEnd(
existing.end,
projectedEnd,
sourceChanged,
el.durationInferred,
);
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||
@@ -544,7 +558,8 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
if (existingAudioIds.has(el.id)) {
const existing = composition.audios.find((a) => a.id === el.id);
if (existing) {
if (existing.src !== src) {
const sourceChanged = existing.src !== src;
if (sourceChanged) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
@@ -552,10 +567,12 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
el.start,
resolveBrowserMediaEnd(el.start, el.end, el.duration),
);
if (projectedEnd > 0) {
existing.end =
existing.end <= 0 ? projectedEnd : Math.min(existing.end, projectedEnd);
}
existing.end = reconcileBrowserMediaEnd(
existing.end,
projectedEnd,
sourceChanged,
el.durationInferred,
);
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||