ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-01 19:42:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: offset nested template video timing (#2859)

Browse Source 
* fix: offset nested template video timing

* test(producer): cover nested sequential video render

* fix: share canonical nested media timing
This commit is contained in:
Miguel Ángel
2026-07-29 03:36:04 +02:00
committed by GitHub
parent 4f344c50b0
commit 9bbb6d50a0
13 changed files with 385 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/src/commands/snapshot.test.ts
+30
View File
@@ -5,6 +5,7 @@ import {
formatSnapshotTimestamp,
parseZoomScale,
requireSnapshotFfmpeg,
resolveSnapshotVideoClipStart,
resolveSnapshotVideoFrameTime,
tailFrameTime,
} from "./snapshot.js";
@@ -154,6 +155,35 @@ describe("resolveSnapshotVideoFrameTime", () => {
});
});
describe("resolveSnapshotVideoClipStart", () => {
it("offsets a scene-local video start by its later template host", () => {
expect(
resolveSnapshotVideoClipStart({
authoredStart: 0,
runtimeResolvedStart: 3,
}),
).toBe(3);
});
it("uses the runtime's recursively resolved start for deeply nested media", () => {
expect(
resolveSnapshotVideoClipStart({
authoredStart: 1,
runtimeResolvedStart: 8,
}),
).toBe(8);
});
it("keeps authored starts as a compatibility fallback", () => {
expect(
resolveSnapshotVideoClipStart({
authoredStart: 3,
runtimeResolvedStart: null,
}),
).toBe(3);
});
});
describe("computeSnapshotTimes (FINDING [7]: tail is always captured)", () => {
it("default frames: last point is the readable tail, never exact duration", () => {
const { times, appendedTail } = computeSnapshotTimes(8, { frames: 5 });
packages/cli/src/commands/snapshot.ts
+39 -13
View File
@@ -93,6 +93,15 @@ export function resolveSnapshotVideoFrameTime(input: {
return Math.max(0, Math.min(relativeTime, sourceEnd - 1 / 30));
}
/** Prefer the runtime's canonical absolute media start. The authored value is
* only a compatibility fallback for pages built with an older runtime. */
export function resolveSnapshotVideoClipStart(input: {
authoredStart: number;
runtimeResolvedStart: number | null;
}): number {
return input.runtimeResolvedStart ?? input.authoredStart;
}
export function requireSnapshotFfmpeg(ffmpegPath: string | undefined): string {
if (ffmpegPath) return ffmpegPath;
throw new Error(
@@ -399,10 +408,14 @@ async function captureSnapshots(
if (cameraExpr) await page.evaluate(cameraExpr);
if (injectVideoFramesBatch && syncVideoFrameVisibility) {
const candidates = await page.evaluate((t: number) => {
return Array.from(document.querySelectorAll("video[data-start]")).map((el) => {
const candidates = await page.evaluate(() => {
const runtimeWindow = window as Window & {
__hfResolveMediaStartSeconds?: (element: Element) => number;
};
return Array.from(document.querySelectorAll("video")).map((el) => {
const v = el as HTMLVideoElement;
const start = parseFloat(v.dataset.start ?? "0") || 0;
const authoredStart = parseFloat(v.dataset.start ?? "0") || 0;
const runtimeResolvedStart = runtimeWindow.__hfResolveMediaStartSeconds?.(v);
const rawRate = v.defaultPlaybackRate;
const playbackRate =
Number.isFinite(rawRate) && rawRate > 0 ? Math.max(0.1, Math.min(5, rawRate)) : 1;
@@ -416,30 +429,43 @@ async function captureSnapshots(
: srcDur > 0
? Math.max(0, (srcDur - mediaStart) / playbackRate)
: Number.POSITIVE_INFINITY;
let relTime = (t - start) * playbackRate + mediaStart;
if (v.loop && srcDur > mediaStart && relTime >= srcDur) {
relTime = mediaStart + ((relTime - mediaStart) % (srcDur - mediaStart));
}
return {
id: v.id,
src: v.currentSrc || v.src,
start,
authoredStart,
runtimeResolvedStart:
runtimeResolvedStart !== undefined && Number.isFinite(runtimeResolvedStart)
? runtimeResolvedStart
: null,
duration,
srcDuration: srcDur,
relTime,
playbackRate,
mediaStart,
loop: v.loop,
};
});
}, time);
});
const active = candidates.flatMap((candidate) => {
const start = resolveSnapshotVideoClipStart(candidate);
let relTime = (time - start) * candidate.playbackRate + candidate.mediaStart;
if (
candidate.loop &&
candidate.srcDuration > candidate.mediaStart &&
relTime >= candidate.srcDuration
) {
relTime =
candidate.mediaStart +
((relTime - candidate.mediaStart) % (candidate.srcDuration - candidate.mediaStart));
}
if (!candidate.id || !candidate.src) return [];
const frameTime = resolveSnapshotVideoFrameTime({
globalTime: time,
clipStart: candidate.start,
clipStart: start,
clipDuration: candidate.duration,
relativeTime: candidate.relTime,
relativeTime: relTime,
sourceDuration: candidate.srcDuration,
});
return frameTime === null ? [] : [{ ...candidate, relTime: frameTime }];
return frameTime === null ? [] : [{ ...candidate, start, relTime: frameTime }];
});
const updates: Array<{ videoId: string; dataUri: string }> = [];
packages/core/src/runtime/init.test.ts
+127
View File
@@ -915,6 +915,130 @@ describe("initSandboxRuntimeModular", () => {
expect(video.currentTime).toBe(5);
});
it("keeps a scene-local video visible inside a later template-mounted host", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
root.setAttribute("data-start", "0");
root.setAttribute("data-duration", "6");
root.setAttribute("data-width", "360");
root.setAttribute("data-height", "640");
document.body.appendChild(root);
const firstHost = document.createElement("div");
firstHost.setAttribute("data-composition-id", "first");
firstHost.setAttribute("data-composition-file", "compositions/first.html");
firstHost.setAttribute("data-start", "0");
firstHost.setAttribute("data-duration", "3");
root.appendChild(firstHost);
const firstVideo = document.createElement("video");
firstVideo.setAttribute("data-start", "0");
firstVideo.setAttribute("data-duration", "3");
firstHost.appendChild(firstVideo);
const secondHost = document.createElement("div");
secondHost.setAttribute("data-composition-id", "second");
secondHost.setAttribute("data-composition-file", "compositions/second.html");
secondHost.setAttribute("data-start", "3");
secondHost.setAttribute("data-duration", "3");
root.appendChild(secondHost);
const secondVideo = document.createElement("video");
secondVideo.setAttribute("data-start", "0");
secondVideo.setAttribute("data-duration", "3");
secondHost.appendChild(secondVideo);
window.__timelines = {
main: createMockTimeline(6),
first: createMockTimeline(3),
second: createMockTimeline(3),
};
initSandboxRuntimeModular();
window.__player?.renderSeek(4);
expect(firstHost.style.visibility).toBe("hidden");
expect(firstVideo.style.visibility).toBe("hidden");
expect(secondHost.style.visibility).toBe("visible");
expect(secondVideo.style.visibility).toBe("visible");
});
it("resolves media starts through arbitrarily nested composition hosts", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
root.setAttribute("data-duration", "10");
document.body.appendChild(root);
const outerHost = document.createElement("div");
outerHost.setAttribute("data-composition-id", "outer");
outerHost.setAttribute("data-composition-file", "outer.html");
outerHost.setAttribute("data-start", "2");
outerHost.setAttribute("data-duration", "6");
root.appendChild(outerHost);
const innerHost = document.createElement("div");
innerHost.setAttribute("data-composition-id", "inner");
innerHost.setAttribute("data-composition-file", "inner.html");
innerHost.setAttribute("data-start", "3");
innerHost.setAttribute("data-duration", "3");
outerHost.appendChild(innerHost);
const video = document.createElement("video");
video.setAttribute("data-start", "1");
video.setAttribute("data-duration", "1");
innerHost.appendChild(video);
window.__timelines = {
main: createMockTimeline(10),
outer: createMockTimeline(6),
inner: createMockTimeline(3),
};
initSandboxRuntimeModular();
expect(window.__hfResolveMediaStartSeconds?.(video)).toBe(6);
window.__player?.renderSeek(5.5);
expect(video.style.visibility).toBe("hidden");
window.__player?.renderSeek(6.5);
expect(video.style.visibility).toBe("visible");
});
it("uses the canonical resolver for reference starts, auto-start media, and inline hosts", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
root.setAttribute("data-duration", "10");
document.body.appendChild(root);
const intro = document.createElement("section");
intro.id = "intro";
intro.setAttribute("data-start", "0");
intro.setAttribute("data-duration", "2");
root.appendChild(intro);
const inlineHost = document.createElement("div");
inlineHost.setAttribute("data-composition-id", "inline");
inlineHost.setAttribute("data-start", "intro + 1");
inlineHost.setAttribute("data-duration", "2");
root.appendChild(inlineHost);
const video = document.createElement("video");
video.setAttribute("data-hf-auto-start", "true");
video.setAttribute("data-duration", "2");
inlineHost.appendChild(video);
window.__timelines = {
main: createMockTimeline(10),
inline: createMockTimeline(2),
};
initSandboxRuntimeModular();
expect(window.__hfResolveMediaStartSeconds?.(video)).toBe(3);
});
it("updates visibility for timed elements inside nested compositions", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
@@ -1431,6 +1555,7 @@ describe("initSandboxRuntimeModular", () => {
const host = document.createElement("div");
host.setAttribute("data-composition-id", "scene-pip");
host.setAttribute("data-composition-file", "compositions/pip.html");
host.setAttribute("data-start", "45.40");
host.setAttribute("data-duration", "7.06");
root.appendChild(host);
@@ -1460,6 +1585,8 @@ describe("initSandboxRuntimeModular", () => {
initSandboxRuntimeModular();
expect(window.__hfResolveMediaStartSeconds?.(pipVideo)).toBeCloseTo(45.4);
const player = (
window as Window & {
__player?: { seek: (timeSeconds: number) => void };
packages/core/src/runtime/init.ts
+55 -35
View File
@@ -53,6 +53,7 @@ import type { PlayerAPI } from "../core.types";
import { swallow } from "./diagnostics";
import { shouldAttemptPeriodicTimelineBind } from "./timelineRebindPolicy";
import { installStudioCustomEase } from "./customEase";
import { parseNumeric } from "./startExpression";
const AUTHORED_DURATION_ATTR = "data-hf-authored-duration";
const AUTHORED_END_ATTR = "data-hf-authored-end";
@@ -606,32 +607,65 @@ export function initSandboxRuntimeModular(): void {
return resolver.resolveDurationForElement(element);
};
const resolveMediaStartSeconds = (element: Element, fallback = 0): number => {
if (!element.hasAttribute("data-hf-auto-start") && element.hasAttribute("data-start")) {
// `data-start` is authored relative to the media element's OWN sub-
// composition, not the root timeline — `fallback` carries the host
// composition's resolved absolute start (see syncMediaForCurrentState's
// inheritedStart), so it must be added, not discarded. Skipping it made
// a nested video play from root t=0 instead of holding until its
// parent scene began (issue #1838) — resolveStartForElement's own
// absolute-expression branch already adds this same host offset, this
// fast literal-value path just didn't.
const own = Math.max(0, Number(element.getAttribute("data-start") ?? 0) || 0);
return own + fallback;
}
return resolveStartForElement(element, fallback);
const resolveMediaCompositionContext = (element: Element) => {
const compositionRoot = element.closest("[data-composition-id]");
const inheritedStart = compositionRoot ? resolveStartForElement(compositionRoot, 0) : null;
const inheritedDuration = compositionRoot
? resolveDurationForElement(compositionRoot, { includeAuthoredTimingAttrs: true })
: null;
return { compositionRoot, inheritedStart, inheritedDuration };
};
const resolveAbsoluteMediaStartSeconds = (element: Element): number => {
const context = resolveMediaCompositionContext(element);
const inheritedStart = context.inheritedStart ?? 0;
const authoredStart = parseNumeric(element.getAttribute("data-start"));
if (
element.hasAttribute("data-hf-auto-start") ||
authoredStart == null ||
inheritedStart <= 0
) {
return resolveStartForElement(element, inheritedStart);
}
// Both timing conventions exist in shipped projects:
// - composition-local media, e.g. host@20 + video@0 => root@20
// - legacy root-global PIP media, e.g. host@45.4 + video@45.4 => root@45.4
// Preserve the global value when its authored window already intersects
// the host's absolute window. Otherwise it is unambiguously local and
// must inherit the recursively-resolved host start.
const authoredDuration = parseNumeric(element.getAttribute("data-duration"));
const hostDuration = context.inheritedDuration;
const hostEnd = hostDuration != null && hostDuration > 0 ? inheritedStart + hostDuration : null;
const authoredEnd =
authoredDuration != null && authoredDuration > 0
? authoredStart + authoredDuration
: authoredStart;
const overlapsHostWindow =
hostEnd == null
? authoredStart >= inheritedStart
: authoredStart < hostEnd &&
(authoredEnd > inheritedStart || authoredStart === inheritedStart);
return overlapsHostWindow ? authoredStart : inheritedStart + authoredStart;
};
window.__hfResolveMediaStartSeconds = resolveAbsoluteMediaStartSeconds;
runtimeCleanupCallbacks.push(() => {
if (window.__hfResolveMediaStartSeconds === resolveAbsoluteMediaStartSeconds) {
delete window.__hfResolveMediaStartSeconds;
}
});
const isTimedElementVisibleAt = (rawNode: HTMLElement, currentTime: number): boolean => {
const tag = rawNode.tagName.toLowerCase();
if (tag === "script" || tag === "style" || tag === "link" || tag === "meta") {
return false;
}
const start =
tag === "video" || tag === "audio"
? resolveMediaStartSeconds(rawNode, 0)
: resolveStartForElement(rawNode, 0);
const isMedia = tag === "video" || tag === "audio";
const start = isMedia
? resolveAbsoluteMediaStartSeconds(rawNode)
: resolveStartForElement(rawNode, 0);
let duration = resolveDurationForElement(rawNode);
const compId = rawNode.getAttribute("data-composition-id");
if (compId) {
@@ -730,7 +764,7 @@ export function initSandboxRuntimeModular(): void {
if (mediaNodes.length === 0) return null;
let maxWindowEndSeconds = 0;
for (const node of mediaNodes) {
const start = resolveMediaStartSeconds(node, 0);
const start = resolveAbsoluteMediaStartSeconds(node);
if (!Number.isFinite(start)) continue;
const duration = resolveMediaElementDurationSeconds(node);
if (duration == null || duration <= MIN_VALID_TIMELINE_DURATION_SECONDS) continue;
@@ -1945,30 +1979,16 @@ export function initSandboxRuntimeModular(): void {
};
const syncMediaForCurrentState = () => {
const resolveMediaCompositionContext = (element: HTMLVideoElement | HTMLAudioElement) => {
const compositionRoot = element.closest("[data-composition-id]");
const inheritedStart = compositionRoot ? resolveStartForElement(compositionRoot, 0) : null;
// Media sync follows the authored host window, matching visibility for
// authored composition hosts. Live child timeline duration only fills in
// when no authored timing exists, so seeks clamp against host clip timing.
const inheritedDuration = compositionRoot
? resolveDurationForElement(compositionRoot, { includeAuthoredTimingAttrs: true })
: null;
return { compositionRoot, inheritedStart, inheritedDuration };
};
const cache = refreshRuntimeMediaCache({
shouldIncludeElement: (element) =>
element.hasAttribute("data-start") ||
Boolean(resolveMediaCompositionContext(element).compositionRoot),
resolveStartSeconds: (element) => {
const context = resolveMediaCompositionContext(
element as HTMLVideoElement | HTMLAudioElement,
);
return resolveMediaStartSeconds(element, context.inheritedStart ?? 0);
return resolveAbsoluteMediaStartSeconds(element);
},
resolveDurationSeconds: (element) => {
const context = resolveMediaCompositionContext(element);
const start = resolveMediaStartSeconds(element, context.inheritedStart ?? 0);
const start = resolveAbsoluteMediaStartSeconds(element);
const mediaStart =
Number.parseFloat(element.dataset.playbackStart ?? element.dataset.mediaStart ?? "0") ||
0;
packages/core/src/runtime/window.d.ts
Vendored
+6
View File
@@ -72,6 +72,12 @@ declare global {
* freshly-injected `__render_frame__` images. See `forceDispatchSeekEvent`.
*/
__hfReseekGpu?: (time: number) => void;
/**
* Canonical root-timeline start for a media element. Snapshot capture uses
* this runtime-owned resolver so reference expressions, authored timing
* restoration, and arbitrary composition nesting cannot drift.
*/
__hfResolveMediaStartSeconds?: (element: Element) => number;
__HF_PICKER_API?: HyperframePickerApi;
gsap?: {
timeline: (params?: { paused?: boolean }) => RuntimeTimelineLike;
packages/producer/tests/nested-sequential-video-local-start/meta.json
+13
View File
@@ -0,0 +1,13 @@
{
"name": "Later nested template video stays visible",
"description": "Two template-mounted hosts play sequentially, and each template owns a scene-local video with data-start=0 that reads its half of one generated source. Before the fix, producer capture evaluated the later video's start against root time without its host offset, hid it for the entire second host window, and rendered black instead of the generated green half.",
"tags": ["video", "sub-composition", "regression"],
"minPsnr": 25,
"maxFrameFailures": 2,
"minAudioCorrelation": 0,
"maxAudioLagWindows": 1,
"renderConfig": {
"fps": 24,
"workers": 1
}
}
packages/producer/tests/nested-sequential-video-local-start/output/compiled.html
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:04e1e32b29828b8cbe7ba114ab66081eb5443359eed7e911cf0e2462044b7cd0
size 2291
packages/producer/tests/nested-sequential-video-local-start/output/output.mp4
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7e56506833244506267cbe6c8bdd971eb2f92fbf96a83ce18c7565c74ecbfcae
size 2829
packages/producer/tests/nested-sequential-video-local-start/src/compositions/first.html
+23
View File
@@ -0,0 +1,23 @@
<template id="first-template">
<div
data-composition-id="first-scene"
data-width="160"
data-height="90"
data-no-timeline
style="position: relative; width: 160px; height: 90px; overflow: hidden; background: #000"
>
<video
id="first-video"
class="clip"
data-start="0"
data-duration="1"
data-media-start="0"
data-track-index="0"
src="../media/source.mp4"
muted
playsinline
preload="auto"
style="display: block; width: 160px; height: 90px; object-fit: cover"
></video>
</div>
</template>
packages/producer/tests/nested-sequential-video-local-start/src/compositions/later.html
+23
View File
@@ -0,0 +1,23 @@
<template id="later-template">
<div
data-composition-id="later-scene"
data-width="160"
data-height="90"
data-no-timeline
style="position: relative; width: 160px; height: 90px; overflow: hidden; background: #000"
>
<video
id="later-video"
class="clip"
data-start="0"
data-duration="1"
data-media-start="1"
data-track-index="0"
src="../media/source.mp4"
muted
playsinline
preload="auto"
style="display: block; width: 160px; height: 90px; object-fit: cover"
></video>
</div>
</template>
packages/producer/tests/nested-sequential-video-local-start/src/index.html
+59
View File
@@ -0,0 +1,59 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=160, height=90" />
<style>
html,
body,
#root {
width: 160px;
height: 90px;
margin: 0;
overflow: hidden;
background: #000;
}
.scene {
position: absolute;
inset: 0;
background: #000;
}
</style>
</head>
<body>
<div
id="root"
data-composition-id="main"
data-start="0"
data-duration="2"
data-width="160"
data-height="90"
data-no-timeline
>
<div
id="first-host"
class="scene"
data-composition-id="first-scene"
data-composition-src="compositions/first.html"
data-start="0"
data-duration="1"
data-track-index="0"
data-no-timeline
></div>
<div
id="later-host"
class="scene"
data-composition-id="later-scene"
data-composition-src="compositions/later.html"
data-start="1"
data-duration="1"
data-track-index="1"
data-no-timeline
></div>
</div>
<script>
window.__timelines = window.__timelines || {};
</script>
</body>
</html>
packages/producer/tests/nested-sequential-video-local-start/src/media/source.mp4
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5b3552325f6a25ea3859be60a6bef131878066aca48d8409e4182f539d144e5f
size 2750
packages/producer/tests/shard-schedule.json
+1
View File
@@ -13,6 +13,7 @@
"distributedShardCount": 1,
"timings": {
"animejs-adapter": 17,
"nested-sequential-video-local-start": 8,
"audio-mux-parity": 32,
"chat": 54,
"css-spinner-render-compat": 15,