diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json
index c520a7e9da..472887c5aa 100644
--- a/packages/core/package-subpaths.json
+++ b/packages/core/package-subpaths.json
@@ -50,6 +50,12 @@
"types": "./dist/compositionContract.d.ts",
"environments": ["browser", "bun", "node"]
},
+ "./studio-preview-mark": {
+ "source": "./src/studioPreviewMark.ts",
+ "runtime": "./dist/studioPreviewMark.js",
+ "types": "./dist/studioPreviewMark.d.ts",
+ "environments": ["browser", "bun", "node"]
+ },
"./media-timing": {
"source": "./src/mediaTiming.ts",
"runtime": "./dist/mediaTiming.js",
diff --git a/packages/core/package.json b/packages/core/package.json
index fffe05154c..e8f255e63f 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -64,6 +64,12 @@
"import": "./src/compositionContract.ts",
"types": "./src/compositionContract.ts"
},
+ "./studio-preview-mark": {
+ "bun": "./src/studioPreviewMark.ts",
+ "node": "./dist/studioPreviewMark.js",
+ "import": "./src/studioPreviewMark.ts",
+ "types": "./src/studioPreviewMark.ts"
+ },
"./media-timing": {
"bun": "./src/mediaTiming.ts",
"node": "./dist/mediaTiming.js",
@@ -466,6 +472,10 @@
"import": "./dist/compositionContract.js",
"types": "./dist/compositionContract.d.ts"
},
+ "./studio-preview-mark": {
+ "import": "./dist/studioPreviewMark.js",
+ "types": "./dist/studioPreviewMark.d.ts"
+ },
"./media-timing": {
"import": "./dist/mediaTiming.js",
"types": "./dist/mediaTiming.d.ts"
diff --git a/packages/core/src/compiler/htmlDocument.test.ts b/packages/core/src/compiler/htmlDocument.test.ts
index c98b00b727..e9ae8a4730 100644
--- a/packages/core/src/compiler/htmlDocument.test.ts
+++ b/packages/core/src/compiler/htmlDocument.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import {
+ findStartTags,
injectScriptsAtHeadStart,
injectScriptsIntoHtml,
injectTagsAtHeadStart,
@@ -221,6 +222,26 @@ describe("htmlDocument helpers", () => {
});
});
+describe("findStartTags", () => {
+ const at = (html: string, name: string) =>
+ findStartTags(html, name).map((i) => html.slice(i, html.indexOf(">", i) + 1));
+
+ it("finds each start tag in any case, and none in comments, raw text or longer names", () => {
+ const html =
+ '' +
+ '
![]()
';
+ expect(at(html, "img")).toEqual(["
", "
"]);
+ });
+
+ it("leaves out template content, nested templates included", () => {
+ const html =
+ "![]()
![]()
![]()
" +
+ '![]()
![]()
';
+ expect(at(html, "img")).toEqual(["
", "
"]);
+ expect(at(html, "template")).toEqual(["", ""]);
+ });
+});
+
describe("injectTagsAtHeadStart on long adversarial input", () => {
const M = 2_000_000;
it.each([
diff --git a/packages/core/src/compiler/htmlDocument.ts b/packages/core/src/compiler/htmlDocument.ts
index dc60e57a74..f18c994dd8 100644
--- a/packages/core/src/compiler/htmlDocument.ts
+++ b/packages/core/src/compiler/htmlDocument.ts
@@ -210,19 +210,38 @@ const RAW_TEXT_TAGS = ["script", "style", "title", "textarea"] as const;
type DocumentTag = "/g;
-function findDocumentTag(html: string, tag: DocumentTag): number {
- const lowered = lowerAscii(html);
+function* markupStarts(lowered: string): Generator {
const unclosedRawText = new Set();
let cursor = 0;
while (cursor !== -1) {
const open = lowered.indexOf("<", cursor);
- if (open === -1) return -1;
- if (isTagAt(lowered, open, tag)) return open;
+ if (open === -1) return;
+ yield open;
cursor = skipMarkup(lowered, open, unclosedRawText);
}
+}
+
+function findDocumentTag(html: string, tag: DocumentTag): number {
+ const lowered = lowerAscii(html);
+ for (const open of markupStarts(lowered)) {
+ if (isTagAt(lowered, open, tag)) return open;
+ }
return -1;
}
+export function findStartTags(html: string, name: string): number[] {
+ const lowered = lowerAscii(html);
+ const token = `<${lowerAscii(name)}`;
+ const starts: number[] = [];
+ let templateDepth = 0;
+ for (const open of markupStarts(lowered)) {
+ if (templateDepth === 0 && isTagAt(lowered, open, token)) starts.push(open);
+ if (isTagAt(lowered, open, " 0 && isTagAt(lowered, open, " {
video.dispatchEvent(new Event("canplay"));
await expect(pending).resolves.toBeUndefined();
});
+
+ it("waits on a later scene's image in the full scan unless it is lazy", () => {
+ const doc = docWith(
+ '
' +
+ '

',
+ );
+ for (const image of doc.querySelectorAll("img")) {
+ Object.defineProperty(image, "complete", { value: false });
+ image.decode = vi.fn().mockResolvedValue(undefined);
+ }
+ const later = doc.querySelector("#later")!;
+ const lazy = doc.querySelector("#lazy")!;
+
+ expect(scanPendingCompositionAssets(doc, { scope: "all" }).pendingImages).toEqual([later]);
+ expect(scanPendingCompositionAssets(doc, { scope: "first-frame" }).pendingImages).toEqual([]);
+ expect(mediaReadinessInput(doc, new AbortController().signal, { scope: "all" })).not.toBeNull();
+ expect(later.decode).toHaveBeenCalled();
+ expect(lazy.decode).not.toHaveBeenCalled();
+ });
});
describe("computeReadinessInput", () => {
diff --git a/packages/core/src/compositionReadiness.ts b/packages/core/src/compositionReadiness.ts
index f5b22eb6da..3dfb9f97e9 100644
--- a/packages/core/src/compositionReadiness.ts
+++ b/packages/core/src/compositionReadiness.ts
@@ -107,8 +107,16 @@ export function scanPendingCompositionAssets(
.filter(isRealmHtmlMediaElement)
.filter((el) => shouldIncludeAsset(el, scope, resolver, runtimeWindow?.__timelines ?? {}))
.filter((el) => el.readyState < HAVE_FUTURE_DATA);
+ // A lazy image off the first frame loads only once shown, so no scope can wait on it.
const pendingImages = Array.from(doc.querySelectorAll("img"))
- .filter((img) => shouldIncludeAsset(img, scope, resolver, runtimeWindow?.__timelines ?? {}))
+ .filter((img) =>
+ shouldIncludeAsset(
+ img,
+ img.getAttribute("loading")?.toLowerCase() === "lazy" ? FIRST_FRAME_READINESS_SCOPE : scope,
+ resolver,
+ runtimeWindow?.__timelines ?? {},
+ ),
+ )
.filter((img) => !img.complete);
const fontsLoading = doc.fonts?.status === "loading";
return { pendingMedia, pendingImages, fontsLoading };
diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts
index 74cc2e98a0..726b4e82b9 100644
--- a/packages/core/src/runtime/entry.test.ts
+++ b/packages/core/src/runtime/entry.test.ts
@@ -1,6 +1,7 @@
// fallow-ignore-file code-duplication
import { afterEach, describe, expect, it, vi } from "vitest";
import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
+import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
import type { RuntimeTimelineLike } from "./types";
function pausedTimeline(duration: number): RuntimeTimelineLike {
@@ -52,9 +53,25 @@ async function evaluateRuntime(): Promise {
}
const visibility = (...els: HTMLElement[]) => els.map((el) => getComputedStyle(el).visibility);
+const imageSkipped = (...clips: HTMLElement[]) =>
+ clips.map((clip) => getComputedStyle(clip.querySelector("img")!).display === "none");
+const withImage = (clip: T): T => {
+ clip.appendChild(document.createElement("img"));
+ return clip;
+};
+// What Studio's preview route serves at the head start; render and player documents never carry it.
+const servePreview = () =>
+ document.head.appendChild(
+ Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
+ );
+const neverDecodes = (clip: HTMLElement) => {
+ withImage(clip).querySelector("img")!.decode = () => new Promise(() => {});
+ return clip;
+};
describe("runtime entry", () => {
afterEach(() => {
+ vi.useRealTimers();
window.__hfRuntimeTeardown?.();
document.head.innerHTML = "";
document.body.innerHTML = "";
@@ -73,10 +90,14 @@ describe("runtime entry", () => {
});
it("paints no timed clip, from script evaluation until the first visibility pass decides it", async () => {
+ servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const later = timed(root, "div", "5");
const poster = timed(root, "img", "0");
+ // Studio serves later scenes' images lazy; laid out, they would load before the first pass.
+ const plate = later.appendChild(document.createElement("img"));
+ plate.setAttribute("loading", "lazy");
// A composition script may write visibility inline before the runtime runs.
later.style.visibility = "visible";
// Readiness, which runs the first pass, waits while GSAP batches timelines.
@@ -86,6 +107,7 @@ describe("runtime entry", () => {
await evaluateRuntime();
expect(window.__player).toBeUndefined();
expect(visibility(current, later)).toEqual(["hidden", "hidden"]);
+ expect(getComputedStyle(plate).display).toBe("none");
delete (document as { readyState?: unknown }).readyState;
document.dispatchEvent(new Event("DOMContentLoaded"));
@@ -96,6 +118,178 @@ describe("runtime entry", () => {
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
expect(window.__renderReady).toBe(true);
expect(visibility(current, later, poster)).toEqual(["visible", "hidden", "visible"]);
+ expect(document.querySelector("style[data-hf-first-pass-hide]")).toBeNull();
+ });
+
+ it("gives a render document neither rule, no look-ahead marks and no held seeks", async () => {
+ const root = mountRoot();
+ const current = timed(root, "div", "0");
+ const later = neverDecodes(timed(root, "div", "5"));
+ const plate = later.querySelector("img")!;
+ plate.setAttribute("loading", "lazy");
+ window.__hfTimelinesBuilding = true;
+ Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" });
+
+ await evaluateRuntime();
+ // A setup-time measure of an authored lazy image must see it laid out.
+ expect(getComputedStyle(plate).display).not.toBe("none");
+ delete (document as { readyState?: unknown }).readyState;
+ document.dispatchEvent(new Event("DOMContentLoaded"));
+ window.__hfTimelinesBuilding = false;
+ window.dispatchEvent(new CustomEvent("hf-timelines-built"));
+ window.__player?.seek(3.5);
+ expect(imageSkipped(later)).toEqual([false]);
+ expect(document.querySelector("[data-hf-upcoming]")).toBeNull();
+ window.__player?.seek(5.5);
+ expect(visibility(current, later)).toEqual(["hidden", "visible"]);
+ });
+
+ it("keys preview mode on the preview meta alone, not on the GSAP fallback script captures keep", async () => {
+ document.head
+ .appendChild(document.createElement("script"))
+ .setAttribute("data-hf-gsap-fallback", "");
+ const root = mountRoot();
+ const current = timed(root, "div", "0");
+ const later = neverDecodes(timed(root, "div", "5"));
+ const plate = later.querySelector("img")!;
+ plate.setAttribute("loading", "lazy");
+ window.__hfTimelinesBuilding = true;
+ Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" });
+
+ await evaluateRuntime();
+ expect(getComputedStyle(plate).display).not.toBe("none");
+ delete (document as { readyState?: unknown }).readyState;
+ document.dispatchEvent(new Event("DOMContentLoaded"));
+ window.__hfTimelinesBuilding = false;
+ window.dispatchEvent(new CustomEvent("hf-timelines-built"));
+ expect(document.querySelector("style[data-hf-skip-hidden-images]")).toBeNull();
+ window.__player?.seek(3.5);
+ expect(imageSkipped(later)).toEqual([false]);
+ expect(document.querySelector("[data-hf-upcoming]")).toBeNull();
+ expect(window.__player?.seek(5.5)).toBeUndefined();
+ expect(visibility(current, later)).toEqual(["hidden", "visible"]);
+ });
+
+ it("skips the images of each hidden clip not due within the look-ahead, until something shows it", async () => {
+ servePreview();
+ const root = mountRoot();
+ const current = withImage(timed(root, "div", "0"));
+ const soon = withImage(timed(root, "div", "1.5"));
+ const later = withImage(timed(root, "div", "5"));
+ // Shorter than the look-ahead: due within it, though already over at its far end.
+ const brief = withImage(timed(root, "div", "5"));
+ brief.setAttribute("data-duration", "0.5");
+
+ await evaluateRuntime();
+ expect(imageSkipped(current, soon, later, brief)).toEqual([false, false, true, true]);
+ window.__player?.seek(3.6);
+ expect(imageSkipped(later, brief)).toEqual([false, false]);
+ window.__player?.seek(0);
+ // How Studio's layer reveal shows a hidden clip.
+ later.style.visibility = "visible";
+ expect(imageSkipped(later)).toEqual([false]);
+ });
+
+ it("holds a paused jump on the previous picture until the next scene's image decodes", async () => {
+ servePreview();
+ const root = mountRoot();
+ const current = timed(root, "div", "0");
+ const later = timed(root, "div", "5");
+ const plate = later.appendChild(document.createElement("img"));
+ plate.setAttribute("loading", "lazy");
+ plate.setAttribute(STUDIO_PREVIEW_LAZY_ATTR, "");
+ let decoded = () => {};
+ plate.decode = () => new Promise((resolve) => (decoded = resolve));
+ const authored = later.appendChild(document.createElement("img"));
+ authored.setAttribute("loading", "lazy");
+ authored.decode = () => Promise.resolve();
+
+ await evaluateRuntime();
+ expect(window.__player?.seek(0.5)).toBeUndefined();
+ const landed = window.__player?.seek(5.5);
+ expect(landed).toBeInstanceOf(Promise);
+ // Any frame painted now shows the previous scene, while the next one is unskipped so its image loads.
+ expect(visibility(current, later)).toEqual(["visible", "hidden"]);
+ expect(imageSkipped(later)).toEqual([false]);
+ expect(window.__player?.getTime()).toBe(5.5);
+ expect([plate, authored].map((img) => img.getAttribute("loading"))).toEqual(["eager", "lazy"]);
+ decoded();
+ await landed;
+ expect(visibility(current, later)).toEqual(["hidden", "visible"]);
+ await window.__hfWaitForSeekCompletion?.();
+ });
+
+ it("applies a held jump after the cap when an image never decodes", async () => {
+ servePreview();
+ const root = mountRoot();
+ const current = timed(root, "div", "0");
+ const later = timed(root, "div", "5");
+ later.appendChild(document.createElement("img")).decode = () => new Promise(() => {});
+
+ await evaluateRuntime();
+ const swallowed: string[] = [];
+ window.__hf = {
+ ...window.__hf,
+ onSwallowed: ({ label }: { label: string }) => swallowed.push(label),
+ };
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ window.__player?.seek(5.5);
+ await vi.advanceTimersByTimeAsync(999);
+ expect(visibility(current, later)).toEqual(["visible", "hidden"]);
+ await vi.advanceTimersByTimeAsync(1);
+ expect(visibility(current, later)).toEqual(["hidden", "visible"]);
+ expect(swallowed).toEqual(["runtime.init.seekHoldCap"]);
+ });
+
+ it("drops a held jump when a newer seek or a render seek lands first", async () => {
+ servePreview();
+ const root = mountRoot();
+ const current = timed(root, "div", "0");
+ const mid = timed(root, "div", "1");
+ const later = timed(root, "div", "5");
+ const plate = later.appendChild(document.createElement("img"));
+ let decoded = () => {};
+ plate.decode = () => new Promise((resolve) => (decoded = resolve));
+
+ await evaluateRuntime();
+ window.__player?.seek(5.5);
+ window.__player?.seek(1);
+ expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
+ decoded();
+ await window.__hfWaitForSeekCompletion?.();
+ expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
+ expect(window.__player?.getTime()).toBe(1);
+
+ window.__player?.seek(0);
+ window.__player?.seek(5.5);
+ window.__player?.renderSeek(1);
+ decoded();
+ await window.__hfWaitForSeekCompletion?.();
+ expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
+ });
+
+ it("never holds a render seek", async () => {
+ servePreview();
+ const root = mountRoot();
+ const current = timed(root, "div", "0");
+ const later = neverDecodes(timed(root, "div", "5"));
+
+ await evaluateRuntime();
+ window.__player?.renderSeek(5.5);
+ expect(visibility(current, later)).toEqual(["hidden", "visible"]);
+ });
+
+ it("shows the jump target at once when play is pressed during the hold", async () => {
+ servePreview();
+ const root = mountRoot();
+ const current = timed(root, "div", "0");
+ const later = timed(root, "div", "5");
+ later.appendChild(document.createElement("img")).decode = () => new Promise(() => {});
+
+ await evaluateRuntime();
+ window.__player?.seek(5.5);
+ window.__player?.play();
+ expect(visibility(current, later)).toEqual(["hidden", "visible"]);
});
it("leaves nothing hidden when the runtime is evaluated a second time", async () => {
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 159f473190..3f01cdf406 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -3,7 +3,8 @@ import { preloadMedia } from "./preloadMedia";
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
import { instantTolerance } from "../clipFacts";
import { isInClipWindow } from "./clipWindow";
-import { revealTimedClipsAfterFirstPass } from "./timedClipHide";
+import { revealTimedClipsAfterFirstPass, SKIPPED_CLIP, skipsHiddenImages } from "./timedClipHide";
+import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark";
import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";
import { injectCompositionCssVariables } from "./getVariables";
import { createCssAdapter } from "./adapters/css";
@@ -21,7 +22,11 @@ import {
patchVideoTextureCompat,
patchWebGLVideoTextureCompat,
} from "./adapters/video-texture-compat";
-import { forceDispatchSeekEvent, waitForSeekCompletion } from "./adapters/seek-dispatch";
+import {
+ forceDispatchSeekEvent,
+ registerSeekCompletion,
+ waitForSeekCompletion,
+} from "./adapters/seek-dispatch";
import { sourceTimeAt } from "../speedRamp";
import { createWaapiAdapter } from "./adapters/waapi";
import {
@@ -86,6 +91,7 @@ import { clampNativeMediaVolume } from "../audioGain";
import { quantizeSeekTime, quantizeTimeToFrame } from "../inline-scripts/parityContract";
import { createManualEditGestureWatch } from "./manualEditGestureWatch";
import type {
+ HeldSeek,
RuntimeDeterministicAdapter,
RuntimeJson,
RuntimeSeekOptions,
@@ -484,7 +490,7 @@ export function initSandboxRuntimeModular(): void {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
- seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => void;
+ seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
@@ -2528,18 +2534,55 @@ export function initSandboxRuntimeModular(): void {
return needsCapture;
};
+ const timedVisibilityAt = (timingRevision?: number) => {
+ const compositionDuration = getSafeTimelineDurationSeconds(
+ state.capturedTimeline,
+ 0,
+ timingRevision,
+ );
+ return (node: HTMLElement, time: number) =>
+ isRuntimeElementVisibleAt(node, {
+ currentTime: time,
+ compositionDuration,
+ canonicalFps: state.canonicalFps,
+ exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
+ timelineRegistry: window.__timelines ?? {},
+ resolver: timingResolverFor(true),
+ });
+ };
+
+ const clipChainVisibleAt = (
+ from: Element | null,
+ time: number,
+ visibleAt: ReturnType,
+ rootComp: HTMLElement | null,
+ ) => {
+ for (let node = from; node && node !== rootComp; node = node.parentElement)
+ if (isHtmlElement(node) && node.hasAttribute("data-start") && !visibleAt(node, time))
+ return false;
+ return true;
+ };
+
+ const hiddenImagesSkipped = skipsHiddenImages();
+ const LOOKAHEAD_SECONDS = 2;
+ // Unskipped while due in the look-ahead window, so a clip shorter than the window still loads first.
+ const dueSoon = (
+ node: HTMLElement,
+ visibleAt: ReturnType,
+ t: number,
+ ) => {
+ const start = resolveStartForElement(node, Number.NaN);
+ return (start > t && start <= t + LOOKAHEAD_SECONDS) || visibleAt(node, t + LOOKAHEAD_SECONDS);
+ };
+
const applyTimedElementVisibility = (
currentTime: number,
visibilityNodes: Element[],
timingRevision?: number,
) => {
const rootComp = resolveRootCompositionElement();
- const compositionDuration = getSafeTimelineDurationSeconds(
- state.capturedTimeline,
- 0,
- timingRevision,
- );
let decidedTimedClip = false;
+ const visibleAt = timedVisibilityAt(timingRevision);
for (const rawNode of visibilityNodes) {
if (!isHtmlElement(rawNode)) continue;
@@ -2561,40 +2604,17 @@ export function initSandboxRuntimeModular(): void {
if (nodeAffectsAudio(rawNode)) hiddenAudioDirty = true;
}
- let isVisibleNow = isRuntimeElementVisibleAt(rawNode, {
- currentTime,
- compositionDuration,
- canonicalFps: state.canonicalFps,
- exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
- timelineRegistry: window.__timelines ?? {},
- resolver: timingResolverFor(true),
- });
// Descendants must not override a hidden ancestor clip. CSS visibility can
// otherwise leak child pixels through inactive scenes because a descendant
// with visibility:visible escapes an ancestor's visibility:hidden.
- if (isVisibleNow) {
- let ancestor = rawNode.parentElement;
- while (ancestor) {
- if (ancestor === rootComp) break;
- if (isHtmlElement(ancestor) && ancestor.hasAttribute("data-start")) {
- if (
- !isRuntimeElementVisibleAt(ancestor, {
- currentTime,
- compositionDuration,
- canonicalFps: state.canonicalFps,
- exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
- timelineRegistry: window.__timelines ?? {},
- resolver: timingResolverFor(true),
- })
- ) {
- isVisibleNow = false;
- break;
- }
- }
- ancestor = ancestor.parentElement;
- }
- }
+ const isVisibleNow =
+ visibleAt(rawNode, currentTime) &&
+ clipChainVisibleAt(rawNode.parentElement, currentTime, visibleAt, rootComp);
rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";
+ rawNode.toggleAttribute(
+ STUDIO_PREVIEW_UPCOMING_ATTR,
+ hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime),
+ );
if (!isMediaElement(rawNode) && !isImageElement(rawNode)) decidedTimedClip = true;
if (isVideoElement(rawNode) || isImageElement(rawNode)) {
colorGradingRuntime?.setSourceVisibility(rawNode, isVisibleNow);
@@ -2634,6 +2654,20 @@ export function initSandboxRuntimeModular(): void {
applyTimedElementVisibility(currentTime, visibilityNodes, timingRevision),
);
+ // Images a seek to `time` would reveal undecoded: skipped with their hidden clip, or still loading.
+ const undecodedImagesShownAt = (time: number): HTMLImageElement[] =>
+ withTimingResolver(() => {
+ if (!hiddenImagesSkipped) return [];
+ const rootComp = resolveRootCompositionElement();
+ const visibleAt = timedVisibilityAt();
+ return Array.from(document.images).filter(
+ (img) =>
+ img.closest('[data-start][style*="visibility: hidden"]') !== null &&
+ (img.closest(SKIPPED_CLIP) !== null || !img.complete) &&
+ clipChainVisibleAt(img, time, visibleAt, rootComp),
+ );
+ });
+
/**
* Clip windows sorted by each endpoint, so a seek can ask which windows it
* crossed instead of asking every clip whether it is active.
@@ -3406,8 +3440,41 @@ export function initSandboxRuntimeModular(): void {
}
};
+ // A paused seek that would reveal undecoded images holds the previous picture until they decode.
+ // Capped: a request that never settles must not freeze the preview; measured jumps settle in ~200 ms.
+ const SEEK_HOLD_CAP_MS = 1000;
+ let heldSeek: { time: number; apply: () => void } | null = null;
+ const flushHeldSeek = () => {
+ const held = heldSeek;
+ heldSeek = null;
+ held?.apply();
+ };
+ const applySeek = (quantized: number, options?: { keepPlaying?: boolean }) => {
+ webAudio.stopAll();
+ clock.detachAudioSource();
+ const wasPlaying = clock.isPlaying();
+ if (wasPlaying) clock.pause();
+ clock.seek(quantized);
+ state.currentTime = clock.now();
+ state.isPlaying = false;
+ state.mediaForceSyncNextTick = true;
+ const tl = state.capturedTimeline;
+ pauseTimelineIfPossible(tl);
+ const pageAnimations = seekTimelineAndAdapters(state.currentTime);
+ runAdapters("pause", 0, pageAnimations);
+ if (options?.keepPlaying && wasPlaying) {
+ transport.play();
+ return;
+ }
+ syncMediaForCurrentState();
+ colorGrading.redraw();
+ paintVfx(state.currentTime);
+ postState(true);
+ };
+
const transport: RuntimePlayerTransport = {
play: () => {
+ flushHeldSeek();
const tl = state.capturedTimeline;
if (clock.isPlaying()) return;
const dur = getSafeTimelineDurationSeconds(tl, 0);
@@ -3460,28 +3527,38 @@ export function initSandboxRuntimeModular(): void {
Math.max(0, Number(timeSeconds) || 0),
state.canonicalFps,
);
- webAudio.stopAll();
- clock.detachAudioSource();
- const wasPlaying = clock.isPlaying();
- if (wasPlaying) clock.pause();
- clock.seek(quantized);
- state.currentTime = clock.now();
- state.isPlaying = false;
- state.mediaForceSyncNextTick = true;
- const tl = state.capturedTimeline;
- pauseTimelineIfPossible(tl);
- const pageAnimations = seekTimelineAndAdapters(state.currentTime);
- runAdapters("pause", 0, pageAnimations);
- if (options?.keepPlaying && wasPlaying) {
- transport.play();
+ heldSeek = null;
+ const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
+ if (undecoded.length === 0) {
+ applySeek(quantized, options);
return;
}
- syncMediaForCurrentState();
- colorGrading.redraw();
- paintVfx(state.currentTime);
- postState(true);
+ const held = { time: quantized, apply: () => applySeek(quantized, options) };
+ heldSeek = held;
+ for (const img of undecoded) {
+ if (img.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) img.setAttribute("loading", "eager");
+ for (let clip = img.closest(SKIPPED_CLIP); clip; clip = img.closest(SKIPPED_CLIP))
+ clip.setAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, "");
+ }
+ let capTimer = 0;
+ const capped = new Promise((resolve) => {
+ capTimer = window.setTimeout(() => {
+ if (heldSeek === held) swallow("runtime.init.seekHoldCap", undecoded);
+ resolve();
+ }, SEEK_HOLD_CAP_MS);
+ });
+ const decoded = Promise.all(
+ undecoded.map((img) => (img.decode ? img.decode().catch(() => {}) : undefined)),
+ );
+ const landed = Promise.race([decoded, capped]).then(() => {
+ window.clearTimeout(capTimer);
+ if (heldSeek === held) flushHeldSeek();
+ });
+ registerSeekCompletion(landed);
+ return landed;
},
renderSeek: (timeSeconds, options) => {
+ heldSeek = null;
renderCaptureSeekStarted = true;
const quantized = quantizeSeekTime(
Math.max(0, Number(timeSeconds) || 0),
@@ -3505,7 +3582,7 @@ export function initSandboxRuntimeModular(): void {
paintVfx(state.currentTime, { engineMode: true });
postState(true);
},
- getTime: () => clock.now(),
+ getTime: () => heldSeek?.time ?? clock.now(),
getDuration: () => {
const dur = clock.getDuration();
return Number.isFinite(dur) ? dur : 0;
diff --git a/packages/core/src/runtime/timedClipHide.ts b/packages/core/src/runtime/timedClipHide.ts
index b590ec57b6..38e14c35cf 100644
--- a/packages/core/src/runtime/timedClipHide.ts
+++ b/packages/core/src/runtime/timedClipHide.ts
@@ -1,26 +1,47 @@
+import { STUDIO_PREVIEW_MARK_META, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark";
+
// Hides timed non-media clips from script evaluation until the first visibility pass decides them;
// before it, a paused page painted every clip at once. Media is left to init's media pass.
// The rule and its flag live on the page, so every runtime copy shares them.
const HIDE_ATTR = "data-hf-first-pass-hide";
const HIDE_UNTIL_FIRST_PASS =
"[data-start]:not(video, audio, img) { visibility: hidden !important; }";
+const PREVIEW_HIDE_UNTIL_FIRST_PASS = 'img[loading="lazy"] { display: none !important; }';
+const SKIP_ATTR = "data-hf-skip-hidden-images";
+export const SKIPPED_CLIP = `[data-start]:not(video, audio, img, [${STUDIO_PREVIEW_UPCOMING_ATTR}])[style*="visibility: hidden"]`;
+const SKIP_HIDDEN_IMAGES = `${SKIPPED_CLIP} img { display: none !important; }`;
+const STUDIO_PREVIEW_MARK = `meta[name="${STUDIO_PREVIEW_MARK_META}"]`;
type FirstPassWindow = Window & {
__hfFirstPassHidden?: boolean;
__hyperframeRuntimeBootstrapped?: boolean;
};
+function appendStyle(parent: Element, attr: string, css: string): void {
+ const style = document.createElement("style");
+ style.setAttribute(attr, "");
+ style.textContent = css;
+ parent.appendChild(style);
+}
+
+/** True in Studio's preview, where hidden clips' images are neither fetched nor decoded until due. */
+export function skipsHiddenImages(): boolean {
+ return document.querySelector(`style[${SKIP_ATTR}]`) !== null;
+}
+
export function hideTimedClipsUntilFirstPass(): void {
if (typeof document === "undefined") return;
+ const parent = document.head ?? document.documentElement;
+ if (!parent) return;
+ const preview = document.querySelector(STUDIO_PREVIEW_MARK) !== null;
+ if (preview && !skipsHiddenImages()) appendStyle(parent, SKIP_ATTR, SKIP_HIDDEN_IMAGES);
const win = window as FirstPassWindow;
// A runtime that already initialised may never run another pass to lift a new rule.
if (win.__hfFirstPassHidden || win.__hyperframeRuntimeBootstrapped) return;
- const parent = document.head ?? document.documentElement;
- if (!parent) return;
- const style = document.createElement("style");
- style.setAttribute(HIDE_ATTR, "");
- style.textContent = HIDE_UNTIL_FIRST_PASS;
- parent.appendChild(style);
+ const css = preview
+ ? `${HIDE_UNTIL_FIRST_PASS} ${PREVIEW_HIDE_UNTIL_FIRST_PASS}`
+ : HIDE_UNTIL_FIRST_PASS;
+ appendStyle(parent, HIDE_ATTR, css);
win.__hfFirstPassHidden = true;
}
diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.ts
index 977cad94cd..2c2dabda8a 100644
--- a/packages/core/src/runtime/types.ts
+++ b/packages/core/src/runtime/types.ts
@@ -260,11 +260,13 @@ export type RuntimeOutboundMessage =
| RuntimePerformanceMessage
| RuntimeGroupLevelsMessage;
+export type HeldSeek = Promise | void;
+
export type RuntimePlayer = {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
- seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => void;
+ seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
renderSeek: (timeSeconds: number, options?: RuntimeSeekOptions) => void;
getTime: () => number;
getDuration: () => number;
diff --git a/packages/core/src/studioPreviewMark.ts b/packages/core/src/studioPreviewMark.ts
new file mode 100644
index 0000000000..bf634a7509
--- /dev/null
+++ b/packages/core/src/studioPreviewMark.ts
@@ -0,0 +1,3 @@
+export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
+export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming";
+export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy";
diff --git a/packages/producer/src/services/coreRuntimeBrowser.test.ts b/packages/producer/src/services/coreRuntimeBrowser.test.ts
index 9a6cc4cba3..5d3d3b05b4 100644
--- a/packages/producer/src/services/coreRuntimeBrowser.test.ts
+++ b/packages/producer/src/services/coreRuntimeBrowser.test.ts
@@ -4,6 +4,8 @@ import { resolve } from "node:path";
import puppeteer, { type Browser, type Page } from "puppeteer";
const RUNTIME_PATH = resolve(import.meta.dirname, "../../../core/dist/hyperframe.runtime.iife.js");
+const PNG_1PX =
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==";
describe("core runtime browser contract", () => {
let browser: Browser;
@@ -207,6 +209,63 @@ describe("core runtime browser contract", () => {
}
}, 30_000);
+ it("renders a later clip's authored lazy image as it is: laid out at setup, fetched before any seek", async () => {
+ // A chunked render starts a worker straight at a later clip, so its image must already be loaded.
+ const assets = "https://assets.test/";
+ const renderPage = await browser.newPage();
+ try {
+ await renderPage.setRequestInterception(true);
+ renderPage.on("request", (request) => {
+ if (request.url() === `${assets}runtime.js`)
+ void request.respond({
+ contentType: "text/javascript",
+ body: readFileSync(RUNTIME_PATH),
+ });
+ else if (request.url() === `${assets}plate.png`)
+ void request.respond({ contentType: "image/png", body: Buffer.from(PNG_1PX, "base64") });
+ else void request.continue();
+ });
+ await renderPage.setContent(`
+
+
+
+
+
+

+
+
+
+ `);
+ await renderPage.waitForFunction(
+ () => (window as unknown as { __renderReady?: boolean }).__renderReady === true,
+ );
+ const loaded = await renderPage
+ .waitForFunction(
+ () => {
+ const plate = document.getElementById("plate") as HTMLImageElement;
+ return plate.complete && plate.naturalWidth > 0;
+ },
+ { timeout: 5_000 },
+ )
+ .then(() => true)
+ .catch(() => false);
+ const atClip = await renderPage.evaluate(() => {
+ const runtime = window as unknown as {
+ __plateWidthAtSetup?: number;
+ __player?: { renderSeek: (timeSeconds: number) => void };
+ };
+ runtime.__player?.renderSeek(2.6);
+ return {
+ setupWidth: runtime.__plateWidthAtSetup,
+ width: document.getElementById("plate")?.offsetWidth,
+ };
+ });
+ expect({ loaded, ...atClip }).toEqual({ loaded: true, setupWidth: 200, width: 200 });
+ } finally {
+ await renderPage.close();
+ }
+ }, 30_000);
+
it("removes the control bridge during teardown", async () => {
const result = await page.evaluate(async () => {
const runtimeWindow = window as unknown as {
diff --git a/packages/studio-server/src/helpers/lazyPreviewImages.test.ts b/packages/studio-server/src/helpers/lazyPreviewImages.test.ts
new file mode 100644
index 0000000000..72e469c1a1
--- /dev/null
+++ b/packages/studio-server/src/helpers/lazyPreviewImages.test.ts
@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import { lazyPreviewImages } from "./lazyPreviewImages.js";
+
+const doc = (body: string) => `${body}`;
+const loadingOf = (html: string, id: string) =>
+ new RegExp(`
]*id="${id}"[^>]*>`).exec(html)?.[0].match(/loading="(\w+)"/)?.[1] ?? null;
+
+describe("lazyPreviewImages", () => {
+ it("marks only images in clips that start after 0 lazy", () => {
+ const html = lazyPreviewImages(
+ doc(
+ '
' +
+ '
' +
+ '
' +
+ '
',
+ ),
+ );
+ expect(
+ ["first", "later", "authored", "referenced", "untimed"].map((id) => loadingOf(html, id)),
+ ).toEqual([null, "lazy", "eager", null, null]);
+ expect(html.match(/
]*data-hf-preview-lazy[^>]*>/g)).toEqual([
+ '
',
+ ]);
+ });
+
+ it("keeps an authored loading attribute in any letter case", () => {
+ const html = doc(
+ '',
+ );
+ expect(lazyPreviewImages(html)).toBe(html);
+ });
+
+ it("marks a later image after a comment, script or template that holds an image", () => {
+ const html = lazyPreviewImages(
+ doc(
+ '' +
+ '
' +
+ '',
+ ),
+ );
+ expect(["cloned", "later"].map((id) => loadingOf(html, id))).toEqual([null, "lazy"]);
+ });
+
+ it("changes nothing when the scanner and the DOM disagree on the images", () => {
+ const html = doc('
');
+ expect(lazyPreviewImages(html)).toBe(html);
+ });
+
+ it("leaves script text and fragments untouched", () => {
+ const script = '';
+ expect(lazyPreviewImages(doc(`${script}`))).toContain(script);
+ const fragment = '';
+ expect(lazyPreviewImages(fragment)).toBe(fragment);
+ });
+
+ it("changes nothing but the attribute, entities included", () => {
+ const head = "a <x & y";
+ const body = '';
+ const html = `${head}${body}`;
+ expect(lazyPreviewImages(html)).toBe(
+ html.replace("
{
+ for (const unit of ["