perf: the preview loads and decodes only the images of scenes on screen or due next (#4578)

* perf(core): skip hidden clips' content so the browser stops decoding their images

Hidden timed clips get content-visibility:hidden while their inline visibility is hidden,
from script evaluation on. A clip due within the look-ahead (2 s by default) keeps its
content live, so its images load before it shows.

* perf(studio-server): load preview images lazily so hidden scenes fetch nothing

* fix(core): stop the readiness scan waiting on and decoding later scenes' images

* perf(studio-server): keep first-frame images eager so the preview opens as fast as before

Only images inside a clip with a numeric start after 0 go lazy; a start the route
cannot read stays eager.

* perf(core): keep clips that start at 0 laid out before the first pass so the first frame is on time

* perf(core): open the preview on time by hiding only lazy images before the first pass

GSAP reads the computed style of the elements it animates while timelines build. Inside a
content-skipped clip every read forces a style update, which cost about 65 ms at open on the
bench composition. Before the first pass the rule now takes only lazy images out of layout, so
hidden later scenes still do not load theirs; after it, the skip rule keeps their content skipped.

* fix(core): hold a paused jump on the previous picture until the next scene's images decode

* fix(studio): swap in a reloaded preview only after its restore seek has painted

* fix(core): apply a held paused jump after 1 s when an image never decodes

* fix(core): keep hidden-scene image skipping and held jumps to Studio's preview, never renders

* fix(studio): reapply manual edits once a held paused jump lands

* fix(studio): copy a clip without the preview's lazy-loading marks

* fix(studio-server): mark preview images lazy without re-encoding the document's entities

* fix(core): share one preview-mark constant between the runtime and the preview route

* fix(studio): reapply manual edits a second time only after a seek that was held

* refactor(core): share the hidden-ancestor walk between clip visibility and image holds

* fix(core): import the studio preview mark from its own subpath

* fix(studio-server): find preview images with the linear html scanner

* chore(core): drop two doc comments the scanner's names already carry

* fix(studio-server): serve the preview mark as its own meta, apart from the GSAP fallback script

* fix(studio-server): capture thumbnails and frames from the preview without lazy images or the mark

* fix(core): let the full readiness scan wait on every image except lazy ones off the first frame

* fix(studio-server): keep an authored loading attribute in any letter case on preview images

* fix(core): leave template content out of findStartTags so preview images match the DOM

* fix(studio): send Ask agent the live markup without the preview loading marks

* chore(core): drop the template note from findStartTags, its test carries it

* perf(studio-server): splice lazy attributes into the preview in one pass

* fix(studio): keep authored image loading on copy and Ask agent by marking preview-made lazy images

* test(studio-server): follow the capture preview URL back to its sub-composition for every project id

* refactor: name the upcoming-clip and GSAP fallback marks once

* fix(studio-server): keep capture builds out of the on-disk preview store
This commit is contained in:
Miguel Ángel
2026-09-28 04:03:36 +00:00
committed by GitHub
parent bc73c411f0
commit 078c493448
29 changed files with 907 additions and 91 deletions
+6
View File
@@ -50,6 +50,12 @@
"types": "./dist/compositionContract.d.ts",
"environments": ["browser", "bun", "node"]
},
"./studio-preview-mark": {
"source": "./src/studioPreviewMark.ts",
"runtime": "./dist/studioPreviewMark.js",
"types": "./dist/studioPreviewMark.d.ts",
"environments": ["browser", "bun", "node"]
},
"./media-timing": {
"source": "./src/mediaTiming.ts",
"runtime": "./dist/mediaTiming.js",
+10
View File
@@ -64,6 +64,12 @@
"import": "./src/compositionContract.ts",
"types": "./src/compositionContract.ts"
},
"./studio-preview-mark": {
"bun": "./src/studioPreviewMark.ts",
"node": "./dist/studioPreviewMark.js",
"import": "./src/studioPreviewMark.ts",
"types": "./src/studioPreviewMark.ts"
},
"./media-timing": {
"bun": "./src/mediaTiming.ts",
"node": "./dist/mediaTiming.js",
@@ -466,6 +472,10 @@
"import": "./dist/compositionContract.js",
"types": "./dist/compositionContract.d.ts"
},
"./studio-preview-mark": {
"import": "./dist/studioPreviewMark.js",
"types": "./dist/studioPreviewMark.d.ts"
},
"./media-timing": {
"import": "./dist/mediaTiming.js",
"types": "./dist/mediaTiming.d.ts"
@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import {
findStartTags,
injectScriptsAtHeadStart,
injectScriptsIntoHtml,
injectTagsAtHeadStart,
@@ -221,6 +222,26 @@ describe("htmlDocument helpers", () => {
});
});
describe("findStartTags", () => {
const at = (html: string, name: string) =>
findStartTags(html, name).map((i) => html.slice(i, html.indexOf(">", i) + 1));
it("finds each start tag in any case, and none in comments, raw text or longer names", () => {
const html =
'<!-- <img a> --><script>"<img b>"</script><textarea><img c></textarea>' +
'<img-card></img-card><IMG src=d><img\nsrc=e><div title="<img f>"></div>';
expect(at(html, "img")).toEqual(["<IMG src=d>", "<img\nsrc=e>"]);
});
it("leaves out template content, nested templates included", () => {
const html =
"<img a><template><img b><template><img c></template><img d></template>" +
'<div data-start="5"><template><img e></template><img f></div>';
expect(at(html, "img")).toEqual(["<img a>", "<img f>"]);
expect(at(html, "template")).toEqual(["<template>", "<template>"]);
});
});
describe("injectTagsAtHeadStart on long adversarial input", () => {
const M = 2_000_000;
it.each([
+23 -4
View File
@@ -210,19 +210,38 @@ const RAW_TEXT_TAGS = ["script", "style", "title", "textarea"] as const;
type DocumentTag = "<head" | "</head" | "<body" | "</body";
const COMMENT_END = /--!?>/g;
function findDocumentTag(html: string, tag: DocumentTag): number {
const lowered = lowerAscii(html);
function* markupStarts(lowered: string): Generator<number> {
const unclosedRawText = new Set<string>();
let cursor = 0;
while (cursor !== -1) {
const open = lowered.indexOf("<", cursor);
if (open === -1) return -1;
if (isTagAt(lowered, open, tag)) return open;
if (open === -1) return;
yield open;
cursor = skipMarkup(lowered, open, unclosedRawText);
}
}
function findDocumentTag(html: string, tag: DocumentTag): number {
const lowered = lowerAscii(html);
for (const open of markupStarts(lowered)) {
if (isTagAt(lowered, open, tag)) return open;
}
return -1;
}
export function findStartTags(html: string, name: string): number[] {
const lowered = lowerAscii(html);
const token = `<${lowerAscii(name)}`;
const starts: number[] = [];
let templateDepth = 0;
for (const open of markupStarts(lowered)) {
if (templateDepth === 0 && isTagAt(lowered, open, token)) starts.push(open);
if (isTagAt(lowered, open, "<template")) templateDepth++;
else if (templateDepth > 0 && isTagAt(lowered, open, "</template")) templateDepth--;
}
return starts;
}
function isTagAt(lowered: string, at: number, token: string): boolean {
return lowered.startsWith(token, at) && isTagBoundary(lowered.charAt(at + token.length));
}
@@ -178,6 +178,25 @@ describe("mediaReadinessInput", () => {
video.dispatchEvent(new Event("canplay"));
await expect(pending).resolves.toBeUndefined();
});
it("waits on a later scene's image in the full scan unless it is lazy", () => {
const doc = docWith(
'<div data-start="30" data-duration="5"><img id="later" src="later.png">' +
'<img id="lazy" loading="LAZY" src="lazy.png"></div>',
);
for (const image of doc.querySelectorAll("img")) {
Object.defineProperty(image, "complete", { value: false });
image.decode = vi.fn().mockResolvedValue(undefined);
}
const later = doc.querySelector<HTMLImageElement>("#later")!;
const lazy = doc.querySelector<HTMLImageElement>("#lazy")!;
expect(scanPendingCompositionAssets(doc, { scope: "all" }).pendingImages).toEqual([later]);
expect(scanPendingCompositionAssets(doc, { scope: "first-frame" }).pendingImages).toEqual([]);
expect(mediaReadinessInput(doc, new AbortController().signal, { scope: "all" })).not.toBeNull();
expect(later.decode).toHaveBeenCalled();
expect(lazy.decode).not.toHaveBeenCalled();
});
});
describe("computeReadinessInput", () => {
+9 -1
View File
@@ -107,8 +107,16 @@ export function scanPendingCompositionAssets(
.filter(isRealmHtmlMediaElement)
.filter((el) => shouldIncludeAsset(el, scope, resolver, runtimeWindow?.__timelines ?? {}))
.filter((el) => el.readyState < HAVE_FUTURE_DATA);
// A lazy image off the first frame loads only once shown, so no scope can wait on it.
const pendingImages = Array.from(doc.querySelectorAll("img"))
.filter((img) => shouldIncludeAsset(img, scope, resolver, runtimeWindow?.__timelines ?? {}))
.filter((img) =>
shouldIncludeAsset(
img,
img.getAttribute("loading")?.toLowerCase() === "lazy" ? FIRST_FRAME_READINESS_SCOPE : scope,
resolver,
runtimeWindow?.__timelines ?? {},
),
)
.filter((img) => !img.complete);
const fontsLoading = doc.fonts?.status === "loading";
return { pendingMedia, pendingImages, fontsLoading };
+194
View File
@@ -1,6 +1,7 @@
// fallow-ignore-file code-duplication
import { afterEach, describe, expect, it, vi } from "vitest";
import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
import type { RuntimeTimelineLike } from "./types";
function pausedTimeline(duration: number): RuntimeTimelineLike {
@@ -52,9 +53,25 @@ async function evaluateRuntime(): Promise<void> {
}
const visibility = (...els: HTMLElement[]) => els.map((el) => getComputedStyle(el).visibility);
const imageSkipped = (...clips: HTMLElement[]) =>
clips.map((clip) => getComputedStyle(clip.querySelector("img")!).display === "none");
const withImage = <T extends HTMLElement>(clip: T): T => {
clip.appendChild(document.createElement("img"));
return clip;
};
// What Studio's preview route serves at the head start; render and player documents never carry it.
const servePreview = () =>
document.head.appendChild(
Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
);
const neverDecodes = (clip: HTMLElement) => {
withImage(clip).querySelector("img")!.decode = () => new Promise<void>(() => {});
return clip;
};
describe("runtime entry", () => {
afterEach(() => {
vi.useRealTimers();
window.__hfRuntimeTeardown?.();
document.head.innerHTML = "";
document.body.innerHTML = "";
@@ -73,10 +90,14 @@ describe("runtime entry", () => {
});
it("paints no timed clip, from script evaluation until the first visibility pass decides it", async () => {
servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const later = timed(root, "div", "5");
const poster = timed(root, "img", "0");
// Studio serves later scenes' images lazy; laid out, they would load before the first pass.
const plate = later.appendChild(document.createElement("img"));
plate.setAttribute("loading", "lazy");
// A composition script may write visibility inline before the runtime runs.
later.style.visibility = "visible";
// Readiness, which runs the first pass, waits while GSAP batches timelines.
@@ -86,6 +107,7 @@ describe("runtime entry", () => {
await evaluateRuntime();
expect(window.__player).toBeUndefined();
expect(visibility(current, later)).toEqual(["hidden", "hidden"]);
expect(getComputedStyle(plate).display).toBe("none");
delete (document as { readyState?: unknown }).readyState;
document.dispatchEvent(new Event("DOMContentLoaded"));
@@ -96,6 +118,178 @@ describe("runtime entry", () => {
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
expect(window.__renderReady).toBe(true);
expect(visibility(current, later, poster)).toEqual(["visible", "hidden", "visible"]);
expect(document.querySelector("style[data-hf-first-pass-hide]")).toBeNull();
});
it("gives a render document neither rule, no look-ahead marks and no held seeks", async () => {
const root = mountRoot();
const current = timed(root, "div", "0");
const later = neverDecodes(timed(root, "div", "5"));
const plate = later.querySelector("img")!;
plate.setAttribute("loading", "lazy");
window.__hfTimelinesBuilding = true;
Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" });
await evaluateRuntime();
// A setup-time measure of an authored lazy image must see it laid out.
expect(getComputedStyle(plate).display).not.toBe("none");
delete (document as { readyState?: unknown }).readyState;
document.dispatchEvent(new Event("DOMContentLoaded"));
window.__hfTimelinesBuilding = false;
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
window.__player?.seek(3.5);
expect(imageSkipped(later)).toEqual([false]);
expect(document.querySelector("[data-hf-upcoming]")).toBeNull();
window.__player?.seek(5.5);
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
});
it("keys preview mode on the preview meta alone, not on the GSAP fallback script captures keep", async () => {
document.head
.appendChild(document.createElement("script"))
.setAttribute("data-hf-gsap-fallback", "");
const root = mountRoot();
const current = timed(root, "div", "0");
const later = neverDecodes(timed(root, "div", "5"));
const plate = later.querySelector("img")!;
plate.setAttribute("loading", "lazy");
window.__hfTimelinesBuilding = true;
Object.defineProperty(document, "readyState", { configurable: true, get: () => "loading" });
await evaluateRuntime();
expect(getComputedStyle(plate).display).not.toBe("none");
delete (document as { readyState?: unknown }).readyState;
document.dispatchEvent(new Event("DOMContentLoaded"));
window.__hfTimelinesBuilding = false;
window.dispatchEvent(new CustomEvent("hf-timelines-built"));
expect(document.querySelector("style[data-hf-skip-hidden-images]")).toBeNull();
window.__player?.seek(3.5);
expect(imageSkipped(later)).toEqual([false]);
expect(document.querySelector("[data-hf-upcoming]")).toBeNull();
expect(window.__player?.seek(5.5)).toBeUndefined();
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
});
it("skips the images of each hidden clip not due within the look-ahead, until something shows it", async () => {
servePreview();
const root = mountRoot();
const current = withImage(timed(root, "div", "0"));
const soon = withImage(timed(root, "div", "1.5"));
const later = withImage(timed(root, "div", "5"));
// Shorter than the look-ahead: due within it, though already over at its far end.
const brief = withImage(timed(root, "div", "5"));
brief.setAttribute("data-duration", "0.5");
await evaluateRuntime();
expect(imageSkipped(current, soon, later, brief)).toEqual([false, false, true, true]);
window.__player?.seek(3.6);
expect(imageSkipped(later, brief)).toEqual([false, false]);
window.__player?.seek(0);
// How Studio's layer reveal shows a hidden clip.
later.style.visibility = "visible";
expect(imageSkipped(later)).toEqual([false]);
});
it("holds a paused jump on the previous picture until the next scene's image decodes", async () => {
servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const later = timed(root, "div", "5");
const plate = later.appendChild(document.createElement("img"));
plate.setAttribute("loading", "lazy");
plate.setAttribute(STUDIO_PREVIEW_LAZY_ATTR, "");
let decoded = () => {};
plate.decode = () => new Promise<void>((resolve) => (decoded = resolve));
const authored = later.appendChild(document.createElement("img"));
authored.setAttribute("loading", "lazy");
authored.decode = () => Promise.resolve();
await evaluateRuntime();
expect(window.__player?.seek(0.5)).toBeUndefined();
const landed = window.__player?.seek(5.5);
expect(landed).toBeInstanceOf(Promise);
// Any frame painted now shows the previous scene, while the next one is unskipped so its image loads.
expect(visibility(current, later)).toEqual(["visible", "hidden"]);
expect(imageSkipped(later)).toEqual([false]);
expect(window.__player?.getTime()).toBe(5.5);
expect([plate, authored].map((img) => img.getAttribute("loading"))).toEqual(["eager", "lazy"]);
decoded();
await landed;
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
await window.__hfWaitForSeekCompletion?.();
});
it("applies a held jump after the cap when an image never decodes", async () => {
servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const later = timed(root, "div", "5");
later.appendChild(document.createElement("img")).decode = () => new Promise<void>(() => {});
await evaluateRuntime();
const swallowed: string[] = [];
window.__hf = {
...window.__hf,
onSwallowed: ({ label }: { label: string }) => swallowed.push(label),
};
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
window.__player?.seek(5.5);
await vi.advanceTimersByTimeAsync(999);
expect(visibility(current, later)).toEqual(["visible", "hidden"]);
await vi.advanceTimersByTimeAsync(1);
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
expect(swallowed).toEqual(["runtime.init.seekHoldCap"]);
});
it("drops a held jump when a newer seek or a render seek lands first", async () => {
servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const mid = timed(root, "div", "1");
const later = timed(root, "div", "5");
const plate = later.appendChild(document.createElement("img"));
let decoded = () => {};
plate.decode = () => new Promise<void>((resolve) => (decoded = resolve));
await evaluateRuntime();
window.__player?.seek(5.5);
window.__player?.seek(1);
expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
decoded();
await window.__hfWaitForSeekCompletion?.();
expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
expect(window.__player?.getTime()).toBe(1);
window.__player?.seek(0);
window.__player?.seek(5.5);
window.__player?.renderSeek(1);
decoded();
await window.__hfWaitForSeekCompletion?.();
expect(visibility(current, mid, later)).toEqual(["visible", "visible", "hidden"]);
});
it("never holds a render seek", async () => {
servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const later = neverDecodes(timed(root, "div", "5"));
await evaluateRuntime();
window.__player?.renderSeek(5.5);
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
});
it("shows the jump target at once when play is pressed during the hold", async () => {
servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const later = timed(root, "div", "5");
later.appendChild(document.createElement("img")).decode = () => new Promise<void>(() => {});
await evaluateRuntime();
window.__player?.seek(5.5);
window.__player?.play();
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
});
it("leaves nothing hidden when the runtime is evaluated a second time", async () => {
+134 -57
View File
@@ -3,7 +3,8 @@ import { preloadMedia } from "./preloadMedia";
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
import { instantTolerance } from "../clipFacts";
import { isInClipWindow } from "./clipWindow";
import { revealTimedClipsAfterFirstPass } from "./timedClipHide";
import { revealTimedClipsAfterFirstPass, SKIPPED_CLIP, skipsHiddenImages } from "./timedClipHide";
import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark";
import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";
import { injectCompositionCssVariables } from "./getVariables";
import { createCssAdapter } from "./adapters/css";
@@ -21,7 +22,11 @@ import {
patchVideoTextureCompat,
patchWebGLVideoTextureCompat,
} from "./adapters/video-texture-compat";
import { forceDispatchSeekEvent, waitForSeekCompletion } from "./adapters/seek-dispatch";
import {
forceDispatchSeekEvent,
registerSeekCompletion,
waitForSeekCompletion,
} from "./adapters/seek-dispatch";
import { sourceTimeAt } from "../speedRamp";
import { createWaapiAdapter } from "./adapters/waapi";
import {
@@ -86,6 +91,7 @@ import { clampNativeMediaVolume } from "../audioGain";
import { quantizeSeekTime, quantizeTimeToFrame } from "../inline-scripts/parityContract";
import { createManualEditGestureWatch } from "./manualEditGestureWatch";
import type {
HeldSeek,
RuntimeDeterministicAdapter,
RuntimeJson,
RuntimeSeekOptions,
@@ -484,7 +490,7 @@ export function initSandboxRuntimeModular(): void {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => void;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
@@ -2528,18 +2534,55 @@ export function initSandboxRuntimeModular(): void {
return needsCapture;
};
const timedVisibilityAt = (timingRevision?: number) => {
const compositionDuration = getSafeTimelineDurationSeconds(
state.capturedTimeline,
0,
timingRevision,
);
return (node: HTMLElement, time: number) =>
isRuntimeElementVisibleAt(node, {
currentTime: time,
compositionDuration,
canonicalFps: state.canonicalFps,
exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
timelineRegistry: window.__timelines ?? {},
resolver: timingResolverFor(true),
});
};
const clipChainVisibleAt = (
from: Element | null,
time: number,
visibleAt: ReturnType<typeof timedVisibilityAt>,
rootComp: HTMLElement | null,
) => {
for (let node = from; node && node !== rootComp; node = node.parentElement)
if (isHtmlElement(node) && node.hasAttribute("data-start") && !visibleAt(node, time))
return false;
return true;
};
const hiddenImagesSkipped = skipsHiddenImages();
const LOOKAHEAD_SECONDS = 2;
// Unskipped while due in the look-ahead window, so a clip shorter than the window still loads first.
const dueSoon = (
node: HTMLElement,
visibleAt: ReturnType<typeof timedVisibilityAt>,
t: number,
) => {
const start = resolveStartForElement(node, Number.NaN);
return (start > t && start <= t + LOOKAHEAD_SECONDS) || visibleAt(node, t + LOOKAHEAD_SECONDS);
};
const applyTimedElementVisibility = (
currentTime: number,
visibilityNodes: Element[],
timingRevision?: number,
) => {
const rootComp = resolveRootCompositionElement();
const compositionDuration = getSafeTimelineDurationSeconds(
state.capturedTimeline,
0,
timingRevision,
);
let decidedTimedClip = false;
const visibleAt = timedVisibilityAt(timingRevision);
for (const rawNode of visibilityNodes) {
if (!isHtmlElement(rawNode)) continue;
@@ -2561,40 +2604,17 @@ export function initSandboxRuntimeModular(): void {
if (nodeAffectsAudio(rawNode)) hiddenAudioDirty = true;
}
let isVisibleNow = isRuntimeElementVisibleAt(rawNode, {
currentTime,
compositionDuration,
canonicalFps: state.canonicalFps,
exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
timelineRegistry: window.__timelines ?? {},
resolver: timingResolverFor(true),
});
// Descendants must not override a hidden ancestor clip. CSS visibility can
// otherwise leak child pixels through inactive scenes because a descendant
// with visibility:visible escapes an ancestor's visibility:hidden.
if (isVisibleNow) {
let ancestor = rawNode.parentElement;
while (ancestor) {
if (ancestor === rootComp) break;
if (isHtmlElement(ancestor) && ancestor.hasAttribute("data-start")) {
if (
!isRuntimeElementVisibleAt(ancestor, {
currentTime,
compositionDuration,
canonicalFps: state.canonicalFps,
exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
timelineRegistry: window.__timelines ?? {},
resolver: timingResolverFor(true),
})
) {
isVisibleNow = false;
break;
}
}
ancestor = ancestor.parentElement;
}
}
const isVisibleNow =
visibleAt(rawNode, currentTime) &&
clipChainVisibleAt(rawNode.parentElement, currentTime, visibleAt, rootComp);
rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";
rawNode.toggleAttribute(
STUDIO_PREVIEW_UPCOMING_ATTR,
hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime),
);
if (!isMediaElement(rawNode) && !isImageElement(rawNode)) decidedTimedClip = true;
if (isVideoElement(rawNode) || isImageElement(rawNode)) {
colorGradingRuntime?.setSourceVisibility(rawNode, isVisibleNow);
@@ -2634,6 +2654,20 @@ export function initSandboxRuntimeModular(): void {
applyTimedElementVisibility(currentTime, visibilityNodes, timingRevision),
);
// Images a seek to `time` would reveal undecoded: skipped with their hidden clip, or still loading.
const undecodedImagesShownAt = (time: number): HTMLImageElement[] =>
withTimingResolver(() => {
if (!hiddenImagesSkipped) return [];
const rootComp = resolveRootCompositionElement();
const visibleAt = timedVisibilityAt();
return Array.from(document.images).filter(
(img) =>
img.closest('[data-start][style*="visibility: hidden"]') !== null &&
(img.closest(SKIPPED_CLIP) !== null || !img.complete) &&
clipChainVisibleAt(img, time, visibleAt, rootComp),
);
});
/**
* Clip windows sorted by each endpoint, so a seek can ask which windows it
* crossed instead of asking every clip whether it is active.
@@ -3406,8 +3440,41 @@ export function initSandboxRuntimeModular(): void {
}
};
// A paused seek that would reveal undecoded images holds the previous picture until they decode.
// Capped: a request that never settles must not freeze the preview; measured jumps settle in ~200 ms.
const SEEK_HOLD_CAP_MS = 1000;
let heldSeek: { time: number; apply: () => void } | null = null;
const flushHeldSeek = () => {
const held = heldSeek;
heldSeek = null;
held?.apply();
};
const applySeek = (quantized: number, options?: { keepPlaying?: boolean }) => {
webAudio.stopAll();
clock.detachAudioSource();
const wasPlaying = clock.isPlaying();
if (wasPlaying) clock.pause();
clock.seek(quantized);
state.currentTime = clock.now();
state.isPlaying = false;
state.mediaForceSyncNextTick = true;
const tl = state.capturedTimeline;
pauseTimelineIfPossible(tl);
const pageAnimations = seekTimelineAndAdapters(state.currentTime);
runAdapters("pause", 0, pageAnimations);
if (options?.keepPlaying && wasPlaying) {
transport.play();
return;
}
syncMediaForCurrentState();
colorGrading.redraw();
paintVfx(state.currentTime);
postState(true);
};
const transport: RuntimePlayerTransport = {
play: () => {
flushHeldSeek();
const tl = state.capturedTimeline;
if (clock.isPlaying()) return;
const dur = getSafeTimelineDurationSeconds(tl, 0);
@@ -3460,28 +3527,38 @@ export function initSandboxRuntimeModular(): void {
Math.max(0, Number(timeSeconds) || 0),
state.canonicalFps,
);
webAudio.stopAll();
clock.detachAudioSource();
const wasPlaying = clock.isPlaying();
if (wasPlaying) clock.pause();
clock.seek(quantized);
state.currentTime = clock.now();
state.isPlaying = false;
state.mediaForceSyncNextTick = true;
const tl = state.capturedTimeline;
pauseTimelineIfPossible(tl);
const pageAnimations = seekTimelineAndAdapters(state.currentTime);
runAdapters("pause", 0, pageAnimations);
if (options?.keepPlaying && wasPlaying) {
transport.play();
heldSeek = null;
const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
if (undecoded.length === 0) {
applySeek(quantized, options);
return;
}
syncMediaForCurrentState();
colorGrading.redraw();
paintVfx(state.currentTime);
postState(true);
const held = { time: quantized, apply: () => applySeek(quantized, options) };
heldSeek = held;
for (const img of undecoded) {
if (img.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) img.setAttribute("loading", "eager");
for (let clip = img.closest(SKIPPED_CLIP); clip; clip = img.closest(SKIPPED_CLIP))
clip.setAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, "");
}
let capTimer = 0;
const capped = new Promise<void>((resolve) => {
capTimer = window.setTimeout(() => {
if (heldSeek === held) swallow("runtime.init.seekHoldCap", undecoded);
resolve();
}, SEEK_HOLD_CAP_MS);
});
const decoded = Promise.all(
undecoded.map((img) => (img.decode ? img.decode().catch(() => {}) : undefined)),
);
const landed = Promise.race([decoded, capped]).then(() => {
window.clearTimeout(capTimer);
if (heldSeek === held) flushHeldSeek();
});
registerSeekCompletion(landed);
return landed;
},
renderSeek: (timeSeconds, options) => {
heldSeek = null;
renderCaptureSeekStarted = true;
const quantized = quantizeSeekTime(
Math.max(0, Number(timeSeconds) || 0),
@@ -3505,7 +3582,7 @@ export function initSandboxRuntimeModular(): void {
paintVfx(state.currentTime, { engineMode: true });
postState(true);
},
getTime: () => clock.now(),
getTime: () => heldSeek?.time ?? clock.now(),
getDuration: () => {
const dur = clock.getDuration();
return Number.isFinite(dur) ? dur : 0;
+27 -6
View File
@@ -1,26 +1,47 @@
import { STUDIO_PREVIEW_MARK_META, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark";
// Hides timed non-media clips from script evaluation until the first visibility pass decides them;
// before it, a paused page painted every clip at once. Media is left to init's media pass.
// The rule and its flag live on the page, so every runtime copy shares them.
const HIDE_ATTR = "data-hf-first-pass-hide";
const HIDE_UNTIL_FIRST_PASS =
"[data-start]:not(video, audio, img) { visibility: hidden !important; }";
const PREVIEW_HIDE_UNTIL_FIRST_PASS = 'img[loading="lazy"] { display: none !important; }';
const SKIP_ATTR = "data-hf-skip-hidden-images";
export const SKIPPED_CLIP = `[data-start]:not(video, audio, img, [${STUDIO_PREVIEW_UPCOMING_ATTR}])[style*="visibility: hidden"]`;
const SKIP_HIDDEN_IMAGES = `${SKIPPED_CLIP} img { display: none !important; }`;
const STUDIO_PREVIEW_MARK = `meta[name="${STUDIO_PREVIEW_MARK_META}"]`;
type FirstPassWindow = Window & {
__hfFirstPassHidden?: boolean;
__hyperframeRuntimeBootstrapped?: boolean;
};
function appendStyle(parent: Element, attr: string, css: string): void {
const style = document.createElement("style");
style.setAttribute(attr, "");
style.textContent = css;
parent.appendChild(style);
}
/** True in Studio's preview, where hidden clips' images are neither fetched nor decoded until due. */
export function skipsHiddenImages(): boolean {
return document.querySelector(`style[${SKIP_ATTR}]`) !== null;
}
export function hideTimedClipsUntilFirstPass(): void {
if (typeof document === "undefined") return;
const parent = document.head ?? document.documentElement;
if (!parent) return;
const preview = document.querySelector(STUDIO_PREVIEW_MARK) !== null;
if (preview && !skipsHiddenImages()) appendStyle(parent, SKIP_ATTR, SKIP_HIDDEN_IMAGES);
const win = window as FirstPassWindow;
// A runtime that already initialised may never run another pass to lift a new rule.
if (win.__hfFirstPassHidden || win.__hyperframeRuntimeBootstrapped) return;
const parent = document.head ?? document.documentElement;
if (!parent) return;
const style = document.createElement("style");
style.setAttribute(HIDE_ATTR, "");
style.textContent = HIDE_UNTIL_FIRST_PASS;
parent.appendChild(style);
const css = preview
? `${HIDE_UNTIL_FIRST_PASS} ${PREVIEW_HIDE_UNTIL_FIRST_PASS}`
: HIDE_UNTIL_FIRST_PASS;
appendStyle(parent, HIDE_ATTR, css);
win.__hfFirstPassHidden = true;
}
+3 -1
View File
@@ -260,11 +260,13 @@ export type RuntimeOutboundMessage =
| RuntimePerformanceMessage
| RuntimeGroupLevelsMessage;
export type HeldSeek = Promise<void> | void;
export type RuntimePlayer = {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => void;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
renderSeek: (timeSeconds: number, options?: RuntimeSeekOptions) => void;
getTime: () => number;
getDuration: () => number;
+3
View File
@@ -0,0 +1,3 @@
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming";
export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy";
@@ -4,6 +4,8 @@ import { resolve } from "node:path";
import puppeteer, { type Browser, type Page } from "puppeteer";
const RUNTIME_PATH = resolve(import.meta.dirname, "../../../core/dist/hyperframe.runtime.iife.js");
const PNG_1PX =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==";
describe("core runtime browser contract", () => {
let browser: Browser;
@@ -207,6 +209,63 @@ describe("core runtime browser contract", () => {
}
}, 30_000);
it("renders a later clip's authored lazy image as it is: laid out at setup, fetched before any seek", async () => {
// A chunked render starts a worker straight at a later clip, so its image must already be loaded.
const assets = "https://assets.test/";
const renderPage = await browser.newPage();
try {
await renderPage.setRequestInterception(true);
renderPage.on("request", (request) => {
if (request.url() === `${assets}runtime.js`)
void request.respond({
contentType: "text/javascript",
body: readFileSync(RUNTIME_PATH),
});
else if (request.url() === `${assets}plate.png`)
void request.respond({ contentType: "image/png", body: Buffer.from(PNG_1PX, "base64") });
else void request.continue();
});
await renderPage.setContent(`<!doctype html><html><head>
<style>.clip { position: absolute; inset: 0; }</style>
<script src="${assets}runtime.js"></script></head><body>
<div data-composition-id="root" data-start="0" data-duration="4" data-width="320" data-height="180">
<div class="clip" data-start="0" data-duration="2.5" data-track-index="1"></div>
<div class="clip" data-start="2.5" data-duration="1.5" data-track-index="1">
<img id="plate" loading="lazy" width="200" height="100" src="${assets}plate.png">
</div>
</div>
<script>window.__plateWidthAtSetup = document.getElementById("plate").offsetWidth;</script>
</body></html>`);
await renderPage.waitForFunction(
() => (window as unknown as { __renderReady?: boolean }).__renderReady === true,
);
const loaded = await renderPage
.waitForFunction(
() => {
const plate = document.getElementById("plate") as HTMLImageElement;
return plate.complete && plate.naturalWidth > 0;
},
{ timeout: 5_000 },
)
.then(() => true)
.catch(() => false);
const atClip = await renderPage.evaluate(() => {
const runtime = window as unknown as {
__plateWidthAtSetup?: number;
__player?: { renderSeek: (timeSeconds: number) => void };
};
runtime.__player?.renderSeek(2.6);
return {
setupWidth: runtime.__plateWidthAtSetup,
width: document.getElementById("plate")?.offsetWidth,
};
});
expect({ loaded, ...atClip }).toEqual({ loaded: true, setupWidth: 200, width: 200 });
} finally {
await renderPage.close();
}
}, 30_000);
it("removes the control bridge during teardown", async () => {
const result = await page.evaluate(async () => {
const runtimeWindow = window as unknown as {
@@ -0,0 +1,76 @@
import { describe, expect, it } from "vitest";
import { lazyPreviewImages } from "./lazyPreviewImages.js";
const doc = (body: string) => `<!DOCTYPE html><html><head></head><body>${body}</body></html>`;
const loadingOf = (html: string, id: string) =>
new RegExp(`<img[^>]*id="${id}"[^>]*>`).exec(html)?.[0].match(/loading="(\w+)"/)?.[1] ?? null;
describe("lazyPreviewImages", () => {
it("marks only images in clips that start after 0 lazy", () => {
const html = lazyPreviewImages(
doc(
'<div data-start="0"><img id="first" src="a.png">' +
'<div data-start="5"><img id="later" src="b.png"><img id="authored" loading="eager" src="c.png"></div>' +
'<div data-start="intro.end"><img id="referenced" src="d.png"></div></div>' +
'<img id="untimed" src="e.png">',
),
);
expect(
["first", "later", "authored", "referenced", "untimed"].map((id) => loadingOf(html, id)),
).toEqual([null, "lazy", "eager", null, null]);
expect(html.match(/<img[^>]*data-hf-preview-lazy[^>]*>/g)).toEqual([
'<img loading="lazy" data-hf-preview-lazy id="later" src="b.png">',
]);
});
it("keeps an authored loading attribute in any letter case", () => {
const html = doc(
'<div data-start="5"><IMG LOADING="eager" src="a.png"><img Loading="eager" src="b.png"></div>',
);
expect(lazyPreviewImages(html)).toBe(html);
});
it("marks a later image after a comment, script or template that holds an image", () => {
const html = lazyPreviewImages(
doc(
'<!-- <img src="x.png"> --><script>el.innerHTML = "<img src=y.png>";</script>' +
'<template><img id="cloned" src="t.png"></template>' +
'<div data-start="5"><template><img src="u.png"></template><img id="later" src="b.png"></div>',
),
);
expect(["cloned", "later"].map((id) => loadingOf(html, id))).toEqual([null, "lazy"]);
});
it("changes nothing when the scanner and the DOM disagree on the images", () => {
const html = doc('<xmp><img src="x.png"></xmp><div data-start="5"><img src="b.png"></div>');
expect(lazyPreviewImages(html)).toBe(html);
});
it("leaves script text and fragments untouched", () => {
const script = '<script>el.innerHTML = "<img src=x.png>";</script>';
expect(lazyPreviewImages(doc(`${script}<div data-start="5"></div>`))).toContain(script);
const fragment = '<div data-start="5"><img src="b.png"></div>';
expect(lazyPreviewImages(fragment)).toBe(fragment);
});
it("changes nothing but the attribute, entities included", () => {
const head = "<head><title>a &amp;lt;x &amp; y</title></head>";
const body = '<div title="a &amp;amp;lt;b" data-start="5"><img src="b.png?x=1&amp;y=2"></div>';
const html = `<!DOCTYPE html><html>${head}<body>${body}</body></html>`;
expect(lazyPreviewImages(html)).toBe(
html.replace("<img ", '<img loading="lazy" data-hf-preview-lazy '),
);
});
it("stays linear on unclosed comments and raw text", () => {
for (const unit of ["<!--", "<script>"]) {
const html = doc(
`<div data-start="5"><img src="a.png"></div>${unit.repeat(2_000_000 / unit.length)}`,
);
const started = performance.now();
const out = lazyPreviewImages(html);
expect(performance.now() - started, unit).toBeLessThan(1000);
expect(out, unit).toBe(html.replace("<img ", '<img loading="lazy" data-hf-preview-lazy '));
}
});
});
@@ -0,0 +1,32 @@
import { findStartTags } from "@hyperframes/core/compiler/html-document";
import { parseHTML } from "linkedom";
import { STUDIO_PREVIEW_LAZY_ATTR } from "@hyperframes/core/studio-preview-mark";
// A start that is not a plain number (a reference) counts as unknown and keeps the image eager.
function startsAfterZero(el: Element): boolean {
for (let node: Element | null = el; node; node = node.parentElement) {
if (Number(node.getAttribute("data-start")) > 0) return true;
}
return false;
}
const hasLoading = (img: Element) =>
Array.from(img.attributes).some((attr) => attr.name.toLowerCase() === "loading");
export function lazyPreviewImages(html: string): string {
if (!/<!doctype|<html[\s>]/i.test(html)) return html;
const images = [...parseHTML(html).document.querySelectorAll("img")];
// The scanner skips comments and raw text, so its `<img` offsets line up with the DOM's images.
const tags = findStartTags(html, "img");
if (tags.length !== images.length) return html;
const parts: string[] = [];
let from = 0;
images.forEach((img, i) => {
if (hasLoading(img) || !startsAfterZero(img)) return;
const at = (tags[i] ?? 0) + 4;
parts.push(html.slice(from, at), ` loading="lazy" ${STUDIO_PREVIEW_LAZY_ATTR}`);
from = at;
});
parts.push(html.slice(from));
return parts.join("");
}
@@ -19,7 +19,8 @@ import {
import { tmpdir } from "node:os";
import { join, parse } from "node:path";
import { ensureHfIds } from "@hyperframes/parsers/hf-ids";
import { PREVIEW_BUNDLE_OPTIONS, registerPreviewRoutes } from "./preview";
import { STUDIO_PREVIEW_MARK_META } from "@hyperframes/core/studio-preview-mark";
import { PREVIEW_BUNDLE_OPTIONS, PREVIEW_CAPTURE_PARAM, registerPreviewRoutes } from "./preview";
import { registerFileRoutes } from "./files";
import { createPreviewDocumentStore } from "../helpers/previewDocumentStore";
import type { StudioApiAdapter } from "../types";
@@ -114,6 +115,37 @@ describe("registerPreviewRoutes", () => {
expect(authored).not.toContain('<base href="/api/projects/demo/preview/">');
});
it("serves the mark the runtime keys preview-only work on, ahead of the runtime script", async () => {
const projectDir = createProjectDir();
const app = new Hono();
registerPreviewRoutes(app, createAdapter(projectDir));
const html = await (await app.request("http://localhost/projects/demo/preview")).text();
const mark = html.indexOf(`<meta name="${STUDIO_PREVIEW_MARK_META}">`);
expect(mark).toBeGreaterThan(-1);
expect(mark).toBeLessThan(html.indexOf("/api/runtime.js"));
expect(html).toContain("<script data-hf-gsap-fallback>");
});
it("serves a later scene's image lazy, and captures every image eager with no mark", async () => {
const projectDir = createProjectDir();
const later =
'<!DOCTYPE html><html><head></head><body><div data-start="5"><img src="b.png"></div></body></html>';
writeFileSync(join(projectDir, "index.html"), later);
writeFileSync(join(projectDir, "scene.html"), later);
const app = new Hono();
registerPreviewRoutes(app, createAdapter(projectDir));
for (const path of ["preview", "preview/comp/scene.html"]) {
const url = `http://localhost/projects/demo/${path}`;
const preview = await (await app.request(url)).text();
const capture = await (await app.request(`${url}?${PREVIEW_CAPTURE_PARAM}=1`)).text();
expect(preview, path).toMatch(/<img loading="lazy" [^>]*src="b.png">/);
expect(preview, path).toContain(STUDIO_PREVIEW_MARK_META);
expect(capture, path).not.toContain("loading=");
expect(capture, path).not.toContain(STUDIO_PREVIEW_MARK_META);
expect(capture, path).toContain("<script data-hf-gsap-fallback>");
}
});
it("injects Studio GSAP motion manifest runtime into project preview", async () => {
const projectDir = createProjectDir();
writeFileSync(
@@ -546,6 +578,27 @@ describe("built preview reuse", () => {
expect(restarted).toEqual({ html: cold.html, builds: 0 });
expect((await serve("build-b")).builds).toBe(1);
});
it("keeps the preview in the document store after a capture build", async () => {
const projectDir = createProjectDir();
const storeDir = join(projectDir, ".hyperframes", "preview");
const session = async (paths: string[]) => {
const bundle = vi.fn(async () => BUILT);
const app = new Hono();
registerPreviewRoutes(
app,
createAdapter(projectDir, {
bundle,
previewDocuments: createPreviewDocumentStore(storeDir, "build-a"),
} as Partial<StudioApiAdapter>),
);
for (const path of paths) await app.request(`http://localhost/projects/demo/${path}`);
return bundle.mock.calls.length;
};
expect(await session(["preview", `preview?${PREVIEW_CAPTURE_PARAM}=1`])).toBe(2);
expect(await session(["preview"])).toBe(0);
});
});
describe("hf-id surfacing in preview route", () => {
+24 -8
View File
@@ -10,6 +10,8 @@ import {
stripEmbeddedRuntimeScripts,
type BundleOptions,
} from "@hyperframes/core/compiler";
import { STUDIO_PREVIEW_MARK_META } from "@hyperframes/core/studio-preview-mark";
import { injectTagsAtHeadStart } from "@hyperframes/core/compiler/html-document";
import { isWithinProjectRoot } from "@hyperframes/parsers/asset-resolution";
import type { ResolvedProject, StudioApiAdapter } from "../types.js";
import { resolveWithinProject } from "../helpers/safePath.js";
@@ -54,6 +56,7 @@ import {
type PreviewApiAdapter,
} from "../helpers/mediaProxyPreview.js";
import { requestSubPath } from "../helpers/requestSubPath.js";
import { lazyPreviewImages } from "../helpers/lazyPreviewImages.js";
const PROJECT_SIGNATURE_META = "hyperframes-project-signature";
const GSAP_CDN_VERSION = "3.15.0";
@@ -189,7 +192,8 @@ function injectStudioMotionScript(
);
}
const GSAP_CDN_FALLBACK_SCRIPT = `<script data-hf-gsap-fallback>
const GSAP_FALLBACK_ATTR = "data-hf-gsap-fallback";
const GSAP_CDN_FALLBACK_SCRIPT = `<script ${GSAP_FALLBACK_ATTR}>
(function(){
var cdnBase="https://cdn.jsdelivr.net/npm/gsap@${GSAP_CDN_VERSION}/dist/";
var loaded={};
@@ -211,7 +215,7 @@ const GSAP_CDN_FALLBACK_SCRIPT = `<script data-hf-gsap-fallback>
</script>`;
function injectGsapCdnFallback(html: string): string {
if (html.includes("data-hf-gsap-fallback")) return html;
if (html.includes(GSAP_FALLBACK_ATTR)) return html;
if (html.includes("<head>")) return html.replace("<head>", "<head>" + GSAP_CDN_FALLBACK_SCRIPT);
return GSAP_CDN_FALLBACK_SCRIPT + html;
}
@@ -260,16 +264,23 @@ function previewVariablesFromRequest(rawVariables: string | undefined):
return { raw: rawVariables, values: parse.values };
}
/** Captures screenshot right after a seek, so they get every image eager and no preview mark. */
export const PREVIEW_CAPTURE_PARAM = "hf-capture";
function injectStudioPreviewAugmentations(
html: string,
adapter: StudioApiAdapter,
projectDir: string,
activeCompositionPath: string,
capture: boolean,
): string {
const marked = capture
? html
: injectTagsAtHeadStart(lazyPreviewImages(html), `<meta name="${STUDIO_PREVIEW_MARK_META}">`);
return injectStudioMotionScript(
injectMotionPathPluginIfNeeded(
injectGsapCdnFallback(
injectProjectSignature(html, resolveProjectSignature(adapter, projectDir)),
injectProjectSignature(marked, resolveProjectSignature(adapter, projectDir)),
),
),
projectDir,
@@ -357,6 +368,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
project: ResolvedProject,
previewVariables: Record<string, unknown> | null,
builtKey: string,
capture: boolean,
): Promise<string | null> {
const diskMain = resolveProjectMainHtml(project.dir, project.id);
const normalizedDisk = diskMain ? ensureHfIds(diskMain.html) : null;
@@ -400,6 +412,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
adapter,
project.dir,
mainCompositionPath,
capture,
);
if (previewVariables) bundled = injectPreviewVariables(bundled, previewVariables);
bundled = await injectMediaCodecMap(
@@ -411,7 +424,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
);
bundled = addScenePartsManifest(bundled, [`meta[name="${PROJECT_SIGNATURE_META}"]`]);
rememberPreview(builtKey, bundled);
adapter.previewDocuments?.write(builtKey, bundled);
if (!capture) adapter.previewDocuments?.write(builtKey, bundled);
return bundled;
} catch {
// Re-read disk on bundle failure so we serve the latest file content,
@@ -424,6 +437,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
adapter,
project.dir,
fallback.compositionPath,
capture,
);
if (previewVariables) {
fallbackAugmented = injectPreviewVariables(fallbackAugmented, previewVariables);
@@ -452,8 +466,9 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
const vars = previewVariablesFromRequest(c.req.query("variables"));
if (vars.error !== undefined) return c.json({ error: vars.error }, 400);
const previewVariables = vars.values;
const capture = c.req.query(PREVIEW_CAPTURE_PARAM) !== undefined;
const etag = `"preview:${signature}${variablesEtagSalt(vars.raw)}"`;
const etag = `"preview:${signature}${variablesEtagSalt(vars.raw)}${capture ? ":capture" : ""}"`;
const ifNoneMatch = c.req.header("If-None-Match");
if (ifNoneMatch === etag) {
return new Response(null, {
@@ -469,7 +484,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
}
let pending = previewBuilds.get(builtKey);
if (!pending) {
pending = buildPreview(project, previewVariables, builtKey).finally(() =>
pending = buildPreview(project, previewVariables, builtKey, capture).finally(() =>
previewBuilds.delete(builtKey),
);
previewBuilds.set(builtKey, pending);
@@ -514,7 +529,8 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
// a pre-pin cached response (preview-only ids, unstamped disk file) must
// not revalidate to a 304 that skips the pin.
const compPathHash = createHash("sha1").update(compPath).digest("hex");
const etag = `"comp:v2:${compPathHash}:${signature}${variablesEtagSalt(vars.raw)}"`;
const capture = c.req.query(PREVIEW_CAPTURE_PARAM) !== undefined;
const etag = `"comp:v2:${compPathHash}:${signature}${variablesEtagSalt(vars.raw)}${capture ? ":capture" : ""}"`;
const ifNoneMatch = c.req.header("If-None-Match");
if (ifNoneMatch === etag) {
return new Response(null, {
@@ -537,7 +553,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
if (!html) return c.text("not found", 404);
recordPreviewReferences(project.dir, html);
html = ensureHfIds(await transformPreviewHtml(html, adapter, project, compPath));
html = injectStudioPreviewAugmentations(html, adapter, project.dir, compPath);
html = injectStudioPreviewAugmentations(html, adapter, project.dir, compPath, capture);
if (previewVariables) html = injectPreviewVariables(html, previewVariables);
html = await injectMediaCodecMap(html, adapter, project.dir, compPath, mediaCodecProbeCache);
return c.html(html, 200, previewCacheHeaders(etag));
@@ -4,7 +4,7 @@ import { mkdtempSync, writeFileSync, rmSync, mkdirSync, statSync } from "node:fs
import { tmpdir } from "node:os";
import { join } from "node:path";
import { registerFileRoutes } from "./files.js";
import { registerPreviewRoutes } from "./preview.js";
import { PREVIEW_CAPTURE_PARAM, registerPreviewRoutes } from "./preview.js";
import { registerThumbnailRoutes } from "./thumbnail.js";
import { registerWaveformRoutes } from "./waveform.js";
import { buildWaveformCacheKey } from "../helpers/waveform.js";
@@ -91,7 +91,7 @@ const RESERVED_NAMES = [
"🎬 film",
];
type ThumbnailCall = { compPath: string; previewUrl: string };
type ThumbnailCall = { compPath: string; previewUrl: string; served?: string };
async function requestProject(projectId: string, route: string, encodedSubPath: string) {
const { dir, cleanup } = projectWithComposition();
@@ -104,15 +104,17 @@ async function requestProject(projectId: string, route: string, encodedSubPath:
"[0.5]",
);
const thumbnails: ThumbnailCall[] = [];
const app = new Hono();
const adapter = {
...createAdapter(dir),
generateThumbnail: async (opts: ThumbnailCall) => {
thumbnails.push({ compPath: opts.compPath, previewUrl: opts.previewUrl });
const page = await app.request(opts.previewUrl.replace("/api/", "/"));
const served = page.ok ? await page.text() : `${page.status}`;
thumbnails.push({ compPath: opts.compPath, previewUrl: opts.previewUrl, served });
return Buffer.from("jpeg");
},
} as StudioApiAdapter;
try {
const app = new Hono();
registerFileRoutes(app, adapter);
registerPreviewRoutes(app, adapter);
registerThumbnailRoutes(app, adapter);
@@ -151,7 +153,8 @@ describe.each(RESERVED_NAMES)("project id %j", (projectId) => {
expect(result.thumbnails).toHaveLength(1);
expect(result.thumbnails[0]?.compPath).toBe("scenes/scene-1.html");
const url = new URL(result.thumbnails[0]?.previewUrl ?? "");
expect(url.search + url.hash).toBe("");
expect(url.search + url.hash).toBe(`?${PREVIEW_CAPTURE_PARAM}=1`);
expect(result.thumbnails[0]?.served).toContain("SCENE ONE");
const segments = url.pathname.split("/").map(decodeURIComponent);
expect(segments).toEqual([
"",
@@ -14,6 +14,7 @@ import {
import { tmpdir } from "node:os";
import { join } from "node:path";
import { pruneThumbnailCache, registerThumbnailRoutes } from "./thumbnail";
import { PREVIEW_CAPTURE_PARAM } from "./preview";
import type { StudioApiAdapter } from "../types";
import { createProjectSignature } from "../helpers/projectSignature.js";
@@ -60,6 +61,23 @@ async function writeComposition(
}
describe("registerThumbnailRoutes", () => {
it("screenshots the capture variant of the preview document", async () => {
const adapter = createAdapter();
await writeComposition(adapter, 1920, 1080);
const app = new Hono();
registerThumbnailRoutes(app, adapter);
const loaded: string[] = [];
vi.mocked(adapter.generateThumbnail!).mockImplementation(async ({ previewUrl }) => {
loaded.push(new URL(previewUrl).search);
return Buffer.from("thumb");
});
await app.request("http://localhost/projects/demo/thumbnail/index.html?t=6");
await app.request("http://localhost/projects/demo/thumbnail/scenes%2Fb.html?t=6");
expect(loaded).toEqual([`?${PREVIEW_CAPTURE_PARAM}=1`, `?${PREVIEW_CAPTURE_PARAM}=1`]);
});
it("forwards selector queries to thumbnail generation", async () => {
const adapter = createAdapter();
const app = new Hono();
@@ -23,6 +23,7 @@ import { STUDIO_MOTION_PATH } from "../helpers/studioMotionRenderScript.js";
import { thumbnailGenerationCoordinator } from "./thumbnailGenerationCoordinator.js";
import { requestSubPath } from "../helpers/requestSubPath.js";
import { resolveWithinProject } from "../helpers/safePath.js";
import { PREVIEW_CAPTURE_PARAM } from "./preview.js";
const THUMBNAIL_CACHE_VERSION = "v4";
const THUMBNAIL_MAX_OUTPUT_WIDTH = 240;
@@ -179,10 +180,11 @@ export function registerThumbnailRoutes(api: Hono, adapter: StudioApiAdapter): v
sourceMtime = Math.max(sourceMtime, manualEdits.mtimeMs, motion.mtimeMs);
const projectUrl = `http://${c.req.header("host")}/api/projects/${encodeURIComponent(project.id)}`;
const previewUrl =
const previewPath =
compPath === "index.html"
? `${projectUrl}/preview`
: `${projectUrl}/preview/comp/${compPath.split("/").map(encodeURIComponent).join("/")}`;
const previewUrl = `${previewPath}?${PREVIEW_CAPTURE_PARAM}=1`;
// Cache
const cacheDir = join(project.dir, ".thumbnails");
@@ -432,6 +432,30 @@ describe("studio manual edits", () => {
expect(applied).toBe(11);
});
it("reapplies once after a seek that applied, and again once a held seek lands", async () => {
const previewWindow = new Window() as unknown as Parameters<
typeof installStudioManualEditSeekReapply
>[0] & { __player: Record<string, unknown> };
let landed = () => {};
const held = new Promise<void>((resolve) => (landed = resolve));
previewWindow.__player = { seek: (time: number) => (time > 5 ? held : undefined) };
let applied = 0;
installStudioManualEditSeekReapply(previewWindow, () => {
applied += 1;
});
const seek = previewWindow.__player.seek as (time: number) => unknown;
seek(1);
await Promise.resolve();
expect(applied).toBe(1);
expect(seek(5.5)).toBe(held);
expect(applied).toBe(2);
landed();
await held;
await Promise.resolve();
expect(applied).toBe(3);
});
it("reapplies manual edits while fresh playback is active", () => {
const window = new Window();
const frames: FrameRequestCallback[] = [];
@@ -78,6 +78,7 @@ function wrapSeekReapplyFunction(
win: StudioManualEditSeekWindow,
owner: Record<string, unknown> | undefined,
key: string,
reapplyWhenHeldSeekLands = false,
): boolean {
const fn = owner?.[key];
if (!owner || typeof fn !== "function") return false;
@@ -87,6 +88,8 @@ function wrapSeekReapplyFunction(
const wrappedSeek = function (this: unknown, ...args: unknown[]): unknown {
const result = seek.apply(this, args);
win.__hfStudioManualEditsApply?.();
const held = reapplyWhenHeldSeekLands ? (result as PromiseLike<void> | undefined) : undefined;
if (typeof held?.then === "function") void held.then(() => win.__hfStudioManualEditsApply?.());
return result;
};
markWrapped(wrappedSeek);
@@ -256,7 +259,7 @@ export function installStudioManualEditSeekReapply(win: Window, apply: () => voi
studioWin[STUDIO_MANUAL_EDITS_APPLY_PROP] = apply;
const wrappedHfSeek = wrapSeekReapplyFunction(studioWin, studioWin.__hf, "seek");
const wrappedPlayerSeek = wrapSeekReapplyFunction(studioWin, studioWin.__player, "seek");
const wrappedPlayerSeek = wrapSeekReapplyFunction(studioWin, studioWin.__player, "seek", true);
const wrappedPlayerRenderSeek = wrapSeekReapplyFunction(
studioWin,
studioWin.__player,
@@ -0,0 +1,78 @@
// @vitest-environment happy-dom
import React, { act } from "react";
import { createRoot } from "react-dom/client";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { DomEditSelection } from "../components/editor/domEditing";
import { useAskAgentModal } from "./useAskAgentModal";
const copied = vi.hoisted(() => [] as string[]);
vi.mock("../utils/clipboard", () => ({
copyTextToClipboard: async (text: string) => copied.push(text) > 0,
}));
(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
afterEach(() => {
document.body.innerHTML = "";
copied.length = 0;
vi.unstubAllGlobals();
});
describe("useAskAgentModal", () => {
it("sends the live element without the preview's lazy loading or look-ahead mark, keeping authored loading", async () => {
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline")));
const element = document.createElement("div");
element.setAttribute("data-hf-upcoming", "");
element.innerHTML =
'<img src="plate.png" loading="lazy" data-hf-preview-lazy><img src="own.png" loading="eager">';
const selection: DomEditSelection = {
element,
label: "Plate",
tagName: "div",
sourceFile: "index.html",
compositionPath: "index.html",
isCompositionHost: false,
isInsideLockedComposition: false,
selector: "div",
selectorIndex: 0,
boundingBox: { x: 0, y: 0, width: 10, height: 10 },
textContent: "",
dataAttributes: {},
inlineStyles: {},
computedStyles: {},
textFields: [],
capabilities: {
canSelect: true,
canEditStyles: true,
canCrop: false,
canMove: true,
canResize: true,
canApplyManualOffset: false,
canApplyManualSize: false,
canApplyManualRotation: false,
},
};
let hook: ReturnType<typeof useAskAgentModal> | null = null;
function Harness() {
hook = useAskAgentModal({
projectId: "demo",
activeCompPath: "index.html",
projectDir: "/demo",
projectIdRef: { current: "demo" },
showToast: () => {},
domEditSelectionRef: { current: selection },
domEditSelection: selection,
});
return null;
}
const host = document.body.appendChild(document.createElement("div"));
await act(async () => createRoot(host).render(<Harness />));
await act(async () => hook!.handleAgentModalSubmit("Make it bigger"));
expect(copied[0]).toContain('<img src="plate.png"><img src="own.png" loading="eager">');
expect(copied[0]).not.toContain("data-hf-preview-lazy");
expect(copied[0]).not.toContain("data-hf-upcoming");
});
});
@@ -4,6 +4,7 @@ import { useState, useCallback, useRef, useEffect } from "react";
import { copyTextToClipboard } from "../utils/clipboard";
import { readTagSnippetByTarget } from "../utils/sourcePatcher";
import { toProjectAbsolutePath, type AgentModalAnchorPoint } from "../utils/studioHelpers";
import { liveMarkupWithoutPreviewMarks } from "../utils/authoredSource";
import { buildElementAgentPrompt, type DomEditSelection } from "../components/editor/domEditing";
import { usePlayerStore } from "../player";
@@ -89,7 +90,8 @@ export function useAskAgentModal({
if (!domEditSelection) return;
const targetPath = domEditSelection.sourceFile || activeCompPath || "index.html";
const tagSnippet = agentPromptTagSnippet ?? domEditSelection.element.outerHTML;
const tagSnippet =
agentPromptTagSnippet ?? liveMarkupWithoutPreviewMarks(domEditSelection.element);
const prompt = buildElementAgentPrompt({
selection: domEditSelection,
currentTime: usePlayerStore.getState().currentTime,
@@ -220,6 +220,27 @@ describe("copy takes a clip's saved markup, not the runtime's live styling", ()
});
});
describe("copy of a clip missing from its saved file", () => {
it("takes the preview's markup without its lazy loading or look-ahead mark, keeping authored loading", async () => {
selectTitle();
const { clipboard, writes, files, iframe } = mountClipboard();
const title = iframe.contentDocument?.querySelector("h1") as Element;
title.setAttribute("data-hf-upcoming", "");
title.insertAdjacentHTML(
"beforeend",
'<img src="plate.png" loading="lazy" data-hf-preview-lazy><img src="own.png" loading="lazy">',
);
files["index.html"] = SAVED.replace(/<h1[\s\S]*<\/h1>/, "");
clipboard().handleCopy();
await clipboard().handlePaste();
expect(writes[0]).toMatch(
/<img src="plate.png" data-hf-id="[^"]+"><img src="own.png" loading="lazy"/,
);
expect(writes[0]).not.toContain("data-hf-preview-lazy");
expect(writes[0]).not.toContain("data-hf-upcoming");
});
});
const SUB_SELECTION = {
hfId: SUB_HF_ID,
selector: "h2",
+4 -1
View File
@@ -23,6 +23,7 @@ import {
authoredMarkup,
findAuthoredElement,
findAuthoredElementById,
liveMarkupWithoutPreviewMarks,
parseSavedSource,
} from "../utils/authoredSource";
import { serializeStudioFileMutations } from "../utils/studioFileMutationCoordinator";
@@ -88,7 +89,9 @@ function getSelectedDomElement(
function savedMarkupElseLive(saved: Document, live: Element, sourceFile: string): string {
const authored = findAuthoredElement(saved, live) ?? findAuthoredElementById(saved, live);
return authored ? authoredMarkup(authored, live, sourceFile) : live.outerHTML;
return authored
? authoredMarkup(authored, live, sourceFile)
: liveMarkupWithoutPreviewMarks(live);
}
async function readSavedMarkup(
@@ -12,7 +12,7 @@ import {
type PreviewIframeSlot,
type UseTimelineSyncCallbacksParams,
} from "./useTimelineSyncCallbacks";
import type { PlaybackAdapter } from "../lib/playbackTypes";
import type { IframeWindow, PlaybackAdapter } from "../lib/playbackTypes";
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
import { usePlayerStore } from "../store/playerStore";
@@ -23,6 +23,11 @@ export const SHADOW_READY_TIMEOUT_MS = 15_000;
// A busy machine can need more than one budget; the shadow keeps loading for this many before it is dropped.
export const SHADOW_READY_BUDGETS = 3;
function restoreSeekPainted(iframe: HTMLIFrameElement | null): Promise<void> | undefined {
const win = iframe?.contentWindow as IframeWindow | null | undefined;
return win?.__hfWaitForSeekCompletion?.().catch(() => {});
}
function isDocumentHidden(): boolean {
return typeof document !== "undefined" && document.visibilityState === "hidden";
}
@@ -136,10 +141,16 @@ export function useShadowPreviewReload({
const getShadowAdapter = useCallback(() => getAdapter(shadowIframeRef.current), [getAdapter]);
const isCurrentShadow = useCallback((gen?: number) => gen === shadowGenRef.current, []);
const markAdapterReady = useCallback(
(_iframe: HTMLIFrameElement | null, gen: number | undefined, commit: () => void) => {
(iframe: HTMLIFrameElement | null, gen: number | undefined, commit: () => void) => {
if (gen == null || gen !== shadowGenRef.current) return;
pendingCommitRef.current = { gen, commit };
promoteWhenReady(gen);
const register = () => {
if (gen !== shadowGenRef.current) return;
pendingCommitRef.current = { gen, commit };
promoteWhenReady(gen);
};
const painted = restoreSeekPainted(iframe);
if (painted) void painted.then(register);
else register();
},
[promoteWhenReady],
);
@@ -325,6 +325,25 @@ describe("shadow reload readiness and failure", () => {
unmount(root);
});
it("promotes only once the shadow's restore seek has painted its frame", async () => {
const { getApi, live, gen, root } = beginReload();
const shadow = makeShadowWithSpies();
let seekPainted = () => {};
Object.assign(shadow.iframe.contentWindow!, {
__hfWaitForSeekCompletion: () => new Promise<void>((resolve) => (seekPainted = resolve)),
});
act(() => {
getApi().setShadowIframeNode(shadow.iframe);
getApi().onShadowIframeLoad(gen);
getApi().onShadowReadyChange(gen, true);
});
expect(getApi().iframeRef.current).toBe(live);
await act(async () => seekPainted());
expect(getApi().iframeRef.current).toBe(shadow.iframe);
unmount(root);
});
it("promotes when the loader clears before the adapter is ready", () => {
const { getApi, gen, root } = beginReload();
const shadow = makeShadowWithSpies();
@@ -62,6 +62,7 @@ export type IframeWindow = Window & {
__timeline?: TimelineLike;
__timelines?: Record<string, TimelineLike>;
__clipManifest?: ClipManifest;
__hfWaitForSeekCompletion?: () => Promise<void>;
/** Declared runtime-side in core's window.d.ts, which this package cannot see.
* Every member stays optional and is optional-called, so a runtime that
* predates the hook degrades to a no-op instead of throwing. */
@@ -1,4 +1,8 @@
import { ensureHfIds, isCompositionTemplate } from "@hyperframes/parsers/hf-ids";
import {
STUDIO_PREVIEW_LAZY_ATTR,
STUDIO_PREVIEW_UPCOMING_ATTR,
} from "@hyperframes/core/studio-preview-mark";
// Stamped as the preview stamps the files it serves, so a live element's hf-id finds its source.
export function parseSavedSource(html: string): Document {
@@ -90,3 +94,14 @@ export function authoredMarkup(authored: Element, live: Element, sourceFile: str
}
return copy.outerHTML;
}
export function liveMarkupWithoutPreviewMarks(live: Element): string {
const copy = live.cloneNode(true) as Element;
for (const el of [copy, ...Array.from(copy.querySelectorAll("*"))]) {
el.removeAttribute(STUDIO_PREVIEW_UPCOMING_ATTR);
if (!el.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) continue;
el.removeAttribute("loading");
el.removeAttribute(STUDIO_PREVIEW_LAZY_ATTR);
}
return copy.outerHTML;
}