perf(studio): capture stills load only the media near their moment, as the preview does

This commit is contained in:
Miguel Angel Simon Sierra
2026-10-19 08:30:42 -04:00
parent cea0bad793
commit 6ad3a98ef3
8 changed files with 51 additions and 16 deletions
@@ -8,7 +8,7 @@ import {
settleFirstFrameCompositionReadiness,
} from "./compositionReadiness.js";
import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
import { DEFER_MEDIA_META, STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
function docWith(bodyHtml: string): Document {
@@ -180,7 +180,7 @@ describe("mediaReadinessInput", () => {
await expect(pending).resolves.toBeUndefined();
});
it("waits on a later clip in Studio's preview only once the runtime has set it to buffer", () => {
it("waits on a later clip in Studio's preview or a capture only once the runtime has set it to buffer", () => {
const html =
'<video id="deferred" data-start="30" data-duration="5" preload="none" src="a.mp4"></video>' +
'<video id="untrimmed" data-start="30" preload="metadata" src="b.mp4"></video>' +
@@ -189,11 +189,14 @@ describe("mediaReadinessInput", () => {
preview.head
.appendChild(preview.createElement("meta"))
.setAttribute("name", STUDIO_PREVIEW_MARK_META);
const capture = docWith(html);
capture.head.appendChild(capture.createElement("meta")).setAttribute("name", DEFER_MEDIA_META);
const plain = docWith(html);
const ids = (doc: Document) =>
scanPendingCompositionAssets(doc, { scope: "all" }).pendingMedia.map((el) => el.id);
expect(ids(preview)).toEqual(["due"]);
expect(ids(capture)).toEqual(["due"]);
expect(ids(plain)).toEqual(["deferred", "untrimmed", "due"]);
});
+3 -3
View File
@@ -1,7 +1,7 @@
import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
import type { RuntimeTimelineLike } from "./runtime/types.js";
import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
import { defersMedia } from "./studioPreviewMark.js";
/** A composition is "ready" once every declared input settles, not just once
* its duration is known. Each input returns null (nothing to wait on) or a
@@ -105,13 +105,13 @@ export function scanPendingCompositionAssets(
includeAuthoredTimingAttrs: true,
});
// Studio's preview buffers a clip off the first frame only once it comes due, as with a lazy image.
const defersMedia = doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null;
const mediaDeferred = defersMedia(doc);
const pendingMedia = Array.from(doc.querySelectorAll("video, audio"))
.filter(isRealmHtmlMediaElement)
.filter((el) =>
shouldIncludeAsset(
el,
defersMedia && el.preload !== "auto" ? FIRST_FRAME_READINESS_SCOPE : scope,
mediaDeferred && el.preload !== "auto" ? FIRST_FRAME_READINESS_SCOPE : scope,
resolver,
runtimeWindow?.__timelines ?? {},
),
+17 -1
View File
@@ -1,7 +1,7 @@
// fallow-ignore-file code-duplication
import { afterEach, describe, expect, it, vi } from "vitest";
import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
import { DEFER_MEDIA_META, STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
import type { RuntimeTimelineLike } from "./types";
function pausedTimeline(duration: number): RuntimeTimelineLike {
@@ -387,6 +387,22 @@ describe("runtime entry", () => {
const armed = (...els: HTMLMediaElement[]) => els.map((el) => el.preload === "auto");
afterEach(() => vi.restoreAllMocks());
it("loads a capture build's media only near its moment, while every image of a hidden clip stays shown", async () => {
spyLoad();
// What a capture build carries instead of the preview mark.
document.head.appendChild(Object.assign(document.createElement("meta"), { name: DEFER_MEDIA_META }));
const root = mountRoot();
const [now, later] = videos(root, "0", "9");
const hidden = withImage(timed(root, "div", "9"));
await evaluateRuntime();
expect(armed(now, later)).toEqual([true, false]);
expect(loadsOf(now, later)).toEqual([1, 0]);
expect(imageSkipped(hidden)).toEqual([false]);
window.__player?.seek(9.5);
expect(armed(later)).toEqual([true]);
});
it("loads a preview's media only on screen or due within the look-ahead, and frees what the playhead left", async () => {
spyLoad();
servePreview();
+9 -4
View File
@@ -18,6 +18,7 @@ import {
RUNTIME_STAMPED_ATTR,
STUDIO_PREVIEW_LAZY_ATTR,
STUDIO_PREVIEW_UPCOMING_ATTR,
defersMedia,
} from "../studioPreviewMark";
import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";
import { injectCompositionCssVariables } from "./getVariables";
@@ -2789,6 +2790,7 @@ export function initSandboxRuntimeModular(): void {
};
const hiddenImagesSkipped = skipsHiddenImages();
const mediaDeferred = defersMedia(document);
const LOOKAHEAD_SECONDS = 2;
const RELEASE_LOOKAHEAD_SECONDS = 4;
// Unskipped while within the window on either side of the playhead, so a step or shuttle back
@@ -2814,7 +2816,7 @@ export function initSandboxRuntimeModular(): void {
// A clip with no authored length loads as in a render: a reload would drop the duration it is
// timed by.
const inPreloadWindow = (el: HTMLMediaElement) =>
hiddenImagesSkipped && el.hasAttribute("data-start") && lengthIsAuthored(el);
mediaDeferred && el.hasAttribute("data-start") && lengthIsAuthored(el);
// Media on screen or due within the look-ahead loads; the rest stops holding a connection to
// the origin that also serves Studio's thumbnails.
@@ -2967,8 +2969,11 @@ export function initSandboxRuntimeModular(): void {
const isRuntimeStamped = isDiscoveryStamped(rawNode);
if (!isRuntimeStamped) rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";
rawNode.toggleAttribute(CLIP_HIDDEN_ATTR, !isRuntimeStamped && !isVisibleNow);
const upcoming =
hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime);
const due =
(hiddenImagesSkipped || mediaDeferred) &&
!isVisibleNow &&
dueSoon(rawNode, visibleAt, currentTime);
const upcoming = hiddenImagesSkipped && due;
if (upcoming && !rawNode.hasAttribute(STUDIO_PREVIEW_UPCOMING_ATTR))
loadPreviewImagesIn(rawNode);
rawNode.toggleAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, upcoming);
@@ -2976,7 +2981,7 @@ export function initSandboxRuntimeModular(): void {
preloadNearPlayhead(
rawNode,
isVisibleNow,
upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),
due || mediaStaysDue(rawNode, visibleAt, currentTime),
);
if (!isMediaElement(rawNode) && !isImageElement(rawNode)) decidedTimedClip = true;
if (isVideoElement(rawNode) || isImageElement(rawNode)) {
+3 -3
View File
@@ -1,6 +1,6 @@
import { parseStrictFiniteTimingNumber } from "./playbackRate";
import { createRuntimeStartTimeResolver } from "./startResolver";
import { skipsHiddenImages } from "./timedClipHide";
import { defersMedia } from "../studioPreviewMark";
type PreloadableMedia = Pick<
HTMLMediaElement,
@@ -34,13 +34,13 @@ export function lengthIsAuthored(media: Element): boolean {
type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver };
/** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
/** Preview and capture builds only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
* One watcher per page: a runtime evaluated again replaces the last one's. */
export function deferMediaUntilDue(): void {
const win = window as DeferralWindow;
win.__hfMediaDeferral?.disconnect();
win.__hfMediaDeferral = undefined;
if (!skipsHiddenImages()) return;
if (!defersMedia(document)) return;
const defer = (el: Element, startInFilm: (el: Element) => number) => {
const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start"));
if (start != null && lengthIsAuthored(el) && startInFilm(el) > 0)
+6
View File
@@ -1,4 +1,6 @@
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
/** Media deferral without the preview's image skipping: a capture's still loads only the media near its moment. */
export const DEFER_MEDIA_META = "hyperframes-defer-media";
export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming";
export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy";
export const STUDIO_PREVIEW_ERRORS = "__hfPreviewErrors";
@@ -18,3 +20,7 @@ export function applyAuthoredTiming(
for (const [name, value] of attrs) el.setAttribute(name, value);
el.removeAttribute(RUNTIME_STAMPED_ATTR);
}
/** Whether `doc` loads media only near the playhead: Studio's preview, or a capture build that asks for that alone. */
export const defersMedia = (doc: Document): boolean =>
doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"], meta[name="${DEFER_MEDIA_META}"]`) !== null;
@@ -23,6 +23,7 @@ import { join, parse } from "node:path";
import { ensureHfIds } from "@hyperframes/parsers/hf-ids";
import {
STUDIO_PREVIEW_ERRORS,
DEFER_MEDIA_META,
STUDIO_PREVIEW_MARK_META,
} from "@hyperframes/core/studio-preview-mark";
import { AFTER_FONTS_SCRIPT_TYPE } from "@hyperframes/core/compiler";
@@ -281,7 +282,7 @@ describe("registerPreviewRoutes", () => {
expect(previewWindow[STUDIO_PREVIEW_ERRORS]).toEqual(["Uncaught Error: GSAP could not load"]);
});
it("serves a later scene's image lazy, and captures every image eager with no mark", async () => {
it("serves a later scene's image lazy, and captures every image eager with only media deferred", async () => {
const projectDir = createProjectDir();
const later =
'<!DOCTYPE html><html><head></head><body><div data-start="5"><img src="b.png"></div></body></html>';
@@ -297,6 +298,8 @@ describe("registerPreviewRoutes", () => {
expect(preview, path).toContain(STUDIO_PREVIEW_MARK_META);
expect(capture, path).not.toContain("loading=");
expect(capture, path).not.toContain(STUDIO_PREVIEW_MARK_META);
expect(capture, path).toContain(`<meta name="${DEFER_MEDIA_META}">`);
expect(preview, path).not.toContain(DEFER_MEDIA_META);
expect(capture, path).toContain("<script data-hf-gsap-fallback>");
}
});
+4 -2
View File
@@ -14,6 +14,7 @@ import {
type BundleOptions,
} from "@hyperframes/core/compiler";
import {
DEFER_MEDIA_META,
STUDIO_PREVIEW_MARK_META,
STUDIO_PREVIEW_ERRORS,
} from "@hyperframes/core/studio-preview-mark";
@@ -320,7 +321,8 @@ function previewVariablesFromRequest(rawVariables: string | undefined):
return { raw: rawVariables, values: parse.values };
}
/** Captures screenshot right after a seek, so they get every image eager and no preview mark. */
/** Captures screenshot right after a seek, so they get every image eager and no preview mark; their media still
* loads only near the moment shot, or a long film's still would load every clip it has. */
export const PREVIEW_CAPTURE_PARAM = "hf-capture";
// Studio's console capture attaches at the iframe's load; this keeps what was raised before it.
@@ -335,7 +337,7 @@ function injectStudioPreviewAugmentations(
capture: boolean,
): string {
const marked = capture
? html
? injectTagsAtHeadStart(html, `<meta name="${DEFER_MEDIA_META}">`)
: injectTagsAtHeadStart(
lazyPreviewImages(html),
`<meta name="${STUDIO_PREVIEW_MARK_META}">${EARLY_ERRORS_SCRIPT}`,