mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-16 06:19:21 +00:00
perf(studio): capture stills load only the media near their moment, as the preview does
This commit is contained in:
@@ -8,7 +8,7 @@ import {
|
||||
settleFirstFrameCompositionReadiness,
|
||||
} from "./compositionReadiness.js";
|
||||
import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
|
||||
import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
|
||||
import { DEFER_MEDIA_META, STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
|
||||
import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
|
||||
|
||||
function docWith(bodyHtml: string): Document {
|
||||
@@ -180,7 +180,7 @@ describe("mediaReadinessInput", () => {
|
||||
await expect(pending).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("waits on a later clip in Studio's preview only once the runtime has set it to buffer", () => {
|
||||
it("waits on a later clip in Studio's preview or a capture only once the runtime has set it to buffer", () => {
|
||||
const html =
|
||||
'<video id="deferred" data-start="30" data-duration="5" preload="none" src="a.mp4"></video>' +
|
||||
'<video id="untrimmed" data-start="30" preload="metadata" src="b.mp4"></video>' +
|
||||
@@ -189,11 +189,14 @@ describe("mediaReadinessInput", () => {
|
||||
preview.head
|
||||
.appendChild(preview.createElement("meta"))
|
||||
.setAttribute("name", STUDIO_PREVIEW_MARK_META);
|
||||
const capture = docWith(html);
|
||||
capture.head.appendChild(capture.createElement("meta")).setAttribute("name", DEFER_MEDIA_META);
|
||||
const plain = docWith(html);
|
||||
const ids = (doc: Document) =>
|
||||
scanPendingCompositionAssets(doc, { scope: "all" }).pendingMedia.map((el) => el.id);
|
||||
|
||||
expect(ids(preview)).toEqual(["due"]);
|
||||
expect(ids(capture)).toEqual(["due"]);
|
||||
expect(ids(plain)).toEqual(["deferred", "untrimmed", "due"]);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
|
||||
import { isRuntimeElementVisibleAt } from "./runtime/timeline.js";
|
||||
import type { RuntimeTimelineLike } from "./runtime/types.js";
|
||||
import { STUDIO_PREVIEW_MARK_META } from "./studioPreviewMark.js";
|
||||
import { defersMedia } from "./studioPreviewMark.js";
|
||||
|
||||
/** A composition is "ready" once every declared input settles, not just once
|
||||
* its duration is known. Each input returns null (nothing to wait on) or a
|
||||
@@ -105,13 +105,13 @@ export function scanPendingCompositionAssets(
|
||||
includeAuthoredTimingAttrs: true,
|
||||
});
|
||||
// Studio's preview buffers a clip off the first frame only once it comes due, as with a lazy image.
|
||||
const defersMedia = doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null;
|
||||
const mediaDeferred = defersMedia(doc);
|
||||
const pendingMedia = Array.from(doc.querySelectorAll("video, audio"))
|
||||
.filter(isRealmHtmlMediaElement)
|
||||
.filter((el) =>
|
||||
shouldIncludeAsset(
|
||||
el,
|
||||
defersMedia && el.preload !== "auto" ? FIRST_FRAME_READINESS_SCOPE : scope,
|
||||
mediaDeferred && el.preload !== "auto" ? FIRST_FRAME_READINESS_SCOPE : scope,
|
||||
resolver,
|
||||
runtimeWindow?.__timelines ?? {},
|
||||
),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading";
|
||||
import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
|
||||
import { DEFER_MEDIA_META, STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
|
||||
import type { RuntimeTimelineLike } from "./types";
|
||||
|
||||
function pausedTimeline(duration: number): RuntimeTimelineLike {
|
||||
@@ -387,6 +387,22 @@ describe("runtime entry", () => {
|
||||
const armed = (...els: HTMLMediaElement[]) => els.map((el) => el.preload === "auto");
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("loads a capture build's media only near its moment, while every image of a hidden clip stays shown", async () => {
|
||||
spyLoad();
|
||||
// What a capture build carries instead of the preview mark.
|
||||
document.head.appendChild(Object.assign(document.createElement("meta"), { name: DEFER_MEDIA_META }));
|
||||
const root = mountRoot();
|
||||
const [now, later] = videos(root, "0", "9");
|
||||
const hidden = withImage(timed(root, "div", "9"));
|
||||
|
||||
await evaluateRuntime();
|
||||
expect(armed(now, later)).toEqual([true, false]);
|
||||
expect(loadsOf(now, later)).toEqual([1, 0]);
|
||||
expect(imageSkipped(hidden)).toEqual([false]);
|
||||
window.__player?.seek(9.5);
|
||||
expect(armed(later)).toEqual([true]);
|
||||
});
|
||||
|
||||
it("loads a preview's media only on screen or due within the look-ahead, and frees what the playhead left", async () => {
|
||||
spyLoad();
|
||||
servePreview();
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
RUNTIME_STAMPED_ATTR,
|
||||
STUDIO_PREVIEW_LAZY_ATTR,
|
||||
STUDIO_PREVIEW_UPCOMING_ATTR,
|
||||
defersMedia,
|
||||
} from "../studioPreviewMark";
|
||||
import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics";
|
||||
import { injectCompositionCssVariables } from "./getVariables";
|
||||
@@ -2789,6 +2790,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
};
|
||||
|
||||
const hiddenImagesSkipped = skipsHiddenImages();
|
||||
const mediaDeferred = defersMedia(document);
|
||||
const LOOKAHEAD_SECONDS = 2;
|
||||
const RELEASE_LOOKAHEAD_SECONDS = 4;
|
||||
// Unskipped while within the window on either side of the playhead, so a step or shuttle back
|
||||
@@ -2814,7 +2816,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
// A clip with no authored length loads as in a render: a reload would drop the duration it is
|
||||
// timed by.
|
||||
const inPreloadWindow = (el: HTMLMediaElement) =>
|
||||
hiddenImagesSkipped && el.hasAttribute("data-start") && lengthIsAuthored(el);
|
||||
mediaDeferred && el.hasAttribute("data-start") && lengthIsAuthored(el);
|
||||
|
||||
// Media on screen or due within the look-ahead loads; the rest stops holding a connection to
|
||||
// the origin that also serves Studio's thumbnails.
|
||||
@@ -2967,8 +2969,11 @@ export function initSandboxRuntimeModular(): void {
|
||||
const isRuntimeStamped = isDiscoveryStamped(rawNode);
|
||||
if (!isRuntimeStamped) rawNode.style.visibility = isVisibleNow ? "visible" : "hidden";
|
||||
rawNode.toggleAttribute(CLIP_HIDDEN_ATTR, !isRuntimeStamped && !isVisibleNow);
|
||||
const upcoming =
|
||||
hiddenImagesSkipped && !isVisibleNow && dueSoon(rawNode, visibleAt, currentTime);
|
||||
const due =
|
||||
(hiddenImagesSkipped || mediaDeferred) &&
|
||||
!isVisibleNow &&
|
||||
dueSoon(rawNode, visibleAt, currentTime);
|
||||
const upcoming = hiddenImagesSkipped && due;
|
||||
if (upcoming && !rawNode.hasAttribute(STUDIO_PREVIEW_UPCOMING_ATTR))
|
||||
loadPreviewImagesIn(rawNode);
|
||||
rawNode.toggleAttribute(STUDIO_PREVIEW_UPCOMING_ATTR, upcoming);
|
||||
@@ -2976,7 +2981,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
preloadNearPlayhead(
|
||||
rawNode,
|
||||
isVisibleNow,
|
||||
upcoming || mediaStaysDue(rawNode, visibleAt, currentTime),
|
||||
due || mediaStaysDue(rawNode, visibleAt, currentTime),
|
||||
);
|
||||
if (!isMediaElement(rawNode) && !isImageElement(rawNode)) decidedTimedClip = true;
|
||||
if (isVideoElement(rawNode) || isImageElement(rawNode)) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { parseStrictFiniteTimingNumber } from "./playbackRate";
|
||||
import { createRuntimeStartTimeResolver } from "./startResolver";
|
||||
import { skipsHiddenImages } from "./timedClipHide";
|
||||
import { defersMedia } from "../studioPreviewMark";
|
||||
|
||||
type PreloadableMedia = Pick<
|
||||
HTMLMediaElement,
|
||||
@@ -34,13 +34,13 @@ export function lengthIsAuthored(media: Element): boolean {
|
||||
|
||||
type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver };
|
||||
|
||||
/** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
|
||||
/** Preview and capture builds only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
|
||||
* One watcher per page: a runtime evaluated again replaces the last one's. */
|
||||
export function deferMediaUntilDue(): void {
|
||||
const win = window as DeferralWindow;
|
||||
win.__hfMediaDeferral?.disconnect();
|
||||
win.__hfMediaDeferral = undefined;
|
||||
if (!skipsHiddenImages()) return;
|
||||
if (!defersMedia(document)) return;
|
||||
const defer = (el: Element, startInFilm: (el: Element) => number) => {
|
||||
const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start"));
|
||||
if (start != null && lengthIsAuthored(el) && startInFilm(el) > 0)
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
|
||||
/** Media deferral without the preview's image skipping: a capture's still loads only the media near its moment. */
|
||||
export const DEFER_MEDIA_META = "hyperframes-defer-media";
|
||||
export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming";
|
||||
export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy";
|
||||
export const STUDIO_PREVIEW_ERRORS = "__hfPreviewErrors";
|
||||
@@ -18,3 +20,7 @@ export function applyAuthoredTiming(
|
||||
for (const [name, value] of attrs) el.setAttribute(name, value);
|
||||
el.removeAttribute(RUNTIME_STAMPED_ATTR);
|
||||
}
|
||||
|
||||
/** Whether `doc` loads media only near the playhead: Studio's preview, or a capture build that asks for that alone. */
|
||||
export const defersMedia = (doc: Document): boolean =>
|
||||
doc.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"], meta[name="${DEFER_MEDIA_META}"]`) !== null;
|
||||
|
||||
@@ -23,6 +23,7 @@ import { join, parse } from "node:path";
|
||||
import { ensureHfIds } from "@hyperframes/parsers/hf-ids";
|
||||
import {
|
||||
STUDIO_PREVIEW_ERRORS,
|
||||
DEFER_MEDIA_META,
|
||||
STUDIO_PREVIEW_MARK_META,
|
||||
} from "@hyperframes/core/studio-preview-mark";
|
||||
import { AFTER_FONTS_SCRIPT_TYPE } from "@hyperframes/core/compiler";
|
||||
@@ -281,7 +282,7 @@ describe("registerPreviewRoutes", () => {
|
||||
expect(previewWindow[STUDIO_PREVIEW_ERRORS]).toEqual(["Uncaught Error: GSAP could not load"]);
|
||||
});
|
||||
|
||||
it("serves a later scene's image lazy, and captures every image eager with no mark", async () => {
|
||||
it("serves a later scene's image lazy, and captures every image eager with only media deferred", async () => {
|
||||
const projectDir = createProjectDir();
|
||||
const later =
|
||||
'<!DOCTYPE html><html><head></head><body><div data-start="5"><img src="b.png"></div></body></html>';
|
||||
@@ -297,6 +298,8 @@ describe("registerPreviewRoutes", () => {
|
||||
expect(preview, path).toContain(STUDIO_PREVIEW_MARK_META);
|
||||
expect(capture, path).not.toContain("loading=");
|
||||
expect(capture, path).not.toContain(STUDIO_PREVIEW_MARK_META);
|
||||
expect(capture, path).toContain(`<meta name="${DEFER_MEDIA_META}">`);
|
||||
expect(preview, path).not.toContain(DEFER_MEDIA_META);
|
||||
expect(capture, path).toContain("<script data-hf-gsap-fallback>");
|
||||
}
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
type BundleOptions,
|
||||
} from "@hyperframes/core/compiler";
|
||||
import {
|
||||
DEFER_MEDIA_META,
|
||||
STUDIO_PREVIEW_MARK_META,
|
||||
STUDIO_PREVIEW_ERRORS,
|
||||
} from "@hyperframes/core/studio-preview-mark";
|
||||
@@ -320,7 +321,8 @@ function previewVariablesFromRequest(rawVariables: string | undefined):
|
||||
return { raw: rawVariables, values: parse.values };
|
||||
}
|
||||
|
||||
/** Captures screenshot right after a seek, so they get every image eager and no preview mark. */
|
||||
/** Captures screenshot right after a seek, so they get every image eager and no preview mark; their media still
|
||||
* loads only near the moment shot, or a long film's still would load every clip it has. */
|
||||
export const PREVIEW_CAPTURE_PARAM = "hf-capture";
|
||||
|
||||
// Studio's console capture attaches at the iframe's load; this keeps what was raised before it.
|
||||
@@ -335,7 +337,7 @@ function injectStudioPreviewAugmentations(
|
||||
capture: boolean,
|
||||
): string {
|
||||
const marked = capture
|
||||
? html
|
||||
? injectTagsAtHeadStart(html, `<meta name="${DEFER_MEDIA_META}">`)
|
||||
: injectTagsAtHeadStart(
|
||||
lazyPreviewImages(html),
|
||||
`<meta name="${STUDIO_PREVIEW_MARK_META}">${EARLY_ERRORS_SCRIPT}`,
|
||||
|
||||
Reference in New Issue
Block a user