diff --git a/.github/workflows/player-perf.yml b/.github/workflows/player-perf.yml index 811f9a41ac..5d6d73fa24 100644 --- a/.github/workflows/player-perf.yml +++ b/.github/workflows/player-perf.yml @@ -124,6 +124,13 @@ jobs: PUPPETEER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} run: bun run test:browser-security + - name: Verify the preview rasters at its shown size (load shard only) + if: matrix.shard == 'load' + working-directory: packages/player + env: + PUPPETEER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} + run: bun run test:browser-raster + - name: Run player perf — ${{ matrix.shard }} (measure mode) working-directory: packages/player env: diff --git a/packages/core/src/runtime/init.swapScenes.test.ts b/packages/core/src/runtime/init.swapScenes.test.ts index dfb37af6dd..c8191865e7 100644 --- a/packages/core/src/runtime/init.swapScenes.test.ts +++ b/packages/core/src/runtime/init.swapScenes.test.ts @@ -691,6 +691,13 @@ describe("__hfSwapScenes", () => { }, ], ["the opacity stamp", '', () => {}, "data-hf-authored-opacity"], + [ + "the preview raster mark", + '', + () => {}, + "data-hf-preview-raster", + ], + ["the upcoming mark", '', () => {}, "data-hf-upcoming"], ])("keeps a video on which the runtime wrote %s", async (_, video, arrange, stamp?: string) => { quietMedia(); arrange(); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 4ba6043977..776ecdf380 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -1,4 +1,5 @@ // fallow-ignore-file code-duplication complexity +import { setPreviewRasterScale } from "./previewRasterHints"; import { refreshSvgSelectorAliases } from "../compiler/svgSelectorAliases"; import { RUNTIME_FILLER } from "./protocol"; import { preloadMedia, releaseMedia, lengthIsAuthored, stopMediaDownload } from "./preloadMedia"; @@ -6,7 +7,11 @@ import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps import { instantTolerance } from "../clipFacts"; import { isInClipWindow } from "./clipWindow"; import { revealTimedClipsAfterFirstPass, SKIPPED_CLIP, skipsHiddenImages } from "./timedClipHide"; -import { STUDIO_PREVIEW_LAZY_ATTR, STUDIO_PREVIEW_UPCOMING_ATTR } from "../studioPreviewMark"; +import { + PREVIEW_ONLY_ATTRS, + STUDIO_PREVIEW_LAZY_ATTR, + STUDIO_PREVIEW_UPCOMING_ATTR, +} from "../studioPreviewMark"; import { initRuntimeAnalytics, emitAnalyticsEvent } from "./analytics"; import { injectCompositionCssVariables } from "./getVariables"; import { createCssAdapter } from "./adapters/css"; @@ -280,13 +285,14 @@ const authoredShape = (el: Element): string => ]); // What the runtime writes on a video or audio: style (reset on a kept one), preload, the opacity stamp, a moved -// element's original translate and its own __hf- classes. A proxied src is compared as written; a bound one is not. +// element's original translate, preview marks and its own __hf- classes. A proxied src is compared as written. const RUNTIME_MEDIA_ATTRS = new Set([ "style", "preload", "src", COLOR_GRADING_AUTHORED_OPACITY_ATTR, EDIT_ORIGINAL_TRANSLATE_ATTR, + ...PREVIEW_ONLY_ATTRS, ]); const ownClasses = (value: string) => value @@ -5024,7 +5030,10 @@ export function initSandboxRuntimeModular(): void { if (state.transportClock) state.transportClock.setRate(state.playbackRate); applyWebAudioRate(); }, - onSetDisplayScale: setProxyDisplayScale, + onSetDisplayScale: (scale) => { + setProxyDisplayScale(scale); + setPreviewRasterScale(scale); + }, onSetIdleHeartbeat: (slow) => { slowIdleHeartbeat = slow; wakeTransport(); diff --git a/packages/core/src/runtime/previewRasterHints.test.ts b/packages/core/src/runtime/previewRasterHints.test.ts new file mode 100644 index 0000000000..c5ef01e714 --- /dev/null +++ b/packages/core/src/runtime/previewRasterHints.test.ts @@ -0,0 +1,178 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark"; +import { setPreviewRasterScale } from "./previewRasterHints"; + +function layer(style: string, parent: Element = document.body): HTMLElement { + const element = document.createElement("div"); + element.setAttribute("style", style); + parent.append(element); + return element; +} + +const POSITIONED = "position: absolute; will-change: transform"; +const marked = (element: Element) => element.hasAttribute(PREVIEW_RASTER_ATTR); + +afterEach(() => { + setPreviewRasterScale(1); + document.body.innerHTML = ""; +}); + +describe("setPreviewRasterScale", () => { + it("touches nothing until the host reports a scale", () => { + const words = layer(POSITIONED); + expect(document.querySelectorAll(`style, [${PREVIEW_RASTER_ATTR}]`)).toHaveLength(0); + expect(getComputedStyle(words).willChange).toBe("transform"); + }); + + it("drops a positioned layer's transform hint while the preview is shown small", () => { + const words = layer(POSITIONED); + const word = layer("position: absolute", words); + const mixed = layer("position: relative; will-change: transform, opacity"); + setPreviewRasterScale(0.275); + expect(marked(words)).toBe(true); + expect(marked(word)).toBe(false); + expect(marked(mixed)).toBe(false); + expect(getComputedStyle(words).willChange).toBe("auto"); + expect(words.getAttribute("style")).toBe(POSITIONED); + }); + + it("keeps the hint where the stacking context or containing block matters", () => { + const unpositioned = layer("will-change: transform"); + const holdsFixed = layer(POSITIONED); + layer("position: fixed", holdsFixed); + const holdsZIndex = layer(POSITIONED); + layer("position: relative; z-index: 2", holdsZIndex); + const holdsBlend = layer(POSITIONED); + layer("mix-blend-mode: multiply", holdsBlend); + const stacked = layer(`${POSITIONED}; z-index: 1`); + layer("position: relative; z-index: 2", stacked); + setPreviewRasterScale(0.5); + expect([unpositioned, holdsFixed, holdsZIndex, holdsBlend].map(marked)).toEqual([ + false, + false, + false, + false, + ]); + expect(marked(stacked)).toBe(true); + }); + + it("restores the document once shown at full size", () => { + const words = layer(POSITIONED); + const before = document.documentElement.outerHTML; + setPreviewRasterScale(0.5); + expect(document.documentElement.outerHTML).not.toBe(before); + setPreviewRasterScale(1); + expect(document.documentElement.outerHTML).toBe(before); + expect(getComputedStyle(words).willChange).toBe("transform"); + }); + + it("follows layers added and classes changed while the preview is small", async () => { + const sheet = document.createElement("style"); + sheet.textContent = ".pinned { position: fixed }"; + document.head.append(sheet); + setPreviewRasterScale(0.5); + const late = layer(POSITIONED); + const child = layer("", late); + await Promise.resolve(); + expect(marked(late)).toBe(true); + child.className = "pinned"; + await Promise.resolve(); + expect(marked(late)).toBe(false); + sheet.remove(); + }); + + it("gives the hint back when an inline style makes a child depend on it", async () => { + const words = layer(POSITIONED); + const word = layer("position: absolute", words); + setPreviewRasterScale(0.5); + expect(marked(words)).toBe(true); + word.style.zIndex = "5"; + await Promise.resolve(); + expect(marked(words)).toBe(false); + }); + + it("re-reads a marked layer's own hints and position when they change", async () => { + const sheet = document.createElement("style"); + sheet.textContent = + ".hinted { will-change: transform } .hinted.fading { will-change: transform, opacity }"; + document.head.append(sheet); + const classed = layer("position: absolute"); + classed.className = "hinted"; + const inline = layer(POSITIONED); + const inlineHint = layer(POSITIONED); + setPreviewRasterScale(0.5); + expect([classed, inline, inlineHint].map(marked)).toEqual([true, true, true]); + classed.className = "hinted fading"; + inline.style.position = "static"; + inlineHint.style.willChange = "transform, opacity"; + await Promise.resolve(); + expect([classed, inline, inlineHint].map(marked)).toEqual([false, false, false]); + sheet.remove(); + }); + + it("follows stylesheets and attribute selectors added while the preview is small", async () => { + const byStylesheet = layer(POSITIONED); + layer("", byStylesheet).className = "late-pinned"; + const byAttribute = layer(POSITIONED); + const scene = layer("", byAttribute); + setPreviewRasterScale(0.5); + expect([byStylesheet, byAttribute].map(marked)).toEqual([true, true]); + const sheet = document.createElement("style"); + sheet.textContent = '.late-pinned, [data-scene="pinned"] { position: fixed }'; + document.head.append(sheet); + await Promise.resolve(); + expect(marked(byStylesheet)).toBe(false); + expect(marked(byAttribute)).toBe(true); + scene.dataset.scene = "pinned"; + await Promise.resolve(); + expect(marked(byAttribute)).toBe(false); + sheet.remove(); + }); + + it("leaves marked layers alone when only something around them changes", async () => { + const scene = layer(""); + const words = [layer(POSITIONED, scene), layer(POSITIONED, scene)]; + setPreviewRasterScale(0.5); + const markChanges: MutationRecord[] = []; + const watch = new MutationObserver((records) => markChanges.push(...records)); + watch.observe(document.body, { subtree: true, attributeFilter: [PREVIEW_RASTER_ATTR] }); + scene.dataset.upcoming = ""; + await Promise.resolve(); + await Promise.resolve(); + watch.disconnect(); + expect(words.map(marked)).toEqual([true, true]); + expect(markChanges).toEqual([]); + }); + + it("does not react to its own marks", async () => { + const words = layer(POSITIONED); + setPreviewRasterScale(0.5); + words.dataset.scene = "intro"; + await Promise.resolve(); + await Promise.resolve(); + const reads = vi.spyOn(window, "getComputedStyle"); + await Promise.resolve(); + await Promise.resolve(); + expect(reads).not.toHaveBeenCalled(); + reads.mockRestore(); + expect(marked(words)).toBe(true); + }); + + it("re-checks the page when a linked stylesheet loads", async () => { + const sheet = document.createElement("style"); + document.head.append(sheet); + const words = layer(POSITIONED); + layer("", words).className = "loaded-pinned"; + setPreviewRasterScale(0.5); + const link = document.createElement("link"); + document.head.append(link); + await Promise.resolve(); + expect(marked(words)).toBe(true); + // Stands in for the linked rules arriving: stylesheet object edits are not observed. + sheet.sheet!.insertRule(".loaded-pinned { position: fixed }"); + link.dispatchEvent(new Event("load")); + expect(marked(words)).toBe(false); + sheet.remove(); + link.remove(); + }); +}); diff --git a/packages/core/src/runtime/previewRasterHints.ts b/packages/core/src/runtime/previewRasterHints.ts new file mode 100644 index 0000000000..e31c3cdf26 --- /dev/null +++ b/packages/core/src/runtime/previewRasterHints.ts @@ -0,0 +1,159 @@ +import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark"; +import { isElementNode } from "./domRealm"; + +// Chromium rasters a `will-change: transform` layer at no less than device scale, so a preview shown +// small rasters each one at full size. Below 1x the hint is dropped wherever nothing depends on the +// stacking context and fixed-position containing block it makes, so the preview paints the same. +const TRANSFORM_HINTS = new Set(["transform", "translate", "rotate", "scale"]); +const MARKED = `[${PREVIEW_RASTER_ATTR}]`; +const DROP_HINTS = `${MARKED} { will-change: auto !important; }`; + +let sheet: HTMLStyleElement | null = null; +let observer: MutationObserver | null = null; +const markedWith = new WeakMap(); + +// Other hints (opacity, filter) make boundaries of their own, so only a pure transform hint goes. +function onlyTransformHints(style: CSSStyleDeclaration): boolean { + return style.willChange.split(",").every((hint) => TRANSFORM_HINTS.has(hint.trim())); +} + +// Fixed boxes always need the hint's containing block; z-indexed and blending boxes need its +// stacking context unless the layer stacks anyway. +function escapes(element: Element, stacks: boolean, self = true): boolean { + for (const pseudo of self ? [null, "::before", "::after"] : ["::before", "::after"]) { + const style = getComputedStyle(element, pseudo); + if (pseudo && style.content === "none") continue; + if (style.position === "fixed") return true; + if (!stacks && (style.zIndex !== "auto" || style.mixBlendMode !== "normal")) return true; + } + return false; +} + +// A positioned element already contains its absolute descendants and paints in the same phase. +function canDropHint(element: Element, style: CSSStyleDeclaration): boolean { + if (style.position === "static") return false; + const stacks = style.zIndex !== "auto"; + if (escapes(element, stacks, false)) return false; + for (const descendant of element.querySelectorAll("*")) { + if (escapes(descendant, stacks)) return false; + } + return true; +} + +// What a script can change inline on a marked layer that decides whether it stays marked. +function inlineFit(element: Element, style: CSSStyleDeclaration): string { + return `${style.zIndex}|${(element as HTMLElement).style?.willChange ?? ""}`; +} + +// A marked element reads `auto` through our rule. Lifting the mark to read its own hints repaints +// the layer, so that only happens when the element itself changed. +function fits(element: Element, readOwnHints: boolean): boolean { + if (element === sheet) return false; + const marked = element.hasAttribute(PREVIEW_RASTER_ATTR); + if (marked && readOwnHints) element.removeAttribute(PREVIEW_RASTER_ATTR); + const style = getComputedStyle(element); + const hintsFit = (marked && !readOwnHints) || onlyTransformHints(style); + const drop = hintsFit && canDropHint(element, style); + if (drop) markedWith.set(element, inlineFit(element, style)); + return drop; +} + +function check(element: Element, readOwnHints: boolean): void { + const drop = fits(element, readOwnHints); + if (drop !== element.hasAttribute(PREVIEW_RASTER_ATTR)) { + element.toggleAttribute(PREVIEW_RASTER_ATTR, drop); + } +} + +function unmarkHolders(element: Element): void { + let holder = element.parentElement?.closest(MARKED); + while (holder) { + if (escapes(element, getComputedStyle(holder).zIndex !== "auto")) { + holder.removeAttribute(PREVIEW_RASTER_ATTR); + } + holder = holder.parentElement?.closest(MARKED); + } +} + +function followTree(root: Element): void { + for (const element of [root, ...root.querySelectorAll("*")]) { + check(element, element === root); + unmarkHolders(element); + } +} + +// Scripts restyle inline every frame, so only what can break a mark is looked at. +function followStyle(element: Element): void { + if (element.hasAttribute(PREVIEW_RASTER_ATTR)) { + const style = getComputedStyle(element); + if (style.position === "static" || inlineFit(element, style) !== markedWith.get(element)) { + check(element, true); + } + } + unmarkHolders(element); +} + +function clearMarks(): void { + for (const element of document.querySelectorAll(MARKED)) { + element.removeAttribute(PREVIEW_RASTER_ATTR); + } +} + +// Every mark is decided before any is written, so the page restyles twice, not once per layer. +function markLayers(): void { + clearMarks(); + const layers = [...document.querySelectorAll("*")].filter((element) => fits(element, true)); + for (const layer of layers) layer.setAttribute(PREVIEW_RASTER_ATTR, ""); +} + +function followAttribute(record: MutationRecord): void { + if (!isElementNode(record.target) || record.attributeName === PREVIEW_RASTER_ATTR) return; + // Any attribute can match a selector; inline style is the per-frame case, so it is cheaper. + if (record.attributeName === "style") followStyle(record.target); + else followTree(record.target); +} + +// New rules can restyle anything, and a linked sheet only applies once it loads. +function addedRules(record: MutationRecord): Element[] { + return [...record.addedNodes].filter( + (node): node is Element => + isElementNode(node) && (node.localName === "style" || node.localName === "link"), + ); +} + +function follow(records: MutationRecord[]): void { + const rules = records.flatMap(addedRules); + for (const rule of rules) { + rule.addEventListener("load", () => observer && markLayers(), { once: true }); + } + if (rules.length) return markLayers(); + for (const record of records) { + if (record.type === "attributes") followAttribute(record); + else for (const node of record.addedNodes) if (isElementNode(node)) followTree(node); + } +} + +function restore(): void { + observer?.disconnect(); + observer = null; + sheet?.remove(); + sheet = null; + clearMarks(); +} + +/** The host reports how large it shows this document; below 1x, transform hints stop forcing full-size rasters. */ +export function setPreviewRasterScale(scale: number): void { + if (!(scale > 0) || !Number.isFinite(scale)) return; + if (scale >= 1) return restore(); + if (observer) return; + sheet = document.createElement("style"); + sheet.textContent = DROP_HINTS; + (document.head ?? document.documentElement).append(sheet); + markLayers(); + observer = new MutationObserver(follow); + observer.observe(document.documentElement, { + subtree: true, + childList: true, + attributes: true, + }); +} diff --git a/packages/core/src/runtime/stackingContext.test.ts b/packages/core/src/runtime/stackingContext.test.ts new file mode 100644 index 0000000000..6df58e250f --- /dev/null +++ b/packages/core/src/runtime/stackingContext.test.ts @@ -0,0 +1,20 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark"; +import { ROOT_CSS_STACKING_CONTEXT_ID, resolveCssStackingContextId } from "./stackingContext"; + +afterEach(() => { + document.body.innerHTML = ""; +}); + +describe("resolveCssStackingContextId", () => { + it("keeps the stacking context of a layer whose hint the small preview dropped", () => { + document.body.innerHTML = ` +
+
+
`; + const clip = document.getElementById("clip")!; + expect(resolveCssStackingContextId(clip)).not.toBe(ROOT_CSS_STACKING_CONTEXT_ID); + document.getElementById("layer")!.removeAttribute(PREVIEW_RASTER_ATTR); + expect(resolveCssStackingContextId(clip)).toBe(ROOT_CSS_STACKING_CONTEXT_ID); + }); +}); diff --git a/packages/core/src/runtime/stackingContext.ts b/packages/core/src/runtime/stackingContext.ts index e27f77c751..9aa226bb59 100644 --- a/packages/core/src/runtime/stackingContext.ts +++ b/packages/core/src/runtime/stackingContext.ts @@ -1,3 +1,5 @@ +import { PREVIEW_RASTER_ATTR } from "../studioPreviewMark"; + export const ROOT_CSS_STACKING_CONTEXT_ID = "css:root"; const NON_NONE_STACKING_CONTEXT_PROPERTIES = [ @@ -67,6 +69,7 @@ function createsWillChangeStackingContext(style: CSSStyleDeclaration): boolean { } function createsCssStackingContext(element: HTMLElement, style: CSSStyleDeclaration): boolean { + if (element.hasAttribute(PREVIEW_RASTER_ATTR)) return true; if (createsPositionedStackingContext(style)) return true; if (createsZIndexedStackingContext(element, style)) return true; if (createsVisualEffectStackingContext(style)) return true; diff --git a/packages/core/src/studioPreviewMark.ts b/packages/core/src/studioPreviewMark.ts index dabeb6ba1b..58dec41377 100644 --- a/packages/core/src/studioPreviewMark.ts +++ b/packages/core/src/studioPreviewMark.ts @@ -2,3 +2,9 @@ export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview"; export const STUDIO_PREVIEW_UPCOMING_ATTR = "data-hf-upcoming"; export const STUDIO_PREVIEW_LAZY_ATTR = "data-hf-preview-lazy"; export const STUDIO_PREVIEW_ERRORS = "__hfPreviewErrors"; +export const PREVIEW_RASTER_ATTR = "data-hf-preview-raster"; +export const PREVIEW_ONLY_ATTRS = [ + STUDIO_PREVIEW_UPCOMING_ATTR, + STUDIO_PREVIEW_LAZY_ATTR, + PREVIEW_RASTER_ATTR, +] as const; diff --git a/packages/player/package.json b/packages/player/package.json index 2cc8bba89c..ea8503547c 100644 --- a/packages/player/package.json +++ b/packages/player/package.json @@ -32,6 +32,7 @@ "typecheck": "tsc --noEmit && tsc --noEmit -p tests/perf/tsconfig.json", "test": "vitest run", "test:browser-security": "bun run tests/browser/sandbox-origin.ts", + "test:browser-raster": "bun run tests/browser/preview-raster-size.ts", "perf": "bun run tests/perf/index.ts" }, "dependencies": { diff --git a/packages/player/tests/browser/preview-raster-size.ts b/packages/player/tests/browser/preview-raster-size.ts new file mode 100644 index 0000000000..0b2cc8e6b0 --- /dev/null +++ b/packages/player/tests/browser/preview-raster-size.ts @@ -0,0 +1,256 @@ +import assert from "node:assert/strict"; + +import { launchBrowser } from "../perf/runner.js"; +import { startServer } from "../perf/server.js"; + +// A small preview must raster its composition's will-change layers near the size it shows them, not at +// full size (which runs heavy compositions out of tile memory), without moving or repainting anything. +const server = startServer({ noCache: true }); +const browser = await launchBrowser({ width: 1920, height: 1080 }); + +type Layer = { + ideal_contents_scale?: number; + raster_scales?: { contents_scale?: [number, number] }; +}; + +try { + const page = await browser.newPage(); + await page.evaluateOnNewDocument(() => { + const sheet = new CSSStyleSheet(); + sheet.replaceSync("hyperframes-player { display: block; width: 1920px; height: 1080px }"); + document.adoptedStyleSheets = [sheet]; + }); + await page.goto(`${server.origin}/host.html?fixture=preview-raster`); + await page.waitForFunction(() => window.__playerReady === true); + const frame = page.frames().find((candidate) => candidate.url().includes("/fixtures/")); + assert(frame, "composition frame loaded"); + const until = (what: string, check: () => boolean) => + frame + .waitForFunction(check, { timeout: 10_000 }) + .catch(() => assert.fail(`timed out: ${what}`)); + const swapped = () => getComputedStyle(document.querySelector(".lyric")!).willChange === "auto"; + const hinted = () => + getComputedStyle(document.querySelector(".lyric")!).willChange === "transform"; + const resize = (width: number, height: number) => + page.evaluate( + (w, h) => + Object.assign(document.getElementById("player")!.style, { + width: `${w}px`, + height: `${h}px`, + }), + width, + height, + ); + // What the player sends; lets the test turn the swap off and on at one shown size. + const sendDisplayScale = (scale: number) => + frame.evaluate( + (value) => + window.postMessage( + { source: "hf-parent", type: "control", action: "set-display-scale", scale: value }, + "*", + ), + scale, + ); + const settle = () => + page.evaluate( + () => new Promise((done) => requestAnimationFrame(() => requestAnimationFrame(done))), + ); + const seek = async (time: number) => { + await page.evaluate( + (t) => (document.getElementById("player") as HTMLElement & { seek(t: number): void }).seek(t), + time, + ); + await settle(); + }; + // Every box, what paints at the stacking and 3D probes, every inline style and computed filter. + const snapshot = () => + frame.evaluate(() => { + const elements = [...document.body.querySelectorAll("*")]; + return { + boxes: elements.map((element) => { + const box = element.getBoundingClientRect(); + return [box.x, box.y, box.width, box.height]; + }), + stacking: document.elementFromPoint(1000, 800)?.id, + flipped: document.elementFromPoint(1500, 800)?.id, + styles: elements.map((element) => (element as HTMLElement).style?.cssText ?? ""), + filters: elements.map((element) => getComputedStyle(element).filter), + }; + }); + // A screenshot of a composition element's interior, 2px in from its antialiased edges. + const shot = async (id: string) => { + const [x, y, width, height] = await frame.evaluate((target) => { + const box = document.getElementById(target)!.getBoundingClientRect(); + return [box.x, box.y, box.width, box.height]; + }, id); + const [left, top, scale] = await page.evaluate(() => { + const box = document.getElementById("player")!.getBoundingClientRect(); + return [box.x, box.y, box.width / 1920]; + }); + const clip = { + x: left + x * scale + 2, + y: top + y * scale + 2, + width: width * scale - 4, + height: height * scale - 4, + }; + return page.screenshot({ encoding: "base64", clip }); + }; + const changedShare = (a: string, b: string) => + page.evaluate( + async (first, second) => { + const pixels = async (png: string) => { + const image = new Image(); + image.src = `data:image/png;base64,${png}`; + await image.decode(); + const canvas = new OffscreenCanvas(image.width, image.height); + const context = canvas.getContext("2d")!; + context.drawImage(image, 0, 0); + return context.getImageData(0, 0, image.width, image.height).data; + }; + const [p, q] = [await pixels(first), await pixels(second)]; + let changed = 0; + for (let i = 0; i < p.length; i += 4) { + const delta = [0, 1, 2].map((c) => Math.abs(p[i + c]! - q[i + c]!)); + if (Math.max(...delta) > 32) changed++; + } + return changed / (p.length / 4); + }, + a, + b, + ); + // A layer snapshot is only written on a compositor draw, so each trace seeks to force one. + const maxOversize = async (attempt: number) => { + await page.tracing.start({ categories: ["disabled-by-default-cc.debug"] }); + await seek(attempt % 6); + const trace = JSON.parse(new TextDecoder().decode(await page.tracing.stop())) as { + traceEvents: { args?: { snapshot?: { active_tree?: { layers?: Layer[] } } } }[]; + }; + const ratios = trace.traceEvents + .flatMap((event) => event.args?.snapshot?.active_tree?.layers ?? []) + .filter((layer) => layer.raster_scales?.contents_scale && layer.ideal_contents_scale) + .map((layer) => layer.raster_scales!.contents_scale![0] / layer.ideal_contents_scale!); + return Math.max(0, ...ratios); + }; + + const full = await snapshot(); + assert.equal(await frame.evaluate(hinted), true, "shown at full size, the hints stay"); + assert.equal(full.stacking, "cover", "the hinted stack keeps its z-index child under a sibling"); + assert.equal(full.flipped, "card", "the flipped card hides its front"); + + await resize(528, 297); + await until("a small preview swaps the hints", swapped); + const small = await snapshot(); + small.boxes.forEach((box, index) => + assert( + box.every((value, edge) => Math.abs(value - full.boxes[index]![edge]!) < 0.01), + `element ${index} moved from ${full.boxes[index]} to ${box}`, + ), + ); + assert.equal(small.stacking, "cover", "paint order is unchanged"); + assert.equal(small.flipped, "card", "a preserve-3d container is not flattened"); + assert.deepEqual(small.filters, full.filters, "no computed filter changes"); + assert.deepEqual(small.styles, full.styles, "no inline style is written"); + assert.deepEqual( + await frame.evaluate(() => + ["wrap", "stack", "card", "glow", "mixed", "glass-wrap", "tilt-wrap"].map( + (id) => getComputedStyle(document.getElementById(id)!).willChange, + ), + ), + ["transform", "transform", "transform", "transform", "transform, opacity", "auto", "auto"], + "only layers nothing depends on lose the hint", + ); + await frame.evaluate(() => { + const late = document.createElement("div"); + late.className = "lyric"; + late.id = "late"; + document.getElementById("lyrics")!.append(late); + }); + await until( + "a layer added later is swapped", + () => getComputedStyle(document.getElementById("late")!).willChange === "auto", + ); + await frame.evaluate(() => document.getElementById("late")!.remove()); + + // The same small preview with the swap off and on: pixels match, and only the swap rasters small. + // A capture can land before re-rastered tiles do, so each shot waits until two in a row agree. + const steadyShot = async (id: string) => { + let last = await shot(id); + for (let attempt = 0; attempt < 20; attempt++) { + const next = await shot(id); + if (next === last) return next; + last = next; + } + return assert.fail(`#${id} never stopped changing`); + }; + const shots = async () => { + await seek(0); + return [await steadyShot("glass"), await steadyShot("tilt-wrap")]; + }; + // Only the swap may differ between the two halves; layers that keep their hint on purpose go. + await frame.evaluate(() => { + for (const id of ["wrap", "stack", "stage3d", "glow", "mixed"]) { + document.getElementById(id)!.remove(); + } + }); + const swappedShots = await shots(); + // Toggling the swap restyles the page and the player answers with its real scale, so its own + // sends are held while the test decides the scale. + await page.evaluate(() => { + const player = document.getElementById("player") as HTMLElement & { _sendDisplayScale(): void }; + player._sendDisplayScale = () => {}; + }); + await sendDisplayScale(1); + await until("the swap turns off", hinted); + let unswapped = 0; + for (let attempt = 0; attempt < 20 && unswapped <= 1.5; attempt++) { + unswapped = await maxOversize(attempt); + } + assert.ok(unswapped > 1.5, `without the swap the trace shows ${unswapped}x, not oversized`); + const nativeShots = await shots(); + assert.deepEqual( + await frame.evaluate(() => + ["glass-wrap", "tilt-wrap"].map( + (id) => getComputedStyle(document.getElementById(id)!).willChange, + ), + ), + ["transform", "transform"], + "the native shots are taken with the swap off", + ); + const glass = await changedShare(swappedShots[0]!, nativeShots[0]!); + assert.ok(glass <= 0.1, `frosted glass changed in ${(glass * 100).toFixed(0)}% of its pixels`); + const tilt = await changedShare(swappedShots[1]!, nativeShots[1]!); + assert.ok(tilt <= 0.1, `a 3D child of a swapped layer changed in ${(tilt * 100).toFixed(0)}%`); + + await sendDisplayScale(528 / 1920); + await until("the swap turns back on", swapped); + for (let clean = 0, attempt = 0; clean < 3; attempt++) { + const oversize = await maxOversize(attempt); + clean = oversize <= 1.5 ? clean + 1 : 0; + assert.ok(attempt < 20, `a layer is still rastered at ${oversize.toFixed(2)}x the size shown`); + } + + // Scripts restyle inline: a child given a z-index gives its layer the hint back. + await frame.evaluate(() => { + document.getElementById("tilted")!.style.zIndex = "5"; + }); + await until( + "an inline z-index gives the hint back", + () => getComputedStyle(document.getElementById("tilt-wrap")!).willChange === "transform", + ); + + await page.evaluate(() => { + delete (document.getElementById("player") as HTMLElement & { _sendDisplayScale?: unknown }) + ._sendDisplayScale; + }); + await resize(1920, 1080); + await until("full size restores the hints", hinted); + assert.equal( + await frame.evaluate(() => document.querySelectorAll("[data-hf-preview-raster]").length), + 0, + "full size removes every mark", + ); + console.log("preview rasters at its shown size with layout and pixels unchanged: PASS"); +} finally { + await browser.close(); + await server.stop(); +} diff --git a/packages/player/tests/perf/fixtures/preview-raster/index.html b/packages/player/tests/perf/fixtures/preview-raster/index.html new file mode 100644 index 0000000000..8cfdadb07c --- /dev/null +++ b/packages/player/tests/perf/fixtures/preview-raster/index.html @@ -0,0 +1,204 @@ + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+

Hidden

+
+ + + diff --git a/packages/studio/src/components/editor/domEditingLayers.ts b/packages/studio/src/components/editor/domEditingLayers.ts index 287c0a1e68..be37659066 100644 --- a/packages/studio/src/components/editor/domEditingLayers.ts +++ b/packages/studio/src/components/editor/domEditingLayers.ts @@ -1,3 +1,4 @@ +import { PREVIEW_ONLY_ATTRS } from "@hyperframes/core/studio-preview-mark"; import { knownSourceAnswer, probeSourceElement, type ProbeTarget } from "./probeSourceElement"; import { isAudibleVideoNode } from "../../player/lib/timelineElementHelpers"; import type { PatchOperation } from "../../utils/sourcePatcher"; @@ -55,6 +56,8 @@ function getTextFieldLabel( return `Text ${index + 1}`; } +const NOT_COPIED_ATTRS = new Set(["style", ...PREVIEW_ONLY_ATTRS]); + function buildTextField( el: HTMLElement, index: number, @@ -70,7 +73,7 @@ function buildTextField( value: el.textContent ?? "", tagName, attributes: Array.from(el.attributes) - .filter((attribute) => attribute.name !== "style") + .filter((attribute) => !NOT_COPIED_ATTRS.has(attribute.name)) .map((attribute) => ({ name: attribute.name, value: attribute.value, diff --git a/packages/studio/src/components/editor/domEditingTextFields.test.ts b/packages/studio/src/components/editor/domEditingTextFields.test.ts index 5c3e7bd23a..e0732bf390 100644 --- a/packages/studio/src/components/editor/domEditingTextFields.test.ts +++ b/packages/studio/src/components/editor/domEditingTextFields.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; +import { Window } from "happy-dom"; +import { + PREVIEW_RASTER_ATTR, + STUDIO_PREVIEW_UPCOMING_ATTR, +} from "@hyperframes/core/studio-preview-mark"; import { serializeDomEditTextFields } from "./domEditing"; +import { collectDomEditTextFields } from "./domEditingLayers"; describe("serializeDomEditTextFields — mixed content", () => { it("round-trips text-node + child element fields", () => { @@ -58,3 +64,15 @@ describe("serializeDomEditTextFields — mixed content", () => { ).toBe("A < B & C > D"); }); }); + +describe("collectDomEditTextFields — preview marks", () => { + it("keeps preview-only attributes out of a saved text edit", () => { + const { document } = new Window(); + document.body.innerHTML = `

Hello world

`; + const heading = document.querySelector("h1") as unknown as HTMLElement; + const saved = serializeDomEditTextFields(collectDomEditTextFields(heading)); + expect(saved).toContain(' { + const live = document.createElement("div"); + live.innerHTML = `
LOUD
`; + live.setAttribute(PREVIEW_RASTER_ATTR, ""); + expect(liveMarkupWithoutPreviewMarks(live)).toBe( + '
LOUD
', + ); + expect(live.hasAttribute(PREVIEW_RASTER_ATTR)).toBe(true); + }); + + it("gives a lazily previewed image back its authored loading", () => { + const live = document.createElement("div"); + live.innerHTML = ``; + expect(liveMarkupWithoutPreviewMarks(live)).toBe('
'); + }); +}); diff --git a/packages/studio/src/utils/authoredSource.ts b/packages/studio/src/utils/authoredSource.ts index 70d699d898..8068d816bb 100644 --- a/packages/studio/src/utils/authoredSource.ts +++ b/packages/studio/src/utils/authoredSource.ts @@ -1,7 +1,7 @@ import { ensureHfIds, isCompositionTemplate } from "@hyperframes/parsers/hf-ids"; import { + PREVIEW_ONLY_ATTRS, STUDIO_PREVIEW_LAZY_ATTR, - STUDIO_PREVIEW_UPCOMING_ATTR, } from "@hyperframes/core/studio-preview-mark"; // Stamped as the preview stamps the files it serves, so a live element's hf-id finds its source. @@ -98,10 +98,8 @@ export function authoredMarkup(authored: Element, live: Element, sourceFile: str export function liveMarkupWithoutPreviewMarks(live: Element): string { const copy = live.cloneNode(true) as Element; for (const el of [copy, ...Array.from(copy.querySelectorAll("*"))]) { - el.removeAttribute(STUDIO_PREVIEW_UPCOMING_ATTR); - if (!el.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) continue; - el.removeAttribute("loading"); - el.removeAttribute(STUDIO_PREVIEW_LAZY_ATTR); + if (el.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) el.removeAttribute("loading"); + for (const attr of PREVIEW_ONLY_ATTRS) el.removeAttribute(attr); } return copy.outerHTML; }