diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5f8789e7fc..499eb6597c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -268,11 +268,20 @@ export { type ToneMapSourceColour, } from "./hdrToneMap"; export { + HEAVY_BITRATE, + HEAVY_PIXEL_SHARE, PREVIEW_PROXY_BOX_PARAM, + PREVIEW_PROXY_LIGHT_PARAM, + PREVIEW_QUALITY_LEVELS, formatPreviewProxyBox, + isHeavyForPreview, + isPreviewQuality, parsePreviewProxyBox, quantizePreviewProxyBox, + scalePreviewProxyBox, type PreviewProxyBox, + type PreviewQuality, + type PreviewSourceFacts, } from "./previewProxyBox"; // Inline scripts diff --git a/packages/core/src/previewProxyBox.test.ts b/packages/core/src/previewProxyBox.test.ts index 3e1a2bab14..183adaab73 100644 --- a/packages/core/src/previewProxyBox.test.ts +++ b/packages/core/src/previewProxyBox.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { parsePreviewProxyBox, quantizePreviewProxyBox } from "./previewProxyBox"; +import { + isHeavyForPreview, + isPreviewQuality, + parsePreviewProxyBox, + quantizePreviewProxyBox, + scalePreviewProxyBox, + type PreviewSourceFacts, +} from "./previewProxyBox"; describe("preview proxy box", () => { it("rounds each side up to the next rung", () => { @@ -28,3 +35,34 @@ describe("preview proxy box", () => { expect(parsePreviewProxyBox("")).toBeNull(); }); }); + +describe("lighter copies of files Chrome can play", () => { + const full: PreviewSourceFacts = { width: 1920, height: 1080, bitDepth: 8, bitrate: 10_000_000 }; + const box = { width: 1218, height: 724 }; + const shown = { width: 1448, height: 862 }; + + it("multiplies the shown box by the level, on the ladder", () => { + expect(scalePreviewProxyBox(box, 1)).toEqual(box); + expect(scalePreviewProxyBox(box, 0.5)).toEqual({ width: 724, height: 362 }); + expect(scalePreviewProxyBox(box, 0.125)).toEqual({ width: 256, height: 256 }); + }); + + it("calls a file heavy for a much larger source, 10-bit or deeper, or 40 Mbit/s and up", () => { + expect(isHeavyForPreview(full, shown)).toBe(false); + expect(isHeavyForPreview({ ...full, width: 3840, height: 2160 }, shown)).toBe(true); + expect(isHeavyForPreview({ ...full, bitDepth: 10 }, shown)).toBe(true); + expect(isHeavyForPreview({ ...full, bitrate: 39_999_999 }, shown)).toBe(false); + expect(isHeavyForPreview({ ...full, bitrate: 40_000_000 }, shown)).toBe(true); + }); + + it("calls a source heavy when the copy box has half its pixels or fewer", () => { + const half = { width: 1448, height: 724 }; + expect(isHeavyForPreview({ ...full, width: 2896, height: 1448 }, half)).toBe(true); + expect(isHeavyForPreview({ ...full, width: 1448, height: 1000 }, half)).toBe(false); + }); + + it("knows the four levels and nothing else", () => { + expect([1, 0.5, 0.25, 0.125].every(isPreviewQuality)).toBe(true); + expect([0, 0.75, 2, "1", null].some(isPreviewQuality)).toBe(false); + }); +}); diff --git a/packages/core/src/previewProxyBox.ts b/packages/core/src/previewProxyBox.ts index 46016422a1..bb6d4d4f85 100644 --- a/packages/core/src/previewProxyBox.ts +++ b/packages/core/src/previewProxyBox.ts @@ -1,6 +1,8 @@ /** Query parameter carrying the largest box, in device pixels, a preview shows a proxied video at. */ export const PREVIEW_PROXY_BOX_PARAM = "hf-proxy-box"; +export const PREVIEW_PROXY_LIGHT_PARAM = "hf-proxy-light"; + export type PreviewProxyBox = { width: number; height: number }; // 256 * 2^(k/4), even: nearby boxes share one cached copy, and a rung overshoots by at most ~19%. @@ -33,3 +35,32 @@ export function parsePreviewProxyBox(raw: string): PreviewProxyBox | null { const height = Number(match[2]); return RUNGS.includes(width) && RUNGS.includes(height) ? { width, height } : null; } + +export const PREVIEW_QUALITY_LEVELS = [1, 0.5, 0.25, 0.125] as const; +export type PreviewQuality = (typeof PREVIEW_QUALITY_LEVELS)[number]; + +export function isPreviewQuality(value: unknown): value is PreviewQuality { + return PREVIEW_QUALITY_LEVELS.some((level) => level === value); +} + +export type PreviewSourceFacts = { + width: number; + height: number; + bitDepth: number; + bitrate: number; +}; + +export const HEAVY_PIXEL_SHARE = 0.5; +export const HEAVY_BITRATE = 40_000_000; + +export function scalePreviewProxyBox(box: PreviewProxyBox, level: PreviewQuality): PreviewProxyBox { + return quantizePreviewProxyBox(box.width * level, box.height * level) ?? box; +} + +export function isHeavyForPreview(facts: PreviewSourceFacts, copyBox: PreviewProxyBox): boolean { + return ( + copyBox.width * copyBox.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height || + facts.bitDepth > 8 || + facts.bitrate >= HEAVY_BITRATE + ); +} diff --git a/packages/core/src/runtime/bridge.ts b/packages/core/src/runtime/bridge.ts index a5a5e89bbd..f665d07d23 100644 --- a/packages/core/src/runtime/bridge.ts +++ b/packages/core/src/runtime/bridge.ts @@ -33,6 +33,7 @@ type BridgeDeps = { onSetRuntimeData?: (channel: string, payload: unknown, requestId?: number) => void; onClearRuntimeData?: (channel: string, requestId?: number) => void; onSetDisplayScale?: (scale: number) => void; + onSetPreviewQuality?: (level: number) => void; getCanonicalFps: () => number; }; @@ -76,6 +77,7 @@ const CONTROL_HANDLERS = new Map( "set-playback-rate": (data, deps) => deps.onSetPlaybackRate(Number(data.playbackRate ?? 1)), "set-idle-heartbeat": (data, deps) => deps.onSetIdleHeartbeat(Boolean(data.slow)), "set-display-scale": (data, deps) => deps.onSetDisplayScale?.(Number(data.scale)), + "set-preview-quality": (data, deps) => deps.onSetPreviewQuality?.(Number(data.level)), "set-root-duration": (data, deps) => deps.onSetRootDuration(Number(data.durationSeconds ?? 0)), "set-play-range": (data, deps) => deps.onSetPlayRange( diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 7550fe35ed..b015a34af9 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -95,6 +95,7 @@ import { handleMetadataForProxy, isRenderMode, maybeProxyProactively, + setPreviewQuality, setProxyDisplayScale, } from "./mediaProxy"; import { @@ -5973,6 +5974,7 @@ export function initSandboxRuntimeModular(): void { setProxyDisplayScale(scale); setPreviewRasterScale(scale); }, + onSetPreviewQuality: setPreviewQuality, onSetIdleHeartbeat: (slow) => { slowIdleHeartbeat = slow; wakeTransport(); diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts index 3a74daef82..e17816da34 100644 --- a/packages/core/src/runtime/mediaProxy.test.ts +++ b/packages/core/src/runtime/mediaProxy.test.ts @@ -869,3 +869,133 @@ describe("a copy sized to how large the video shows", () => { expect(boxOf(el)).toBe("2436x4096"); }); }); + +describe("preview quality: lighter copies of heavy files", () => { + const HEAVY = { width: 3840, height: 2160, bitDepth: 8, bitrate: 10_000_000 }; + const LIGHT = { width: 1280, height: 720, bitDepth: 8, bitrate: 10_000_000 }; + + // A framed page showing a 1920x1080 stage at full size on a 1x screen: a 2048x1218 box. + async function stage(facts: Record, codecMap?: Record) { + vi.resetModules(); + vi.spyOn(window, "parent", "get").mockReturnValue({} as Window); + const mod = await import("./mediaProxy"); + window.__HF_MEDIA_SOURCE_FACTS__ = facts; + if (codecMap) window.__HF_MEDIA_CODEC_MAP__ = codecMap as never; + Object.defineProperty(document.documentElement, "clientWidth", { + configurable: true, + value: 1920, + }); + Object.defineProperty(document.documentElement, "clientHeight", { + configurable: true, + value: 1080, + }); + vi.spyOn(window, "devicePixelRatio", "get").mockReturnValue(1); + const video = (src = "/video.mp4") => { + const el = createVideo(src); + stubCanPlayType(el, "probably"); + el.getBoundingClientRect = () => ({ width: 1920, height: 1080 }) as DOMRect; + return el; + }; + mod.setProxyDisplayScale(1); + return { ...mod, video }; + } + + const params = (el: HTMLMediaElement) => new URL(el.src, document.baseURI).searchParams; + + afterEach(() => { + delete (window as { __HF_MEDIA_SOURCE_FACTS__?: unknown }).__HF_MEDIA_SOURCE_FACTS__; + }); + + it("changes nothing at full quality", async () => { + const { video, setPreviewQuality, maybeProxyProactively } = await stage({ + "/video.mp4": HEAVY, + }); + const el = video(); + maybeProxyProactively(el); + setPreviewQuality(1); + await settle(); + + expect(isProxied(el)).toBe(false); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("swaps a heavy file Chrome can play to a copy at the shown box times the level", async () => { + const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY }); + const el = video(); + + setPreviewQuality(0.5); + await settle(); + + expect(params(el).get("hf-proxy")).toBe("auto"); + expect(params(el).get("hf-proxy-box")).toBe("1024x724"); + expect(params(el).get("hf-proxy-light")).toBe("1"); + }); + + it("leaves a file alone that is not heavy at that box", async () => { + const { video, setPreviewQuality } = await stage({ "/video.mp4": LIGHT }); + const el = video(); + + setPreviewQuality(0.5); + await settle(); + + expect(isProxied(el)).toBe(false); + }); + + it("puts the original back at full quality", async () => { + const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY }); + const el = video(); + setPreviewQuality(0.5); + await settle(); + expect(isProxied(el)).toBe(true); + + setPreviewQuality(1); + + expect(el.getAttribute("src")).toBe("/video.mp4"); + expect(el.load).toHaveBeenCalledTimes(2); + }); + + it("lightens a video that appears after the level was lowered", async () => { + const { video, setPreviewQuality, maybeProxyProactively } = await stage({ + "/video.mp4": HEAVY, + }); + setPreviewQuality(0.25); + const el = video(); + + maybeProxyProactively(el); + await settle(); + + expect(params(el).get("hf-proxy-box")).toBe("512x362"); + }); + + it("sizes the copy a file cannot play without by the level too, and resizes it as the level moves", async () => { + const { video, setPreviewQuality, maybeProxyProactively } = await stage( + { "/video.mp4": HEAVY }, + { "/video.mp4": HEVC_ENTRY }, + ); + const el = video(); + stubCanPlayType(el, ""); + maybeProxyProactively(el); + await settle(); + expect(params(el).get("hf-proxy-box")).toBe("2048x1218"); + expect(params(el).has("hf-proxy-light")).toBe(false); + + setPreviewQuality(0.5); + await settle(); + expect(params(el).get("hf-proxy-box")).toBe("1024x724"); + expect(params(el).has("hf-proxy-light")).toBe(false); + + setPreviewQuality(1); + await settle(); + expect(params(el).get("hf-proxy-box")).toBe("2048x1218"); + }); + + it("ignores a level that is not one of the four", async () => { + const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY }); + const el = video(); + + setPreviewQuality(0.7); + await settle(); + + expect(isProxied(el)).toBe(false); + }); +}); diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts index 33aed24341..0b99fe50e6 100644 --- a/packages/core/src/runtime/mediaProxy.ts +++ b/packages/core/src/runtime/mediaProxy.ts @@ -9,9 +9,15 @@ import { waitForServedProxy } from "./proxyWait"; import { registerSeekCompletion } from "./adapters/seek-dispatch"; import { formatPreviewProxyBox, + isHeavyForPreview, + isPreviewQuality, PREVIEW_PROXY_BOX_PARAM, + PREVIEW_PROXY_LIGHT_PARAM, quantizePreviewProxyBox, + scalePreviewProxyBox, type PreviewProxyBox, + type PreviewQuality, + type PreviewSourceFacts, } from "../previewProxyBox"; /** @@ -34,6 +40,7 @@ export type MediaCodecMapEntry = { declare global { interface Window { __HF_MEDIA_CODEC_MAP__?: Record; + __HF_MEDIA_SOURCE_FACTS__?: Record; } } @@ -45,7 +52,7 @@ const DIAGNOSTIC_FALLBACK_CODE = "runtime_media_proxy_fallback"; * did) proxy it — a remote asset, or the proxy URL itself failing. */ const DIAGNOSTIC_UNAVAILABLE_CODE = "runtime_media_proxy_unavailable"; -type ProxyTrigger = "proactive" | "reactive" | "tertiary"; +type ProxyTrigger = "proactive" | "reactive" | "tertiary" | "light"; // `swappedElements` gates every trigger, so a second undecodable-video signal never re-swaps. @@ -63,6 +70,10 @@ function currentSrcValue(el: HTMLMediaElement): string { return el.currentSrc || el.src; } +// Below 1, heavy files get lighter copies at the shown box times this level; 1 is the preview as it always was. +let previewQuality: PreviewQuality = 1; +const lightElements = new WeakSet(); + // How large the host shows this document, in its CSS pixels per ours (player fit x page zoom). // A top-level page shows itself at 1; a framed one waits for its host to say. let displayScale: number | null = window.parent === window ? 1 : null; @@ -72,7 +83,13 @@ const displayScaleKnown = new Promise((arrived) => (displayScaleArrived = const DISPLAY_SCALE_WAIT_MS = 1000; // Elements on a size-bounded copy: the source, the copy's URL and the box it was made for. -type BoxedCopy = { originalSrc: string; variant: string; href: string; box: PreviewProxyBox }; +type BoxedCopy = { + originalSrc: string; + variant: string; + href: string; + box: PreviewProxyBox; + light: boolean; +}; const boxedElements = new Map(); // The larger copy each element waits for; a newer ask replaces it and the older wait stops. const pendingUpgrades = new WeakMap(); @@ -106,6 +123,19 @@ export function setProxyDisplayScale(scale: number): void { for (const el of boxedElements.keys()) keepSized(el, scale); } +function codecEntryOf(el: HTMLMediaElement): MediaCodecMapEntry | null { + const map = window.__HF_MEDIA_CODEC_MAP__; + const key = deriveCodecMapKey(el); + return map && key !== null ? lookupCodecMapEntry(key, map) : null; +} + +function sourceFactsOf(el: HTMLMediaElement): PreviewSourceFacts | null { + const facts = window.__HF_MEDIA_SOURCE_FACTS__; + const key = deriveCodecMapKey(el); + return facts && key !== null ? lookupCodecMapEntry(key, facts) : null; +} + +// fallow-ignore-next-line complexity function keepSized(el: HTMLMediaElement, scale: number): void { const copy = boxedElements.get(el); if (!copy) return; @@ -113,32 +143,68 @@ function keepSized(el: HTMLMediaElement, scale: number): void { boxedElements.delete(el); return; } - const needed = shownBox(el, scale); - if (needed && needed.width <= copy.box.width && needed.height <= copy.box.height) return; - // Never smaller on either side than the copy it replaces. - const box = needed && { - width: Math.max(needed.width, copy.box.width), - height: Math.max(needed.height, copy.box.height), + const shown = shownBox(el, scale); + const lowered = previewQuality < 1; + const needed = shown && lowered ? scalePreviewProxyBox(shown, previewQuality) : shown; + if (lowered && !needed) return; + if (needed && lowered && copy.light && lightCopyNoLongerNeeded(el, needed)) { + return restoreOriginal(el); + } + if (boxCovers(copy.box, needed, lowered)) return; + requestBox(el, copy, lowered || !needed ? needed : growOnly(needed, copy.box)); +} + +function lightCopyNoLongerNeeded(el: HTMLMediaElement, needed: PreviewProxyBox): boolean { + const facts = sourceFactsOf(el); + return facts !== null && !isHeavyForPreview(facts, needed); +} + +function boxCovers(current: PreviewProxyBox, needed: PreviewProxyBox | null, exact: boolean) { + if (exact) return needed?.width === current.width && needed.height === current.height; + return needed !== null && needed.width <= current.width && needed.height <= current.height; +} + +function growOnly(needed: PreviewProxyBox, current: PreviewProxyBox): PreviewProxyBox { + return { + width: Math.max(needed.width, current.width), + height: Math.max(needed.height, current.height), }; - const href = appendProxyParam(copy.originalSrc, copy.variant, box); +} + +function requestBox(el: HTMLMediaElement, copy: BoxedCopy, box: PreviewProxyBox | null): void { + const href = appendProxyParam(copy.originalSrc, copy.variant, box, copy.light); if (pendingUpgrades.get(el) === href) return; pendingUpgrades.set(el, href); const wanted = () => pendingUpgrades.get(el) === href && el.isConnected && el.getAttribute("src") === copy.href; void waitForServedProxy(href, wanted).then((served) => { - if (served && wanted()) loadProxy(el, href, copy.originalSrc, copy.variant, box); + if (served && wanted()) loadProxy(el, href, copy.originalSrc, copy.variant, box, copy.light); if (pendingUpgrades.get(el) === href) pendingUpgrades.delete(el); }); } +/** Puts a video that only got a lighter copy back on its original. */ +function restoreOriginal(el: HTMLMediaElement): void { + const original = swappedElements.get(el) ?? null; + for (const state of [boxedElements, pendingUpgrades, proxyRequested, swappedElements]) { + state.delete(el); + } + lightElements.delete(el); + evictMediaSyncState(el); + if (original === null) el.removeAttribute("src"); + else el.setAttribute("src", original); + el.load(); +} + function loadProxy( el: HTMLMediaElement, href: string, originalSrc: string, variant: string, box: PreviewProxyBox | null, + light = false, ): void { - if (box) boxedElements.set(el, { originalSrc, variant, href, box }); + if (box) boxedElements.set(el, { originalSrc, variant, href, box, light }); else boxedElements.delete(el); // Sync state measured on the previous file would read the new file's buffering as drift. evictMediaSyncState(el); @@ -202,10 +268,7 @@ export function deriveCodecMapKey(el: HTMLMediaElement): string | null { * ending in `"/notfoo.mp4"`, since that would require the literal substring * `"/foo.mp4"` to appear where the last `/` already fell a segment later). */ -function lookupLongestExactSuffix( - pathname: string, - map: Record, -): MediaCodecMapEntry | null { +function lookupLongestExactSuffix(pathname: string, map: Record): T | null { let bestKey: string | null = null; for (const key of Object.keys(map)) { if (!pathname.endsWith(key)) continue; @@ -214,10 +277,7 @@ function lookupLongestExactSuffix( return bestKey ? (map[bestKey] ?? null) : null; } -function lookupNormalizedSuffix( - pathname: string, - map: Record, -): MediaCodecMapEntry | null { +function lookupNormalizedSuffix(pathname: string, map: Record): T | null { // Case-insensitive filesystems and Unicode-normalizing filesystems can // serve an authored URL whose spelling differs from the canonical path // ffprobe returned (Clip.mp4 vs clip.mp4, NFC vs NFD). Exact matching stays @@ -226,7 +286,7 @@ function lookupNormalizedSuffix( // spellings never select the wrong asset. const normalizedPathname = pathname.normalize("NFC").toLowerCase(); let normalizedBestLength = -1; - let normalizedBest: MediaCodecMapEntry | null = null; + let normalizedBest: T | null = null; let ambiguous = false; for (const [key, entry] of Object.entries(map)) { const normalizedKey = key.normalize("NFC").toLowerCase(); @@ -242,10 +302,7 @@ function lookupNormalizedSuffix( return ambiguous ? null : normalizedBest; } -function lookupCodecMapEntry( - pathname: string, - map: Record, -): MediaCodecMapEntry | null { +function lookupCodecMapEntry(pathname: string, map: Record): T | null { return ( map[pathname] ?? lookupLongestExactSuffix(pathname, map) ?? @@ -253,10 +310,16 @@ function lookupCodecMapEntry( ); } -function appendProxyParam(src: string, variant: string, box: PreviewProxyBox | null): string { +function appendProxyParam( + src: string, + variant: string, + box: PreviewProxyBox | null, + light = false, +): string { const url = new URL(src, document.baseURI); url.searchParams.set(PROXY_QUERY_PARAM, variant); if (box) url.searchParams.set(PREVIEW_PROXY_BOX_PARAM, formatPreviewProxyBox(box)); + if (light && box) url.searchParams.set(PREVIEW_PROXY_LIGHT_PARAM, "1"); return url.href; } @@ -300,6 +363,49 @@ function emitUnavailableDiagnostic( console.info(`[hyperframes] ${DIAGNOSTIC_UNAVAILABLE_CODE}: "${asset}" (${reason}): ${note}`); } +type CopyJob = { + originalSrc: string; + originalAttr: string | null; + variant: string; + lightFacts?: PreviewSourceFacts; + level: PreviewQuality; + live: () => boolean; +}; + +// fallow-ignore-next-line complexity +async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null) { + const { originalSrc, originalAttr, variant, lightFacts, level, live } = job; + // Asked once the shown size is known, so the copy is made at that size and only once. + const shown = scale === null ? null : shownBox(el, scale); + const box = shown && level < 1 ? scalePreviewProxyBox(shown, level) : shown; + if (lightFacts && !(box && isHeavyForPreview(lightFacts, box))) { + if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el); + return; + } + const proxiedSrc = appendProxyParam(originalSrc, variant, box, lightFacts !== undefined); + const served = await waitForServedProxy(proxiedSrc, live); + if (!live()) { + if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el); + if (lightFacts && el.isConnected && el.getAttribute("src") === originalAttr) { + applyPreviewQuality(el); + } + return; + } + if (!served) { + emitUnavailableDiagnostic(el, "proxy_playback_failed", originalSrc); + return; + } + swappedElements.set(el, originalAttr); + if (lightFacts) lightElements.add(el); + loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined); + // The shown size may have grown while this copy was being made. + if (displayScale !== null) keepSized(el, displayScale); + return new Promise((landed) => { + el.addEventListener("loadeddata", landed, { once: true }); + el.addEventListener("error", landed, { once: true }); + }); +} + /** * Swap `el` to its alpha-aware proxy URL, evict stale per-source sync state, and * emit the one-time diagnostic + console line. Safe to call from any of the @@ -309,6 +415,7 @@ export function swapToProxy( el: HTMLMediaElement, entry: MediaCodecMapEntry | null = null, trigger: ProxyTrigger = "reactive", + lightFacts?: PreviewSourceFacts, ): void { if (swappedElements.has(el)) return; if (proxyRequested.has(el) && proxyRequested.get(el) === el.getAttribute("src")) return; @@ -323,29 +430,15 @@ export function swapToProxy( } const originalAttr = el.getAttribute("src"); proxyRequested.set(el, originalAttr); - const live = () => el.isConnected && el.getAttribute("src") === originalAttr; - const swap = shownScale().then(async (scale) => { - // Asked once the shown size is known, so the copy is made at that size and only once. - const box = scale === null ? null : shownBox(el, scale); - const proxiedSrc = appendProxyParam(originalSrc, variant, box); - const served = await waitForServedProxy(proxiedSrc, live); - if (!live()) { - if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el); - return; - } - if (!served) { - emitUnavailableDiagnostic(el, "proxy_playback_failed", originalSrc); - return; - } - swappedElements.set(el, originalAttr); - loadProxy(el, proxiedSrc, originalSrc, variant, box); - // The shown size may have grown while this copy was being made. - if (displayScale !== null) keepSized(el, displayScale); - return new Promise((landed) => { - el.addEventListener("loadeddata", landed, { once: true }); - el.addEventListener("error", landed, { once: true }); - }); - }); + const level = previewQuality; + // A lighter copy waits for no level but its own; a copy the file cannot play without finishes at any level. + const live = () => + el.isConnected && + el.getAttribute("src") === originalAttr && + (!lightFacts || previewQuality === level); + const job: CopyJob = { originalSrc, originalAttr, variant, lightFacts, level, live }; + const swap = shownScale().then((scale) => makeCopy(el, job, scale)); + if (lightFacts) return; // A frame capture holds for the copy as it held for a loading video before, up to the same cap. registerSeekCompletion( Promise.race([swap, new Promise((cap) => setTimeout(cap, HOLD_CAP_MS))]), @@ -383,15 +476,31 @@ export function maybeProxyProactively(el: HTMLMediaElement): void { if (isRenderMode(el)) return; if (!isVideoElement(el)) return; if (swappedElements.has(el)) return; - const map = window.__HF_MEDIA_CODEC_MAP__; - if (!map) return; - const key = deriveCodecMapKey(el); - if (key === null) return; - const entry = lookupCodecMapEntry(key, map); - if (!entry || !entry.browserHostile) return; - const canPlay = entry.representativeMime ? el.canPlayType(entry.representativeMime) : ""; - if (canPlay === "probably" || canPlay === "maybe") return; - swapToProxy(el, entry, "proactive"); + const entry = codecEntryOf(el); + const canPlay = entry?.representativeMime ? el.canPlayType(entry.representativeMime) : ""; + const playable = canPlay === "probably" || canPlay === "maybe"; + if (entry?.browserHostile && !playable) swapToProxy(el, entry, "proactive"); + else if (previewQuality < 1) lightenIfHeavy(el, entry); +} + +function lightenIfHeavy(el: HTMLMediaElement, entry: MediaCodecMapEntry | null): void { + if (swappedElements.has(el) || proxyRequested.has(el)) return; + const facts = sourceFactsOf(el); + if (facts) swapToProxy(el, entry, "light", facts); +} + +function applyPreviewQuality(el: HTMLMediaElement): void { + if (isRenderMode(el) || !isVideoElement(el)) return; + if (previewQuality === 1 && lightElements.has(el)) return restoreOriginal(el); + if (boxedElements.has(el) && displayScale !== null) return keepSized(el, displayScale); + if (previewQuality < 1) lightenIfHeavy(el, codecEntryOf(el)); +} + +/** The host's preview quality: below 1, heavy videos play lighter copies sized to the shown box times the level. */ +export function setPreviewQuality(level: number): void { + if (!isPreviewQuality(level) || level === previewQuality) return; + previewQuality = level; + for (const el of document.querySelectorAll("video")) applyPreviewQuality(el); } /** diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.ts index 35ed38aea7..d1f4e27f76 100644 --- a/packages/core/src/runtime/types.ts +++ b/packages/core/src/runtime/types.ts @@ -22,6 +22,7 @@ type RuntimeBridgeControlActionBase = | "set-web-audio-media-disabled" | "set-idle-heartbeat" | "set-display-scale" + | "set-preview-quality" | "set-root-duration" | "set-play-range" | "stop-media" @@ -41,6 +42,7 @@ type RuntimeBridgeControlMessageBase = { disabled?: boolean; slow?: boolean; scale?: number; + level?: number; playbackRate?: number; target?: HfColorGradingTarget | string | null; grading?: RuntimeJson; diff --git a/packages/studio-server/src/helpers/mediaCodecMap.test.ts b/packages/studio-server/src/helpers/mediaCodecMap.test.ts index 3ab8ab549e..60caaaa8e0 100644 --- a/packages/studio-server/src/helpers/mediaCodecMap.test.ts +++ b/packages/studio-server/src/helpers/mediaCodecMap.test.ts @@ -10,6 +10,7 @@ import { probeAssetCodec, proxyVariantFor, resolveProxyVariantRequest, + scanProjectMedia, scanProjectMediaCodecMap, shouldPrewarmProxy, type AssetCodecFacts, @@ -24,7 +25,14 @@ const FAKE_FFPROBE_PATH = process.execPath; function makeRunner( codecByPath: Record< string, - string | { codecName: string; pixFmt?: string; tags?: Record } + | string + | { + codecName: string; + pixFmt?: string; + tags?: Record; + size?: [number, number]; + bitRate?: string; + } >, ): FfprobeRunner { return (_command, args) => { @@ -38,6 +46,9 @@ function makeRunner( codec_name: normalized.codecName, pix_fmt: normalized.pixFmt, tags: normalized.tags, + width: normalized.size?.[0], + height: normalized.size?.[1], + bit_rate: normalized.bitRate, }, ] : []; @@ -328,6 +339,68 @@ describe("decideMediaProxyEligibility", () => { }); }); +describe("a lighter copy of a file Chrome can play", () => { + const playable = { + codecName: "h264", + browserHostile: false, + representativeMime: null, + hasAlpha: false, + source: { width: 3840, height: 2160, bitDepth: 8, bitrate: 10_000_000 }, + }; + + it("is allowed only for the box it is asked at, and only when the file is heavy there", () => { + expect(decideMediaProxyEligibility(playable)).toEqual({ + eligible: false, + reason: "browser_safe_codec", + }); + expect(decideMediaProxyEligibility(playable, { width: 724, height: 362 })).toEqual({ + eligible: true, + }); + expect( + decideMediaProxyEligibility( + { ...playable, source: { ...playable.source, width: 1920, height: 1080 } }, + { width: 2048, height: 1218 }, + ), + ).toEqual({ eligible: false, reason: "browser_safe_codec" }); + }); + + it("reads the size, bit depth and bitrate off the probe, and keeps them with the codec facts", async () => { + const project = tmpProject(); + const videoPath = join(project, "cam.mov"); + writeFileSync(videoPath, "fake video bytes"); + + const facts = await probeAssetCodec( + videoPath, + makeRunner({ + [videoPath]: { + codecName: "hevc", + pixFmt: "yuv420p10le", + size: [3840, 2160], + bitRate: "60000000", + }, + }), + ); + + expect(facts?.source).toEqual({ width: 3840, height: 2160, bitDepth: 10, bitrate: 60_000_000 }); + }); + + it("scans the sizes of the playable videos too, while the codec map keeps only the hostile ones", async () => { + const project = tmpProject(); + writeFileSync(join(project, "cam.mp4"), "fake video bytes"); + writeFileSync(join(project, "broll.mov"), "fake video bytes"); + + const scan = await scanProjectMedia(project, [{ html: videoHtml("cam.mp4", "broll.mov") }], { + runner: makeRunner({ + [join(project, "cam.mp4")]: { codecName: "h264", size: [1920, 1080] }, + [join(project, "broll.mov")]: { codecName: "hevc", size: [3840, 2160] }, + }), + }); + + expect(Object.keys(scan.codecMap)).toEqual(["/broll.mov"]); + expect(Object.keys(scan.sourceFacts).sort()).toEqual(["/broll.mov", "/cam.mp4"]); + }); +}); + describe("proxyVariantFor", () => { it("uses VP8 for alpha and H.264 otherwise", () => { const facts = { diff --git a/packages/studio-server/src/helpers/mediaCodecMap.ts b/packages/studio-server/src/helpers/mediaCodecMap.ts index 8966e22422..c14e43335c 100644 --- a/packages/studio-server/src/helpers/mediaCodecMap.ts +++ b/packages/studio-server/src/helpers/mediaCodecMap.ts @@ -9,6 +9,11 @@ import { isUnresolvedAssetPlaceholder, resolveExistingLocalAsset as resolveAsset, } from "@hyperframes/parsers/asset-resolution"; +import { + isHeavyForPreview, + type PreviewProxyBox, + type PreviewSourceFacts, +} from "@hyperframes/core"; import { probeMediaMetadata, type FfprobeRunner } from "./mediaMetadata.js"; /** @@ -27,6 +32,8 @@ export interface AssetCodecFacts { /** Source carries an alpha channel (`MediaMetadata.hasAlpha`). Alpha sources use a * VP8/WebM proxy so their transparency is preserved across Chromium builds. */ hasAlpha: boolean; + /** Size, bit depth and bitrate, for deciding whether a playable file deserves a lighter copy. */ + source?: PreviewSourceFacts; } /** Server-root-relative URL pathname -> that asset's codec facts. */ @@ -176,20 +183,34 @@ export type MediaProxyEligibility = | { eligible: true } | { eligible: false; reason: MediaProxyIneligibilityReason }; -/** Single policy gate shared by proactive scans and on-demand proxy routes. */ -export function decideMediaProxyEligibility(facts: AssetCodecFacts | null): MediaProxyEligibility { +/** + * Single policy gate shared by proactive scans and on-demand proxy routes. A file Chrome can play is + * eligible only for a lighter copy (`lightBox`, the box that request asks for) and only when it is heavy there. + */ +export function decideMediaProxyEligibility( + facts: AssetCodecFacts | null, + lightBox?: PreviewProxyBox, +): MediaProxyEligibility { if (!facts) return { eligible: false, reason: "unknown_codec" }; - if (!facts.browserHostile) return { eligible: false, reason: "browser_safe_codec" }; - return { eligible: true }; + if (facts.browserHostile) return { eligible: true }; + if (lightBox && facts.source && isHeavyForPreview(facts.source, lightBox)) { + return { eligible: true }; + } + return { eligible: false, reason: "browser_safe_codec" }; } -function codecFactsFor(codecName: string, hasAlpha: boolean): AssetCodecFacts { +function codecFactsFor( + codecName: string, + hasAlpha: boolean, + source?: PreviewSourceFacts, +): AssetCodecFacts { const hostile = hostileCodecEntry(codecName); return { codecName, browserHostile: hostile !== undefined, representativeMime: hostile?.representativeMime ?? null, hasAlpha, + ...(source && { source }), }; } @@ -205,7 +226,7 @@ async function probeCodecFacts( if (metadata.kind !== "video") return null; const codecName = metadata.color.codecName; if (!codecName) return null; - return codecFactsFor(codecName, metadata.hasAlpha === true); + return codecFactsFor(codecName, metadata.hasAlpha === true, metadata.source); } interface CachedAssetProbe { @@ -347,20 +368,12 @@ export interface ScanProjectMediaCodecMapOptions { runner?: FfprobeRunner; } -/** - * Scans a project's composition HTML for local `