From 2f0ade181585323ebb2b53a2e15d68098655c90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 14 Nov 2026 08:09:33 +0000 Subject: [PATCH] fix(core): at full quality a 4K clip plays a copy at the size the preview shows it (#7723) * fix(core): at full quality a 4K clip plays a copy at the size the preview shows it * fix(core): names carry the copy rule, not comments * fix(core): a clip whose copy fails is left held as it was, so its decoder is still freed when idle * fix(core): the copy hold lives with the copy code, keeping core inside its size budget --- packages/core/src/previewProxyBox.ts | 16 ++- packages/core/src/runtime/mediaProxy.test.ts | 139 +++++++++++++++++-- packages/core/src/runtime/mediaProxy.ts | 83 ++++++++--- packages/core/src/runtime/preloadMedia.ts | 3 +- packages/core/src/runtime/proxySrc.ts | 8 +- 5 files changed, 211 insertions(+), 38 deletions(-) diff --git a/packages/core/src/previewProxyBox.ts b/packages/core/src/previewProxyBox.ts index be011b27cc..1b27e536de 100644 --- a/packages/core/src/previewProxyBox.ts +++ b/packages/core/src/previewProxyBox.ts @@ -66,10 +66,20 @@ export function scalePreviewProxyBox(box: PreviewProxyBox, level: PreviewQuality return quantizePreviewProxyBox(box.width * level, box.height * level) ?? box; } +export function isOversizedForPreview(facts: PreviewSourceFacts, box: PreviewProxyBox): boolean { + return box.width * box.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height; +} + export function isHeavyForPreview(facts: PreviewSourceFacts, copyBox: PreviewProxyBox): boolean { return ( - copyBox.width * copyBox.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height || - facts.bitDepth > 8 || - facts.bitrate >= HEAVY_BITRATE + isOversizedForPreview(facts, copyBox) || facts.bitDepth > 8 || facts.bitrate >= HEAVY_BITRATE ); } + +export function wantsLighterCopy( + facts: PreviewSourceFacts, + box: PreviewProxyBox, + level: PreviewQuality, +): boolean { + return level < 1 ? isHeavyForPreview(facts, box) : isOversizedForPreview(facts, box); +} diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts index af637fa8ca..60ef1fdf08 100644 --- a/packages/core/src/runtime/mediaProxy.test.ts +++ b/packages/core/src/runtime/mediaProxy.test.ts @@ -1044,6 +1044,8 @@ describe("a copy sized to how large the video shows", () => { describe("preview quality: lighter copies of heavy files", () => { const HEAVY = { width: 3840, height: 2160, bitDepth: 8, bitrate: 10_000_000 }; const LIGHT = { width: 1280, height: 720, bitDepth: 8, bitrate: 10_000_000 }; + // Heavy at a lower level's box, not oversized for the full 2048x1218 box. + const MID = { width: 2560, height: 1440, bitDepth: 8, bitrate: 10_000_000 }; // A framed page showing a 1920x1080 stage at full size on a 1x screen: a 2048x1218 box. async function stage(facts: Record, codecMap?: Record) { @@ -1193,19 +1195,135 @@ describe("preview quality: lighter copies of heavy files", () => { }); }); - it("changes nothing at full quality", async () => { - const { video, setPreviewQuality, maybeProxyProactively } = await stage({ - "/video.mp4": HEAVY, + it("changes nothing at full quality for a file not oversized for its box, 10-bit included", async () => { + const { video, maybeProxyProactively } = await stage({ + "/mid.mp4": MID, + "/ten-bit.mp4": { ...MID, bitDepth: 10 }, }); - const el = video(); - maybeProxyProactively(el); - setPreviewQuality(1); + const els = [video("/mid.mp4"), video("/ten-bit.mp4")]; + for (const el of els) maybeProxyProactively(el); await settle(); - expect(isProxied(el)).toBe(false); + expect(els.map(isProxied)).toEqual([false, false]); expect(fetch).not.toHaveBeenCalled(); }); + describe("at full quality, a file with at least twice the pixels it shows", () => { + async function waitingForItsCopy(answer: () => Response) { + vi.useFakeTimers(); + let copyAsks = 0; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => { + if (!String(url).includes("hf-proxy")) return new Response("x", { status: 206 }); + copyAsks += 1; + return answer(); + }), + ); + const staged = await stage({ "/video.mp4": HEAVY }); + const proxySrc = await import("./proxySrc"); + const preload = await import("./preloadMedia"); + const el = staged.video(); + staged.maybeProxyProactively(el); + await vi.advanceTimersByTimeAsync(0); + return { ...staged, el, proxySrc, preload, asks: () => copyAsks }; + } + const making = () => new Response(null, { status: 202, headers: { "Retry-After": "2" } }); + + it("never loads the file while its copy is made, and plays a copy at the shown box", async () => { + let made = false; + const { el, proxySrc } = await waitingForItsCopy(() => + made ? new Response("x", { status: 206 }) : making(), + ); + + expect(el.hasAttribute("src")).toBe(false); + expect(proxySrc.unproxiedSrc(el)).toBe("/video.mp4"); + + made = true; + await vi.advanceTimersByTimeAsync(2_000); + + expect(params(el).get("hf-proxy-box")).toBe("2048x1218"); + expect(params(el).get("hf-proxy-light")).toBe("1"); + expect(proxySrc.unproxiedSrc(el)).toBe("/video.mp4"); + }); + + it("is not given its file back as its clip comes due", async () => { + const { el, preload } = await waitingForItsCopy(making); + + preload.giveBackWhenDue(el); + + expect(el.hasAttribute("src")).toBe(false); + }); + + type Prepare = ( + el: HTMLVideoElement, + proxySrc: typeof import("./proxySrc"), + proactively: (el: HTMLMediaElement) => void, + ) => void; + async function copyFailsAfter(prepare: Prepare) { + vi.useFakeTimers(); + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => + String(url).includes("hf-proxy") + ? new Response(null, { status: 500 }) + : new Response("x", { status: 206 }), + ), + ); + const staged = await stage({ "/video.mp4": HEAVY }); + const proxySrc = await import("./proxySrc"); + const el = staged.video(); + prepare(el, proxySrc, staged.maybeProxyProactively); + await vi.advanceTimersByTimeAsync(60_000); + return { el, proxySrc }; + } + + it("stays held until due when its copy fails, as it was before the copy", async () => { + const { el, proxySrc } = await copyFailsAfter((el, proxySrc, proactively) => { + proactively(el); + proxySrc.recordOpenedAtInPoint(el, "/video.mp4", null, true); + el.removeAttribute("src"); + }); + + expect(el.getAttribute("src")).toBeNull(); + expect(proxySrc.isHeldUntilDue(el)).toBe(true); + }); + + it("keeps its idle hold when its copy fails", async () => { + const { el, proxySrc } = await copyFailsAfter((el, proxySrc, proactively) => { + proxySrc.recordOpenedAtInPoint(el, "/video.mp4", "/video.mp4", true); + proactively(el); + }); + + expect(el.getAttribute("src")).toBe("/video.mp4"); + expect(proxySrc.holdsWhenIdle(el)).toBe(true); + }); + + it("gets its file back when the copy cannot be made", async () => { + const { el, asks } = await waitingForItsCopy(() => new Response(null, { status: 500 })); + + await vi.advanceTimersByTimeAsync(60_000); + + expect(asks()).toBe(4); + expect(el.getAttribute("src")).toBe("/video.mp4"); + }); + + it("waits for the lower level's copy instead, still not loading the file, when the level drops", async () => { + const { el, setPreviewQuality } = await waitingForItsCopy(making); + + setPreviewQuality(0.5); + await vi.advanceTimersByTimeAsync(2_000); + + const boxes = vi + .mocked(fetch) + .mock.calls.map(([url]) => + new URL(String(url), document.baseURI).searchParams.get("hf-proxy-box"), + ); + expect(boxes.at(-1)).toBe("1024x724"); + expect(el.hasAttribute("src")).toBe(false); + }); + }); + it("swaps a heavy file Chrome can play to a copy at the shown box times the level", async () => { const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY }); const el = video(); @@ -1229,7 +1347,7 @@ describe("preview quality: lighter copies of heavy files", () => { }); it("puts the original back at full quality", async () => { - const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY }); + const { video, setPreviewQuality } = await stage({ "/video.mp4": MID }); const el = video(); setPreviewQuality(0.5); await settle(); @@ -1245,7 +1363,10 @@ describe("preview quality: lighter copies of heavy files", () => { }); it("lets go of a held still when the original has no src to put back", async () => { - const { video, setPreviewQuality, setProxyDisplayScale } = await stage({ "/video.mp4": HEAVY }); + // Heavy below Full by its bit depth alone, so Full puts the original back at any shown size. + const { video, setPreviewQuality, setProxyDisplayScale } = await stage({ + "/video.mp4": { ...LIGHT, bitDepth: 10 }, + }); const el = video(); el.removeAttribute("src"); el.append(Object.assign(document.createElement("source"), { src: "/video.mp4" })); diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts index 6100df4223..3fd05e9322 100644 --- a/packages/core/src/runtime/mediaProxy.ts +++ b/packages/core/src/runtime/mediaProxy.ts @@ -16,8 +16,15 @@ import { lookupCodecMapEntry, sourceFactsOf, } from "./mediaSourceLookup"; -import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc"; -import { giveBackWhenDue } from "./preloadMedia"; +import { + heldForCopy, + heldPosters, + inPointOf, + recordOpenedAtInPoint, + swappedElements, + unproxiedSrc, +} from "./proxySrc"; +import { giveBackWhenDue, stopMediaDownload } from "./preloadMedia"; import { restartPlaybackHealth } from "./playbackHealth"; import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies"; import { waitForServedProxy } from "./proxyWait"; @@ -25,6 +32,7 @@ import { registerSeekCompletion } from "./adapters/seek-dispatch"; import { formatPreviewProxyBox, isHeavyForPreview, + isOversizedForPreview, isPreviewQuality, PREVIEW_QUALITY_LEVELS, PREVIEW_PROXY_BOX_PARAM, @@ -36,6 +44,7 @@ import { type PreviewProxyBox, type PreviewQuality, type PreviewSourceFacts, + wantsLighterCopy, } from "../previewProxyBox"; /** @@ -83,9 +92,8 @@ const unavailableDiagnosedElements = new WeakSet(); // keep that src, so nothing that reads it (timeline, player mirror) loads a copy still being made. const proxyRequested = new WeakMap(); -// Below 1, heavy files get lighter copies at the shown box times this level; 1 is the preview as it always was. +// Below 1, heavy files get lighter copies at the shown box times this level; at 1, files oversized for their box. let previewQuality: PreviewQuality = 1; -const lightElements = new WeakSet(); // Elements whose next ask for a copy tells the server to forget its failure first. const retryNext = new WeakSet(); @@ -156,7 +164,7 @@ function keepSized(el: HTMLMediaElement, scale: number): void { const lowered = previewQuality < 1; const needed = shown && lowered ? scalePreviewProxyBox(shown, previewQuality) : shown; if (lowered && !needed) return; - if (needed && lowered && copy.light && lightCopyNoLongerNeeded(el, needed)) { + if (needed && copy.light && lightCopyNoLongerNeeded(el, needed)) { return restoreOriginal(el); } if (boxCovers(copy.box, needed, lowered)) return dropUnneededCopy(el); @@ -169,7 +177,7 @@ function dropUnneededCopy(el: HTMLMediaElement): void { function lightCopyNoLongerNeeded(el: HTMLMediaElement, needed: PreviewProxyBox): boolean { const facts = sourceFactsOf(el); - return facts !== null && !isHeavyForPreview(facts, needed); + return facts !== null && !wantsLighterCopy(facts, needed, previewQuality); } function boxCovers(current: PreviewProxyBox, needed: PreviewProxyBox | null, exact: boolean) { @@ -215,7 +223,6 @@ function restoreOriginal(el: HTMLMediaElement): void { for (const state of [boxedElements, pendingUpgrades, proxyRequested, swappedElements]) { state.delete(el); } - lightElements.delete(el); evictMediaSyncState(el); restartPlaybackHealth(); if (original === null) { @@ -386,6 +393,40 @@ function emitUnavailableDiagnostic( console.info(`[hyperframes] ${DIAGNOSTIC_UNAVAILABLE_CODE}: "${asset}" (${reason}): ${note}`); } +/** Holds `el` with no src while its copy is made (readers still see its file); false with no src to hold. */ +function holdForCopy(el: HTMLMediaElement): boolean { + const authored = unproxiedSrc(el); + if (authored === null) return false; + heldForCopy.set(el, inPointOf(el)); + recordOpenedAtInPoint(el, authored, null); + stopMediaDownload(el); + holdFrameCaptures( + new Promise((landed) => { + el.addEventListener("loadeddata", landed, { once: true }); + el.addEventListener("error", landed, { once: true }); + }), + ); + return true; +} + +/** Without a copy (`giveBack` set), a clip held until due stays held; any other gets its src back as the runtime's. */ +function endCopyHold(el: HTMLMediaElement, giveBack: string | null): void { + if (!heldForCopy.has(el)) return; + const before = heldForCopy.get(el); + heldForCopy.delete(el); + if (giveBack === null || el.getAttribute("src") !== null) return; + const src = before ? before.written : giveBack; + recordOpenedAtInPoint(el, before?.authored ?? giveBack, src, before?.holdsWhenIdle); + if (src !== null) el.setAttribute("src", src); +} + +function holdFrameCaptures(until: Promise): void { + registerSeekCompletion( + Promise.race([until, new Promise((cap) => wallSetTimeout(cap, HOLD_CAP_MS))]), + "video-frame", + ); +} + type CopyJob = { originalSrc: string; originalAttr: string | null; @@ -401,10 +442,11 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null // Asked once the shown size is known, so the copy is made at that size and only once. const shown = scale === null ? null : shownBox(el, scale); const box = shown && level < 1 ? scalePreviewProxyBox(shown, level) : shown; - if (lightFacts && !(box && isHeavyForPreview(lightFacts, box))) { + if (lightFacts && !(box && wantsLighterCopy(lightFacts, box, level))) { if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el); return; } + const held = lightFacts && box && isOversizedForPreview(lightFacts, box) && holdForCopy(el); const proxiedSrc = appendProxyParam( originalSrc, variant, @@ -420,9 +462,11 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null (fraction) => noteCopy(el, "making", fraction, light), () => isActiveAtPlayhead(el), false, - light ? undefined : () => showStill(el, originalSrc, variant, box), + light && !held ? undefined : () => showStill(el, originalSrc, variant, box), ); - if (!live()) { + const wanted = live(); + if (held) endCopyHold(el, served && wanted ? null : originalAttr); + if (!wanted) { dropStill(el); if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el); if (lightFacts && el.isConnected && el.getAttribute("src") === originalAttr) { @@ -431,13 +475,13 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null return; } if (!served) { + dropStill(el); noteCopy(el, "failed", null, light); emitUnavailableDiagnostic(el, "proxy_playback_failed", originalSrc); return; } noteCopy(el, "ready", null, light); - swappedElements.set(el, unproxiedSrc(el)); - if (lightFacts) lightElements.add(el); + swappedElements.set(el, held ? originalAttr : unproxiedSrc(el)); loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined); // The shown size may have grown while this copy was being made. if (displayScale !== null) keepSized(el, displayScale); @@ -481,16 +525,14 @@ export function swapToProxy( // A lighter copy waits for no level but its own; a copy the file cannot play without finishes at any level. const live = () => el.isConnected && - el.getAttribute("src") === originalAttr && + (el.getAttribute("src") === originalAttr || + (heldForCopy.has(el) && el.getAttribute("src") === null)) && (!lightFacts || previewQuality === level); const job: CopyJob = { originalSrc, originalAttr, variant, lightFacts, level, live }; const swap = shownScale().then((scale) => makeCopy(el, job, scale)); if (lightFacts) return; // A frame capture holds for the copy as it held for a loading video before, up to the same cap. - registerSeekCompletion( - Promise.race([swap, new Promise((cap) => wallSetTimeout(cap, HOLD_CAP_MS))]), - "video-frame", - ); + holdFrameCaptures(swap); const codecName = entry?.codecName ?? null; const details: Record = { asset: originalSrc, @@ -527,7 +569,7 @@ export function maybeProxyProactively(el: HTMLMediaElement): void { const canPlay = entry?.representativeMime ? el.canPlayType(entry.representativeMime) : ""; const playable = canPlay === "probably" || canPlay === "maybe"; if (entry?.browserHostile && !playable) swapToProxy(el, entry, "proactive"); - else if (previewQuality < 1) lightenIfHeavy(el, entry); + else lightenIfHeavy(el, entry); } function lightenIfHeavy(el: HTMLMediaElement, entry: MediaCodecMapEntry | null): void { @@ -538,12 +580,11 @@ function lightenIfHeavy(el: HTMLMediaElement, entry: MediaCodecMapEntry | null): function applyPreviewQuality(el: HTMLMediaElement): void { if (isRenderMode(el) || !isVideoElement(el)) return; - if (previewQuality === 1 && lightElements.has(el)) return restoreOriginal(el); if (boxedElements.has(el) && displayScale !== null) return keepSized(el, displayScale); - if (previewQuality < 1) lightenIfHeavy(el, codecEntryOf(el)); + lightenIfHeavy(el, codecEntryOf(el)); } -/** The host's preview quality: below 1, heavy videos play lighter copies sized to the shown box times the level. */ +/** The host's preview quality: below 1, heavy videos play lighter copies at the shown box times the level. */ export function setPreviewQuality(level: number): void { if (!isPreviewQuality(level) || level === previewQuality) return; previewQuality = level; diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts index 935f0a1371..fb102ef4be 100644 --- a/packages/core/src/runtime/preloadMedia.ts +++ b/packages/core/src/runtime/preloadMedia.ts @@ -5,6 +5,7 @@ import { isHeldUntilDue, recordOpenedAtInPoint, unproxiedSrc, + heldForCopy, } from "./proxySrc"; import { createRuntimeStartTimeResolver } from "./startResolver"; import { readMediaRenderId } from "./renderFrameSibling"; @@ -171,7 +172,7 @@ function holdUntilDue(el: HTMLMediaElement): void { } export function giveBackWhenDue(el: HTMLMediaElement): void { - if (!isHeldUntilDue(el)) return; + if (!isHeldUntilDue(el) || heldForCopy.has(el)) return; const authored = authoredSrcOf(el)!; const start = readMediaStart(el); const written = start > 0 ? `${authored}#t=${start}` : authored; diff --git a/packages/core/src/runtime/proxySrc.ts b/packages/core/src/runtime/proxySrc.ts index eae5759c8e..31e09254e5 100644 --- a/packages/core/src/runtime/proxySrc.ts +++ b/packages/core/src/runtime/proxySrc.ts @@ -3,10 +3,10 @@ export const swappedElements = new WeakMap(); /** Media elements the runtime opened at their in-point: the src the author wrote, the one written in its * place (null while the clip is held with no src until it is due), and whether it is held again when idle. */ -const openedAtInPoint = new WeakMap< - Element, - { authored: string; written: string | null; holdsWhenIdle: boolean } ->(); +type InPointRecord = { authored: string; written: string | null; holdsWhenIdle: boolean }; +const openedAtInPoint = new WeakMap(); +export const inPointOf = (el: Element) => openedAtInPoint.get(el); +export const heldForCopy = new WeakMap(); export function recordOpenedAtInPoint( el: Element,