fix(core): at full quality a 4K clip plays a copy at the size the preview shows it (#7723)

* fix(core): at full quality a 4K clip plays a copy at the size the preview shows it

* fix(core): names carry the copy rule, not comments

* fix(core): a clip whose copy fails is left held as it was, so its decoder is still freed when idle

* fix(core): the copy hold lives with the copy code, keeping core inside its size budget
This commit is contained in:
Miguel Ángel
2026-11-14 08:09:33 +00:00
committed by GitHub
parent 58ef24c6d8
commit 2f0ade1815
5 changed files with 211 additions and 38 deletions
+13 -3
View File
@@ -66,10 +66,20 @@ export function scalePreviewProxyBox(box: PreviewProxyBox, level: PreviewQuality
return quantizePreviewProxyBox(box.width * level, box.height * level) ?? box;
}
export function isOversizedForPreview(facts: PreviewSourceFacts, box: PreviewProxyBox): boolean {
return box.width * box.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height;
}
export function isHeavyForPreview(facts: PreviewSourceFacts, copyBox: PreviewProxyBox): boolean {
return (
copyBox.width * copyBox.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height ||
facts.bitDepth > 8 ||
facts.bitrate >= HEAVY_BITRATE
isOversizedForPreview(facts, copyBox) || facts.bitDepth > 8 || facts.bitrate >= HEAVY_BITRATE
);
}
export function wantsLighterCopy(
facts: PreviewSourceFacts,
box: PreviewProxyBox,
level: PreviewQuality,
): boolean {
return level < 1 ? isHeavyForPreview(facts, box) : isOversizedForPreview(facts, box);
}
+130 -9
View File
@@ -1044,6 +1044,8 @@ describe("a copy sized to how large the video shows", () => {
describe("preview quality: lighter copies of heavy files", () => {
const HEAVY = { width: 3840, height: 2160, bitDepth: 8, bitrate: 10_000_000 };
const LIGHT = { width: 1280, height: 720, bitDepth: 8, bitrate: 10_000_000 };
// Heavy at a lower level's box, not oversized for the full 2048x1218 box.
const MID = { width: 2560, height: 1440, bitDepth: 8, bitrate: 10_000_000 };
// A framed page showing a 1920x1080 stage at full size on a 1x screen: a 2048x1218 box.
async function stage(facts: Record<string, typeof HEAVY>, codecMap?: Record<string, unknown>) {
@@ -1193,19 +1195,135 @@ describe("preview quality: lighter copies of heavy files", () => {
});
});
it("changes nothing at full quality", async () => {
const { video, setPreviewQuality, maybeProxyProactively } = await stage({
"/video.mp4": HEAVY,
it("changes nothing at full quality for a file not oversized for its box, 10-bit included", async () => {
const { video, maybeProxyProactively } = await stage({
"/mid.mp4": MID,
"/ten-bit.mp4": { ...MID, bitDepth: 10 },
});
const el = video();
maybeProxyProactively(el);
setPreviewQuality(1);
const els = [video("/mid.mp4"), video("/ten-bit.mp4")];
for (const el of els) maybeProxyProactively(el);
await settle();
expect(isProxied(el)).toBe(false);
expect(els.map(isProxied)).toEqual([false, false]);
expect(fetch).not.toHaveBeenCalled();
});
describe("at full quality, a file with at least twice the pixels it shows", () => {
async function waitingForItsCopy(answer: () => Response) {
vi.useFakeTimers();
let copyAsks = 0;
vi.stubGlobal(
"fetch",
vi.fn(async (url: string) => {
if (!String(url).includes("hf-proxy")) return new Response("x", { status: 206 });
copyAsks += 1;
return answer();
}),
);
const staged = await stage({ "/video.mp4": HEAVY });
const proxySrc = await import("./proxySrc");
const preload = await import("./preloadMedia");
const el = staged.video();
staged.maybeProxyProactively(el);
await vi.advanceTimersByTimeAsync(0);
return { ...staged, el, proxySrc, preload, asks: () => copyAsks };
}
const making = () => new Response(null, { status: 202, headers: { "Retry-After": "2" } });
it("never loads the file while its copy is made, and plays a copy at the shown box", async () => {
let made = false;
const { el, proxySrc } = await waitingForItsCopy(() =>
made ? new Response("x", { status: 206 }) : making(),
);
expect(el.hasAttribute("src")).toBe(false);
expect(proxySrc.unproxiedSrc(el)).toBe("/video.mp4");
made = true;
await vi.advanceTimersByTimeAsync(2_000);
expect(params(el).get("hf-proxy-box")).toBe("2048x1218");
expect(params(el).get("hf-proxy-light")).toBe("1");
expect(proxySrc.unproxiedSrc(el)).toBe("/video.mp4");
});
it("is not given its file back as its clip comes due", async () => {
const { el, preload } = await waitingForItsCopy(making);
preload.giveBackWhenDue(el);
expect(el.hasAttribute("src")).toBe(false);
});
type Prepare = (
el: HTMLVideoElement,
proxySrc: typeof import("./proxySrc"),
proactively: (el: HTMLMediaElement) => void,
) => void;
async function copyFailsAfter(prepare: Prepare) {
vi.useFakeTimers();
vi.stubGlobal(
"fetch",
vi.fn(async (url: string) =>
String(url).includes("hf-proxy")
? new Response(null, { status: 500 })
: new Response("x", { status: 206 }),
),
);
const staged = await stage({ "/video.mp4": HEAVY });
const proxySrc = await import("./proxySrc");
const el = staged.video();
prepare(el, proxySrc, staged.maybeProxyProactively);
await vi.advanceTimersByTimeAsync(60_000);
return { el, proxySrc };
}
it("stays held until due when its copy fails, as it was before the copy", async () => {
const { el, proxySrc } = await copyFailsAfter((el, proxySrc, proactively) => {
proactively(el);
proxySrc.recordOpenedAtInPoint(el, "/video.mp4", null, true);
el.removeAttribute("src");
});
expect(el.getAttribute("src")).toBeNull();
expect(proxySrc.isHeldUntilDue(el)).toBe(true);
});
it("keeps its idle hold when its copy fails", async () => {
const { el, proxySrc } = await copyFailsAfter((el, proxySrc, proactively) => {
proxySrc.recordOpenedAtInPoint(el, "/video.mp4", "/video.mp4", true);
proactively(el);
});
expect(el.getAttribute("src")).toBe("/video.mp4");
expect(proxySrc.holdsWhenIdle(el)).toBe(true);
});
it("gets its file back when the copy cannot be made", async () => {
const { el, asks } = await waitingForItsCopy(() => new Response(null, { status: 500 }));
await vi.advanceTimersByTimeAsync(60_000);
expect(asks()).toBe(4);
expect(el.getAttribute("src")).toBe("/video.mp4");
});
it("waits for the lower level's copy instead, still not loading the file, when the level drops", async () => {
const { el, setPreviewQuality } = await waitingForItsCopy(making);
setPreviewQuality(0.5);
await vi.advanceTimersByTimeAsync(2_000);
const boxes = vi
.mocked(fetch)
.mock.calls.map(([url]) =>
new URL(String(url), document.baseURI).searchParams.get("hf-proxy-box"),
);
expect(boxes.at(-1)).toBe("1024x724");
expect(el.hasAttribute("src")).toBe(false);
});
});
it("swaps a heavy file Chrome can play to a copy at the shown box times the level", async () => {
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
const el = video();
@@ -1229,7 +1347,7 @@ describe("preview quality: lighter copies of heavy files", () => {
});
it("puts the original back at full quality", async () => {
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
const { video, setPreviewQuality } = await stage({ "/video.mp4": MID });
const el = video();
setPreviewQuality(0.5);
await settle();
@@ -1245,7 +1363,10 @@ describe("preview quality: lighter copies of heavy files", () => {
});
it("lets go of a held still when the original has no src to put back", async () => {
const { video, setPreviewQuality, setProxyDisplayScale } = await stage({ "/video.mp4": HEAVY });
// Heavy below Full by its bit depth alone, so Full puts the original back at any shown size.
const { video, setPreviewQuality, setProxyDisplayScale } = await stage({
"/video.mp4": { ...LIGHT, bitDepth: 10 },
});
const el = video();
el.removeAttribute("src");
el.append(Object.assign(document.createElement("source"), { src: "/video.mp4" }));
+62 -21
View File
@@ -16,8 +16,15 @@ import {
lookupCodecMapEntry,
sourceFactsOf,
} from "./mediaSourceLookup";
import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc";
import { giveBackWhenDue } from "./preloadMedia";
import {
heldForCopy,
heldPosters,
inPointOf,
recordOpenedAtInPoint,
swappedElements,
unproxiedSrc,
} from "./proxySrc";
import { giveBackWhenDue, stopMediaDownload } from "./preloadMedia";
import { restartPlaybackHealth } from "./playbackHealth";
import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies";
import { waitForServedProxy } from "./proxyWait";
@@ -25,6 +32,7 @@ import { registerSeekCompletion } from "./adapters/seek-dispatch";
import {
formatPreviewProxyBox,
isHeavyForPreview,
isOversizedForPreview,
isPreviewQuality,
PREVIEW_QUALITY_LEVELS,
PREVIEW_PROXY_BOX_PARAM,
@@ -36,6 +44,7 @@ import {
type PreviewProxyBox,
type PreviewQuality,
type PreviewSourceFacts,
wantsLighterCopy,
} from "../previewProxyBox";
/**
@@ -83,9 +92,8 @@ const unavailableDiagnosedElements = new WeakSet<HTMLMediaElement>();
// keep that src, so nothing that reads it (timeline, player mirror) loads a copy still being made.
const proxyRequested = new WeakMap<HTMLMediaElement, string | null>();
// Below 1, heavy files get lighter copies at the shown box times this level; 1 is the preview as it always was.
// Below 1, heavy files get lighter copies at the shown box times this level; at 1, files oversized for their box.
let previewQuality: PreviewQuality = 1;
const lightElements = new WeakSet<HTMLMediaElement>();
// Elements whose next ask for a copy tells the server to forget its failure first.
const retryNext = new WeakSet<HTMLMediaElement>();
@@ -156,7 +164,7 @@ function keepSized(el: HTMLMediaElement, scale: number): void {
const lowered = previewQuality < 1;
const needed = shown && lowered ? scalePreviewProxyBox(shown, previewQuality) : shown;
if (lowered && !needed) return;
if (needed && lowered && copy.light && lightCopyNoLongerNeeded(el, needed)) {
if (needed && copy.light && lightCopyNoLongerNeeded(el, needed)) {
return restoreOriginal(el);
}
if (boxCovers(copy.box, needed, lowered)) return dropUnneededCopy(el);
@@ -169,7 +177,7 @@ function dropUnneededCopy(el: HTMLMediaElement): void {
function lightCopyNoLongerNeeded(el: HTMLMediaElement, needed: PreviewProxyBox): boolean {
const facts = sourceFactsOf(el);
return facts !== null && !isHeavyForPreview(facts, needed);
return facts !== null && !wantsLighterCopy(facts, needed, previewQuality);
}
function boxCovers(current: PreviewProxyBox, needed: PreviewProxyBox | null, exact: boolean) {
@@ -215,7 +223,6 @@ function restoreOriginal(el: HTMLMediaElement): void {
for (const state of [boxedElements, pendingUpgrades, proxyRequested, swappedElements]) {
state.delete(el);
}
lightElements.delete(el);
evictMediaSyncState(el);
restartPlaybackHealth();
if (original === null) {
@@ -386,6 +393,40 @@ function emitUnavailableDiagnostic(
console.info(`[hyperframes] ${DIAGNOSTIC_UNAVAILABLE_CODE}: "${asset}" (${reason}): ${note}`);
}
/** Holds `el` with no src while its copy is made (readers still see its file); false with no src to hold. */
function holdForCopy(el: HTMLMediaElement): boolean {
const authored = unproxiedSrc(el);
if (authored === null) return false;
heldForCopy.set(el, inPointOf(el));
recordOpenedAtInPoint(el, authored, null);
stopMediaDownload(el);
holdFrameCaptures(
new Promise((landed) => {
el.addEventListener("loadeddata", landed, { once: true });
el.addEventListener("error", landed, { once: true });
}),
);
return true;
}
/** Without a copy (`giveBack` set), a clip held until due stays held; any other gets its src back as the runtime's. */
function endCopyHold(el: HTMLMediaElement, giveBack: string | null): void {
if (!heldForCopy.has(el)) return;
const before = heldForCopy.get(el);
heldForCopy.delete(el);
if (giveBack === null || el.getAttribute("src") !== null) return;
const src = before ? before.written : giveBack;
recordOpenedAtInPoint(el, before?.authored ?? giveBack, src, before?.holdsWhenIdle);
if (src !== null) el.setAttribute("src", src);
}
function holdFrameCaptures(until: Promise<unknown>): void {
registerSeekCompletion(
Promise.race([until, new Promise((cap) => wallSetTimeout(cap, HOLD_CAP_MS))]),
"video-frame",
);
}
type CopyJob = {
originalSrc: string;
originalAttr: string | null;
@@ -401,10 +442,11 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
// Asked once the shown size is known, so the copy is made at that size and only once.
const shown = scale === null ? null : shownBox(el, scale);
const box = shown && level < 1 ? scalePreviewProxyBox(shown, level) : shown;
if (lightFacts && !(box && isHeavyForPreview(lightFacts, box))) {
if (lightFacts && !(box && wantsLighterCopy(lightFacts, box, level))) {
if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el);
return;
}
const held = lightFacts && box && isOversizedForPreview(lightFacts, box) && holdForCopy(el);
const proxiedSrc = appendProxyParam(
originalSrc,
variant,
@@ -420,9 +462,11 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
(fraction) => noteCopy(el, "making", fraction, light),
() => isActiveAtPlayhead(el),
false,
light ? undefined : () => showStill(el, originalSrc, variant, box),
light && !held ? undefined : () => showStill(el, originalSrc, variant, box),
);
if (!live()) {
const wanted = live();
if (held) endCopyHold(el, served && wanted ? null : originalAttr);
if (!wanted) {
dropStill(el);
if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el);
if (lightFacts && el.isConnected && el.getAttribute("src") === originalAttr) {
@@ -431,13 +475,13 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
return;
}
if (!served) {
dropStill(el);
noteCopy(el, "failed", null, light);
emitUnavailableDiagnostic(el, "proxy_playback_failed", originalSrc);
return;
}
noteCopy(el, "ready", null, light);
swappedElements.set(el, unproxiedSrc(el));
if (lightFacts) lightElements.add(el);
swappedElements.set(el, held ? originalAttr : unproxiedSrc(el));
loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined);
// The shown size may have grown while this copy was being made.
if (displayScale !== null) keepSized(el, displayScale);
@@ -481,16 +525,14 @@ export function swapToProxy(
// A lighter copy waits for no level but its own; a copy the file cannot play without finishes at any level.
const live = () =>
el.isConnected &&
el.getAttribute("src") === originalAttr &&
(el.getAttribute("src") === originalAttr ||
(heldForCopy.has(el) && el.getAttribute("src") === null)) &&
(!lightFacts || previewQuality === level);
const job: CopyJob = { originalSrc, originalAttr, variant, lightFacts, level, live };
const swap = shownScale().then((scale) => makeCopy(el, job, scale));
if (lightFacts) return;
// A frame capture holds for the copy as it held for a loading video before, up to the same cap.
registerSeekCompletion(
Promise.race([swap, new Promise((cap) => wallSetTimeout(cap, HOLD_CAP_MS))]),
"video-frame",
);
holdFrameCaptures(swap);
const codecName = entry?.codecName ?? null;
const details: Record<string, RuntimeJson> = {
asset: originalSrc,
@@ -527,7 +569,7 @@ export function maybeProxyProactively(el: HTMLMediaElement): void {
const canPlay = entry?.representativeMime ? el.canPlayType(entry.representativeMime) : "";
const playable = canPlay === "probably" || canPlay === "maybe";
if (entry?.browserHostile && !playable) swapToProxy(el, entry, "proactive");
else if (previewQuality < 1) lightenIfHeavy(el, entry);
else lightenIfHeavy(el, entry);
}
function lightenIfHeavy(el: HTMLMediaElement, entry: MediaCodecMapEntry | null): void {
@@ -538,12 +580,11 @@ function lightenIfHeavy(el: HTMLMediaElement, entry: MediaCodecMapEntry | null):
function applyPreviewQuality(el: HTMLMediaElement): void {
if (isRenderMode(el) || !isVideoElement(el)) return;
if (previewQuality === 1 && lightElements.has(el)) return restoreOriginal(el);
if (boxedElements.has(el) && displayScale !== null) return keepSized(el, displayScale);
if (previewQuality < 1) lightenIfHeavy(el, codecEntryOf(el));
lightenIfHeavy(el, codecEntryOf(el));
}
/** The host's preview quality: below 1, heavy videos play lighter copies sized to the shown box times the level. */
/** The host's preview quality: below 1, heavy videos play lighter copies at the shown box times the level. */
export function setPreviewQuality(level: number): void {
if (!isPreviewQuality(level) || level === previewQuality) return;
previewQuality = level;
+2 -1
View File
@@ -5,6 +5,7 @@ import {
isHeldUntilDue,
recordOpenedAtInPoint,
unproxiedSrc,
heldForCopy,
} from "./proxySrc";
import { createRuntimeStartTimeResolver } from "./startResolver";
import { readMediaRenderId } from "./renderFrameSibling";
@@ -171,7 +172,7 @@ function holdUntilDue(el: HTMLMediaElement): void {
}
export function giveBackWhenDue(el: HTMLMediaElement): void {
if (!isHeldUntilDue(el)) return;
if (!isHeldUntilDue(el) || heldForCopy.has(el)) return;
const authored = authoredSrcOf(el)!;
const start = readMediaStart(el);
const written = start > 0 ? `${authored}#t=${start}` : authored;
+4 -4
View File
@@ -3,10 +3,10 @@ export const swappedElements = new WeakMap<Element, string | null>();
/** Media elements the runtime opened at their in-point: the src the author wrote, the one written in its
* place (null while the clip is held with no src until it is due), and whether it is held again when idle. */
const openedAtInPoint = new WeakMap<
Element,
{ authored: string; written: string | null; holdsWhenIdle: boolean }
>();
type InPointRecord = { authored: string; written: string | null; holdsWhenIdle: boolean };
const openedAtInPoint = new WeakMap<Element, InPointRecord>();
export const inPointOf = (el: Element) => openedAtInPoint.get(el);
export const heldForCopy = new WeakMap<Element, InPointRecord | undefined>();
export function recordOpenedAtInPoint(
el: Element,