mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(core): at full quality a 4K clip plays a copy at the size the preview shows it (#7723)
* fix(core): at full quality a 4K clip plays a copy at the size the preview shows it * fix(core): names carry the copy rule, not comments * fix(core): a clip whose copy fails is left held as it was, so its decoder is still freed when idle * fix(core): the copy hold lives with the copy code, keeping core inside its size budget
This commit is contained in:
@@ -66,10 +66,20 @@ export function scalePreviewProxyBox(box: PreviewProxyBox, level: PreviewQuality
|
||||
return quantizePreviewProxyBox(box.width * level, box.height * level) ?? box;
|
||||
}
|
||||
|
||||
export function isOversizedForPreview(facts: PreviewSourceFacts, box: PreviewProxyBox): boolean {
|
||||
return box.width * box.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height;
|
||||
}
|
||||
|
||||
export function isHeavyForPreview(facts: PreviewSourceFacts, copyBox: PreviewProxyBox): boolean {
|
||||
return (
|
||||
copyBox.width * copyBox.height <= HEAVY_PIXEL_SHARE * facts.width * facts.height ||
|
||||
facts.bitDepth > 8 ||
|
||||
facts.bitrate >= HEAVY_BITRATE
|
||||
isOversizedForPreview(facts, copyBox) || facts.bitDepth > 8 || facts.bitrate >= HEAVY_BITRATE
|
||||
);
|
||||
}
|
||||
|
||||
export function wantsLighterCopy(
|
||||
facts: PreviewSourceFacts,
|
||||
box: PreviewProxyBox,
|
||||
level: PreviewQuality,
|
||||
): boolean {
|
||||
return level < 1 ? isHeavyForPreview(facts, box) : isOversizedForPreview(facts, box);
|
||||
}
|
||||
|
||||
@@ -1044,6 +1044,8 @@ describe("a copy sized to how large the video shows", () => {
|
||||
describe("preview quality: lighter copies of heavy files", () => {
|
||||
const HEAVY = { width: 3840, height: 2160, bitDepth: 8, bitrate: 10_000_000 };
|
||||
const LIGHT = { width: 1280, height: 720, bitDepth: 8, bitrate: 10_000_000 };
|
||||
// Heavy at a lower level's box, not oversized for the full 2048x1218 box.
|
||||
const MID = { width: 2560, height: 1440, bitDepth: 8, bitrate: 10_000_000 };
|
||||
|
||||
// A framed page showing a 1920x1080 stage at full size on a 1x screen: a 2048x1218 box.
|
||||
async function stage(facts: Record<string, typeof HEAVY>, codecMap?: Record<string, unknown>) {
|
||||
@@ -1193,19 +1195,135 @@ describe("preview quality: lighter copies of heavy files", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("changes nothing at full quality", async () => {
|
||||
const { video, setPreviewQuality, maybeProxyProactively } = await stage({
|
||||
"/video.mp4": HEAVY,
|
||||
it("changes nothing at full quality for a file not oversized for its box, 10-bit included", async () => {
|
||||
const { video, maybeProxyProactively } = await stage({
|
||||
"/mid.mp4": MID,
|
||||
"/ten-bit.mp4": { ...MID, bitDepth: 10 },
|
||||
});
|
||||
const el = video();
|
||||
maybeProxyProactively(el);
|
||||
setPreviewQuality(1);
|
||||
const els = [video("/mid.mp4"), video("/ten-bit.mp4")];
|
||||
for (const el of els) maybeProxyProactively(el);
|
||||
await settle();
|
||||
|
||||
expect(isProxied(el)).toBe(false);
|
||||
expect(els.map(isProxied)).toEqual([false, false]);
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe("at full quality, a file with at least twice the pixels it shows", () => {
|
||||
async function waitingForItsCopy(answer: () => Response) {
|
||||
vi.useFakeTimers();
|
||||
let copyAsks = 0;
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (url: string) => {
|
||||
if (!String(url).includes("hf-proxy")) return new Response("x", { status: 206 });
|
||||
copyAsks += 1;
|
||||
return answer();
|
||||
}),
|
||||
);
|
||||
const staged = await stage({ "/video.mp4": HEAVY });
|
||||
const proxySrc = await import("./proxySrc");
|
||||
const preload = await import("./preloadMedia");
|
||||
const el = staged.video();
|
||||
staged.maybeProxyProactively(el);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
return { ...staged, el, proxySrc, preload, asks: () => copyAsks };
|
||||
}
|
||||
const making = () => new Response(null, { status: 202, headers: { "Retry-After": "2" } });
|
||||
|
||||
it("never loads the file while its copy is made, and plays a copy at the shown box", async () => {
|
||||
let made = false;
|
||||
const { el, proxySrc } = await waitingForItsCopy(() =>
|
||||
made ? new Response("x", { status: 206 }) : making(),
|
||||
);
|
||||
|
||||
expect(el.hasAttribute("src")).toBe(false);
|
||||
expect(proxySrc.unproxiedSrc(el)).toBe("/video.mp4");
|
||||
|
||||
made = true;
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
|
||||
expect(params(el).get("hf-proxy-box")).toBe("2048x1218");
|
||||
expect(params(el).get("hf-proxy-light")).toBe("1");
|
||||
expect(proxySrc.unproxiedSrc(el)).toBe("/video.mp4");
|
||||
});
|
||||
|
||||
it("is not given its file back as its clip comes due", async () => {
|
||||
const { el, preload } = await waitingForItsCopy(making);
|
||||
|
||||
preload.giveBackWhenDue(el);
|
||||
|
||||
expect(el.hasAttribute("src")).toBe(false);
|
||||
});
|
||||
|
||||
type Prepare = (
|
||||
el: HTMLVideoElement,
|
||||
proxySrc: typeof import("./proxySrc"),
|
||||
proactively: (el: HTMLMediaElement) => void,
|
||||
) => void;
|
||||
async function copyFailsAfter(prepare: Prepare) {
|
||||
vi.useFakeTimers();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (url: string) =>
|
||||
String(url).includes("hf-proxy")
|
||||
? new Response(null, { status: 500 })
|
||||
: new Response("x", { status: 206 }),
|
||||
),
|
||||
);
|
||||
const staged = await stage({ "/video.mp4": HEAVY });
|
||||
const proxySrc = await import("./proxySrc");
|
||||
const el = staged.video();
|
||||
prepare(el, proxySrc, staged.maybeProxyProactively);
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
return { el, proxySrc };
|
||||
}
|
||||
|
||||
it("stays held until due when its copy fails, as it was before the copy", async () => {
|
||||
const { el, proxySrc } = await copyFailsAfter((el, proxySrc, proactively) => {
|
||||
proactively(el);
|
||||
proxySrc.recordOpenedAtInPoint(el, "/video.mp4", null, true);
|
||||
el.removeAttribute("src");
|
||||
});
|
||||
|
||||
expect(el.getAttribute("src")).toBeNull();
|
||||
expect(proxySrc.isHeldUntilDue(el)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps its idle hold when its copy fails", async () => {
|
||||
const { el, proxySrc } = await copyFailsAfter((el, proxySrc, proactively) => {
|
||||
proxySrc.recordOpenedAtInPoint(el, "/video.mp4", "/video.mp4", true);
|
||||
proactively(el);
|
||||
});
|
||||
|
||||
expect(el.getAttribute("src")).toBe("/video.mp4");
|
||||
expect(proxySrc.holdsWhenIdle(el)).toBe(true);
|
||||
});
|
||||
|
||||
it("gets its file back when the copy cannot be made", async () => {
|
||||
const { el, asks } = await waitingForItsCopy(() => new Response(null, { status: 500 }));
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
|
||||
expect(asks()).toBe(4);
|
||||
expect(el.getAttribute("src")).toBe("/video.mp4");
|
||||
});
|
||||
|
||||
it("waits for the lower level's copy instead, still not loading the file, when the level drops", async () => {
|
||||
const { el, setPreviewQuality } = await waitingForItsCopy(making);
|
||||
|
||||
setPreviewQuality(0.5);
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
|
||||
const boxes = vi
|
||||
.mocked(fetch)
|
||||
.mock.calls.map(([url]) =>
|
||||
new URL(String(url), document.baseURI).searchParams.get("hf-proxy-box"),
|
||||
);
|
||||
expect(boxes.at(-1)).toBe("1024x724");
|
||||
expect(el.hasAttribute("src")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it("swaps a heavy file Chrome can play to a copy at the shown box times the level", async () => {
|
||||
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
|
||||
const el = video();
|
||||
@@ -1229,7 +1347,7 @@ describe("preview quality: lighter copies of heavy files", () => {
|
||||
});
|
||||
|
||||
it("puts the original back at full quality", async () => {
|
||||
const { video, setPreviewQuality } = await stage({ "/video.mp4": HEAVY });
|
||||
const { video, setPreviewQuality } = await stage({ "/video.mp4": MID });
|
||||
const el = video();
|
||||
setPreviewQuality(0.5);
|
||||
await settle();
|
||||
@@ -1245,7 +1363,10 @@ describe("preview quality: lighter copies of heavy files", () => {
|
||||
});
|
||||
|
||||
it("lets go of a held still when the original has no src to put back", async () => {
|
||||
const { video, setPreviewQuality, setProxyDisplayScale } = await stage({ "/video.mp4": HEAVY });
|
||||
// Heavy below Full by its bit depth alone, so Full puts the original back at any shown size.
|
||||
const { video, setPreviewQuality, setProxyDisplayScale } = await stage({
|
||||
"/video.mp4": { ...LIGHT, bitDepth: 10 },
|
||||
});
|
||||
const el = video();
|
||||
el.removeAttribute("src");
|
||||
el.append(Object.assign(document.createElement("source"), { src: "/video.mp4" }));
|
||||
|
||||
@@ -16,8 +16,15 @@ import {
|
||||
lookupCodecMapEntry,
|
||||
sourceFactsOf,
|
||||
} from "./mediaSourceLookup";
|
||||
import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc";
|
||||
import { giveBackWhenDue } from "./preloadMedia";
|
||||
import {
|
||||
heldForCopy,
|
||||
heldPosters,
|
||||
inPointOf,
|
||||
recordOpenedAtInPoint,
|
||||
swappedElements,
|
||||
unproxiedSrc,
|
||||
} from "./proxySrc";
|
||||
import { giveBackWhenDue, stopMediaDownload } from "./preloadMedia";
|
||||
import { restartPlaybackHealth } from "./playbackHealth";
|
||||
import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies";
|
||||
import { waitForServedProxy } from "./proxyWait";
|
||||
@@ -25,6 +32,7 @@ import { registerSeekCompletion } from "./adapters/seek-dispatch";
|
||||
import {
|
||||
formatPreviewProxyBox,
|
||||
isHeavyForPreview,
|
||||
isOversizedForPreview,
|
||||
isPreviewQuality,
|
||||
PREVIEW_QUALITY_LEVELS,
|
||||
PREVIEW_PROXY_BOX_PARAM,
|
||||
@@ -36,6 +44,7 @@ import {
|
||||
type PreviewProxyBox,
|
||||
type PreviewQuality,
|
||||
type PreviewSourceFacts,
|
||||
wantsLighterCopy,
|
||||
} from "../previewProxyBox";
|
||||
|
||||
/**
|
||||
@@ -83,9 +92,8 @@ const unavailableDiagnosedElements = new WeakSet<HTMLMediaElement>();
|
||||
// keep that src, so nothing that reads it (timeline, player mirror) loads a copy still being made.
|
||||
const proxyRequested = new WeakMap<HTMLMediaElement, string | null>();
|
||||
|
||||
// Below 1, heavy files get lighter copies at the shown box times this level; 1 is the preview as it always was.
|
||||
// Below 1, heavy files get lighter copies at the shown box times this level; at 1, files oversized for their box.
|
||||
let previewQuality: PreviewQuality = 1;
|
||||
const lightElements = new WeakSet<HTMLMediaElement>();
|
||||
// Elements whose next ask for a copy tells the server to forget its failure first.
|
||||
const retryNext = new WeakSet<HTMLMediaElement>();
|
||||
|
||||
@@ -156,7 +164,7 @@ function keepSized(el: HTMLMediaElement, scale: number): void {
|
||||
const lowered = previewQuality < 1;
|
||||
const needed = shown && lowered ? scalePreviewProxyBox(shown, previewQuality) : shown;
|
||||
if (lowered && !needed) return;
|
||||
if (needed && lowered && copy.light && lightCopyNoLongerNeeded(el, needed)) {
|
||||
if (needed && copy.light && lightCopyNoLongerNeeded(el, needed)) {
|
||||
return restoreOriginal(el);
|
||||
}
|
||||
if (boxCovers(copy.box, needed, lowered)) return dropUnneededCopy(el);
|
||||
@@ -169,7 +177,7 @@ function dropUnneededCopy(el: HTMLMediaElement): void {
|
||||
|
||||
function lightCopyNoLongerNeeded(el: HTMLMediaElement, needed: PreviewProxyBox): boolean {
|
||||
const facts = sourceFactsOf(el);
|
||||
return facts !== null && !isHeavyForPreview(facts, needed);
|
||||
return facts !== null && !wantsLighterCopy(facts, needed, previewQuality);
|
||||
}
|
||||
|
||||
function boxCovers(current: PreviewProxyBox, needed: PreviewProxyBox | null, exact: boolean) {
|
||||
@@ -215,7 +223,6 @@ function restoreOriginal(el: HTMLMediaElement): void {
|
||||
for (const state of [boxedElements, pendingUpgrades, proxyRequested, swappedElements]) {
|
||||
state.delete(el);
|
||||
}
|
||||
lightElements.delete(el);
|
||||
evictMediaSyncState(el);
|
||||
restartPlaybackHealth();
|
||||
if (original === null) {
|
||||
@@ -386,6 +393,40 @@ function emitUnavailableDiagnostic(
|
||||
console.info(`[hyperframes] ${DIAGNOSTIC_UNAVAILABLE_CODE}: "${asset}" (${reason}): ${note}`);
|
||||
}
|
||||
|
||||
/** Holds `el` with no src while its copy is made (readers still see its file); false with no src to hold. */
|
||||
function holdForCopy(el: HTMLMediaElement): boolean {
|
||||
const authored = unproxiedSrc(el);
|
||||
if (authored === null) return false;
|
||||
heldForCopy.set(el, inPointOf(el));
|
||||
recordOpenedAtInPoint(el, authored, null);
|
||||
stopMediaDownload(el);
|
||||
holdFrameCaptures(
|
||||
new Promise((landed) => {
|
||||
el.addEventListener("loadeddata", landed, { once: true });
|
||||
el.addEventListener("error", landed, { once: true });
|
||||
}),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Without a copy (`giveBack` set), a clip held until due stays held; any other gets its src back as the runtime's. */
|
||||
function endCopyHold(el: HTMLMediaElement, giveBack: string | null): void {
|
||||
if (!heldForCopy.has(el)) return;
|
||||
const before = heldForCopy.get(el);
|
||||
heldForCopy.delete(el);
|
||||
if (giveBack === null || el.getAttribute("src") !== null) return;
|
||||
const src = before ? before.written : giveBack;
|
||||
recordOpenedAtInPoint(el, before?.authored ?? giveBack, src, before?.holdsWhenIdle);
|
||||
if (src !== null) el.setAttribute("src", src);
|
||||
}
|
||||
|
||||
function holdFrameCaptures(until: Promise<unknown>): void {
|
||||
registerSeekCompletion(
|
||||
Promise.race([until, new Promise((cap) => wallSetTimeout(cap, HOLD_CAP_MS))]),
|
||||
"video-frame",
|
||||
);
|
||||
}
|
||||
|
||||
type CopyJob = {
|
||||
originalSrc: string;
|
||||
originalAttr: string | null;
|
||||
@@ -401,10 +442,11 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
|
||||
// Asked once the shown size is known, so the copy is made at that size and only once.
|
||||
const shown = scale === null ? null : shownBox(el, scale);
|
||||
const box = shown && level < 1 ? scalePreviewProxyBox(shown, level) : shown;
|
||||
if (lightFacts && !(box && isHeavyForPreview(lightFacts, box))) {
|
||||
if (lightFacts && !(box && wantsLighterCopy(lightFacts, box, level))) {
|
||||
if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el);
|
||||
return;
|
||||
}
|
||||
const held = lightFacts && box && isOversizedForPreview(lightFacts, box) && holdForCopy(el);
|
||||
const proxiedSrc = appendProxyParam(
|
||||
originalSrc,
|
||||
variant,
|
||||
@@ -420,9 +462,11 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
|
||||
(fraction) => noteCopy(el, "making", fraction, light),
|
||||
() => isActiveAtPlayhead(el),
|
||||
false,
|
||||
light ? undefined : () => showStill(el, originalSrc, variant, box),
|
||||
light && !held ? undefined : () => showStill(el, originalSrc, variant, box),
|
||||
);
|
||||
if (!live()) {
|
||||
const wanted = live();
|
||||
if (held) endCopyHold(el, served && wanted ? null : originalAttr);
|
||||
if (!wanted) {
|
||||
dropStill(el);
|
||||
if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el);
|
||||
if (lightFacts && el.isConnected && el.getAttribute("src") === originalAttr) {
|
||||
@@ -431,13 +475,13 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
|
||||
return;
|
||||
}
|
||||
if (!served) {
|
||||
dropStill(el);
|
||||
noteCopy(el, "failed", null, light);
|
||||
emitUnavailableDiagnostic(el, "proxy_playback_failed", originalSrc);
|
||||
return;
|
||||
}
|
||||
noteCopy(el, "ready", null, light);
|
||||
swappedElements.set(el, unproxiedSrc(el));
|
||||
if (lightFacts) lightElements.add(el);
|
||||
swappedElements.set(el, held ? originalAttr : unproxiedSrc(el));
|
||||
loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined);
|
||||
// The shown size may have grown while this copy was being made.
|
||||
if (displayScale !== null) keepSized(el, displayScale);
|
||||
@@ -481,16 +525,14 @@ export function swapToProxy(
|
||||
// A lighter copy waits for no level but its own; a copy the file cannot play without finishes at any level.
|
||||
const live = () =>
|
||||
el.isConnected &&
|
||||
el.getAttribute("src") === originalAttr &&
|
||||
(el.getAttribute("src") === originalAttr ||
|
||||
(heldForCopy.has(el) && el.getAttribute("src") === null)) &&
|
||||
(!lightFacts || previewQuality === level);
|
||||
const job: CopyJob = { originalSrc, originalAttr, variant, lightFacts, level, live };
|
||||
const swap = shownScale().then((scale) => makeCopy(el, job, scale));
|
||||
if (lightFacts) return;
|
||||
// A frame capture holds for the copy as it held for a loading video before, up to the same cap.
|
||||
registerSeekCompletion(
|
||||
Promise.race([swap, new Promise((cap) => wallSetTimeout(cap, HOLD_CAP_MS))]),
|
||||
"video-frame",
|
||||
);
|
||||
holdFrameCaptures(swap);
|
||||
const codecName = entry?.codecName ?? null;
|
||||
const details: Record<string, RuntimeJson> = {
|
||||
asset: originalSrc,
|
||||
@@ -527,7 +569,7 @@ export function maybeProxyProactively(el: HTMLMediaElement): void {
|
||||
const canPlay = entry?.representativeMime ? el.canPlayType(entry.representativeMime) : "";
|
||||
const playable = canPlay === "probably" || canPlay === "maybe";
|
||||
if (entry?.browserHostile && !playable) swapToProxy(el, entry, "proactive");
|
||||
else if (previewQuality < 1) lightenIfHeavy(el, entry);
|
||||
else lightenIfHeavy(el, entry);
|
||||
}
|
||||
|
||||
function lightenIfHeavy(el: HTMLMediaElement, entry: MediaCodecMapEntry | null): void {
|
||||
@@ -538,12 +580,11 @@ function lightenIfHeavy(el: HTMLMediaElement, entry: MediaCodecMapEntry | null):
|
||||
|
||||
function applyPreviewQuality(el: HTMLMediaElement): void {
|
||||
if (isRenderMode(el) || !isVideoElement(el)) return;
|
||||
if (previewQuality === 1 && lightElements.has(el)) return restoreOriginal(el);
|
||||
if (boxedElements.has(el) && displayScale !== null) return keepSized(el, displayScale);
|
||||
if (previewQuality < 1) lightenIfHeavy(el, codecEntryOf(el));
|
||||
lightenIfHeavy(el, codecEntryOf(el));
|
||||
}
|
||||
|
||||
/** The host's preview quality: below 1, heavy videos play lighter copies sized to the shown box times the level. */
|
||||
/** The host's preview quality: below 1, heavy videos play lighter copies at the shown box times the level. */
|
||||
export function setPreviewQuality(level: number): void {
|
||||
if (!isPreviewQuality(level) || level === previewQuality) return;
|
||||
previewQuality = level;
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
isHeldUntilDue,
|
||||
recordOpenedAtInPoint,
|
||||
unproxiedSrc,
|
||||
heldForCopy,
|
||||
} from "./proxySrc";
|
||||
import { createRuntimeStartTimeResolver } from "./startResolver";
|
||||
import { readMediaRenderId } from "./renderFrameSibling";
|
||||
@@ -171,7 +172,7 @@ function holdUntilDue(el: HTMLMediaElement): void {
|
||||
}
|
||||
|
||||
export function giveBackWhenDue(el: HTMLMediaElement): void {
|
||||
if (!isHeldUntilDue(el)) return;
|
||||
if (!isHeldUntilDue(el) || heldForCopy.has(el)) return;
|
||||
const authored = authoredSrcOf(el)!;
|
||||
const start = readMediaStart(el);
|
||||
const written = start > 0 ? `${authored}#t=${start}` : authored;
|
||||
|
||||
@@ -3,10 +3,10 @@ export const swappedElements = new WeakMap<Element, string | null>();
|
||||
|
||||
/** Media elements the runtime opened at their in-point: the src the author wrote, the one written in its
|
||||
* place (null while the clip is held with no src until it is due), and whether it is held again when idle. */
|
||||
const openedAtInPoint = new WeakMap<
|
||||
Element,
|
||||
{ authored: string; written: string | null; holdsWhenIdle: boolean }
|
||||
>();
|
||||
type InPointRecord = { authored: string; written: string | null; holdsWhenIdle: boolean };
|
||||
const openedAtInPoint = new WeakMap<Element, InPointRecord>();
|
||||
export const inPointOf = (el: Element) => openedAtInPoint.get(el);
|
||||
export const heldForCopy = new WeakMap<Element, InPointRecord | undefined>();
|
||||
|
||||
export function recordOpenedAtInPoint(
|
||||
el: Element,
|
||||
|
||||
Reference in New Issue
Block a user