fix(core): show a paused video frame with the same color as the playing video on Mac (#7584)

* fix(core): show a paused video frame with the same color as the playing video on Mac

On macOS Chromium the video layer shows 8-bit SDR video with Apple's video gamma while a canvas shows the decoded pixels as sRGB, so a paused frame looked lighter-to-darker than the playing video. One module now owns the conversion and the paused stand-in canvas is displayed through it.

* fix(core): drop a comment line to keep the file's comment share down

* fix(core): correct a paused frame only for a source the scan reports as 8-bit, on both hold paths

Source facts now come from the shared media lookup (origin-aware, exact then longest suffix then normalized), unknown depth is left alone, and the native-only seek hold gets the same display helper.

* fix(core): shorten two comments to the line limit
This commit is contained in:
Miguel Ángel
2026-11-12 05:44:27 +00:00
committed by GitHub
parent 4fad43db20
commit 303a4c4771
9 changed files with 347 additions and 95 deletions
+1 -1
View File
@@ -10,7 +10,6 @@ const syncRuntimeMedia = (
) => syncRuntimeMediaWithDuration({ getCompositionDuration: () => 0, ...params });
import {
deriveCodecMapKey,
handleErrorForProxy,
describeVideoSources,
handleMetadataForProxy,
@@ -18,6 +17,7 @@ import {
swapToProxy,
type MediaCodecMapEntry,
} from "./mediaProxy";
import { deriveCodecMapKey } from "./mediaSourceLookup";
import { recordOpenedAtInPoint, unproxiedSrc } from "./proxySrc";
vi.mock("./bridge", () => ({
+6 -94
View File
@@ -10,6 +10,12 @@ import {
import { findInjectedRenderFrame } from "./renderFrameSibling";
import type { RuntimeJson } from "./types";
import { isVideoElement } from "./domRealm";
import {
currentSrcValue,
deriveCodecMapKey,
lookupCodecMapEntry,
sourceFactsOf,
} from "./mediaSourceLookup";
import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc";
import { giveBackWhenDue } from "./preloadMedia";
import { restartPlaybackHealth } from "./playbackHealth";
@@ -52,7 +58,6 @@ export type MediaCodecMapEntry = {
declare global {
interface Window {
__HF_MEDIA_CODEC_MAP__?: Record<string, MediaCodecMapEntry>;
__HF_MEDIA_SOURCE_FACTS__?: Record<string, PreviewSourceFacts>;
}
}
@@ -78,10 +83,6 @@ const unavailableDiagnosedElements = new WeakSet<HTMLMediaElement>();
// keep that src, so nothing that reads it (timeline, player mirror) loads a copy still being made.
const proxyRequested = new WeakMap<HTMLMediaElement, string | null>();
function currentSrcValue(el: HTMLMediaElement): string {
return el.currentSrc || el.src;
}
// Below 1, heavy files get lighter copies at the shown box times this level; 1 is the preview as it always was.
let previewQuality: PreviewQuality = 1;
const lightElements = new WeakSet<HTMLMediaElement>();
@@ -143,12 +144,6 @@ function codecEntryOf(el: HTMLMediaElement): MediaCodecMapEntry | null {
return map && key !== null ? lookupCodecMapEntry(key, map) : null;
}
function sourceFactsOf(el: HTMLMediaElement): PreviewSourceFacts | null {
const facts = window.__HF_MEDIA_SOURCE_FACTS__;
const key = deriveCodecMapKey(el);
return facts && key !== null ? lookupCodecMapEntry(key, facts) : null;
}
// fallow-ignore-next-line complexity
function keepSized(el: HTMLMediaElement, scale: number): void {
const copy = boxedElements.get(el);
@@ -315,89 +310,6 @@ export function isRenderMode(el: HTMLMediaElement): boolean {
return isVideoElement(el) && !!findInjectedRenderFrame(el);
}
/**
* Resolve an element's codec-map key: the served-root-relative,
* percent-decoded, query-string-stripped URL pathname (per the server
* contract KTD). Returns null when there's no usable src, or the src is
* cross-origin — the server can't have scanned (or proxy) a file it doesn't
* host, so there is never a map entry to look up.
*/
export function deriveCodecMapKey(el: HTMLMediaElement): string | null {
const raw = currentSrcValue(el);
if (!raw) return null;
let url: URL;
try {
url = new URL(raw, document.baseURI);
} catch {
return null;
}
if (url.origin !== window.location.origin) return null;
try {
return decodeURIComponent(url.pathname);
} catch {
return url.pathname;
}
}
/**
* The composition is served under a route prefix on some surfaces —
* studio-server's preview route injects `<base href="/api/projects/:id/preview/">`
* (packages/studio-server/src/routes/preview.ts) so every relative asset
* resolves through that prefix — while the codec map (per the server
* contract) is keyed by project-root-relative pathnames with no prefix.
* `play` and the CLI's static project server serve from the root with no
* such prefix, where an exact match already succeeds. So: try exact first
* (covers unprefixed servers), then fall back to the longest map key that
* is a suffix of the pathname. No separate segment-boundary check is
* needed: every key is contractually root-relative and leading-slash
* (`"/assets/x.mp4"`, never `"assets/x.mp4"`), so a suffix match's boundary
* is always that leading `/` itself — `pathname.endsWith(key)` can't match
* a partial segment (e.g. key `"/foo.mp4"` can never match a pathname
* ending in `"/notfoo.mp4"`, since that would require the literal substring
* `"/foo.mp4"` to appear where the last `/` already fell a segment later).
*/
function lookupLongestExactSuffix<T>(pathname: string, map: Record<string, T>): T | null {
let bestKey: string | null = null;
for (const key of Object.keys(map)) {
if (!pathname.endsWith(key)) continue;
if (bestKey === null || key.length > bestKey.length) bestKey = key;
}
return bestKey ? (map[bestKey] ?? null) : null;
}
function lookupNormalizedSuffix<T>(pathname: string, map: Record<string, T>): T | null {
// Case-insensitive filesystems and Unicode-normalizing filesystems can
// serve an authored URL whose spelling differs from the canonical path
// ffprobe returned (Clip.mp4 vs clip.mp4, NFC vs NFD). Exact matching stays
// authoritative; this normalized fallback is used only when it finds one
// unambiguous longest suffix, so case-sensitive projects containing both
// spellings never select the wrong asset.
const normalizedPathname = pathname.normalize("NFC").toLowerCase();
let normalizedBestLength = -1;
let normalizedBest: T | null = null;
let ambiguous = false;
for (const [key, entry] of Object.entries(map)) {
const normalizedKey = key.normalize("NFC").toLowerCase();
if (!normalizedPathname.endsWith(normalizedKey)) continue;
if (normalizedKey.length > normalizedBestLength) {
normalizedBestLength = normalizedKey.length;
normalizedBest = entry;
ambiguous = false;
} else if (normalizedKey.length === normalizedBestLength) {
ambiguous = true;
}
}
return ambiguous ? null : normalizedBest;
}
function lookupCodecMapEntry<T>(pathname: string, map: Record<string, T>): T | null {
return (
map[pathname] ??
lookupLongestExactSuffix(pathname, map) ??
lookupNormalizedSuffix(pathname, map)
);
}
const stills = new WeakSet<HTMLMediaElement>();
/** While a file the browser cannot play is being copied, its own frame at the playhead stands in as the poster. */
@@ -0,0 +1,73 @@
import type { PreviewSourceFacts } from "../previewProxyBox";
declare global {
interface Window {
__HF_MEDIA_SOURCE_FACTS__?: Record<string, PreviewSourceFacts>;
}
}
export function currentSrcValue(el: HTMLMediaElement): string {
return el.currentSrc || el.src;
}
/** The decoded, query-free pathname the server's maps are keyed by; null with no src or a cross-origin one. */
export function deriveCodecMapKey(el: HTMLMediaElement): string | null {
const raw = currentSrcValue(el);
if (!raw) return null;
let url: URL;
try {
url = new URL(raw, document.baseURI);
} catch {
return null;
}
if (url.origin !== window.location.origin) return null;
try {
return decodeURIComponent(url.pathname);
} catch {
return url.pathname;
}
}
/** The keys carry no route prefix, so the longest key that ends the pathname wins. */
function lookupLongestExactSuffix<T>(pathname: string, map: Record<string, T>): T | null {
let bestKey: string | null = null;
for (const key of Object.keys(map)) {
if (!pathname.endsWith(key)) continue;
if (bestKey === null || key.length > bestKey.length) bestKey = key;
}
return bestKey ? (map[bestKey] ?? null) : null;
}
function lookupNormalizedSuffix<T>(pathname: string, map: Record<string, T>): T | null {
// Exact stays authoritative; a case or Unicode spelling difference matches only when one longest key is unambiguous.
const normalizedPathname = pathname.normalize("NFC").toLowerCase();
let normalizedBestLength = -1;
let normalizedBest: T | null = null;
let ambiguous = false;
for (const [key, entry] of Object.entries(map)) {
const normalizedKey = key.normalize("NFC").toLowerCase();
if (!normalizedPathname.endsWith(normalizedKey)) continue;
if (normalizedKey.length > normalizedBestLength) {
normalizedBestLength = normalizedKey.length;
normalizedBest = entry;
ambiguous = false;
} else if (normalizedKey.length === normalizedBestLength) {
ambiguous = true;
}
}
return ambiguous ? null : normalizedBest;
}
export function lookupCodecMapEntry<T>(pathname: string, map: Record<string, T>): T | null {
return (
map[pathname] ??
lookupLongestExactSuffix(pathname, map) ??
lookupNormalizedSuffix(pathname, map)
);
}
export function sourceFactsOf(el: HTMLMediaElement): PreviewSourceFacts | null {
const facts = window.__HF_MEDIA_SOURCE_FACTS__;
const key = deriveCodecMapKey(el);
return facts && key !== null ? lookupCodecMapEntry(key, facts) : null;
}
@@ -0,0 +1,128 @@
import { afterEach, expect, it, vi } from "vitest";
import {
NATIVE_VIDEO_LOOK_FILTER_ID,
applyNativeVideoLook,
nativeVideoLookNeeded,
nativeVideoLookValue,
} from "./nativeVideoLook";
const MAC_CHROME =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
const MAC_SAFARI =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15";
const WINDOWS_CHROME =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
const asBrowser = (userAgent: string) => vi.stubGlobal("navigator", { userAgent });
afterEach(() => {
vi.unstubAllGlobals();
document.body.innerHTML = "";
});
// A 16-step gray ramp (BT.709 limited range, steps of 17) measured on screen on a Mac: what a canvas shows
// for each step, and what the video layer shows for the same step.
const MEASURED_ON_CANVAS = [
0, 17, 33, 51, 67, 85, 102, 118, 136, 152, 170, 187, 203, 221, 237, 255,
];
const MEASURED_ON_VIDEO_LAYER = [
0, 15, 37, 58, 76, 95, 113, 130, 147, 162, 179, 195, 209, 225, 239, 255,
];
it("turns what a canvas shows into what the video layer shows, within one step on every measured gray", () => {
MEASURED_ON_CANVAS.forEach((shown, step) => {
expect(
Math.abs(nativeVideoLookValue(shown) - MEASURED_ON_VIDEO_LAYER[step]!),
).toBeLessThanOrEqual(1);
});
});
it("keeps black and white where they are", () => {
expect([nativeVideoLookValue(0), nativeVideoLookValue(255)]).toEqual([0, 255]);
});
it("applies to Chromium on a Mac and to nothing else", () => {
asBrowser(MAC_CHROME);
expect(nativeVideoLookNeeded()).toBe(true);
asBrowser(MAC_SAFARI);
expect(nativeVideoLookNeeded()).toBe(false);
asBrowser(WINDOWS_CHROME);
expect(nativeVideoLookNeeded()).toBe(false);
});
const factsFor = (entries: Record<string, number | undefined>) => {
window.__HF_MEDIA_SOURCE_FACTS__ = Object.fromEntries(
Object.entries(entries).map(([path, bitDepth]) => [
path,
{ width: 64, height: 36, bitrate: 1, bitDepth: bitDepth as number },
]),
);
};
const videoAt = (src: string) => {
const video = document.createElement("video");
video.setAttribute("src", src);
document.body.append(video);
return video;
};
const lookOf = (video: HTMLVideoElement, authored = "none") => {
const canvas = document.createElement("canvas");
applyNativeVideoLook(canvas, video, authored);
return canvas.style.filter;
};
const LOOK = new RegExp(`^url\\("?#${NATIVE_VIDEO_LOOK_FILTER_ID}"?\\)`);
afterEach(() => {
delete window.__HF_MEDIA_SOURCE_FACTS__;
});
it("puts the look ahead of an authored filter and defines it once", () => {
asBrowser(MAC_CHROME);
factsFor({ "/a.mp4": 8 });
const video = videoAt("/a.mp4");
expect(lookOf(video)).toMatch(new RegExp(`${LOOK.source}$`));
expect(lookOf(video, "blur(2px)")).toMatch(new RegExp(`${LOOK.source} blur\\(2px\\)$`));
expect(document.querySelectorAll(`#${NATIVE_VIDEO_LOOK_FILTER_ID}`)).toHaveLength(1);
// Saved source drops every element carrying the preview-canvas mark, so the filter never reaches a project file.
expect(document.querySelector("svg")?.hasAttribute("data-hf-preview-video-canvas")).toBe(true);
});
it("leaves the canvas alone where the video layer already agrees with it", () => {
asBrowser(WINDOWS_CHROME);
factsFor({ "/a.mp4": 8 });
expect(lookOf(videoAt("/a.mp4"), "blur(2px)")).toBe("");
expect(document.getElementById(NATIVE_VIDEO_LOOK_FILTER_ID)).toBeNull();
});
it("applies only to a source known to be 8-bit", () => {
asBrowser(MAC_CHROME);
factsFor({ "/ten.mp4": 10, "/unknown.mp4": undefined });
expect(lookOf(videoAt("/ten.mp4"))).toBe("");
expect(lookOf(videoAt("/unknown.mp4"))).toBe("");
});
it("leaves a source alone when the scan has no facts for it", () => {
asBrowser(MAC_CHROME);
expect(lookOf(videoAt("/a.mp4"))).toBe("");
factsFor({ "/other.mp4": 8 });
expect(lookOf(videoAt("/a.mp4"))).toBe("");
});
it("leaves a remote source alone even when its path matches a scanned file", () => {
asBrowser(MAC_CHROME);
factsFor({ "/a.mp4": 8 });
expect(lookOf(videoAt("https://cdn.example.com/a.mp4"))).toBe("");
});
it("finds a source's facts the way the media proxy does: prefix, case and normalization", () => {
asBrowser(MAC_CHROME);
factsFor({ "/assets/a.mp4": 8, "/Original.WEBM": 10, "/Café.mp4": 8 });
expect(lookOf(videoAt("/api/projects/p/preview/assets/a.mp4"))).toMatch(LOOK);
expect(lookOf(videoAt("/original.webm"))).toBe("");
expect(lookOf(videoAt(`/Cafe\u0301.mp4`))).toMatch(LOOK);
});
it("does not guess between two equally good normalized matches", () => {
asBrowser(MAC_CHROME);
factsFor({ "/Clip.mp4": 8, "/clip.MP4": 8 });
expect(lookOf(videoAt("/CLIP.mp4"))).toBe("");
});
@@ -0,0 +1,59 @@
import { isWebKit } from "./mediaClock";
import { PREVIEW_VIDEO_CANVAS_ATTR } from "../studioPreviewMark";
import { sourceFactsOf } from "./mediaSourceLookup";
/** The one owner of making a canvas frame look like the browser's own video layer, which on macOS Chromium
* shows 8-bit SDR video with Apple's gamma (about 1.961) where a canvas shows sRGB. Display-only, so canvas
* readers keep the pixels a playing video gives them. */
export const NATIVE_VIDEO_LOOK_FILTER_ID = "__hf_native_video_look";
const VIDEO_LAYER_GAMMA = 1.961;
const srgbEncode = (linear: number): number =>
linear <= 0.0031308 ? 12.92 * linear : 1.055 * Math.pow(linear, 1 / 2.4) - 0.055;
/** The 8-bit value the video layer shows for a canvas value, 0 to 255. */
export function nativeVideoLookValue(canvasValue: number): number {
return Math.round(255 * srgbEncode(Math.pow(canvasValue / 255, VIDEO_LAYER_GAMMA)));
}
/** Chromium on macOS: the only browser measured to draw a canvas differently from its video layer. */
export function nativeVideoLookNeeded(): boolean {
const { userAgent } = navigator;
return /Macintosh/.test(userAgent) && !isWebKit() && /Chrom(e|ium)\//.test(userAgent);
}
/** Only a source the studio's scan positively reports as 8-bit is measured; unknown or higher depth is left alone. */
function isKnownSdrSource(video: HTMLVideoElement): boolean {
const bitDepth = sourceFactsOf(video)?.bitDepth;
return typeof bitDepth === "number" && bitDepth > 0 && bitDepth <= 8;
}
function ensureFilter(doc: Document): void {
if (doc.getElementById(NATIVE_VIDEO_LOOK_FILTER_ID)) return;
const values = Array.from({ length: 256 }, (_, level) =>
(nativeVideoLookValue(level) / 255).toFixed(4),
).join(" ");
const channel = (name: string) => `<feFunc${name} type="table" tableValues="${values}"/>`;
const holder = doc.createElement("div");
holder.innerHTML =
`<svg width="0" height="0" style="position:absolute" aria-hidden="true" data-hyperframes-ignore data-hf-ignore ` +
`data-layout-ignore data-hyperframes-picker-ignore ${PREVIEW_VIDEO_CANVAS_ATTR}><filter id="${NATIVE_VIDEO_LOOK_FILTER_ID}" ` +
`color-interpolation-filters="sRGB"><feComponentTransfer>${channel("R")}${channel("G")}${channel("B")}` +
`</feComponentTransfer></filter></svg>`;
(doc.body ?? doc.documentElement).append(holder.firstElementChild as Element);
}
/** Gives the visible stand-in canvas the look its video layer has, ahead of any authored filter. */
export function applyNativeVideoLook(
canvas: HTMLCanvasElement,
video: HTMLVideoElement,
authoredFilter: string,
): void {
if (!nativeVideoLookNeeded() || !isKnownSdrSource(video)) return;
ensureFilter(canvas.ownerDocument);
const look = `url(#${NATIVE_VIDEO_LOOK_FILTER_ID})`;
canvas.style.setProperty(
"filter",
authoredFilter === "none" ? look : `${look} ${authoredFilter}`,
);
}
@@ -267,3 +267,47 @@ it("does not retire an adopted decoded picture when its native snapshot lease en
expect(ownsPausedVideoTime(video)).toBe(true);
expect(video.currentTime).toBe(3);
});
const MAC_CHROME =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
it("shows a paused frame with the look its video layer has, and the authored filter after it", async () => {
vi.stubGlobal("navigator", { userAgent: MAC_CHROME });
window.__HF_MEDIA_SOURCE_FACTS__ = {
"/original.webm": { width: 64, height: 36, bitDepth: 8, bitrate: 1 },
};
const { session, video, request } = fixture();
video.style.filter = "blur(2px)";
const batch = await session.prepare(request, new AbortController().signal);
batch!.present();
batch!.refresh();
expect(getPausedVideoFrame(video)!.style.filter).toMatch(
/^url\("?#__hf_native_video_look"?\) blur\(2px\)$/,
);
delete window.__HF_MEDIA_SOURCE_FACTS__;
vi.unstubAllGlobals();
});
it("leaves the paused frame of an HDR source as decoded", async () => {
vi.stubGlobal("navigator", { userAgent: MAC_CHROME });
window.__HF_MEDIA_SOURCE_FACTS__ = {
"/original.webm": { width: 64, height: 36, bitDepth: 10, bitrate: 1 },
};
const { session, video, request } = fixture();
const batch = await session.prepare(request, new AbortController().signal);
batch!.present();
batch!.refresh();
expect(getPausedVideoFrame(video)!.style.filter).not.toContain("__hf_native_video_look");
delete window.__HF_MEDIA_SOURCE_FACTS__;
vi.unstubAllGlobals();
});
it("leaves the paused frame of a source the scan did not report as it was decoded", async () => {
vi.stubGlobal("navigator", { userAgent: MAC_CHROME });
const { session, video, request } = fixture();
const batch = await session.prepare(request, new AbortController().signal);
batch!.present();
batch!.refresh();
expect(getPausedVideoFrame(video)!.style.filter).not.toContain("__hf_native_video_look");
vi.unstubAllGlobals();
});
@@ -1,5 +1,6 @@
import { MEDIA_ELEMENT_LEAD_SECONDS } from "./mediaClock";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { applyNativeVideoLook } from "./nativeVideoLook";
import {
installPausedVideoCallbacks,
stagePausedVideoCallbacks,
@@ -156,6 +157,7 @@ function refresh(entry: Presented): void {
canvas.style.top = `${video.offsetTop}px`;
canvas.style.margin = "0";
}
applyNativeVideoLook(canvas, video, css.filter || "none");
canvas.style.setProperty("pointer-events", "none", "important");
canvas.style.setProperty("animation", "none", "important");
canvas.style.setProperty("transition", "none", "important");
@@ -237,3 +237,35 @@ it("retires the hard Play timer after native readiness publishes", async () => {
expect(commit).toHaveBeenCalledWith(3);
expect(vi.getTimerCount()).toBe(0);
});
it("gives the native-only held picture the look of its video layer", async () => {
vi.stubGlobal("navigator", {
userAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
});
window.__HF_MEDIA_SOURCE_FACTS__ = {
"/clip.mp4": { width: 64, height: 36, bitDepth: 8, bitrate: 1 },
};
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
drawImage: vi.fn(),
} as unknown as CanvasRenderingContext2D);
const { job, video, host, native } = fixture();
video.setAttribute("src", "/clip.mp4");
let ready = 4;
Object.defineProperties(video, {
readyState: { get: () => ready },
videoWidth: { value: 64 },
videoHeight: { value: 36 },
});
host.videosToHold = () => [video];
host.frames.available = () => false;
native.mockImplementation(() => {
ready = 1;
});
const settled = job.handoff(3);
expect(document.querySelector("canvas")?.style.filter).toMatch(/__hf_native_video_look/);
job.cancel();
await settled;
delete window.__HF_MEDIA_SOURCE_FACTS__;
vi.unstubAllGlobals();
});
+2
View File
@@ -3,6 +3,7 @@ import { isUnplayable, mediaSourceTimeAt } from "./media";
import type { RuntimeMediaClip } from "./types";
import { isVideoElement } from "./domRealm";
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
import { applyNativeVideoLook } from "./nativeVideoLook";
import {
nativeSnapshotStyle,
captureNativeVideoFrame,
@@ -20,6 +21,7 @@ function holdVideoFrame(video: HTMLVideoElement): (() => void) | undefined {
const canvas = captureNativeVideoFrame(video);
if (!canvas) return;
for (const name of Array.from(css)) canvas.style.setProperty(name, css.getPropertyValue(name));
applyNativeVideoLook(canvas, video, css.filter || "none");
canvas.style.setProperty("animation", "none", "important");
canvas.style.setProperty("transition", "none", "important");
canvas.style.setProperty("pointer-events", "none", "important");