feat(core): preview health names the media element that held play (#7159)

This commit is contained in:
Miguel Ángel
2026-11-08 02:26:53 +00:00
committed by GitHub
parent 26956ea1f4
commit fd05a3860c
8 changed files with 152 additions and 12 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
// Types
export type { RuntimeTimelineClipIdentity } from "./runtime/types.js";
export type { PlaybackHeldBy, RuntimeTimelineClipIdentity } from "./runtime/types.js";
export type {
ExecutionMode,
Orientation,
@@ -228,7 +228,7 @@ it("a hold for data does not reload a later clip", () => {
});
it("a film held for data reads as choppy, so the app can lower it", () => {
const reports: Array<{ choppy: boolean; waitingShare: number }> = [];
const reports: Array<{ choppy: boolean; waitingShare: number; heldBy: unknown }> = [];
const { voiceReady, playFor } = playPodcast(0, "video");
voiceReady.value = 4;
playFor(0.5);
@@ -239,12 +239,13 @@ it("a film held for data reads as choppy, so the app can lower it", () => {
for (const [message] of post.mock.calls) {
const m = message as {
type?: string;
health: { choppy: boolean; waitingShare: number } | null;
health: { choppy: boolean; waitingShare: number; heldBy: unknown } | null;
};
if (m?.type === "preview-health" && m.health) reports.push(m.health);
}
expect(reports.at(-1)?.choppy).toBe(true);
expect(reports.at(-1)?.waitingShare).toBeGreaterThan(0.9);
expect(reports.at(-1)?.heldBy).toMatchObject({ kind: "video", readyState: 1 });
});
it("a video far behind the film lands on it when it starves", () => {
+16 -7
View File
@@ -26,7 +26,12 @@ import {
lengthIsAuthored,
stopMediaDownload,
} from "./preloadMedia";
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
import {
heldByMedia,
notePlaybackFrame,
noteDisplayScale,
watchMediaFetches,
} from "./playbackHealth";
import { hasFrozenPicture } from "./frozenPicture";
import { createPreviewSeek } from "./previewSeek";
import { createPausedVideoFrames, releasePausedVideoFrame } from "./pausedVideoFrames";
@@ -3819,6 +3824,7 @@ export function initSandboxRuntimeModular(): void {
document.removeEventListener("play", onMediaPlayWakeTransport, true);
});
runtimeCleanupCallbacks.push(watchAudioLoadErrors(document));
runtimeCleanupCallbacks.push(watchMediaFetches(document));
let lastVisibilityPassSeconds: number | null = null;
const syncMediaForCurrentState = (timingRevision?: number) => {
@@ -6237,18 +6243,21 @@ export function initSandboxRuntimeModular(): void {
const t = clock.now();
state.currentTime = t;
if (!window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
const nowMs = performance.now();
const holder =
bufferingFor[0] ??
mediaClipsInWindow.find(
({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
)?.el;
notePlaybackFrame({
nowMs: performance.now(),
nowMs,
filmTime: t,
fps: state.canonicalFps,
rate: state.playbackRate,
playing: playing || bufferingFor.length > 0,
hidden: document.visibilityState === "hidden",
waiting:
bufferingFor.length > 0 ||
mediaClipsInWindow.some(
({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
),
waiting: holder !== undefined,
heldBy: holder ? heldByMedia(holder, nowMs) : null,
frozen: hasFrozenPicture(mediaClipsInWindow, t),
clock: clock.followedClock(),
layoutKey: `${window.innerWidth}x${window.innerHeight}@${window.devicePixelRatio}`,
@@ -4,11 +4,13 @@ vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() }));
import { postRuntimeMessage } from "./bridge";
import {
heldByMedia,
notePlaybackFrame,
noteDisplayScale,
PLAYBACK_HEALTH,
restartPlaybackHealth,
type PlaybackHealthInput,
watchMediaFetches,
} from "./playbackHealth";
const posted = vi.mocked(postRuntimeMessage);
@@ -25,6 +27,7 @@ type Health = {
gapP50Ms: number;
gapP95Ms: number;
clockSource: string;
heldBy: unknown;
} | null;
const healths = (): Health[] =>
posted.mock.calls.map(([message]) =>
@@ -116,6 +119,50 @@ describe("playback health", () => {
expect(waiting?.gapP95Ms).toBeLessThan(20);
});
it("names the element that last held play in the window, and none when nothing held it", () => {
const voice = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 4000 } as const;
const clip = { kind: "video", readyState: 2, networkState: 2, fetchMs: null } as const;
play(1000, 3, { waiting: true, heldBy: clip });
play(4000 + 1000 / 60, 3, { waiting: true, heldBy: voice });
expect(healths().at(-1)?.heldBy).toEqual(voice);
restartPlaybackHealth();
posted.mockClear();
play(20_000, 6);
expect(healths().at(-1)?.heldBy).toBeNull();
});
it("times an element's fetch from its loadstart, else its resource timing, and says null for one never seen", () => {
const now = vi.spyOn(performance, "now").mockReturnValue(1000);
const stop = watchMediaFetches(document);
const voice = document.createElement("audio");
const clip = document.createElement("video");
document.body.append(voice, clip);
voice.dispatchEvent(new Event("loadstart"));
expect(heldByMedia(voice, 4500)).toEqual({
kind: "audio",
readyState: voice.readyState,
networkState: voice.networkState,
fetchMs: 3500,
});
expect(heldByMedia(clip, 4500)).toMatchObject({ kind: "video", fetchMs: null });
const early = document.createElement("audio");
Object.defineProperty(early, "currentSrc", { value: "http://film/voice.mp3" });
const entries = vi
.spyOn(performance, "getEntriesByName")
.mockImplementation((name) =>
name === "http://film/voice.mp3" ? [{ startTime: 500 } as PerformanceEntry] : [],
);
expect(heldByMedia(early, 4500).fetchMs).toBe(4000);
entries.mockRestore();
stop();
now.mockReturnValue(2000);
clip.dispatchEvent(new Event("loadstart"));
expect(heldByMedia(clip, 4500).fetchMs).toBeNull();
voice.remove();
clip.remove();
now.mockRestore();
});
it("says which clock the film's time came from", () => {
play(1000, 6);
expect(healths().at(-1)?.clockSource).toBe("monotonic");
+36 -2
View File
@@ -1,5 +1,5 @@
import { postRuntimeMessage } from "./bridge";
import type { RuntimePreviewHealthMessage } from "./types";
import type { PlaybackHeldBy, RuntimePreviewHealthMessage } from "./types";
/** What counts as a choppy preview; every number of the rule lives here. */
export const PLAYBACK_HEALTH = {
@@ -18,13 +18,21 @@ export type PlaybackHealthInput = {
hidden: boolean;
/** A playing video is waiting for data. */
waiting: boolean;
heldBy?: PlaybackHeldBy | null;
frozen?: boolean;
clock: "audio" | "webaudio" | "monotonic";
/** Anything that changes how heavy a frame is to draw (window size, zoom). */
layoutKey: string;
};
type Sample = { atMs: number; gapMs: number; drawn: boolean; waiting: boolean; frozen: boolean };
type Sample = {
atMs: number;
gapMs: number;
drawn: boolean;
waiting: boolean;
frozen: boolean;
heldBy: PlaybackHeldBy | null;
};
type Last = { ms: number; film: number; frame: number };
const FORWARD_JUMP_SECONDS = 0.5;
@@ -35,6 +43,28 @@ let key = "";
let posted = false;
let postedAtMs = 0;
let clockSource: PlaybackHealthInput["clock"] = "monotonic";
const fetchStartedAt = new WeakMap<EventTarget, number>();
export function watchMediaFetches(doc: Document): () => void {
const onStart = (event: Event) => {
if (event.target) fetchStartedAt.set(event.target, performance.now());
};
doc.addEventListener("loadstart", onStart, true);
return () => doc.removeEventListener("loadstart", onStart, true);
}
const resourceFetchStart = (src: string) => performance.getEntriesByName(src).at(-1)?.startTime;
export function heldByMedia(el: HTMLMediaElement, nowMs: number): PlaybackHeldBy {
const started =
fetchStartedAt.get(el) ?? (el.currentSrc ? resourceFetchStart(el.currentSrc) : undefined);
return {
kind: el.tagName === "VIDEO" ? "video" : "audio",
readyState: el.readyState,
networkState: el.networkState,
fetchMs: started === undefined ? null : Math.round(nowMs - started),
};
}
function post(health: RuntimePreviewHealthMessage["health"]): void {
postRuntimeMessage({ source: "hf-preview", type: "preview-health", health });
@@ -75,6 +105,8 @@ function measure(nowMs: number, fps: number, rate: number): void {
const waitingMs = samples.reduce((sum, s) => sum + (s.waiting ? s.gapMs : 0), 0);
const frozenMs = samples.reduce((sum, s) => sum + (s.frozen ? s.gapMs : 0), 0);
const droppedShare = Math.max(skipped, waitingMs / spanMs);
let heldBy: PlaybackHeldBy | null = null;
for (const sample of samples) heldBy = sample.heldBy ?? heldBy;
const gaps = samples.map((sample) => sample.gapMs).sort((a, b) => a - b);
const gapAt = (share: number) => Math.round(gaps[Math.ceil(share * gaps.length) - 1] ?? 0);
posted = true;
@@ -91,6 +123,7 @@ function measure(nowMs: number, fps: number, rate: number): void {
gapP50Ms: gapAt(0.5),
gapP95Ms: gapAt(0.95),
clockSource,
heldBy,
mode: "play",
});
}
@@ -130,6 +163,7 @@ export function notePlaybackFrame(input: PlaybackHealthInput): void {
drawn: frame !== previous.frame,
waiting: input.waiting,
frozen: input.frozen === true,
heldBy: input.heldBy ?? null,
});
measure(nowMs, fps, rate);
}
+10
View File
@@ -269,6 +269,15 @@ export type RuntimePreviewCopiesMessage = {
} | null;
};
export type FetchNotSeenToStart = null;
export type PlaybackHeldBy = {
kind: "audio" | "video";
readyState: number;
networkState: number;
fetchMs: number | FetchNotSeenToStart;
};
export type RuntimePreviewHealthMessage = {
source: "hf-preview";
type: "preview-health";
@@ -285,6 +294,7 @@ export type RuntimePreviewHealthMessage = {
gapP50Ms: number;
gapP95Ms: number;
clockSource: "audio" | "webaudio" | "monotonic";
heldBy: PlaybackHeldBy | null;
mode: "play";
} | null;
};
@@ -87,4 +87,31 @@ describe("preview quality in the player store", () => {
).toBeUndefined();
expect(parsePlaybackHealth(health)?.clockSource).toBeUndefined();
});
it("keeps what held play when the runtime names it in its own shape, and drops any other shape", () => {
const health = {
droppedShare: 1,
seconds: 3,
choppy: true,
fps: 0,
worstFrameMs: 20,
mode: "play",
};
const heldBy = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 3500 };
expect(parsePlaybackHealth({ ...health, heldBy })?.heldBy).toEqual(heldBy);
expect(
parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: null } })?.heldBy,
).toEqual({
...heldBy,
fetchMs: null,
});
expect(parsePlaybackHealth({ ...health, heldBy: null })?.heldBy).toBeNull();
expect(
parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: "3500" } })?.heldBy,
).toBeUndefined();
expect(
parsePlaybackHealth({ ...health, heldBy: { ...heldBy, kind: "/a/b.mp3" } })?.heldBy,
).toBeUndefined();
expect(parsePlaybackHealth(health)?.heldBy).toBeUndefined();
});
});
@@ -1,5 +1,6 @@
import type { StoreApi } from "zustand";
import { isPreviewQuality, type PreviewQuality } from "@hyperframes/core/preview-quality";
import type { PlaybackHeldBy } from "@hyperframes/core";
import type { MediaLoadFailure } from "@hyperframes/core/runtime/protocol";
import { readStudioUiPreferences, writeStudioUiPreferences } from "../../utils/studioUiPreferences";
@@ -25,6 +26,7 @@ export interface PlaybackHealth {
gapP50Ms?: number;
gapP95Ms?: number;
clockSource?: "audio" | "webaudio" | "monotonic";
heldBy?: PlaybackHeldBy | null;
mode: "play";
}
@@ -118,6 +120,15 @@ export function parsePreviewCopies(value: unknown): PreviewCopies | null {
const optionalNumber = (n: unknown) => (typeof n === "number" ? n : undefined);
const CLOCK_SOURCES = ["audio", "webaudio", "monotonic"] as const;
const optionalClock = (clock: unknown) => CLOCK_SOURCES.find((known) => known === clock);
function parseHeldBy(value: unknown): PlaybackHealth["heldBy"] {
if (value === null) return null;
if (typeof value !== "object") return undefined;
const { kind, readyState, networkState, fetchMs } = value as Record<string, unknown>;
if (kind !== "audio" && kind !== "video") return undefined;
if (typeof readyState !== "number" || typeof networkState !== "number") return undefined;
if (typeof fetchMs !== "number" && fetchMs !== null) return undefined;
return { kind, readyState, networkState, fetchMs };
}
export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
if (typeof value !== "object" || value === null) return null;
@@ -137,6 +148,7 @@ export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
gapP50Ms: optionalNumber(extra.gapP50Ms),
gapP95Ms: optionalNumber(extra.gapP95Ms),
clockSource: optionalClock(extra.clockSource),
heldBy: parseHeldBy(extra.heldBy),
mode: "play",
};
}