mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
feat(core): preview health names the media element that held play (#7159)
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// Types
|
||||
export type { RuntimeTimelineClipIdentity } from "./runtime/types.js";
|
||||
export type { PlaybackHeldBy, RuntimeTimelineClipIdentity } from "./runtime/types.js";
|
||||
export type {
|
||||
ExecutionMode,
|
||||
Orientation,
|
||||
|
||||
@@ -228,7 +228,7 @@ it("a hold for data does not reload a later clip", () => {
|
||||
});
|
||||
|
||||
it("a film held for data reads as choppy, so the app can lower it", () => {
|
||||
const reports: Array<{ choppy: boolean; waitingShare: number }> = [];
|
||||
const reports: Array<{ choppy: boolean; waitingShare: number; heldBy: unknown }> = [];
|
||||
const { voiceReady, playFor } = playPodcast(0, "video");
|
||||
voiceReady.value = 4;
|
||||
playFor(0.5);
|
||||
@@ -239,12 +239,13 @@ it("a film held for data reads as choppy, so the app can lower it", () => {
|
||||
for (const [message] of post.mock.calls) {
|
||||
const m = message as {
|
||||
type?: string;
|
||||
health: { choppy: boolean; waitingShare: number } | null;
|
||||
health: { choppy: boolean; waitingShare: number; heldBy: unknown } | null;
|
||||
};
|
||||
if (m?.type === "preview-health" && m.health) reports.push(m.health);
|
||||
}
|
||||
expect(reports.at(-1)?.choppy).toBe(true);
|
||||
expect(reports.at(-1)?.waitingShare).toBeGreaterThan(0.9);
|
||||
expect(reports.at(-1)?.heldBy).toMatchObject({ kind: "video", readyState: 1 });
|
||||
});
|
||||
|
||||
it("a video far behind the film lands on it when it starves", () => {
|
||||
|
||||
@@ -26,7 +26,12 @@ import {
|
||||
lengthIsAuthored,
|
||||
stopMediaDownload,
|
||||
} from "./preloadMedia";
|
||||
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
|
||||
import {
|
||||
heldByMedia,
|
||||
notePlaybackFrame,
|
||||
noteDisplayScale,
|
||||
watchMediaFetches,
|
||||
} from "./playbackHealth";
|
||||
import { hasFrozenPicture } from "./frozenPicture";
|
||||
import { createPreviewSeek } from "./previewSeek";
|
||||
import { createPausedVideoFrames, releasePausedVideoFrame } from "./pausedVideoFrames";
|
||||
@@ -3819,6 +3824,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
document.removeEventListener("play", onMediaPlayWakeTransport, true);
|
||||
});
|
||||
runtimeCleanupCallbacks.push(watchAudioLoadErrors(document));
|
||||
runtimeCleanupCallbacks.push(watchMediaFetches(document));
|
||||
|
||||
let lastVisibilityPassSeconds: number | null = null;
|
||||
const syncMediaForCurrentState = (timingRevision?: number) => {
|
||||
@@ -6237,18 +6243,21 @@ export function initSandboxRuntimeModular(): void {
|
||||
const t = clock.now();
|
||||
state.currentTime = t;
|
||||
if (!window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
|
||||
const nowMs = performance.now();
|
||||
const holder =
|
||||
bufferingFor[0] ??
|
||||
mediaClipsInWindow.find(
|
||||
({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
|
||||
)?.el;
|
||||
notePlaybackFrame({
|
||||
nowMs: performance.now(),
|
||||
nowMs,
|
||||
filmTime: t,
|
||||
fps: state.canonicalFps,
|
||||
rate: state.playbackRate,
|
||||
playing: playing || bufferingFor.length > 0,
|
||||
hidden: document.visibilityState === "hidden",
|
||||
waiting:
|
||||
bufferingFor.length > 0 ||
|
||||
mediaClipsInWindow.some(
|
||||
({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
|
||||
),
|
||||
waiting: holder !== undefined,
|
||||
heldBy: holder ? heldByMedia(holder, nowMs) : null,
|
||||
frozen: hasFrozenPicture(mediaClipsInWindow, t),
|
||||
clock: clock.followedClock(),
|
||||
layoutKey: `${window.innerWidth}x${window.innerHeight}@${window.devicePixelRatio}`,
|
||||
|
||||
@@ -4,11 +4,13 @@ vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() }));
|
||||
|
||||
import { postRuntimeMessage } from "./bridge";
|
||||
import {
|
||||
heldByMedia,
|
||||
notePlaybackFrame,
|
||||
noteDisplayScale,
|
||||
PLAYBACK_HEALTH,
|
||||
restartPlaybackHealth,
|
||||
type PlaybackHealthInput,
|
||||
watchMediaFetches,
|
||||
} from "./playbackHealth";
|
||||
|
||||
const posted = vi.mocked(postRuntimeMessage);
|
||||
@@ -25,6 +27,7 @@ type Health = {
|
||||
gapP50Ms: number;
|
||||
gapP95Ms: number;
|
||||
clockSource: string;
|
||||
heldBy: unknown;
|
||||
} | null;
|
||||
const healths = (): Health[] =>
|
||||
posted.mock.calls.map(([message]) =>
|
||||
@@ -116,6 +119,50 @@ describe("playback health", () => {
|
||||
expect(waiting?.gapP95Ms).toBeLessThan(20);
|
||||
});
|
||||
|
||||
it("names the element that last held play in the window, and none when nothing held it", () => {
|
||||
const voice = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 4000 } as const;
|
||||
const clip = { kind: "video", readyState: 2, networkState: 2, fetchMs: null } as const;
|
||||
play(1000, 3, { waiting: true, heldBy: clip });
|
||||
play(4000 + 1000 / 60, 3, { waiting: true, heldBy: voice });
|
||||
expect(healths().at(-1)?.heldBy).toEqual(voice);
|
||||
restartPlaybackHealth();
|
||||
posted.mockClear();
|
||||
play(20_000, 6);
|
||||
expect(healths().at(-1)?.heldBy).toBeNull();
|
||||
});
|
||||
|
||||
it("times an element's fetch from its loadstart, else its resource timing, and says null for one never seen", () => {
|
||||
const now = vi.spyOn(performance, "now").mockReturnValue(1000);
|
||||
const stop = watchMediaFetches(document);
|
||||
const voice = document.createElement("audio");
|
||||
const clip = document.createElement("video");
|
||||
document.body.append(voice, clip);
|
||||
voice.dispatchEvent(new Event("loadstart"));
|
||||
expect(heldByMedia(voice, 4500)).toEqual({
|
||||
kind: "audio",
|
||||
readyState: voice.readyState,
|
||||
networkState: voice.networkState,
|
||||
fetchMs: 3500,
|
||||
});
|
||||
expect(heldByMedia(clip, 4500)).toMatchObject({ kind: "video", fetchMs: null });
|
||||
const early = document.createElement("audio");
|
||||
Object.defineProperty(early, "currentSrc", { value: "http://film/voice.mp3" });
|
||||
const entries = vi
|
||||
.spyOn(performance, "getEntriesByName")
|
||||
.mockImplementation((name) =>
|
||||
name === "http://film/voice.mp3" ? [{ startTime: 500 } as PerformanceEntry] : [],
|
||||
);
|
||||
expect(heldByMedia(early, 4500).fetchMs).toBe(4000);
|
||||
entries.mockRestore();
|
||||
stop();
|
||||
now.mockReturnValue(2000);
|
||||
clip.dispatchEvent(new Event("loadstart"));
|
||||
expect(heldByMedia(clip, 4500).fetchMs).toBeNull();
|
||||
voice.remove();
|
||||
clip.remove();
|
||||
now.mockRestore();
|
||||
});
|
||||
|
||||
it("says which clock the film's time came from", () => {
|
||||
play(1000, 6);
|
||||
expect(healths().at(-1)?.clockSource).toBe("monotonic");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { postRuntimeMessage } from "./bridge";
|
||||
import type { RuntimePreviewHealthMessage } from "./types";
|
||||
import type { PlaybackHeldBy, RuntimePreviewHealthMessage } from "./types";
|
||||
|
||||
/** What counts as a choppy preview; every number of the rule lives here. */
|
||||
export const PLAYBACK_HEALTH = {
|
||||
@@ -18,13 +18,21 @@ export type PlaybackHealthInput = {
|
||||
hidden: boolean;
|
||||
/** A playing video is waiting for data. */
|
||||
waiting: boolean;
|
||||
heldBy?: PlaybackHeldBy | null;
|
||||
frozen?: boolean;
|
||||
clock: "audio" | "webaudio" | "monotonic";
|
||||
/** Anything that changes how heavy a frame is to draw (window size, zoom). */
|
||||
layoutKey: string;
|
||||
};
|
||||
|
||||
type Sample = { atMs: number; gapMs: number; drawn: boolean; waiting: boolean; frozen: boolean };
|
||||
type Sample = {
|
||||
atMs: number;
|
||||
gapMs: number;
|
||||
drawn: boolean;
|
||||
waiting: boolean;
|
||||
frozen: boolean;
|
||||
heldBy: PlaybackHeldBy | null;
|
||||
};
|
||||
type Last = { ms: number; film: number; frame: number };
|
||||
|
||||
const FORWARD_JUMP_SECONDS = 0.5;
|
||||
@@ -35,6 +43,28 @@ let key = "";
|
||||
let posted = false;
|
||||
let postedAtMs = 0;
|
||||
let clockSource: PlaybackHealthInput["clock"] = "monotonic";
|
||||
const fetchStartedAt = new WeakMap<EventTarget, number>();
|
||||
|
||||
export function watchMediaFetches(doc: Document): () => void {
|
||||
const onStart = (event: Event) => {
|
||||
if (event.target) fetchStartedAt.set(event.target, performance.now());
|
||||
};
|
||||
doc.addEventListener("loadstart", onStart, true);
|
||||
return () => doc.removeEventListener("loadstart", onStart, true);
|
||||
}
|
||||
|
||||
const resourceFetchStart = (src: string) => performance.getEntriesByName(src).at(-1)?.startTime;
|
||||
|
||||
export function heldByMedia(el: HTMLMediaElement, nowMs: number): PlaybackHeldBy {
|
||||
const started =
|
||||
fetchStartedAt.get(el) ?? (el.currentSrc ? resourceFetchStart(el.currentSrc) : undefined);
|
||||
return {
|
||||
kind: el.tagName === "VIDEO" ? "video" : "audio",
|
||||
readyState: el.readyState,
|
||||
networkState: el.networkState,
|
||||
fetchMs: started === undefined ? null : Math.round(nowMs - started),
|
||||
};
|
||||
}
|
||||
|
||||
function post(health: RuntimePreviewHealthMessage["health"]): void {
|
||||
postRuntimeMessage({ source: "hf-preview", type: "preview-health", health });
|
||||
@@ -75,6 +105,8 @@ function measure(nowMs: number, fps: number, rate: number): void {
|
||||
const waitingMs = samples.reduce((sum, s) => sum + (s.waiting ? s.gapMs : 0), 0);
|
||||
const frozenMs = samples.reduce((sum, s) => sum + (s.frozen ? s.gapMs : 0), 0);
|
||||
const droppedShare = Math.max(skipped, waitingMs / spanMs);
|
||||
let heldBy: PlaybackHeldBy | null = null;
|
||||
for (const sample of samples) heldBy = sample.heldBy ?? heldBy;
|
||||
const gaps = samples.map((sample) => sample.gapMs).sort((a, b) => a - b);
|
||||
const gapAt = (share: number) => Math.round(gaps[Math.ceil(share * gaps.length) - 1] ?? 0);
|
||||
posted = true;
|
||||
@@ -91,6 +123,7 @@ function measure(nowMs: number, fps: number, rate: number): void {
|
||||
gapP50Ms: gapAt(0.5),
|
||||
gapP95Ms: gapAt(0.95),
|
||||
clockSource,
|
||||
heldBy,
|
||||
mode: "play",
|
||||
});
|
||||
}
|
||||
@@ -130,6 +163,7 @@ export function notePlaybackFrame(input: PlaybackHealthInput): void {
|
||||
drawn: frame !== previous.frame,
|
||||
waiting: input.waiting,
|
||||
frozen: input.frozen === true,
|
||||
heldBy: input.heldBy ?? null,
|
||||
});
|
||||
measure(nowMs, fps, rate);
|
||||
}
|
||||
|
||||
@@ -269,6 +269,15 @@ export type RuntimePreviewCopiesMessage = {
|
||||
} | null;
|
||||
};
|
||||
|
||||
export type FetchNotSeenToStart = null;
|
||||
|
||||
export type PlaybackHeldBy = {
|
||||
kind: "audio" | "video";
|
||||
readyState: number;
|
||||
networkState: number;
|
||||
fetchMs: number | FetchNotSeenToStart;
|
||||
};
|
||||
|
||||
export type RuntimePreviewHealthMessage = {
|
||||
source: "hf-preview";
|
||||
type: "preview-health";
|
||||
@@ -285,6 +294,7 @@ export type RuntimePreviewHealthMessage = {
|
||||
gapP50Ms: number;
|
||||
gapP95Ms: number;
|
||||
clockSource: "audio" | "webaudio" | "monotonic";
|
||||
heldBy: PlaybackHeldBy | null;
|
||||
mode: "play";
|
||||
} | null;
|
||||
};
|
||||
|
||||
@@ -87,4 +87,31 @@ describe("preview quality in the player store", () => {
|
||||
).toBeUndefined();
|
||||
expect(parsePlaybackHealth(health)?.clockSource).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps what held play when the runtime names it in its own shape, and drops any other shape", () => {
|
||||
const health = {
|
||||
droppedShare: 1,
|
||||
seconds: 3,
|
||||
choppy: true,
|
||||
fps: 0,
|
||||
worstFrameMs: 20,
|
||||
mode: "play",
|
||||
};
|
||||
const heldBy = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 3500 };
|
||||
expect(parsePlaybackHealth({ ...health, heldBy })?.heldBy).toEqual(heldBy);
|
||||
expect(
|
||||
parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: null } })?.heldBy,
|
||||
).toEqual({
|
||||
...heldBy,
|
||||
fetchMs: null,
|
||||
});
|
||||
expect(parsePlaybackHealth({ ...health, heldBy: null })?.heldBy).toBeNull();
|
||||
expect(
|
||||
parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: "3500" } })?.heldBy,
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
parsePlaybackHealth({ ...health, heldBy: { ...heldBy, kind: "/a/b.mp3" } })?.heldBy,
|
||||
).toBeUndefined();
|
||||
expect(parsePlaybackHealth(health)?.heldBy).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { StoreApi } from "zustand";
|
||||
import { isPreviewQuality, type PreviewQuality } from "@hyperframes/core/preview-quality";
|
||||
import type { PlaybackHeldBy } from "@hyperframes/core";
|
||||
import type { MediaLoadFailure } from "@hyperframes/core/runtime/protocol";
|
||||
import { readStudioUiPreferences, writeStudioUiPreferences } from "../../utils/studioUiPreferences";
|
||||
|
||||
@@ -25,6 +26,7 @@ export interface PlaybackHealth {
|
||||
gapP50Ms?: number;
|
||||
gapP95Ms?: number;
|
||||
clockSource?: "audio" | "webaudio" | "monotonic";
|
||||
heldBy?: PlaybackHeldBy | null;
|
||||
mode: "play";
|
||||
}
|
||||
|
||||
@@ -118,6 +120,15 @@ export function parsePreviewCopies(value: unknown): PreviewCopies | null {
|
||||
const optionalNumber = (n: unknown) => (typeof n === "number" ? n : undefined);
|
||||
const CLOCK_SOURCES = ["audio", "webaudio", "monotonic"] as const;
|
||||
const optionalClock = (clock: unknown) => CLOCK_SOURCES.find((known) => known === clock);
|
||||
function parseHeldBy(value: unknown): PlaybackHealth["heldBy"] {
|
||||
if (value === null) return null;
|
||||
if (typeof value !== "object") return undefined;
|
||||
const { kind, readyState, networkState, fetchMs } = value as Record<string, unknown>;
|
||||
if (kind !== "audio" && kind !== "video") return undefined;
|
||||
if (typeof readyState !== "number" || typeof networkState !== "number") return undefined;
|
||||
if (typeof fetchMs !== "number" && fetchMs !== null) return undefined;
|
||||
return { kind, readyState, networkState, fetchMs };
|
||||
}
|
||||
|
||||
export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
|
||||
if (typeof value !== "object" || value === null) return null;
|
||||
@@ -137,6 +148,7 @@ export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
|
||||
gapP50Ms: optionalNumber(extra.gapP50Ms),
|
||||
gapP95Ms: optionalNumber(extra.gapP95Ms),
|
||||
clockSource: optionalClock(extra.clockSource),
|
||||
heldBy: parseHeldBy(extra.heldBy),
|
||||
mode: "play",
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user