feat(studio): scrub full-quality video with decoded preview frames (#7379)

* feat(studio): scrub full-quality video with decoded preview frames

* fix(studio): release scrub decoders before native playback

* fix(studio): bound paused fallback and tolerate decoder load failure

* fix(studio): recover stalled native preview handoffs

* fix(runtime): recognize cross-realm preview video elements

* fix(studio): bound Play handoffs and drain transparent preview samples

* fix(preview): recover retained video without chasing playback clock

* fix(studio): drain finite alpha queries before disposing readers
This commit is contained in:
James Russo
2026-11-08 00:17:12 +00:00
committed by GitHub
parent b87a01f6a8
commit b7157bdabb
57 changed files with 4474 additions and 153 deletions
+2 -2
View File
@@ -389,7 +389,7 @@
"cn": "0.2.4",
"dockview-react": "8.3.1",
"gsap": "^3.13.0",
"mediabunny": "^1.45.3",
"mediabunny": "^1.65.2",
},
"devDependencies": {
"@hyperframes/producer": "workspace:*",
@@ -1856,7 +1856,7 @@
"mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="],
"mediabunny": ["mediabunny@1.45.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-GUCPYjR+5olLM7DRmupCXCmZkkSrKHVl1gyW2RztpObqLfrix19kWGf/9WgWzDW0g49DvfGXpl+zfArCx5HDMQ=="],
"mediabunny": ["mediabunny@1.65.2", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-goZpibvo/5hkc/56WVNm93Ini+o7K4y8aXA20c/BN1B+m3L5aHQa8MTvO+S1APg/YI8kMILtrX6QCqUtxLW7bA=="],
"meow": ["meow@13.2.0", "", {}, "sha512-pxQJQzB6djGPXh08dacEloMFopsOqGVRKFPYvPOt9XDZ1HasbgDZA74CJGreSU4G3Ak7EFJGoiH2auq+yXISgA=="],
+25
View File
@@ -106,6 +106,31 @@ depends on the browser playing in real time, while render captures frames one at
time. A heavy project can therefore stutter during preview and still render every
frame.
Paused video scrubbing decodes and stages the visible video layers before
advancing the composition's animations. Completed frames appear during a drag,
with newer pointer positions coalesced into the next decode. Canvas and GPU
effects consume the same staged pixels. Continuous playback uses native video;
Studio keeps the paused picture visible while those videos catch up before Play.
A native handoff waits for ready frames for up to ten seconds; Pause or a new
seek cancels it. At the deadline, the rest plays while each unready video keeps
its last full-quality picture. During playback, native frame presentation or
playback progress releases that picture; normal transport sync corrects drift.
Paused recovery still waits for its requested frame. A skipped source keeps its
picture on subsequent Play requests until it recovers.
A paused fallback that outlasts its wait retains the presented clock and picture,
while paused sync continues. It can still publish when the video becomes ready;
a new gesture cancels that recovery. Studio requests paused-decoder release
before preparing native playback and reopens readers when paused scrubbing resumes.
Transparent video queries drain their alpha merges before input disposal. If one
never settles, its cleanup can outlast the Play deadline; the remaining playback
still starts while that video's last picture stays visible.
Scrubbing and paused frame steps decode the original video at source resolution
throughout. Automatic playback quality reductions do not reduce these frames,
and the frame decoder does not substitute re-encoded seek copies. Playback
quality settings do not reduce these paused frames. Browsers or
media that cannot use the frame decoder fall back to native seeking. Long-GOP
sources can take longer to seek, especially backward or across distant cuts.
## Tailwind preset
```typescript
@@ -469,6 +469,14 @@ describe("insertRuntimeTag", () => {
expect(out.indexOf("window.author")).toBeLessThan(out.indexOf(runtime));
});
it("keeps the tracker and runtime ahead of author scripts for Studio previews", () => {
const out = insertRuntimeTag(page('data-motion-blur="on"'), runtime, true);
const tracker = out.indexOf("data-hf-accel-tracker");
expect(tracker).toBeGreaterThan(-1);
expect(tracker).toBeLessThan(out.indexOf(runtime));
expect(out.indexOf(runtime)).toBeLessThan(out.indexOf("window.author"));
});
it("adds the tracker once, however many times the runtime is placed", () => {
const once = insertRuntimeTag(page('data-motion-blur="on"'), runtime);
const twice = insertRuntimeTag(once, runtime);
+2 -2
View File
@@ -393,8 +393,8 @@ function insertTagInHead(html: string, tag: string): string {
const ACCEL_TRACKER_ATTR = "data-hf-accel-tracker";
const OPTS_INTO_MOTION_BLUR = /\sdata-motion-blur\s*=\s*["']?on[\s"'>]/i;
export function insertRuntimeTag(html: string, tag: string): string {
const withRuntime = insertTagInHead(html, tag);
export function insertRuntimeTag(html: string, tag: string, atHeadStart = false): string {
const withRuntime = atHeadStart ? injectTagsAtHeadStart(html, tag) : insertTagInHead(html, tag);
if (!OPTS_INTO_MOTION_BLUR.test(html) || html.includes(ACCEL_TRACKER_ATTR)) return withRuntime;
const tracker = `<script ${ACCEL_TRACKER_ATTR}>${acceleratedCanvasTrackerSource()}</script>`;
return injectTagsAtHeadStart(withRuntime, tracker);
+5
View File
@@ -308,6 +308,11 @@ export {
type PreviewQuality,
type PreviewSourceFacts,
} from "./previewProxyBox";
export type {
PausedVideoFrameProvider,
PausedVideoFrameRequest,
PausedVideoFrame,
} from "./runtime/pausedVideoFrames";
// Inline scripts
export {
+2
View File
@@ -55,6 +55,8 @@ export type PreviewSourceFacts = {
bitDepth: number;
bitrate: number;
codec?: string;
frameRate?: number;
variableFrameRate?: boolean;
};
export const HEAVY_PIXEL_SHARE = 0.5;
@@ -0,0 +1,238 @@
import { afterEach, expect, it, vi } from "vitest";
import { createPausedVideoFrames, getPausedVideoFrame } from "../pausedVideoFrames";
import {
patchCanvasVideoFrameCompat,
patchExternalVideoTextureCompat,
patchVideoTextureCompat,
patchWebGLVideoTextureCompat,
} from "./video-texture-compat";
const sessions: ReturnType<typeof createPausedVideoFrames>[] = [];
afterEach(() => {
for (const session of sessions.splice(0)) session.destroy();
document.body.innerHTML = "";
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
function surfaces() {
const draw = vi.fn();
const offscreenDraw = vi.fn();
const image = vi.fn();
const subImage = vi.fn();
const copy = vi.fn();
const imported = vi.fn((_descriptor: unknown) => ({ texture: true }));
const constructed = vi.fn();
class Context2D {
drawImage(...args: unknown[]) {
draw.apply(this, args);
}
}
class Offscreen2D {
drawImage(...args: unknown[]) {
offscreenDraw.apply(this, args);
}
}
class GL {
texImage2D(...args: unknown[]) {
image.apply(this, args);
}
texSubImage2D(...args: unknown[]) {
subImage.apply(this, args);
}
}
class GL2 extends GL {}
class Queue {
copyExternalImageToTexture(...args: unknown[]) {
copy.apply(this, args);
}
}
class Device {
importExternalTexture(descriptor: unknown) {
return imported.call(this, descriptor);
}
}
class Frame {
close = vi.fn();
constructor(source: unknown, options: unknown) {
constructed(source, options);
}
}
vi.stubGlobal("CanvasRenderingContext2D", Context2D);
vi.stubGlobal("OffscreenCanvasRenderingContext2D", Offscreen2D);
vi.stubGlobal("WebGLRenderingContext", GL);
vi.stubGlobal("WebGL2RenderingContext", GL2);
vi.stubGlobal("GPUQueue", Queue);
vi.stubGlobal("GPUDevice", Device);
vi.stubGlobal("VideoFrame", Frame);
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
clearRect: vi.fn(),
drawImage: vi.fn(),
} as unknown as CanvasRenderingContext2D);
return {
Context2D,
Offscreen2D,
GL,
GL2,
Queue,
Device,
draw,
offscreenDraw,
image,
subImage,
copy,
imported,
constructed,
};
}
async function presented() {
const session = createPausedVideoFrames();
sessions.push(session);
const video = document.createElement("video");
video.src = "/original.mp4";
document.body.append(video);
Object.defineProperties(video, {
videoWidth: { value: 1920 },
videoHeight: { value: 1080 },
});
const source = document.createElement("canvas");
source.width = 320;
source.height = 180;
session.setProvider({
decode: async (requests) =>
requests.map(({ time }) => ({
source,
timestamp: time,
duration: 1 / 30,
width: 1920,
height: 1080,
release: vi.fn(),
})),
dispose: vi.fn(),
});
const show = async (time: number) => {
const batch = await session.prepare(
[{ video, source: video.src, time }],
new AbortController().signal,
);
expect(batch?.present()).toBe(true);
batch!.refresh();
};
await show(3);
const canvas = getPausedVideoFrame(video)!;
expect([canvas.width, canvas.height]).toEqual([video.videoWidth, video.videoHeight]);
return { session, video, canvas, show };
}
it.each([
["three arguments", [11, 17]],
["five arguments", [11, 17, 640, 360]],
["nine arguments", [120, 60, 960, 540, 11, 17, 640, 360]],
])("preserves %s and source coordinates for both canvas surfaces", async (_name, coordinates) => {
const { Context2D, Offscreen2D, draw, offscreenDraw } = surfaces();
const { session, video, canvas } = await presented();
patchCanvasVideoFrameCompat();
patchCanvasVideoFrameCompat();
const context = new Context2D();
const offscreen = new Offscreen2D();
context.drawImage(video, ...coordinates);
offscreen.drawImage(video, ...coordinates);
expect(draw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates);
expect(offscreenDraw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates);
expect(draw.mock.contexts[0]).toBe(context);
expect(offscreenDraw.mock.contexts[0]).toBe(offscreen);
session.clear();
context.drawImage(video, ...coordinates);
expect(draw).toHaveBeenLastCalledWith(video, ...coordinates);
const image = document.createElement("img");
offscreen.drawImage(image, ...coordinates);
expect(offscreenDraw).toHaveBeenLastCalledWith(image, ...coordinates);
});
it("uploads paused pixels through both WebGL versions without changing crop or upload parameters", async () => {
const { GL, GL2, image, subImage } = surfaces();
const { session, video, canvas } = await presented();
patchWebGLVideoTextureCompat();
for (const Context of [GL, GL2]) {
const gl = new Context();
const imageArgs = [0x0de1, 2, 0x1908, 0x1908, 0x1401];
const subArgs = [0x0de1, 2, 13, 29, 0x1908, 0x1401];
gl.texImage2D(...imageArgs, video);
gl.texSubImage2D(...subArgs, video);
expect(image).toHaveBeenLastCalledWith(...imageArgs, canvas);
expect(subImage).toHaveBeenLastCalledWith(...subArgs, canvas);
expect(image.mock.contexts.at(-1)).toBe(gl);
const pixels = new Uint8Array([10, 20, 30, 255]);
gl.texImage2D(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels);
expect(image).toHaveBeenLastCalledWith(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels);
}
session.clear();
new GL().texImage2D(0x0de1, 0, 0x1908, 0x1908, 0x1401, video);
expect(image.mock.lastCall?.at(-1)).toBe(video);
});
it("preserves WebGPU crop, orientation, destination and full intrinsic copy extent", async () => {
const { Queue, copy } = surfaces();
const { session, video, canvas } = await presented();
patchVideoTextureCompat();
const queue = new Queue();
const source = { source: video, origin: { x: 37, y: 41 }, flipY: true };
const destination = { texture: {}, origin: [5, 7], mipLevel: 1, premultipliedAlpha: true };
const extent = {
width: video.videoWidth - 37,
height: video.videoHeight - 41,
depthOrArrayLayers: 1,
};
queue.copyExternalImageToTexture(source, destination, extent);
expect(copy).toHaveBeenCalledExactlyOnceWith({ ...source, source: canvas }, destination, extent);
expect(copy.mock.contexts[0]).toBe(queue);
expect(source.source).toBe(video);
expect(copy.mock.calls[0]?.[1]).toBe(destination);
expect(copy.mock.calls[0]?.[2]).toBe(extent);
session.clear();
queue.copyExternalImageToTexture(source, destination, extent);
expect(copy).toHaveBeenLastCalledWith(source, destination, extent);
expect(copy.mock.lastCall?.[0]).toBe(source);
});
it("reuses an external VideoFrame until presentation changes, then closes it on replacement and clear", async () => {
const { Device, imported, constructed } = surfaces();
const { session, video, canvas, show } = await presented();
patchExternalVideoTextureCompat();
const once = Device.prototype.importExternalTexture;
patchExternalVideoTextureCompat();
expect(Device.prototype.importExternalTexture).toBe(once);
const device = new Device();
const descriptor = { source: video, colorSpace: "srgb", label: "authored texture" };
const texture = device.importExternalTexture(descriptor);
const first = imported.mock.lastCall?.[0] as {
source: { close: ReturnType<typeof vi.fn> };
colorSpace: string;
label: string;
};
expect(texture).toEqual({ texture: true });
expect(constructed).toHaveBeenCalledExactlyOnceWith(canvas, { timestamp: 3_000_000 });
expect(first).toEqual({ ...descriptor, source: first.source });
expect(first.source).not.toBe(video);
expect(imported.mock.contexts[0]).toBe(device);
expect(descriptor.source).toBe(video);
device.importExternalTexture(descriptor);
expect(imported.mock.lastCall?.[0]).toEqual(first);
expect(constructed).toHaveBeenCalledTimes(1);
expect(first.source.close).not.toHaveBeenCalled();
await show(4);
expect(first.source.close).toHaveBeenCalledTimes(1);
device.importExternalTexture(descriptor);
const second = imported.mock.lastCall?.[0] as typeof first;
expect(second.source).not.toBe(first.source);
expect(constructed).toHaveBeenLastCalledWith(getPausedVideoFrame(video), {
timestamp: 4_000_000,
});
session.clear();
expect(second.source.close).toHaveBeenCalledTimes(1);
device.importExternalTexture(descriptor);
expect(imported).toHaveBeenLastCalledWith(descriptor);
expect(imported.mock.lastCall?.[0]).toBe(descriptor);
expect(constructed).toHaveBeenCalledTimes(2);
});
@@ -15,6 +15,7 @@
import { findInjectedRenderFrame } from "../renderFrameSibling.js";
import { isImageElement, isVideoElement } from "../domRealm";
import { getPausedExternalVideoFrame, getPausedVideoFrame } from "../pausedVideoFrames";
/**
* Resolve the decoded render-frame `<img>` for a source `<video>`, if the
@@ -26,7 +27,9 @@ import { isImageElement, isVideoElement } from "../domRealm";
* first (cheap) and fall back to an id lookup in case a node was inserted
* between them.
*/
function resolveRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null {
function resolveRenderFrameImage(
video: HTMLVideoElement,
): HTMLImageElement | HTMLCanvasElement | null {
const sibling = video.nextElementSibling;
if (
isImageElement(sibling) &&
@@ -40,7 +43,48 @@ function resolveRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | nu
if (byId && byId.complete && byId.naturalWidth > 0) {
return byId;
}
return null;
return getPausedVideoFrame(video);
}
export function patchCanvasVideoFrameCompat(): void {
for (const ctor of [
globalThis.CanvasRenderingContext2D,
globalThis.OffscreenCanvasRenderingContext2D,
]) {
if (!ctor?.prototype?.drawImage) continue;
const original = ctor.prototype.drawImage as ((...args: unknown[]) => void) & {
__hfVideoPatched?: boolean;
};
if (original.__hfVideoPatched) continue;
const patched = function (this: unknown, ...args: unknown[]) {
if (isVideoElement(args[0])) args[0] = getPausedVideoFrame(args[0]) ?? args[0];
original.apply(this, args);
};
patched.__hfVideoPatched = true;
ctor.prototype.drawImage = patched;
}
}
export function patchExternalVideoTextureCompat(): void {
const ctor = (globalThis as Record<string, unknown>).GPUDevice as
| {
prototype: {
importExternalTexture?: ((source: Record<string, unknown>) => unknown) & {
__hfVideoPatched?: boolean;
};
};
}
| undefined;
const original = ctor?.prototype.importExternalTexture;
if (!ctor || typeof original !== "function" || original.__hfVideoPatched) return;
const patched = function (this: unknown, descriptor: Record<string, unknown>) {
const frame = isVideoElement(descriptor.source)
? getPausedExternalVideoFrame(descriptor.source)
: null;
return original.call(this, frame ? { ...descriptor, source: frame } : descriptor);
};
patched.__hfVideoPatched = true;
ctor.prototype.importExternalTexture = patched;
}
export function patchVideoTextureCompat(): void {
+4 -1
View File
@@ -49,6 +49,7 @@ import { mediaElementTime } from "./mediaClock";
import { swallow } from "./diagnostics";
import { createRecordFilter } from "./mutationRecords";
import { findInjectedRenderFrame } from "./renderFrameSibling";
import { getPausedVideoFrame } from "./pausedVideoFrames";
import {
isCanvasElement,
isElementNode,
@@ -2807,6 +2808,8 @@ function getDrawableSource(element: ColorGradingMediaElement): TexImageSource |
if (isVideoElement(element)) {
const renderFrame = findRenderFrameImage(element);
if (renderFrame) return renderFrame;
const previewFrame = getPausedVideoFrame(element);
if (previewFrame) return previewFrame;
}
return isDrawableSource(element) ? element : null;
}
@@ -3542,7 +3545,7 @@ function placeCanvas(
layout: { width: number; height: number };
uv: ReturnType<typeof calculateObjectFitUv>;
} | null {
const styleSource = isHtmlElement(source) ? source : entry.element;
const styleSource = isRenderFrameImage(source) ? source : entry.element;
const layout = updateCanvasLayout(entry, styleSource);
if (!layout) return null;
const style = window.getComputedStyle(styleSource);
+2
View File
@@ -329,6 +329,8 @@ describe("runtime entry", () => {
let ready = 1;
Object.defineProperty(video, "readyState", { configurable: true, get: () => ready });
Object.defineProperty(video, "networkState", { configurable: true, value: 2 });
Object.defineProperty(video, "videoWidth", { configurable: true, value: 64 });
Object.defineProperty(video, "videoHeight", { configurable: true, value: 36 });
await evaluateRuntime();
let settled = false;
const landed = Promise.resolve(window.__player!.seek(5.5)).then(() => {
+2
View File
@@ -19,6 +19,7 @@ import { hasFrameSources, registerFrameSource } from "./frameSources";
import { createFilmBridge } from "./filmBridge";
import { trackAcceleratedCanvases } from "./captureMotionBlur";
import { parkFarScenesAtBoot } from "./sceneWindow";
import { installPausedVideoCallbacks } from "./pausedVideoCallbacks";
type HyperframeWindow = Window & {
__hyperframeRuntimeBootstrapped?: boolean;
@@ -44,6 +45,7 @@ type HyperframeWindow = Window & {
// at script evaluation time, while the document is still parsing.
installAuthoredOpacityCapture();
installAuthoredMediaCapture();
installPausedVideoCallbacks();
installFlatGsapTransforms();
trackAcceleratedCanvases();
window.__hfHasFrameSources = hasFrameSources;
@@ -1,8 +1,14 @@
import { afterEach, expect, it, vi } from "vitest";
import { initSandboxRuntimeModular } from "./init";
import { hideTimedClipsUntilFirstPass } from "./timedClipHide";
import { resetSeekDispatchState } from "./adapters/seek-dispatch";
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
import type {
PausedVideoFrame,
PausedVideoFrameProvider,
PausedVideoFrameRequest,
} from "./pausedVideoFrames";
function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
let time = 0;
@@ -34,6 +40,17 @@ function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
return {
video,
seeks,
metadataAt: (position: number) => {
time = position;
ready = 1;
seeking = false;
},
presentAt: (position: number) => {
time = position;
ready = 4;
seeking = false;
video.dispatchEvent(new Event("timeupdate"));
},
land: () => {
ready = 4;
seeking = false;
@@ -57,6 +74,7 @@ function film(withImage = false) {
const drawImage = vi.fn();
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
drawImage,
clearRect: vi.fn(),
} as unknown as CanvasRenderingContext2D);
document.body.innerHTML =
'<div data-composition-id="main" data-root="true" data-duration="10">' +
@@ -83,6 +101,47 @@ function film(withImage = false) {
return { main, old: videos.slice(0, 2), next: videos.slice(2), drawImage };
}
function frameProvider(videos: ReturnType<typeof decoder>[]) {
const clocks = new Map(
videos.map(({ video }) => [video, Object.getOwnPropertyDescriptor(video, "currentTime")!]),
);
for (const { video } of videos) Reflect.deleteProperty(video, "currentTime");
vi.spyOn(HTMLMediaElement.prototype, "currentTime", "get").mockImplementation(
function (this: HTMLMediaElement) {
return clocks.get(this as HTMLVideoElement)?.get?.();
},
);
vi.spyOn(HTMLMediaElement.prototype, "currentTime", "set").mockImplementation(function (
this: HTMLMediaElement,
time: number,
) {
clocks.get(this as HTMLVideoElement)?.set?.(time);
});
const pending: Array<() => void> = [];
const provider: PausedVideoFrameProvider = {
decode: vi.fn(
(requests: PausedVideoFrameRequest[]) =>
new Promise<PausedVideoFrame[]>((resolve) => {
pending.push(() =>
resolve(
requests.map(({ time }) => ({
source: document.createElement("canvas"),
timestamp: time,
duration: 1 / 30,
width: 64,
height: 36,
release: vi.fn(),
})),
),
);
}),
),
dispose: vi.fn(),
};
window.__hf!.setPausedVideoFrameProvider!(provider);
return { provider, pending };
}
it("keeps the old picture and page until both destination videos have decoded", async () => {
const { main, old, next, drawImage } = film();
const landed = window.__player!.seek(5.5);
@@ -112,6 +171,8 @@ it("coalesces same-clip requests and publishes complete intermediate pictures",
expect(next.map((v) => v.seeks.length)).toEqual(initialSeeks);
for (const video of next) video.land();
expect(main.time?.()).toBe(5.5);
expect(window.__player!.getTime()).toBe(7.5);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
expect(next.every((v) => Math.abs(v.video.currentTime - 2.5) < 0.001)).toBe(true);
for (const video of next) video.land();
await landed;
@@ -270,6 +331,547 @@ it("caps a stalled decoder without leaving the picture hidden", async () => {
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
});
it("exposes the presented clock to animation callbacks while retaining the latest scrub destination", async () => {
const { main, next } = film();
const times: number[] = [];
const original = main.totalTime!;
main.totalTime = (time?: number) => {
if (time !== undefined) times.push(window.__player!.getTime());
return original(time);
};
const done = window.__player!.seek(5.5);
window.__player!.seek(7.5);
for (const video of next) video.land();
expect(times).toContain(5.5);
expect(window.__player!.getTime()).toBe(7.5);
for (const video of next) video.land();
await done;
});
it.each(["stop-media", "set-native-media-sync-disabled"])(
"does not resume a pending playback handoff after %s",
async (action) => {
const { old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const landed = window.__player!.seek(1);
pending.shift()!();
await landed;
window.__player!.play();
expect(window.__player!.isPlaying()).toBe(false);
window.dispatchEvent(
new MessageEvent("message", {
source: window,
data: { source: "hf-parent", type: "control", action, disabled: true },
}),
);
await Promise.resolve();
for (const video of old) video.land();
await Promise.resolve();
expect(window.__player!.isPlaying()).toBe(false);
},
);
it("keeps the complete picture and buffers beyond the handoff cap until native frames are ready", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const { main, old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const landed = window.__player!.seek(1);
pending.shift()!();
await landed;
for (const video of old) video.metadataAt(1);
window.__player!.play();
expect(old.every(({ video }) => video.preload === "auto")).toBe(true);
await vi.advanceTimersByTimeAsync(5100);
expect(window.__player!.isPlaying()).toBe(false);
expect(main.time?.()).toBe(1);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
window.__player!.seek(1);
for (const video of old) video.land();
await vi.advanceTimersByTimeAsync(0);
expect(window.__player!.isPlaying()).toBe(true);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
});
it("settles a stalled paused fallback while retaining the last complete picture", async () => {
resetSeekDispatchState();
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const { main, old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(1);
pending.shift()!();
await initial;
vi.mocked(provider.decode).mockRejectedValueOnce(new Error("decoder unavailable"));
const stalled = window.__player!.seek(6);
const retargeted = window.__player!.seek(7);
const wait = window.__hfWaitForSeekCompletion!();
await vi.advanceTimersByTimeAsync(5100);
await stalled;
await retargeted;
await wait;
expect(main.time?.()).toBe(1);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
const retry = window.__player!.seek(8);
pending.shift()!();
await retry;
expect(main.time?.()).toBe(8);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
});
it.each([
["already metadata-only, idle", false, 1],
["ready, then metadata-only idle", true, 1],
["ready, then loading forever", true, 2],
["metadata-only, loading forever", false, 2],
] as const)(
"bounds Play for %s while retaining its full-quality picture",
async (_, initiallyReady, networkState) => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
const { main, old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(3);
pending.shift()!();
await initial;
const stalled = old[0]!;
if (initiallyReady) stalled.land();
else stalled.metadataAt(3);
Object.defineProperty(stalled.video, "networkState", {
value: networkState,
configurable: true,
});
window.__player!.play();
old[1]!.land();
await vi.advanceTimersByTimeAsync(5100);
if (initiallyReady || networkState === 2) expect(window.__player!.isPlaying()).toBe(false);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(
window.__player!.isPlaying() ? 1 : 2,
);
stalled.video.dispatchEvent(new Event("progress"));
await vi.advanceTimersByTimeAsync(4900);
expect(window.__player!.isPlaying()).toBe(true);
expect(main.time?.()).toBeGreaterThanOrEqual(3);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(1);
const held = document.querySelector("[data-hf-preview-video-canvas]")!;
expect([held.getAttribute("width"), held.getAttribute("height")]).toEqual(["64", "36"]);
},
);
it("keeps every unready video picture at the hard deadline", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
const { old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(3);
pending.shift()!();
await initial;
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
window.__player!.play();
await vi.advanceTimersByTimeAsync(10000);
expect(window.__player!.isPlaying()).toBe(true);
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
window.__player!.pause();
pending.shift()!();
await Promise.resolve();
await Promise.resolve();
window.__player!.play();
await Promise.resolve();
await Promise.resolve();
expect(window.__player!.isPlaying()).toBe(true);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
});
it("reveals a delayed playing frame without chasing the moving clock with more seeks", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
const { old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(3);
pending.shift()!();
await initial;
const stalled = old[0]!;
stalled.metadataAt(3);
let error: { code: number } | null = { code: 4 };
Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
window.__player!.play();
old[1]!.land();
await vi.advanceTimersByTimeAsync(0);
error = null;
const seeks = stalled.seeks.length;
Object.defineProperty(stalled.video, "paused", { configurable: true, value: false });
for (let i = 0; i < 3; i++) {
await vi.advanceTimersByTimeAsync(50);
stalled.land();
if (i === 0) stalled.video.dispatchEvent(new Event("playing"));
stalled.presentAt(3 + (i + 1) / 30);
}
expect(stalled.seeks).toHaveLength(seeks);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
expect(window.__player!.isPlaying()).toBe(true);
});
it("keeps a paused recovery covered until its static target seek lands", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
const { old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(3);
pending.shift()!();
await initial;
const stalled = old[0]!;
stalled.metadataAt(3);
let error: { code: number } | null = { code: 4 };
Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
window.__player!.play();
old[1]!.land();
await vi.advanceTimersByTimeAsync(500);
expect(window.__player!.isPlaying()).toBe(true);
window.__hf!.setPausedVideoFrameProvider!(null);
window.__player!.pause();
const picture = document.querySelector("[data-hf-preview-video-canvas]");
expect(picture).not.toBeNull();
error = null;
stalled.metadataAt(3);
stalled.land();
expect(stalled.video.previousElementSibling).toBe(picture);
expect(stalled.video.currentTime).toBeCloseTo(3.5, 2);
stalled.land();
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
});
it("publishes a late native recovery after a paused fallback wait has settled", async () => {
resetSeekDispatchState();
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const { main, old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(1);
pending.shift()!();
await initial;
vi.mocked(provider.decode).mockRejectedValueOnce(new Error("decoder unavailable"));
const stalled = window.__player!.seek(6);
await vi.advanceTimersByTimeAsync(5100);
await stalled;
expect(main.time?.()).toBe(1);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
expect(window.__player!.getTime()).toBe(1);
const priorSeeks = old[0]!.seeks.length;
const destinationSeeks = next.map((video) => video.seeks.length);
old[0]!.reload();
expect(old[0]!.seeks.length).toBeGreaterThan(priorSeeks);
expect(next.map((video) => video.seeks.length)).toEqual(destinationSeeks);
for (const video of next) video.land();
await Promise.resolve();
expect(main.time?.()).toBe(6);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
});
it("keeps a hold-only snapshot visible while Play awaits decoder suspension", async () => {
const { next, old } = film();
const { provider } = frameProvider([...old, ...next]);
let suspended = () => {};
provider.suspend = () =>
new Promise<void>((resolve) => {
suspended = resolve;
});
const seek = window.__player!.seek(6);
const pictures = [...document.querySelectorAll("canvas")];
expect(pictures).toHaveLength(2);
window.__player!.play();
await seek;
expect([...document.querySelectorAll("canvas")]).toEqual(pictures);
suspended();
await Promise.resolve();
for (const video of next) video.land();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(window.__player!.isPlaying()).toBe(true);
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
});
it("resumes a pending Play after its frame provider is replaced", async () => {
const { old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(1);
pending.shift()!();
await initial;
window.__player!.play();
window.__hf!.setPausedVideoFrameProvider!({ decode: provider.decode, dispose: vi.fn() });
pending.shift()!();
await new Promise((resolve) => setTimeout(resolve, 0));
for (const video of old) video.land();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(window.__player!.isPlaying()).toBe(true);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
});
it("cancels the hard Play deadline when Pause restores provider ownership", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
const { old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(3);
pending.shift()!();
await initial;
window.__player!.play();
await vi.advanceTimersByTimeAsync(9000);
window.__player!.pause();
pending.shift()!();
await vi.advanceTimersByTimeAsync(20000);
expect(window.__player!.isPlaying()).toBe(false);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
});
it("bounds a deferred decoder release and ignores its late completion", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
const { old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
let released = () => {};
provider.suspend = () =>
new Promise<void>((resolve) => {
released = resolve;
});
const initial = window.__player!.seek(3);
pending.shift()!();
await initial;
window.__player!.play();
await vi.advanceTimersByTimeAsync(10000);
expect(window.__player!.isPlaying()).toBe(true);
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
expect(pictures).toHaveLength(2);
released();
await Promise.resolve();
await Promise.resolve();
expect(window.__player!.isPlaying()).toBe(true);
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
});
it("ignores decoder release after Pause cancels a pending Play", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
const { old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
let released = () => {};
provider.suspend = () =>
new Promise<void>((resolve) => {
released = resolve;
});
const initial = window.__player!.seek(3);
pending.shift()!();
await initial;
window.__player!.play();
await vi.advanceTimersByTimeAsync(9000);
window.__player!.pause();
pending.shift()!();
await Promise.resolve();
await Promise.resolve();
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
const seeks = old.map((video) => video.seeks.length);
released();
await Promise.resolve();
await Promise.resolve();
expect(old.map((video) => video.seeks.length)).toEqual(seeks);
await vi.advanceTimersByTimeAsync(1000);
expect(window.__player!.isPlaying()).toBe(false);
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
});
it("does not wait for a video frame from an audio-only video element", async () => {
const { main, next } = film();
for (const { video } of next)
Object.defineProperties(video, {
readyState: { value: 1, configurable: true },
videoWidth: { value: 0, configurable: true },
error: { value: null, configurable: true },
});
expect(window.__player!.seek(6)).toBeUndefined();
expect(main.time?.()).toBe(6);
window.__player!.play();
expect(window.__player!.isPlaying()).toBe(true);
});
it("waits for paused decoder suspension before preparing native playback", async () => {
const { old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
let releaseDecoders = () => {};
provider.suspend = vi.fn(
() =>
new Promise<void>((resolve) => {
releaseDecoders = resolve;
}),
);
const initial = window.__player!.seek(1);
pending.shift()!();
await initial;
const seeks = old.map((v) => v.seeks.length);
window.__player!.play();
expect(provider.suspend).toHaveBeenCalledOnce();
expect(old.map((v) => v.seeks.length)).toEqual(seeks);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
releaseDecoders();
await Promise.resolve();
for (const video of old) video.land();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(window.__player!.isPlaying()).toBe(true);
});
it("restores paused ownership when Pause cancels an unfinished decoder suspension", async () => {
const { old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
let releaseDecoders = () => {};
provider.suspend = () =>
new Promise<void>((resolve) => {
releaseDecoders = resolve;
});
const initial = window.__player!.seek(1);
pending.shift()!();
await initial;
window.__player!.play();
window.__player!.pause();
expect(pending).toHaveLength(1);
pending.shift()!();
releaseDecoders();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(window.__player!.isPlaying()).toBe(false);
expect(
old.every(({ video }) => Object.hasOwn(video, "currentTime") && video.currentTime === 1),
).toBe(true);
});
it("warms the restart destination before Play when paused at the end", async () => {
const { main, old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const landed = window.__player!.seek(10, { settled: true });
pending.shift()!();
await landed;
expect(main.time?.()).toBe(10);
for (const video of old) video.metadataAt(0);
const seeks = old.map((v) => v.seeks.length);
window.__player!.play();
expect(old.map((v) => v.seeks.length)).toEqual(seeks.map((n) => n + 1));
expect(window.__player!.isPlaying()).toBe(false);
for (const video of old) video.land();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(window.__player!.isPlaying()).toBe(true);
expect(window.__player!.getTime()).toBeLessThan(1);
expect(old.every(({ video }) => video.style.visibility === "visible")).toBe(true);
});
it("retains a complete paused picture while replacing or removing its frame provider", async () => {
const { main, old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
const initial = window.__player!.seek(1);
pending.shift()!();
await initial;
const replacement = { decode: provider.decode, dispose: vi.fn() };
window.__hf!.setPausedVideoFrameProvider!(replacement);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
pending.shift()!();
await Promise.resolve();
await Promise.resolve();
expect(main.time?.()).toBe(1);
window.__hf!.setPausedVideoFrameProvider!(null);
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
for (const video of old) video.land();
await Promise.resolve();
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
});
it("retains an in-flight destination when the host replaces its frame provider", async () => {
const { main, old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
const first = window.__player!.seek(6);
const picture = document.querySelector("[data-hf-preview-video-canvas]");
expect(picture).not.toBeNull();
const replacement = { decode: provider.decode, dispose: vi.fn() };
window.__hf!.setPausedVideoFrameProvider!(replacement);
expect(document.querySelector("[data-hf-preview-video-canvas]")).toBe(picture);
expect(provider.dispose).toHaveBeenCalledTimes(1);
expect(window.__player!.getTime()).toBe(6);
pending.shift()!();
pending.shift()!();
await first;
await Promise.resolve();
expect(main.time?.()).toBe(6);
});
it("loads original source metadata before publishing provider frames used by intrinsic-size effects", async () => {
const { main, old, next } = film();
const { pending } = frameProvider([...old, ...next]);
for (const { video } of next)
Object.defineProperty(video, "videoWidth", { value: 0, configurable: true });
const landed = window.__player!.seek(6);
pending.shift()!();
await Promise.resolve();
expect(main.time?.()).toBe(0);
for (const { video } of next) {
Object.defineProperty(video, "videoWidth", { value: 64, configurable: true });
video.dispatchEvent(new Event("loadedmetadata"));
}
await landed;
expect(main.time?.()).toBe(6);
});
it("refines the paused frame once after input settles, then warms native playback", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const { old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
const landed = window.__player!.seek(1);
pending.shift()!();
await landed;
await vi.advanceTimersByTimeAsync(120);
expect(provider.decode).toHaveBeenCalledTimes(2);
expect(vi.mocked(provider.decode).mock.calls[1]![0].every((request) => request.settled)).toBe(
true,
);
pending.shift()!();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(120);
expect(old.every(({ video }) => !Object.hasOwn(video, "currentTime"))).toBe(true);
await vi.advanceTimersByTimeAsync(1000);
expect(provider.decode).toHaveBeenCalledTimes(2);
expect(window.__player!.isPlaying()).toBe(false);
});
it("decodes frame-step seeks at settled quality immediately without a second refinement", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const { old, next } = film();
const { provider, pending } = frameProvider([...old, ...next]);
const landed = window.__player!.seek(1, { settled: true });
expect(vi.mocked(provider.decode).mock.calls[0]![0].every((request) => request.settled)).toBe(
true,
);
pending.shift()!();
await landed;
await vi.advanceTimersByTimeAsync(1000);
expect(provider.decode).toHaveBeenCalledTimes(1);
expect(window.__player!.isPlaying()).toBe(false);
});
it("notifies video-driven effects on the committed clock before animation redraw", async () => {
const proto = HTMLVideoElement.prototype;
const request = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
const cancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
Object.defineProperties(proto, {
requestVideoFrameCallback: { value: vi.fn(() => 1), configurable: true, writable: true },
cancelVideoFrameCallback: { value: vi.fn(), configurable: true, writable: true },
});
try {
const { main, old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const notifications: number[] = [];
old[0]!.video.requestVideoFrameCallback(() => notifications.push(window.__player!.getTime()));
const original = main.totalTime!;
main.totalTime = (time?: number) => {
if (time === 1) expect(notifications).toEqual([1]);
return original(time);
};
const landed = window.__player!.seek(1);
pending.shift()!();
await landed;
expect(notifications).toEqual([1]);
} finally {
if (request) Object.defineProperty(proto, "requestVideoFrameCallback", request);
else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
if (cancel) Object.defineProperty(proto, "cancelVideoFrameCallback", cancel);
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
}
});
afterEach(() => {
window.__hfRuntimeTeardown?.();
document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`)?.remove();
@@ -281,3 +883,21 @@ afterEach(() => {
vi.useRealTimers();
resetRuntimeFixtureDom();
});
it("releases a native hold when stop-media cancels its first provider seek", async () => {
const { old, next } = film();
const { pending } = frameProvider([...old, ...next]);
const seek = window.__player!.seek(3);
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(2);
window.dispatchEvent(
new MessageEvent("message", {
source: window,
data: { source: "hf-parent", type: "control", action: "stop-media" },
}),
);
await seek;
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
pending.shift()!();
await Promise.resolve();
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
});
@@ -1879,6 +1879,8 @@ describe("the preview's scene window", () => {
return ready.has(this) ? 4 : 1;
});
vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
vi.spyOn(HTMLVideoElement.prototype, "videoWidth", "get").mockReturnValue(64);
vi.spyOn(HTMLVideoElement.prototype, "videoHeight", "get").mockReturnValue(36);
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
const { root } = trackingRoot();
+208 -28
View File
@@ -29,6 +29,7 @@ import {
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
import { hasFrozenPicture } from "./frozenPicture";
import { createPreviewSeek } from "./previewSeek";
import { createPausedVideoFrames, releasePausedVideoFrame } from "./pausedVideoFrames";
import { copiesReady } from "./previewCopies";
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
import { instantTolerance } from "../clipFacts";
@@ -77,6 +78,8 @@ import { createFrameSourceAdapter } from "./frameSources";
import {
patchVideoTextureCompat,
patchWebGLVideoTextureCompat,
patchCanvasVideoFrameCompat,
patchExternalVideoTextureCompat,
} from "./adapters/video-texture-compat";
import {
forceDispatchSeekEvent,
@@ -530,6 +533,10 @@ function loadPreviewImagesIn(root: Element): void {
export function initSandboxRuntimeModular(): void {
const state = createRuntimeState();
let previewSeek: ReturnType<typeof createPreviewSeek> | undefined;
const pausedFrames = createPausedVideoFrames();
let nativePlayGeneration = 0;
let nativePlayTime: number | undefined;
let nativeWarmTimer: ReturnType<typeof wallSetTimeout> | undefined;
const stampsForStudio = window.parent !== window;
// Studio's discovery stamps time a clip for the timeline only; nothing hides it at their end.
const isDiscoveryStamped = (el: Element) =>
@@ -740,6 +747,13 @@ export function initSandboxRuntimeModular(): void {
const registerRuntimeCleanup = (callback: () => void) => {
runtimeCleanupCallbacks.push(callback);
};
registerRuntimeCleanup(() => {
nativePlayGeneration++;
nativePlayTime = undefined;
wallClearTimeout(nativeWarmTimer);
pausedFrames.destroy();
delete window.__hf?.setPausedVideoFrameProvider;
});
const postRuntimeDiagnosticOnce = (
code: string,
details: Record<string, RuntimeJson>,
@@ -761,7 +775,7 @@ export function initSandboxRuntimeModular(): void {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => HeldSeek;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
@@ -3687,6 +3701,7 @@ export function initSandboxRuntimeModular(): void {
* exists for. On `window.__hf` because the Studio is across the iframe boundary. */
const pausedMediaLeases = new WeakSet<HTMLMediaElement>();
const leasePausedMedia = (el: HTMLMediaElement): void => {
if (isVideoElement(el)) releasePausedVideoFrame(el);
pausedMediaLeases.add(el);
};
const releasePausedMedia = (el: HTMLMediaElement): void => {
@@ -3695,10 +3710,33 @@ export function initSandboxRuntimeModular(): void {
};
window.__hf.leasePausedMedia = leasePausedMedia;
window.__hf.releasePausedMedia = releasePausedMedia;
window.__hf.setPausedVideoFrameProvider = (provider) => {
const resumePlay = nativePlayTime !== undefined;
const destination =
previewSeek?.targetTime ?? (pausedFrames.active ? state.currentTime : undefined);
nativePlayGeneration++;
nativePlayTime = undefined;
wallClearTimeout(nativeWarmTimer);
previewSeek?.cancel(true);
pausedFrames.setProvider(provider);
if (!provider) pausedFrames.prepareNative();
if (destination !== undefined) {
const landed = seekWhenReady(destination);
const generation = nativePlayGeneration;
if (resumePlay)
void Promise.resolve(landed).then(() => {
if (!state.tornDown && generation === nativePlayGeneration) transport.play();
});
}
};
// Clips the transport may move now (not a paused lease); null while the host drives media.
const transportMovableClips = (clips: RuntimeMediaClip[]): RuntimeMediaClip[] | null => {
if (state.nativeMediaSyncDisabled) return null;
return state.isPlaying ? clips : clips.filter((clip) => !pausedMediaLeases.has(clip.el));
return state.isPlaying
? clips
: clips.filter(
(clip) => !pausedMediaLeases.has(clip.el) && !previewSeek?.ownsRecovery(clip.el),
);
};
window.__hf.setLoopStart = (seconds) => {
loopStartSeconds = seconds != null && Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
@@ -3784,7 +3822,7 @@ export function initSandboxRuntimeModular(): void {
let lastVisibilityPassSeconds: number | null = null;
const syncMediaForCurrentState = (timingRevision?: number) => {
if (previewSeek?.time !== undefined) return;
if (previewSeek?.time !== undefined && !previewSeek.committing) return;
// Scope 1 of 3 (see `withTimingResolver`). Closes before `syncRuntimeMedia`,
// which may call `el.load()` and invalidate every cached duration.
//
@@ -4493,7 +4531,7 @@ export function initSandboxRuntimeModular(): void {
// Redraw the current frame as a seek does; a playing film cut to end at or before it stops there, as at its end.
const cutShort = clock.isPlaying() && duration > 0 && state.currentTime >= duration;
if (joining && clock.isPlaying() && !cutShort) joinPlayback(swappedHosts);
else if (previewSeek?.time !== undefined) previewSeek.refresh();
else if (previewSeek?.targetTime !== undefined) previewSeek.refresh();
else transport.seek(cutShort ? duration : state.currentTime, { keepPlaying: !cutShort });
if (previewSeek?.time === undefined) syncTimedElementVisibility(state.currentTime);
postTimeline();
@@ -4803,7 +4841,7 @@ export function initSandboxRuntimeModular(): void {
.sort((a, b) => Math.abs(a.start - time) - Math.abs(b.start - time))[0];
const toMount = landed ?? neighbour;
const toRetire =
toMount || previewSeek?.time !== undefined
toMount || previewSeek?.targetTime !== undefined
? []
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
if (!toMount && toRetire.length === 0) {
@@ -4812,9 +4850,11 @@ export function initSandboxRuntimeModular(): void {
}
const wantedNow = (name: string) =>
heldScenes.has(name) ||
scenesWanted(windowScenes, previewSeek?.time ?? state.currentTime, clock.isPlaying()).has(
name,
);
scenesWanted(
windowScenes,
previewSeek?.targetTime ?? state.currentTime,
clock.isPlaying(),
).has(name);
const parkedBeforeStep = parkedNames.size;
windowWork = (async () => {
if (landed) await mountScene(landed.name);
@@ -4827,7 +4867,7 @@ export function initSandboxRuntimeModular(): void {
windowWork = null;
const stuckStepKeepsItsPlan = parkedNames.size === parkedBeforeStep;
windowNear = stuckStepKeepsItsPlan ? key : "";
moveSceneWindow(previewSeek?.time ?? state.currentTime);
moveSceneWindow(previewSeek?.targetTime ?? state.currentTime);
});
// A seek is not done until the scene it lands in is mounted.
if (landed) registerSeekCompletion(windowWork, "scene-window");
@@ -4934,6 +4974,7 @@ export function initSandboxRuntimeModular(): void {
if (!keepPlaying) syncMediaForCurrentState();
const tl = state.capturedTimeline;
pauseTimelineIfPossible(tl);
pausedFrames.notify();
const pageAnimations = seekTimelineAndAdapters(state.currentTime);
if (!keepPlaying) decideClipsUnderScriptHidden(state.currentTime);
runAdapters("pause", 0, pageAnimations);
@@ -4968,7 +5009,65 @@ export function initSandboxRuntimeModular(): void {
postState(true);
};
const previewVideoClipsAt = (time: number) =>
withTimingResolver(() => {
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
const visibleAt = timedVisibilityAt();
const root = resolveRootCompositionElement();
return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
(clip) =>
isVideoElement(clip.el) &&
!(clip.el.readyState >= 1 && clip.el.videoWidth === 0) &&
!isSilencedByHidden(clip.el) &&
(clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
isClipVisibleAt(time, clip.start, clip.end, duration) &&
clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
);
});
previewSeek = createPreviewSeek({
frames: {
available: pausedFrames.available,
active: () => pausedFrames.active,
skipped: pausedFrames.skipped,
hold: (videos) => {
if (pausedFrames.available()) return pausedFrames.holdNative(videos);
},
prepare: (clips, time, signal, settled) => {
moveSceneWindow(time);
for (const { el } of clips) {
if (!isVideoElement(el) || el.videoWidth > 0) continue;
if (el.preload === "none") el.preload = "metadata";
if (el.networkState !== 2) el.load();
}
const requests = clips.flatMap((clip) =>
isVideoElement(clip.el)
? [
{
video: clip.el,
source: clip.el.src || clip.el.currentSrc,
time: mediaSourceTimeAt(clip, time),
settled,
},
]
: [],
);
return pausedFrames.prepare(requests, signal);
},
suspend: pausedFrames.suspendProvider,
prepareNative: pausedFrames.prepareNative,
clear: (keep) =>
pausedFrames.clear(keep, (video, presented) => {
if (clock.isPlaying()) return presented;
const time = clock.now();
const clip = previewVideoClipsAt(time).find(({ el }) => el === video);
if (!clip) return true;
const expected = mediaSourceTimeAt(clip, time);
if (Math.abs(video.currentTime - expected) < 0.02) return true;
video.currentTime = mediaElementTime(expected);
return false;
}),
},
videosToHold: () =>
withTimingResolver(() =>
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? [])
@@ -4978,23 +5077,19 @@ export function initSandboxRuntimeModular(): void {
scenesReadyAt: (time) =>
!windowScenes.some(
({ name, start, end }) => start <= time && time <= end && parkedNames.has(name),
) &&
previewVideoClipsAt(time).every(
({ el }) => !isVideoElement(el) || el.videoWidth > 0 || isUnplayable(el),
),
clipsAt: (time) =>
withTimingResolver(() => {
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
const visibleAt = timedVisibilityAt();
const root = resolveRootCompositionElement();
return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
(clip) =>
isVideoElement(clip.el) &&
!isSilencedByHidden(clip.el) &&
(clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
isClipVisibleAt(time, clip.start, clip.end, duration) &&
clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
);
}),
clipsAt: previewVideoClipsAt,
prepare: (clips, time) => {
moveSceneWindow(time);
for (const clip of clips) {
const { el } = clip;
if (el.preload !== "auto") el.preload = "auto";
if (el.readyState === 1 && !el.seeking)
el.currentTime = mediaElementTime(mediaSourceTimeAt(clip, time));
}
syncRuntimeMedia({
clips,
timeSeconds: time,
@@ -5004,10 +5099,30 @@ export function initSandboxRuntimeModular(): void {
getCompositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
});
},
commit: (time) => applySeek(time),
commit: (time) => {
applySeek(time);
wallClearTimeout(nativeWarmTimer);
const refined = previewSeek?.refining;
if (pausedFrames.active)
nativeWarmTimer = wallSetTimeout(() => {
if (state.tornDown || clock.isPlaying() || previewSeek?.time !== undefined) return;
if (!refined) {
const landed = previewSeek?.refine(time);
if (landed) registerSeekCompletion(landed, "video-frame");
return;
}
pausedFrames.prepareNative();
state.mediaForceSyncNextTick = true;
syncMediaForCurrentState();
}, 120);
},
});
const seekWhenReady = (time: number, options?: { keepPlaying?: boolean }) => {
const seekWhenReady = (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => {
if (nativePlayTime !== undefined && Math.abs(time - nativePlayTime) < 1e-5) return;
nativePlayGeneration++;
nativePlayTime = undefined;
wallClearTimeout(nativeWarmTimer);
if (
!mediaDeferred ||
state.nativeMediaSyncDisabled ||
@@ -5016,6 +5131,7 @@ export function initSandboxRuntimeModular(): void {
(options?.keepPlaying && clock.isPlaying())
) {
previewSeek?.cancel();
pausedFrames.clear();
applySeek(time, options);
return;
}
@@ -5030,7 +5146,7 @@ export function initSandboxRuntimeModular(): void {
for (const { el } of transportMovableClips(clips) ?? []) {
if (!el.paused) el.pause();
}
const landed = previewSeek?.seek(time);
const landed = options?.settled ? previewSeek?.refine(time) : previewSeek?.seek(time);
if (landed) registerSeekCompletion(landed, "video-frame");
postState(true);
return landed;
@@ -5039,6 +5155,33 @@ export function initSandboxRuntimeModular(): void {
const transport: RuntimePlayerTransport = {
play: () => {
flushHeldSeek();
if (
(pausedFrames.active || (pausedFrames.available() && previewSeek?.time !== undefined)) &&
!clock.isPlaying()
) {
wallClearTimeout(nativeWarmTimer);
const generation = ++nativePlayGeneration;
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
if (duration > 0) clock.setDuration(duration);
nativePlayTime =
previewSeek?.targetTime ??
(clock.reachedEnd() ? clock.getPlayStart() : state.currentTime);
const landed = previewSeek?.handoff(nativePlayTime);
if (landed) {
bufferingFor = withTimingResolver(() =>
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).map(({ el }) => el),
);
postState(true);
void landed.then(() => {
if (state.tornDown || generation !== nativePlayGeneration) return;
bufferingFor = [];
nativePlayTime = undefined;
transport.play();
});
return;
}
nativePlayTime = undefined;
}
const destination = previewSeek?.time;
previewSeek?.cancel();
if (destination !== undefined) applySeek(destination);
@@ -5076,9 +5219,22 @@ export function initSandboxRuntimeModular(): void {
postState(true);
},
pause: () => {
const destination = previewSeek?.targetTime ?? nativePlayTime;
nativePlayGeneration++;
nativePlayTime = undefined;
previewSeek?.cancel(true);
const wasBuffering = bufferingFor.length > 0;
bufferingFor = [];
if (clock.isPlaying()) haltPlayback();
if (clock.isPlaying()) {
if (
mediaDeferred &&
pausedFrames.available() &&
!state.nativeMediaSyncDisabled &&
!isExportRenderDrivingFrames()
)
void seekWhenReady(clock.now());
else haltPlayback();
} else if (destination !== undefined) void seekWhenReady(destination);
else if (wasBuffering) postState(true);
},
seek: (timeSeconds, options) => {
@@ -5086,6 +5242,10 @@ export function initSandboxRuntimeModular(): void {
Math.max(0, Number(timeSeconds) || 0),
state.canonicalFps,
);
if (nativePlayTime !== undefined && Math.abs(quantized - nativePlayTime) < 1e-5) return;
nativePlayGeneration++;
nativePlayTime = undefined;
wallClearTimeout(nativeWarmTimer);
heldSeek = null;
const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
if (undecoded.length === 0) {
@@ -5116,6 +5276,10 @@ export function initSandboxRuntimeModular(): void {
return landed;
},
renderSeek: (timeSeconds, options) => {
nativePlayGeneration++;
nativePlayTime = undefined;
wallClearTimeout(nativeWarmTimer);
pausedFrames.clear();
heldSeek = null;
previewSeek?.cancel();
renderCaptureSeekStarted = true;
@@ -5142,7 +5306,8 @@ export function initSandboxRuntimeModular(): void {
if (blurred) registerSeekCompletion(blurred, "motion-blur");
postState(true);
},
getTime: () => heldSeek?.time ?? previewSeek?.time ?? clock.now(),
getTime: () =>
previewSeek?.committing ? clock.now() : (heldSeek?.time ?? previewSeek?.time ?? clock.now()),
getDuration: () => {
const dur = clock.getDuration();
return Number.isFinite(dur) ? dur : 0;
@@ -5239,6 +5404,8 @@ export function initSandboxRuntimeModular(): void {
] as RuntimeDeterministicAdapter[];
patchVideoTextureCompat();
patchWebGLVideoTextureCompat();
patchCanvasVideoFrameCompat();
patchExternalVideoTextureCompat();
// Lets the engine re-render GPU compositions after it injects decoded video
// frames, so video-textured WebGL/WebGPU scenes sample the correct frame.
window.__hfReseekGpu = (time: number) => {
@@ -5969,6 +6136,7 @@ export function initSandboxRuntimeModular(): void {
!ready(el) &&
isPlayableTrack(el) &&
!stillStandsIn(el) &&
!pausedFrames.retained(el) &&
!webAudio.ownsElement(el),
)
.map(({ el }) => el);
@@ -6388,7 +6556,11 @@ export function initSandboxRuntimeModular(): void {
emitAnalyticsEvent("composition_paused", { time: player.getTime() });
},
onStopMedia: () => {
nativePlayGeneration++;
nativePlayTime = undefined;
wallClearTimeout(nativeWarmTimer);
previewSeek?.cancel();
bufferingFor = [];
webAudio.stopAll();
const mediaEls = document.querySelectorAll("video, audio");
for (const el of mediaEls) {
@@ -6440,6 +6612,14 @@ export function initSandboxRuntimeModular(): void {
}
},
onSetNativeMediaSyncDisabled: (disabled) => {
if (disabled) {
nativePlayGeneration++;
nativePlayTime = undefined;
wallClearTimeout(nativeWarmTimer);
previewSeek?.cancel();
pausedFrames.clear();
bufferingFor = [];
}
if (state.nativeMediaSyncDisabled === disabled) return;
state.nativeMediaSyncDisabled = disabled;
state.mediaForceSyncNextTick = true;
+5
View File
@@ -12,6 +12,7 @@ import type { RuntimeMediaClip } from "./types.js";
import { clampAudioGain, clipGain, gainTimesLane } from "../audioGain.js";
import { isMemberGroupHidden } from "../audioGroups.js";
import { findInjectedRenderFrame } from "./renderFrameSibling.js";
import { ownsPausedVideoTime } from "./pausedVideoFrames";
import { isRenderFrameVideo } from "./preloadMedia.js";
import { registerSeekCompletion } from "./adapters/seek-dispatch.js";
import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";
@@ -683,6 +684,10 @@ export function syncRuntimeMedia(params: {
// (no-op when already "auto") and catches elements whose preload
// was overridden after init.ts set it.
if (el.preload !== "auto" && !isRenderFrameVideo(el)) el.preload = "auto";
if (!params.playing && ownsPausedVideoTime(el)) {
if (!el.paused) el.pause();
continue;
}
// Per-element rate × global transport rate
const baseRate = rateAt(clipRate, params.timeSeconds - origin) * params.playbackRate;
// Too slow for the browser to play: kept paused and stepped onto the playhead by seeks.
@@ -0,0 +1,329 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import {
flushPausedVideoCallbacks,
installPausedVideoCallbacks,
observeNativeVideoFrames,
releasePausedVideoCallbacks,
stagePausedVideoCallbacks,
} from "./pausedVideoCallbacks";
const proto = HTMLVideoElement.prototype;
const originalRequest = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
const originalCancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
const pending = new Map<number, { video: HTMLVideoElement; callback: VideoFrameRequestCallback }>();
let nextHandle = 100;
const nativeRequest = vi.fn(function (this: HTMLVideoElement, callback: VideoFrameRequestCallback) {
const handle = ++nextHandle;
pending.set(handle, { video: this, callback });
return handle;
});
const nativeCancel = vi.fn(function (this: HTMLVideoElement, handle: number) {
if (pending.get(handle)?.video === this) pending.delete(handle);
});
beforeEach(() => {
pending.clear();
nextHandle = 100;
nativeRequest.mockClear();
nativeCancel.mockClear();
Object.defineProperty(proto, "requestVideoFrameCallback", {
configurable: true,
writable: true,
value: nativeRequest,
});
Object.defineProperty(proto, "cancelVideoFrameCallback", {
configurable: true,
writable: true,
value: nativeCancel,
});
installPausedVideoCallbacks();
});
afterEach(() => {
if (originalRequest) Object.defineProperty(proto, "requestVideoFrameCallback", originalRequest);
else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
if (originalCancel) Object.defineProperty(proto, "cancelVideoFrameCallback", originalCancel);
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
pending.clear();
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
function metadata(mediaTime: number, presentedFrames: number): VideoFrameCallbackMetadata {
return {
mediaTime,
presentedFrames,
width: 1920,
height: 1080,
presentationTime: 123,
expectedDisplayTime: 125,
processingDuration: 0.003,
};
}
function deliverNative(handle: number, frame: VideoFrameCallbackMetadata) {
const request = pending.get(handle);
expect(request).toBeDefined();
pending.delete(handle);
request!.callback(120, frame);
}
const frame = (mediaTime: number) => ({ mediaTime, width: 1920, height: 1080 });
it("forwards native timing and cancellation when no paused frame owns the video", () => {
const video = document.createElement("video");
const callback = vi.fn();
const id = video.requestVideoFrameCallback(callback);
expect(nativeRequest.mock.contexts[0]).toBe(video);
expect(id).toBe(101);
const nativeFrame = metadata(0.5, 15);
deliverNative(id, nativeFrame);
expect(callback).toHaveBeenCalledExactlyOnceWith(120, nativeFrame);
expect(callback.mock.lastCall?.[1]).toBe(nativeFrame);
const cancelled = video.requestVideoFrameCallback(callback);
video.cancelVideoFrameCallback(cancelled);
expect(nativeCancel).toHaveBeenLastCalledWith(cancelled);
expect(nativeCancel.mock.contexts.at(-1)).toBe(video);
expect(pending.has(cancelled)).toBe(false);
video.cancelVideoFrameCallback(999);
expect(nativeCancel).toHaveBeenLastCalledWith(999);
});
it("reports one presented frame to multiple native observers without counting each callback as a frame", () => {
const video = document.createElement("video");
const first = vi.fn();
const second = vi.fn();
const firstId = video.requestVideoFrameCallback(first);
const secondId = video.requestVideoFrameCallback(second);
deliverNative(firstId, metadata(0.2, 6));
deliverNative(secondId, metadata(0.2, 6));
expect(first.mock.lastCall?.[1].presentedFrames).toBe(6);
expect(second.mock.lastCall?.[1].presentedFrames).toBe(6);
const afterReload = vi.fn();
const next = video.requestVideoFrameCallback(afterReload);
deliverNative(next, metadata(0, 1));
expect(afterReload.mock.lastCall?.[1].presentedFrames).toBe(1);
});
it("preserves native callback receiver and reports authored failures without skipping other observers", () => {
const video = document.createElement("video");
const callback = vi.fn();
const id = video.requestVideoFrameCallback(callback);
const receiver = {};
const native = pending.get(id)!;
native.callback.call(receiver, 120, metadata(0.2, 6));
expect(callback.mock.contexts[0]).toBe(receiver);
const report = vi.fn();
vi.stubGlobal("reportError", report);
const failure = new Error("authored effect failed");
video.requestVideoFrameCallback(() => {
throw failure;
});
const later = vi.fn();
video.requestVideoFrameCallback(later);
stagePausedVideoCallbacks(video, frame(1));
flushPausedVideoCallbacks(video);
expect(report).toHaveBeenCalledExactlyOnceWith(failure);
expect(later).toHaveBeenCalledTimes(1);
});
it("publishes one-shot synthetic callbacks with the staged dimensions and source clock", () => {
vi.spyOn(performance, "now").mockReturnValue(300);
const video = document.createElement("video");
const callback = vi.fn();
const id = video.requestVideoFrameCallback(callback);
stagePausedVideoCallbacks(video, frame(3.5));
expect(callback).not.toHaveBeenCalled();
flushPausedVideoCallbacks(video);
expect(callback).toHaveBeenCalledExactlyOnceWith(300, {
...frame(3.5),
presentedFrames: 1,
presentationTime: 300,
expectedDisplayTime: 300,
processingDuration: 0,
});
expect(nativeCancel).toHaveBeenCalledWith(id);
expect(pending.has(id)).toBe(false);
flushPausedVideoCallbacks(video);
stagePausedVideoCallbacks(video, frame(4));
flushPausedVideoCallbacks(video);
expect(callback).toHaveBeenCalledTimes(1);
});
it("defers a callback registered by a callback until the next published frame", () => {
const video = document.createElement("video");
const observed: number[] = [];
const callback: VideoFrameRequestCallback = (_now, info) => {
observed.push(info.mediaTime);
video.requestVideoFrameCallback(callback);
};
video.requestVideoFrameCallback(callback);
stagePausedVideoCallbacks(video, frame(1));
flushPausedVideoCallbacks(video);
flushPausedVideoCallbacks(video);
expect(observed).toEqual([1]);
stagePausedVideoCallbacks(video, frame(2));
flushPausedVideoCallbacks(video);
expect(observed).toEqual([1, 2]);
expect(pending.size).toBe(1);
});
it("honors cancellation of a later observer while a synthetic notification batch is running", () => {
const video = document.createElement("video");
const cancelled = vi.fn();
video.requestVideoFrameCallback(() => video.cancelVideoFrameCallback(later));
const later = video.requestVideoFrameCallback(cancelled);
stagePausedVideoCallbacks(video, frame(1));
flushPausedVideoCallbacks(video);
expect(cancelled).not.toHaveBeenCalled();
expect(nativeCancel).toHaveBeenCalledWith(later);
expect(pending.size).toBe(0);
});
it("suppresses native metadata while a paused frame owns the pixels and publishes the staged frame instead", () => {
const video = document.createElement("video");
const callback = vi.fn();
const id = video.requestVideoFrameCallback(callback);
stagePausedVideoCallbacks(video, frame(4));
deliverNative(id, metadata(0.5, 15));
expect(callback).not.toHaveBeenCalled();
flushPausedVideoCallbacks(video);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback.mock.lastCall?.[1]).toMatchObject({ mediaTime: 4, presentedFrames: 1 });
expect(nativeCancel).not.toHaveBeenCalled();
});
it("rearms a native callback consumed during pause and still cancels it using the original public handle", () => {
const video = document.createElement("video");
const callback = vi.fn();
const publicId = video.requestVideoFrameCallback(callback);
stagePausedVideoCallbacks(video, frame(3));
deliverNative(publicId, metadata(1, 30));
releasePausedVideoCallbacks(video);
expect(callback).not.toHaveBeenCalled();
expect(nativeRequest).toHaveBeenCalledTimes(2);
const rearmedId = nativeRequest.mock.results.at(-1)!.value;
expect(rearmedId).not.toBe(publicId);
expect(pending.has(rearmedId)).toBe(true);
releasePausedVideoCallbacks(video);
expect(nativeRequest).toHaveBeenCalledTimes(2);
video.cancelVideoFrameCallback(publicId);
expect(nativeCancel).toHaveBeenLastCalledWith(rearmedId);
expect(pending.size).toBe(0);
});
it("resumes the pending observer on native frames after handoff", () => {
const video = document.createElement("video");
const callback = vi.fn();
const publicId = video.requestVideoFrameCallback(callback);
stagePausedVideoCallbacks(video, frame(3));
deliverNative(publicId, metadata(1, 30));
releasePausedVideoCallbacks(video);
const rearmedId = nativeRequest.mock.results.at(-1)!.value;
deliverNative(rearmedId, metadata(3.033, 31));
expect(callback).toHaveBeenCalledExactlyOnceWith(120, metadata(3.033, 31));
expect(pending.size).toBe(0);
});
it("installs once without double-wrapping request or cancellation", () => {
const request = proto.requestVideoFrameCallback;
const cancel = proto.cancelVideoFrameCallback;
installPausedVideoCallbacks();
expect(proto.requestVideoFrameCallback).toBe(request);
expect(proto.cancelVideoFrameCallback).toBe(cancel);
const callback = vi.fn();
const video = document.createElement("video");
video.requestVideoFrameCallback(callback);
stagePausedVideoCallbacks(video, frame(1));
flushPausedVideoCallbacks(video);
expect(callback).toHaveBeenCalledTimes(1);
expect(nativeRequest).toHaveBeenCalledTimes(1);
});
it("invalidates the actual vendored Three VideoTexture before redraw and continues natively after release", async () => {
const { VideoTexture } =
await import("../../../../registry/examples/path-traced-3d/assets/vendor/three-path-tracer.min.js");
const video = document.createElement("video");
Object.defineProperty(video, "readyState", { value: 4 });
const texture = new VideoTexture(video);
texture.update();
expect(texture.version).toBe(0);
stagePausedVideoCallbacks(video, frame(3));
flushPausedVideoCallbacks(video);
expect(texture.version).toBe(1);
texture.update();
expect(texture.version).toBe(1);
stagePausedVideoCallbacks(video, frame(4));
flushPausedVideoCallbacks(video);
expect(texture.version).toBe(2);
// The native decoder can finish a warm seek while provider pixels remain displayed.
const waitingId = [...pending.keys()][0]!;
deliverNative(waitingId, metadata(4, 120));
expect(texture.version).toBe(2);
releasePausedVideoCallbacks(video);
const resumedId = [...pending.keys()][0]!;
deliverNative(resumedId, metadata(4.033, 121));
expect(texture.version).toBe(3);
texture.dispose();
expect(pending.size).toBe(0);
});
it("observes native frames independently of staged authored callbacks", () => {
const video = document.createElement("video");
const authored = vi.fn();
video.requestVideoFrameCallback(authored);
stagePausedVideoCallbacks(video, frame(3));
const recovered = vi.fn(() => true);
const stop = observeNativeVideoFrames(video, recovered);
deliverNative(102, metadata(3.05, 2));
expect(recovered).toHaveBeenCalledOnce();
expect(authored).not.toHaveBeenCalled();
flushPausedVideoCallbacks(video);
expect(authored).toHaveBeenCalledOnce();
expect(authored.mock.lastCall?.[1].mediaTime).toBe(3);
stop();
});
it("cancels a re-armed native observer and ignores an already queued callback", () => {
const video = document.createElement("video");
const recovered = vi.fn(() => false);
const stop = observeNativeVideoFrames(video, recovered);
deliverNative(101, metadata(3, 1));
const queued = pending.get(102)!;
stop();
expect(pending.has(102)).toBe(false);
queued.callback(130, metadata(3.05, 2));
expect(recovered).toHaveBeenCalledOnce();
expect(pending.size).toBe(0);
});
it("uses playing and progressing time when opacity defers a native callback", () => {
const video = document.createElement("video");
let paused = true;
let time = 3;
Object.defineProperties(video, {
paused: { get: () => paused },
currentTime: { get: () => time },
});
const recovered = vi.fn(() => true);
const stop = observeNativeVideoFrames(video, recovered);
video.dispatchEvent(new Event("timeupdate"));
expect(recovered).not.toHaveBeenCalled();
paused = false;
video.dispatchEvent(new Event("playing"));
video.dispatchEvent(new Event("timeupdate"));
expect(recovered).not.toHaveBeenCalled();
time += 1 / 30;
video.dispatchEvent(new Event("timeupdate"));
expect(recovered).toHaveBeenCalledOnce();
expect(pending.size).toBe(0);
stop();
});
it("keeps waiting when no native frame or playback progress arrives", () => {
const video = document.createElement("video");
const recovered = vi.fn(() => true);
const stop = observeNativeVideoFrames(video, recovered);
for (let i = 0; i < 3; i++) video.dispatchEvent(new Event("timeupdate"));
expect(recovered).not.toHaveBeenCalled();
expect(pending.size).toBe(1);
stop();
expect(pending.size).toBe(0);
});
@@ -0,0 +1,171 @@
type Frame = Pick<VideoFrameCallbackMetadata, "mediaTime" | "width" | "height">;
type Pending = { callback: VideoFrameRequestCallback; native: number | null };
type State = {
pending: Map<number, Pending>;
frame: Frame | null;
count: number;
nativeCount: number;
synthetic: boolean;
dirty: boolean;
};
type Request = typeof HTMLVideoElement.prototype.requestVideoFrameCallback;
type Cancel = typeof HTMLVideoElement.prototype.cancelVideoFrameCallback;
type Registry = { states: WeakMap<HTMLVideoElement, State>; request: Request; cancel: Cancel };
type TrackedRequest = Request & { __hfPausedCallbacks?: Registry };
const registry = () =>
(HTMLVideoElement.prototype.requestVideoFrameCallback as TrackedRequest)?.__hfPausedCallbacks;
function stateFor(owner: Registry, video: HTMLVideoElement): State {
let state = owner.states.get(video);
if (!state) {
state = {
pending: new Map(),
frame: null,
count: 0,
nativeCount: 0,
synthetic: false,
dirty: false,
};
owner.states.set(video, state);
}
return state;
}
function requestNative(
owner: Registry,
video: HTMLVideoElement,
state: State,
pending: Pending,
id: () => number,
): number {
return owner.request.call(video, function (this: unknown, now, metadata) {
pending.native = null;
const advance = Math.max(0, metadata.presentedFrames - state.nativeCount);
state.nativeCount = metadata.presentedFrames;
if (state.frame) return;
state.pending.delete(id());
if (!state.synthetic) {
state.count = metadata.presentedFrames;
pending.callback.call(this, now, metadata);
return;
}
state.count += advance;
pending.callback.call(this, now, { ...metadata, presentedFrames: state.count });
});
}
/** Install before authored scripts: Three's VideoTexture registers its first callback in its constructor. */
export function installPausedVideoCallbacks(): void {
const proto = HTMLVideoElement.prototype;
const request = proto.requestVideoFrameCallback as TrackedRequest;
if (typeof request !== "function" || request.__hfPausedCallbacks) return;
const owner: Registry = {
states: new WeakMap(),
request,
cancel: proto.cancelVideoFrameCallback,
};
const patched: TrackedRequest = function (this: HTMLVideoElement, callback) {
const state = stateFor(owner, this);
const pending: Pending = { callback, native: null };
const id = requestNative(owner, this, state, pending, () => id);
pending.native = id;
state.pending.set(id, pending);
return id;
};
patched.__hfPausedCallbacks = owner;
proto.requestVideoFrameCallback = patched;
proto.cancelVideoFrameCallback = function (id) {
const pending = owner.states.get(this)?.pending.get(id);
if (!pending) return owner.cancel.call(this, id);
owner.states.get(this)?.pending.delete(id);
if (pending.native !== null) owner.cancel.call(this, pending.native);
};
}
export function stagePausedVideoCallbacks(video: HTMLVideoElement, frame: Frame): void {
const owner = registry();
if (!owner) return;
const state = stateFor(owner, video);
state.frame = frame;
state.dirty = true;
}
/** Notify once before composition redraw; callbacks registered from a callback wait for the next frame. */
export function flushPausedVideoCallbacks(video: HTMLVideoElement): void {
const owner = registry();
const state = owner?.states.get(video);
if (!owner || !state?.frame || !state.dirty) return;
state.dirty = false;
state.synthetic = true;
state.count++;
const now = performance.now();
const metadata: VideoFrameCallbackMetadata = {
...state.frame,
presentedFrames: state.count,
presentationTime: now,
expectedDisplayTime: now,
processingDuration: 0,
};
for (const [id, pending] of [...state.pending]) {
if (!state.pending.delete(id)) continue;
if (pending.native !== null) owner.cancel.call(video, pending.native);
try {
pending.callback.call(undefined, now, metadata);
} catch (error) {
reportError(error);
}
}
}
export function releasePausedVideoCallbacks(video: HTMLVideoElement): void {
const owner = registry();
const state = owner?.states.get(video);
if (!owner || !state) return;
state.frame = null;
state.dirty = false;
for (const [id, pending] of state.pending) {
if (pending.native === null)
pending.native = requestNative(owner, video, state, pending, () => id);
}
}
/** Recovery must observe native frames even while authored callbacks still see staged pixels. */
export function observeNativeVideoFrames(
video: HTMLVideoElement,
onFrame: () => boolean,
): () => void {
const owner = registry();
const request = owner?.request ?? video.requestVideoFrameCallback;
const cancel = owner?.cancel ?? video.cancelVideoFrameCallback;
let handle: number | null = null;
let stopped = false;
let playingAt: number | null = video.paused ? null : video.currentTime;
const stop = () => {
if (stopped) return;
stopped = true;
if (handle !== null && typeof cancel === "function") cancel.call(video, handle);
video.removeEventListener("playing", playing);
video.removeEventListener("timeupdate", progress);
};
const arm = () => {
if (!stopped && typeof request === "function") handle = request.call(video, presented);
};
const presented = () => {
handle = null;
if (stopped) return;
if (onFrame()) stop();
else arm();
};
const playing = () => {
playingAt = video.currentTime;
};
const progress = () => {
if (stopped || playingAt === null || video.paused || video.currentTime === playingAt) return;
if (onFrame()) stop();
};
video.addEventListener("playing", playing);
video.addEventListener("timeupdate", progress);
arm();
return stop;
}
@@ -0,0 +1,256 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import {
createPausedVideoFrames,
getPausedVideoFrame,
ownsPausedVideoTime,
type PausedVideoFrameProvider,
} from "./pausedVideoFrames";
const sessions: ReturnType<typeof createPausedVideoFrames>[] = [];
beforeEach(() => {
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
clearRect: vi.fn(),
drawImage: vi.fn(),
} as unknown as CanvasRenderingContext2D);
});
afterEach(() => {
for (const session of sessions.splice(0)) session.destroy();
document.body.innerHTML = "";
vi.restoreAllMocks();
});
function fixture() {
const session = createPausedVideoFrames();
sessions.push(session);
const video = document.createElement("video");
video.src = "/original.webm";
video.style.cssText = "position:absolute;width:64px;height:36px;transform:translateX(10px)";
document.body.append(video);
Object.defineProperty(video, "videoWidth", { value: 64 });
Object.defineProperty(video, "videoHeight", { value: 36 });
const release = vi.fn();
const provider: PausedVideoFrameProvider = {
decode: vi.fn(async () => [
{
source: document.createElement("canvas"),
timestamp: 3,
duration: 1 / 30,
width: 64,
height: 36,
release,
},
]),
dispose: vi.fn(),
};
session.setProvider(provider);
const request = [{ video, source: video.src, time: 3 }];
return { session, video, provider, request, release };
}
it("presents decoded pixels with the authored source, dimensions, layout and displayed source clock", async () => {
const { session, video, provider, request, release } = fixture();
const originalSrc = video.src;
const batch = await session.prepare(request, new AbortController().signal);
batch!.present();
video.style.transform = "translateX(25px)";
batch!.refresh();
const canvas = getPausedVideoFrame(video)!;
expect([video.src, video.videoWidth, video.videoHeight, video.currentTime]).toEqual([
originalSrc,
64,
36,
3,
]);
expect(canvas.style.transform).toBe("translateX(25px)");
expect(canvas.nextElementSibling).toBe(video);
expect(ownsPausedVideoTime(video)).toBe(true);
expect(release).toHaveBeenCalledTimes(1);
session.prepareNative();
expect(Object.hasOwn(video, "currentTime")).toBe(false);
expect(ownsPausedVideoTime(video)).toBe(false);
expect(getPausedVideoFrame(video)).toBe(canvas);
session.clear();
expect(getPausedVideoFrame(video)).toBeNull();
expect(canvas.width).toBe(0);
expect(video.hasAttribute("data-hf-preview-video-frame")).toBe(false);
session.destroy();
expect(provider.dispose).toHaveBeenCalledTimes(1);
});
it("releases a late decode when its request is aborted or its source changes", async () => {
const { session, video, request, release } = fixture();
const controller = new AbortController();
const batch = session.prepare(request, controller.signal);
controller.abort();
expect(await batch).toBeNull();
expect(release).toHaveBeenCalledTimes(1);
const changed = session.prepare(request, new AbortController().signal);
video.src = "/replacement.webm";
expect(await changed).toBeNull();
expect(release).toHaveBeenCalledTimes(2);
expect(getPausedVideoFrame(video)).toBeNull();
});
it("does not surrender decoded presentation for the runtime's native seek padding", async () => {
const { session, video, request } = fixture();
const batch = await session.prepare(request, new AbortController().signal);
batch!.present();
batch!.refresh();
const canvas = getPausedVideoFrame(video);
video.currentTime = 3.00001;
expect(ownsPausedVideoTime(video)).toBe(true);
expect(getPausedVideoFrame(video)).toBe(canvas);
expect(video.currentTime).toBe(3);
});
it("gives an authored seek back to native media and preserves an existing currentTime override", async () => {
const { session, video, request } = fixture();
const batch = await session.prepare(request, new AbortController().signal);
batch!.present();
video.currentTime = 7;
expect(video.currentTime).toBe(7);
expect(getPausedVideoFrame(video)).toBeNull();
expect(await session.prepare(request, new AbortController().signal)).toBeNull();
const custom = document.createElement("video");
Object.defineProperty(custom, "currentTime", { value: 2, configurable: true });
expect(
await session.prepare([{ ...request[0]!, video: custom }], new AbortController().signal),
).toBeNull();
expect(custom.currentTime).toBe(2);
});
it("rejects a neighbouring frame that does not cover the requested presentation time", async () => {
const { session, provider, request, release } = fixture();
vi.mocked(provider.decode).mockResolvedValue([
{
source: document.createElement("canvas"),
timestamp: 2,
duration: 1 / 30,
width: 64,
height: 36,
release,
},
]);
expect(await session.prepare(request, new AbortController().signal)).toBeNull();
expect(release).toHaveBeenCalledTimes(1);
});
it("retains the whole previous presentation when staging a new layer fails", async () => {
const { session, video, request, release } = fixture();
const first = await session.prepare(request, new AbortController().signal);
first!.present();
first!.refresh();
const previous = getPausedVideoFrame(video);
const second = await session.prepare(request, new AbortController().signal);
vi.mocked(HTMLCanvasElement.prototype.getContext).mockReturnValueOnce(null);
expect(second!.present()).toBe(false);
expect(getPausedVideoFrame(video)).toBe(previous);
expect(video.currentTime).toBe(3);
expect(release).toHaveBeenCalledTimes(2);
});
it("revalidates source identity after preparing a frame and before publishing it", async () => {
const { session, video, request, release } = fixture();
const prepared = await session.prepare(request, new AbortController().signal);
video.src = "/edited.mp4";
expect(prepared!.valid()).toBe(false);
expect(prepared!.present()).toBe(false);
expect(getPausedVideoFrame(video)).toBeNull();
expect(release).toHaveBeenCalledTimes(1);
});
it("reuses display surfaces across updates and removes them when source children are replaced", async () => {
const { session, video, request } = fixture();
const first = await session.prepare(request, new AbortController().signal);
first!.present();
first!.refresh();
const canvas = getPausedVideoFrame(video);
const next = await session.prepare(request, new AbortController().signal);
next!.present();
next!.refresh();
expect(getPausedVideoFrame(video)).toBe(canvas);
video.append(document.createElement("source"));
await Promise.resolve();
expect(getPausedVideoFrame(video)).toBeNull();
expect(Object.hasOwn(video, "currentTime")).toBe(false);
});
it("keeps the displayed surface in sync with paused editing without releasing decoded pixels", async () => {
const { session, video, request } = fixture();
const batch = await session.prepare(request, new AbortController().signal);
batch!.present();
batch!.refresh();
const canvas = getPausedVideoFrame(video)!;
video.style.transform = "translateX(80px)";
video.style.opacity = "0.4";
video.style.width = "80px";
await Promise.resolve();
expect(getPausedVideoFrame(video)).toBe(canvas);
expect(canvas.style.transform).toBe("translateX(80px)");
expect(canvas.style.opacity).toBe("0.4");
expect(canvas.style.width).toBe("80px");
expect(video.currentTime).toBe(3);
});
it("holds a full-size native snapshot through original-source provider replacement", async () => {
const { session, video, request } = fixture();
Object.defineProperty(video, "readyState", { value: 4 });
session.holdNative([video]);
const snapshot = getPausedVideoFrame(video)!;
expect([snapshot.width, snapshot.height]).toEqual([64, 36]);
expect(snapshot.hasAttribute("data-hyperframes-picker-ignore")).toBe(true);
expect(ownsPausedVideoTime(video)).toBe(false);
const decoded = await session.prepare(request, new AbortController().signal);
expect(decoded!.present()).toBe(true);
expect(getPausedVideoFrame(video)).toBe(snapshot);
expect(ownsPausedVideoTime(video)).toBe(true);
expect(video.currentTime).toBe(3);
});
it("cancels a retained native observer when a new paused picture replaces it", async () => {
const proto = HTMLVideoElement.prototype;
const original = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
const originalCancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
let queued: VideoFrameRequestCallback = () => {};
const cancel = vi.fn();
Object.defineProperty(proto, "requestVideoFrameCallback", {
configurable: true,
writable: true,
value: (callback: VideoFrameRequestCallback) => {
queued = callback;
return 123;
},
});
Object.defineProperty(proto, "cancelVideoFrameCallback", {
configurable: true,
writable: true,
value: cancel,
});
try {
const { session, video, request } = fixture();
const first = await session.prepare(request, new AbortController().signal);
first!.present();
session.prepareNative();
session.clear(new Set([video]), () => false);
const replacement = await session.prepare(request, new AbortController().signal);
replacement!.present();
const picture = getPausedVideoFrame(video);
expect(cancel).toHaveBeenCalledWith(123);
queued(0, {
mediaTime: 3,
width: 64,
height: 36,
presentedFrames: 1,
presentationTime: 0,
expectedDisplayTime: 0,
processingDuration: 0,
});
expect(getPausedVideoFrame(video)).toBe(picture);
expect(ownsPausedVideoTime(video)).toBe(true);
} finally {
if (original) Object.defineProperty(proto, "requestVideoFrameCallback", original);
else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
if (originalCancel) Object.defineProperty(proto, "cancelVideoFrameCallback", originalCancel);
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
}
});
@@ -0,0 +1,423 @@
import { MEDIA_ELEMENT_LEAD_SECONDS } from "./mediaClock";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import {
installPausedVideoCallbacks,
stagePausedVideoCallbacks,
flushPausedVideoCallbacks,
releasePausedVideoCallbacks,
observeNativeVideoFrames,
} from "./pausedVideoCallbacks";
import {
PREVIEW_VIDEO_FRAME_ATTR as FRAME_ATTR,
PREVIEW_VIDEO_SOURCE_ATTR,
PREVIEW_VIDEO_CANVAS_ATTR,
} from "../studioPreviewMark";
export interface PausedVideoFrameRequest {
video: HTMLVideoElement;
source: string;
time: number;
settled?: boolean;
}
export interface PausedVideoFrame {
source: CanvasImageSource;
timestamp: number;
duration: number;
width: number;
height: number;
release(): void;
}
/** Decoding lives in the host; core owns composition timing and presentation. */
export interface PausedVideoFrameProvider {
decode(requests: PausedVideoFrameRequest[], signal: AbortSignal): Promise<PausedVideoFrame[]>;
suspend?(): Promise<void>;
dispose(): void;
}
export interface PreparedPausedVideoFrames {
valid(): boolean;
present(): boolean;
refresh(): void;
release(): void;
}
const SURFACE_BUDGET = 128 * 1024 * 1024;
type Presented = {
video: HTMLVideoElement;
canvas: HTMLCanvasElement;
time: number;
driving: boolean;
restoreTime: () => void;
gpuFrame?: VideoFrame;
stopWaiting?: () => void;
// Paused frame replacement does not retry a source native playback already skipped.
nativeSkipped?: boolean;
};
const presented = new Map<HTMLMediaElement, Presented>();
const authoredSeeks = new WeakSet<HTMLVideoElement>();
export function getPausedVideoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
return presented.get(video)?.canvas ?? null;
}
export function ownsPausedVideoTime(video: HTMLMediaElement): boolean {
return presented.get(video)?.driving === true;
}
export function getPausedExternalVideoFrame(video: HTMLVideoElement): VideoFrame | null {
const entry = presented.get(video);
if (!entry || typeof VideoFrame === "undefined") return null;
entry.gpuFrame ??= new VideoFrame(entry.canvas, { timestamp: Math.round(entry.time * 1e6) });
return entry.gpuFrame;
}
export function captureNativeVideoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const context = canvas.getContext("2d");
if (!context) return null;
try {
context.drawImage(video, 0, 0);
return canvas;
} catch {
return null;
}
}
export function nativeSnapshotStyle(video: HTMLVideoElement): CSSStyleDeclaration | null {
if (getPausedVideoFrame(video)) return null;
// Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return null;
const css = getComputedStyle(video);
if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return null;
if (!video.videoWidth || !video.videoHeight) return null;
return css;
}
function virtualTime(entry: Presented): void {
let proto = Object.getPrototypeOf(entry.video);
let descriptor: PropertyDescriptor | undefined;
while (proto && !descriptor) {
descriptor = Object.getOwnPropertyDescriptor(proto, "currentTime");
proto = Object.getPrototypeOf(proto);
}
const nativeSet = descriptor?.set;
Object.defineProperty(entry.video, "currentTime", {
configurable: true,
get: () => entry.time,
set: (value: number) => {
if (Math.abs(value - entry.time) <= MEDIA_ELEMENT_LEAD_SECONDS * 2) return;
authoredSeeks.add(entry.video);
remove(entry);
nativeSet?.call(entry.video, value);
},
});
entry.restoreTime = () => {
Reflect.deleteProperty(entry.video, "currentTime");
};
}
function remove(entry: Presented): void {
entry.stopWaiting?.();
entry.restoreTime();
releasePausedVideoCallbacks(entry.video);
entry.gpuFrame?.close();
entry.video.removeAttribute(FRAME_ATTR);
entry.video.removeAttribute(PREVIEW_VIDEO_SOURCE_ATTR);
entry.canvas.remove();
entry.canvas.width = entry.canvas.height = 0;
presented.delete(entry.video);
}
export function releasePausedVideoFrame(video: HTMLVideoElement): void {
const entry = presented.get(video);
if (entry) remove(entry);
}
function refresh(entry: Presented): void {
const { video, canvas } = entry;
video.removeAttribute(FRAME_ATTR);
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) {
canvas.remove();
return;
}
const css = getComputedStyle(video);
for (const key of Array.from(css)) {
const value = css.getPropertyValue(key);
if (canvas.style.getPropertyValue(key) !== value) canvas.style.setProperty(key, value);
}
if (css.position !== "absolute" && css.position !== "fixed") {
canvas.style.position = "absolute";
canvas.style.inset = "auto";
canvas.style.left = `${video.offsetLeft}px`;
canvas.style.top = `${video.offsetTop}px`;
canvas.style.margin = "0";
}
canvas.style.setProperty("pointer-events", "none", "important");
canvas.style.setProperty("animation", "none", "important");
canvas.style.setProperty("transition", "none", "important");
if (video.previousElementSibling !== canvas) video.before(canvas);
video.setAttribute(FRAME_ATTR, "");
}
type ReusablePresentation = { entry: Presented; context: CanvasRenderingContext2D };
function markFrameCanvas(canvas: HTMLCanvasElement): void {
canvas.setAttribute(PREVIEW_VIDEO_CANVAS_ATTR, "");
canvas.setAttribute("data-layout-ignore", "");
canvas.setAttribute("data-hyperframes-ignore", "");
canvas.setAttribute("data-hyperframes-picker-ignore", "");
canvas.setAttribute("data-hf-ignore", "");
canvas.setAttribute("aria-hidden", "true");
}
function adoptStagedFrame(next: Presented, existing?: ReusablePresentation): void {
if (existing) {
const { entry, context } = existing;
entry.stopWaiting?.();
entry.stopWaiting = undefined;
entry.gpuFrame?.close();
entry.gpuFrame = undefined;
context.clearRect(0, 0, entry.canvas.width, entry.canvas.height);
context.drawImage(next.canvas, 0, 0);
next.canvas.width = next.canvas.height = 0;
entry.time = next.time;
if (!entry.driving) virtualTime(entry);
entry.driving = true;
return;
}
const { video, canvas } = next;
releasePausedVideoFrame(video);
markFrameCanvas(canvas);
presented.set(video, next);
video.setAttribute(PREVIEW_VIDEO_SOURCE_ATTR, "");
virtualTime(next);
}
/** Keeps original nodes, URLs, intrinsic sizes and the displayed source clock intact. */
export function createPausedVideoFrames() {
installPausedVideoCallbacks();
let provider: PausedVideoFrameProvider | null = null;
const style = document.createElement("style");
style.textContent = `[${FRAME_ATTR}] { opacity: 0 !important; }`;
const observer = new MutationObserver((records) => {
for (const entry of [...presented.values()]) {
if (
!entry.video.isConnected ||
records.some(
(record) =>
(record.type === "childList" && record.target === entry.video) ||
(record.type === "attributes" &&
record.attributeName === "src" &&
(record.target === entry.video || record.target.parentElement === entry.video)),
)
)
remove(entry);
else if (records.some((record) => record.target.contains(entry.video))) refresh(entry);
}
});
observer.observe(document.body, {
childList: true,
subtree: true,
attributes: true,
attributeFilter: ["src", "style", "class", "hidden", "data-hidden"],
});
const clear = (
keep?: Set<HTMLVideoElement>,
current?: (video: HTMLVideoElement, presented: boolean) => boolean,
) => {
for (const entry of [...presented.values()]) {
if (!keep?.has(entry.video)) {
remove(entry);
continue;
}
entry.nativeSkipped = true;
const reveal = (nativeFrame: boolean) => {
if (presented.get(entry.video) !== entry) return true;
if (
entry.video.readyState < 2 ||
entry.video.seeking ||
(current && !current(entry.video, nativeFrame))
)
return false;
remove(entry);
return true;
};
const ready = () => {
reveal(false);
};
const events = ["loadeddata", "canplay", "seeked"];
entry.stopWaiting?.();
for (const event of events) entry.video.addEventListener(event, ready);
const stopFrames = observeNativeVideoFrames(entry.video, () => reveal(true));
entry.stopWaiting = () => {
for (const event of events) entry.video.removeEventListener(event, ready);
stopFrames();
};
}
if (presented.size === 0) style.remove();
};
return {
get active() {
// A skipped native clip may keep a fallback picture without owning paused playback.
return [...presented.values()].some((entry) => !entry.stopWaiting);
},
holdNative(videos: HTMLVideoElement[]) {
const held: Presented[] = [];
for (const video of videos) {
if (!nativeSnapshotStyle(video)) continue;
const canvas = captureNativeVideoFrame(video);
if (!canvas) continue;
const entry: Presented = {
video,
canvas,
time: video.currentTime,
driving: false,
restoreTime: () => {},
};
held.push(entry);
markFrameCanvas(canvas);
presented.set(video, entry);
video.setAttribute(PREVIEW_VIDEO_SOURCE_ATTR, "");
if (!style.isConnected) document.head.append(style);
refresh(entry);
}
return () => {
for (const entry of held)
if (presented.get(entry.video) === entry && !entry.driving) remove(entry);
};
},
skipped: (el: HTMLMediaElement) => presented.get(el)?.nativeSkipped === true,
retained: (el: HTMLMediaElement) => presented.get(el)?.stopWaiting !== undefined,
available: () => provider !== null,
setProvider(next: PausedVideoFrameProvider | null) {
if (provider === next) return;
provider?.dispose();
provider = next;
},
async prepare(requests: PausedVideoFrameRequest[], signal: AbortSignal) {
if (
!provider ||
requests.length === 0 ||
requests.some(
({ video }) =>
authoredSeeks.has(video) ||
(!presented.has(video) && Object.hasOwn(video, "currentTime")),
)
)
return null;
const owner = provider;
const identity = (video: HTMLVideoElement) =>
JSON.stringify([
video.src,
...Array.from(video.querySelectorAll("source"), (source) => source.src),
]);
const sources = requests.map(({ video }) => identity(video));
const frames = await owner.decode(requests, signal);
let released = false;
const release = () => {
if (released) return;
released = true;
for (const frame of frames) frame.release();
};
const current = () =>
!signal.aborted &&
provider === owner &&
requests.every(
({ video }, index) => video.isConnected && identity(video) === sources[index],
);
if (!current()) {
release();
return null;
}
const valid =
frames.reduce((sum, frame) => sum + frame.width * frame.height * 4, 0) <= SURFACE_BUDGET &&
frames.length === requests.length &&
frames.every((frame, index) => {
const time = requests[index]!.time;
return (
frame.width > 0 &&
frame.height > 0 &&
frame.timestamp <= time + 1e-5 &&
time < frame.timestamp + frame.duration + 1e-5
);
});
if (!valid) {
release();
return null;
}
return {
valid: current,
present() {
if (!current()) {
release();
return false;
}
const staged: Presented[] = [];
const reusable = new Map<Presented, ReusablePresentation>();
try {
frames.forEach((frame, index) => {
const { video, time } = requests[index]!;
const canvas = document.createElement("canvas");
canvas.width = frame.width;
canvas.height = frame.height;
staged.push({ video, canvas, time, driving: true, restoreTime: () => {} });
const context = canvas.getContext("2d");
if (!context) throw new Error("Preview canvas unavailable");
context.drawImage(frame.source, 0, 0, canvas.width, canvas.height);
const prior = presented.get(video);
if (
prior &&
prior.canvas.width === frame.width &&
prior.canvas.height === frame.height
) {
const destination = prior.canvas.getContext("2d");
if (!destination) throw new Error("Preview canvas unavailable");
reusable.set(staged[staged.length - 1]!, { entry: prior, context: destination });
}
});
} catch {
for (const { canvas } of staged) canvas.width = canvas.height = 0;
return false;
} finally {
release();
}
const wanted = new Set(requests.map(({ video }) => video));
for (const entry of [...presented.values()]) if (!wanted.has(entry.video)) remove(entry);
if (!style.isConnected) document.head.append(style);
for (const [index, next] of staged.entries()) {
adoptStagedFrame(next, reusable.get(next));
const { timestamp, width, height } = frames[index]!;
stagePausedVideoCallbacks(next.video, { mediaTime: timestamp, width, height });
}
return true;
},
refresh() {
for (const entry of presented.values()) refresh(entry);
},
release,
};
},
suspendProvider: () => provider?.suspend?.(),
prepareNative() {
for (const entry of presented.values()) {
entry.driving = false;
entry.restoreTime();
}
},
notify() {
for (const { video } of presented.values()) flushPausedVideoCallbacks(video);
},
clear,
destroy() {
observer.disconnect();
clear();
provider?.dispose();
provider = null;
},
};
}
+11
View File
@@ -302,6 +302,17 @@ describe("createPickerModule", () => {
}
});
it("picks a video whose decoded preview surface hides the native pixels", () => {
const picker = createPickerModule({ postMessage: createMockPostMessage() });
picker.installPickerApi();
document.body.innerHTML =
'<video id="video" data-hf-preview-video-frame style="opacity:0;width:100px;height:100px"></video>';
const video = document.querySelector("video")!;
expect(candidatesUnder(() => [video])).toContain("#video");
video.removeAttribute("data-hf-preview-video-frame");
expect(candidatesUnder(() => [video])).not.toContain("#video");
});
it("still picks an image inside a faint texture wrapper", () => {
const picker = createPickerModule({ postMessage: createMockPostMessage() });
picker.installPickerApi();
+3 -1
View File
@@ -1,4 +1,5 @@
import type { RuntimeJson, RuntimeOutboundMessage, RuntimePickerElementInfo } from "./types";
import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { swallow } from "./diagnostics";
import { isElementNode } from "./domRealm";
@@ -191,7 +192,8 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule {
if (
Number.isFinite(opacity) &&
opacity <= 0.01 &&
!current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)
!current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR) &&
!current.hasAttribute(PREVIEW_VIDEO_FRAME_ATTR)
)
return true;
current = current.parentElement;
+1 -1
View File
@@ -211,7 +211,7 @@ export function createRuntimePlayer(deps: PlayerDeps): RuntimePlayer {
deps.onRenderFrameSeek(time);
deps.onStatePost(true);
},
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => {
seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => {
const timeline = deps.getTimeline();
if (!timeline) return;
const safeTime = Math.max(0, Number(timeSeconds) || 0);
@@ -0,0 +1,226 @@
import { afterEach, expect, it, vi } from "vitest";
import { createPreviewSeek } from "./previewSeek";
import type { PreparedPausedVideoFrames } from "./pausedVideoFrames";
import { isUnplayable, syncRuntimeMedia } from "./media";
import type { RuntimeMediaClip } from "./types";
const jobs: ReturnType<typeof createPreviewSeek>[] = [];
afterEach(() => {
for (const job of jobs.splice(0)) job.cancel();
document.body.innerHTML = "";
vi.restoreAllMocks();
vi.useRealTimers();
});
const batch = (): PreparedPausedVideoFrames => ({
valid: () => true,
present: vi.fn(() => true),
refresh: vi.fn(),
release: vi.fn(),
});
function fixture() {
const video = document.createElement("video");
document.body.append(video);
const clip: RuntimeMediaClip = {
el: video,
start: 0,
duration: 10,
end: 10,
mediaStart: 0,
playbackRate: 1,
volume: null,
loop: false,
sourceDuration: 10,
};
const pending: Array<{
signal: AbortSignal;
resolve: (result: PreparedPausedVideoFrames | null) => void;
}> = [];
const commit = vi.fn();
const native = vi.fn();
const prepare = vi.fn(
(_clips: RuntimeMediaClip[], _time: number, signal: AbortSignal) =>
new Promise<PreparedPausedVideoFrames | null>((resolve) => pending.push({ signal, resolve })),
);
const host = {
clipsAt: () => [clip],
videosToHold: (): HTMLVideoElement[] => [],
scenesReadyAt: () => true,
prepare: native,
commit,
frames: {
active: () => false,
available: () => true,
prepare,
prepareNative: vi.fn(),
clear: vi.fn(),
},
};
const job = createPreviewSeek(host);
jobs.push(job);
return { job, commit, native, prepare, pending, video, clip, host };
}
it("presents coherent intermediate work while retaining the latest requested destination", async () => {
const { job, commit, pending } = fixture();
const done = job.seek(1);
expect(job.seek(2)).toBe(done);
const first = batch();
pending[0]!.resolve(first);
await Promise.resolve();
expect(commit).toHaveBeenCalledWith(1);
expect(job.time).toBe(2);
expect(first.present).toHaveBeenCalledTimes(1);
const last = batch();
pending[1]!.resolve(last);
await done;
expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 2]);
expect(job.time).toBeUndefined();
});
it("does not rebuild document structure when no native snapshots need hiding", async () => {
const { job, pending } = fixture();
const mutations: MutationRecord[] = [];
const observer = new MutationObserver((records) => mutations.push(...records));
observer.observe(document.head, { childList: true });
const done = job.seek(1);
job.seek(2);
pending[0]!.resolve(batch());
await Promise.resolve();
pending[1]!.resolve(batch());
await done;
expect([...mutations, ...observer.takeRecords()]).toEqual([]);
observer.disconnect();
});
it("resumes provider decoding when a scrub interrupts a native play handoff", async () => {
const { job, commit, prepare, native, pending } = fixture();
const initial = job.seek(1);
pending[0]!.resolve(batch());
await initial;
const play = job.handoff(2);
expect(native).toHaveBeenCalledTimes(1);
const scrub = job.seek(3);
await play;
expect(prepare).toHaveBeenLastCalledWith(expect.any(Array), 3, expect.any(AbortSignal), false);
pending[1]!.resolve(batch());
await scrub;
expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 3]);
});
it("releases a decode that arrives after cancellation without presenting it", async () => {
const { job, commit, pending } = fixture();
const done = job.seek(1);
job.cancel();
expect(pending[0]!.signal.aborted).toBe(true);
const late = batch();
pending[0]!.resolve(late);
await Promise.resolve();
await done;
expect(late.release).toHaveBeenCalledTimes(1);
expect(late.present).not.toHaveBeenCalled();
expect(commit).not.toHaveBeenCalled();
});
it("lets a new gesture supersede a full-quality refinement without changing its destination", async () => {
const { job, prepare, pending, commit } = fixture();
const refining = job.refine(1);
expect(prepare).toHaveBeenLastCalledWith(expect.any(Array), 1, expect.any(AbortSignal), true);
const next = job.seek(2);
expect(pending[0]!.signal.aborted).toBe(true);
pending[0]!.resolve(batch());
pending[1]!.resolve(batch());
await Promise.all([refining, next]);
expect(commit.mock.calls.map(([time]) => time)).toEqual([2]);
});
it("does not erase a seek requested synchronously while publishing the previous one", async () => {
const { job, commit, pending } = fixture();
commit.mockImplementationOnce(() => {
job.seek(4);
});
const first = job.seek(1);
pending[0]!.resolve(batch());
await first;
expect(job.time).toBe(4);
pending[1]!.resolve(batch());
await Promise.resolve();
expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 4]);
});
it("keeps a request-wide hard deadline through refresh and same-clip retarget", async () => {
vi.useFakeTimers();
const { job, commit } = fixture();
const done = job.handoff(3);
await vi.advanceTimersByTimeAsync(4900);
job.refresh();
await vi.advanceTimersByTimeAsync(4100);
job.seek(4, true);
expect(commit).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1000);
await done;
expect(commit.mock.calls.map(([time]) => time)).toEqual([4]);
expect(job.time).toBeUndefined();
});
it("does not reset a source already known to be unplayable on another Play", async () => {
vi.useFakeTimers();
const { job, commit, video, clip } = fixture();
video.src = "/stopped.mp4";
Object.defineProperties(video, {
readyState: { value: 1, configurable: true },
networkState: { value: 1, configurable: true },
});
video.play = vi.fn(async () => {});
video.pause = vi.fn();
syncRuntimeMedia({
clips: [clip],
timeSeconds: 0,
playing: true,
playbackRate: 1,
getCompositionDuration: () => 10,
});
await vi.advanceTimersByTimeAsync(5000);
expect(isUnplayable(video)).toBe(true);
const done = job.handoff(3);
await Promise.resolve();
await done;
expect(commit).toHaveBeenLastCalledWith(3);
expect(job.time).toBeUndefined();
});
it("preserves the held native picture when a new gesture cancels expired background work", async () => {
vi.useFakeTimers();
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
drawImage: vi.fn(),
} as unknown as CanvasRenderingContext2D);
const { job, video, native, pending, prepare, host } = fixture();
let ready = 4;
let time = 0;
Object.defineProperties(video, {
readyState: { get: () => ready },
networkState: { value: 2 },
videoWidth: { value: 64 },
videoHeight: { value: 36 },
currentTime: { get: () => time },
});
host.videosToHold = () => [video];
host.frames.active = () => true;
native.mockImplementation((_clips, target) => {
ready = 1;
time = target;
});
prepare.mockResolvedValueOnce(null);
const expired = job.seek(3);
await vi.advanceTimersByTimeAsync(5000);
await expired;
expect(job.time).toBeUndefined();
expect(job.targetTime).toBe(3);
const picture = document.querySelector("canvas");
expect(picture).not.toBeNull();
const next = job.seek(4);
expect(document.querySelector("canvas")).toBe(picture);
pending[0]!.resolve(batch());
await next;
expect(document.querySelector("canvas")).toBeNull();
});
+281 -84
View File
@@ -1,35 +1,23 @@
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { isUnplayable, mediaSourceTimeAt } from "./media";
import type { RuntimeMediaClip } from "./types";
import { isVideoElement } from "./domRealm";
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
import {
nativeSnapshotStyle,
captureNativeVideoFrame,
type PreparedPausedVideoFrames,
} from "./pausedVideoFrames";
const HELD_VIDEO_ATTR = "data-hf-seek-video";
const SEEK_CAP_MS = 5000;
const EVENTS = ["loadedmetadata", "loadeddata", "canplay", "seeked", "error"] as const;
function videoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const context = canvas.getContext("2d");
if (!context) return null;
try {
context.drawImage(video, 0, 0);
return canvas;
} catch {
return null;
}
}
/** Keep a displayed video in its layout slot while its decoder moves independently of the page. */
function holdVideoFrame(video: HTMLVideoElement): (() => void) | undefined {
// Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return;
const css = getComputedStyle(video);
if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return;
if (!video.videoWidth || !video.videoHeight) return;
const canvas = videoFrame(video);
const css = nativeSnapshotStyle(video);
if (!css) return;
const canvas = captureNativeVideoFrame(video);
if (!canvas) return;
for (const name of Array.from(css)) canvas.style.setProperty(name, css.getPropertyValue(name));
canvas.style.setProperty("animation", "none", "important");
@@ -49,7 +37,7 @@ function holdVideoFrames(videos: HTMLVideoElement[]): () => void {
const restore = videos.map(holdVideoFrame);
const style = document.createElement("style");
style.textContent = `[${HELD_VIDEO_ATTR}] { display: none !important; }`;
document.head.appendChild(style);
if (restore.some(Boolean)) document.head.appendChild(style);
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
return () => {
for (const putBack of restore) putBack?.();
@@ -64,72 +52,226 @@ type SeekHost = {
scenesReadyAt: (time: number) => boolean;
prepare: (clips: RuntimeMediaClip[], time: number) => void;
commit: (time: number) => void;
frames?: {
available(): boolean;
active?(): boolean;
skipped?(el: HTMLMediaElement): boolean;
hold?(videos: HTMLVideoElement[]): (() => void) | undefined;
prepare(
clips: RuntimeMediaClip[],
time: number,
signal: AbortSignal,
settled?: boolean,
): Promise<PreparedPausedVideoFrames | null>;
suspend?(): Promise<void> | undefined;
prepareNative(): void;
clear(keep?: Set<HTMLVideoElement>): void;
};
};
/** One decoder job at a time; requests arriving during it replace the queued destination. */
/** One preparation at a time; publish coherent progress and coalesce newer destinations. */
export function createPreviewSeek(host: SeekHost) {
let pending: { time: number; done: Promise<void>; resolve: () => void } | null = null;
let releasePicture = () => {};
type SeekRequest = {
time: number;
native: boolean;
settled: boolean;
deadline?: ReturnType<typeof wallSetTimeout>;
decoderRelease?: Promise<void> | null;
force: () => void;
done: Promise<void>;
resolve: () => void;
};
let pending: SeekRequest | null = null;
// Recovery keeps its listeners without blocking paused transport work.
let recovery: SeekRequest | null = null;
let releasePicture: (() => void) | null = null;
const frameHolds: Array<() => void> = [];
let stopWaiting = () => {};
let runningClips: RuntimeMediaClip[] = [];
let cancelWork = () => {};
let committing = false;
let committingSettled = false;
let runId = 0;
const needsPreparation = (native: boolean) => native || host.frames?.available();
const nativeReady = ({ el, ...clip }: RuntimeMediaClip, time: number) =>
!el.isConnected ||
(el.readyState >= 2 &&
!el.seeking &&
Math.abs(el.currentTime - mediaSourceTimeAt({ el, ...clip }, time)) < 0.02);
const ready = (clips: RuntimeMediaClip[], time: number) =>
host.scenesReadyAt(time) &&
clips.every((clip) => {
const { el } = clip;
return (
!el.isConnected ||
isUnplayable(el) ||
(el.readyState >= 2 &&
!el.seeking &&
Math.abs(el.currentTime - mediaSourceTimeAt(clip, time)) < 0.02)
);
});
clips.every(
(clip) => isUnplayable(clip.el) || host.frames?.skipped?.(clip.el) || nativeReady(clip, time),
);
const cancel = () => {
const request = pending;
pending = null;
const releaseHeldPicture = () => {
releasePicture?.();
releasePicture = null;
};
const clearFrames = (clips: RuntimeMediaClip[], native: boolean, time: number) =>
host.frames?.clear(
native
? new Set(
clips.flatMap((clip) =>
isVideoElement(clip.el) && !nativeReady(clip, time) ? [clip.el] : [],
),
)
: undefined,
);
const retireRequest = (request: SeekRequest | null) => {
if (pending === request) pending = null;
if (recovery === request) recovery = null;
wallClearTimeout(request?.deadline);
};
const holdPicture = () => {
const videos = host.videosToHold();
const release = host.frames?.hold?.(videos);
if (release) frameHolds.push(release);
releasePicture ??= holdVideoFrames(videos);
};
const cancel = (keepPicture = false) => {
const request = pending ?? recovery;
retireRequest(request);
runId++;
stopWaiting();
stopWaiting = () => {};
releasePicture();
releasePicture = () => {};
cancelWork();
cancelWork = () => {};
if (!keepPicture) {
releaseHeldPicture();
for (const release of frameHolds.splice(0)) release();
}
request?.resolve();
};
const afterDecoderRelease = (request: SeekRequest, start: () => void) => {
if (request.decoderRelease === undefined) {
try {
request.decoderRelease = host.frames?.suspend?.() ?? null;
} catch {
request.decoderRelease = null;
}
}
if (request.decoderRelease) void request.decoderRelease.then(start, start);
else start();
};
const run = () => {
const request = pending;
const request = pending ?? recovery;
if (!request) return;
cancelWork();
const id = ++runId;
const time = request.time;
const clips = host.clipsAt(time);
runningClips = clips;
const selection = clips.map((clip) => ({ el: clip.el, time: mediaSourceTimeAt(clip, time) }));
const controller = new AbortController();
let frame: PreparedPausedVideoFrames | null = null;
let preparing = true;
let nativePhase = false;
let finished = false;
const live = () => (pending === request || recovery === request) && runId === id && !finished;
cancelWork = () => {
controller.abort();
frame?.release();
};
const publish = () => {
frameHolds.length = 0;
releaseHeldPicture();
if (!frame) clearFrames(host.clipsAt(time), request.native, time);
committing = true;
committingSettled = request.settled;
try {
host.commit(time);
frame?.refresh();
} finally {
committing = false;
committingSettled = false;
}
};
const selectionChanged = () => {
const current = host.clipsAt(time);
return (
current.length !== selection.length ||
current.some(
(clip, index) =>
clip.el !== selection[index]?.el ||
Math.abs(mediaSourceTimeAt(clip, time) - selection[index]!.time) > 1e-8,
)
);
};
const frameReadyToPublish = () => {
if (!frame) return true;
if (selectionChanged()) {
stopWaiting();
run();
return false;
}
if (!frame.valid() || !frame.present()) {
native();
return false;
}
return true;
};
const finish = () => {
if (finished || pending !== request || id !== runId) return;
if (!live() || !frameReadyToPublish()) return;
finished = true;
stopWaiting();
stopWaiting = () => {};
if (request.time !== time) {
if (ready(host.clipsAt(time), time)) {
releasePicture();
host.commit(time);
releasePicture = holdVideoFrames(host.videosToHold());
if (frame || ready(host.clipsAt(time), time)) {
publish();
if (pending !== request && recovery !== request) return;
holdPicture();
}
run();
return;
}
cancel();
host.commit(time);
retireRequest(request);
runId++;
const cleanup = cancelWork;
cancelWork = () => {};
try {
publish();
} finally {
cleanup();
request.resolve();
}
};
request.force = () => {
if (!live()) return;
if (request.time === time) finish();
else {
stopWaiting();
run();
request.force();
}
};
const check = () => {
if (!preparing && ready(host.clipsAt(time), time)) finish();
if (preparing || !live()) return;
if (frame ? host.scenesReadyAt(time) : ready(host.clipsAt(time), time)) finish();
};
const native = () => {
if (!live() || nativePhase) return;
nativePhase = true;
controller.abort();
frame?.release();
frame = null;
host.frames?.prepareNative();
host.prepare(host.clipsAt(time), time);
preparing = false;
check();
};
const metadata = () => {
if (pending !== request || finished || id !== runId) return;
// A source/proxy change resets currentTime; the parked clock still describes the old picture.
if (!live()) return;
if (!nativePhase) {
check();
return;
}
preparing = true;
host.prepare(host.clipsAt(time), time);
preparing = false;
@@ -147,51 +289,106 @@ export function createPreviewSeek(host: SeekHost) {
);
};
});
const cap = wallSetTimeout(finish, SEEK_CAP_MS);
const expire = () => {
if (!live()) return;
if (request.native) check();
else if (!nativePhase && !frame) {
native();
if (live()) cap = wallSetTimeout(expire, SEEK_CAP_MS);
} else if (host.frames?.active?.()) {
pending = null;
recovery = request;
request.resolve();
} else finish();
};
let cap = wallSetTimeout(expire, SEEK_CAP_MS);
stopWaiting = () => {
wallClearTimeout(cap);
for (const remove of listeners) remove();
};
host.prepare(clips, time);
preparing = false;
check();
if (request.native) afterDecoderRelease(request, native);
else if (recovery !== request && host.frames && needsPreparation(request.native)) {
void host.frames
.prepare(clips, time, controller.signal, request.settled)
.then((decoded) => {
if (!live() || controller.signal.aborted) {
decoded?.release();
return;
}
if (!decoded) return native();
frame = decoded;
preparing = false;
check();
})
.catch(native);
} else native();
};
const retarget = (time: number) => {
const request = pending!;
request.time = time;
const clips = host.clipsAt(time);
if (
clips.length !== runningClips.length ||
clips.some((clip, index) => clip.el !== runningClips[index]?.el)
) {
stopWaiting();
run();
}
return request.done;
};
const seek = (time: number, native = false, settled = false): Promise<void> | void => {
if (recovery) cancel(true);
if (pending && (pending.native !== native || pending.settled !== settled)) cancel(true);
if (pending) return retarget(time);
if (!needsPreparation(native) && ready(host.clipsAt(time), time)) {
releaseHeldPicture();
clearFrames(host.clipsAt(time), native, time);
host.commit(time);
return;
}
let resolve = () => {};
const done = new Promise<void>((settle) => {
resolve = settle;
});
const request = { time, native, settled, done, resolve, force: () => {} };
pending = request;
// Refreshing a run must not restart the wall-clock budget for Play.
if (native) pending.deadline = wallSetTimeout(() => request.force(), SEEK_CAP_MS * 2);
holdPicture();
run();
return done;
};
return {
get committing() {
return committing;
},
get refining() {
return committingSettled;
},
get time() {
return pending?.time;
},
seek(time: number): Promise<void> | void {
if (pending) {
const request = pending;
request.time = time;
const clips = host.clipsAt(time);
if (
clips.length !== runningClips.length ||
clips.some((clip, index) => clip.el !== runningClips[index]?.el)
) {
stopWaiting();
run();
}
return request.done;
}
if (ready(host.clipsAt(time), time)) {
host.commit(time);
return;
}
let resolve = () => {};
const done = new Promise<void>((settle) => {
resolve = settle;
});
pending = { time, done, resolve };
releasePicture = holdVideoFrames(host.videosToHold());
run();
return done;
get targetTime() {
return (pending ?? recovery)?.time;
},
ownsRecovery(el: HTMLMediaElement) {
return recovery !== null && runningClips.some((clip) => clip.el === el);
},
seek,
refine(time: number) {
return seek(time, false, true);
},
refresh() {
if (!pending) return;
stopWaiting();
run();
if (pending || recovery) {
stopWaiting();
run();
}
},
handoff(time: number) {
cancel(true);
host.frames?.prepareNative();
return seek(time, true);
},
cancel,
};
+1 -1
View File
@@ -337,7 +337,7 @@ export type RuntimePlayer = {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => HeldSeek;
renderSeek: (timeSeconds: number, options?: RuntimeSeekOptions) => void;
getTime: () => number;
getDuration: () => number;
+2
View File
@@ -11,6 +11,7 @@ import type { PlayerAPI } from "../core.types";
import type { ClipTree } from "./clipTree";
import type { CompositionLengthSource } from "./compositionLength";
import type { SvgSelectorAliases } from "../compiler/svgSelectorAliases";
import type { PausedVideoFrameProvider } from "./pausedVideoFrames";
type ThreeClockLike = {
elapsedTime: number;
@@ -79,6 +80,7 @@ declare global {
* runtime's paused-side enforcement leaves it alone. Always release. */
leasePausedMedia?: (el: HTMLMediaElement) => void;
releasePausedMedia?: (el: HTMLMediaElement) => void;
setPausedVideoFrameProvider?: (provider: PausedVideoFrameProvider | null) => void;
/** Where Studio's loop wraps to while it plays, or null: the clips due there stay loaded. */
setLoopStart?: (seconds: number | null) => void;
/** Read-only level taps for the Studio meters: nothing exists until
+6
View File
@@ -2,6 +2,9 @@ import { isWebKit } from "./runtime/mediaClock.js";
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
export const PREVIEW_SEEK_HOLD_ATTR = "data-hf-preview-seek-hold";
export const PREVIEW_VIDEO_FRAME_ATTR = "data-hf-preview-video-frame";
export const PREVIEW_VIDEO_SOURCE_ATTR = "data-hf-preview-video-source";
export const PREVIEW_VIDEO_CANVAS_ATTR = "data-hf-preview-video-canvas";
export const STUDIO_PREVIEW_CAPTURE_MODE = "capture";
export const PLAYER_FRAME_ATTR = "data-hf-player-frame";
@@ -31,6 +34,9 @@ export const CLIP_HIDDEN_ATTR = "data-hf-clip-hidden";
export const STUDIO_PREVIEW_FAR_SCENE_ATTR = "data-hf-far-scene";
export const STUDIO_PREVIEW_PARKED_SCENE_ATTR = "data-hf-parked-scene";
export const PREVIEW_ONLY_ATTRS = [
PREVIEW_VIDEO_FRAME_ATTR,
PREVIEW_VIDEO_SOURCE_ATTR,
PREVIEW_VIDEO_CANVAS_ATTR,
STUDIO_PREVIEW_UPCOMING_ATTR,
STUDIO_PREVIEW_FAR_SCENE_ATTR,
STUDIO_PREVIEW_PARKED_SCENE_ATTR,
@@ -168,6 +168,20 @@ describe("probeMediaMetadata", () => {
.source?.bitDepth,
).toBe(12);
expect((await probe({ pix_fmt: "yuv420p" })).source).toBeUndefined();
expect(
(
await probe({
width: 1920,
height: 1080,
r_frame_rate: "30000/1001",
avg_frame_rate: "30000/1001",
})
).source,
).toMatchObject({ frameRate: 30000 / 1001, variableFrameRate: false });
expect(
(await probe({ width: 1920, height: 1080, r_frame_rate: "60/1", avg_frame_rate: "24/1" }))
.source,
).toMatchObject({ frameRate: 60, variableFrameRate: true });
});
it("lists every stream with its flags, for a failure report", async () => {
@@ -101,6 +101,8 @@ interface FfprobeStream {
width?: number;
height?: number;
bit_rate?: string;
r_frame_rate?: string;
avg_frame_rate?: string;
disposition?: Record<string, number | undefined>;
tags?: Record<string, string | undefined>;
}
@@ -226,11 +228,25 @@ function sourceFactsOf(
): PreviewSourceFacts | undefined {
if (!stream?.width || !stream.height) return undefined;
const bitrate = Number(stream.bit_rate ?? formatBitrate);
const rate = (raw: string | undefined) => {
const [numerator, denominator = "1"] = (raw ?? "0").split("/");
const value = Number(numerator) / Number(denominator);
return Number.isFinite(value) && value > 0 ? value : 0;
};
const nominal = rate(stream.r_frame_rate);
const average = rate(stream.avg_frame_rate);
return {
width: stream.width,
height: stream.height,
bitDepth: bitDepthOf(stream),
bitrate: Number.isFinite(bitrate) ? bitrate : 0,
...(nominal > 0
? {
frameRate: nominal,
variableFrameRate:
average > 0 && Math.abs(nominal - average) / Math.max(nominal, average) > 0.1,
}
: {}),
};
}
@@ -317,7 +333,7 @@ export async function probeMediaMetadata(
"-v",
"error",
"-show_entries",
"stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample,width,height,bit_rate:stream_disposition:stream_tags=alpha_mode:format=duration,bit_rate",
"stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample,width,height,bit_rate,r_frame_rate,avg_frame_rate:stream_disposition:stream_tags=alpha_mode:format=duration,bit_rate",
"-of",
"json",
"--",
@@ -33,7 +33,10 @@ async function loadHelper(
}> {
vi.resetModules();
const resolveProxy = vi.fn(async () => "/unused-prewarm-proxy-path");
vi.doMock("./proxyTranscoder.js", () => ({ resolveProxy, PROXY_PARAMS_VERSION: "v1" }));
vi.doMock("./proxyTranscoder.js", () => ({
resolveProxy,
PROXY_PARAMS_VERSION: "v1",
}));
// The real codec table and pre-warm gate; only the ffprobe-backed scan is
// replaced, so this asserts the shipped policy rather than a copy of it.
vi.doMock("./mediaCodecMap.js", async () => ({
@@ -172,19 +175,20 @@ describe("injectMediaCodecMapIntoHtml pre-warm", () => {
});
describe("injectMediaCodecMapIntoHtml placement", () => {
it("puts the map before the document's own </head>, not inside an inlined script that prints one", async () => {
it("puts facts before runtime and authored head scripts without confusing script contents for document tags", async () => {
const { injectMediaCodecMapIntoHtml } = await loadHelper({
"/audio/seg01.webm": hostile("vp9", 'video/webm; codecs="vp09.00.10.08"'),
});
const vendor = 'p.print("<head>"),p.print("</head>"),p.print("<body>")';
const html = await injectMediaCodecMapIntoHtml(
`<html><head><script>${vendor}</script></head><body></body></html>`,
`<html><head><script src="/api/runtime.js"></script><script>${vendor}</script></head><body></body></html>`,
tmpProject(),
[{ html: "<html></html>" }],
);
expect(html).toContain(`<script>${vendor}</script>`);
expect(html.indexOf("data-hf-media-codec-map")).toBeGreaterThan(html.indexOf(vendor));
expect(html.indexOf("data-hf-media-codec-map")).toBeLessThan(html.indexOf("/api/runtime.js"));
expect(html.indexOf("data-hf-media-codec-map")).toBeLessThan(html.indexOf(vendor));
});
});
@@ -1,6 +1,6 @@
import { resolve } from "node:path";
import { formatPreviewProxyBox, type PreviewProxyBox } from "@hyperframes/core";
import { insertBeforeCloseTag } from "@hyperframes/core/compiler/html-document";
import { injectTagsAtHeadStart } from "@hyperframes/core/compiler/html-document";
import type { StudioApiAdapter } from "../types.js";
import {
createMediaCodecProbeCache,
@@ -134,7 +134,7 @@ export async function injectMediaCodecMapIntoHtml(
`window.__HF_MEDIA_SOURCE_FACTS__=${safeJson(sourceFacts)};`,
];
const tag = `<script data-hf-media-codec-map>${assignments.join("")}</script>`;
return insertBeforeCloseTag(html, "head", `${tag}\n`) ?? `${tag}\n${html}`;
return injectTagsAtHeadStart(html, tag);
}
/**
@@ -282,6 +282,24 @@ describe("registerPreviewRoutes", () => {
expect(html).toContain("window.AUTHOR_SEEN = 1");
});
it("keeps the accelerated-canvas tracker before authored scripts in blur previews", async () => {
const projectDir = createProjectDir();
writeFileSync(
join(projectDir, "index.html"),
'<html><head><script>window.AUTHOR_CANVAS = 1</script></head><body data-motion-blur="on"></body></html>',
);
const app = new Hono();
registerPreviewRoutes(app, createAdapter(projectDir));
const html = await (await app.request("http://localhost/projects/demo/preview")).text();
expect(html.indexOf("data-hf-accel-tracker")).toBeGreaterThan(-1);
expect(html.indexOf("data-hf-accel-tracker")).toBeLessThan(
html.indexOf("window.AUTHOR_CANVAS"),
);
expect(html.indexOf('src="/api/runtime.js"')).toBeLessThan(
html.indexOf("window.AUTHOR_CANVAS"),
);
});
it("serves one preview runtime when the page on disk already links one", async () => {
const projectDir = createProjectDir();
writeFileSync(
@@ -1626,6 +1644,71 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {
return import("./preview.js");
}
it.each(["bundled", "disk after null bundle", "disk after failed bundle"])(
"installs preview hooks before authored head consumers with flags, facts and error capture already present (%s)",
async (path) => {
const projectDir = createProjectDir();
const authored = `<!doctype html><html><head>
<script data-hf-accel-tracker>window.TRACKER_KEPT = true;</script>
<script data-hyperframes-preview-runtime="1" src="/stale-runtime.js"></script>
<script id="head-consumer">window.HEAD_CONSUMER = document.createElement("video").requestVideoFrameCallback;</script>
<script id="head-library" src="assets/head-library.js"></script>
</head><body><main data-composition-id="main" data-duration="2" data-motion-blur="on"></main>
<script id="body-consumer">window.BODY_CONSUMER = true;</script>
</body></html>`;
writeFileSync(join(projectDir, "index.html"), authored);
writeFileSync(
join(projectDir, "scene.html"),
`<template><main data-composition-id="scene" data-duration="2"></main><script id="body-consumer">window.BODY_CONSUMER = true;</script></template>`,
);
const { registerPreviewRoutes: register } = await loadPreviewModule({
scanMapImpl: async () => ({
"/clip.webm": {
codecName: "vp9",
browserHostile: true,
representativeMime: 'video/webm; codecs="vp09.00.10.08"',
},
}),
});
const app = new Hono();
register(
app,
createAdapter(projectDir, {
bundle: async () => {
if (path === "disk after failed bundle") throw new Error("bundler unavailable");
return path === "bundled" ? authored : null;
},
}),
);
for (const suffix of ["", "/comp/scene.html"]) {
const response = await app.request("http://localhost/projects/demo/preview" + suffix);
expect(response.status).toBe(200);
const html = await response.text();
const runtime = html.indexOf('src="/api/runtime.js"');
expect(runtime).toBeGreaterThan(-1);
for (const early of [
`name="${STUDIO_PREVIEW_MARK_META}"`,
`window.${STUDIO_PREVIEW_ERRORS}=`,
"window.__HF_MEDIA_CODEC_MAP__=",
"window.__HF_MEDIA_SOURCE_FACTS__=",
]) {
expect(html.indexOf(early)).toBeGreaterThan(-1);
expect(html.indexOf(early)).toBeLessThan(runtime);
}
for (const authoredId of ["head-consumer", "head-library", "body-consumer"])
expect(runtime).toBeLessThan(html.indexOf(`id="${authoredId}"`));
expect(html).not.toContain("stale-runtime.js");
expect(
parseHTML(html).document.querySelectorAll("script[data-hyperframes-preview-runtime]"),
).toHaveLength(1);
expect(
parseHTML(html).document.querySelectorAll("script[data-hf-accel-tracker]"),
).toHaveLength(1);
expect(html).toContain("window.TRACKER_KEPT = true;");
}
},
);
describe("?hf-proxy=h264 on the static asset route", () => {
it("serves proxy bytes with Accept-Ranges on a full request, and a 206 range slice on a Range request", async () => {
const projectDir = createProjectDir();
+3 -3
View File
@@ -150,10 +150,10 @@ function parseStudioMotionManifestContent(content: string): {
}
}
/** Swaps any runtime already in the page for the preview runtime, placed as the bundler places it. */
/** Preview hooks must install before authored head scripts register media consumers. */
function withPreviewRuntime(html: string, runtimeUrl: string): string {
const tag = `<script ${RUNTIME_BOOTSTRAP_ATTR}="1" src="${runtimeUrl}"></script>`;
return insertRuntimeTag(stripEmbeddedRuntimeScripts(html), tag);
return insertRuntimeTag(stripEmbeddedRuntimeScripts(html), tag, true);
}
function injectScriptTagIntoHead(html: string, scriptTag: string): string {
@@ -685,7 +685,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
gsapBaseUrl,
);
if (!html) return c.text("not found", 404);
html = withRootImportMap(html, baseHref);
html = withPreviewRuntime(withRootImportMap(html, baseHref), adapter.runtimeUrl);
recordPreviewReferences(project.dir, html);
html = ensureHfIds(await transformPreviewHtml(html, adapter, project, compPath));
html = injectStudioPreviewAugmentations(html, adapter, signature, project.dir, compPath, {
+1 -1
View File
@@ -114,7 +114,7 @@
"cn": "0.2.4",
"dockview-react": "8.3.1",
"gsap": "^3.13.0",
"mediabunny": "^1.45.3"
"mediabunny": "^1.65.2"
},
"devDependencies": {
"@hyperframes/producer": "workspace:*",
@@ -0,0 +1,75 @@
// @vitest-environment jsdom
import { afterEach, expect, it } from "vitest";
import {
PREVIEW_VIDEO_FRAME_ATTR,
PREVIEW_VIDEO_CANVAS_ATTR,
} from "@hyperframes/core/studio-preview-mark";
import { getCuratedComputedStyles, isElementVisibleThroughAncestors } from "./domEditingDom";
import {
collectDomEditLayerItems,
resolveDomEditSelection,
resolveVisualDomEditSelectionTarget,
} from "./domEditing";
const options = { activeCompositionPath: "index.html", isMasterView: true, skipSourceProbe: true };
afterEach(() => {
document.body.innerHTML = "";
document.head.innerHTML = "";
});
function preview(opacity: string) {
document.head.innerHTML = `<style>[${PREVIEW_VIDEO_FRAME_ATTR}] { opacity: 0 !important; }</style>`;
document.body.innerHTML = `<main data-composition-id="main"><canvas ${PREVIEW_VIDEO_CANVAS_ATTR} style="opacity:${opacity}" data-hf-ignore></canvas><video id="clip" ${PREVIEW_VIDEO_FRAME_ATTR} style="opacity:${opacity}" src="clip.mp4"></video></main>`;
const video = document.querySelector("video")!;
const canvas = document.querySelector("canvas")!;
const root = document.querySelector("main")!;
Object.defineProperty(video, "getBoundingClientRect", {
value: () => ({
x: 10,
y: 20,
left: 10,
top: 20,
right: 210,
bottom: 120,
width: 200,
height: 100,
}),
});
return { video, canvas, root };
}
it("keeps a provider-backed video selectable and listed with its logical opacity", async () => {
const { video, canvas, root } = preview("0.65");
expect(getComputedStyle(video).opacity).toBe("0");
expect(isElementVisibleThroughAncestors(video)).toBe(true);
expect(resolveVisualDomEditSelectionTarget([canvas, video], options)).toBe(video);
expect(collectDomEditLayerItems(root, options).map((item) => item.element)).toEqual([video]);
const selected = await resolveDomEditSelection(video, options);
expect(selected?.computedStyles.opacity).toBe("0.65");
expect(selected?.inlineStyles.opacity).toBe("0.65");
expect(getComputedStyle(video).opacity).toBe("0");
});
it("keeps authored zero opacity invisible and reports zero to the inspector", async () => {
const { video, canvas } = preview("0");
expect(isElementVisibleThroughAncestors(video)).toBe(false);
expect(resolveVisualDomEditSelectionTarget([canvas, video], options)).toBeNull();
expect(getCuratedComputedStyles(video).opacity).toBe("0");
const selected = await resolveDomEditSelection(video, options);
expect(selected?.computedStyles.opacity).toBe("0");
expect(video.style.opacity).toBe("0");
});
it("still respects a hidden ancestor when the preview frame itself is visible", () => {
const { video, root } = preview("0.65");
root.style.opacity = "0";
expect(isElementVisibleThroughAncestors(video)).toBe(false);
expect(resolveVisualDomEditSelectionTarget([video], options)).toBeNull();
});
it("does not treat an unrelated preceding canvas as the video's visible frame", () => {
const { video, canvas } = preview("0.65");
canvas.removeAttribute(PREVIEW_VIDEO_CANVAS_ATTR);
expect(isElementVisibleThroughAncestors(video)).toBe(false);
expect(getCuratedComputedStyles(video).opacity).toBe("0");
});
@@ -3,6 +3,10 @@
* selector utilities, and composition source resolution.
* No imports from other domEditing* modules — safe to import from anywhere.
*/
import {
PREVIEW_VIDEO_FRAME_ATTR,
PREVIEW_VIDEO_CANVAS_ATTR,
} from "@hyperframes/core/studio-preview-mark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading";
import { getSourceScopedSelectorIndex } from "../../utils/sourceScopedSelectorIndex";
import { CURATED_STYLE_PROPERTIES } from "./domEditingTypes";
@@ -34,12 +38,24 @@ export function isEditableTextLeaf(el: HTMLElement): boolean {
return isTextBearingTag(el.tagName.toLowerCase()) && el.children.length === 0;
}
/** The preview canvas keeps the source's authored opacity while its native pixels are hidden. */
export function getLogicalOpacity(el: Element, computed: CSSStyleDeclaration): string {
const overlay = el.previousElementSibling;
if (
el.hasAttribute(PREVIEW_VIDEO_FRAME_ATTR) &&
overlay?.localName === "canvas" &&
overlay.hasAttribute(PREVIEW_VIDEO_CANVAS_ATTR)
)
return el.ownerDocument.defaultView?.getComputedStyle(overlay).opacity ?? computed.opacity;
return computed.opacity;
}
/** Does this node render AT ALL, ignoring what it inherits? Sole owner of the
* rule; the walk below only decides which nodes to ask it about. */
function elementRendersItself(win: Window, el: HTMLElement): boolean {
const computed = win.getComputedStyle(el);
if (computed.display === "none" || computed.visibility === "hidden") return false;
const opacity = Number.parseFloat(computed.opacity);
const opacity = Number.parseFloat(getLogicalOpacity(el, computed));
return !(
Number.isFinite(opacity) &&
opacity <= 0.01 &&
@@ -101,7 +117,8 @@ export function getCuratedComputedStyles(el: HTMLElement): Record<string, string
if (!computed) return styles;
for (const prop of CURATED_STYLE_PROPERTIES) {
const value = computed.getPropertyValue(prop);
const value =
prop === "opacity" ? getLogicalOpacity(el, computed) : computed.getPropertyValue(prop);
if (value) styles[prop] = value;
}
@@ -23,7 +23,10 @@ export interface NLEContextValue {
play: () => void;
pause: () => void;
togglePlay: () => void;
seek: (time: number, options?: { keepPlaying?: boolean; follow?: boolean }) => boolean;
seek: (
time: number,
options?: { keepPlaying?: boolean; follow?: boolean; settled?: boolean },
) => boolean;
refreshPlayer: () => void;
onIframeLoad: () => void;
// The hidden reload iframe NLEPreview renders next to the live one during a full reload.
@@ -115,6 +115,19 @@ function mountPicker(
};
}
it.each(["0.65", "0"])(
"reads provider-backed video opacity %s in the picked-element inspector",
(opacity) => {
const mounted = `<style>[data-hf-preview-video-frame] { opacity: 0 !important; }</style><canvas data-hf-preview-video-canvas style="opacity:${opacity}"></canvas><video id="provider-video" data-hf-preview-video-frame style="opacity:${opacity}"></video>`;
const { iframe, picker } = mountPicker({ "index.html": SAVED }, "#provider-video", mounted);
expect(
iframe.contentWindow!.getComputedStyle(iframe.contentDocument!.querySelector("video")!)
.opacity,
).toBe("0");
expect(picker().pickedElement?.computedStyles.opacity).toBe(opacity);
},
);
describe("pick mode cancellation and hover feedback", () => {
it("cancels pick mode when Escape is pressed with Studio focus", () => {
const { picker } = mountPicker({ "index.html": SAVED });
@@ -12,7 +12,7 @@ import {
postRuntimeControlMessage,
} from "../player/lib/runtimeProtocol";
import { compositionPathOfPreviewUrl } from "../player/components/CompositionThumbnail";
import { getSourceFileForElement } from "../components/editor/domEditingDom";
import { getLogicalOpacity, getSourceFileForElement } from "../components/editor/domEditingDom";
import {
createElementPickerHoverLabel,
type ElementPickerHoverInfo,
@@ -589,7 +589,8 @@ function readComputedStyles(iframe: HTMLIFrameElement, selector: string): Record
];
for (const prop of props) {
const val = computed.getPropertyValue(prop);
const val =
prop === "opacity" ? getLogicalOpacity(el, computed) : computed.getPropertyValue(prop);
if (val) styles[prop] = val;
}
} catch {
@@ -57,7 +57,7 @@ describe("PlayerControls preview seeking", () => {
setValue.call(input, String(time));
input.dispatchEvent(new Event("input", { bubbles: true }));
});
expect(onSeek).toHaveBeenLastCalledWith(time);
expect(onSeek).toHaveBeenLastCalledWith(time, { settled: true });
}
expect(onSeek).toHaveBeenCalledTimes(4);
});
@@ -161,7 +161,7 @@ const FullscreenButton = memo(function FullscreenButton({
interface PlayerControlsProps {
onTogglePlay: () => void;
onSeek: (time: number) => void;
onSeek: (time: number, options?: { settled?: boolean }) => void;
disabled?: boolean;
isFullscreen?: boolean;
onToggleFullscreen?: () => void;
@@ -0,0 +1,42 @@
// @vitest-environment happy-dom
import React, { act } from "react";
import { expect, it, vi } from "vitest";
import { PreviewSeekBar } from "./PreviewSeekBar";
import { createHappyDomRootHarness } from "./testRootHarness";
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
const { mount } = createHappyDomRootHarness();
it("marks keyboard and released seeks as settled and pointer scrubbing as interactive", () => {
const host = document.createElement("div");
document.body.append(host);
const onSeek = vi.fn();
act(() => mount(host).render(<PreviewSeekBar duration={10} disabled={false} onSeek={onSeek} />));
const input = host.querySelector("input")!;
const change = (value: number) => {
Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(
input,
String(value),
);
input.dispatchEvent(new Event("input", { bubbles: true }));
};
act(() => {
input.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
change(1);
});
expect(onSeek).toHaveBeenLastCalledWith(1, { settled: true });
act(() => {
input.dispatchEvent(new PointerEvent("pointerdown", { bubbles: true }));
change(2);
});
expect(onSeek).toHaveBeenLastCalledWith(2, { settled: false });
act(() => {
input.dispatchEvent(new PointerEvent("pointerup", { bubbles: true }));
});
expect(onSeek).toHaveBeenLastCalledWith(2, { settled: true });
act(() => {
input.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
change(3);
});
expect(onSeek).toHaveBeenLastCalledWith(3, { settled: true });
});
@@ -1,4 +1,4 @@
import { memo, useEffect, useRef } from "react";
import { memo, useEffect, useRef, type PointerEvent } from "react";
import { formatTime } from "../lib/time";
import { liveTime, usePlayerStore } from "../store/playerStore";
@@ -9,9 +9,16 @@ export const PreviewSeekBar = memo(function PreviewSeekBar({
}: {
duration: number;
disabled: boolean;
onSeek: (time: number) => void;
onSeek: (time: number, options?: { settled?: boolean }) => void;
}) {
const inputRef = useRef<HTMLInputElement>(null);
const settledRef = useRef(true);
const finishPointerSeek = (event: PointerEvent<HTMLInputElement>) => {
if (settledRef.current) return;
settledRef.current = true;
onSeek(event.currentTarget.valueAsNumber, { settled: true });
};
useEffect(() => {
const input = inputRef.current;
@@ -39,7 +46,17 @@ export const PreviewSeekBar = memo(function PreviewSeekBar({
defaultValue={0}
disabled={disabled || duration <= 0}
aria-label="Seek preview"
onChange={(event) => onSeek(event.currentTarget.valueAsNumber)}
onPointerDown={() => {
settledRef.current = false;
}}
onPointerUp={finishPointerSeek}
onPointerCancel={finishPointerSeek}
onKeyDown={() => {
settledRef.current = true;
}}
onChange={(event) =>
onSeek(event.currentTarget.valueAsNumber, { settled: settledRef.current })
}
className="h-6 w-full cursor-pointer accent-neutral-300 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-accent-ink disabled:cursor-default disabled:opacity-30"
/>
</div>
@@ -331,7 +331,7 @@ describe("usePlaybackKeyboard — frame step", () => {
dispatch(keydown({ code: "ArrowRight", key: "ArrowRight" }));
});
expect(spies.seek).toHaveBeenCalledWith((frame + 1) / fps);
expect(spies.seek).toHaveBeenCalledWith((frame + 1) / fps, { settled: true });
});
});
@@ -30,7 +30,7 @@ interface UsePlaybackKeyboardParams {
play: () => void;
playBackward: (rate: number) => void;
pause: () => void;
seek: (time: number, options?: { keepPlaying?: boolean }) => void;
seek: (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => void;
}
const FRAME_STEPS = {
@@ -112,6 +112,7 @@ export function usePlaybackKeyboard({
const currentTime = adapter?.getTime() ?? usePlayerStore.getState().currentTime;
seek(
stepFrameTime(currentTime, deltaFrames, useLinkedClipPreferences.getState().compositionFps),
{ settled: true },
);
},
[getAdapter, seek],
@@ -28,6 +28,7 @@ export {
} from "../lib/playbackShortcuts";
import type { PlaybackAdapter, IframeWindow } from "../lib/playbackTypes";
import { attachPausedVideoProvider } from "../lib/pausedVideoProvider";
import { releaseStaticSeekCache, type StaticSeekCacheEntry } from "../lib/playbackAdapter";
import { mergeTimelineElementsPreservingDowngrades } from "../lib/timelineDOM";
import { findTimelineElementInIframe } from "../../hooks/timelineEditingHelpers";
@@ -150,6 +151,7 @@ export function useTimelinePlayer({
const iframe = overrideIframe !== undefined ? overrideIframe : iframeRef.current;
const win = iframe?.contentWindow as IframeWindow | null;
if (!iframe || !win) return null;
attachPausedVideoProvider(iframe, win);
const adapter = resolvePlaybackAdapter(iframe, win, {
cache: staticSeekAdapterRef,
warned: staticSeekWarnedRef,
@@ -295,7 +297,7 @@ export function useTimelinePlayer({
stopRAFLoop();
}, [getAdapter, setCurrentTime, setIsPlaying, stopRAFLoop, stopReverseLoop]);
const seek = useCallback(
(time: number, options?: { keepPlaying?: boolean; follow?: boolean }) => {
(time: number, options?: { keepPlaying?: boolean; follow?: boolean; settled?: boolean }) => {
const wasReverseShuttle = shuttleDirectionRef.current === "backward";
stopReverseLoop();
const adapter = getAdapter();
@@ -611,8 +611,7 @@ describe("mediaUrlSourceOptions", () => {
expect(failure.retries).toBe(0);
});
// mediabunny rethrows a failed read-ahead nobody awaits "so it doesn't get swallowed": that copy is typed too.
it("fails a real mediabunny read, and its unawaited read-ahead, with the typed error", async () => {
it("fails a real mediabunny read with the typed error without a duplicate unhandled rejection", async () => {
vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("", { status: 404 })));
const mb = await import("mediabunny");
const runnerListeners = process.listeners("unhandledRejection");
@@ -629,8 +628,8 @@ describe("mediaUrlSourceOptions", () => {
await new Promise(setImmediate);
expect(error).toMatchObject({ name: "MediaFetchError", status: 404, retries: 0 });
expect(unhandled.length).toBeGreaterThan(0);
for (const reason of unhandled) expect(reason).toBeInstanceOf(MediaFetchError);
expect(error).toBeInstanceOf(MediaFetchError);
expect(unhandled).toEqual([]);
} finally {
process.removeAllListeners("unhandledRejection");
for (const listener of runnerListeners) process.on("unhandledRejection", listener);
@@ -0,0 +1,644 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createPausedVideoDecoder } from "./pausedVideoDecoder";
const inputs: { source: string; dispose: ReturnType<typeof vi.fn> }[] = [];
const getTrack = vi.fn();
const getSample = vi.fn();
const cancelReads = vi.fn();
const iterateSamples = vi.fn();
const finiteSamples = vi.fn();
const canvasContext = vi.fn();
const createBitmap = vi.fn();
let qualityChosenBy = "person";
let previewQuality = 1;
let displayScale = 1;
vi.mock("mediabunny", () => ({
ALL_FORMATS: [],
UrlSource: class {
constructor(readonly url: string) {}
},
Input: class {
dispose = vi.fn();
constructor(readonly options: { source: { url: string } }) {
inputs.push({ source: options.source.url, dispose: this.dispose });
}
getPrimaryVideoTrack() {
return getTrack(this.options.source.url);
}
},
VideoSampleSink: class {
constructor(readonly track: { source: string }) {}
getSample(time: number) {
return getSample(this.track.source, time);
}
samples(start: number, end: number) {
return iterateSamples(this.track.source, start, end);
}
samplesAtTimestamps(times: Iterable<number>) {
return finiteSamples(this.track.source, times);
}
},
}));
vi.mock("./mediaFetch", () => ({ mediaUrlSourceOptions: () => ({ cancelReads }) }));
vi.mock("../store/playerStore", () => ({
usePlayerStore: {
getState: () => ({ previewQuality, previewQualityChosenBy: qualityChosenBy }),
},
}));
vi.mock("@hyperframes/core/runtime/protocol", () => ({ frameDisplayScale: () => displayScale }));
const providers: ReturnType<typeof createPausedVideoDecoder>[] = [];
function provider(
facts?: Record<
string,
{ width: number; height: number; frameRate?: number; variableFrameRate?: boolean }
>,
rootSize?: { width: number; height: number },
) {
const iframe = document.createElement("iframe");
document.body.append(iframe);
if (facts) Object.assign(iframe.contentWindow, { __HF_MEDIA_SOURCE_FACTS__: facts });
if (rootSize)
Object.defineProperties(iframe.contentDocument!.documentElement, {
clientWidth: { value: rootSize.width, configurable: true },
clientHeight: { value: rootSize.height, configurable: true },
});
const result = createPausedVideoDecoder(iframe);
providers.push(result);
return result;
}
const request = (source: string) => ({
source: `http://localhost/${source}.webm`,
time: 0,
video: document.createElement("video"),
});
const signal = () => new AbortController().signal;
const sample = (timestamp = 0) => ({
timestamp,
duration: 1 / 30,
close: vi.fn(),
drawWithFit: vi.fn(),
});
const bitmap = (bytes = 4) => ({ width: bytes / 4, height: 1, close: vi.fn() });
const track = (source: string) => ({
source,
getDisplayWidth: async () => 100,
getDisplayHeight: async () => 100,
getFirstTimestamp: async () => 0,
getTimeResolution: async () => 1000,
async getSquarePixelWidth() {
return this.getDisplayWidth();
},
async getSquarePixelHeight() {
return this.getDisplayHeight();
},
getRotation: async () => 0,
getFlip: async () => false,
canBeTransparent: async () => false,
});
beforeEach(() => {
inputs.length = 0;
getTrack.mockReset().mockImplementation(async (source: string) => track(source));
getSample.mockReset().mockImplementation(async () => sample());
iterateSamples.mockReset();
finiteSamples.mockReset().mockImplementation(async function* (
source: string,
times: Iterable<number>,
) {
for (const time of times) yield await getSample(source, time);
});
canvasContext.mockReset().mockReturnValue({ clearRect: vi.fn(), imageSmoothingQuality: "low" });
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockImplementation(canvasContext);
createBitmap.mockReset().mockImplementation(async () => bitmap());
cancelReads.mockClear();
qualityChosenBy = "person";
previewQuality = 1;
displayScale = 1;
vi.stubGlobal("createImageBitmap", createBitmap);
});
afterEach(async () => {
for (const owner of providers.splice(0)) owner.dispose();
await Promise.resolve();
document.body.innerHTML = "";
vi.useRealTimers();
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
it("rejects a two-source frame set that cannot fit without closing a leased bitmap", async () => {
const first = bitmap(40 * 1024 * 1024),
second = bitmap(40 * 1024 * 1024);
createBitmap.mockResolvedValueOnce(first).mockResolvedValueOnce(second);
await expect(provider().decode([request("a"), request("b")], signal())).rejects.toThrow(
"Preview frame budget reached",
);
expect(first.close).not.toHaveBeenCalled();
expect(second.close).toHaveBeenCalledOnce();
});
it("reserves the reader budget before opening and keeps concurrent same-source leases alive", async () => {
let finishOpen: (() => void) | undefined;
const opened = new Promise<void>((resolve) => {
finishOpen = resolve;
});
getTrack.mockImplementation(async (source: string) => {
await opened;
return track(source);
});
const owner = provider();
const first = owner.decode([request("a"), request("b"), request("c"), request("d")], signal());
const shared = owner.decode([request("a")], signal());
await expect(owner.decode([request("e")], signal())).rejects.toThrow(
"Preview decoder budget reached",
);
expect(inputs).toHaveLength(4);
expect(inputs.every((input) => input.dispose.mock.calls.length === 0)).toBe(true);
finishOpen?.();
const frames = await first,
other = await shared;
expect(frames).toHaveLength(4);
expect(other).toHaveLength(1);
expect(inputs.every((input) => input.dispose.mock.calls.length === 0)).toBe(true);
for (const frame of [...frames, ...other]) frame.release();
});
it("releases successful source leases when another source fails", async () => {
const first = bitmap(40 * 1024 * 1024),
next = bitmap(40 * 1024 * 1024);
createBitmap.mockResolvedValueOnce(first).mockResolvedValueOnce(next);
getSample.mockImplementation(async (source: string) => {
if (source.endsWith("/b.webm")) throw new Error("decoder failed");
return sample();
});
const owner = provider();
await expect(owner.decode([request("a"), request("b")], signal())).rejects.toThrow(
"decoder failed",
);
const [frame] = await owner.decode([request("c")], signal());
expect(first.close).toHaveBeenCalledOnce();
expect(next.close).not.toHaveBeenCalled();
frame!.release();
});
it("retires idle readers even when decoding fails before a frame is returned", async () => {
vi.useFakeTimers();
getSample.mockRejectedValueOnce(new Error("decoder failed"));
await expect(provider().decode([request("a")], signal())).rejects.toThrow("decoder failed");
expect(inputs[0]!.dispose).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(10000);
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
expect(cancelReads).toHaveBeenCalledOnce();
});
it("closes a bitmap completed after provider disposal and rejects later requests without opening", async () => {
let finish: ((image: ReturnType<typeof bitmap>) => void) | undefined;
createBitmap.mockImplementationOnce(
() =>
new Promise((resolve) => {
finish = resolve;
}),
);
const owner = provider(),
pending = owner.decode([request("a")], signal());
await vi.waitFor(() => expect(createBitmap).toHaveBeenCalledOnce());
owner.dispose();
const image = bitmap();
finish?.(image);
await expect(pending).rejects.toThrow("Preview disposed");
expect(image.close).toHaveBeenCalledOnce();
await expect(owner.decode([request("b")], signal())).rejects.toMatchObject({
name: "AbortError",
});
expect(inputs).toHaveLength(1);
});
it("shares full-resolution readers when animation changes only an oversized display box", async () => {
getTrack.mockImplementation(async (source: string) => ({
...track(source),
getDisplayWidth: async () => 1000,
getDisplayHeight: async () => 500,
}));
const owner = provider({ "a.webm": { width: 1000, height: 500 } }),
item = request("a");
const rect = vi.spyOn(item.video, "getBoundingClientRect");
rect.mockReturnValue(new DOMRect(0, 0, 2600, 1800));
const first = await owner.decode([item], signal());
first[0]!.release();
rect.mockReturnValue(new DOMRect(0, 0, 1900, 1300));
const second = await owner.decode([item], signal());
second[0]!.release();
expect(inputs).toHaveLength(1);
});
it("keeps original resolution across hidden cuts, viewport changes and automatic playback quality", async () => {
qualityChosenBy = "app";
getTrack.mockImplementation(async (source: string) => ({
...track(source),
getDisplayWidth: async () => 2100,
getDisplayHeight: async () => 1800,
}));
const owner = provider({ "a.webm": { width: 2100, height: 1800 } });
for (const [width, scale, quality] of [
[2300, 1, 1],
[0, 0.5, 0.25],
[400, 0.25, 0.5],
]) {
displayScale = scale!;
previewQuality = quality!;
const item = request("a");
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, width, 200));
const frames = await owner.decode([item], signal());
frames[0]!.release();
}
expect(inputs).toHaveLength(1);
expect(inputs[0]!.source).toBe("http://localhost/a.webm");
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width: 2100, height: 1800 });
});
const cfrFacts = { "a.webm": { width: 100, height: 100, frameRate: 30, variableFrameRate: false } };
it("closes skipped raw samples and rasterizes only selected frames across a 0.3 second forward gap", async () => {
const one = sample(0.1),
skipA = sample(0.2),
skipB = sample(0.3),
last = sample(0.4);
iterateSamples.mockImplementation(async function* () {
yield one;
yield skipA;
yield skipB;
yield last;
});
const owner = provider(cfrFacts),
item = request("a");
for (const time of [0, 0.1, 0.4]) {
const frames = await owner.decode([{ ...item, time }], signal());
frames[0]!.release();
}
expect(iterateSamples).toHaveBeenCalledOnce();
for (const skipped of [skipA, skipB]) {
expect(skipped.close).toHaveBeenCalledOnce();
expect(skipped.drawWithFit).not.toHaveBeenCalled();
}
for (const selected of [one, last]) {
expect(selected.close).toHaveBeenCalledOnce();
expect(selected.drawWithFit).toHaveBeenCalledWith(expect.anything(), {
fit: "fill",
rotation: 0,
flip: false,
});
}
expect(createBitmap).toHaveBeenCalledTimes(3);
expect(canvasContext).toHaveBeenCalledWith("2d", { alpha: true });
});
it("closes the sequential reader and uses demand decoding beyond the bounded forward gap", async () => {
const returned = vi.fn();
iterateSamples.mockImplementation(async function* () {
try {
yield sample(0.1);
yield sample(0.2);
} finally {
returned();
}
});
getSample.mockImplementation(async (_source: string, time: number) =>
sample(Math.floor(time * 30) / 30),
);
const owner = provider(cfrFacts),
item = request("a");
for (const time of [0, 0.1, 1]) {
const frames = await owner.decode([{ ...item, time }], signal());
frames[0]!.release();
}
expect(returned).toHaveBeenCalledOnce();
expect(getSample).toHaveBeenCalledTimes(2);
expect(getSample).toHaveBeenLastCalledWith(item.source, expect.closeTo(1.00001, 8));
});
it("closes a raw sample arriving after cancellation without drawing it", async () => {
let finish: ((result: { done: false; value: ReturnType<typeof sample> }) => void) | undefined;
const next = vi.fn(
() =>
new Promise((resolve) => {
finish = resolve;
}),
),
returned = vi.fn().mockResolvedValue({ done: true });
iterateSamples.mockReturnValue({ next, return: returned });
const owner = provider(cfrFacts),
item = request("a");
const initial = await owner.decode([item], signal());
initial[0]!.release();
const controller = new AbortController(),
pending = owner.decode([{ ...item, time: 0.1 }], controller.signal);
await vi.waitFor(() => expect(next).toHaveBeenCalledOnce());
controller.abort();
const late = sample(0.1);
finish?.({ done: false, value: late });
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
expect(late.close).toHaveBeenCalledOnce();
expect(late.drawWithFit).not.toHaveBeenCalled();
expect(returned).toHaveBeenCalledOnce();
});
it("closes selected samples when rasterization or bitmap copying fails", async () => {
const drawing = sample();
drawing.drawWithFit.mockImplementationOnce(() => {
throw new Error("drawing failed");
});
getSample.mockResolvedValueOnce(drawing);
const owner = provider();
await expect(owner.decode([request("a")], signal())).rejects.toThrow("drawing failed");
expect(drawing.close).toHaveBeenCalledOnce();
const copying = sample();
getSample.mockResolvedValueOnce(copying);
createBitmap.mockRejectedValueOnce(new Error("copying failed"));
await expect(owner.decode([request("b")], signal())).rejects.toThrow("copying failed");
expect(copying.close).toHaveBeenCalledOnce();
});
it("closes a demand sample completed after provider disposal before rasterization", async () => {
let finish: ((value: ReturnType<typeof sample>) => void) | undefined;
getSample.mockImplementationOnce(
() =>
new Promise((resolve) => {
finish = resolve;
}),
);
const owner = provider(),
pending = owner.decode([request("a")], signal());
await vi.waitFor(() => expect(getSample).toHaveBeenCalledOnce());
owner.dispose();
const late = sample();
finish?.(late);
await expect(pending).rejects.toThrow("Preview disposed");
expect(late.close).toHaveBeenCalledOnce();
expect(late.drawWithFit).not.toHaveBeenCalled();
});
it("keeps original pixels throughout automatic dragging and settling", async () => {
qualityChosenBy = "auto";
getTrack.mockImplementation(async (source: string) => ({
...track(source),
getDisplayWidth: async () => 1000,
getDisplayHeight: async () => 500,
}));
const owner = provider({ "a.webm": { width: 1000, height: 500 } }),
item = request("a");
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 2600, 1800));
const drag = await owner.decode([{ ...item, settled: false }], signal());
drag[0]!.release();
const settled = await owner.decode([{ ...item, settled: true }], signal());
settled[0]!.release();
expect(createBitmap.mock.calls.map(([canvas]) => [canvas.width, canvas.height])).toEqual([
[1000, 500],
]);
});
it("restores full paused resolution after automatic playback quality was lowered", async () => {
qualityChosenBy = "app";
previewQuality = 0.25;
getTrack.mockImplementation(async (source: string) => ({
...track(source),
getDisplayWidth: async () => 1000,
getDisplayHeight: async () => 500,
}));
const owner = provider({ "a.webm": { width: 1000, height: 500 } });
const item = request("a");
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 1000, 500));
const frame = await owner.decode([{ ...item, settled: true }], signal());
frame[0]!.release();
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width: 1000, height: 500 });
});
it("uses original resolution even when a lower playback quality was saved", async () => {
previewQuality = 0.25;
getTrack.mockImplementation(async (source: string) => ({
...track(source),
getDisplayWidth: async () => 1000,
getDisplayHeight: async () => 500,
}));
const owner = provider({ "a.webm": { width: 1000, height: 500 } }),
item = request("a");
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 2600, 1800));
const drag = await owner.decode([{ ...item, settled: false }], signal());
drag[0]!.release();
const settled = await owner.decode([{ ...item, settled: true }], signal());
settled[0]!.release();
expect(createBitmap).toHaveBeenCalledOnce();
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width: 1000, height: 500 });
expect(inputs).toHaveLength(1);
});
it.each([
{ rotation: 0, flip: false, width: 320, height: 200 },
{ rotation: 90, flip: true, width: 200, height: 320 },
])(
"preserves track rotation $rotation, flip and non-square-pixel display geometry",
async ({ rotation, flip, width, height }) => {
getTrack.mockImplementation(async (source: string) => ({
...track(source),
getDisplayWidth: async () => width,
getDisplayHeight: async () => height,
getSquarePixelWidth: async () => 320,
getSquarePixelHeight: async () => 200,
getRotation: async () => rotation,
getFlip: async () => flip,
}));
const decoded = sample();
getSample.mockResolvedValueOnce(decoded);
const [frame] = await provider().decode([request("a")], signal());
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width, height });
expect(decoded.drawWithFit).toHaveBeenCalledWith(expect.anything(), {
fit: "fill",
rotation,
flip,
});
expect(frame).toMatchObject({ width, height });
frame!.release();
},
);
it("releases sequential decoders before native playback and lazily recreates them", async () => {
const returned = vi.fn();
iterateSamples.mockImplementation(async function* () {
try {
yield sample(0.1);
yield sample(0.2);
} finally {
returned();
}
});
getSample.mockImplementation(async (_source: string, time: number) =>
sample(Math.floor(time * 30) / 30),
);
const owner = provider(cfrFacts),
item = request("a");
for (const time of [0, 0.1]) {
const [frame] = await owner.decode([{ ...item, time }], signal());
frame!.release();
}
expect(returned).not.toHaveBeenCalled();
await owner.suspend!();
expect(returned).toHaveBeenCalledOnce();
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
expect(cancelReads).toHaveBeenCalledOnce();
const [next] = await owner.decode([{ ...item, time: 0.2 }], signal());
expect(inputs).toHaveLength(2);
expect([next!.width, next!.height]).toEqual([100, 100]);
next!.release();
});
it("closes a bitmap copied after native-play suspension without retaining its decoder", async () => {
let finish: ((image: ReturnType<typeof bitmap>) => void) | undefined;
createBitmap.mockImplementationOnce(
() =>
new Promise((resolve) => {
finish = resolve;
}),
);
const owner = provider();
const pending = owner.decode([request("a")], signal());
await vi.waitFor(() => expect(createBitmap).toHaveBeenCalledOnce());
await owner.suspend!();
const image = bitmap();
finish!(image);
await expect(pending).rejects.toThrow("Preview disposed");
expect(image.close).toHaveBeenCalledOnce();
const [next] = await owner.decode([request("a")], signal());
expect(inputs).toHaveLength(2);
next!.release();
});
it("does not poison a reopened source when an older suspended input settles late", async () => {
let rejectOpen: (error: Error) => void = () => {};
getTrack.mockImplementationOnce(
() =>
new Promise((_resolve, reject) => {
rejectOpen = reject;
}),
);
const owner = provider();
const old = owner.decode([request("a")], signal());
await vi.waitFor(() => expect(inputs).toHaveLength(1));
const suspended = owner.suspend!();
expect(inputs[0]!.dispose).toHaveBeenCalled();
const [current] = await owner.decode([request("a")], signal());
current!.release();
rejectOpen(new DOMException("Suspended", "AbortError"));
await expect(old).rejects.toMatchObject({ name: "AbortError" });
await suspended;
const [cached] = await owner.decode([request("a")], signal());
expect(inputs).toHaveLength(2);
expect(inputs[1]!.dispose).not.toHaveBeenCalled();
cached!.release();
});
it("uses finite queries for alpha tracks while preserving original dimensions and closing samples", async () => {
getTrack.mockImplementation(async (source: string) => ({
...track(source),
canBeTransparent: async () => true,
}));
const samples = [sample(0), sample(0.1), sample(0.2)];
getSample.mockImplementation(async () => samples[getSample.mock.calls.length - 1]);
iterateSamples.mockImplementation(async function* () {
yield sample(0.1);
yield sample(0.2);
});
const owner = provider(cfrFacts),
item = request("a");
for (const time of [0, 0.1, 0.2]) {
const [frame] = await owner.decode([{ ...item, time }], signal());
expect([frame!.width, frame!.height]).toEqual([100, 100]);
frame!.release();
}
expect(iterateSamples).not.toHaveBeenCalled();
expect(getSample).toHaveBeenCalledTimes(3);
for (const frame of samples) expect(frame.close).toHaveBeenCalledOnce();
await owner.suspend!();
});
it.each(["suspend", "dispose"] as const)(
"drains an in-flight alpha query before %s",
async (action) => {
getTrack.mockImplementation(async (source: string) => ({
...track(source),
canBeTransparent: async () => true,
}));
let finish: (value: ReturnType<typeof sample>) => void = () => {};
getSample.mockImplementationOnce(
() =>
new Promise((resolve) => {
finish = resolve;
}),
);
const owner = provider();
const pending = owner.decode([request("a")], signal());
await vi.waitFor(() => expect(getSample).toHaveBeenCalledOnce());
const rejected = expect(pending).rejects.toThrow("Preview disposed");
const suspended = action === "suspend" ? owner.suspend!() : owner.dispose();
await Promise.resolve();
expect(inputs[0]!.dispose).not.toHaveBeenCalled();
expect(cancelReads).not.toHaveBeenCalled();
if (action === "suspend") {
const [next] = await owner.decode([request("a")], signal());
expect(inputs).toHaveLength(2);
next!.release();
}
const final = sample();
finish(final);
await rejected;
await suspended;
expect(final.close).toHaveBeenCalledOnce();
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
if (action === "suspend") expect(inputs[1]!.dispose).not.toHaveBeenCalled();
},
);
it("finishes the alpha timestamp iterator before publishing its selected sample", async () => {
getTrack.mockImplementation(async (source: string) => ({
...track(source),
canBeTransparent: async () => true,
}));
let drain: () => void = () => {};
const drained = new Promise<void>((resolve) => {
drain = resolve;
});
const selected = sample();
finiteSamples.mockImplementation(async function* () {
yield selected;
await drained;
});
const owner = provider();
let published = false;
const decoding = owner.decode([request("a")], signal()).then((frames) => {
published = true;
return frames;
});
await vi.waitFor(() => expect(finiteSamples).toHaveBeenCalledOnce());
await Promise.resolve();
expect(published).toBe(false);
expect(createBitmap).not.toHaveBeenCalled();
drain();
const [frame] = await decoding;
expect(selected.close).toHaveBeenCalledOnce();
frame!.release();
});
it("closes the selected alpha sample when the finite iterator fails while draining", async () => {
getTrack.mockImplementation(async (source: string) => ({
...track(source),
canBeTransparent: async () => true,
}));
const selected = sample();
finiteSamples.mockImplementation(async function* () {
yield selected;
throw new Error("Alpha drain failed");
});
const owner = provider();
await expect(owner.decode([request("a")], signal())).rejects.toThrow("Alpha drain failed");
expect(selected.close).toHaveBeenCalledOnce();
});
@@ -0,0 +1,428 @@
import {
ALL_FORMATS,
Input,
UrlSource,
VideoSampleSink,
type Rotation,
type VideoSample,
} from "mediabunny";
import type {
PausedVideoFrame,
PausedVideoFrameProvider,
PausedVideoFrameRequest,
} from "@hyperframes/core";
import { mediaUrlSourceOptions } from "./mediaFetch";
import type { IframeWindow } from "./playbackTypes";
import {
previewDecodeTime,
previewPresentationTime,
type PreviewVideoTiming,
} from "./previewVideoTime";
const FRAME_BUDGET = 64 * 1024 * 1024;
const MAX_READERS = 4;
const NEARBY_FORWARD_SECONDS = 0.5;
type Cached = {
image: ImageBitmap;
timestamp: number;
duration: number;
bytes: number;
pins: number;
};
type Reader = {
key: string;
input: Input;
cancelReads(): void;
sink: VideoSampleSink;
canvas: HTMLCanvasElement;
rotation: Rotation;
flip: boolean;
width: number;
height: number;
lead: number;
ticks: number;
transparent: boolean;
tail: Promise<void>;
iterator: AsyncGenerator<VideoSample, void, unknown> | null;
iteratorEnd: number;
lastTime: number;
cache: Map<number, Cached>;
users: number;
disposed: boolean;
idleTimer?: ReturnType<typeof setTimeout>;
};
async function readSingleSample(reader: Reader, query: number): Promise<VideoSample | null> {
if (!reader.transparent) return reader.sink.getSample(query);
let selected: VideoSample | null = null;
try {
for await (const next of reader.sink.samplesAtTimestamps([query])) {
selected?.close();
selected = next;
}
return selected;
} catch (error) {
selected?.close();
throw error;
}
}
export function createPausedVideoDecoder(iframe: HTMLIFrameElement): PausedVideoFrameProvider {
const readers = new Map<string, Promise<Reader>>();
const opened = new Map<string, Reader>();
const opening = new Map<Input, () => void>();
const failed = new Map<string, number>();
const cacheOrder = new Map<Cached, Reader>();
let bytes = 0;
let disposed = false;
const sourceFacts = (source: string) => {
let path = new URL(source, location.href).pathname;
try {
path = decodeURIComponent(path);
} catch {}
return Object.entries(
(iframe.contentWindow as IframeWindow | null)?.__HF_MEDIA_SOURCE_FACTS__ ?? {},
)
.filter(([suffix]) => path.endsWith(suffix))
.sort(([a], [b]) => b.length - a.length)[0]?.[1];
};
const trim = (budget = FRAME_BUDGET) => {
for (const [entry, reader] of cacheOrder) {
if (bytes <= budget) break;
if (entry.pins > 0) continue;
cacheOrder.delete(entry);
reader.cache.delete(entry.timestamp);
bytes -= entry.bytes;
entry.image.close();
}
};
const disposeReader = async (reader: Reader) => {
if (reader.disposed) return;
reader.disposed = true;
clearTimeout(reader.idleTimer);
if (opened.get(reader.key) === reader) opened.delete(reader.key);
// Input disposal must not cut off an in-flight alpha merger.
if (reader.transparent) await reader.tail;
reader.cancelReads();
reader.input.dispose();
await closeIterator(reader).catch(() => {});
for (const entry of reader.cache.values()) {
cacheOrder.delete(entry);
bytes -= entry.bytes;
entry.image.close();
}
reader.cache.clear();
reader.canvas.width = reader.canvas.height = 0;
};
const scheduleIdle = (reader: Reader) => {
clearTimeout(reader.idleTimer);
if (
reader.disposed ||
reader.users > 0 ||
[...reader.cache.values()].some((entry) => entry.pins > 0)
)
return;
reader.idleTimer = setTimeout(() => {
if (reader.users > 0 || [...reader.cache.values()].some((entry) => entry.pins > 0)) return;
if (opened.get(reader.key) !== reader) return;
readers.delete(reader.key);
void disposeReader(reader);
}, 10000);
};
const open = async (source: string): Promise<Reader> => {
const options = mediaUrlSourceOptions("thumbnail");
const input = new Input({ source: new UrlSource(source, options), formats: ALL_FORMATS });
opening.set(input, options.cancelReads);
try {
const track = await input.getPrimaryVideoTrack();
if (!track || disposed) throw new Error("Preview video unavailable");
const [width, height, lead, ticks, squareWidth, squareHeight, rotation, flip, transparent] =
await Promise.all([
track.getDisplayWidth(),
track.getDisplayHeight(),
track.getFirstTimestamp(),
track.getTimeResolution(),
track.getSquarePixelWidth(),
track.getSquarePixelHeight(),
track.getRotation(),
track.getFlip(),
track.canBeTransparent(),
]);
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, Math.ceil(width));
const aspect = rotation % 180 === 0 ? squareWidth / squareHeight : squareHeight / squareWidth;
canvas.height = Math.max(1, Math.round(canvas.width / aspect));
return {
key: source,
input,
cancelReads: options.cancelReads,
sink: new VideoSampleSink(track),
canvas,
rotation,
flip,
width,
height,
lead,
ticks,
transparent,
tail: Promise.resolve(),
iterator: null,
iteratorEnd: 0,
lastTime: -Infinity,
cache: new Map(),
users: 0,
disposed: false,
};
} catch (error) {
input.dispose();
options.cancelReads();
throw error;
} finally {
opening.delete(input);
}
};
const reserveReader = () => {
for (const [oldKey] of readers) {
if (readers.size < MAX_READERS) break;
const old = opened.get(oldKey);
if (old && old.users === 0 && ![...old.cache.values()].some((entry) => entry.pins > 0)) {
readers.delete(oldKey);
void disposeReader(old);
}
}
if (readers.size >= MAX_READERS) throw new Error("Preview decoder budget reached");
};
const readerFor = async (source: string): Promise<Reader> => {
const key = source;
if ((failed.get(key) ?? 0) > performance.now()) throw new Error("Preview decoder cooling down");
let promise = readers.get(key);
if (!promise) {
reserveReader();
promise = open(source);
readers.set(key, promise);
}
readers.delete(key);
readers.set(key, promise);
let reader: Reader;
try {
reader = await promise;
} catch (error) {
if (readers.get(key) === promise) {
readers.delete(key);
failed.set(key, performance.now() + 5000);
}
throw error;
}
if (disposed || reader.disposed) throw new DOMException("Preview disposed", "AbortError");
opened.set(key, reader);
reader.users++;
clearTimeout(reader.idleTimer);
return reader;
};
const closeIterator = async (reader: Reader) => {
const iterator = reader.iterator;
reader.iterator = null;
await iterator?.return();
};
const checkActive = (reader: Reader, signal: AbortSignal) => {
signal.throwIfAborted();
if (disposed || reader.disposed) throw new Error("Preview disposed");
};
const readForward = async (reader: Reader, query: number, signal: AbortSignal) => {
if (!reader.iterator) {
reader.iteratorEnd = query + 8;
reader.iterator = reader.sink.samples(query, reader.iteratorEnd);
}
for (;;) {
checkActive(reader, signal);
const next = await reader.iterator.next();
if (next.done) return null;
const sample = next.value;
let selected = false;
try {
checkActive(reader, signal);
if (sample.timestamp > query + 1e-6) return null;
selected = sample.timestamp + sample.duration > query + 1e-6;
if (selected) return sample;
} finally {
if (!selected) sample.close();
}
}
};
const pin = (reader: Reader, entry: Cached) => {
entry.pins++;
cacheOrder.delete(entry);
cacheOrder.set(entry, reader);
return entry;
};
const cacheFrame = async (
reader: Reader,
sample: VideoSample,
time: number,
timing: PreviewVideoTiming,
signal: AbortSignal,
): Promise<Cached> => {
const context = reader.canvas.getContext("2d", { alpha: true });
if (!context) throw new Error("Preview canvas unavailable");
context.clearRect(0, 0, reader.canvas.width, reader.canvas.height);
context.imageSmoothingQuality = "high";
sample.drawWithFit(context, { fit: "fill", rotation: reader.rotation, flip: reader.flip });
const image = await createImageBitmap(reader.canvas);
if (signal.aborted || disposed || reader.disposed) {
image.close();
signal.throwIfAborted();
throw new Error("Preview disposed");
}
const entry = {
image,
...previewPresentationTime(sample.timestamp, time, timing),
bytes: image.width * image.height * 4,
pins: 1,
};
const previous = reader.cache.get(entry.timestamp);
if (previous) {
previous.duration = Math.max(previous.duration, entry.duration);
image.close();
return pin(reader, previous);
}
trim(FRAME_BUDGET - entry.bytes);
if (bytes + entry.bytes > FRAME_BUDGET) {
image.close();
throw new Error("Preview frame budget reached");
}
reader.cache.set(entry.timestamp, entry);
cacheOrder.set(entry, reader);
bytes += entry.bytes;
return entry;
};
const nearbyForward = (reader: Reader, time: number, timing: PreviewVideoTiming) =>
// Mediabunny 1.65.2 can orphan alpha samples when a range is cancelled.
// Finite timestamp iterators drain the merger before closing it.
!reader.transparent &&
!timing.variableFrameRate &&
(timing.frameRate ?? 0) > 0 &&
time > reader.lastTime + 1e-6 &&
time - reader.lastTime <= NEARBY_FORWARD_SECONDS;
const readFrame = async (
reader: Reader,
time: number,
timing: PreviewVideoTiming,
signal: AbortSignal,
): Promise<Cached> => {
checkActive(reader, signal);
const cached = [...reader.cache.values()].find(
(entry) => time >= entry.timestamp - 1e-6 && time < entry.timestamp + entry.duration - 1e-6,
);
if (cached) return pin(reader, cached);
const query = previewDecodeTime(time, timing),
forward = nearbyForward(reader, time, timing);
let sample: VideoSample | null = null;
try {
if (!forward || query >= reader.iteratorEnd) await closeIterator(reader);
sample = forward ? await readForward(reader, query, signal) : null;
if (!sample) {
await closeIterator(reader);
sample = await readSingleSample(reader, query);
}
if (!sample) throw new Error("No preview frame at requested time");
checkActive(reader, signal);
reader.lastTime = time;
return await cacheFrame(reader, sample, time, timing, signal);
} finally {
sample?.close();
if (signal.aborted || disposed) await closeIterator(reader).catch(() => {});
}
};
const sourceTiming = (request: PausedVideoFrameRequest, reader: Reader) => {
const facts = sourceFacts(request.source);
return {
frameRate: facts?.frameRate,
variableFrameRate: facts?.variableFrameRate,
sourceLead: reader.lead,
inputLead: reader.lead,
timeResolution: reader.ticks,
};
};
const decodeOne = async (
request: PausedVideoFrameRequest,
signal: AbortSignal,
): Promise<PausedVideoFrame> => {
signal.throwIfAborted();
const source = request.source;
if (!source) throw new Error("Preview source unavailable");
const reader = await readerFor(source);
try {
const timing = sourceTiming(request, reader);
const job = reader.tail.then(() => readFrame(reader, request.time, timing, signal));
reader.tail = job.then(
() => {},
() => {},
);
const entry = await job;
let released = false;
trim();
return {
source: entry.image,
timestamp: entry.timestamp,
duration: entry.duration,
width: reader.width,
height: reader.height,
release() {
if (released) return;
released = true;
entry.pins--;
trim();
scheduleIdle(reader);
},
};
} finally {
reader.users--;
scheduleIdle(reader);
}
};
return {
async decode(requests, signal) {
if (disposed) throw new DOMException("Preview disposed", "AbortError");
if (new Set(requests.map(({ source }) => source)).size > MAX_READERS)
throw new Error("Preview decoder budget reached");
const results = await Promise.allSettled(
requests.map((request) => decodeOne(request, signal)),
);
const failure = results.find((result) => result.status === "rejected");
const frames = results.flatMap((result) =>
result.status === "fulfilled" ? [result.value] : [],
);
if (failure || disposed || signal.aborted) {
for (const frame of frames) frame.release();
if (failure) throw failure.reason;
throw new DOMException("Preview cancelled", "AbortError");
}
return frames;
},
async suspend() {
const closing = [...readers.values()].map((promise) =>
promise.then(disposeReader).catch(() => {}),
);
readers.clear();
for (const [input, cancel] of opening) {
cancel();
input.dispose();
}
await Promise.all(closing);
},
dispose() {
if (disposed) return;
disposed = true;
for (const [input, cancel] of opening) {
cancel();
input.dispose();
}
for (const promise of readers.values()) void promise.then(disposeReader).catch(() => {});
readers.clear();
failed.clear();
},
};
}
@@ -0,0 +1,53 @@
// @vitest-environment happy-dom
import { expect, it, vi } from "vitest";
import type { PausedVideoFrameProvider } from "@hyperframes/core";
import { attachPausedVideoProvider } from "./pausedVideoProvider";
const decoder = vi.hoisted(() => ({
decode: vi.fn(async () => []),
suspend: vi.fn(async () => {}),
dispose: vi.fn(),
}));
vi.mock("./pausedVideoDecoder", () => ({ createPausedVideoDecoder: () => decoder }));
it("forwards native-play suspension through the lazy provider facade", async () => {
const iframe = document.createElement("iframe");
document.body.append(iframe);
const install = vi.fn<(provider: PausedVideoFrameProvider) => void>();
const win = iframe.contentWindow!;
Object.assign(win, {
__hf: {
setPausedVideoFrameProvider: install,
},
});
attachPausedVideoProvider(iframe, win);
const provider = install.mock.calls[0]![0];
await provider.decode([], new AbortController().signal);
await provider.suspend?.();
expect(decoder.suspend).toHaveBeenCalledOnce();
await provider.decode([], new AbortController().signal);
expect(decoder.decode).toHaveBeenCalledTimes(2);
win.dispatchEvent(new Event("pagehide"));
await Promise.resolve();
expect(decoder.dispose).toHaveBeenCalledOnce();
iframe.remove();
});
it("allows native-play suspension after the lazy decoder chunk rejects", async () => {
vi.resetModules();
vi.doMock("./pausedVideoDecoder", () => {
throw new Error("decoder chunk unavailable");
});
const { attachPausedVideoProvider: attach } = await import("./pausedVideoProvider");
const iframe = document.createElement("iframe");
document.body.append(iframe);
const install = vi.fn<(provider: PausedVideoFrameProvider) => void>();
const win = iframe.contentWindow!;
Object.assign(win, { __hf: { setPausedVideoFrameProvider: install } });
attach(iframe, win);
const provider = install.mock.calls[0]![0];
await expect(provider.decode([], new AbortController().signal)).rejects.toThrow();
await expect(provider.suspend!()).resolves.toBeUndefined();
win.dispatchEvent(new Event("pagehide"));
iframe.remove();
});
@@ -0,0 +1,54 @@
import type { PausedVideoFrameProvider } from "@hyperframes/core";
import type { IframeWindow } from "./playbackTypes";
const installed = new WeakMap<
HTMLIFrameElement,
{
document: Document;
provider: PausedVideoFrameProvider;
setter: (provider: PausedVideoFrameProvider | null) => void;
}
>();
/** A lazy facade also installs in Studio's hidden reload frame before its first seek. */
export function attachPausedVideoProvider(iframe: HTMLIFrameElement, win: IframeWindow): void {
if (!win.__hf?.setPausedVideoFrameProvider || !iframe.contentDocument) return;
const previous = installed.get(iframe);
const setter = win.__hf.setPausedVideoFrameProvider;
if (previous?.document === iframe.contentDocument && previous.setter === setter) return;
previous?.provider.dispose();
let disposed = false;
let decoder: Promise<PausedVideoFrameProvider> | null = null;
const provider: PausedVideoFrameProvider = {
async decode(requests, signal) {
signal.throwIfAborted();
if (disposed) throw new DOMException("Preview disposed", "AbortError");
decoder ??= import("./pausedVideoDecoder").then(({ createPausedVideoDecoder }) =>
createPausedVideoDecoder(iframe),
);
const actual = await decoder;
if (disposed) {
actual.dispose();
throw new DOMException("Preview disposed", "AbortError");
}
return actual.decode(requests, signal);
},
async suspend() {
const actual = await decoder?.catch(() => null);
await actual?.suspend?.();
},
dispose() {
if (disposed) return;
disposed = true;
void decoder?.then((actual) => actual.dispose()).catch(() => {});
win.removeEventListener("pagehide", onPageHide);
},
};
const onPageHide = () => {
provider.dispose();
installed.delete(iframe);
};
win.addEventListener("pagehide", onPageHide, { once: true });
installed.set(iframe, { document: iframe.contentDocument, provider, setter });
setter(provider);
}
@@ -4,12 +4,16 @@
* from here without creating circular dependencies.
*/
import type { RuntimeTimelineClipIdentity } from "@hyperframes/core";
import type {
RuntimeTimelineClipIdentity,
PausedVideoFrameProvider,
PreviewSourceFacts,
} from "@hyperframes/core";
export interface PlaybackAdapter {
play: () => void;
pause: () => void;
seek: (time: number, options?: { keepPlaying?: boolean }) => void;
seek: (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => void;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
@@ -58,6 +62,7 @@ export interface ClipManifest {
}
export type IframeWindow = Window & {
__HF_MEDIA_SOURCE_FACTS__?: Record<string, PreviewSourceFacts>;
__player?: RuntimePlaybackAdapter;
__playerReady?: boolean;
__timeline?: TimelineLike;
@@ -70,6 +75,7 @@ export type IframeWindow = Window & {
__hf?: {
leasePausedMedia?: (el: HTMLMediaElement) => void;
releasePausedMedia?: (el: HTMLMediaElement) => void;
setPausedVideoFrameProvider?: (provider: PausedVideoFrameProvider | null) => void;
setLoopStart?: (seconds: number | null) => void;
};
};
@@ -0,0 +1,38 @@
import { expect, it } from "vitest";
import { previewDecodeTime, previewPresentationTime } from "./previewVideoTime";
const cfr = {
frameRate: 30,
variableFrameRate: false,
sourceLead: 0,
inputLead: 0,
timeResolution: 1000,
};
it("matches export at rounded WebM ticks without selecting the nearest future frame", () => {
expect(previewDecodeTime(2 / 30, cfr)).toBeCloseTo(0.06701, 8);
expect(previewPresentationTime(0.067, 2 / 30, cfr).timestamp).toBeCloseTo(2 / 30, 8);
expect(previewDecodeTime(3.33, cfr)).toBeCloseTo(3.30001, 8);
expect(previewDecodeTime(0.0666, cfr)).toBeCloseTo(0.03301, 8);
});
it("preserves a nonzero source lead when a prepared file starts at zero", () => {
const timing = { ...cfr, sourceLead: 5, inputLead: 0 };
expect(previewDecodeTime(5 + 2 / 30, timing)).toBeCloseTo(0.06701, 8);
expect(previewPresentationTime(0.067, 5 + 2 / 30, timing).timestamp).toBeCloseTo(5 + 2 / 30, 8);
});
it("retains raw VFR times and holds a prior frame through a timestamp gap", () => {
const timing = { ...cfr, variableFrameRate: true };
expect(previewDecodeTime(0.1, timing)).toBeCloseTo(0.1, 8);
const held = previewPresentationTime(0.04, 0.1, timing);
expect(held.timestamp).toBe(0.04);
expect(held.timestamp + held.duration).toBeGreaterThan(0.1);
expect(held.timestamp + held.duration).toBeLessThan(0.101);
});
it("does not advance to the next CFR frame just before its boundary", () => {
expect(previewDecodeTime(2 / 30 - 0.000005, cfr)).toBeCloseTo(0.03301, 8);
const timing = { ...cfr, frameRate: 30000 / 1001, sourceLead: 5 };
expect(previewDecodeTime(5 + (2 * 1001) / 30000 - 0.000005, timing)).toBeCloseTo(0.03301, 8);
});
it("does not cross a real VFR timestamp when querying just before it", () => {
const timing = { ...cfr, variableFrameRate: true, sourceLead: 5, inputLead: 2 };
expect(previewDecodeTime(5.1 - 0.000005, timing)).toBeLessThan(2.1);
});
@@ -0,0 +1,39 @@
export interface PreviewVideoTiming {
frameRate?: number;
variableFrameRate?: boolean;
sourceLead: number;
inputLead: number;
timeResolution: number;
}
const LEAD = 1e-5;
const FRAME_EPSILON = 1e-9;
const constantRate = (timing: PreviewVideoTiming) =>
!timing.variableFrameRate && (timing.frameRate ?? 0) > 0;
/** Match export: floor the source-frame grid, then undo container tick rounding. */
export function previewDecodeTime(time: number, timing: PreviewVideoTiming): number {
const { sourceLead, inputLead, timeResolution, frameRate = 0 } = timing;
if (!constantRate(timing))
return Math.max(inputLead, time - sourceLead + inputLead) + FRAME_EPSILON;
const index = Math.max(0, Math.floor((time - sourceLead) * frameRate + FRAME_EPSILON));
const ideal = inputLead + index / frameRate;
return (timeResolution > 0 ? Math.round(ideal * timeResolution) / timeResolution : ideal) + LEAD;
}
export function previewPresentationTime(
timestamp: number,
requested: number,
timing: PreviewVideoTiming,
) {
const { frameRate = 0, sourceLead, inputLead } = timing;
const logical = constantRate(timing)
? Math.round((timestamp - inputLead) * frameRate) / frameRate + sourceLead
: timestamp - inputLead + sourceLead;
const start = Math.min(requested, logical);
// A decoded prior frame proves its coverage through this query, including a VFR gap.
return {
timestamp: start,
duration: Math.max(constantRate(timing) ? 1 / frameRate : 0, requested - start + LEAD),
};
}
@@ -4,10 +4,21 @@ import {
PREVIEW_RASTER_ATTR,
RUNTIME_STAMPED_ATTR,
STUDIO_PREVIEW_LAZY_ATTR,
PREVIEW_VIDEO_CANVAS_ATTR,
PREVIEW_VIDEO_FRAME_ATTR,
PREVIEW_VIDEO_SOURCE_ATTR,
} from "@hyperframes/core/studio-preview-mark";
import { liveMarkupWithoutPreviewMarks } from "./authoredSource";
describe("liveMarkupWithoutPreviewMarks", () => {
it("omits generated video surfaces and their hiding marks from copied composition markup", () => {
const live = document.createElement("div");
live.innerHTML = `<canvas ${PREVIEW_VIDEO_CANVAS_ATTR}></canvas><video src="original.webm" ${PREVIEW_VIDEO_FRAME_ATTR} ${PREVIEW_VIDEO_SOURCE_ATTR}></video><canvas id="authored"></canvas>`;
expect(liveMarkupWithoutPreviewMarks(live)).toBe(
'<div><video src="original.webm"></video><canvas id="authored"></canvas></div>',
);
expect(live.querySelectorAll("canvas")).toHaveLength(2);
});
it("drops the preview's raster mark from an element and its descendants", () => {
const live = document.createElement("div");
live.innerHTML = `<div class="line" ${PREVIEW_RASTER_ATTR}><span class="word">LOUD</span></div>`;
@@ -1,6 +1,7 @@
import { ensureHfIds, isCompositionTemplate } from "@hyperframes/parsers/hf-ids";
import {
PREVIEW_ONLY_ATTRS,
PREVIEW_VIDEO_CANVAS_ATTR,
STUDIO_PREVIEW_LAZY_ATTR,
} from "@hyperframes/core/studio-preview-mark";
@@ -97,6 +98,7 @@ export function authoredMarkup(authored: Element, live: Element, sourceFile: str
export function liveMarkupWithoutPreviewMarks(live: Element): string {
const copy = live.cloneNode(true) as Element;
for (const canvas of copy.querySelectorAll(`[${PREVIEW_VIDEO_CANVAS_ATTR}]`)) canvas.remove();
for (const el of [copy, ...Array.from(copy.querySelectorAll("*"))]) {
if (el.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) el.removeAttribute("loading");
for (const attr of PREVIEW_ONLY_ATTRS) el.removeAttribute(attr);
+1 -1
View File
@@ -36,7 +36,7 @@
"unpacked": 541601
},
"@hyperframes/producer": {
"packed": 857000,
"packed": 870000,
"unpacked": 3130000
},
"@hyperframes/sdk": {