mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
feat(studio): scrub full-quality video with decoded preview frames (#7379)
* feat(studio): scrub full-quality video with decoded preview frames * fix(studio): release scrub decoders before native playback * fix(studio): bound paused fallback and tolerate decoder load failure * fix(studio): recover stalled native preview handoffs * fix(runtime): recognize cross-realm preview video elements * fix(studio): bound Play handoffs and drain transparent preview samples * fix(preview): recover retained video without chasing playback clock * fix(studio): drain finite alpha queries before disposing readers
This commit is contained in:
@@ -389,7 +389,7 @@
|
||||
"cn": "0.2.4",
|
||||
"dockview-react": "8.3.1",
|
||||
"gsap": "^3.13.0",
|
||||
"mediabunny": "^1.45.3",
|
||||
"mediabunny": "^1.65.2",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@hyperframes/producer": "workspace:*",
|
||||
@@ -1856,7 +1856,7 @@
|
||||
|
||||
"mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="],
|
||||
|
||||
"mediabunny": ["mediabunny@1.45.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-GUCPYjR+5olLM7DRmupCXCmZkkSrKHVl1gyW2RztpObqLfrix19kWGf/9WgWzDW0g49DvfGXpl+zfArCx5HDMQ=="],
|
||||
"mediabunny": ["mediabunny@1.65.2", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-goZpibvo/5hkc/56WVNm93Ini+o7K4y8aXA20c/BN1B+m3L5aHQa8MTvO+S1APg/YI8kMILtrX6QCqUtxLW7bA=="],
|
||||
|
||||
"meow": ["meow@13.2.0", "", {}, "sha512-pxQJQzB6djGPXh08dacEloMFopsOqGVRKFPYvPOt9XDZ1HasbgDZA74CJGreSU4G3Ak7EFJGoiH2auq+yXISgA=="],
|
||||
|
||||
|
||||
@@ -106,6 +106,31 @@ depends on the browser playing in real time, while render captures frames one at
|
||||
time. A heavy project can therefore stutter during preview and still render every
|
||||
frame.
|
||||
|
||||
Paused video scrubbing decodes and stages the visible video layers before
|
||||
advancing the composition's animations. Completed frames appear during a drag,
|
||||
with newer pointer positions coalesced into the next decode. Canvas and GPU
|
||||
effects consume the same staged pixels. Continuous playback uses native video;
|
||||
Studio keeps the paused picture visible while those videos catch up before Play.
|
||||
A native handoff waits for ready frames for up to ten seconds; Pause or a new
|
||||
seek cancels it. At the deadline, the rest plays while each unready video keeps
|
||||
its last full-quality picture. During playback, native frame presentation or
|
||||
playback progress releases that picture; normal transport sync corrects drift.
|
||||
Paused recovery still waits for its requested frame. A skipped source keeps its
|
||||
picture on subsequent Play requests until it recovers.
|
||||
A paused fallback that outlasts its wait retains the presented clock and picture,
|
||||
while paused sync continues. It can still publish when the video becomes ready;
|
||||
a new gesture cancels that recovery. Studio requests paused-decoder release
|
||||
before preparing native playback and reopens readers when paused scrubbing resumes.
|
||||
Transparent video queries drain their alpha merges before input disposal. If one
|
||||
never settles, its cleanup can outlast the Play deadline; the remaining playback
|
||||
still starts while that video's last picture stays visible.
|
||||
Scrubbing and paused frame steps decode the original video at source resolution
|
||||
throughout. Automatic playback quality reductions do not reduce these frames,
|
||||
and the frame decoder does not substitute re-encoded seek copies. Playback
|
||||
quality settings do not reduce these paused frames. Browsers or
|
||||
media that cannot use the frame decoder fall back to native seeking. Long-GOP
|
||||
sources can take longer to seek, especially backward or across distant cuts.
|
||||
|
||||
## Tailwind preset
|
||||
|
||||
```typescript
|
||||
|
||||
@@ -469,6 +469,14 @@ describe("insertRuntimeTag", () => {
|
||||
expect(out.indexOf("window.author")).toBeLessThan(out.indexOf(runtime));
|
||||
});
|
||||
|
||||
it("keeps the tracker and runtime ahead of author scripts for Studio previews", () => {
|
||||
const out = insertRuntimeTag(page('data-motion-blur="on"'), runtime, true);
|
||||
const tracker = out.indexOf("data-hf-accel-tracker");
|
||||
expect(tracker).toBeGreaterThan(-1);
|
||||
expect(tracker).toBeLessThan(out.indexOf(runtime));
|
||||
expect(out.indexOf(runtime)).toBeLessThan(out.indexOf("window.author"));
|
||||
});
|
||||
|
||||
it("adds the tracker once, however many times the runtime is placed", () => {
|
||||
const once = insertRuntimeTag(page('data-motion-blur="on"'), runtime);
|
||||
const twice = insertRuntimeTag(once, runtime);
|
||||
|
||||
@@ -393,8 +393,8 @@ function insertTagInHead(html: string, tag: string): string {
|
||||
const ACCEL_TRACKER_ATTR = "data-hf-accel-tracker";
|
||||
const OPTS_INTO_MOTION_BLUR = /\sdata-motion-blur\s*=\s*["']?on[\s"'>]/i;
|
||||
|
||||
export function insertRuntimeTag(html: string, tag: string): string {
|
||||
const withRuntime = insertTagInHead(html, tag);
|
||||
export function insertRuntimeTag(html: string, tag: string, atHeadStart = false): string {
|
||||
const withRuntime = atHeadStart ? injectTagsAtHeadStart(html, tag) : insertTagInHead(html, tag);
|
||||
if (!OPTS_INTO_MOTION_BLUR.test(html) || html.includes(ACCEL_TRACKER_ATTR)) return withRuntime;
|
||||
const tracker = `<script ${ACCEL_TRACKER_ATTR}>${acceleratedCanvasTrackerSource()}</script>`;
|
||||
return injectTagsAtHeadStart(withRuntime, tracker);
|
||||
|
||||
@@ -308,6 +308,11 @@ export {
|
||||
type PreviewQuality,
|
||||
type PreviewSourceFacts,
|
||||
} from "./previewProxyBox";
|
||||
export type {
|
||||
PausedVideoFrameProvider,
|
||||
PausedVideoFrameRequest,
|
||||
PausedVideoFrame,
|
||||
} from "./runtime/pausedVideoFrames";
|
||||
|
||||
// Inline scripts
|
||||
export {
|
||||
|
||||
@@ -55,6 +55,8 @@ export type PreviewSourceFacts = {
|
||||
bitDepth: number;
|
||||
bitrate: number;
|
||||
codec?: string;
|
||||
frameRate?: number;
|
||||
variableFrameRate?: boolean;
|
||||
};
|
||||
|
||||
export const HEAVY_PIXEL_SHARE = 0.5;
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createPausedVideoFrames, getPausedVideoFrame } from "../pausedVideoFrames";
|
||||
import {
|
||||
patchCanvasVideoFrameCompat,
|
||||
patchExternalVideoTextureCompat,
|
||||
patchVideoTextureCompat,
|
||||
patchWebGLVideoTextureCompat,
|
||||
} from "./video-texture-compat";
|
||||
|
||||
const sessions: ReturnType<typeof createPausedVideoFrames>[] = [];
|
||||
afterEach(() => {
|
||||
for (const session of sessions.splice(0)) session.destroy();
|
||||
document.body.innerHTML = "";
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function surfaces() {
|
||||
const draw = vi.fn();
|
||||
const offscreenDraw = vi.fn();
|
||||
const image = vi.fn();
|
||||
const subImage = vi.fn();
|
||||
const copy = vi.fn();
|
||||
const imported = vi.fn((_descriptor: unknown) => ({ texture: true }));
|
||||
const constructed = vi.fn();
|
||||
class Context2D {
|
||||
drawImage(...args: unknown[]) {
|
||||
draw.apply(this, args);
|
||||
}
|
||||
}
|
||||
class Offscreen2D {
|
||||
drawImage(...args: unknown[]) {
|
||||
offscreenDraw.apply(this, args);
|
||||
}
|
||||
}
|
||||
class GL {
|
||||
texImage2D(...args: unknown[]) {
|
||||
image.apply(this, args);
|
||||
}
|
||||
texSubImage2D(...args: unknown[]) {
|
||||
subImage.apply(this, args);
|
||||
}
|
||||
}
|
||||
class GL2 extends GL {}
|
||||
class Queue {
|
||||
copyExternalImageToTexture(...args: unknown[]) {
|
||||
copy.apply(this, args);
|
||||
}
|
||||
}
|
||||
class Device {
|
||||
importExternalTexture(descriptor: unknown) {
|
||||
return imported.call(this, descriptor);
|
||||
}
|
||||
}
|
||||
class Frame {
|
||||
close = vi.fn();
|
||||
constructor(source: unknown, options: unknown) {
|
||||
constructed(source, options);
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("CanvasRenderingContext2D", Context2D);
|
||||
vi.stubGlobal("OffscreenCanvasRenderingContext2D", Offscreen2D);
|
||||
vi.stubGlobal("WebGLRenderingContext", GL);
|
||||
vi.stubGlobal("WebGL2RenderingContext", GL2);
|
||||
vi.stubGlobal("GPUQueue", Queue);
|
||||
vi.stubGlobal("GPUDevice", Device);
|
||||
vi.stubGlobal("VideoFrame", Frame);
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
clearRect: vi.fn(),
|
||||
drawImage: vi.fn(),
|
||||
} as unknown as CanvasRenderingContext2D);
|
||||
return {
|
||||
Context2D,
|
||||
Offscreen2D,
|
||||
GL,
|
||||
GL2,
|
||||
Queue,
|
||||
Device,
|
||||
draw,
|
||||
offscreenDraw,
|
||||
image,
|
||||
subImage,
|
||||
copy,
|
||||
imported,
|
||||
constructed,
|
||||
};
|
||||
}
|
||||
|
||||
async function presented() {
|
||||
const session = createPausedVideoFrames();
|
||||
sessions.push(session);
|
||||
const video = document.createElement("video");
|
||||
video.src = "/original.mp4";
|
||||
document.body.append(video);
|
||||
Object.defineProperties(video, {
|
||||
videoWidth: { value: 1920 },
|
||||
videoHeight: { value: 1080 },
|
||||
});
|
||||
const source = document.createElement("canvas");
|
||||
source.width = 320;
|
||||
source.height = 180;
|
||||
session.setProvider({
|
||||
decode: async (requests) =>
|
||||
requests.map(({ time }) => ({
|
||||
source,
|
||||
timestamp: time,
|
||||
duration: 1 / 30,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
release: vi.fn(),
|
||||
})),
|
||||
dispose: vi.fn(),
|
||||
});
|
||||
const show = async (time: number) => {
|
||||
const batch = await session.prepare(
|
||||
[{ video, source: video.src, time }],
|
||||
new AbortController().signal,
|
||||
);
|
||||
expect(batch?.present()).toBe(true);
|
||||
batch!.refresh();
|
||||
};
|
||||
await show(3);
|
||||
const canvas = getPausedVideoFrame(video)!;
|
||||
expect([canvas.width, canvas.height]).toEqual([video.videoWidth, video.videoHeight]);
|
||||
return { session, video, canvas, show };
|
||||
}
|
||||
|
||||
it.each([
|
||||
["three arguments", [11, 17]],
|
||||
["five arguments", [11, 17, 640, 360]],
|
||||
["nine arguments", [120, 60, 960, 540, 11, 17, 640, 360]],
|
||||
])("preserves %s and source coordinates for both canvas surfaces", async (_name, coordinates) => {
|
||||
const { Context2D, Offscreen2D, draw, offscreenDraw } = surfaces();
|
||||
const { session, video, canvas } = await presented();
|
||||
patchCanvasVideoFrameCompat();
|
||||
patchCanvasVideoFrameCompat();
|
||||
const context = new Context2D();
|
||||
const offscreen = new Offscreen2D();
|
||||
context.drawImage(video, ...coordinates);
|
||||
offscreen.drawImage(video, ...coordinates);
|
||||
expect(draw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates);
|
||||
expect(offscreenDraw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates);
|
||||
expect(draw.mock.contexts[0]).toBe(context);
|
||||
expect(offscreenDraw.mock.contexts[0]).toBe(offscreen);
|
||||
session.clear();
|
||||
context.drawImage(video, ...coordinates);
|
||||
expect(draw).toHaveBeenLastCalledWith(video, ...coordinates);
|
||||
const image = document.createElement("img");
|
||||
offscreen.drawImage(image, ...coordinates);
|
||||
expect(offscreenDraw).toHaveBeenLastCalledWith(image, ...coordinates);
|
||||
});
|
||||
|
||||
it("uploads paused pixels through both WebGL versions without changing crop or upload parameters", async () => {
|
||||
const { GL, GL2, image, subImage } = surfaces();
|
||||
const { session, video, canvas } = await presented();
|
||||
patchWebGLVideoTextureCompat();
|
||||
for (const Context of [GL, GL2]) {
|
||||
const gl = new Context();
|
||||
const imageArgs = [0x0de1, 2, 0x1908, 0x1908, 0x1401];
|
||||
const subArgs = [0x0de1, 2, 13, 29, 0x1908, 0x1401];
|
||||
gl.texImage2D(...imageArgs, video);
|
||||
gl.texSubImage2D(...subArgs, video);
|
||||
expect(image).toHaveBeenLastCalledWith(...imageArgs, canvas);
|
||||
expect(subImage).toHaveBeenLastCalledWith(...subArgs, canvas);
|
||||
expect(image.mock.contexts.at(-1)).toBe(gl);
|
||||
const pixels = new Uint8Array([10, 20, 30, 255]);
|
||||
gl.texImage2D(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels);
|
||||
expect(image).toHaveBeenLastCalledWith(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels);
|
||||
}
|
||||
session.clear();
|
||||
new GL().texImage2D(0x0de1, 0, 0x1908, 0x1908, 0x1401, video);
|
||||
expect(image.mock.lastCall?.at(-1)).toBe(video);
|
||||
});
|
||||
|
||||
it("preserves WebGPU crop, orientation, destination and full intrinsic copy extent", async () => {
|
||||
const { Queue, copy } = surfaces();
|
||||
const { session, video, canvas } = await presented();
|
||||
patchVideoTextureCompat();
|
||||
const queue = new Queue();
|
||||
const source = { source: video, origin: { x: 37, y: 41 }, flipY: true };
|
||||
const destination = { texture: {}, origin: [5, 7], mipLevel: 1, premultipliedAlpha: true };
|
||||
const extent = {
|
||||
width: video.videoWidth - 37,
|
||||
height: video.videoHeight - 41,
|
||||
depthOrArrayLayers: 1,
|
||||
};
|
||||
queue.copyExternalImageToTexture(source, destination, extent);
|
||||
expect(copy).toHaveBeenCalledExactlyOnceWith({ ...source, source: canvas }, destination, extent);
|
||||
expect(copy.mock.contexts[0]).toBe(queue);
|
||||
expect(source.source).toBe(video);
|
||||
expect(copy.mock.calls[0]?.[1]).toBe(destination);
|
||||
expect(copy.mock.calls[0]?.[2]).toBe(extent);
|
||||
session.clear();
|
||||
queue.copyExternalImageToTexture(source, destination, extent);
|
||||
expect(copy).toHaveBeenLastCalledWith(source, destination, extent);
|
||||
expect(copy.mock.lastCall?.[0]).toBe(source);
|
||||
});
|
||||
|
||||
it("reuses an external VideoFrame until presentation changes, then closes it on replacement and clear", async () => {
|
||||
const { Device, imported, constructed } = surfaces();
|
||||
const { session, video, canvas, show } = await presented();
|
||||
patchExternalVideoTextureCompat();
|
||||
const once = Device.prototype.importExternalTexture;
|
||||
patchExternalVideoTextureCompat();
|
||||
expect(Device.prototype.importExternalTexture).toBe(once);
|
||||
const device = new Device();
|
||||
const descriptor = { source: video, colorSpace: "srgb", label: "authored texture" };
|
||||
const texture = device.importExternalTexture(descriptor);
|
||||
const first = imported.mock.lastCall?.[0] as {
|
||||
source: { close: ReturnType<typeof vi.fn> };
|
||||
colorSpace: string;
|
||||
label: string;
|
||||
};
|
||||
expect(texture).toEqual({ texture: true });
|
||||
expect(constructed).toHaveBeenCalledExactlyOnceWith(canvas, { timestamp: 3_000_000 });
|
||||
expect(first).toEqual({ ...descriptor, source: first.source });
|
||||
expect(first.source).not.toBe(video);
|
||||
expect(imported.mock.contexts[0]).toBe(device);
|
||||
expect(descriptor.source).toBe(video);
|
||||
device.importExternalTexture(descriptor);
|
||||
expect(imported.mock.lastCall?.[0]).toEqual(first);
|
||||
expect(constructed).toHaveBeenCalledTimes(1);
|
||||
expect(first.source.close).not.toHaveBeenCalled();
|
||||
await show(4);
|
||||
expect(first.source.close).toHaveBeenCalledTimes(1);
|
||||
device.importExternalTexture(descriptor);
|
||||
const second = imported.mock.lastCall?.[0] as typeof first;
|
||||
expect(second.source).not.toBe(first.source);
|
||||
expect(constructed).toHaveBeenLastCalledWith(getPausedVideoFrame(video), {
|
||||
timestamp: 4_000_000,
|
||||
});
|
||||
session.clear();
|
||||
expect(second.source.close).toHaveBeenCalledTimes(1);
|
||||
device.importExternalTexture(descriptor);
|
||||
expect(imported).toHaveBeenLastCalledWith(descriptor);
|
||||
expect(imported.mock.lastCall?.[0]).toBe(descriptor);
|
||||
expect(constructed).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
import { findInjectedRenderFrame } from "../renderFrameSibling.js";
|
||||
import { isImageElement, isVideoElement } from "../domRealm";
|
||||
import { getPausedExternalVideoFrame, getPausedVideoFrame } from "../pausedVideoFrames";
|
||||
|
||||
/**
|
||||
* Resolve the decoded render-frame `<img>` for a source `<video>`, if the
|
||||
@@ -26,7 +27,9 @@ import { isImageElement, isVideoElement } from "../domRealm";
|
||||
* first (cheap) and fall back to an id lookup in case a node was inserted
|
||||
* between them.
|
||||
*/
|
||||
function resolveRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null {
|
||||
function resolveRenderFrameImage(
|
||||
video: HTMLVideoElement,
|
||||
): HTMLImageElement | HTMLCanvasElement | null {
|
||||
const sibling = video.nextElementSibling;
|
||||
if (
|
||||
isImageElement(sibling) &&
|
||||
@@ -40,7 +43,48 @@ function resolveRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | nu
|
||||
if (byId && byId.complete && byId.naturalWidth > 0) {
|
||||
return byId;
|
||||
}
|
||||
return null;
|
||||
return getPausedVideoFrame(video);
|
||||
}
|
||||
|
||||
export function patchCanvasVideoFrameCompat(): void {
|
||||
for (const ctor of [
|
||||
globalThis.CanvasRenderingContext2D,
|
||||
globalThis.OffscreenCanvasRenderingContext2D,
|
||||
]) {
|
||||
if (!ctor?.prototype?.drawImage) continue;
|
||||
const original = ctor.prototype.drawImage as ((...args: unknown[]) => void) & {
|
||||
__hfVideoPatched?: boolean;
|
||||
};
|
||||
if (original.__hfVideoPatched) continue;
|
||||
const patched = function (this: unknown, ...args: unknown[]) {
|
||||
if (isVideoElement(args[0])) args[0] = getPausedVideoFrame(args[0]) ?? args[0];
|
||||
original.apply(this, args);
|
||||
};
|
||||
patched.__hfVideoPatched = true;
|
||||
ctor.prototype.drawImage = patched;
|
||||
}
|
||||
}
|
||||
|
||||
export function patchExternalVideoTextureCompat(): void {
|
||||
const ctor = (globalThis as Record<string, unknown>).GPUDevice as
|
||||
| {
|
||||
prototype: {
|
||||
importExternalTexture?: ((source: Record<string, unknown>) => unknown) & {
|
||||
__hfVideoPatched?: boolean;
|
||||
};
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
const original = ctor?.prototype.importExternalTexture;
|
||||
if (!ctor || typeof original !== "function" || original.__hfVideoPatched) return;
|
||||
const patched = function (this: unknown, descriptor: Record<string, unknown>) {
|
||||
const frame = isVideoElement(descriptor.source)
|
||||
? getPausedExternalVideoFrame(descriptor.source)
|
||||
: null;
|
||||
return original.call(this, frame ? { ...descriptor, source: frame } : descriptor);
|
||||
};
|
||||
patched.__hfVideoPatched = true;
|
||||
ctor.prototype.importExternalTexture = patched;
|
||||
}
|
||||
|
||||
export function patchVideoTextureCompat(): void {
|
||||
|
||||
@@ -49,6 +49,7 @@ import { mediaElementTime } from "./mediaClock";
|
||||
import { swallow } from "./diagnostics";
|
||||
import { createRecordFilter } from "./mutationRecords";
|
||||
import { findInjectedRenderFrame } from "./renderFrameSibling";
|
||||
import { getPausedVideoFrame } from "./pausedVideoFrames";
|
||||
import {
|
||||
isCanvasElement,
|
||||
isElementNode,
|
||||
@@ -2807,6 +2808,8 @@ function getDrawableSource(element: ColorGradingMediaElement): TexImageSource |
|
||||
if (isVideoElement(element)) {
|
||||
const renderFrame = findRenderFrameImage(element);
|
||||
if (renderFrame) return renderFrame;
|
||||
const previewFrame = getPausedVideoFrame(element);
|
||||
if (previewFrame) return previewFrame;
|
||||
}
|
||||
return isDrawableSource(element) ? element : null;
|
||||
}
|
||||
@@ -3542,7 +3545,7 @@ function placeCanvas(
|
||||
layout: { width: number; height: number };
|
||||
uv: ReturnType<typeof calculateObjectFitUv>;
|
||||
} | null {
|
||||
const styleSource = isHtmlElement(source) ? source : entry.element;
|
||||
const styleSource = isRenderFrameImage(source) ? source : entry.element;
|
||||
const layout = updateCanvasLayout(entry, styleSource);
|
||||
if (!layout) return null;
|
||||
const style = window.getComputedStyle(styleSource);
|
||||
|
||||
@@ -329,6 +329,8 @@ describe("runtime entry", () => {
|
||||
let ready = 1;
|
||||
Object.defineProperty(video, "readyState", { configurable: true, get: () => ready });
|
||||
Object.defineProperty(video, "networkState", { configurable: true, value: 2 });
|
||||
Object.defineProperty(video, "videoWidth", { configurable: true, value: 64 });
|
||||
Object.defineProperty(video, "videoHeight", { configurable: true, value: 36 });
|
||||
await evaluateRuntime();
|
||||
let settled = false;
|
||||
const landed = Promise.resolve(window.__player!.seek(5.5)).then(() => {
|
||||
|
||||
@@ -19,6 +19,7 @@ import { hasFrameSources, registerFrameSource } from "./frameSources";
|
||||
import { createFilmBridge } from "./filmBridge";
|
||||
import { trackAcceleratedCanvases } from "./captureMotionBlur";
|
||||
import { parkFarScenesAtBoot } from "./sceneWindow";
|
||||
import { installPausedVideoCallbacks } from "./pausedVideoCallbacks";
|
||||
|
||||
type HyperframeWindow = Window & {
|
||||
__hyperframeRuntimeBootstrapped?: boolean;
|
||||
@@ -44,6 +45,7 @@ type HyperframeWindow = Window & {
|
||||
// at script evaluation time, while the document is still parsing.
|
||||
installAuthoredOpacityCapture();
|
||||
installAuthoredMediaCapture();
|
||||
installPausedVideoCallbacks();
|
||||
installFlatGsapTransforms();
|
||||
trackAcceleratedCanvases();
|
||||
window.__hfHasFrameSources = hasFrameSources;
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { initSandboxRuntimeModular } from "./init";
|
||||
import { hideTimedClipsUntilFirstPass } from "./timedClipHide";
|
||||
import { resetSeekDispatchState } from "./adapters/seek-dispatch";
|
||||
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
|
||||
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
|
||||
import type {
|
||||
PausedVideoFrame,
|
||||
PausedVideoFrameProvider,
|
||||
PausedVideoFrameRequest,
|
||||
} from "./pausedVideoFrames";
|
||||
|
||||
function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
|
||||
let time = 0;
|
||||
@@ -34,6 +40,17 @@ function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
|
||||
return {
|
||||
video,
|
||||
seeks,
|
||||
metadataAt: (position: number) => {
|
||||
time = position;
|
||||
ready = 1;
|
||||
seeking = false;
|
||||
},
|
||||
presentAt: (position: number) => {
|
||||
time = position;
|
||||
ready = 4;
|
||||
seeking = false;
|
||||
video.dispatchEvent(new Event("timeupdate"));
|
||||
},
|
||||
land: () => {
|
||||
ready = 4;
|
||||
seeking = false;
|
||||
@@ -57,6 +74,7 @@ function film(withImage = false) {
|
||||
const drawImage = vi.fn();
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
drawImage,
|
||||
clearRect: vi.fn(),
|
||||
} as unknown as CanvasRenderingContext2D);
|
||||
document.body.innerHTML =
|
||||
'<div data-composition-id="main" data-root="true" data-duration="10">' +
|
||||
@@ -83,6 +101,47 @@ function film(withImage = false) {
|
||||
return { main, old: videos.slice(0, 2), next: videos.slice(2), drawImage };
|
||||
}
|
||||
|
||||
function frameProvider(videos: ReturnType<typeof decoder>[]) {
|
||||
const clocks = new Map(
|
||||
videos.map(({ video }) => [video, Object.getOwnPropertyDescriptor(video, "currentTime")!]),
|
||||
);
|
||||
for (const { video } of videos) Reflect.deleteProperty(video, "currentTime");
|
||||
vi.spyOn(HTMLMediaElement.prototype, "currentTime", "get").mockImplementation(
|
||||
function (this: HTMLMediaElement) {
|
||||
return clocks.get(this as HTMLVideoElement)?.get?.();
|
||||
},
|
||||
);
|
||||
vi.spyOn(HTMLMediaElement.prototype, "currentTime", "set").mockImplementation(function (
|
||||
this: HTMLMediaElement,
|
||||
time: number,
|
||||
) {
|
||||
clocks.get(this as HTMLVideoElement)?.set?.(time);
|
||||
});
|
||||
const pending: Array<() => void> = [];
|
||||
const provider: PausedVideoFrameProvider = {
|
||||
decode: vi.fn(
|
||||
(requests: PausedVideoFrameRequest[]) =>
|
||||
new Promise<PausedVideoFrame[]>((resolve) => {
|
||||
pending.push(() =>
|
||||
resolve(
|
||||
requests.map(({ time }) => ({
|
||||
source: document.createElement("canvas"),
|
||||
timestamp: time,
|
||||
duration: 1 / 30,
|
||||
width: 64,
|
||||
height: 36,
|
||||
release: vi.fn(),
|
||||
})),
|
||||
),
|
||||
);
|
||||
}),
|
||||
),
|
||||
dispose: vi.fn(),
|
||||
};
|
||||
window.__hf!.setPausedVideoFrameProvider!(provider);
|
||||
return { provider, pending };
|
||||
}
|
||||
|
||||
it("keeps the old picture and page until both destination videos have decoded", async () => {
|
||||
const { main, old, next, drawImage } = film();
|
||||
const landed = window.__player!.seek(5.5);
|
||||
@@ -112,6 +171,8 @@ it("coalesces same-clip requests and publishes complete intermediate pictures",
|
||||
expect(next.map((v) => v.seeks.length)).toEqual(initialSeeks);
|
||||
for (const video of next) video.land();
|
||||
expect(main.time?.()).toBe(5.5);
|
||||
expect(window.__player!.getTime()).toBe(7.5);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
|
||||
expect(next.every((v) => Math.abs(v.video.currentTime - 2.5) < 0.001)).toBe(true);
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
@@ -270,6 +331,547 @@ it("caps a stalled decoder without leaving the picture hidden", async () => {
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("exposes the presented clock to animation callbacks while retaining the latest scrub destination", async () => {
|
||||
const { main, next } = film();
|
||||
const times: number[] = [];
|
||||
const original = main.totalTime!;
|
||||
main.totalTime = (time?: number) => {
|
||||
if (time !== undefined) times.push(window.__player!.getTime());
|
||||
return original(time);
|
||||
};
|
||||
const done = window.__player!.seek(5.5);
|
||||
window.__player!.seek(7.5);
|
||||
for (const video of next) video.land();
|
||||
expect(times).toContain(5.5);
|
||||
expect(window.__player!.getTime()).toBe(7.5);
|
||||
for (const video of next) video.land();
|
||||
await done;
|
||||
});
|
||||
|
||||
it.each(["stop-media", "set-native-media-sync-disabled"])(
|
||||
"does not resume a pending playback handoff after %s",
|
||||
async (action) => {
|
||||
const { old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const landed = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await landed;
|
||||
window.__player!.play();
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
window.dispatchEvent(
|
||||
new MessageEvent("message", {
|
||||
source: window,
|
||||
data: { source: "hf-parent", type: "control", action, disabled: true },
|
||||
}),
|
||||
);
|
||||
await Promise.resolve();
|
||||
for (const video of old) video.land();
|
||||
await Promise.resolve();
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the complete picture and buffers beyond the handoff cap until native frames are ready", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const { main, old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const landed = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await landed;
|
||||
for (const video of old) video.metadataAt(1);
|
||||
window.__player!.play();
|
||||
expect(old.every(({ video }) => video.preload === "auto")).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(5100);
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
expect(main.time?.()).toBe(1);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
window.__player!.seek(1);
|
||||
for (const video of old) video.land();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("settles a stalled paused fallback while retaining the last complete picture", async () => {
|
||||
resetSeekDispatchState();
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const { main, old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
vi.mocked(provider.decode).mockRejectedValueOnce(new Error("decoder unavailable"));
|
||||
const stalled = window.__player!.seek(6);
|
||||
const retargeted = window.__player!.seek(7);
|
||||
const wait = window.__hfWaitForSeekCompletion!();
|
||||
await vi.advanceTimersByTimeAsync(5100);
|
||||
await stalled;
|
||||
await retargeted;
|
||||
await wait;
|
||||
expect(main.time?.()).toBe(1);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
|
||||
const retry = window.__player!.seek(8);
|
||||
pending.shift()!();
|
||||
await retry;
|
||||
expect(main.time?.()).toBe(8);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["already metadata-only, idle", false, 1],
|
||||
["ready, then metadata-only idle", true, 1],
|
||||
["ready, then loading forever", true, 2],
|
||||
["metadata-only, loading forever", false, 2],
|
||||
] as const)(
|
||||
"bounds Play for %s while retaining its full-quality picture",
|
||||
async (_, initiallyReady, networkState) => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { main, old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(3);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
const stalled = old[0]!;
|
||||
if (initiallyReady) stalled.land();
|
||||
else stalled.metadataAt(3);
|
||||
Object.defineProperty(stalled.video, "networkState", {
|
||||
value: networkState,
|
||||
configurable: true,
|
||||
});
|
||||
window.__player!.play();
|
||||
old[1]!.land();
|
||||
await vi.advanceTimersByTimeAsync(5100);
|
||||
if (initiallyReady || networkState === 2) expect(window.__player!.isPlaying()).toBe(false);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(
|
||||
window.__player!.isPlaying() ? 1 : 2,
|
||||
);
|
||||
stalled.video.dispatchEvent(new Event("progress"));
|
||||
await vi.advanceTimersByTimeAsync(4900);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect(main.time?.()).toBeGreaterThanOrEqual(3);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(1);
|
||||
const held = document.querySelector("[data-hf-preview-video-canvas]")!;
|
||||
expect([held.getAttribute("width"), held.getAttribute("height")]).toEqual(["64", "36"]);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps every unready video picture at the hard deadline", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(3);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
|
||||
window.__player!.play();
|
||||
await vi.advanceTimersByTimeAsync(10000);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
|
||||
window.__player!.pause();
|
||||
pending.shift()!();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
window.__player!.play();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("reveals a delayed playing frame without chasing the moving clock with more seeks", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(3);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
const stalled = old[0]!;
|
||||
stalled.metadataAt(3);
|
||||
let error: { code: number } | null = { code: 4 };
|
||||
Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
|
||||
window.__player!.play();
|
||||
old[1]!.land();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
error = null;
|
||||
const seeks = stalled.seeks.length;
|
||||
Object.defineProperty(stalled.video, "paused", { configurable: true, value: false });
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
stalled.land();
|
||||
if (i === 0) stalled.video.dispatchEvent(new Event("playing"));
|
||||
stalled.presentAt(3 + (i + 1) / 30);
|
||||
}
|
||||
expect(stalled.seeks).toHaveLength(seeks);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps a paused recovery covered until its static target seek lands", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(3);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
const stalled = old[0]!;
|
||||
stalled.metadataAt(3);
|
||||
let error: { code: number } | null = { code: 4 };
|
||||
Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
|
||||
window.__player!.play();
|
||||
old[1]!.land();
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
window.__hf!.setPausedVideoFrameProvider!(null);
|
||||
window.__player!.pause();
|
||||
const picture = document.querySelector("[data-hf-preview-video-canvas]");
|
||||
expect(picture).not.toBeNull();
|
||||
error = null;
|
||||
stalled.metadataAt(3);
|
||||
stalled.land();
|
||||
expect(stalled.video.previousElementSibling).toBe(picture);
|
||||
expect(stalled.video.currentTime).toBeCloseTo(3.5, 2);
|
||||
stalled.land();
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("publishes a late native recovery after a paused fallback wait has settled", async () => {
|
||||
resetSeekDispatchState();
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const { main, old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
vi.mocked(provider.decode).mockRejectedValueOnce(new Error("decoder unavailable"));
|
||||
const stalled = window.__player!.seek(6);
|
||||
await vi.advanceTimersByTimeAsync(5100);
|
||||
await stalled;
|
||||
expect(main.time?.()).toBe(1);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
|
||||
expect(window.__player!.getTime()).toBe(1);
|
||||
const priorSeeks = old[0]!.seeks.length;
|
||||
const destinationSeeks = next.map((video) => video.seeks.length);
|
||||
old[0]!.reload();
|
||||
expect(old[0]!.seeks.length).toBeGreaterThan(priorSeeks);
|
||||
expect(next.map((video) => video.seeks.length)).toEqual(destinationSeeks);
|
||||
for (const video of next) video.land();
|
||||
await Promise.resolve();
|
||||
expect(main.time?.()).toBe(6);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps a hold-only snapshot visible while Play awaits decoder suspension", async () => {
|
||||
const { next, old } = film();
|
||||
const { provider } = frameProvider([...old, ...next]);
|
||||
let suspended = () => {};
|
||||
provider.suspend = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
suspended = resolve;
|
||||
});
|
||||
const seek = window.__player!.seek(6);
|
||||
const pictures = [...document.querySelectorAll("canvas")];
|
||||
expect(pictures).toHaveLength(2);
|
||||
window.__player!.play();
|
||||
await seek;
|
||||
expect([...document.querySelectorAll("canvas")]).toEqual(pictures);
|
||||
suspended();
|
||||
await Promise.resolve();
|
||||
for (const video of next) video.land();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("resumes a pending Play after its frame provider is replaced", async () => {
|
||||
const { old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
window.__player!.play();
|
||||
window.__hf!.setPausedVideoFrameProvider!({ decode: provider.decode, dispose: vi.fn() });
|
||||
pending.shift()!();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
for (const video of old) video.land();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("cancels the hard Play deadline when Pause restores provider ownership", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(3);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
window.__player!.play();
|
||||
await vi.advanceTimersByTimeAsync(9000);
|
||||
window.__player!.pause();
|
||||
pending.shift()!();
|
||||
await vi.advanceTimersByTimeAsync(20000);
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("bounds a deferred decoder release and ignores its late completion", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
let released = () => {};
|
||||
provider.suspend = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
released = resolve;
|
||||
});
|
||||
const initial = window.__player!.seek(3);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
window.__player!.play();
|
||||
await vi.advanceTimersByTimeAsync(10000);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
|
||||
expect(pictures).toHaveLength(2);
|
||||
released();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
|
||||
});
|
||||
|
||||
it("ignores decoder release after Pause cancels a pending Play", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
let released = () => {};
|
||||
provider.suspend = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
released = resolve;
|
||||
});
|
||||
const initial = window.__player!.seek(3);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
window.__player!.play();
|
||||
await vi.advanceTimersByTimeAsync(9000);
|
||||
window.__player!.pause();
|
||||
pending.shift()!();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
|
||||
const seeks = old.map((video) => video.seeks.length);
|
||||
released();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(old.map((video) => video.seeks.length)).toEqual(seeks);
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
|
||||
});
|
||||
|
||||
it("does not wait for a video frame from an audio-only video element", async () => {
|
||||
const { main, next } = film();
|
||||
for (const { video } of next)
|
||||
Object.defineProperties(video, {
|
||||
readyState: { value: 1, configurable: true },
|
||||
videoWidth: { value: 0, configurable: true },
|
||||
error: { value: null, configurable: true },
|
||||
});
|
||||
expect(window.__player!.seek(6)).toBeUndefined();
|
||||
expect(main.time?.()).toBe(6);
|
||||
window.__player!.play();
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
});
|
||||
|
||||
it("waits for paused decoder suspension before preparing native playback", async () => {
|
||||
const { old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
let releaseDecoders = () => {};
|
||||
provider.suspend = vi.fn(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseDecoders = resolve;
|
||||
}),
|
||||
);
|
||||
const initial = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
const seeks = old.map((v) => v.seeks.length);
|
||||
window.__player!.play();
|
||||
expect(provider.suspend).toHaveBeenCalledOnce();
|
||||
expect(old.map((v) => v.seeks.length)).toEqual(seeks);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
releaseDecoders();
|
||||
await Promise.resolve();
|
||||
for (const video of old) video.land();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
});
|
||||
|
||||
it("restores paused ownership when Pause cancels an unfinished decoder suspension", async () => {
|
||||
const { old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
let releaseDecoders = () => {};
|
||||
provider.suspend = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseDecoders = resolve;
|
||||
});
|
||||
const initial = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
window.__player!.play();
|
||||
window.__player!.pause();
|
||||
expect(pending).toHaveLength(1);
|
||||
pending.shift()!();
|
||||
releaseDecoders();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
expect(
|
||||
old.every(({ video }) => Object.hasOwn(video, "currentTime") && video.currentTime === 1),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("warms the restart destination before Play when paused at the end", async () => {
|
||||
const { main, old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const landed = window.__player!.seek(10, { settled: true });
|
||||
pending.shift()!();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(10);
|
||||
for (const video of old) video.metadataAt(0);
|
||||
const seeks = old.map((v) => v.seeks.length);
|
||||
window.__player!.play();
|
||||
expect(old.map((v) => v.seeks.length)).toEqual(seeks.map((n) => n + 1));
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
for (const video of old) video.land();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect(window.__player!.getTime()).toBeLessThan(1);
|
||||
expect(old.every(({ video }) => video.style.visibility === "visible")).toBe(true);
|
||||
});
|
||||
|
||||
it("retains a complete paused picture while replacing or removing its frame provider", async () => {
|
||||
const { main, old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
const replacement = { decode: provider.decode, dispose: vi.fn() };
|
||||
window.__hf!.setPausedVideoFrameProvider!(replacement);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
pending.shift()!();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(main.time?.()).toBe(1);
|
||||
window.__hf!.setPausedVideoFrameProvider!(null);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
for (const video of old) video.land();
|
||||
await Promise.resolve();
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("retains an in-flight destination when the host replaces its frame provider", async () => {
|
||||
const { main, old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
const first = window.__player!.seek(6);
|
||||
const picture = document.querySelector("[data-hf-preview-video-canvas]");
|
||||
expect(picture).not.toBeNull();
|
||||
const replacement = { decode: provider.decode, dispose: vi.fn() };
|
||||
window.__hf!.setPausedVideoFrameProvider!(replacement);
|
||||
expect(document.querySelector("[data-hf-preview-video-canvas]")).toBe(picture);
|
||||
expect(provider.dispose).toHaveBeenCalledTimes(1);
|
||||
expect(window.__player!.getTime()).toBe(6);
|
||||
pending.shift()!();
|
||||
pending.shift()!();
|
||||
await first;
|
||||
await Promise.resolve();
|
||||
expect(main.time?.()).toBe(6);
|
||||
});
|
||||
|
||||
it("loads original source metadata before publishing provider frames used by intrinsic-size effects", async () => {
|
||||
const { main, old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
for (const { video } of next)
|
||||
Object.defineProperty(video, "videoWidth", { value: 0, configurable: true });
|
||||
const landed = window.__player!.seek(6);
|
||||
pending.shift()!();
|
||||
await Promise.resolve();
|
||||
expect(main.time?.()).toBe(0);
|
||||
for (const { video } of next) {
|
||||
Object.defineProperty(video, "videoWidth", { value: 64, configurable: true });
|
||||
video.dispatchEvent(new Event("loadedmetadata"));
|
||||
}
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(6);
|
||||
});
|
||||
|
||||
it("refines the paused frame once after input settles, then warms native playback", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const { old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
const landed = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await landed;
|
||||
await vi.advanceTimersByTimeAsync(120);
|
||||
expect(provider.decode).toHaveBeenCalledTimes(2);
|
||||
expect(vi.mocked(provider.decode).mock.calls[1]![0].every((request) => request.settled)).toBe(
|
||||
true,
|
||||
);
|
||||
pending.shift()!();
|
||||
await Promise.resolve();
|
||||
await vi.advanceTimersByTimeAsync(120);
|
||||
expect(old.every(({ video }) => !Object.hasOwn(video, "currentTime"))).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(provider.decode).toHaveBeenCalledTimes(2);
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
});
|
||||
|
||||
it("decodes frame-step seeks at settled quality immediately without a second refinement", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const { old, next } = film();
|
||||
const { provider, pending } = frameProvider([...old, ...next]);
|
||||
const landed = window.__player!.seek(1, { settled: true });
|
||||
expect(vi.mocked(provider.decode).mock.calls[0]![0].every((request) => request.settled)).toBe(
|
||||
true,
|
||||
);
|
||||
pending.shift()!();
|
||||
await landed;
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(provider.decode).toHaveBeenCalledTimes(1);
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
});
|
||||
|
||||
it("notifies video-driven effects on the committed clock before animation redraw", async () => {
|
||||
const proto = HTMLVideoElement.prototype;
|
||||
const request = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
|
||||
const cancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
|
||||
Object.defineProperties(proto, {
|
||||
requestVideoFrameCallback: { value: vi.fn(() => 1), configurable: true, writable: true },
|
||||
cancelVideoFrameCallback: { value: vi.fn(), configurable: true, writable: true },
|
||||
});
|
||||
try {
|
||||
const { main, old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const notifications: number[] = [];
|
||||
old[0]!.video.requestVideoFrameCallback(() => notifications.push(window.__player!.getTime()));
|
||||
const original = main.totalTime!;
|
||||
main.totalTime = (time?: number) => {
|
||||
if (time === 1) expect(notifications).toEqual([1]);
|
||||
return original(time);
|
||||
};
|
||||
const landed = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await landed;
|
||||
expect(notifications).toEqual([1]);
|
||||
} finally {
|
||||
if (request) Object.defineProperty(proto, "requestVideoFrameCallback", request);
|
||||
else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
|
||||
if (cancel) Object.defineProperty(proto, "cancelVideoFrameCallback", cancel);
|
||||
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
window.__hfRuntimeTeardown?.();
|
||||
document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`)?.remove();
|
||||
@@ -281,3 +883,21 @@ afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
resetRuntimeFixtureDom();
|
||||
});
|
||||
|
||||
it("releases a native hold when stop-media cancels its first provider seek", async () => {
|
||||
const { old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const seek = window.__player!.seek(3);
|
||||
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(2);
|
||||
window.dispatchEvent(
|
||||
new MessageEvent("message", {
|
||||
source: window,
|
||||
data: { source: "hf-parent", type: "control", action: "stop-media" },
|
||||
}),
|
||||
);
|
||||
await seek;
|
||||
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
pending.shift()!();
|
||||
await Promise.resolve();
|
||||
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
@@ -1879,6 +1879,8 @@ describe("the preview's scene window", () => {
|
||||
return ready.has(this) ? 4 : 1;
|
||||
});
|
||||
vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
|
||||
vi.spyOn(HTMLVideoElement.prototype, "videoWidth", "get").mockReturnValue(64);
|
||||
vi.spyOn(HTMLVideoElement.prototype, "videoHeight", "get").mockReturnValue(36);
|
||||
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
|
||||
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
|
||||
const { root } = trackingRoot();
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
|
||||
import { hasFrozenPicture } from "./frozenPicture";
|
||||
import { createPreviewSeek } from "./previewSeek";
|
||||
import { createPausedVideoFrames, releasePausedVideoFrame } from "./pausedVideoFrames";
|
||||
import { copiesReady } from "./previewCopies";
|
||||
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
|
||||
import { instantTolerance } from "../clipFacts";
|
||||
@@ -77,6 +78,8 @@ import { createFrameSourceAdapter } from "./frameSources";
|
||||
import {
|
||||
patchVideoTextureCompat,
|
||||
patchWebGLVideoTextureCompat,
|
||||
patchCanvasVideoFrameCompat,
|
||||
patchExternalVideoTextureCompat,
|
||||
} from "./adapters/video-texture-compat";
|
||||
import {
|
||||
forceDispatchSeekEvent,
|
||||
@@ -530,6 +533,10 @@ function loadPreviewImagesIn(root: Element): void {
|
||||
export function initSandboxRuntimeModular(): void {
|
||||
const state = createRuntimeState();
|
||||
let previewSeek: ReturnType<typeof createPreviewSeek> | undefined;
|
||||
const pausedFrames = createPausedVideoFrames();
|
||||
let nativePlayGeneration = 0;
|
||||
let nativePlayTime: number | undefined;
|
||||
let nativeWarmTimer: ReturnType<typeof wallSetTimeout> | undefined;
|
||||
const stampsForStudio = window.parent !== window;
|
||||
// Studio's discovery stamps time a clip for the timeline only; nothing hides it at their end.
|
||||
const isDiscoveryStamped = (el: Element) =>
|
||||
@@ -740,6 +747,13 @@ export function initSandboxRuntimeModular(): void {
|
||||
const registerRuntimeCleanup = (callback: () => void) => {
|
||||
runtimeCleanupCallbacks.push(callback);
|
||||
};
|
||||
registerRuntimeCleanup(() => {
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
pausedFrames.destroy();
|
||||
delete window.__hf?.setPausedVideoFrameProvider;
|
||||
});
|
||||
const postRuntimeDiagnosticOnce = (
|
||||
code: string,
|
||||
details: Record<string, RuntimeJson>,
|
||||
@@ -761,7 +775,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
_timeline: RuntimeTimelineLike | null;
|
||||
play: () => void;
|
||||
pause: () => void;
|
||||
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
|
||||
seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => HeldSeek;
|
||||
getTime: () => number;
|
||||
getDuration: () => number;
|
||||
isPlaying: () => boolean;
|
||||
@@ -3687,6 +3701,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
* exists for. On `window.__hf` because the Studio is across the iframe boundary. */
|
||||
const pausedMediaLeases = new WeakSet<HTMLMediaElement>();
|
||||
const leasePausedMedia = (el: HTMLMediaElement): void => {
|
||||
if (isVideoElement(el)) releasePausedVideoFrame(el);
|
||||
pausedMediaLeases.add(el);
|
||||
};
|
||||
const releasePausedMedia = (el: HTMLMediaElement): void => {
|
||||
@@ -3695,10 +3710,33 @@ export function initSandboxRuntimeModular(): void {
|
||||
};
|
||||
window.__hf.leasePausedMedia = leasePausedMedia;
|
||||
window.__hf.releasePausedMedia = releasePausedMedia;
|
||||
window.__hf.setPausedVideoFrameProvider = (provider) => {
|
||||
const resumePlay = nativePlayTime !== undefined;
|
||||
const destination =
|
||||
previewSeek?.targetTime ?? (pausedFrames.active ? state.currentTime : undefined);
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
previewSeek?.cancel(true);
|
||||
pausedFrames.setProvider(provider);
|
||||
if (!provider) pausedFrames.prepareNative();
|
||||
if (destination !== undefined) {
|
||||
const landed = seekWhenReady(destination);
|
||||
const generation = nativePlayGeneration;
|
||||
if (resumePlay)
|
||||
void Promise.resolve(landed).then(() => {
|
||||
if (!state.tornDown && generation === nativePlayGeneration) transport.play();
|
||||
});
|
||||
}
|
||||
};
|
||||
// Clips the transport may move now (not a paused lease); null while the host drives media.
|
||||
const transportMovableClips = (clips: RuntimeMediaClip[]): RuntimeMediaClip[] | null => {
|
||||
if (state.nativeMediaSyncDisabled) return null;
|
||||
return state.isPlaying ? clips : clips.filter((clip) => !pausedMediaLeases.has(clip.el));
|
||||
return state.isPlaying
|
||||
? clips
|
||||
: clips.filter(
|
||||
(clip) => !pausedMediaLeases.has(clip.el) && !previewSeek?.ownsRecovery(clip.el),
|
||||
);
|
||||
};
|
||||
window.__hf.setLoopStart = (seconds) => {
|
||||
loopStartSeconds = seconds != null && Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
|
||||
@@ -3784,7 +3822,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
|
||||
let lastVisibilityPassSeconds: number | null = null;
|
||||
const syncMediaForCurrentState = (timingRevision?: number) => {
|
||||
if (previewSeek?.time !== undefined) return;
|
||||
if (previewSeek?.time !== undefined && !previewSeek.committing) return;
|
||||
// Scope 1 of 3 (see `withTimingResolver`). Closes before `syncRuntimeMedia`,
|
||||
// which may call `el.load()` and invalidate every cached duration.
|
||||
//
|
||||
@@ -4493,7 +4531,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
// Redraw the current frame as a seek does; a playing film cut to end at or before it stops there, as at its end.
|
||||
const cutShort = clock.isPlaying() && duration > 0 && state.currentTime >= duration;
|
||||
if (joining && clock.isPlaying() && !cutShort) joinPlayback(swappedHosts);
|
||||
else if (previewSeek?.time !== undefined) previewSeek.refresh();
|
||||
else if (previewSeek?.targetTime !== undefined) previewSeek.refresh();
|
||||
else transport.seek(cutShort ? duration : state.currentTime, { keepPlaying: !cutShort });
|
||||
if (previewSeek?.time === undefined) syncTimedElementVisibility(state.currentTime);
|
||||
postTimeline();
|
||||
@@ -4803,7 +4841,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
.sort((a, b) => Math.abs(a.start - time) - Math.abs(b.start - time))[0];
|
||||
const toMount = landed ?? neighbour;
|
||||
const toRetire =
|
||||
toMount || previewSeek?.time !== undefined
|
||||
toMount || previewSeek?.targetTime !== undefined
|
||||
? []
|
||||
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
|
||||
if (!toMount && toRetire.length === 0) {
|
||||
@@ -4812,9 +4850,11 @@ export function initSandboxRuntimeModular(): void {
|
||||
}
|
||||
const wantedNow = (name: string) =>
|
||||
heldScenes.has(name) ||
|
||||
scenesWanted(windowScenes, previewSeek?.time ?? state.currentTime, clock.isPlaying()).has(
|
||||
name,
|
||||
);
|
||||
scenesWanted(
|
||||
windowScenes,
|
||||
previewSeek?.targetTime ?? state.currentTime,
|
||||
clock.isPlaying(),
|
||||
).has(name);
|
||||
const parkedBeforeStep = parkedNames.size;
|
||||
windowWork = (async () => {
|
||||
if (landed) await mountScene(landed.name);
|
||||
@@ -4827,7 +4867,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
windowWork = null;
|
||||
const stuckStepKeepsItsPlan = parkedNames.size === parkedBeforeStep;
|
||||
windowNear = stuckStepKeepsItsPlan ? key : "";
|
||||
moveSceneWindow(previewSeek?.time ?? state.currentTime);
|
||||
moveSceneWindow(previewSeek?.targetTime ?? state.currentTime);
|
||||
});
|
||||
// A seek is not done until the scene it lands in is mounted.
|
||||
if (landed) registerSeekCompletion(windowWork, "scene-window");
|
||||
@@ -4934,6 +4974,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
if (!keepPlaying) syncMediaForCurrentState();
|
||||
const tl = state.capturedTimeline;
|
||||
pauseTimelineIfPossible(tl);
|
||||
pausedFrames.notify();
|
||||
const pageAnimations = seekTimelineAndAdapters(state.currentTime);
|
||||
if (!keepPlaying) decideClipsUnderScriptHidden(state.currentTime);
|
||||
runAdapters("pause", 0, pageAnimations);
|
||||
@@ -4968,7 +5009,65 @@ export function initSandboxRuntimeModular(): void {
|
||||
postState(true);
|
||||
};
|
||||
|
||||
const previewVideoClipsAt = (time: number) =>
|
||||
withTimingResolver(() => {
|
||||
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
|
||||
const visibleAt = timedVisibilityAt();
|
||||
const root = resolveRootCompositionElement();
|
||||
return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
|
||||
(clip) =>
|
||||
isVideoElement(clip.el) &&
|
||||
!(clip.el.readyState >= 1 && clip.el.videoWidth === 0) &&
|
||||
!isSilencedByHidden(clip.el) &&
|
||||
(clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
|
||||
isClipVisibleAt(time, clip.start, clip.end, duration) &&
|
||||
clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
|
||||
);
|
||||
});
|
||||
|
||||
previewSeek = createPreviewSeek({
|
||||
frames: {
|
||||
available: pausedFrames.available,
|
||||
active: () => pausedFrames.active,
|
||||
skipped: pausedFrames.skipped,
|
||||
hold: (videos) => {
|
||||
if (pausedFrames.available()) return pausedFrames.holdNative(videos);
|
||||
},
|
||||
prepare: (clips, time, signal, settled) => {
|
||||
moveSceneWindow(time);
|
||||
for (const { el } of clips) {
|
||||
if (!isVideoElement(el) || el.videoWidth > 0) continue;
|
||||
if (el.preload === "none") el.preload = "metadata";
|
||||
if (el.networkState !== 2) el.load();
|
||||
}
|
||||
const requests = clips.flatMap((clip) =>
|
||||
isVideoElement(clip.el)
|
||||
? [
|
||||
{
|
||||
video: clip.el,
|
||||
source: clip.el.src || clip.el.currentSrc,
|
||||
time: mediaSourceTimeAt(clip, time),
|
||||
settled,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
);
|
||||
return pausedFrames.prepare(requests, signal);
|
||||
},
|
||||
suspend: pausedFrames.suspendProvider,
|
||||
prepareNative: pausedFrames.prepareNative,
|
||||
clear: (keep) =>
|
||||
pausedFrames.clear(keep, (video, presented) => {
|
||||
if (clock.isPlaying()) return presented;
|
||||
const time = clock.now();
|
||||
const clip = previewVideoClipsAt(time).find(({ el }) => el === video);
|
||||
if (!clip) return true;
|
||||
const expected = mediaSourceTimeAt(clip, time);
|
||||
if (Math.abs(video.currentTime - expected) < 0.02) return true;
|
||||
video.currentTime = mediaElementTime(expected);
|
||||
return false;
|
||||
}),
|
||||
},
|
||||
videosToHold: () =>
|
||||
withTimingResolver(() =>
|
||||
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? [])
|
||||
@@ -4978,23 +5077,19 @@ export function initSandboxRuntimeModular(): void {
|
||||
scenesReadyAt: (time) =>
|
||||
!windowScenes.some(
|
||||
({ name, start, end }) => start <= time && time <= end && parkedNames.has(name),
|
||||
) &&
|
||||
previewVideoClipsAt(time).every(
|
||||
({ el }) => !isVideoElement(el) || el.videoWidth > 0 || isUnplayable(el),
|
||||
),
|
||||
clipsAt: (time) =>
|
||||
withTimingResolver(() => {
|
||||
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
|
||||
const visibleAt = timedVisibilityAt();
|
||||
const root = resolveRootCompositionElement();
|
||||
return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
|
||||
(clip) =>
|
||||
isVideoElement(clip.el) &&
|
||||
!isSilencedByHidden(clip.el) &&
|
||||
(clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
|
||||
isClipVisibleAt(time, clip.start, clip.end, duration) &&
|
||||
clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
|
||||
);
|
||||
}),
|
||||
clipsAt: previewVideoClipsAt,
|
||||
prepare: (clips, time) => {
|
||||
moveSceneWindow(time);
|
||||
for (const clip of clips) {
|
||||
const { el } = clip;
|
||||
if (el.preload !== "auto") el.preload = "auto";
|
||||
if (el.readyState === 1 && !el.seeking)
|
||||
el.currentTime = mediaElementTime(mediaSourceTimeAt(clip, time));
|
||||
}
|
||||
syncRuntimeMedia({
|
||||
clips,
|
||||
timeSeconds: time,
|
||||
@@ -5004,10 +5099,30 @@ export function initSandboxRuntimeModular(): void {
|
||||
getCompositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
|
||||
});
|
||||
},
|
||||
commit: (time) => applySeek(time),
|
||||
commit: (time) => {
|
||||
applySeek(time);
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
const refined = previewSeek?.refining;
|
||||
if (pausedFrames.active)
|
||||
nativeWarmTimer = wallSetTimeout(() => {
|
||||
if (state.tornDown || clock.isPlaying() || previewSeek?.time !== undefined) return;
|
||||
if (!refined) {
|
||||
const landed = previewSeek?.refine(time);
|
||||
if (landed) registerSeekCompletion(landed, "video-frame");
|
||||
return;
|
||||
}
|
||||
pausedFrames.prepareNative();
|
||||
state.mediaForceSyncNextTick = true;
|
||||
syncMediaForCurrentState();
|
||||
}, 120);
|
||||
},
|
||||
});
|
||||
|
||||
const seekWhenReady = (time: number, options?: { keepPlaying?: boolean }) => {
|
||||
const seekWhenReady = (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => {
|
||||
if (nativePlayTime !== undefined && Math.abs(time - nativePlayTime) < 1e-5) return;
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
if (
|
||||
!mediaDeferred ||
|
||||
state.nativeMediaSyncDisabled ||
|
||||
@@ -5016,6 +5131,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
(options?.keepPlaying && clock.isPlaying())
|
||||
) {
|
||||
previewSeek?.cancel();
|
||||
pausedFrames.clear();
|
||||
applySeek(time, options);
|
||||
return;
|
||||
}
|
||||
@@ -5030,7 +5146,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
for (const { el } of transportMovableClips(clips) ?? []) {
|
||||
if (!el.paused) el.pause();
|
||||
}
|
||||
const landed = previewSeek?.seek(time);
|
||||
const landed = options?.settled ? previewSeek?.refine(time) : previewSeek?.seek(time);
|
||||
if (landed) registerSeekCompletion(landed, "video-frame");
|
||||
postState(true);
|
||||
return landed;
|
||||
@@ -5039,6 +5155,33 @@ export function initSandboxRuntimeModular(): void {
|
||||
const transport: RuntimePlayerTransport = {
|
||||
play: () => {
|
||||
flushHeldSeek();
|
||||
if (
|
||||
(pausedFrames.active || (pausedFrames.available() && previewSeek?.time !== undefined)) &&
|
||||
!clock.isPlaying()
|
||||
) {
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
const generation = ++nativePlayGeneration;
|
||||
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
|
||||
if (duration > 0) clock.setDuration(duration);
|
||||
nativePlayTime =
|
||||
previewSeek?.targetTime ??
|
||||
(clock.reachedEnd() ? clock.getPlayStart() : state.currentTime);
|
||||
const landed = previewSeek?.handoff(nativePlayTime);
|
||||
if (landed) {
|
||||
bufferingFor = withTimingResolver(() =>
|
||||
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).map(({ el }) => el),
|
||||
);
|
||||
postState(true);
|
||||
void landed.then(() => {
|
||||
if (state.tornDown || generation !== nativePlayGeneration) return;
|
||||
bufferingFor = [];
|
||||
nativePlayTime = undefined;
|
||||
transport.play();
|
||||
});
|
||||
return;
|
||||
}
|
||||
nativePlayTime = undefined;
|
||||
}
|
||||
const destination = previewSeek?.time;
|
||||
previewSeek?.cancel();
|
||||
if (destination !== undefined) applySeek(destination);
|
||||
@@ -5076,9 +5219,22 @@ export function initSandboxRuntimeModular(): void {
|
||||
postState(true);
|
||||
},
|
||||
pause: () => {
|
||||
const destination = previewSeek?.targetTime ?? nativePlayTime;
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
previewSeek?.cancel(true);
|
||||
const wasBuffering = bufferingFor.length > 0;
|
||||
bufferingFor = [];
|
||||
if (clock.isPlaying()) haltPlayback();
|
||||
if (clock.isPlaying()) {
|
||||
if (
|
||||
mediaDeferred &&
|
||||
pausedFrames.available() &&
|
||||
!state.nativeMediaSyncDisabled &&
|
||||
!isExportRenderDrivingFrames()
|
||||
)
|
||||
void seekWhenReady(clock.now());
|
||||
else haltPlayback();
|
||||
} else if (destination !== undefined) void seekWhenReady(destination);
|
||||
else if (wasBuffering) postState(true);
|
||||
},
|
||||
seek: (timeSeconds, options) => {
|
||||
@@ -5086,6 +5242,10 @@ export function initSandboxRuntimeModular(): void {
|
||||
Math.max(0, Number(timeSeconds) || 0),
|
||||
state.canonicalFps,
|
||||
);
|
||||
if (nativePlayTime !== undefined && Math.abs(quantized - nativePlayTime) < 1e-5) return;
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
heldSeek = null;
|
||||
const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
|
||||
if (undecoded.length === 0) {
|
||||
@@ -5116,6 +5276,10 @@ export function initSandboxRuntimeModular(): void {
|
||||
return landed;
|
||||
},
|
||||
renderSeek: (timeSeconds, options) => {
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
pausedFrames.clear();
|
||||
heldSeek = null;
|
||||
previewSeek?.cancel();
|
||||
renderCaptureSeekStarted = true;
|
||||
@@ -5142,7 +5306,8 @@ export function initSandboxRuntimeModular(): void {
|
||||
if (blurred) registerSeekCompletion(blurred, "motion-blur");
|
||||
postState(true);
|
||||
},
|
||||
getTime: () => heldSeek?.time ?? previewSeek?.time ?? clock.now(),
|
||||
getTime: () =>
|
||||
previewSeek?.committing ? clock.now() : (heldSeek?.time ?? previewSeek?.time ?? clock.now()),
|
||||
getDuration: () => {
|
||||
const dur = clock.getDuration();
|
||||
return Number.isFinite(dur) ? dur : 0;
|
||||
@@ -5239,6 +5404,8 @@ export function initSandboxRuntimeModular(): void {
|
||||
] as RuntimeDeterministicAdapter[];
|
||||
patchVideoTextureCompat();
|
||||
patchWebGLVideoTextureCompat();
|
||||
patchCanvasVideoFrameCompat();
|
||||
patchExternalVideoTextureCompat();
|
||||
// Lets the engine re-render GPU compositions after it injects decoded video
|
||||
// frames, so video-textured WebGL/WebGPU scenes sample the correct frame.
|
||||
window.__hfReseekGpu = (time: number) => {
|
||||
@@ -5969,6 +6136,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
!ready(el) &&
|
||||
isPlayableTrack(el) &&
|
||||
!stillStandsIn(el) &&
|
||||
!pausedFrames.retained(el) &&
|
||||
!webAudio.ownsElement(el),
|
||||
)
|
||||
.map(({ el }) => el);
|
||||
@@ -6388,7 +6556,11 @@ export function initSandboxRuntimeModular(): void {
|
||||
emitAnalyticsEvent("composition_paused", { time: player.getTime() });
|
||||
},
|
||||
onStopMedia: () => {
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
previewSeek?.cancel();
|
||||
bufferingFor = [];
|
||||
webAudio.stopAll();
|
||||
const mediaEls = document.querySelectorAll("video, audio");
|
||||
for (const el of mediaEls) {
|
||||
@@ -6440,6 +6612,14 @@ export function initSandboxRuntimeModular(): void {
|
||||
}
|
||||
},
|
||||
onSetNativeMediaSyncDisabled: (disabled) => {
|
||||
if (disabled) {
|
||||
nativePlayGeneration++;
|
||||
nativePlayTime = undefined;
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
previewSeek?.cancel();
|
||||
pausedFrames.clear();
|
||||
bufferingFor = [];
|
||||
}
|
||||
if (state.nativeMediaSyncDisabled === disabled) return;
|
||||
state.nativeMediaSyncDisabled = disabled;
|
||||
state.mediaForceSyncNextTick = true;
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { RuntimeMediaClip } from "./types.js";
|
||||
import { clampAudioGain, clipGain, gainTimesLane } from "../audioGain.js";
|
||||
import { isMemberGroupHidden } from "../audioGroups.js";
|
||||
import { findInjectedRenderFrame } from "./renderFrameSibling.js";
|
||||
import { ownsPausedVideoTime } from "./pausedVideoFrames";
|
||||
import { isRenderFrameVideo } from "./preloadMedia.js";
|
||||
import { registerSeekCompletion } from "./adapters/seek-dispatch.js";
|
||||
import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";
|
||||
@@ -683,6 +684,10 @@ export function syncRuntimeMedia(params: {
|
||||
// (no-op when already "auto") and catches elements whose preload
|
||||
// was overridden after init.ts set it.
|
||||
if (el.preload !== "auto" && !isRenderFrameVideo(el)) el.preload = "auto";
|
||||
if (!params.playing && ownsPausedVideoTime(el)) {
|
||||
if (!el.paused) el.pause();
|
||||
continue;
|
||||
}
|
||||
// Per-element rate × global transport rate
|
||||
const baseRate = rateAt(clipRate, params.timeSeconds - origin) * params.playbackRate;
|
||||
// Too slow for the browser to play: kept paused and stepped onto the playhead by seeks.
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import {
|
||||
flushPausedVideoCallbacks,
|
||||
installPausedVideoCallbacks,
|
||||
observeNativeVideoFrames,
|
||||
releasePausedVideoCallbacks,
|
||||
stagePausedVideoCallbacks,
|
||||
} from "./pausedVideoCallbacks";
|
||||
|
||||
const proto = HTMLVideoElement.prototype;
|
||||
const originalRequest = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
|
||||
const originalCancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
|
||||
const pending = new Map<number, { video: HTMLVideoElement; callback: VideoFrameRequestCallback }>();
|
||||
let nextHandle = 100;
|
||||
const nativeRequest = vi.fn(function (this: HTMLVideoElement, callback: VideoFrameRequestCallback) {
|
||||
const handle = ++nextHandle;
|
||||
pending.set(handle, { video: this, callback });
|
||||
return handle;
|
||||
});
|
||||
const nativeCancel = vi.fn(function (this: HTMLVideoElement, handle: number) {
|
||||
if (pending.get(handle)?.video === this) pending.delete(handle);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
pending.clear();
|
||||
nextHandle = 100;
|
||||
nativeRequest.mockClear();
|
||||
nativeCancel.mockClear();
|
||||
Object.defineProperty(proto, "requestVideoFrameCallback", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: nativeRequest,
|
||||
});
|
||||
Object.defineProperty(proto, "cancelVideoFrameCallback", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: nativeCancel,
|
||||
});
|
||||
installPausedVideoCallbacks();
|
||||
});
|
||||
afterEach(() => {
|
||||
if (originalRequest) Object.defineProperty(proto, "requestVideoFrameCallback", originalRequest);
|
||||
else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
|
||||
if (originalCancel) Object.defineProperty(proto, "cancelVideoFrameCallback", originalCancel);
|
||||
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
|
||||
pending.clear();
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function metadata(mediaTime: number, presentedFrames: number): VideoFrameCallbackMetadata {
|
||||
return {
|
||||
mediaTime,
|
||||
presentedFrames,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
presentationTime: 123,
|
||||
expectedDisplayTime: 125,
|
||||
processingDuration: 0.003,
|
||||
};
|
||||
}
|
||||
function deliverNative(handle: number, frame: VideoFrameCallbackMetadata) {
|
||||
const request = pending.get(handle);
|
||||
expect(request).toBeDefined();
|
||||
pending.delete(handle);
|
||||
request!.callback(120, frame);
|
||||
}
|
||||
const frame = (mediaTime: number) => ({ mediaTime, width: 1920, height: 1080 });
|
||||
|
||||
it("forwards native timing and cancellation when no paused frame owns the video", () => {
|
||||
const video = document.createElement("video");
|
||||
const callback = vi.fn();
|
||||
const id = video.requestVideoFrameCallback(callback);
|
||||
expect(nativeRequest.mock.contexts[0]).toBe(video);
|
||||
expect(id).toBe(101);
|
||||
const nativeFrame = metadata(0.5, 15);
|
||||
deliverNative(id, nativeFrame);
|
||||
expect(callback).toHaveBeenCalledExactlyOnceWith(120, nativeFrame);
|
||||
expect(callback.mock.lastCall?.[1]).toBe(nativeFrame);
|
||||
const cancelled = video.requestVideoFrameCallback(callback);
|
||||
video.cancelVideoFrameCallback(cancelled);
|
||||
expect(nativeCancel).toHaveBeenLastCalledWith(cancelled);
|
||||
expect(nativeCancel.mock.contexts.at(-1)).toBe(video);
|
||||
expect(pending.has(cancelled)).toBe(false);
|
||||
video.cancelVideoFrameCallback(999);
|
||||
expect(nativeCancel).toHaveBeenLastCalledWith(999);
|
||||
});
|
||||
|
||||
it("reports one presented frame to multiple native observers without counting each callback as a frame", () => {
|
||||
const video = document.createElement("video");
|
||||
const first = vi.fn();
|
||||
const second = vi.fn();
|
||||
const firstId = video.requestVideoFrameCallback(first);
|
||||
const secondId = video.requestVideoFrameCallback(second);
|
||||
deliverNative(firstId, metadata(0.2, 6));
|
||||
deliverNative(secondId, metadata(0.2, 6));
|
||||
expect(first.mock.lastCall?.[1].presentedFrames).toBe(6);
|
||||
expect(second.mock.lastCall?.[1].presentedFrames).toBe(6);
|
||||
const afterReload = vi.fn();
|
||||
const next = video.requestVideoFrameCallback(afterReload);
|
||||
deliverNative(next, metadata(0, 1));
|
||||
expect(afterReload.mock.lastCall?.[1].presentedFrames).toBe(1);
|
||||
});
|
||||
|
||||
it("preserves native callback receiver and reports authored failures without skipping other observers", () => {
|
||||
const video = document.createElement("video");
|
||||
const callback = vi.fn();
|
||||
const id = video.requestVideoFrameCallback(callback);
|
||||
const receiver = {};
|
||||
const native = pending.get(id)!;
|
||||
native.callback.call(receiver, 120, metadata(0.2, 6));
|
||||
expect(callback.mock.contexts[0]).toBe(receiver);
|
||||
const report = vi.fn();
|
||||
vi.stubGlobal("reportError", report);
|
||||
const failure = new Error("authored effect failed");
|
||||
video.requestVideoFrameCallback(() => {
|
||||
throw failure;
|
||||
});
|
||||
const later = vi.fn();
|
||||
video.requestVideoFrameCallback(later);
|
||||
stagePausedVideoCallbacks(video, frame(1));
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(report).toHaveBeenCalledExactlyOnceWith(failure);
|
||||
expect(later).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("publishes one-shot synthetic callbacks with the staged dimensions and source clock", () => {
|
||||
vi.spyOn(performance, "now").mockReturnValue(300);
|
||||
const video = document.createElement("video");
|
||||
const callback = vi.fn();
|
||||
const id = video.requestVideoFrameCallback(callback);
|
||||
stagePausedVideoCallbacks(video, frame(3.5));
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(callback).toHaveBeenCalledExactlyOnceWith(300, {
|
||||
...frame(3.5),
|
||||
presentedFrames: 1,
|
||||
presentationTime: 300,
|
||||
expectedDisplayTime: 300,
|
||||
processingDuration: 0,
|
||||
});
|
||||
expect(nativeCancel).toHaveBeenCalledWith(id);
|
||||
expect(pending.has(id)).toBe(false);
|
||||
flushPausedVideoCallbacks(video);
|
||||
stagePausedVideoCallbacks(video, frame(4));
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(callback).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("defers a callback registered by a callback until the next published frame", () => {
|
||||
const video = document.createElement("video");
|
||||
const observed: number[] = [];
|
||||
const callback: VideoFrameRequestCallback = (_now, info) => {
|
||||
observed.push(info.mediaTime);
|
||||
video.requestVideoFrameCallback(callback);
|
||||
};
|
||||
video.requestVideoFrameCallback(callback);
|
||||
stagePausedVideoCallbacks(video, frame(1));
|
||||
flushPausedVideoCallbacks(video);
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(observed).toEqual([1]);
|
||||
stagePausedVideoCallbacks(video, frame(2));
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(observed).toEqual([1, 2]);
|
||||
expect(pending.size).toBe(1);
|
||||
});
|
||||
|
||||
it("honors cancellation of a later observer while a synthetic notification batch is running", () => {
|
||||
const video = document.createElement("video");
|
||||
const cancelled = vi.fn();
|
||||
video.requestVideoFrameCallback(() => video.cancelVideoFrameCallback(later));
|
||||
const later = video.requestVideoFrameCallback(cancelled);
|
||||
stagePausedVideoCallbacks(video, frame(1));
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(cancelled).not.toHaveBeenCalled();
|
||||
expect(nativeCancel).toHaveBeenCalledWith(later);
|
||||
expect(pending.size).toBe(0);
|
||||
});
|
||||
|
||||
it("suppresses native metadata while a paused frame owns the pixels and publishes the staged frame instead", () => {
|
||||
const video = document.createElement("video");
|
||||
const callback = vi.fn();
|
||||
const id = video.requestVideoFrameCallback(callback);
|
||||
stagePausedVideoCallbacks(video, frame(4));
|
||||
deliverNative(id, metadata(0.5, 15));
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(callback).toHaveBeenCalledTimes(1);
|
||||
expect(callback.mock.lastCall?.[1]).toMatchObject({ mediaTime: 4, presentedFrames: 1 });
|
||||
expect(nativeCancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rearms a native callback consumed during pause and still cancels it using the original public handle", () => {
|
||||
const video = document.createElement("video");
|
||||
const callback = vi.fn();
|
||||
const publicId = video.requestVideoFrameCallback(callback);
|
||||
stagePausedVideoCallbacks(video, frame(3));
|
||||
deliverNative(publicId, metadata(1, 30));
|
||||
releasePausedVideoCallbacks(video);
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
expect(nativeRequest).toHaveBeenCalledTimes(2);
|
||||
const rearmedId = nativeRequest.mock.results.at(-1)!.value;
|
||||
expect(rearmedId).not.toBe(publicId);
|
||||
expect(pending.has(rearmedId)).toBe(true);
|
||||
releasePausedVideoCallbacks(video);
|
||||
expect(nativeRequest).toHaveBeenCalledTimes(2);
|
||||
video.cancelVideoFrameCallback(publicId);
|
||||
expect(nativeCancel).toHaveBeenLastCalledWith(rearmedId);
|
||||
expect(pending.size).toBe(0);
|
||||
});
|
||||
|
||||
it("resumes the pending observer on native frames after handoff", () => {
|
||||
const video = document.createElement("video");
|
||||
const callback = vi.fn();
|
||||
const publicId = video.requestVideoFrameCallback(callback);
|
||||
stagePausedVideoCallbacks(video, frame(3));
|
||||
deliverNative(publicId, metadata(1, 30));
|
||||
releasePausedVideoCallbacks(video);
|
||||
const rearmedId = nativeRequest.mock.results.at(-1)!.value;
|
||||
deliverNative(rearmedId, metadata(3.033, 31));
|
||||
expect(callback).toHaveBeenCalledExactlyOnceWith(120, metadata(3.033, 31));
|
||||
expect(pending.size).toBe(0);
|
||||
});
|
||||
|
||||
it("installs once without double-wrapping request or cancellation", () => {
|
||||
const request = proto.requestVideoFrameCallback;
|
||||
const cancel = proto.cancelVideoFrameCallback;
|
||||
installPausedVideoCallbacks();
|
||||
expect(proto.requestVideoFrameCallback).toBe(request);
|
||||
expect(proto.cancelVideoFrameCallback).toBe(cancel);
|
||||
const callback = vi.fn();
|
||||
const video = document.createElement("video");
|
||||
video.requestVideoFrameCallback(callback);
|
||||
stagePausedVideoCallbacks(video, frame(1));
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(callback).toHaveBeenCalledTimes(1);
|
||||
expect(nativeRequest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("invalidates the actual vendored Three VideoTexture before redraw and continues natively after release", async () => {
|
||||
const { VideoTexture } =
|
||||
await import("../../../../registry/examples/path-traced-3d/assets/vendor/three-path-tracer.min.js");
|
||||
const video = document.createElement("video");
|
||||
Object.defineProperty(video, "readyState", { value: 4 });
|
||||
const texture = new VideoTexture(video);
|
||||
texture.update();
|
||||
expect(texture.version).toBe(0);
|
||||
stagePausedVideoCallbacks(video, frame(3));
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(texture.version).toBe(1);
|
||||
texture.update();
|
||||
expect(texture.version).toBe(1);
|
||||
stagePausedVideoCallbacks(video, frame(4));
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(texture.version).toBe(2);
|
||||
// The native decoder can finish a warm seek while provider pixels remain displayed.
|
||||
const waitingId = [...pending.keys()][0]!;
|
||||
deliverNative(waitingId, metadata(4, 120));
|
||||
expect(texture.version).toBe(2);
|
||||
releasePausedVideoCallbacks(video);
|
||||
const resumedId = [...pending.keys()][0]!;
|
||||
deliverNative(resumedId, metadata(4.033, 121));
|
||||
expect(texture.version).toBe(3);
|
||||
texture.dispose();
|
||||
expect(pending.size).toBe(0);
|
||||
});
|
||||
|
||||
it("observes native frames independently of staged authored callbacks", () => {
|
||||
const video = document.createElement("video");
|
||||
const authored = vi.fn();
|
||||
video.requestVideoFrameCallback(authored);
|
||||
stagePausedVideoCallbacks(video, frame(3));
|
||||
const recovered = vi.fn(() => true);
|
||||
const stop = observeNativeVideoFrames(video, recovered);
|
||||
deliverNative(102, metadata(3.05, 2));
|
||||
expect(recovered).toHaveBeenCalledOnce();
|
||||
expect(authored).not.toHaveBeenCalled();
|
||||
flushPausedVideoCallbacks(video);
|
||||
expect(authored).toHaveBeenCalledOnce();
|
||||
expect(authored.mock.lastCall?.[1].mediaTime).toBe(3);
|
||||
stop();
|
||||
});
|
||||
|
||||
it("cancels a re-armed native observer and ignores an already queued callback", () => {
|
||||
const video = document.createElement("video");
|
||||
const recovered = vi.fn(() => false);
|
||||
const stop = observeNativeVideoFrames(video, recovered);
|
||||
deliverNative(101, metadata(3, 1));
|
||||
const queued = pending.get(102)!;
|
||||
stop();
|
||||
expect(pending.has(102)).toBe(false);
|
||||
queued.callback(130, metadata(3.05, 2));
|
||||
expect(recovered).toHaveBeenCalledOnce();
|
||||
expect(pending.size).toBe(0);
|
||||
});
|
||||
|
||||
it("uses playing and progressing time when opacity defers a native callback", () => {
|
||||
const video = document.createElement("video");
|
||||
let paused = true;
|
||||
let time = 3;
|
||||
Object.defineProperties(video, {
|
||||
paused: { get: () => paused },
|
||||
currentTime: { get: () => time },
|
||||
});
|
||||
const recovered = vi.fn(() => true);
|
||||
const stop = observeNativeVideoFrames(video, recovered);
|
||||
video.dispatchEvent(new Event("timeupdate"));
|
||||
expect(recovered).not.toHaveBeenCalled();
|
||||
paused = false;
|
||||
video.dispatchEvent(new Event("playing"));
|
||||
video.dispatchEvent(new Event("timeupdate"));
|
||||
expect(recovered).not.toHaveBeenCalled();
|
||||
time += 1 / 30;
|
||||
video.dispatchEvent(new Event("timeupdate"));
|
||||
expect(recovered).toHaveBeenCalledOnce();
|
||||
expect(pending.size).toBe(0);
|
||||
stop();
|
||||
});
|
||||
|
||||
it("keeps waiting when no native frame or playback progress arrives", () => {
|
||||
const video = document.createElement("video");
|
||||
const recovered = vi.fn(() => true);
|
||||
const stop = observeNativeVideoFrames(video, recovered);
|
||||
for (let i = 0; i < 3; i++) video.dispatchEvent(new Event("timeupdate"));
|
||||
expect(recovered).not.toHaveBeenCalled();
|
||||
expect(pending.size).toBe(1);
|
||||
stop();
|
||||
expect(pending.size).toBe(0);
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
type Frame = Pick<VideoFrameCallbackMetadata, "mediaTime" | "width" | "height">;
|
||||
type Pending = { callback: VideoFrameRequestCallback; native: number | null };
|
||||
type State = {
|
||||
pending: Map<number, Pending>;
|
||||
frame: Frame | null;
|
||||
count: number;
|
||||
nativeCount: number;
|
||||
synthetic: boolean;
|
||||
dirty: boolean;
|
||||
};
|
||||
type Request = typeof HTMLVideoElement.prototype.requestVideoFrameCallback;
|
||||
type Cancel = typeof HTMLVideoElement.prototype.cancelVideoFrameCallback;
|
||||
type Registry = { states: WeakMap<HTMLVideoElement, State>; request: Request; cancel: Cancel };
|
||||
type TrackedRequest = Request & { __hfPausedCallbacks?: Registry };
|
||||
|
||||
const registry = () =>
|
||||
(HTMLVideoElement.prototype.requestVideoFrameCallback as TrackedRequest)?.__hfPausedCallbacks;
|
||||
|
||||
function stateFor(owner: Registry, video: HTMLVideoElement): State {
|
||||
let state = owner.states.get(video);
|
||||
if (!state) {
|
||||
state = {
|
||||
pending: new Map(),
|
||||
frame: null,
|
||||
count: 0,
|
||||
nativeCount: 0,
|
||||
synthetic: false,
|
||||
dirty: false,
|
||||
};
|
||||
owner.states.set(video, state);
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
function requestNative(
|
||||
owner: Registry,
|
||||
video: HTMLVideoElement,
|
||||
state: State,
|
||||
pending: Pending,
|
||||
id: () => number,
|
||||
): number {
|
||||
return owner.request.call(video, function (this: unknown, now, metadata) {
|
||||
pending.native = null;
|
||||
const advance = Math.max(0, metadata.presentedFrames - state.nativeCount);
|
||||
state.nativeCount = metadata.presentedFrames;
|
||||
if (state.frame) return;
|
||||
state.pending.delete(id());
|
||||
if (!state.synthetic) {
|
||||
state.count = metadata.presentedFrames;
|
||||
pending.callback.call(this, now, metadata);
|
||||
return;
|
||||
}
|
||||
state.count += advance;
|
||||
pending.callback.call(this, now, { ...metadata, presentedFrames: state.count });
|
||||
});
|
||||
}
|
||||
|
||||
/** Install before authored scripts: Three's VideoTexture registers its first callback in its constructor. */
|
||||
export function installPausedVideoCallbacks(): void {
|
||||
const proto = HTMLVideoElement.prototype;
|
||||
const request = proto.requestVideoFrameCallback as TrackedRequest;
|
||||
if (typeof request !== "function" || request.__hfPausedCallbacks) return;
|
||||
const owner: Registry = {
|
||||
states: new WeakMap(),
|
||||
request,
|
||||
cancel: proto.cancelVideoFrameCallback,
|
||||
};
|
||||
const patched: TrackedRequest = function (this: HTMLVideoElement, callback) {
|
||||
const state = stateFor(owner, this);
|
||||
const pending: Pending = { callback, native: null };
|
||||
const id = requestNative(owner, this, state, pending, () => id);
|
||||
pending.native = id;
|
||||
state.pending.set(id, pending);
|
||||
return id;
|
||||
};
|
||||
patched.__hfPausedCallbacks = owner;
|
||||
proto.requestVideoFrameCallback = patched;
|
||||
proto.cancelVideoFrameCallback = function (id) {
|
||||
const pending = owner.states.get(this)?.pending.get(id);
|
||||
if (!pending) return owner.cancel.call(this, id);
|
||||
owner.states.get(this)?.pending.delete(id);
|
||||
if (pending.native !== null) owner.cancel.call(this, pending.native);
|
||||
};
|
||||
}
|
||||
|
||||
export function stagePausedVideoCallbacks(video: HTMLVideoElement, frame: Frame): void {
|
||||
const owner = registry();
|
||||
if (!owner) return;
|
||||
const state = stateFor(owner, video);
|
||||
state.frame = frame;
|
||||
state.dirty = true;
|
||||
}
|
||||
|
||||
/** Notify once before composition redraw; callbacks registered from a callback wait for the next frame. */
|
||||
export function flushPausedVideoCallbacks(video: HTMLVideoElement): void {
|
||||
const owner = registry();
|
||||
const state = owner?.states.get(video);
|
||||
if (!owner || !state?.frame || !state.dirty) return;
|
||||
state.dirty = false;
|
||||
state.synthetic = true;
|
||||
state.count++;
|
||||
const now = performance.now();
|
||||
const metadata: VideoFrameCallbackMetadata = {
|
||||
...state.frame,
|
||||
presentedFrames: state.count,
|
||||
presentationTime: now,
|
||||
expectedDisplayTime: now,
|
||||
processingDuration: 0,
|
||||
};
|
||||
for (const [id, pending] of [...state.pending]) {
|
||||
if (!state.pending.delete(id)) continue;
|
||||
if (pending.native !== null) owner.cancel.call(video, pending.native);
|
||||
try {
|
||||
pending.callback.call(undefined, now, metadata);
|
||||
} catch (error) {
|
||||
reportError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function releasePausedVideoCallbacks(video: HTMLVideoElement): void {
|
||||
const owner = registry();
|
||||
const state = owner?.states.get(video);
|
||||
if (!owner || !state) return;
|
||||
state.frame = null;
|
||||
state.dirty = false;
|
||||
for (const [id, pending] of state.pending) {
|
||||
if (pending.native === null)
|
||||
pending.native = requestNative(owner, video, state, pending, () => id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Recovery must observe native frames even while authored callbacks still see staged pixels. */
|
||||
export function observeNativeVideoFrames(
|
||||
video: HTMLVideoElement,
|
||||
onFrame: () => boolean,
|
||||
): () => void {
|
||||
const owner = registry();
|
||||
const request = owner?.request ?? video.requestVideoFrameCallback;
|
||||
const cancel = owner?.cancel ?? video.cancelVideoFrameCallback;
|
||||
let handle: number | null = null;
|
||||
let stopped = false;
|
||||
let playingAt: number | null = video.paused ? null : video.currentTime;
|
||||
const stop = () => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
if (handle !== null && typeof cancel === "function") cancel.call(video, handle);
|
||||
video.removeEventListener("playing", playing);
|
||||
video.removeEventListener("timeupdate", progress);
|
||||
};
|
||||
const arm = () => {
|
||||
if (!stopped && typeof request === "function") handle = request.call(video, presented);
|
||||
};
|
||||
const presented = () => {
|
||||
handle = null;
|
||||
if (stopped) return;
|
||||
if (onFrame()) stop();
|
||||
else arm();
|
||||
};
|
||||
const playing = () => {
|
||||
playingAt = video.currentTime;
|
||||
};
|
||||
const progress = () => {
|
||||
if (stopped || playingAt === null || video.paused || video.currentTime === playingAt) return;
|
||||
if (onFrame()) stop();
|
||||
};
|
||||
video.addEventListener("playing", playing);
|
||||
video.addEventListener("timeupdate", progress);
|
||||
arm();
|
||||
return stop;
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createPausedVideoFrames,
|
||||
getPausedVideoFrame,
|
||||
ownsPausedVideoTime,
|
||||
type PausedVideoFrameProvider,
|
||||
} from "./pausedVideoFrames";
|
||||
|
||||
const sessions: ReturnType<typeof createPausedVideoFrames>[] = [];
|
||||
beforeEach(() => {
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
clearRect: vi.fn(),
|
||||
drawImage: vi.fn(),
|
||||
} as unknown as CanvasRenderingContext2D);
|
||||
});
|
||||
afterEach(() => {
|
||||
for (const session of sessions.splice(0)) session.destroy();
|
||||
document.body.innerHTML = "";
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function fixture() {
|
||||
const session = createPausedVideoFrames();
|
||||
sessions.push(session);
|
||||
const video = document.createElement("video");
|
||||
video.src = "/original.webm";
|
||||
video.style.cssText = "position:absolute;width:64px;height:36px;transform:translateX(10px)";
|
||||
document.body.append(video);
|
||||
Object.defineProperty(video, "videoWidth", { value: 64 });
|
||||
Object.defineProperty(video, "videoHeight", { value: 36 });
|
||||
const release = vi.fn();
|
||||
const provider: PausedVideoFrameProvider = {
|
||||
decode: vi.fn(async () => [
|
||||
{
|
||||
source: document.createElement("canvas"),
|
||||
timestamp: 3,
|
||||
duration: 1 / 30,
|
||||
width: 64,
|
||||
height: 36,
|
||||
release,
|
||||
},
|
||||
]),
|
||||
dispose: vi.fn(),
|
||||
};
|
||||
session.setProvider(provider);
|
||||
const request = [{ video, source: video.src, time: 3 }];
|
||||
return { session, video, provider, request, release };
|
||||
}
|
||||
|
||||
it("presents decoded pixels with the authored source, dimensions, layout and displayed source clock", async () => {
|
||||
const { session, video, provider, request, release } = fixture();
|
||||
const originalSrc = video.src;
|
||||
const batch = await session.prepare(request, new AbortController().signal);
|
||||
batch!.present();
|
||||
video.style.transform = "translateX(25px)";
|
||||
batch!.refresh();
|
||||
const canvas = getPausedVideoFrame(video)!;
|
||||
expect([video.src, video.videoWidth, video.videoHeight, video.currentTime]).toEqual([
|
||||
originalSrc,
|
||||
64,
|
||||
36,
|
||||
3,
|
||||
]);
|
||||
expect(canvas.style.transform).toBe("translateX(25px)");
|
||||
expect(canvas.nextElementSibling).toBe(video);
|
||||
expect(ownsPausedVideoTime(video)).toBe(true);
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
session.prepareNative();
|
||||
expect(Object.hasOwn(video, "currentTime")).toBe(false);
|
||||
expect(ownsPausedVideoTime(video)).toBe(false);
|
||||
expect(getPausedVideoFrame(video)).toBe(canvas);
|
||||
session.clear();
|
||||
expect(getPausedVideoFrame(video)).toBeNull();
|
||||
expect(canvas.width).toBe(0);
|
||||
expect(video.hasAttribute("data-hf-preview-video-frame")).toBe(false);
|
||||
session.destroy();
|
||||
expect(provider.dispose).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("releases a late decode when its request is aborted or its source changes", async () => {
|
||||
const { session, video, request, release } = fixture();
|
||||
const controller = new AbortController();
|
||||
const batch = session.prepare(request, controller.signal);
|
||||
controller.abort();
|
||||
expect(await batch).toBeNull();
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
const changed = session.prepare(request, new AbortController().signal);
|
||||
video.src = "/replacement.webm";
|
||||
expect(await changed).toBeNull();
|
||||
expect(release).toHaveBeenCalledTimes(2);
|
||||
expect(getPausedVideoFrame(video)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not surrender decoded presentation for the runtime's native seek padding", async () => {
|
||||
const { session, video, request } = fixture();
|
||||
const batch = await session.prepare(request, new AbortController().signal);
|
||||
batch!.present();
|
||||
batch!.refresh();
|
||||
const canvas = getPausedVideoFrame(video);
|
||||
video.currentTime = 3.00001;
|
||||
expect(ownsPausedVideoTime(video)).toBe(true);
|
||||
expect(getPausedVideoFrame(video)).toBe(canvas);
|
||||
expect(video.currentTime).toBe(3);
|
||||
});
|
||||
|
||||
it("gives an authored seek back to native media and preserves an existing currentTime override", async () => {
|
||||
const { session, video, request } = fixture();
|
||||
const batch = await session.prepare(request, new AbortController().signal);
|
||||
batch!.present();
|
||||
video.currentTime = 7;
|
||||
expect(video.currentTime).toBe(7);
|
||||
expect(getPausedVideoFrame(video)).toBeNull();
|
||||
expect(await session.prepare(request, new AbortController().signal)).toBeNull();
|
||||
const custom = document.createElement("video");
|
||||
Object.defineProperty(custom, "currentTime", { value: 2, configurable: true });
|
||||
expect(
|
||||
await session.prepare([{ ...request[0]!, video: custom }], new AbortController().signal),
|
||||
).toBeNull();
|
||||
expect(custom.currentTime).toBe(2);
|
||||
});
|
||||
|
||||
it("rejects a neighbouring frame that does not cover the requested presentation time", async () => {
|
||||
const { session, provider, request, release } = fixture();
|
||||
vi.mocked(provider.decode).mockResolvedValue([
|
||||
{
|
||||
source: document.createElement("canvas"),
|
||||
timestamp: 2,
|
||||
duration: 1 / 30,
|
||||
width: 64,
|
||||
height: 36,
|
||||
release,
|
||||
},
|
||||
]);
|
||||
expect(await session.prepare(request, new AbortController().signal)).toBeNull();
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("retains the whole previous presentation when staging a new layer fails", async () => {
|
||||
const { session, video, request, release } = fixture();
|
||||
const first = await session.prepare(request, new AbortController().signal);
|
||||
first!.present();
|
||||
first!.refresh();
|
||||
const previous = getPausedVideoFrame(video);
|
||||
const second = await session.prepare(request, new AbortController().signal);
|
||||
vi.mocked(HTMLCanvasElement.prototype.getContext).mockReturnValueOnce(null);
|
||||
expect(second!.present()).toBe(false);
|
||||
expect(getPausedVideoFrame(video)).toBe(previous);
|
||||
expect(video.currentTime).toBe(3);
|
||||
expect(release).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("revalidates source identity after preparing a frame and before publishing it", async () => {
|
||||
const { session, video, request, release } = fixture();
|
||||
const prepared = await session.prepare(request, new AbortController().signal);
|
||||
video.src = "/edited.mp4";
|
||||
expect(prepared!.valid()).toBe(false);
|
||||
expect(prepared!.present()).toBe(false);
|
||||
expect(getPausedVideoFrame(video)).toBeNull();
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reuses display surfaces across updates and removes them when source children are replaced", async () => {
|
||||
const { session, video, request } = fixture();
|
||||
const first = await session.prepare(request, new AbortController().signal);
|
||||
first!.present();
|
||||
first!.refresh();
|
||||
const canvas = getPausedVideoFrame(video);
|
||||
const next = await session.prepare(request, new AbortController().signal);
|
||||
next!.present();
|
||||
next!.refresh();
|
||||
expect(getPausedVideoFrame(video)).toBe(canvas);
|
||||
video.append(document.createElement("source"));
|
||||
await Promise.resolve();
|
||||
expect(getPausedVideoFrame(video)).toBeNull();
|
||||
expect(Object.hasOwn(video, "currentTime")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps the displayed surface in sync with paused editing without releasing decoded pixels", async () => {
|
||||
const { session, video, request } = fixture();
|
||||
const batch = await session.prepare(request, new AbortController().signal);
|
||||
batch!.present();
|
||||
batch!.refresh();
|
||||
const canvas = getPausedVideoFrame(video)!;
|
||||
video.style.transform = "translateX(80px)";
|
||||
video.style.opacity = "0.4";
|
||||
video.style.width = "80px";
|
||||
await Promise.resolve();
|
||||
expect(getPausedVideoFrame(video)).toBe(canvas);
|
||||
expect(canvas.style.transform).toBe("translateX(80px)");
|
||||
expect(canvas.style.opacity).toBe("0.4");
|
||||
expect(canvas.style.width).toBe("80px");
|
||||
expect(video.currentTime).toBe(3);
|
||||
});
|
||||
|
||||
it("holds a full-size native snapshot through original-source provider replacement", async () => {
|
||||
const { session, video, request } = fixture();
|
||||
Object.defineProperty(video, "readyState", { value: 4 });
|
||||
session.holdNative([video]);
|
||||
const snapshot = getPausedVideoFrame(video)!;
|
||||
expect([snapshot.width, snapshot.height]).toEqual([64, 36]);
|
||||
expect(snapshot.hasAttribute("data-hyperframes-picker-ignore")).toBe(true);
|
||||
expect(ownsPausedVideoTime(video)).toBe(false);
|
||||
const decoded = await session.prepare(request, new AbortController().signal);
|
||||
expect(decoded!.present()).toBe(true);
|
||||
expect(getPausedVideoFrame(video)).toBe(snapshot);
|
||||
expect(ownsPausedVideoTime(video)).toBe(true);
|
||||
expect(video.currentTime).toBe(3);
|
||||
});
|
||||
|
||||
it("cancels a retained native observer when a new paused picture replaces it", async () => {
|
||||
const proto = HTMLVideoElement.prototype;
|
||||
const original = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
|
||||
const originalCancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
|
||||
let queued: VideoFrameRequestCallback = () => {};
|
||||
const cancel = vi.fn();
|
||||
Object.defineProperty(proto, "requestVideoFrameCallback", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: (callback: VideoFrameRequestCallback) => {
|
||||
queued = callback;
|
||||
return 123;
|
||||
},
|
||||
});
|
||||
Object.defineProperty(proto, "cancelVideoFrameCallback", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: cancel,
|
||||
});
|
||||
try {
|
||||
const { session, video, request } = fixture();
|
||||
const first = await session.prepare(request, new AbortController().signal);
|
||||
first!.present();
|
||||
session.prepareNative();
|
||||
session.clear(new Set([video]), () => false);
|
||||
const replacement = await session.prepare(request, new AbortController().signal);
|
||||
replacement!.present();
|
||||
const picture = getPausedVideoFrame(video);
|
||||
expect(cancel).toHaveBeenCalledWith(123);
|
||||
queued(0, {
|
||||
mediaTime: 3,
|
||||
width: 64,
|
||||
height: 36,
|
||||
presentedFrames: 1,
|
||||
presentationTime: 0,
|
||||
expectedDisplayTime: 0,
|
||||
processingDuration: 0,
|
||||
});
|
||||
expect(getPausedVideoFrame(video)).toBe(picture);
|
||||
expect(ownsPausedVideoTime(video)).toBe(true);
|
||||
} finally {
|
||||
if (original) Object.defineProperty(proto, "requestVideoFrameCallback", original);
|
||||
else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
|
||||
if (originalCancel) Object.defineProperty(proto, "cancelVideoFrameCallback", originalCancel);
|
||||
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,423 @@
|
||||
import { MEDIA_ELEMENT_LEAD_SECONDS } from "./mediaClock";
|
||||
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
|
||||
import {
|
||||
installPausedVideoCallbacks,
|
||||
stagePausedVideoCallbacks,
|
||||
flushPausedVideoCallbacks,
|
||||
releasePausedVideoCallbacks,
|
||||
observeNativeVideoFrames,
|
||||
} from "./pausedVideoCallbacks";
|
||||
import {
|
||||
PREVIEW_VIDEO_FRAME_ATTR as FRAME_ATTR,
|
||||
PREVIEW_VIDEO_SOURCE_ATTR,
|
||||
PREVIEW_VIDEO_CANVAS_ATTR,
|
||||
} from "../studioPreviewMark";
|
||||
|
||||
export interface PausedVideoFrameRequest {
|
||||
video: HTMLVideoElement;
|
||||
source: string;
|
||||
time: number;
|
||||
settled?: boolean;
|
||||
}
|
||||
|
||||
export interface PausedVideoFrame {
|
||||
source: CanvasImageSource;
|
||||
timestamp: number;
|
||||
duration: number;
|
||||
width: number;
|
||||
height: number;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
/** Decoding lives in the host; core owns composition timing and presentation. */
|
||||
export interface PausedVideoFrameProvider {
|
||||
decode(requests: PausedVideoFrameRequest[], signal: AbortSignal): Promise<PausedVideoFrame[]>;
|
||||
suspend?(): Promise<void>;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
export interface PreparedPausedVideoFrames {
|
||||
valid(): boolean;
|
||||
present(): boolean;
|
||||
refresh(): void;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
const SURFACE_BUDGET = 128 * 1024 * 1024;
|
||||
type Presented = {
|
||||
video: HTMLVideoElement;
|
||||
canvas: HTMLCanvasElement;
|
||||
time: number;
|
||||
driving: boolean;
|
||||
restoreTime: () => void;
|
||||
gpuFrame?: VideoFrame;
|
||||
stopWaiting?: () => void;
|
||||
// Paused frame replacement does not retry a source native playback already skipped.
|
||||
nativeSkipped?: boolean;
|
||||
};
|
||||
const presented = new Map<HTMLMediaElement, Presented>();
|
||||
const authoredSeeks = new WeakSet<HTMLVideoElement>();
|
||||
|
||||
export function getPausedVideoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
|
||||
return presented.get(video)?.canvas ?? null;
|
||||
}
|
||||
|
||||
export function ownsPausedVideoTime(video: HTMLMediaElement): boolean {
|
||||
return presented.get(video)?.driving === true;
|
||||
}
|
||||
|
||||
export function getPausedExternalVideoFrame(video: HTMLVideoElement): VideoFrame | null {
|
||||
const entry = presented.get(video);
|
||||
if (!entry || typeof VideoFrame === "undefined") return null;
|
||||
entry.gpuFrame ??= new VideoFrame(entry.canvas, { timestamp: Math.round(entry.time * 1e6) });
|
||||
return entry.gpuFrame;
|
||||
}
|
||||
|
||||
export function captureNativeVideoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = video.videoWidth;
|
||||
canvas.height = video.videoHeight;
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) return null;
|
||||
try {
|
||||
context.drawImage(video, 0, 0);
|
||||
return canvas;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function nativeSnapshotStyle(video: HTMLVideoElement): CSSStyleDeclaration | null {
|
||||
if (getPausedVideoFrame(video)) return null;
|
||||
// Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
|
||||
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return null;
|
||||
const css = getComputedStyle(video);
|
||||
if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return null;
|
||||
if (!video.videoWidth || !video.videoHeight) return null;
|
||||
return css;
|
||||
}
|
||||
|
||||
function virtualTime(entry: Presented): void {
|
||||
let proto = Object.getPrototypeOf(entry.video);
|
||||
let descriptor: PropertyDescriptor | undefined;
|
||||
while (proto && !descriptor) {
|
||||
descriptor = Object.getOwnPropertyDescriptor(proto, "currentTime");
|
||||
proto = Object.getPrototypeOf(proto);
|
||||
}
|
||||
const nativeSet = descriptor?.set;
|
||||
Object.defineProperty(entry.video, "currentTime", {
|
||||
configurable: true,
|
||||
get: () => entry.time,
|
||||
set: (value: number) => {
|
||||
if (Math.abs(value - entry.time) <= MEDIA_ELEMENT_LEAD_SECONDS * 2) return;
|
||||
authoredSeeks.add(entry.video);
|
||||
remove(entry);
|
||||
nativeSet?.call(entry.video, value);
|
||||
},
|
||||
});
|
||||
entry.restoreTime = () => {
|
||||
Reflect.deleteProperty(entry.video, "currentTime");
|
||||
};
|
||||
}
|
||||
|
||||
function remove(entry: Presented): void {
|
||||
entry.stopWaiting?.();
|
||||
entry.restoreTime();
|
||||
releasePausedVideoCallbacks(entry.video);
|
||||
entry.gpuFrame?.close();
|
||||
entry.video.removeAttribute(FRAME_ATTR);
|
||||
entry.video.removeAttribute(PREVIEW_VIDEO_SOURCE_ATTR);
|
||||
entry.canvas.remove();
|
||||
entry.canvas.width = entry.canvas.height = 0;
|
||||
presented.delete(entry.video);
|
||||
}
|
||||
|
||||
export function releasePausedVideoFrame(video: HTMLVideoElement): void {
|
||||
const entry = presented.get(video);
|
||||
if (entry) remove(entry);
|
||||
}
|
||||
|
||||
function refresh(entry: Presented): void {
|
||||
const { video, canvas } = entry;
|
||||
video.removeAttribute(FRAME_ATTR);
|
||||
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) {
|
||||
canvas.remove();
|
||||
return;
|
||||
}
|
||||
const css = getComputedStyle(video);
|
||||
for (const key of Array.from(css)) {
|
||||
const value = css.getPropertyValue(key);
|
||||
if (canvas.style.getPropertyValue(key) !== value) canvas.style.setProperty(key, value);
|
||||
}
|
||||
if (css.position !== "absolute" && css.position !== "fixed") {
|
||||
canvas.style.position = "absolute";
|
||||
canvas.style.inset = "auto";
|
||||
canvas.style.left = `${video.offsetLeft}px`;
|
||||
canvas.style.top = `${video.offsetTop}px`;
|
||||
canvas.style.margin = "0";
|
||||
}
|
||||
canvas.style.setProperty("pointer-events", "none", "important");
|
||||
canvas.style.setProperty("animation", "none", "important");
|
||||
canvas.style.setProperty("transition", "none", "important");
|
||||
if (video.previousElementSibling !== canvas) video.before(canvas);
|
||||
video.setAttribute(FRAME_ATTR, "");
|
||||
}
|
||||
|
||||
type ReusablePresentation = { entry: Presented; context: CanvasRenderingContext2D };
|
||||
|
||||
function markFrameCanvas(canvas: HTMLCanvasElement): void {
|
||||
canvas.setAttribute(PREVIEW_VIDEO_CANVAS_ATTR, "");
|
||||
canvas.setAttribute("data-layout-ignore", "");
|
||||
canvas.setAttribute("data-hyperframes-ignore", "");
|
||||
canvas.setAttribute("data-hyperframes-picker-ignore", "");
|
||||
canvas.setAttribute("data-hf-ignore", "");
|
||||
canvas.setAttribute("aria-hidden", "true");
|
||||
}
|
||||
|
||||
function adoptStagedFrame(next: Presented, existing?: ReusablePresentation): void {
|
||||
if (existing) {
|
||||
const { entry, context } = existing;
|
||||
entry.stopWaiting?.();
|
||||
entry.stopWaiting = undefined;
|
||||
entry.gpuFrame?.close();
|
||||
entry.gpuFrame = undefined;
|
||||
context.clearRect(0, 0, entry.canvas.width, entry.canvas.height);
|
||||
context.drawImage(next.canvas, 0, 0);
|
||||
next.canvas.width = next.canvas.height = 0;
|
||||
entry.time = next.time;
|
||||
if (!entry.driving) virtualTime(entry);
|
||||
entry.driving = true;
|
||||
return;
|
||||
}
|
||||
const { video, canvas } = next;
|
||||
releasePausedVideoFrame(video);
|
||||
markFrameCanvas(canvas);
|
||||
presented.set(video, next);
|
||||
video.setAttribute(PREVIEW_VIDEO_SOURCE_ATTR, "");
|
||||
virtualTime(next);
|
||||
}
|
||||
|
||||
/** Keeps original nodes, URLs, intrinsic sizes and the displayed source clock intact. */
|
||||
export function createPausedVideoFrames() {
|
||||
installPausedVideoCallbacks();
|
||||
let provider: PausedVideoFrameProvider | null = null;
|
||||
const style = document.createElement("style");
|
||||
style.textContent = `[${FRAME_ATTR}] { opacity: 0 !important; }`;
|
||||
const observer = new MutationObserver((records) => {
|
||||
for (const entry of [...presented.values()]) {
|
||||
if (
|
||||
!entry.video.isConnected ||
|
||||
records.some(
|
||||
(record) =>
|
||||
(record.type === "childList" && record.target === entry.video) ||
|
||||
(record.type === "attributes" &&
|
||||
record.attributeName === "src" &&
|
||||
(record.target === entry.video || record.target.parentElement === entry.video)),
|
||||
)
|
||||
)
|
||||
remove(entry);
|
||||
else if (records.some((record) => record.target.contains(entry.video))) refresh(entry);
|
||||
}
|
||||
});
|
||||
observer.observe(document.body, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: true,
|
||||
attributeFilter: ["src", "style", "class", "hidden", "data-hidden"],
|
||||
});
|
||||
|
||||
const clear = (
|
||||
keep?: Set<HTMLVideoElement>,
|
||||
current?: (video: HTMLVideoElement, presented: boolean) => boolean,
|
||||
) => {
|
||||
for (const entry of [...presented.values()]) {
|
||||
if (!keep?.has(entry.video)) {
|
||||
remove(entry);
|
||||
continue;
|
||||
}
|
||||
entry.nativeSkipped = true;
|
||||
const reveal = (nativeFrame: boolean) => {
|
||||
if (presented.get(entry.video) !== entry) return true;
|
||||
if (
|
||||
entry.video.readyState < 2 ||
|
||||
entry.video.seeking ||
|
||||
(current && !current(entry.video, nativeFrame))
|
||||
)
|
||||
return false;
|
||||
remove(entry);
|
||||
return true;
|
||||
};
|
||||
const ready = () => {
|
||||
reveal(false);
|
||||
};
|
||||
const events = ["loadeddata", "canplay", "seeked"];
|
||||
entry.stopWaiting?.();
|
||||
for (const event of events) entry.video.addEventListener(event, ready);
|
||||
const stopFrames = observeNativeVideoFrames(entry.video, () => reveal(true));
|
||||
entry.stopWaiting = () => {
|
||||
for (const event of events) entry.video.removeEventListener(event, ready);
|
||||
stopFrames();
|
||||
};
|
||||
}
|
||||
if (presented.size === 0) style.remove();
|
||||
};
|
||||
return {
|
||||
get active() {
|
||||
// A skipped native clip may keep a fallback picture without owning paused playback.
|
||||
return [...presented.values()].some((entry) => !entry.stopWaiting);
|
||||
},
|
||||
holdNative(videos: HTMLVideoElement[]) {
|
||||
const held: Presented[] = [];
|
||||
for (const video of videos) {
|
||||
if (!nativeSnapshotStyle(video)) continue;
|
||||
const canvas = captureNativeVideoFrame(video);
|
||||
if (!canvas) continue;
|
||||
const entry: Presented = {
|
||||
video,
|
||||
canvas,
|
||||
time: video.currentTime,
|
||||
driving: false,
|
||||
restoreTime: () => {},
|
||||
};
|
||||
held.push(entry);
|
||||
markFrameCanvas(canvas);
|
||||
presented.set(video, entry);
|
||||
video.setAttribute(PREVIEW_VIDEO_SOURCE_ATTR, "");
|
||||
if (!style.isConnected) document.head.append(style);
|
||||
refresh(entry);
|
||||
}
|
||||
return () => {
|
||||
for (const entry of held)
|
||||
if (presented.get(entry.video) === entry && !entry.driving) remove(entry);
|
||||
};
|
||||
},
|
||||
skipped: (el: HTMLMediaElement) => presented.get(el)?.nativeSkipped === true,
|
||||
retained: (el: HTMLMediaElement) => presented.get(el)?.stopWaiting !== undefined,
|
||||
available: () => provider !== null,
|
||||
setProvider(next: PausedVideoFrameProvider | null) {
|
||||
if (provider === next) return;
|
||||
provider?.dispose();
|
||||
provider = next;
|
||||
},
|
||||
async prepare(requests: PausedVideoFrameRequest[], signal: AbortSignal) {
|
||||
if (
|
||||
!provider ||
|
||||
requests.length === 0 ||
|
||||
requests.some(
|
||||
({ video }) =>
|
||||
authoredSeeks.has(video) ||
|
||||
(!presented.has(video) && Object.hasOwn(video, "currentTime")),
|
||||
)
|
||||
)
|
||||
return null;
|
||||
const owner = provider;
|
||||
const identity = (video: HTMLVideoElement) =>
|
||||
JSON.stringify([
|
||||
video.src,
|
||||
...Array.from(video.querySelectorAll("source"), (source) => source.src),
|
||||
]);
|
||||
const sources = requests.map(({ video }) => identity(video));
|
||||
const frames = await owner.decode(requests, signal);
|
||||
let released = false;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
for (const frame of frames) frame.release();
|
||||
};
|
||||
const current = () =>
|
||||
!signal.aborted &&
|
||||
provider === owner &&
|
||||
requests.every(
|
||||
({ video }, index) => video.isConnected && identity(video) === sources[index],
|
||||
);
|
||||
if (!current()) {
|
||||
release();
|
||||
return null;
|
||||
}
|
||||
const valid =
|
||||
frames.reduce((sum, frame) => sum + frame.width * frame.height * 4, 0) <= SURFACE_BUDGET &&
|
||||
frames.length === requests.length &&
|
||||
frames.every((frame, index) => {
|
||||
const time = requests[index]!.time;
|
||||
return (
|
||||
frame.width > 0 &&
|
||||
frame.height > 0 &&
|
||||
frame.timestamp <= time + 1e-5 &&
|
||||
time < frame.timestamp + frame.duration + 1e-5
|
||||
);
|
||||
});
|
||||
if (!valid) {
|
||||
release();
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
valid: current,
|
||||
present() {
|
||||
if (!current()) {
|
||||
release();
|
||||
return false;
|
||||
}
|
||||
const staged: Presented[] = [];
|
||||
const reusable = new Map<Presented, ReusablePresentation>();
|
||||
try {
|
||||
frames.forEach((frame, index) => {
|
||||
const { video, time } = requests[index]!;
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = frame.width;
|
||||
canvas.height = frame.height;
|
||||
staged.push({ video, canvas, time, driving: true, restoreTime: () => {} });
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Preview canvas unavailable");
|
||||
context.drawImage(frame.source, 0, 0, canvas.width, canvas.height);
|
||||
const prior = presented.get(video);
|
||||
if (
|
||||
prior &&
|
||||
prior.canvas.width === frame.width &&
|
||||
prior.canvas.height === frame.height
|
||||
) {
|
||||
const destination = prior.canvas.getContext("2d");
|
||||
if (!destination) throw new Error("Preview canvas unavailable");
|
||||
reusable.set(staged[staged.length - 1]!, { entry: prior, context: destination });
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
for (const { canvas } of staged) canvas.width = canvas.height = 0;
|
||||
return false;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
const wanted = new Set(requests.map(({ video }) => video));
|
||||
for (const entry of [...presented.values()]) if (!wanted.has(entry.video)) remove(entry);
|
||||
if (!style.isConnected) document.head.append(style);
|
||||
for (const [index, next] of staged.entries()) {
|
||||
adoptStagedFrame(next, reusable.get(next));
|
||||
const { timestamp, width, height } = frames[index]!;
|
||||
stagePausedVideoCallbacks(next.video, { mediaTime: timestamp, width, height });
|
||||
}
|
||||
return true;
|
||||
},
|
||||
refresh() {
|
||||
for (const entry of presented.values()) refresh(entry);
|
||||
},
|
||||
release,
|
||||
};
|
||||
},
|
||||
suspendProvider: () => provider?.suspend?.(),
|
||||
prepareNative() {
|
||||
for (const entry of presented.values()) {
|
||||
entry.driving = false;
|
||||
entry.restoreTime();
|
||||
}
|
||||
},
|
||||
notify() {
|
||||
for (const { video } of presented.values()) flushPausedVideoCallbacks(video);
|
||||
},
|
||||
clear,
|
||||
destroy() {
|
||||
observer.disconnect();
|
||||
clear();
|
||||
provider?.dispose();
|
||||
provider = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -302,6 +302,17 @@ describe("createPickerModule", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("picks a video whose decoded preview surface hides the native pixels", () => {
|
||||
const picker = createPickerModule({ postMessage: createMockPostMessage() });
|
||||
picker.installPickerApi();
|
||||
document.body.innerHTML =
|
||||
'<video id="video" data-hf-preview-video-frame style="opacity:0;width:100px;height:100px"></video>';
|
||||
const video = document.querySelector("video")!;
|
||||
expect(candidatesUnder(() => [video])).toContain("#video");
|
||||
video.removeAttribute("data-hf-preview-video-frame");
|
||||
expect(candidatesUnder(() => [video])).not.toContain("#video");
|
||||
});
|
||||
|
||||
it("still picks an image inside a faint texture wrapper", () => {
|
||||
const picker = createPickerModule({ postMessage: createMockPostMessage() });
|
||||
picker.installPickerApi();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { RuntimeJson, RuntimeOutboundMessage, RuntimePickerElementInfo } from "./types";
|
||||
import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark";
|
||||
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
|
||||
import { swallow } from "./diagnostics";
|
||||
import { isElementNode } from "./domRealm";
|
||||
@@ -191,7 +192,8 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule {
|
||||
if (
|
||||
Number.isFinite(opacity) &&
|
||||
opacity <= 0.01 &&
|
||||
!current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)
|
||||
!current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR) &&
|
||||
!current.hasAttribute(PREVIEW_VIDEO_FRAME_ATTR)
|
||||
)
|
||||
return true;
|
||||
current = current.parentElement;
|
||||
|
||||
@@ -211,7 +211,7 @@ export function createRuntimePlayer(deps: PlayerDeps): RuntimePlayer {
|
||||
deps.onRenderFrameSeek(time);
|
||||
deps.onStatePost(true);
|
||||
},
|
||||
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => {
|
||||
seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => {
|
||||
const timeline = deps.getTimeline();
|
||||
if (!timeline) return;
|
||||
const safeTime = Math.max(0, Number(timeSeconds) || 0);
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createPreviewSeek } from "./previewSeek";
|
||||
import type { PreparedPausedVideoFrames } from "./pausedVideoFrames";
|
||||
import { isUnplayable, syncRuntimeMedia } from "./media";
|
||||
import type { RuntimeMediaClip } from "./types";
|
||||
|
||||
const jobs: ReturnType<typeof createPreviewSeek>[] = [];
|
||||
afterEach(() => {
|
||||
for (const job of jobs.splice(0)) job.cancel();
|
||||
document.body.innerHTML = "";
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
const batch = (): PreparedPausedVideoFrames => ({
|
||||
valid: () => true,
|
||||
present: vi.fn(() => true),
|
||||
refresh: vi.fn(),
|
||||
release: vi.fn(),
|
||||
});
|
||||
function fixture() {
|
||||
const video = document.createElement("video");
|
||||
document.body.append(video);
|
||||
const clip: RuntimeMediaClip = {
|
||||
el: video,
|
||||
start: 0,
|
||||
duration: 10,
|
||||
end: 10,
|
||||
mediaStart: 0,
|
||||
playbackRate: 1,
|
||||
volume: null,
|
||||
loop: false,
|
||||
sourceDuration: 10,
|
||||
};
|
||||
const pending: Array<{
|
||||
signal: AbortSignal;
|
||||
resolve: (result: PreparedPausedVideoFrames | null) => void;
|
||||
}> = [];
|
||||
const commit = vi.fn();
|
||||
const native = vi.fn();
|
||||
const prepare = vi.fn(
|
||||
(_clips: RuntimeMediaClip[], _time: number, signal: AbortSignal) =>
|
||||
new Promise<PreparedPausedVideoFrames | null>((resolve) => pending.push({ signal, resolve })),
|
||||
);
|
||||
const host = {
|
||||
clipsAt: () => [clip],
|
||||
videosToHold: (): HTMLVideoElement[] => [],
|
||||
scenesReadyAt: () => true,
|
||||
prepare: native,
|
||||
commit,
|
||||
frames: {
|
||||
active: () => false,
|
||||
available: () => true,
|
||||
prepare,
|
||||
prepareNative: vi.fn(),
|
||||
clear: vi.fn(),
|
||||
},
|
||||
};
|
||||
const job = createPreviewSeek(host);
|
||||
jobs.push(job);
|
||||
return { job, commit, native, prepare, pending, video, clip, host };
|
||||
}
|
||||
|
||||
it("presents coherent intermediate work while retaining the latest requested destination", async () => {
|
||||
const { job, commit, pending } = fixture();
|
||||
const done = job.seek(1);
|
||||
expect(job.seek(2)).toBe(done);
|
||||
const first = batch();
|
||||
pending[0]!.resolve(first);
|
||||
await Promise.resolve();
|
||||
expect(commit).toHaveBeenCalledWith(1);
|
||||
expect(job.time).toBe(2);
|
||||
expect(first.present).toHaveBeenCalledTimes(1);
|
||||
const last = batch();
|
||||
pending[1]!.resolve(last);
|
||||
await done;
|
||||
expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 2]);
|
||||
expect(job.time).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not rebuild document structure when no native snapshots need hiding", async () => {
|
||||
const { job, pending } = fixture();
|
||||
const mutations: MutationRecord[] = [];
|
||||
const observer = new MutationObserver((records) => mutations.push(...records));
|
||||
observer.observe(document.head, { childList: true });
|
||||
const done = job.seek(1);
|
||||
job.seek(2);
|
||||
pending[0]!.resolve(batch());
|
||||
await Promise.resolve();
|
||||
pending[1]!.resolve(batch());
|
||||
await done;
|
||||
expect([...mutations, ...observer.takeRecords()]).toEqual([]);
|
||||
observer.disconnect();
|
||||
});
|
||||
|
||||
it("resumes provider decoding when a scrub interrupts a native play handoff", async () => {
|
||||
const { job, commit, prepare, native, pending } = fixture();
|
||||
const initial = job.seek(1);
|
||||
pending[0]!.resolve(batch());
|
||||
await initial;
|
||||
const play = job.handoff(2);
|
||||
expect(native).toHaveBeenCalledTimes(1);
|
||||
const scrub = job.seek(3);
|
||||
await play;
|
||||
expect(prepare).toHaveBeenLastCalledWith(expect.any(Array), 3, expect.any(AbortSignal), false);
|
||||
pending[1]!.resolve(batch());
|
||||
await scrub;
|
||||
expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 3]);
|
||||
});
|
||||
|
||||
it("releases a decode that arrives after cancellation without presenting it", async () => {
|
||||
const { job, commit, pending } = fixture();
|
||||
const done = job.seek(1);
|
||||
job.cancel();
|
||||
expect(pending[0]!.signal.aborted).toBe(true);
|
||||
const late = batch();
|
||||
pending[0]!.resolve(late);
|
||||
await Promise.resolve();
|
||||
await done;
|
||||
expect(late.release).toHaveBeenCalledTimes(1);
|
||||
expect(late.present).not.toHaveBeenCalled();
|
||||
expect(commit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("lets a new gesture supersede a full-quality refinement without changing its destination", async () => {
|
||||
const { job, prepare, pending, commit } = fixture();
|
||||
const refining = job.refine(1);
|
||||
expect(prepare).toHaveBeenLastCalledWith(expect.any(Array), 1, expect.any(AbortSignal), true);
|
||||
const next = job.seek(2);
|
||||
expect(pending[0]!.signal.aborted).toBe(true);
|
||||
pending[0]!.resolve(batch());
|
||||
pending[1]!.resolve(batch());
|
||||
await Promise.all([refining, next]);
|
||||
expect(commit.mock.calls.map(([time]) => time)).toEqual([2]);
|
||||
});
|
||||
|
||||
it("does not erase a seek requested synchronously while publishing the previous one", async () => {
|
||||
const { job, commit, pending } = fixture();
|
||||
commit.mockImplementationOnce(() => {
|
||||
job.seek(4);
|
||||
});
|
||||
const first = job.seek(1);
|
||||
pending[0]!.resolve(batch());
|
||||
await first;
|
||||
expect(job.time).toBe(4);
|
||||
pending[1]!.resolve(batch());
|
||||
await Promise.resolve();
|
||||
expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 4]);
|
||||
});
|
||||
|
||||
it("keeps a request-wide hard deadline through refresh and same-clip retarget", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { job, commit } = fixture();
|
||||
const done = job.handoff(3);
|
||||
await vi.advanceTimersByTimeAsync(4900);
|
||||
job.refresh();
|
||||
await vi.advanceTimersByTimeAsync(4100);
|
||||
job.seek(4, true);
|
||||
expect(commit).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await done;
|
||||
expect(commit.mock.calls.map(([time]) => time)).toEqual([4]);
|
||||
expect(job.time).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not reset a source already known to be unplayable on another Play", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { job, commit, video, clip } = fixture();
|
||||
video.src = "/stopped.mp4";
|
||||
Object.defineProperties(video, {
|
||||
readyState: { value: 1, configurable: true },
|
||||
networkState: { value: 1, configurable: true },
|
||||
});
|
||||
video.play = vi.fn(async () => {});
|
||||
video.pause = vi.fn();
|
||||
syncRuntimeMedia({
|
||||
clips: [clip],
|
||||
timeSeconds: 0,
|
||||
playing: true,
|
||||
playbackRate: 1,
|
||||
getCompositionDuration: () => 10,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(isUnplayable(video)).toBe(true);
|
||||
const done = job.handoff(3);
|
||||
await Promise.resolve();
|
||||
await done;
|
||||
expect(commit).toHaveBeenLastCalledWith(3);
|
||||
expect(job.time).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves the held native picture when a new gesture cancels expired background work", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
drawImage: vi.fn(),
|
||||
} as unknown as CanvasRenderingContext2D);
|
||||
const { job, video, native, pending, prepare, host } = fixture();
|
||||
let ready = 4;
|
||||
let time = 0;
|
||||
Object.defineProperties(video, {
|
||||
readyState: { get: () => ready },
|
||||
networkState: { value: 2 },
|
||||
videoWidth: { value: 64 },
|
||||
videoHeight: { value: 36 },
|
||||
currentTime: { get: () => time },
|
||||
});
|
||||
host.videosToHold = () => [video];
|
||||
host.frames.active = () => true;
|
||||
native.mockImplementation((_clips, target) => {
|
||||
ready = 1;
|
||||
time = target;
|
||||
});
|
||||
prepare.mockResolvedValueOnce(null);
|
||||
const expired = job.seek(3);
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
await expired;
|
||||
expect(job.time).toBeUndefined();
|
||||
expect(job.targetTime).toBe(3);
|
||||
const picture = document.querySelector("canvas");
|
||||
expect(picture).not.toBeNull();
|
||||
const next = job.seek(4);
|
||||
expect(document.querySelector("canvas")).toBe(picture);
|
||||
pending[0]!.resolve(batch());
|
||||
await next;
|
||||
expect(document.querySelector("canvas")).toBeNull();
|
||||
});
|
||||
@@ -1,35 +1,23 @@
|
||||
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
|
||||
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
|
||||
import { isUnplayable, mediaSourceTimeAt } from "./media";
|
||||
import type { RuntimeMediaClip } from "./types";
|
||||
import { isVideoElement } from "./domRealm";
|
||||
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
|
||||
import {
|
||||
nativeSnapshotStyle,
|
||||
captureNativeVideoFrame,
|
||||
type PreparedPausedVideoFrames,
|
||||
} from "./pausedVideoFrames";
|
||||
|
||||
const HELD_VIDEO_ATTR = "data-hf-seek-video";
|
||||
const SEEK_CAP_MS = 5000;
|
||||
const EVENTS = ["loadedmetadata", "loadeddata", "canplay", "seeked", "error"] as const;
|
||||
|
||||
function videoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = video.videoWidth;
|
||||
canvas.height = video.videoHeight;
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) return null;
|
||||
try {
|
||||
context.drawImage(video, 0, 0);
|
||||
return canvas;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Keep a displayed video in its layout slot while its decoder moves independently of the page. */
|
||||
function holdVideoFrame(video: HTMLVideoElement): (() => void) | undefined {
|
||||
// Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
|
||||
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return;
|
||||
const css = getComputedStyle(video);
|
||||
if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return;
|
||||
if (!video.videoWidth || !video.videoHeight) return;
|
||||
const canvas = videoFrame(video);
|
||||
const css = nativeSnapshotStyle(video);
|
||||
if (!css) return;
|
||||
const canvas = captureNativeVideoFrame(video);
|
||||
if (!canvas) return;
|
||||
for (const name of Array.from(css)) canvas.style.setProperty(name, css.getPropertyValue(name));
|
||||
canvas.style.setProperty("animation", "none", "important");
|
||||
@@ -49,7 +37,7 @@ function holdVideoFrames(videos: HTMLVideoElement[]): () => void {
|
||||
const restore = videos.map(holdVideoFrame);
|
||||
const style = document.createElement("style");
|
||||
style.textContent = `[${HELD_VIDEO_ATTR}] { display: none !important; }`;
|
||||
document.head.appendChild(style);
|
||||
if (restore.some(Boolean)) document.head.appendChild(style);
|
||||
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
|
||||
return () => {
|
||||
for (const putBack of restore) putBack?.();
|
||||
@@ -64,72 +52,226 @@ type SeekHost = {
|
||||
scenesReadyAt: (time: number) => boolean;
|
||||
prepare: (clips: RuntimeMediaClip[], time: number) => void;
|
||||
commit: (time: number) => void;
|
||||
frames?: {
|
||||
available(): boolean;
|
||||
active?(): boolean;
|
||||
skipped?(el: HTMLMediaElement): boolean;
|
||||
hold?(videos: HTMLVideoElement[]): (() => void) | undefined;
|
||||
prepare(
|
||||
clips: RuntimeMediaClip[],
|
||||
time: number,
|
||||
signal: AbortSignal,
|
||||
settled?: boolean,
|
||||
): Promise<PreparedPausedVideoFrames | null>;
|
||||
suspend?(): Promise<void> | undefined;
|
||||
prepareNative(): void;
|
||||
clear(keep?: Set<HTMLVideoElement>): void;
|
||||
};
|
||||
};
|
||||
|
||||
/** One decoder job at a time; requests arriving during it replace the queued destination. */
|
||||
/** One preparation at a time; publish coherent progress and coalesce newer destinations. */
|
||||
export function createPreviewSeek(host: SeekHost) {
|
||||
let pending: { time: number; done: Promise<void>; resolve: () => void } | null = null;
|
||||
let releasePicture = () => {};
|
||||
type SeekRequest = {
|
||||
time: number;
|
||||
native: boolean;
|
||||
settled: boolean;
|
||||
deadline?: ReturnType<typeof wallSetTimeout>;
|
||||
decoderRelease?: Promise<void> | null;
|
||||
force: () => void;
|
||||
done: Promise<void>;
|
||||
resolve: () => void;
|
||||
};
|
||||
let pending: SeekRequest | null = null;
|
||||
// Recovery keeps its listeners without blocking paused transport work.
|
||||
let recovery: SeekRequest | null = null;
|
||||
let releasePicture: (() => void) | null = null;
|
||||
const frameHolds: Array<() => void> = [];
|
||||
let stopWaiting = () => {};
|
||||
let runningClips: RuntimeMediaClip[] = [];
|
||||
let cancelWork = () => {};
|
||||
let committing = false;
|
||||
let committingSettled = false;
|
||||
let runId = 0;
|
||||
|
||||
const needsPreparation = (native: boolean) => native || host.frames?.available();
|
||||
|
||||
const nativeReady = ({ el, ...clip }: RuntimeMediaClip, time: number) =>
|
||||
!el.isConnected ||
|
||||
(el.readyState >= 2 &&
|
||||
!el.seeking &&
|
||||
Math.abs(el.currentTime - mediaSourceTimeAt({ el, ...clip }, time)) < 0.02);
|
||||
const ready = (clips: RuntimeMediaClip[], time: number) =>
|
||||
host.scenesReadyAt(time) &&
|
||||
clips.every((clip) => {
|
||||
const { el } = clip;
|
||||
return (
|
||||
!el.isConnected ||
|
||||
isUnplayable(el) ||
|
||||
(el.readyState >= 2 &&
|
||||
!el.seeking &&
|
||||
Math.abs(el.currentTime - mediaSourceTimeAt(clip, time)) < 0.02)
|
||||
);
|
||||
});
|
||||
clips.every(
|
||||
(clip) => isUnplayable(clip.el) || host.frames?.skipped?.(clip.el) || nativeReady(clip, time),
|
||||
);
|
||||
|
||||
const cancel = () => {
|
||||
const request = pending;
|
||||
pending = null;
|
||||
const releaseHeldPicture = () => {
|
||||
releasePicture?.();
|
||||
releasePicture = null;
|
||||
};
|
||||
const clearFrames = (clips: RuntimeMediaClip[], native: boolean, time: number) =>
|
||||
host.frames?.clear(
|
||||
native
|
||||
? new Set(
|
||||
clips.flatMap((clip) =>
|
||||
isVideoElement(clip.el) && !nativeReady(clip, time) ? [clip.el] : [],
|
||||
),
|
||||
)
|
||||
: undefined,
|
||||
);
|
||||
|
||||
const retireRequest = (request: SeekRequest | null) => {
|
||||
if (pending === request) pending = null;
|
||||
if (recovery === request) recovery = null;
|
||||
wallClearTimeout(request?.deadline);
|
||||
};
|
||||
const holdPicture = () => {
|
||||
const videos = host.videosToHold();
|
||||
const release = host.frames?.hold?.(videos);
|
||||
if (release) frameHolds.push(release);
|
||||
releasePicture ??= holdVideoFrames(videos);
|
||||
};
|
||||
const cancel = (keepPicture = false) => {
|
||||
const request = pending ?? recovery;
|
||||
retireRequest(request);
|
||||
runId++;
|
||||
stopWaiting();
|
||||
stopWaiting = () => {};
|
||||
releasePicture();
|
||||
releasePicture = () => {};
|
||||
cancelWork();
|
||||
cancelWork = () => {};
|
||||
if (!keepPicture) {
|
||||
releaseHeldPicture();
|
||||
for (const release of frameHolds.splice(0)) release();
|
||||
}
|
||||
request?.resolve();
|
||||
};
|
||||
|
||||
const afterDecoderRelease = (request: SeekRequest, start: () => void) => {
|
||||
if (request.decoderRelease === undefined) {
|
||||
try {
|
||||
request.decoderRelease = host.frames?.suspend?.() ?? null;
|
||||
} catch {
|
||||
request.decoderRelease = null;
|
||||
}
|
||||
}
|
||||
if (request.decoderRelease) void request.decoderRelease.then(start, start);
|
||||
else start();
|
||||
};
|
||||
|
||||
const run = () => {
|
||||
const request = pending;
|
||||
const request = pending ?? recovery;
|
||||
if (!request) return;
|
||||
cancelWork();
|
||||
const id = ++runId;
|
||||
const time = request.time;
|
||||
const clips = host.clipsAt(time);
|
||||
runningClips = clips;
|
||||
const selection = clips.map((clip) => ({ el: clip.el, time: mediaSourceTimeAt(clip, time) }));
|
||||
const controller = new AbortController();
|
||||
let frame: PreparedPausedVideoFrames | null = null;
|
||||
let preparing = true;
|
||||
let nativePhase = false;
|
||||
let finished = false;
|
||||
const live = () => (pending === request || recovery === request) && runId === id && !finished;
|
||||
cancelWork = () => {
|
||||
controller.abort();
|
||||
frame?.release();
|
||||
};
|
||||
|
||||
const publish = () => {
|
||||
frameHolds.length = 0;
|
||||
releaseHeldPicture();
|
||||
if (!frame) clearFrames(host.clipsAt(time), request.native, time);
|
||||
committing = true;
|
||||
committingSettled = request.settled;
|
||||
try {
|
||||
host.commit(time);
|
||||
frame?.refresh();
|
||||
} finally {
|
||||
committing = false;
|
||||
committingSettled = false;
|
||||
}
|
||||
};
|
||||
const selectionChanged = () => {
|
||||
const current = host.clipsAt(time);
|
||||
return (
|
||||
current.length !== selection.length ||
|
||||
current.some(
|
||||
(clip, index) =>
|
||||
clip.el !== selection[index]?.el ||
|
||||
Math.abs(mediaSourceTimeAt(clip, time) - selection[index]!.time) > 1e-8,
|
||||
)
|
||||
);
|
||||
};
|
||||
const frameReadyToPublish = () => {
|
||||
if (!frame) return true;
|
||||
if (selectionChanged()) {
|
||||
stopWaiting();
|
||||
run();
|
||||
return false;
|
||||
}
|
||||
if (!frame.valid() || !frame.present()) {
|
||||
native();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
const finish = () => {
|
||||
if (finished || pending !== request || id !== runId) return;
|
||||
if (!live() || !frameReadyToPublish()) return;
|
||||
finished = true;
|
||||
stopWaiting();
|
||||
stopWaiting = () => {};
|
||||
if (request.time !== time) {
|
||||
if (ready(host.clipsAt(time), time)) {
|
||||
releasePicture();
|
||||
host.commit(time);
|
||||
releasePicture = holdVideoFrames(host.videosToHold());
|
||||
if (frame || ready(host.clipsAt(time), time)) {
|
||||
publish();
|
||||
if (pending !== request && recovery !== request) return;
|
||||
holdPicture();
|
||||
}
|
||||
run();
|
||||
return;
|
||||
}
|
||||
cancel();
|
||||
host.commit(time);
|
||||
retireRequest(request);
|
||||
runId++;
|
||||
const cleanup = cancelWork;
|
||||
cancelWork = () => {};
|
||||
try {
|
||||
publish();
|
||||
} finally {
|
||||
cleanup();
|
||||
request.resolve();
|
||||
}
|
||||
};
|
||||
request.force = () => {
|
||||
if (!live()) return;
|
||||
if (request.time === time) finish();
|
||||
else {
|
||||
stopWaiting();
|
||||
run();
|
||||
request.force();
|
||||
}
|
||||
};
|
||||
const check = () => {
|
||||
if (!preparing && ready(host.clipsAt(time), time)) finish();
|
||||
if (preparing || !live()) return;
|
||||
if (frame ? host.scenesReadyAt(time) : ready(host.clipsAt(time), time)) finish();
|
||||
};
|
||||
const native = () => {
|
||||
if (!live() || nativePhase) return;
|
||||
nativePhase = true;
|
||||
controller.abort();
|
||||
frame?.release();
|
||||
frame = null;
|
||||
host.frames?.prepareNative();
|
||||
host.prepare(host.clipsAt(time), time);
|
||||
preparing = false;
|
||||
check();
|
||||
};
|
||||
const metadata = () => {
|
||||
if (pending !== request || finished || id !== runId) return;
|
||||
// A source/proxy change resets currentTime; the parked clock still describes the old picture.
|
||||
if (!live()) return;
|
||||
if (!nativePhase) {
|
||||
check();
|
||||
return;
|
||||
}
|
||||
preparing = true;
|
||||
host.prepare(host.clipsAt(time), time);
|
||||
preparing = false;
|
||||
@@ -147,51 +289,106 @@ export function createPreviewSeek(host: SeekHost) {
|
||||
);
|
||||
};
|
||||
});
|
||||
const cap = wallSetTimeout(finish, SEEK_CAP_MS);
|
||||
const expire = () => {
|
||||
if (!live()) return;
|
||||
if (request.native) check();
|
||||
else if (!nativePhase && !frame) {
|
||||
native();
|
||||
if (live()) cap = wallSetTimeout(expire, SEEK_CAP_MS);
|
||||
} else if (host.frames?.active?.()) {
|
||||
pending = null;
|
||||
recovery = request;
|
||||
request.resolve();
|
||||
} else finish();
|
||||
};
|
||||
let cap = wallSetTimeout(expire, SEEK_CAP_MS);
|
||||
stopWaiting = () => {
|
||||
wallClearTimeout(cap);
|
||||
for (const remove of listeners) remove();
|
||||
};
|
||||
host.prepare(clips, time);
|
||||
preparing = false;
|
||||
check();
|
||||
if (request.native) afterDecoderRelease(request, native);
|
||||
else if (recovery !== request && host.frames && needsPreparation(request.native)) {
|
||||
void host.frames
|
||||
.prepare(clips, time, controller.signal, request.settled)
|
||||
.then((decoded) => {
|
||||
if (!live() || controller.signal.aborted) {
|
||||
decoded?.release();
|
||||
return;
|
||||
}
|
||||
if (!decoded) return native();
|
||||
frame = decoded;
|
||||
preparing = false;
|
||||
check();
|
||||
})
|
||||
.catch(native);
|
||||
} else native();
|
||||
};
|
||||
|
||||
const retarget = (time: number) => {
|
||||
const request = pending!;
|
||||
request.time = time;
|
||||
const clips = host.clipsAt(time);
|
||||
if (
|
||||
clips.length !== runningClips.length ||
|
||||
clips.some((clip, index) => clip.el !== runningClips[index]?.el)
|
||||
) {
|
||||
stopWaiting();
|
||||
run();
|
||||
}
|
||||
return request.done;
|
||||
};
|
||||
const seek = (time: number, native = false, settled = false): Promise<void> | void => {
|
||||
if (recovery) cancel(true);
|
||||
if (pending && (pending.native !== native || pending.settled !== settled)) cancel(true);
|
||||
if (pending) return retarget(time);
|
||||
if (!needsPreparation(native) && ready(host.clipsAt(time), time)) {
|
||||
releaseHeldPicture();
|
||||
clearFrames(host.clipsAt(time), native, time);
|
||||
host.commit(time);
|
||||
return;
|
||||
}
|
||||
let resolve = () => {};
|
||||
const done = new Promise<void>((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
const request = { time, native, settled, done, resolve, force: () => {} };
|
||||
pending = request;
|
||||
// Refreshing a run must not restart the wall-clock budget for Play.
|
||||
if (native) pending.deadline = wallSetTimeout(() => request.force(), SEEK_CAP_MS * 2);
|
||||
holdPicture();
|
||||
run();
|
||||
return done;
|
||||
};
|
||||
return {
|
||||
get committing() {
|
||||
return committing;
|
||||
},
|
||||
get refining() {
|
||||
return committingSettled;
|
||||
},
|
||||
get time() {
|
||||
return pending?.time;
|
||||
},
|
||||
seek(time: number): Promise<void> | void {
|
||||
if (pending) {
|
||||
const request = pending;
|
||||
request.time = time;
|
||||
const clips = host.clipsAt(time);
|
||||
if (
|
||||
clips.length !== runningClips.length ||
|
||||
clips.some((clip, index) => clip.el !== runningClips[index]?.el)
|
||||
) {
|
||||
stopWaiting();
|
||||
run();
|
||||
}
|
||||
return request.done;
|
||||
}
|
||||
if (ready(host.clipsAt(time), time)) {
|
||||
host.commit(time);
|
||||
return;
|
||||
}
|
||||
let resolve = () => {};
|
||||
const done = new Promise<void>((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
pending = { time, done, resolve };
|
||||
releasePicture = holdVideoFrames(host.videosToHold());
|
||||
run();
|
||||
return done;
|
||||
get targetTime() {
|
||||
return (pending ?? recovery)?.time;
|
||||
},
|
||||
ownsRecovery(el: HTMLMediaElement) {
|
||||
return recovery !== null && runningClips.some((clip) => clip.el === el);
|
||||
},
|
||||
seek,
|
||||
refine(time: number) {
|
||||
return seek(time, false, true);
|
||||
},
|
||||
refresh() {
|
||||
if (!pending) return;
|
||||
stopWaiting();
|
||||
run();
|
||||
if (pending || recovery) {
|
||||
stopWaiting();
|
||||
run();
|
||||
}
|
||||
},
|
||||
handoff(time: number) {
|
||||
cancel(true);
|
||||
host.frames?.prepareNative();
|
||||
return seek(time, true);
|
||||
},
|
||||
cancel,
|
||||
};
|
||||
|
||||
@@ -337,7 +337,7 @@ export type RuntimePlayer = {
|
||||
_timeline: RuntimeTimelineLike | null;
|
||||
play: () => void;
|
||||
pause: () => void;
|
||||
seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
|
||||
seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => HeldSeek;
|
||||
renderSeek: (timeSeconds: number, options?: RuntimeSeekOptions) => void;
|
||||
getTime: () => number;
|
||||
getDuration: () => number;
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import type { PlayerAPI } from "../core.types";
|
||||
import type { ClipTree } from "./clipTree";
|
||||
import type { CompositionLengthSource } from "./compositionLength";
|
||||
import type { SvgSelectorAliases } from "../compiler/svgSelectorAliases";
|
||||
import type { PausedVideoFrameProvider } from "./pausedVideoFrames";
|
||||
|
||||
type ThreeClockLike = {
|
||||
elapsedTime: number;
|
||||
@@ -79,6 +80,7 @@ declare global {
|
||||
* runtime's paused-side enforcement leaves it alone. Always release. */
|
||||
leasePausedMedia?: (el: HTMLMediaElement) => void;
|
||||
releasePausedMedia?: (el: HTMLMediaElement) => void;
|
||||
setPausedVideoFrameProvider?: (provider: PausedVideoFrameProvider | null) => void;
|
||||
/** Where Studio's loop wraps to while it plays, or null: the clips due there stay loaded. */
|
||||
setLoopStart?: (seconds: number | null) => void;
|
||||
/** Read-only level taps for the Studio meters: nothing exists until
|
||||
|
||||
@@ -2,6 +2,9 @@ import { isWebKit } from "./runtime/mediaClock.js";
|
||||
|
||||
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
|
||||
export const PREVIEW_SEEK_HOLD_ATTR = "data-hf-preview-seek-hold";
|
||||
export const PREVIEW_VIDEO_FRAME_ATTR = "data-hf-preview-video-frame";
|
||||
export const PREVIEW_VIDEO_SOURCE_ATTR = "data-hf-preview-video-source";
|
||||
export const PREVIEW_VIDEO_CANVAS_ATTR = "data-hf-preview-video-canvas";
|
||||
export const STUDIO_PREVIEW_CAPTURE_MODE = "capture";
|
||||
export const PLAYER_FRAME_ATTR = "data-hf-player-frame";
|
||||
|
||||
@@ -31,6 +34,9 @@ export const CLIP_HIDDEN_ATTR = "data-hf-clip-hidden";
|
||||
export const STUDIO_PREVIEW_FAR_SCENE_ATTR = "data-hf-far-scene";
|
||||
export const STUDIO_PREVIEW_PARKED_SCENE_ATTR = "data-hf-parked-scene";
|
||||
export const PREVIEW_ONLY_ATTRS = [
|
||||
PREVIEW_VIDEO_FRAME_ATTR,
|
||||
PREVIEW_VIDEO_SOURCE_ATTR,
|
||||
PREVIEW_VIDEO_CANVAS_ATTR,
|
||||
STUDIO_PREVIEW_UPCOMING_ATTR,
|
||||
STUDIO_PREVIEW_FAR_SCENE_ATTR,
|
||||
STUDIO_PREVIEW_PARKED_SCENE_ATTR,
|
||||
|
||||
@@ -168,6 +168,20 @@ describe("probeMediaMetadata", () => {
|
||||
.source?.bitDepth,
|
||||
).toBe(12);
|
||||
expect((await probe({ pix_fmt: "yuv420p" })).source).toBeUndefined();
|
||||
expect(
|
||||
(
|
||||
await probe({
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
r_frame_rate: "30000/1001",
|
||||
avg_frame_rate: "30000/1001",
|
||||
})
|
||||
).source,
|
||||
).toMatchObject({ frameRate: 30000 / 1001, variableFrameRate: false });
|
||||
expect(
|
||||
(await probe({ width: 1920, height: 1080, r_frame_rate: "60/1", avg_frame_rate: "24/1" }))
|
||||
.source,
|
||||
).toMatchObject({ frameRate: 60, variableFrameRate: true });
|
||||
});
|
||||
|
||||
it("lists every stream with its flags, for a failure report", async () => {
|
||||
|
||||
@@ -101,6 +101,8 @@ interface FfprobeStream {
|
||||
width?: number;
|
||||
height?: number;
|
||||
bit_rate?: string;
|
||||
r_frame_rate?: string;
|
||||
avg_frame_rate?: string;
|
||||
disposition?: Record<string, number | undefined>;
|
||||
tags?: Record<string, string | undefined>;
|
||||
}
|
||||
@@ -226,11 +228,25 @@ function sourceFactsOf(
|
||||
): PreviewSourceFacts | undefined {
|
||||
if (!stream?.width || !stream.height) return undefined;
|
||||
const bitrate = Number(stream.bit_rate ?? formatBitrate);
|
||||
const rate = (raw: string | undefined) => {
|
||||
const [numerator, denominator = "1"] = (raw ?? "0").split("/");
|
||||
const value = Number(numerator) / Number(denominator);
|
||||
return Number.isFinite(value) && value > 0 ? value : 0;
|
||||
};
|
||||
const nominal = rate(stream.r_frame_rate);
|
||||
const average = rate(stream.avg_frame_rate);
|
||||
return {
|
||||
width: stream.width,
|
||||
height: stream.height,
|
||||
bitDepth: bitDepthOf(stream),
|
||||
bitrate: Number.isFinite(bitrate) ? bitrate : 0,
|
||||
...(nominal > 0
|
||||
? {
|
||||
frameRate: nominal,
|
||||
variableFrameRate:
|
||||
average > 0 && Math.abs(nominal - average) / Math.max(nominal, average) > 0.1,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -317,7 +333,7 @@ export async function probeMediaMetadata(
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample,width,height,bit_rate:stream_disposition:stream_tags=alpha_mode:format=duration,bit_rate",
|
||||
"stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample,width,height,bit_rate,r_frame_rate,avg_frame_rate:stream_disposition:stream_tags=alpha_mode:format=duration,bit_rate",
|
||||
"-of",
|
||||
"json",
|
||||
"--",
|
||||
|
||||
@@ -33,7 +33,10 @@ async function loadHelper(
|
||||
}> {
|
||||
vi.resetModules();
|
||||
const resolveProxy = vi.fn(async () => "/unused-prewarm-proxy-path");
|
||||
vi.doMock("./proxyTranscoder.js", () => ({ resolveProxy, PROXY_PARAMS_VERSION: "v1" }));
|
||||
vi.doMock("./proxyTranscoder.js", () => ({
|
||||
resolveProxy,
|
||||
PROXY_PARAMS_VERSION: "v1",
|
||||
}));
|
||||
// The real codec table and pre-warm gate; only the ffprobe-backed scan is
|
||||
// replaced, so this asserts the shipped policy rather than a copy of it.
|
||||
vi.doMock("./mediaCodecMap.js", async () => ({
|
||||
@@ -172,19 +175,20 @@ describe("injectMediaCodecMapIntoHtml pre-warm", () => {
|
||||
});
|
||||
|
||||
describe("injectMediaCodecMapIntoHtml placement", () => {
|
||||
it("puts the map before the document's own </head>, not inside an inlined script that prints one", async () => {
|
||||
it("puts facts before runtime and authored head scripts without confusing script contents for document tags", async () => {
|
||||
const { injectMediaCodecMapIntoHtml } = await loadHelper({
|
||||
"/audio/seg01.webm": hostile("vp9", 'video/webm; codecs="vp09.00.10.08"'),
|
||||
});
|
||||
const vendor = 'p.print("<head>"),p.print("</head>"),p.print("<body>")';
|
||||
|
||||
const html = await injectMediaCodecMapIntoHtml(
|
||||
`<html><head><script>${vendor}</script></head><body></body></html>`,
|
||||
`<html><head><script src="/api/runtime.js"></script><script>${vendor}</script></head><body></body></html>`,
|
||||
tmpProject(),
|
||||
[{ html: "<html></html>" }],
|
||||
);
|
||||
|
||||
expect(html).toContain(`<script>${vendor}</script>`);
|
||||
expect(html.indexOf("data-hf-media-codec-map")).toBeGreaterThan(html.indexOf(vendor));
|
||||
expect(html.indexOf("data-hf-media-codec-map")).toBeLessThan(html.indexOf("/api/runtime.js"));
|
||||
expect(html.indexOf("data-hf-media-codec-map")).toBeLessThan(html.indexOf(vendor));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { resolve } from "node:path";
|
||||
import { formatPreviewProxyBox, type PreviewProxyBox } from "@hyperframes/core";
|
||||
import { insertBeforeCloseTag } from "@hyperframes/core/compiler/html-document";
|
||||
import { injectTagsAtHeadStart } from "@hyperframes/core/compiler/html-document";
|
||||
import type { StudioApiAdapter } from "../types.js";
|
||||
import {
|
||||
createMediaCodecProbeCache,
|
||||
@@ -134,7 +134,7 @@ export async function injectMediaCodecMapIntoHtml(
|
||||
`window.__HF_MEDIA_SOURCE_FACTS__=${safeJson(sourceFacts)};`,
|
||||
];
|
||||
const tag = `<script data-hf-media-codec-map>${assignments.join("")}</script>`;
|
||||
return insertBeforeCloseTag(html, "head", `${tag}\n`) ?? `${tag}\n${html}`;
|
||||
return injectTagsAtHeadStart(html, tag);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -282,6 +282,24 @@ describe("registerPreviewRoutes", () => {
|
||||
expect(html).toContain("window.AUTHOR_SEEN = 1");
|
||||
});
|
||||
|
||||
it("keeps the accelerated-canvas tracker before authored scripts in blur previews", async () => {
|
||||
const projectDir = createProjectDir();
|
||||
writeFileSync(
|
||||
join(projectDir, "index.html"),
|
||||
'<html><head><script>window.AUTHOR_CANVAS = 1</script></head><body data-motion-blur="on"></body></html>',
|
||||
);
|
||||
const app = new Hono();
|
||||
registerPreviewRoutes(app, createAdapter(projectDir));
|
||||
const html = await (await app.request("http://localhost/projects/demo/preview")).text();
|
||||
expect(html.indexOf("data-hf-accel-tracker")).toBeGreaterThan(-1);
|
||||
expect(html.indexOf("data-hf-accel-tracker")).toBeLessThan(
|
||||
html.indexOf("window.AUTHOR_CANVAS"),
|
||||
);
|
||||
expect(html.indexOf('src="/api/runtime.js"')).toBeLessThan(
|
||||
html.indexOf("window.AUTHOR_CANVAS"),
|
||||
);
|
||||
});
|
||||
|
||||
it("serves one preview runtime when the page on disk already links one", async () => {
|
||||
const projectDir = createProjectDir();
|
||||
writeFileSync(
|
||||
@@ -1626,6 +1644,71 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {
|
||||
return import("./preview.js");
|
||||
}
|
||||
|
||||
it.each(["bundled", "disk after null bundle", "disk after failed bundle"])(
|
||||
"installs preview hooks before authored head consumers with flags, facts and error capture already present (%s)",
|
||||
async (path) => {
|
||||
const projectDir = createProjectDir();
|
||||
const authored = `<!doctype html><html><head>
|
||||
<script data-hf-accel-tracker>window.TRACKER_KEPT = true;</script>
|
||||
<script data-hyperframes-preview-runtime="1" src="/stale-runtime.js"></script>
|
||||
<script id="head-consumer">window.HEAD_CONSUMER = document.createElement("video").requestVideoFrameCallback;</script>
|
||||
<script id="head-library" src="assets/head-library.js"></script>
|
||||
</head><body><main data-composition-id="main" data-duration="2" data-motion-blur="on"></main>
|
||||
<script id="body-consumer">window.BODY_CONSUMER = true;</script>
|
||||
</body></html>`;
|
||||
writeFileSync(join(projectDir, "index.html"), authored);
|
||||
writeFileSync(
|
||||
join(projectDir, "scene.html"),
|
||||
`<template><main data-composition-id="scene" data-duration="2"></main><script id="body-consumer">window.BODY_CONSUMER = true;</script></template>`,
|
||||
);
|
||||
const { registerPreviewRoutes: register } = await loadPreviewModule({
|
||||
scanMapImpl: async () => ({
|
||||
"/clip.webm": {
|
||||
codecName: "vp9",
|
||||
browserHostile: true,
|
||||
representativeMime: 'video/webm; codecs="vp09.00.10.08"',
|
||||
},
|
||||
}),
|
||||
});
|
||||
const app = new Hono();
|
||||
register(
|
||||
app,
|
||||
createAdapter(projectDir, {
|
||||
bundle: async () => {
|
||||
if (path === "disk after failed bundle") throw new Error("bundler unavailable");
|
||||
return path === "bundled" ? authored : null;
|
||||
},
|
||||
}),
|
||||
);
|
||||
for (const suffix of ["", "/comp/scene.html"]) {
|
||||
const response = await app.request("http://localhost/projects/demo/preview" + suffix);
|
||||
expect(response.status).toBe(200);
|
||||
const html = await response.text();
|
||||
const runtime = html.indexOf('src="/api/runtime.js"');
|
||||
expect(runtime).toBeGreaterThan(-1);
|
||||
for (const early of [
|
||||
`name="${STUDIO_PREVIEW_MARK_META}"`,
|
||||
`window.${STUDIO_PREVIEW_ERRORS}=`,
|
||||
"window.__HF_MEDIA_CODEC_MAP__=",
|
||||
"window.__HF_MEDIA_SOURCE_FACTS__=",
|
||||
]) {
|
||||
expect(html.indexOf(early)).toBeGreaterThan(-1);
|
||||
expect(html.indexOf(early)).toBeLessThan(runtime);
|
||||
}
|
||||
for (const authoredId of ["head-consumer", "head-library", "body-consumer"])
|
||||
expect(runtime).toBeLessThan(html.indexOf(`id="${authoredId}"`));
|
||||
expect(html).not.toContain("stale-runtime.js");
|
||||
expect(
|
||||
parseHTML(html).document.querySelectorAll("script[data-hyperframes-preview-runtime]"),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
parseHTML(html).document.querySelectorAll("script[data-hf-accel-tracker]"),
|
||||
).toHaveLength(1);
|
||||
expect(html).toContain("window.TRACKER_KEPT = true;");
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
describe("?hf-proxy=h264 on the static asset route", () => {
|
||||
it("serves proxy bytes with Accept-Ranges on a full request, and a 206 range slice on a Range request", async () => {
|
||||
const projectDir = createProjectDir();
|
||||
|
||||
@@ -150,10 +150,10 @@ function parseStudioMotionManifestContent(content: string): {
|
||||
}
|
||||
}
|
||||
|
||||
/** Swaps any runtime already in the page for the preview runtime, placed as the bundler places it. */
|
||||
/** Preview hooks must install before authored head scripts register media consumers. */
|
||||
function withPreviewRuntime(html: string, runtimeUrl: string): string {
|
||||
const tag = `<script ${RUNTIME_BOOTSTRAP_ATTR}="1" src="${runtimeUrl}"></script>`;
|
||||
return insertRuntimeTag(stripEmbeddedRuntimeScripts(html), tag);
|
||||
return insertRuntimeTag(stripEmbeddedRuntimeScripts(html), tag, true);
|
||||
}
|
||||
|
||||
function injectScriptTagIntoHead(html: string, scriptTag: string): string {
|
||||
@@ -685,7 +685,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
|
||||
gsapBaseUrl,
|
||||
);
|
||||
if (!html) return c.text("not found", 404);
|
||||
html = withRootImportMap(html, baseHref);
|
||||
html = withPreviewRuntime(withRootImportMap(html, baseHref), adapter.runtimeUrl);
|
||||
recordPreviewReferences(project.dir, html);
|
||||
html = ensureHfIds(await transformPreviewHtml(html, adapter, project, compPath));
|
||||
html = injectStudioPreviewAugmentations(html, adapter, signature, project.dir, compPath, {
|
||||
|
||||
@@ -114,7 +114,7 @@
|
||||
"cn": "0.2.4",
|
||||
"dockview-react": "8.3.1",
|
||||
"gsap": "^3.13.0",
|
||||
"mediabunny": "^1.45.3"
|
||||
"mediabunny": "^1.65.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@hyperframes/producer": "workspace:*",
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
// @vitest-environment jsdom
|
||||
import { afterEach, expect, it } from "vitest";
|
||||
import {
|
||||
PREVIEW_VIDEO_FRAME_ATTR,
|
||||
PREVIEW_VIDEO_CANVAS_ATTR,
|
||||
} from "@hyperframes/core/studio-preview-mark";
|
||||
import { getCuratedComputedStyles, isElementVisibleThroughAncestors } from "./domEditingDom";
|
||||
import {
|
||||
collectDomEditLayerItems,
|
||||
resolveDomEditSelection,
|
||||
resolveVisualDomEditSelectionTarget,
|
||||
} from "./domEditing";
|
||||
|
||||
const options = { activeCompositionPath: "index.html", isMasterView: true, skipSourceProbe: true };
|
||||
afterEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
document.head.innerHTML = "";
|
||||
});
|
||||
|
||||
function preview(opacity: string) {
|
||||
document.head.innerHTML = `<style>[${PREVIEW_VIDEO_FRAME_ATTR}] { opacity: 0 !important; }</style>`;
|
||||
document.body.innerHTML = `<main data-composition-id="main"><canvas ${PREVIEW_VIDEO_CANVAS_ATTR} style="opacity:${opacity}" data-hf-ignore></canvas><video id="clip" ${PREVIEW_VIDEO_FRAME_ATTR} style="opacity:${opacity}" src="clip.mp4"></video></main>`;
|
||||
const video = document.querySelector("video")!;
|
||||
const canvas = document.querySelector("canvas")!;
|
||||
const root = document.querySelector("main")!;
|
||||
Object.defineProperty(video, "getBoundingClientRect", {
|
||||
value: () => ({
|
||||
x: 10,
|
||||
y: 20,
|
||||
left: 10,
|
||||
top: 20,
|
||||
right: 210,
|
||||
bottom: 120,
|
||||
width: 200,
|
||||
height: 100,
|
||||
}),
|
||||
});
|
||||
return { video, canvas, root };
|
||||
}
|
||||
|
||||
it("keeps a provider-backed video selectable and listed with its logical opacity", async () => {
|
||||
const { video, canvas, root } = preview("0.65");
|
||||
expect(getComputedStyle(video).opacity).toBe("0");
|
||||
expect(isElementVisibleThroughAncestors(video)).toBe(true);
|
||||
expect(resolveVisualDomEditSelectionTarget([canvas, video], options)).toBe(video);
|
||||
expect(collectDomEditLayerItems(root, options).map((item) => item.element)).toEqual([video]);
|
||||
const selected = await resolveDomEditSelection(video, options);
|
||||
expect(selected?.computedStyles.opacity).toBe("0.65");
|
||||
expect(selected?.inlineStyles.opacity).toBe("0.65");
|
||||
expect(getComputedStyle(video).opacity).toBe("0");
|
||||
});
|
||||
|
||||
it("keeps authored zero opacity invisible and reports zero to the inspector", async () => {
|
||||
const { video, canvas } = preview("0");
|
||||
expect(isElementVisibleThroughAncestors(video)).toBe(false);
|
||||
expect(resolveVisualDomEditSelectionTarget([canvas, video], options)).toBeNull();
|
||||
expect(getCuratedComputedStyles(video).opacity).toBe("0");
|
||||
const selected = await resolveDomEditSelection(video, options);
|
||||
expect(selected?.computedStyles.opacity).toBe("0");
|
||||
expect(video.style.opacity).toBe("0");
|
||||
});
|
||||
|
||||
it("still respects a hidden ancestor when the preview frame itself is visible", () => {
|
||||
const { video, root } = preview("0.65");
|
||||
root.style.opacity = "0";
|
||||
expect(isElementVisibleThroughAncestors(video)).toBe(false);
|
||||
expect(resolveVisualDomEditSelectionTarget([video], options)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not treat an unrelated preceding canvas as the video's visible frame", () => {
|
||||
const { video, canvas } = preview("0.65");
|
||||
canvas.removeAttribute(PREVIEW_VIDEO_CANVAS_ATTR);
|
||||
expect(isElementVisibleThroughAncestors(video)).toBe(false);
|
||||
expect(getCuratedComputedStyles(video).opacity).toBe("0");
|
||||
});
|
||||
@@ -3,6 +3,10 @@
|
||||
* selector utilities, and composition source resolution.
|
||||
* No imports from other domEditing* modules — safe to import from anywhere.
|
||||
*/
|
||||
import {
|
||||
PREVIEW_VIDEO_FRAME_ATTR,
|
||||
PREVIEW_VIDEO_CANVAS_ATTR,
|
||||
} from "@hyperframes/core/studio-preview-mark";
|
||||
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading";
|
||||
import { getSourceScopedSelectorIndex } from "../../utils/sourceScopedSelectorIndex";
|
||||
import { CURATED_STYLE_PROPERTIES } from "./domEditingTypes";
|
||||
@@ -34,12 +38,24 @@ export function isEditableTextLeaf(el: HTMLElement): boolean {
|
||||
return isTextBearingTag(el.tagName.toLowerCase()) && el.children.length === 0;
|
||||
}
|
||||
|
||||
/** The preview canvas keeps the source's authored opacity while its native pixels are hidden. */
|
||||
export function getLogicalOpacity(el: Element, computed: CSSStyleDeclaration): string {
|
||||
const overlay = el.previousElementSibling;
|
||||
if (
|
||||
el.hasAttribute(PREVIEW_VIDEO_FRAME_ATTR) &&
|
||||
overlay?.localName === "canvas" &&
|
||||
overlay.hasAttribute(PREVIEW_VIDEO_CANVAS_ATTR)
|
||||
)
|
||||
return el.ownerDocument.defaultView?.getComputedStyle(overlay).opacity ?? computed.opacity;
|
||||
return computed.opacity;
|
||||
}
|
||||
|
||||
/** Does this node render AT ALL, ignoring what it inherits? Sole owner of the
|
||||
* rule; the walk below only decides which nodes to ask it about. */
|
||||
function elementRendersItself(win: Window, el: HTMLElement): boolean {
|
||||
const computed = win.getComputedStyle(el);
|
||||
if (computed.display === "none" || computed.visibility === "hidden") return false;
|
||||
const opacity = Number.parseFloat(computed.opacity);
|
||||
const opacity = Number.parseFloat(getLogicalOpacity(el, computed));
|
||||
return !(
|
||||
Number.isFinite(opacity) &&
|
||||
opacity <= 0.01 &&
|
||||
@@ -101,7 +117,8 @@ export function getCuratedComputedStyles(el: HTMLElement): Record<string, string
|
||||
if (!computed) return styles;
|
||||
|
||||
for (const prop of CURATED_STYLE_PROPERTIES) {
|
||||
const value = computed.getPropertyValue(prop);
|
||||
const value =
|
||||
prop === "opacity" ? getLogicalOpacity(el, computed) : computed.getPropertyValue(prop);
|
||||
if (value) styles[prop] = value;
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,10 @@ export interface NLEContextValue {
|
||||
play: () => void;
|
||||
pause: () => void;
|
||||
togglePlay: () => void;
|
||||
seek: (time: number, options?: { keepPlaying?: boolean; follow?: boolean }) => boolean;
|
||||
seek: (
|
||||
time: number,
|
||||
options?: { keepPlaying?: boolean; follow?: boolean; settled?: boolean },
|
||||
) => boolean;
|
||||
refreshPlayer: () => void;
|
||||
onIframeLoad: () => void;
|
||||
// The hidden reload iframe NLEPreview renders next to the live one during a full reload.
|
||||
|
||||
@@ -115,6 +115,19 @@ function mountPicker(
|
||||
};
|
||||
}
|
||||
|
||||
it.each(["0.65", "0"])(
|
||||
"reads provider-backed video opacity %s in the picked-element inspector",
|
||||
(opacity) => {
|
||||
const mounted = `<style>[data-hf-preview-video-frame] { opacity: 0 !important; }</style><canvas data-hf-preview-video-canvas style="opacity:${opacity}"></canvas><video id="provider-video" data-hf-preview-video-frame style="opacity:${opacity}"></video>`;
|
||||
const { iframe, picker } = mountPicker({ "index.html": SAVED }, "#provider-video", mounted);
|
||||
expect(
|
||||
iframe.contentWindow!.getComputedStyle(iframe.contentDocument!.querySelector("video")!)
|
||||
.opacity,
|
||||
).toBe("0");
|
||||
expect(picker().pickedElement?.computedStyles.opacity).toBe(opacity);
|
||||
},
|
||||
);
|
||||
|
||||
describe("pick mode cancellation and hover feedback", () => {
|
||||
it("cancels pick mode when Escape is pressed with Studio focus", () => {
|
||||
const { picker } = mountPicker({ "index.html": SAVED });
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
postRuntimeControlMessage,
|
||||
} from "../player/lib/runtimeProtocol";
|
||||
import { compositionPathOfPreviewUrl } from "../player/components/CompositionThumbnail";
|
||||
import { getSourceFileForElement } from "../components/editor/domEditingDom";
|
||||
import { getLogicalOpacity, getSourceFileForElement } from "../components/editor/domEditingDom";
|
||||
import {
|
||||
createElementPickerHoverLabel,
|
||||
type ElementPickerHoverInfo,
|
||||
@@ -589,7 +589,8 @@ function readComputedStyles(iframe: HTMLIFrameElement, selector: string): Record
|
||||
];
|
||||
|
||||
for (const prop of props) {
|
||||
const val = computed.getPropertyValue(prop);
|
||||
const val =
|
||||
prop === "opacity" ? getLogicalOpacity(el, computed) : computed.getPropertyValue(prop);
|
||||
if (val) styles[prop] = val;
|
||||
}
|
||||
} catch {
|
||||
|
||||
@@ -57,7 +57,7 @@ describe("PlayerControls preview seeking", () => {
|
||||
setValue.call(input, String(time));
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
});
|
||||
expect(onSeek).toHaveBeenLastCalledWith(time);
|
||||
expect(onSeek).toHaveBeenLastCalledWith(time, { settled: true });
|
||||
}
|
||||
expect(onSeek).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
@@ -161,7 +161,7 @@ const FullscreenButton = memo(function FullscreenButton({
|
||||
|
||||
interface PlayerControlsProps {
|
||||
onTogglePlay: () => void;
|
||||
onSeek: (time: number) => void;
|
||||
onSeek: (time: number, options?: { settled?: boolean }) => void;
|
||||
disabled?: boolean;
|
||||
isFullscreen?: boolean;
|
||||
onToggleFullscreen?: () => void;
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// @vitest-environment happy-dom
|
||||
import React, { act } from "react";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { PreviewSeekBar } from "./PreviewSeekBar";
|
||||
import { createHappyDomRootHarness } from "./testRootHarness";
|
||||
|
||||
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
|
||||
const { mount } = createHappyDomRootHarness();
|
||||
|
||||
it("marks keyboard and released seeks as settled and pointer scrubbing as interactive", () => {
|
||||
const host = document.createElement("div");
|
||||
document.body.append(host);
|
||||
const onSeek = vi.fn();
|
||||
act(() => mount(host).render(<PreviewSeekBar duration={10} disabled={false} onSeek={onSeek} />));
|
||||
const input = host.querySelector("input")!;
|
||||
const change = (value: number) => {
|
||||
Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(
|
||||
input,
|
||||
String(value),
|
||||
);
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
};
|
||||
act(() => {
|
||||
input.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
|
||||
change(1);
|
||||
});
|
||||
expect(onSeek).toHaveBeenLastCalledWith(1, { settled: true });
|
||||
act(() => {
|
||||
input.dispatchEvent(new PointerEvent("pointerdown", { bubbles: true }));
|
||||
change(2);
|
||||
});
|
||||
expect(onSeek).toHaveBeenLastCalledWith(2, { settled: false });
|
||||
act(() => {
|
||||
input.dispatchEvent(new PointerEvent("pointerup", { bubbles: true }));
|
||||
});
|
||||
expect(onSeek).toHaveBeenLastCalledWith(2, { settled: true });
|
||||
act(() => {
|
||||
input.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
|
||||
change(3);
|
||||
});
|
||||
expect(onSeek).toHaveBeenLastCalledWith(3, { settled: true });
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { memo, useEffect, useRef } from "react";
|
||||
import { memo, useEffect, useRef, type PointerEvent } from "react";
|
||||
import { formatTime } from "../lib/time";
|
||||
import { liveTime, usePlayerStore } from "../store/playerStore";
|
||||
|
||||
@@ -9,9 +9,16 @@ export const PreviewSeekBar = memo(function PreviewSeekBar({
|
||||
}: {
|
||||
duration: number;
|
||||
disabled: boolean;
|
||||
onSeek: (time: number) => void;
|
||||
onSeek: (time: number, options?: { settled?: boolean }) => void;
|
||||
}) {
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const settledRef = useRef(true);
|
||||
|
||||
const finishPointerSeek = (event: PointerEvent<HTMLInputElement>) => {
|
||||
if (settledRef.current) return;
|
||||
settledRef.current = true;
|
||||
onSeek(event.currentTarget.valueAsNumber, { settled: true });
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const input = inputRef.current;
|
||||
@@ -39,7 +46,17 @@ export const PreviewSeekBar = memo(function PreviewSeekBar({
|
||||
defaultValue={0}
|
||||
disabled={disabled || duration <= 0}
|
||||
aria-label="Seek preview"
|
||||
onChange={(event) => onSeek(event.currentTarget.valueAsNumber)}
|
||||
onPointerDown={() => {
|
||||
settledRef.current = false;
|
||||
}}
|
||||
onPointerUp={finishPointerSeek}
|
||||
onPointerCancel={finishPointerSeek}
|
||||
onKeyDown={() => {
|
||||
settledRef.current = true;
|
||||
}}
|
||||
onChange={(event) =>
|
||||
onSeek(event.currentTarget.valueAsNumber, { settled: settledRef.current })
|
||||
}
|
||||
className="h-6 w-full cursor-pointer accent-neutral-300 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-accent-ink disabled:cursor-default disabled:opacity-30"
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -331,7 +331,7 @@ describe("usePlaybackKeyboard — frame step", () => {
|
||||
dispatch(keydown({ code: "ArrowRight", key: "ArrowRight" }));
|
||||
});
|
||||
|
||||
expect(spies.seek).toHaveBeenCalledWith((frame + 1) / fps);
|
||||
expect(spies.seek).toHaveBeenCalledWith((frame + 1) / fps, { settled: true });
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ interface UsePlaybackKeyboardParams {
|
||||
play: () => void;
|
||||
playBackward: (rate: number) => void;
|
||||
pause: () => void;
|
||||
seek: (time: number, options?: { keepPlaying?: boolean }) => void;
|
||||
seek: (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => void;
|
||||
}
|
||||
|
||||
const FRAME_STEPS = {
|
||||
@@ -112,6 +112,7 @@ export function usePlaybackKeyboard({
|
||||
const currentTime = adapter?.getTime() ?? usePlayerStore.getState().currentTime;
|
||||
seek(
|
||||
stepFrameTime(currentTime, deltaFrames, useLinkedClipPreferences.getState().compositionFps),
|
||||
{ settled: true },
|
||||
);
|
||||
},
|
||||
[getAdapter, seek],
|
||||
|
||||
@@ -28,6 +28,7 @@ export {
|
||||
} from "../lib/playbackShortcuts";
|
||||
|
||||
import type { PlaybackAdapter, IframeWindow } from "../lib/playbackTypes";
|
||||
import { attachPausedVideoProvider } from "../lib/pausedVideoProvider";
|
||||
import { releaseStaticSeekCache, type StaticSeekCacheEntry } from "../lib/playbackAdapter";
|
||||
import { mergeTimelineElementsPreservingDowngrades } from "../lib/timelineDOM";
|
||||
import { findTimelineElementInIframe } from "../../hooks/timelineEditingHelpers";
|
||||
@@ -150,6 +151,7 @@ export function useTimelinePlayer({
|
||||
const iframe = overrideIframe !== undefined ? overrideIframe : iframeRef.current;
|
||||
const win = iframe?.contentWindow as IframeWindow | null;
|
||||
if (!iframe || !win) return null;
|
||||
attachPausedVideoProvider(iframe, win);
|
||||
const adapter = resolvePlaybackAdapter(iframe, win, {
|
||||
cache: staticSeekAdapterRef,
|
||||
warned: staticSeekWarnedRef,
|
||||
@@ -295,7 +297,7 @@ export function useTimelinePlayer({
|
||||
stopRAFLoop();
|
||||
}, [getAdapter, setCurrentTime, setIsPlaying, stopRAFLoop, stopReverseLoop]);
|
||||
const seek = useCallback(
|
||||
(time: number, options?: { keepPlaying?: boolean; follow?: boolean }) => {
|
||||
(time: number, options?: { keepPlaying?: boolean; follow?: boolean; settled?: boolean }) => {
|
||||
const wasReverseShuttle = shuttleDirectionRef.current === "backward";
|
||||
stopReverseLoop();
|
||||
const adapter = getAdapter();
|
||||
|
||||
@@ -611,8 +611,7 @@ describe("mediaUrlSourceOptions", () => {
|
||||
expect(failure.retries).toBe(0);
|
||||
});
|
||||
|
||||
// mediabunny rethrows a failed read-ahead nobody awaits "so it doesn't get swallowed": that copy is typed too.
|
||||
it("fails a real mediabunny read, and its unawaited read-ahead, with the typed error", async () => {
|
||||
it("fails a real mediabunny read with the typed error without a duplicate unhandled rejection", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("", { status: 404 })));
|
||||
const mb = await import("mediabunny");
|
||||
const runnerListeners = process.listeners("unhandledRejection");
|
||||
@@ -629,8 +628,8 @@ describe("mediaUrlSourceOptions", () => {
|
||||
await new Promise(setImmediate);
|
||||
|
||||
expect(error).toMatchObject({ name: "MediaFetchError", status: 404, retries: 0 });
|
||||
expect(unhandled.length).toBeGreaterThan(0);
|
||||
for (const reason of unhandled) expect(reason).toBeInstanceOf(MediaFetchError);
|
||||
expect(error).toBeInstanceOf(MediaFetchError);
|
||||
expect(unhandled).toEqual([]);
|
||||
} finally {
|
||||
process.removeAllListeners("unhandledRejection");
|
||||
for (const listener of runnerListeners) process.on("unhandledRejection", listener);
|
||||
|
||||
@@ -0,0 +1,644 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { createPausedVideoDecoder } from "./pausedVideoDecoder";
|
||||
|
||||
const inputs: { source: string; dispose: ReturnType<typeof vi.fn> }[] = [];
|
||||
const getTrack = vi.fn();
|
||||
const getSample = vi.fn();
|
||||
const cancelReads = vi.fn();
|
||||
const iterateSamples = vi.fn();
|
||||
const finiteSamples = vi.fn();
|
||||
const canvasContext = vi.fn();
|
||||
const createBitmap = vi.fn();
|
||||
let qualityChosenBy = "person";
|
||||
let previewQuality = 1;
|
||||
let displayScale = 1;
|
||||
|
||||
vi.mock("mediabunny", () => ({
|
||||
ALL_FORMATS: [],
|
||||
UrlSource: class {
|
||||
constructor(readonly url: string) {}
|
||||
},
|
||||
Input: class {
|
||||
dispose = vi.fn();
|
||||
constructor(readonly options: { source: { url: string } }) {
|
||||
inputs.push({ source: options.source.url, dispose: this.dispose });
|
||||
}
|
||||
getPrimaryVideoTrack() {
|
||||
return getTrack(this.options.source.url);
|
||||
}
|
||||
},
|
||||
VideoSampleSink: class {
|
||||
constructor(readonly track: { source: string }) {}
|
||||
getSample(time: number) {
|
||||
return getSample(this.track.source, time);
|
||||
}
|
||||
samples(start: number, end: number) {
|
||||
return iterateSamples(this.track.source, start, end);
|
||||
}
|
||||
samplesAtTimestamps(times: Iterable<number>) {
|
||||
return finiteSamples(this.track.source, times);
|
||||
}
|
||||
},
|
||||
}));
|
||||
vi.mock("./mediaFetch", () => ({ mediaUrlSourceOptions: () => ({ cancelReads }) }));
|
||||
vi.mock("../store/playerStore", () => ({
|
||||
usePlayerStore: {
|
||||
getState: () => ({ previewQuality, previewQualityChosenBy: qualityChosenBy }),
|
||||
},
|
||||
}));
|
||||
vi.mock("@hyperframes/core/runtime/protocol", () => ({ frameDisplayScale: () => displayScale }));
|
||||
|
||||
const providers: ReturnType<typeof createPausedVideoDecoder>[] = [];
|
||||
function provider(
|
||||
facts?: Record<
|
||||
string,
|
||||
{ width: number; height: number; frameRate?: number; variableFrameRate?: boolean }
|
||||
>,
|
||||
rootSize?: { width: number; height: number },
|
||||
) {
|
||||
const iframe = document.createElement("iframe");
|
||||
document.body.append(iframe);
|
||||
if (facts) Object.assign(iframe.contentWindow, { __HF_MEDIA_SOURCE_FACTS__: facts });
|
||||
if (rootSize)
|
||||
Object.defineProperties(iframe.contentDocument!.documentElement, {
|
||||
clientWidth: { value: rootSize.width, configurable: true },
|
||||
clientHeight: { value: rootSize.height, configurable: true },
|
||||
});
|
||||
const result = createPausedVideoDecoder(iframe);
|
||||
providers.push(result);
|
||||
return result;
|
||||
}
|
||||
const request = (source: string) => ({
|
||||
source: `http://localhost/${source}.webm`,
|
||||
time: 0,
|
||||
video: document.createElement("video"),
|
||||
});
|
||||
const signal = () => new AbortController().signal;
|
||||
const sample = (timestamp = 0) => ({
|
||||
timestamp,
|
||||
duration: 1 / 30,
|
||||
close: vi.fn(),
|
||||
drawWithFit: vi.fn(),
|
||||
});
|
||||
const bitmap = (bytes = 4) => ({ width: bytes / 4, height: 1, close: vi.fn() });
|
||||
const track = (source: string) => ({
|
||||
source,
|
||||
getDisplayWidth: async () => 100,
|
||||
getDisplayHeight: async () => 100,
|
||||
getFirstTimestamp: async () => 0,
|
||||
getTimeResolution: async () => 1000,
|
||||
async getSquarePixelWidth() {
|
||||
return this.getDisplayWidth();
|
||||
},
|
||||
async getSquarePixelHeight() {
|
||||
return this.getDisplayHeight();
|
||||
},
|
||||
getRotation: async () => 0,
|
||||
getFlip: async () => false,
|
||||
canBeTransparent: async () => false,
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
inputs.length = 0;
|
||||
getTrack.mockReset().mockImplementation(async (source: string) => track(source));
|
||||
getSample.mockReset().mockImplementation(async () => sample());
|
||||
iterateSamples.mockReset();
|
||||
finiteSamples.mockReset().mockImplementation(async function* (
|
||||
source: string,
|
||||
times: Iterable<number>,
|
||||
) {
|
||||
for (const time of times) yield await getSample(source, time);
|
||||
});
|
||||
canvasContext.mockReset().mockReturnValue({ clearRect: vi.fn(), imageSmoothingQuality: "low" });
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockImplementation(canvasContext);
|
||||
createBitmap.mockReset().mockImplementation(async () => bitmap());
|
||||
cancelReads.mockClear();
|
||||
qualityChosenBy = "person";
|
||||
previewQuality = 1;
|
||||
displayScale = 1;
|
||||
vi.stubGlobal("createImageBitmap", createBitmap);
|
||||
});
|
||||
afterEach(async () => {
|
||||
for (const owner of providers.splice(0)) owner.dispose();
|
||||
await Promise.resolve();
|
||||
document.body.innerHTML = "";
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("rejects a two-source frame set that cannot fit without closing a leased bitmap", async () => {
|
||||
const first = bitmap(40 * 1024 * 1024),
|
||||
second = bitmap(40 * 1024 * 1024);
|
||||
createBitmap.mockResolvedValueOnce(first).mockResolvedValueOnce(second);
|
||||
await expect(provider().decode([request("a"), request("b")], signal())).rejects.toThrow(
|
||||
"Preview frame budget reached",
|
||||
);
|
||||
expect(first.close).not.toHaveBeenCalled();
|
||||
expect(second.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("reserves the reader budget before opening and keeps concurrent same-source leases alive", async () => {
|
||||
let finishOpen: (() => void) | undefined;
|
||||
const opened = new Promise<void>((resolve) => {
|
||||
finishOpen = resolve;
|
||||
});
|
||||
getTrack.mockImplementation(async (source: string) => {
|
||||
await opened;
|
||||
return track(source);
|
||||
});
|
||||
const owner = provider();
|
||||
const first = owner.decode([request("a"), request("b"), request("c"), request("d")], signal());
|
||||
const shared = owner.decode([request("a")], signal());
|
||||
await expect(owner.decode([request("e")], signal())).rejects.toThrow(
|
||||
"Preview decoder budget reached",
|
||||
);
|
||||
expect(inputs).toHaveLength(4);
|
||||
expect(inputs.every((input) => input.dispose.mock.calls.length === 0)).toBe(true);
|
||||
finishOpen?.();
|
||||
const frames = await first,
|
||||
other = await shared;
|
||||
expect(frames).toHaveLength(4);
|
||||
expect(other).toHaveLength(1);
|
||||
expect(inputs.every((input) => input.dispose.mock.calls.length === 0)).toBe(true);
|
||||
for (const frame of [...frames, ...other]) frame.release();
|
||||
});
|
||||
|
||||
it("releases successful source leases when another source fails", async () => {
|
||||
const first = bitmap(40 * 1024 * 1024),
|
||||
next = bitmap(40 * 1024 * 1024);
|
||||
createBitmap.mockResolvedValueOnce(first).mockResolvedValueOnce(next);
|
||||
getSample.mockImplementation(async (source: string) => {
|
||||
if (source.endsWith("/b.webm")) throw new Error("decoder failed");
|
||||
return sample();
|
||||
});
|
||||
const owner = provider();
|
||||
await expect(owner.decode([request("a"), request("b")], signal())).rejects.toThrow(
|
||||
"decoder failed",
|
||||
);
|
||||
const [frame] = await owner.decode([request("c")], signal());
|
||||
expect(first.close).toHaveBeenCalledOnce();
|
||||
expect(next.close).not.toHaveBeenCalled();
|
||||
frame!.release();
|
||||
});
|
||||
|
||||
it("retires idle readers even when decoding fails before a frame is returned", async () => {
|
||||
vi.useFakeTimers();
|
||||
getSample.mockRejectedValueOnce(new Error("decoder failed"));
|
||||
await expect(provider().decode([request("a")], signal())).rejects.toThrow("decoder failed");
|
||||
expect(inputs[0]!.dispose).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(10000);
|
||||
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
|
||||
expect(cancelReads).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("closes a bitmap completed after provider disposal and rejects later requests without opening", async () => {
|
||||
let finish: ((image: ReturnType<typeof bitmap>) => void) | undefined;
|
||||
createBitmap.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const owner = provider(),
|
||||
pending = owner.decode([request("a")], signal());
|
||||
await vi.waitFor(() => expect(createBitmap).toHaveBeenCalledOnce());
|
||||
owner.dispose();
|
||||
const image = bitmap();
|
||||
finish?.(image);
|
||||
await expect(pending).rejects.toThrow("Preview disposed");
|
||||
expect(image.close).toHaveBeenCalledOnce();
|
||||
await expect(owner.decode([request("b")], signal())).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(inputs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("shares full-resolution readers when animation changes only an oversized display box", async () => {
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
getDisplayWidth: async () => 1000,
|
||||
getDisplayHeight: async () => 500,
|
||||
}));
|
||||
const owner = provider({ "a.webm": { width: 1000, height: 500 } }),
|
||||
item = request("a");
|
||||
const rect = vi.spyOn(item.video, "getBoundingClientRect");
|
||||
rect.mockReturnValue(new DOMRect(0, 0, 2600, 1800));
|
||||
const first = await owner.decode([item], signal());
|
||||
first[0]!.release();
|
||||
rect.mockReturnValue(new DOMRect(0, 0, 1900, 1300));
|
||||
const second = await owner.decode([item], signal());
|
||||
second[0]!.release();
|
||||
expect(inputs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps original resolution across hidden cuts, viewport changes and automatic playback quality", async () => {
|
||||
qualityChosenBy = "app";
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
getDisplayWidth: async () => 2100,
|
||||
getDisplayHeight: async () => 1800,
|
||||
}));
|
||||
const owner = provider({ "a.webm": { width: 2100, height: 1800 } });
|
||||
for (const [width, scale, quality] of [
|
||||
[2300, 1, 1],
|
||||
[0, 0.5, 0.25],
|
||||
[400, 0.25, 0.5],
|
||||
]) {
|
||||
displayScale = scale!;
|
||||
previewQuality = quality!;
|
||||
const item = request("a");
|
||||
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, width, 200));
|
||||
const frames = await owner.decode([item], signal());
|
||||
frames[0]!.release();
|
||||
}
|
||||
expect(inputs).toHaveLength(1);
|
||||
expect(inputs[0]!.source).toBe("http://localhost/a.webm");
|
||||
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width: 2100, height: 1800 });
|
||||
});
|
||||
|
||||
const cfrFacts = { "a.webm": { width: 100, height: 100, frameRate: 30, variableFrameRate: false } };
|
||||
|
||||
it("closes skipped raw samples and rasterizes only selected frames across a 0.3 second forward gap", async () => {
|
||||
const one = sample(0.1),
|
||||
skipA = sample(0.2),
|
||||
skipB = sample(0.3),
|
||||
last = sample(0.4);
|
||||
iterateSamples.mockImplementation(async function* () {
|
||||
yield one;
|
||||
yield skipA;
|
||||
yield skipB;
|
||||
yield last;
|
||||
});
|
||||
const owner = provider(cfrFacts),
|
||||
item = request("a");
|
||||
for (const time of [0, 0.1, 0.4]) {
|
||||
const frames = await owner.decode([{ ...item, time }], signal());
|
||||
frames[0]!.release();
|
||||
}
|
||||
expect(iterateSamples).toHaveBeenCalledOnce();
|
||||
for (const skipped of [skipA, skipB]) {
|
||||
expect(skipped.close).toHaveBeenCalledOnce();
|
||||
expect(skipped.drawWithFit).not.toHaveBeenCalled();
|
||||
}
|
||||
for (const selected of [one, last]) {
|
||||
expect(selected.close).toHaveBeenCalledOnce();
|
||||
expect(selected.drawWithFit).toHaveBeenCalledWith(expect.anything(), {
|
||||
fit: "fill",
|
||||
rotation: 0,
|
||||
flip: false,
|
||||
});
|
||||
}
|
||||
expect(createBitmap).toHaveBeenCalledTimes(3);
|
||||
expect(canvasContext).toHaveBeenCalledWith("2d", { alpha: true });
|
||||
});
|
||||
|
||||
it("closes the sequential reader and uses demand decoding beyond the bounded forward gap", async () => {
|
||||
const returned = vi.fn();
|
||||
iterateSamples.mockImplementation(async function* () {
|
||||
try {
|
||||
yield sample(0.1);
|
||||
yield sample(0.2);
|
||||
} finally {
|
||||
returned();
|
||||
}
|
||||
});
|
||||
getSample.mockImplementation(async (_source: string, time: number) =>
|
||||
sample(Math.floor(time * 30) / 30),
|
||||
);
|
||||
const owner = provider(cfrFacts),
|
||||
item = request("a");
|
||||
for (const time of [0, 0.1, 1]) {
|
||||
const frames = await owner.decode([{ ...item, time }], signal());
|
||||
frames[0]!.release();
|
||||
}
|
||||
expect(returned).toHaveBeenCalledOnce();
|
||||
expect(getSample).toHaveBeenCalledTimes(2);
|
||||
expect(getSample).toHaveBeenLastCalledWith(item.source, expect.closeTo(1.00001, 8));
|
||||
});
|
||||
|
||||
it("closes a raw sample arriving after cancellation without drawing it", async () => {
|
||||
let finish: ((result: { done: false; value: ReturnType<typeof sample> }) => void) | undefined;
|
||||
const next = vi.fn(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
),
|
||||
returned = vi.fn().mockResolvedValue({ done: true });
|
||||
iterateSamples.mockReturnValue({ next, return: returned });
|
||||
const owner = provider(cfrFacts),
|
||||
item = request("a");
|
||||
const initial = await owner.decode([item], signal());
|
||||
initial[0]!.release();
|
||||
const controller = new AbortController(),
|
||||
pending = owner.decode([{ ...item, time: 0.1 }], controller.signal);
|
||||
await vi.waitFor(() => expect(next).toHaveBeenCalledOnce());
|
||||
controller.abort();
|
||||
const late = sample(0.1);
|
||||
finish?.({ done: false, value: late });
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(late.close).toHaveBeenCalledOnce();
|
||||
expect(late.drawWithFit).not.toHaveBeenCalled();
|
||||
expect(returned).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("closes selected samples when rasterization or bitmap copying fails", async () => {
|
||||
const drawing = sample();
|
||||
drawing.drawWithFit.mockImplementationOnce(() => {
|
||||
throw new Error("drawing failed");
|
||||
});
|
||||
getSample.mockResolvedValueOnce(drawing);
|
||||
const owner = provider();
|
||||
await expect(owner.decode([request("a")], signal())).rejects.toThrow("drawing failed");
|
||||
expect(drawing.close).toHaveBeenCalledOnce();
|
||||
const copying = sample();
|
||||
getSample.mockResolvedValueOnce(copying);
|
||||
createBitmap.mockRejectedValueOnce(new Error("copying failed"));
|
||||
await expect(owner.decode([request("b")], signal())).rejects.toThrow("copying failed");
|
||||
expect(copying.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("closes a demand sample completed after provider disposal before rasterization", async () => {
|
||||
let finish: ((value: ReturnType<typeof sample>) => void) | undefined;
|
||||
getSample.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const owner = provider(),
|
||||
pending = owner.decode([request("a")], signal());
|
||||
await vi.waitFor(() => expect(getSample).toHaveBeenCalledOnce());
|
||||
owner.dispose();
|
||||
const late = sample();
|
||||
finish?.(late);
|
||||
await expect(pending).rejects.toThrow("Preview disposed");
|
||||
expect(late.close).toHaveBeenCalledOnce();
|
||||
expect(late.drawWithFit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps original pixels throughout automatic dragging and settling", async () => {
|
||||
qualityChosenBy = "auto";
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
getDisplayWidth: async () => 1000,
|
||||
getDisplayHeight: async () => 500,
|
||||
}));
|
||||
const owner = provider({ "a.webm": { width: 1000, height: 500 } }),
|
||||
item = request("a");
|
||||
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 2600, 1800));
|
||||
const drag = await owner.decode([{ ...item, settled: false }], signal());
|
||||
drag[0]!.release();
|
||||
const settled = await owner.decode([{ ...item, settled: true }], signal());
|
||||
settled[0]!.release();
|
||||
expect(createBitmap.mock.calls.map(([canvas]) => [canvas.width, canvas.height])).toEqual([
|
||||
[1000, 500],
|
||||
]);
|
||||
});
|
||||
|
||||
it("restores full paused resolution after automatic playback quality was lowered", async () => {
|
||||
qualityChosenBy = "app";
|
||||
previewQuality = 0.25;
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
getDisplayWidth: async () => 1000,
|
||||
getDisplayHeight: async () => 500,
|
||||
}));
|
||||
const owner = provider({ "a.webm": { width: 1000, height: 500 } });
|
||||
const item = request("a");
|
||||
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 1000, 500));
|
||||
const frame = await owner.decode([{ ...item, settled: true }], signal());
|
||||
frame[0]!.release();
|
||||
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width: 1000, height: 500 });
|
||||
});
|
||||
|
||||
it("uses original resolution even when a lower playback quality was saved", async () => {
|
||||
previewQuality = 0.25;
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
getDisplayWidth: async () => 1000,
|
||||
getDisplayHeight: async () => 500,
|
||||
}));
|
||||
const owner = provider({ "a.webm": { width: 1000, height: 500 } }),
|
||||
item = request("a");
|
||||
vi.spyOn(item.video, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 2600, 1800));
|
||||
const drag = await owner.decode([{ ...item, settled: false }], signal());
|
||||
drag[0]!.release();
|
||||
const settled = await owner.decode([{ ...item, settled: true }], signal());
|
||||
settled[0]!.release();
|
||||
expect(createBitmap).toHaveBeenCalledOnce();
|
||||
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width: 1000, height: 500 });
|
||||
expect(inputs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ rotation: 0, flip: false, width: 320, height: 200 },
|
||||
{ rotation: 90, flip: true, width: 200, height: 320 },
|
||||
])(
|
||||
"preserves track rotation $rotation, flip and non-square-pixel display geometry",
|
||||
async ({ rotation, flip, width, height }) => {
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
getDisplayWidth: async () => width,
|
||||
getDisplayHeight: async () => height,
|
||||
getSquarePixelWidth: async () => 320,
|
||||
getSquarePixelHeight: async () => 200,
|
||||
getRotation: async () => rotation,
|
||||
getFlip: async () => flip,
|
||||
}));
|
||||
const decoded = sample();
|
||||
getSample.mockResolvedValueOnce(decoded);
|
||||
const [frame] = await provider().decode([request("a")], signal());
|
||||
expect(createBitmap.mock.calls[0]![0]).toMatchObject({ width, height });
|
||||
expect(decoded.drawWithFit).toHaveBeenCalledWith(expect.anything(), {
|
||||
fit: "fill",
|
||||
rotation,
|
||||
flip,
|
||||
});
|
||||
expect(frame).toMatchObject({ width, height });
|
||||
frame!.release();
|
||||
},
|
||||
);
|
||||
|
||||
it("releases sequential decoders before native playback and lazily recreates them", async () => {
|
||||
const returned = vi.fn();
|
||||
iterateSamples.mockImplementation(async function* () {
|
||||
try {
|
||||
yield sample(0.1);
|
||||
yield sample(0.2);
|
||||
} finally {
|
||||
returned();
|
||||
}
|
||||
});
|
||||
getSample.mockImplementation(async (_source: string, time: number) =>
|
||||
sample(Math.floor(time * 30) / 30),
|
||||
);
|
||||
const owner = provider(cfrFacts),
|
||||
item = request("a");
|
||||
for (const time of [0, 0.1]) {
|
||||
const [frame] = await owner.decode([{ ...item, time }], signal());
|
||||
frame!.release();
|
||||
}
|
||||
expect(returned).not.toHaveBeenCalled();
|
||||
await owner.suspend!();
|
||||
expect(returned).toHaveBeenCalledOnce();
|
||||
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
|
||||
expect(cancelReads).toHaveBeenCalledOnce();
|
||||
const [next] = await owner.decode([{ ...item, time: 0.2 }], signal());
|
||||
expect(inputs).toHaveLength(2);
|
||||
expect([next!.width, next!.height]).toEqual([100, 100]);
|
||||
next!.release();
|
||||
});
|
||||
|
||||
it("closes a bitmap copied after native-play suspension without retaining its decoder", async () => {
|
||||
let finish: ((image: ReturnType<typeof bitmap>) => void) | undefined;
|
||||
createBitmap.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const owner = provider();
|
||||
const pending = owner.decode([request("a")], signal());
|
||||
await vi.waitFor(() => expect(createBitmap).toHaveBeenCalledOnce());
|
||||
await owner.suspend!();
|
||||
const image = bitmap();
|
||||
finish!(image);
|
||||
await expect(pending).rejects.toThrow("Preview disposed");
|
||||
expect(image.close).toHaveBeenCalledOnce();
|
||||
const [next] = await owner.decode([request("a")], signal());
|
||||
expect(inputs).toHaveLength(2);
|
||||
next!.release();
|
||||
});
|
||||
|
||||
it("does not poison a reopened source when an older suspended input settles late", async () => {
|
||||
let rejectOpen: (error: Error) => void = () => {};
|
||||
getTrack.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((_resolve, reject) => {
|
||||
rejectOpen = reject;
|
||||
}),
|
||||
);
|
||||
const owner = provider();
|
||||
const old = owner.decode([request("a")], signal());
|
||||
await vi.waitFor(() => expect(inputs).toHaveLength(1));
|
||||
const suspended = owner.suspend!();
|
||||
expect(inputs[0]!.dispose).toHaveBeenCalled();
|
||||
const [current] = await owner.decode([request("a")], signal());
|
||||
current!.release();
|
||||
rejectOpen(new DOMException("Suspended", "AbortError"));
|
||||
await expect(old).rejects.toMatchObject({ name: "AbortError" });
|
||||
await suspended;
|
||||
const [cached] = await owner.decode([request("a")], signal());
|
||||
expect(inputs).toHaveLength(2);
|
||||
expect(inputs[1]!.dispose).not.toHaveBeenCalled();
|
||||
cached!.release();
|
||||
});
|
||||
|
||||
it("uses finite queries for alpha tracks while preserving original dimensions and closing samples", async () => {
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
canBeTransparent: async () => true,
|
||||
}));
|
||||
const samples = [sample(0), sample(0.1), sample(0.2)];
|
||||
getSample.mockImplementation(async () => samples[getSample.mock.calls.length - 1]);
|
||||
iterateSamples.mockImplementation(async function* () {
|
||||
yield sample(0.1);
|
||||
yield sample(0.2);
|
||||
});
|
||||
const owner = provider(cfrFacts),
|
||||
item = request("a");
|
||||
for (const time of [0, 0.1, 0.2]) {
|
||||
const [frame] = await owner.decode([{ ...item, time }], signal());
|
||||
expect([frame!.width, frame!.height]).toEqual([100, 100]);
|
||||
frame!.release();
|
||||
}
|
||||
expect(iterateSamples).not.toHaveBeenCalled();
|
||||
expect(getSample).toHaveBeenCalledTimes(3);
|
||||
for (const frame of samples) expect(frame.close).toHaveBeenCalledOnce();
|
||||
await owner.suspend!();
|
||||
});
|
||||
|
||||
it.each(["suspend", "dispose"] as const)(
|
||||
"drains an in-flight alpha query before %s",
|
||||
async (action) => {
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
canBeTransparent: async () => true,
|
||||
}));
|
||||
let finish: (value: ReturnType<typeof sample>) => void = () => {};
|
||||
getSample.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const owner = provider();
|
||||
const pending = owner.decode([request("a")], signal());
|
||||
await vi.waitFor(() => expect(getSample).toHaveBeenCalledOnce());
|
||||
const rejected = expect(pending).rejects.toThrow("Preview disposed");
|
||||
const suspended = action === "suspend" ? owner.suspend!() : owner.dispose();
|
||||
await Promise.resolve();
|
||||
expect(inputs[0]!.dispose).not.toHaveBeenCalled();
|
||||
expect(cancelReads).not.toHaveBeenCalled();
|
||||
if (action === "suspend") {
|
||||
const [next] = await owner.decode([request("a")], signal());
|
||||
expect(inputs).toHaveLength(2);
|
||||
next!.release();
|
||||
}
|
||||
const final = sample();
|
||||
finish(final);
|
||||
await rejected;
|
||||
await suspended;
|
||||
expect(final.close).toHaveBeenCalledOnce();
|
||||
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
|
||||
if (action === "suspend") expect(inputs[1]!.dispose).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("finishes the alpha timestamp iterator before publishing its selected sample", async () => {
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
canBeTransparent: async () => true,
|
||||
}));
|
||||
let drain: () => void = () => {};
|
||||
const drained = new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
const selected = sample();
|
||||
finiteSamples.mockImplementation(async function* () {
|
||||
yield selected;
|
||||
await drained;
|
||||
});
|
||||
const owner = provider();
|
||||
let published = false;
|
||||
const decoding = owner.decode([request("a")], signal()).then((frames) => {
|
||||
published = true;
|
||||
return frames;
|
||||
});
|
||||
await vi.waitFor(() => expect(finiteSamples).toHaveBeenCalledOnce());
|
||||
await Promise.resolve();
|
||||
expect(published).toBe(false);
|
||||
expect(createBitmap).not.toHaveBeenCalled();
|
||||
drain();
|
||||
const [frame] = await decoding;
|
||||
expect(selected.close).toHaveBeenCalledOnce();
|
||||
frame!.release();
|
||||
});
|
||||
|
||||
it("closes the selected alpha sample when the finite iterator fails while draining", async () => {
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
canBeTransparent: async () => true,
|
||||
}));
|
||||
const selected = sample();
|
||||
finiteSamples.mockImplementation(async function* () {
|
||||
yield selected;
|
||||
throw new Error("Alpha drain failed");
|
||||
});
|
||||
const owner = provider();
|
||||
await expect(owner.decode([request("a")], signal())).rejects.toThrow("Alpha drain failed");
|
||||
expect(selected.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
@@ -0,0 +1,428 @@
|
||||
import {
|
||||
ALL_FORMATS,
|
||||
Input,
|
||||
UrlSource,
|
||||
VideoSampleSink,
|
||||
type Rotation,
|
||||
type VideoSample,
|
||||
} from "mediabunny";
|
||||
import type {
|
||||
PausedVideoFrame,
|
||||
PausedVideoFrameProvider,
|
||||
PausedVideoFrameRequest,
|
||||
} from "@hyperframes/core";
|
||||
import { mediaUrlSourceOptions } from "./mediaFetch";
|
||||
import type { IframeWindow } from "./playbackTypes";
|
||||
import {
|
||||
previewDecodeTime,
|
||||
previewPresentationTime,
|
||||
type PreviewVideoTiming,
|
||||
} from "./previewVideoTime";
|
||||
|
||||
const FRAME_BUDGET = 64 * 1024 * 1024;
|
||||
const MAX_READERS = 4;
|
||||
const NEARBY_FORWARD_SECONDS = 0.5;
|
||||
|
||||
type Cached = {
|
||||
image: ImageBitmap;
|
||||
timestamp: number;
|
||||
duration: number;
|
||||
bytes: number;
|
||||
pins: number;
|
||||
};
|
||||
type Reader = {
|
||||
key: string;
|
||||
input: Input;
|
||||
cancelReads(): void;
|
||||
sink: VideoSampleSink;
|
||||
canvas: HTMLCanvasElement;
|
||||
rotation: Rotation;
|
||||
flip: boolean;
|
||||
width: number;
|
||||
height: number;
|
||||
lead: number;
|
||||
ticks: number;
|
||||
transparent: boolean;
|
||||
tail: Promise<void>;
|
||||
iterator: AsyncGenerator<VideoSample, void, unknown> | null;
|
||||
iteratorEnd: number;
|
||||
lastTime: number;
|
||||
cache: Map<number, Cached>;
|
||||
users: number;
|
||||
disposed: boolean;
|
||||
idleTimer?: ReturnType<typeof setTimeout>;
|
||||
};
|
||||
|
||||
async function readSingleSample(reader: Reader, query: number): Promise<VideoSample | null> {
|
||||
if (!reader.transparent) return reader.sink.getSample(query);
|
||||
let selected: VideoSample | null = null;
|
||||
try {
|
||||
for await (const next of reader.sink.samplesAtTimestamps([query])) {
|
||||
selected?.close();
|
||||
selected = next;
|
||||
}
|
||||
return selected;
|
||||
} catch (error) {
|
||||
selected?.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function createPausedVideoDecoder(iframe: HTMLIFrameElement): PausedVideoFrameProvider {
|
||||
const readers = new Map<string, Promise<Reader>>();
|
||||
const opened = new Map<string, Reader>();
|
||||
const opening = new Map<Input, () => void>();
|
||||
const failed = new Map<string, number>();
|
||||
const cacheOrder = new Map<Cached, Reader>();
|
||||
let bytes = 0;
|
||||
let disposed = false;
|
||||
|
||||
const sourceFacts = (source: string) => {
|
||||
let path = new URL(source, location.href).pathname;
|
||||
try {
|
||||
path = decodeURIComponent(path);
|
||||
} catch {}
|
||||
return Object.entries(
|
||||
(iframe.contentWindow as IframeWindow | null)?.__HF_MEDIA_SOURCE_FACTS__ ?? {},
|
||||
)
|
||||
.filter(([suffix]) => path.endsWith(suffix))
|
||||
.sort(([a], [b]) => b.length - a.length)[0]?.[1];
|
||||
};
|
||||
|
||||
const trim = (budget = FRAME_BUDGET) => {
|
||||
for (const [entry, reader] of cacheOrder) {
|
||||
if (bytes <= budget) break;
|
||||
if (entry.pins > 0) continue;
|
||||
cacheOrder.delete(entry);
|
||||
reader.cache.delete(entry.timestamp);
|
||||
bytes -= entry.bytes;
|
||||
entry.image.close();
|
||||
}
|
||||
};
|
||||
const disposeReader = async (reader: Reader) => {
|
||||
if (reader.disposed) return;
|
||||
reader.disposed = true;
|
||||
clearTimeout(reader.idleTimer);
|
||||
if (opened.get(reader.key) === reader) opened.delete(reader.key);
|
||||
// Input disposal must not cut off an in-flight alpha merger.
|
||||
if (reader.transparent) await reader.tail;
|
||||
reader.cancelReads();
|
||||
reader.input.dispose();
|
||||
await closeIterator(reader).catch(() => {});
|
||||
for (const entry of reader.cache.values()) {
|
||||
cacheOrder.delete(entry);
|
||||
bytes -= entry.bytes;
|
||||
entry.image.close();
|
||||
}
|
||||
reader.cache.clear();
|
||||
reader.canvas.width = reader.canvas.height = 0;
|
||||
};
|
||||
const scheduleIdle = (reader: Reader) => {
|
||||
clearTimeout(reader.idleTimer);
|
||||
if (
|
||||
reader.disposed ||
|
||||
reader.users > 0 ||
|
||||
[...reader.cache.values()].some((entry) => entry.pins > 0)
|
||||
)
|
||||
return;
|
||||
reader.idleTimer = setTimeout(() => {
|
||||
if (reader.users > 0 || [...reader.cache.values()].some((entry) => entry.pins > 0)) return;
|
||||
if (opened.get(reader.key) !== reader) return;
|
||||
readers.delete(reader.key);
|
||||
void disposeReader(reader);
|
||||
}, 10000);
|
||||
};
|
||||
const open = async (source: string): Promise<Reader> => {
|
||||
const options = mediaUrlSourceOptions("thumbnail");
|
||||
const input = new Input({ source: new UrlSource(source, options), formats: ALL_FORMATS });
|
||||
opening.set(input, options.cancelReads);
|
||||
try {
|
||||
const track = await input.getPrimaryVideoTrack();
|
||||
if (!track || disposed) throw new Error("Preview video unavailable");
|
||||
const [width, height, lead, ticks, squareWidth, squareHeight, rotation, flip, transparent] =
|
||||
await Promise.all([
|
||||
track.getDisplayWidth(),
|
||||
track.getDisplayHeight(),
|
||||
track.getFirstTimestamp(),
|
||||
track.getTimeResolution(),
|
||||
track.getSquarePixelWidth(),
|
||||
track.getSquarePixelHeight(),
|
||||
track.getRotation(),
|
||||
track.getFlip(),
|
||||
track.canBeTransparent(),
|
||||
]);
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.ceil(width));
|
||||
const aspect = rotation % 180 === 0 ? squareWidth / squareHeight : squareHeight / squareWidth;
|
||||
canvas.height = Math.max(1, Math.round(canvas.width / aspect));
|
||||
return {
|
||||
key: source,
|
||||
input,
|
||||
cancelReads: options.cancelReads,
|
||||
sink: new VideoSampleSink(track),
|
||||
canvas,
|
||||
rotation,
|
||||
flip,
|
||||
width,
|
||||
height,
|
||||
lead,
|
||||
ticks,
|
||||
transparent,
|
||||
tail: Promise.resolve(),
|
||||
iterator: null,
|
||||
iteratorEnd: 0,
|
||||
lastTime: -Infinity,
|
||||
cache: new Map(),
|
||||
users: 0,
|
||||
disposed: false,
|
||||
};
|
||||
} catch (error) {
|
||||
input.dispose();
|
||||
options.cancelReads();
|
||||
throw error;
|
||||
} finally {
|
||||
opening.delete(input);
|
||||
}
|
||||
};
|
||||
const reserveReader = () => {
|
||||
for (const [oldKey] of readers) {
|
||||
if (readers.size < MAX_READERS) break;
|
||||
const old = opened.get(oldKey);
|
||||
if (old && old.users === 0 && ![...old.cache.values()].some((entry) => entry.pins > 0)) {
|
||||
readers.delete(oldKey);
|
||||
void disposeReader(old);
|
||||
}
|
||||
}
|
||||
if (readers.size >= MAX_READERS) throw new Error("Preview decoder budget reached");
|
||||
};
|
||||
const readerFor = async (source: string): Promise<Reader> => {
|
||||
const key = source;
|
||||
if ((failed.get(key) ?? 0) > performance.now()) throw new Error("Preview decoder cooling down");
|
||||
let promise = readers.get(key);
|
||||
if (!promise) {
|
||||
reserveReader();
|
||||
promise = open(source);
|
||||
readers.set(key, promise);
|
||||
}
|
||||
readers.delete(key);
|
||||
readers.set(key, promise);
|
||||
let reader: Reader;
|
||||
try {
|
||||
reader = await promise;
|
||||
} catch (error) {
|
||||
if (readers.get(key) === promise) {
|
||||
readers.delete(key);
|
||||
failed.set(key, performance.now() + 5000);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (disposed || reader.disposed) throw new DOMException("Preview disposed", "AbortError");
|
||||
opened.set(key, reader);
|
||||
reader.users++;
|
||||
clearTimeout(reader.idleTimer);
|
||||
return reader;
|
||||
};
|
||||
const closeIterator = async (reader: Reader) => {
|
||||
const iterator = reader.iterator;
|
||||
reader.iterator = null;
|
||||
await iterator?.return();
|
||||
};
|
||||
const checkActive = (reader: Reader, signal: AbortSignal) => {
|
||||
signal.throwIfAborted();
|
||||
if (disposed || reader.disposed) throw new Error("Preview disposed");
|
||||
};
|
||||
const readForward = async (reader: Reader, query: number, signal: AbortSignal) => {
|
||||
if (!reader.iterator) {
|
||||
reader.iteratorEnd = query + 8;
|
||||
reader.iterator = reader.sink.samples(query, reader.iteratorEnd);
|
||||
}
|
||||
for (;;) {
|
||||
checkActive(reader, signal);
|
||||
const next = await reader.iterator.next();
|
||||
if (next.done) return null;
|
||||
const sample = next.value;
|
||||
let selected = false;
|
||||
try {
|
||||
checkActive(reader, signal);
|
||||
if (sample.timestamp > query + 1e-6) return null;
|
||||
selected = sample.timestamp + sample.duration > query + 1e-6;
|
||||
if (selected) return sample;
|
||||
} finally {
|
||||
if (!selected) sample.close();
|
||||
}
|
||||
}
|
||||
};
|
||||
const pin = (reader: Reader, entry: Cached) => {
|
||||
entry.pins++;
|
||||
cacheOrder.delete(entry);
|
||||
cacheOrder.set(entry, reader);
|
||||
return entry;
|
||||
};
|
||||
const cacheFrame = async (
|
||||
reader: Reader,
|
||||
sample: VideoSample,
|
||||
time: number,
|
||||
timing: PreviewVideoTiming,
|
||||
signal: AbortSignal,
|
||||
): Promise<Cached> => {
|
||||
const context = reader.canvas.getContext("2d", { alpha: true });
|
||||
if (!context) throw new Error("Preview canvas unavailable");
|
||||
context.clearRect(0, 0, reader.canvas.width, reader.canvas.height);
|
||||
context.imageSmoothingQuality = "high";
|
||||
sample.drawWithFit(context, { fit: "fill", rotation: reader.rotation, flip: reader.flip });
|
||||
const image = await createImageBitmap(reader.canvas);
|
||||
if (signal.aborted || disposed || reader.disposed) {
|
||||
image.close();
|
||||
signal.throwIfAborted();
|
||||
throw new Error("Preview disposed");
|
||||
}
|
||||
const entry = {
|
||||
image,
|
||||
...previewPresentationTime(sample.timestamp, time, timing),
|
||||
bytes: image.width * image.height * 4,
|
||||
pins: 1,
|
||||
};
|
||||
const previous = reader.cache.get(entry.timestamp);
|
||||
if (previous) {
|
||||
previous.duration = Math.max(previous.duration, entry.duration);
|
||||
image.close();
|
||||
return pin(reader, previous);
|
||||
}
|
||||
trim(FRAME_BUDGET - entry.bytes);
|
||||
if (bytes + entry.bytes > FRAME_BUDGET) {
|
||||
image.close();
|
||||
throw new Error("Preview frame budget reached");
|
||||
}
|
||||
reader.cache.set(entry.timestamp, entry);
|
||||
cacheOrder.set(entry, reader);
|
||||
bytes += entry.bytes;
|
||||
return entry;
|
||||
};
|
||||
const nearbyForward = (reader: Reader, time: number, timing: PreviewVideoTiming) =>
|
||||
// Mediabunny 1.65.2 can orphan alpha samples when a range is cancelled.
|
||||
// Finite timestamp iterators drain the merger before closing it.
|
||||
!reader.transparent &&
|
||||
!timing.variableFrameRate &&
|
||||
(timing.frameRate ?? 0) > 0 &&
|
||||
time > reader.lastTime + 1e-6 &&
|
||||
time - reader.lastTime <= NEARBY_FORWARD_SECONDS;
|
||||
const readFrame = async (
|
||||
reader: Reader,
|
||||
time: number,
|
||||
timing: PreviewVideoTiming,
|
||||
signal: AbortSignal,
|
||||
): Promise<Cached> => {
|
||||
checkActive(reader, signal);
|
||||
const cached = [...reader.cache.values()].find(
|
||||
(entry) => time >= entry.timestamp - 1e-6 && time < entry.timestamp + entry.duration - 1e-6,
|
||||
);
|
||||
if (cached) return pin(reader, cached);
|
||||
const query = previewDecodeTime(time, timing),
|
||||
forward = nearbyForward(reader, time, timing);
|
||||
let sample: VideoSample | null = null;
|
||||
try {
|
||||
if (!forward || query >= reader.iteratorEnd) await closeIterator(reader);
|
||||
sample = forward ? await readForward(reader, query, signal) : null;
|
||||
if (!sample) {
|
||||
await closeIterator(reader);
|
||||
sample = await readSingleSample(reader, query);
|
||||
}
|
||||
if (!sample) throw new Error("No preview frame at requested time");
|
||||
checkActive(reader, signal);
|
||||
reader.lastTime = time;
|
||||
return await cacheFrame(reader, sample, time, timing, signal);
|
||||
} finally {
|
||||
sample?.close();
|
||||
if (signal.aborted || disposed) await closeIterator(reader).catch(() => {});
|
||||
}
|
||||
};
|
||||
const sourceTiming = (request: PausedVideoFrameRequest, reader: Reader) => {
|
||||
const facts = sourceFacts(request.source);
|
||||
return {
|
||||
frameRate: facts?.frameRate,
|
||||
variableFrameRate: facts?.variableFrameRate,
|
||||
sourceLead: reader.lead,
|
||||
inputLead: reader.lead,
|
||||
timeResolution: reader.ticks,
|
||||
};
|
||||
};
|
||||
const decodeOne = async (
|
||||
request: PausedVideoFrameRequest,
|
||||
signal: AbortSignal,
|
||||
): Promise<PausedVideoFrame> => {
|
||||
signal.throwIfAborted();
|
||||
const source = request.source;
|
||||
if (!source) throw new Error("Preview source unavailable");
|
||||
const reader = await readerFor(source);
|
||||
try {
|
||||
const timing = sourceTiming(request, reader);
|
||||
const job = reader.tail.then(() => readFrame(reader, request.time, timing, signal));
|
||||
reader.tail = job.then(
|
||||
() => {},
|
||||
() => {},
|
||||
);
|
||||
const entry = await job;
|
||||
let released = false;
|
||||
trim();
|
||||
return {
|
||||
source: entry.image,
|
||||
timestamp: entry.timestamp,
|
||||
duration: entry.duration,
|
||||
width: reader.width,
|
||||
height: reader.height,
|
||||
release() {
|
||||
if (released) return;
|
||||
released = true;
|
||||
entry.pins--;
|
||||
trim();
|
||||
scheduleIdle(reader);
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
reader.users--;
|
||||
scheduleIdle(reader);
|
||||
}
|
||||
};
|
||||
return {
|
||||
async decode(requests, signal) {
|
||||
if (disposed) throw new DOMException("Preview disposed", "AbortError");
|
||||
if (new Set(requests.map(({ source }) => source)).size > MAX_READERS)
|
||||
throw new Error("Preview decoder budget reached");
|
||||
const results = await Promise.allSettled(
|
||||
requests.map((request) => decodeOne(request, signal)),
|
||||
);
|
||||
const failure = results.find((result) => result.status === "rejected");
|
||||
const frames = results.flatMap((result) =>
|
||||
result.status === "fulfilled" ? [result.value] : [],
|
||||
);
|
||||
if (failure || disposed || signal.aborted) {
|
||||
for (const frame of frames) frame.release();
|
||||
if (failure) throw failure.reason;
|
||||
throw new DOMException("Preview cancelled", "AbortError");
|
||||
}
|
||||
return frames;
|
||||
},
|
||||
async suspend() {
|
||||
const closing = [...readers.values()].map((promise) =>
|
||||
promise.then(disposeReader).catch(() => {}),
|
||||
);
|
||||
readers.clear();
|
||||
for (const [input, cancel] of opening) {
|
||||
cancel();
|
||||
input.dispose();
|
||||
}
|
||||
await Promise.all(closing);
|
||||
},
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
for (const [input, cancel] of opening) {
|
||||
cancel();
|
||||
input.dispose();
|
||||
}
|
||||
for (const promise of readers.values()) void promise.then(disposeReader).catch(() => {});
|
||||
readers.clear();
|
||||
failed.clear();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { expect, it, vi } from "vitest";
|
||||
import type { PausedVideoFrameProvider } from "@hyperframes/core";
|
||||
import { attachPausedVideoProvider } from "./pausedVideoProvider";
|
||||
|
||||
const decoder = vi.hoisted(() => ({
|
||||
decode: vi.fn(async () => []),
|
||||
suspend: vi.fn(async () => {}),
|
||||
dispose: vi.fn(),
|
||||
}));
|
||||
vi.mock("./pausedVideoDecoder", () => ({ createPausedVideoDecoder: () => decoder }));
|
||||
|
||||
it("forwards native-play suspension through the lazy provider facade", async () => {
|
||||
const iframe = document.createElement("iframe");
|
||||
document.body.append(iframe);
|
||||
const install = vi.fn<(provider: PausedVideoFrameProvider) => void>();
|
||||
const win = iframe.contentWindow!;
|
||||
Object.assign(win, {
|
||||
__hf: {
|
||||
setPausedVideoFrameProvider: install,
|
||||
},
|
||||
});
|
||||
attachPausedVideoProvider(iframe, win);
|
||||
const provider = install.mock.calls[0]![0];
|
||||
await provider.decode([], new AbortController().signal);
|
||||
await provider.suspend?.();
|
||||
expect(decoder.suspend).toHaveBeenCalledOnce();
|
||||
await provider.decode([], new AbortController().signal);
|
||||
expect(decoder.decode).toHaveBeenCalledTimes(2);
|
||||
win.dispatchEvent(new Event("pagehide"));
|
||||
await Promise.resolve();
|
||||
expect(decoder.dispose).toHaveBeenCalledOnce();
|
||||
iframe.remove();
|
||||
});
|
||||
|
||||
it("allows native-play suspension after the lazy decoder chunk rejects", async () => {
|
||||
vi.resetModules();
|
||||
vi.doMock("./pausedVideoDecoder", () => {
|
||||
throw new Error("decoder chunk unavailable");
|
||||
});
|
||||
const { attachPausedVideoProvider: attach } = await import("./pausedVideoProvider");
|
||||
const iframe = document.createElement("iframe");
|
||||
document.body.append(iframe);
|
||||
const install = vi.fn<(provider: PausedVideoFrameProvider) => void>();
|
||||
const win = iframe.contentWindow!;
|
||||
Object.assign(win, { __hf: { setPausedVideoFrameProvider: install } });
|
||||
attach(iframe, win);
|
||||
const provider = install.mock.calls[0]![0];
|
||||
await expect(provider.decode([], new AbortController().signal)).rejects.toThrow();
|
||||
await expect(provider.suspend!()).resolves.toBeUndefined();
|
||||
win.dispatchEvent(new Event("pagehide"));
|
||||
iframe.remove();
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { PausedVideoFrameProvider } from "@hyperframes/core";
|
||||
import type { IframeWindow } from "./playbackTypes";
|
||||
|
||||
const installed = new WeakMap<
|
||||
HTMLIFrameElement,
|
||||
{
|
||||
document: Document;
|
||||
provider: PausedVideoFrameProvider;
|
||||
setter: (provider: PausedVideoFrameProvider | null) => void;
|
||||
}
|
||||
>();
|
||||
|
||||
/** A lazy facade also installs in Studio's hidden reload frame before its first seek. */
|
||||
export function attachPausedVideoProvider(iframe: HTMLIFrameElement, win: IframeWindow): void {
|
||||
if (!win.__hf?.setPausedVideoFrameProvider || !iframe.contentDocument) return;
|
||||
const previous = installed.get(iframe);
|
||||
const setter = win.__hf.setPausedVideoFrameProvider;
|
||||
if (previous?.document === iframe.contentDocument && previous.setter === setter) return;
|
||||
previous?.provider.dispose();
|
||||
let disposed = false;
|
||||
let decoder: Promise<PausedVideoFrameProvider> | null = null;
|
||||
const provider: PausedVideoFrameProvider = {
|
||||
async decode(requests, signal) {
|
||||
signal.throwIfAborted();
|
||||
if (disposed) throw new DOMException("Preview disposed", "AbortError");
|
||||
decoder ??= import("./pausedVideoDecoder").then(({ createPausedVideoDecoder }) =>
|
||||
createPausedVideoDecoder(iframe),
|
||||
);
|
||||
const actual = await decoder;
|
||||
if (disposed) {
|
||||
actual.dispose();
|
||||
throw new DOMException("Preview disposed", "AbortError");
|
||||
}
|
||||
return actual.decode(requests, signal);
|
||||
},
|
||||
async suspend() {
|
||||
const actual = await decoder?.catch(() => null);
|
||||
await actual?.suspend?.();
|
||||
},
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
void decoder?.then((actual) => actual.dispose()).catch(() => {});
|
||||
win.removeEventListener("pagehide", onPageHide);
|
||||
},
|
||||
};
|
||||
const onPageHide = () => {
|
||||
provider.dispose();
|
||||
installed.delete(iframe);
|
||||
};
|
||||
win.addEventListener("pagehide", onPageHide, { once: true });
|
||||
installed.set(iframe, { document: iframe.contentDocument, provider, setter });
|
||||
setter(provider);
|
||||
}
|
||||
@@ -4,12 +4,16 @@
|
||||
* from here without creating circular dependencies.
|
||||
*/
|
||||
|
||||
import type { RuntimeTimelineClipIdentity } from "@hyperframes/core";
|
||||
import type {
|
||||
RuntimeTimelineClipIdentity,
|
||||
PausedVideoFrameProvider,
|
||||
PreviewSourceFacts,
|
||||
} from "@hyperframes/core";
|
||||
|
||||
export interface PlaybackAdapter {
|
||||
play: () => void;
|
||||
pause: () => void;
|
||||
seek: (time: number, options?: { keepPlaying?: boolean }) => void;
|
||||
seek: (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => void;
|
||||
getTime: () => number;
|
||||
getDuration: () => number;
|
||||
isPlaying: () => boolean;
|
||||
@@ -58,6 +62,7 @@ export interface ClipManifest {
|
||||
}
|
||||
|
||||
export type IframeWindow = Window & {
|
||||
__HF_MEDIA_SOURCE_FACTS__?: Record<string, PreviewSourceFacts>;
|
||||
__player?: RuntimePlaybackAdapter;
|
||||
__playerReady?: boolean;
|
||||
__timeline?: TimelineLike;
|
||||
@@ -70,6 +75,7 @@ export type IframeWindow = Window & {
|
||||
__hf?: {
|
||||
leasePausedMedia?: (el: HTMLMediaElement) => void;
|
||||
releasePausedMedia?: (el: HTMLMediaElement) => void;
|
||||
setPausedVideoFrameProvider?: (provider: PausedVideoFrameProvider | null) => void;
|
||||
setLoopStart?: (seconds: number | null) => void;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { previewDecodeTime, previewPresentationTime } from "./previewVideoTime";
|
||||
|
||||
const cfr = {
|
||||
frameRate: 30,
|
||||
variableFrameRate: false,
|
||||
sourceLead: 0,
|
||||
inputLead: 0,
|
||||
timeResolution: 1000,
|
||||
};
|
||||
it("matches export at rounded WebM ticks without selecting the nearest future frame", () => {
|
||||
expect(previewDecodeTime(2 / 30, cfr)).toBeCloseTo(0.06701, 8);
|
||||
expect(previewPresentationTime(0.067, 2 / 30, cfr).timestamp).toBeCloseTo(2 / 30, 8);
|
||||
expect(previewDecodeTime(3.33, cfr)).toBeCloseTo(3.30001, 8);
|
||||
expect(previewDecodeTime(0.0666, cfr)).toBeCloseTo(0.03301, 8);
|
||||
});
|
||||
it("preserves a nonzero source lead when a prepared file starts at zero", () => {
|
||||
const timing = { ...cfr, sourceLead: 5, inputLead: 0 };
|
||||
expect(previewDecodeTime(5 + 2 / 30, timing)).toBeCloseTo(0.06701, 8);
|
||||
expect(previewPresentationTime(0.067, 5 + 2 / 30, timing).timestamp).toBeCloseTo(5 + 2 / 30, 8);
|
||||
});
|
||||
it("retains raw VFR times and holds a prior frame through a timestamp gap", () => {
|
||||
const timing = { ...cfr, variableFrameRate: true };
|
||||
expect(previewDecodeTime(0.1, timing)).toBeCloseTo(0.1, 8);
|
||||
const held = previewPresentationTime(0.04, 0.1, timing);
|
||||
expect(held.timestamp).toBe(0.04);
|
||||
expect(held.timestamp + held.duration).toBeGreaterThan(0.1);
|
||||
expect(held.timestamp + held.duration).toBeLessThan(0.101);
|
||||
});
|
||||
it("does not advance to the next CFR frame just before its boundary", () => {
|
||||
expect(previewDecodeTime(2 / 30 - 0.000005, cfr)).toBeCloseTo(0.03301, 8);
|
||||
const timing = { ...cfr, frameRate: 30000 / 1001, sourceLead: 5 };
|
||||
expect(previewDecodeTime(5 + (2 * 1001) / 30000 - 0.000005, timing)).toBeCloseTo(0.03301, 8);
|
||||
});
|
||||
it("does not cross a real VFR timestamp when querying just before it", () => {
|
||||
const timing = { ...cfr, variableFrameRate: true, sourceLead: 5, inputLead: 2 };
|
||||
expect(previewDecodeTime(5.1 - 0.000005, timing)).toBeLessThan(2.1);
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
export interface PreviewVideoTiming {
|
||||
frameRate?: number;
|
||||
variableFrameRate?: boolean;
|
||||
sourceLead: number;
|
||||
inputLead: number;
|
||||
timeResolution: number;
|
||||
}
|
||||
|
||||
const LEAD = 1e-5;
|
||||
const FRAME_EPSILON = 1e-9;
|
||||
const constantRate = (timing: PreviewVideoTiming) =>
|
||||
!timing.variableFrameRate && (timing.frameRate ?? 0) > 0;
|
||||
|
||||
/** Match export: floor the source-frame grid, then undo container tick rounding. */
|
||||
export function previewDecodeTime(time: number, timing: PreviewVideoTiming): number {
|
||||
const { sourceLead, inputLead, timeResolution, frameRate = 0 } = timing;
|
||||
if (!constantRate(timing))
|
||||
return Math.max(inputLead, time - sourceLead + inputLead) + FRAME_EPSILON;
|
||||
const index = Math.max(0, Math.floor((time - sourceLead) * frameRate + FRAME_EPSILON));
|
||||
const ideal = inputLead + index / frameRate;
|
||||
return (timeResolution > 0 ? Math.round(ideal * timeResolution) / timeResolution : ideal) + LEAD;
|
||||
}
|
||||
|
||||
export function previewPresentationTime(
|
||||
timestamp: number,
|
||||
requested: number,
|
||||
timing: PreviewVideoTiming,
|
||||
) {
|
||||
const { frameRate = 0, sourceLead, inputLead } = timing;
|
||||
const logical = constantRate(timing)
|
||||
? Math.round((timestamp - inputLead) * frameRate) / frameRate + sourceLead
|
||||
: timestamp - inputLead + sourceLead;
|
||||
const start = Math.min(requested, logical);
|
||||
// A decoded prior frame proves its coverage through this query, including a VFR gap.
|
||||
return {
|
||||
timestamp: start,
|
||||
duration: Math.max(constantRate(timing) ? 1 / frameRate : 0, requested - start + LEAD),
|
||||
};
|
||||
}
|
||||
@@ -4,10 +4,21 @@ import {
|
||||
PREVIEW_RASTER_ATTR,
|
||||
RUNTIME_STAMPED_ATTR,
|
||||
STUDIO_PREVIEW_LAZY_ATTR,
|
||||
PREVIEW_VIDEO_CANVAS_ATTR,
|
||||
PREVIEW_VIDEO_FRAME_ATTR,
|
||||
PREVIEW_VIDEO_SOURCE_ATTR,
|
||||
} from "@hyperframes/core/studio-preview-mark";
|
||||
import { liveMarkupWithoutPreviewMarks } from "./authoredSource";
|
||||
|
||||
describe("liveMarkupWithoutPreviewMarks", () => {
|
||||
it("omits generated video surfaces and their hiding marks from copied composition markup", () => {
|
||||
const live = document.createElement("div");
|
||||
live.innerHTML = `<canvas ${PREVIEW_VIDEO_CANVAS_ATTR}></canvas><video src="original.webm" ${PREVIEW_VIDEO_FRAME_ATTR} ${PREVIEW_VIDEO_SOURCE_ATTR}></video><canvas id="authored"></canvas>`;
|
||||
expect(liveMarkupWithoutPreviewMarks(live)).toBe(
|
||||
'<div><video src="original.webm"></video><canvas id="authored"></canvas></div>',
|
||||
);
|
||||
expect(live.querySelectorAll("canvas")).toHaveLength(2);
|
||||
});
|
||||
it("drops the preview's raster mark from an element and its descendants", () => {
|
||||
const live = document.createElement("div");
|
||||
live.innerHTML = `<div class="line" ${PREVIEW_RASTER_ATTR}><span class="word">LOUD</span></div>`;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { ensureHfIds, isCompositionTemplate } from "@hyperframes/parsers/hf-ids";
|
||||
import {
|
||||
PREVIEW_ONLY_ATTRS,
|
||||
PREVIEW_VIDEO_CANVAS_ATTR,
|
||||
STUDIO_PREVIEW_LAZY_ATTR,
|
||||
} from "@hyperframes/core/studio-preview-mark";
|
||||
|
||||
@@ -97,6 +98,7 @@ export function authoredMarkup(authored: Element, live: Element, sourceFile: str
|
||||
|
||||
export function liveMarkupWithoutPreviewMarks(live: Element): string {
|
||||
const copy = live.cloneNode(true) as Element;
|
||||
for (const canvas of copy.querySelectorAll(`[${PREVIEW_VIDEO_CANVAS_ATTR}]`)) canvas.remove();
|
||||
for (const el of [copy, ...Array.from(copy.querySelectorAll("*"))]) {
|
||||
if (el.hasAttribute(STUDIO_PREVIEW_LAZY_ATTR)) el.removeAttribute("loading");
|
||||
for (const attr of PREVIEW_ONLY_ATTRS) el.removeAttribute(attr);
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
"unpacked": 541601
|
||||
},
|
||||
"@hyperframes/producer": {
|
||||
"packed": 857000,
|
||||
"packed": 870000,
|
||||
"unpacked": 3130000
|
||||
},
|
||||
"@hyperframes/sdk": {
|
||||
|
||||
Reference in New Issue
Block a user