' +
@@ -83,6 +101,47 @@ function film(withImage = false) {
return { main, old: videos.slice(0, 2), next: videos.slice(2), drawImage };
}
+function frameProvider(videos: ReturnType
[]) {
+ const clocks = new Map(
+ videos.map(({ video }) => [video, Object.getOwnPropertyDescriptor(video, "currentTime")!]),
+ );
+ for (const { video } of videos) Reflect.deleteProperty(video, "currentTime");
+ vi.spyOn(HTMLMediaElement.prototype, "currentTime", "get").mockImplementation(
+ function (this: HTMLMediaElement) {
+ return clocks.get(this as HTMLVideoElement)?.get?.();
+ },
+ );
+ vi.spyOn(HTMLMediaElement.prototype, "currentTime", "set").mockImplementation(function (
+ this: HTMLMediaElement,
+ time: number,
+ ) {
+ clocks.get(this as HTMLVideoElement)?.set?.(time);
+ });
+ const pending: Array<() => void> = [];
+ const provider: PausedVideoFrameProvider = {
+ decode: vi.fn(
+ (requests: PausedVideoFrameRequest[]) =>
+ new Promise((resolve) => {
+ pending.push(() =>
+ resolve(
+ requests.map(({ time }) => ({
+ source: document.createElement("canvas"),
+ timestamp: time,
+ duration: 1 / 30,
+ width: 64,
+ height: 36,
+ release: vi.fn(),
+ })),
+ ),
+ );
+ }),
+ ),
+ dispose: vi.fn(),
+ };
+ window.__hf!.setPausedVideoFrameProvider!(provider);
+ return { provider, pending };
+}
+
it("keeps the old picture and page until both destination videos have decoded", async () => {
const { main, old, next, drawImage } = film();
const landed = window.__player!.seek(5.5);
@@ -112,6 +171,8 @@ it("coalesces same-clip requests and publishes complete intermediate pictures",
expect(next.map((v) => v.seeks.length)).toEqual(initialSeeks);
for (const video of next) video.land();
expect(main.time?.()).toBe(5.5);
+ expect(window.__player!.getTime()).toBe(7.5);
+ expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
expect(next.every((v) => Math.abs(v.video.currentTime - 2.5) < 0.001)).toBe(true);
for (const video of next) video.land();
await landed;
@@ -270,6 +331,547 @@ it("caps a stalled decoder without leaving the picture hidden", async () => {
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
});
+it("exposes the presented clock to animation callbacks while retaining the latest scrub destination", async () => {
+ const { main, next } = film();
+ const times: number[] = [];
+ const original = main.totalTime!;
+ main.totalTime = (time?: number) => {
+ if (time !== undefined) times.push(window.__player!.getTime());
+ return original(time);
+ };
+ const done = window.__player!.seek(5.5);
+ window.__player!.seek(7.5);
+ for (const video of next) video.land();
+ expect(times).toContain(5.5);
+ expect(window.__player!.getTime()).toBe(7.5);
+ for (const video of next) video.land();
+ await done;
+});
+
+it.each(["stop-media", "set-native-media-sync-disabled"])(
+ "does not resume a pending playback handoff after %s",
+ async (action) => {
+ const { old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const landed = window.__player!.seek(1);
+ pending.shift()!();
+ await landed;
+ window.__player!.play();
+ expect(window.__player!.isPlaying()).toBe(false);
+ window.dispatchEvent(
+ new MessageEvent("message", {
+ source: window,
+ data: { source: "hf-parent", type: "control", action, disabled: true },
+ }),
+ );
+ await Promise.resolve();
+ for (const video of old) video.land();
+ await Promise.resolve();
+ expect(window.__player!.isPlaying()).toBe(false);
+ },
+);
+
+it("keeps the complete picture and buffers beyond the handoff cap until native frames are ready", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ const { main, old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const landed = window.__player!.seek(1);
+ pending.shift()!();
+ await landed;
+ for (const video of old) video.metadataAt(1);
+ window.__player!.play();
+ expect(old.every(({ video }) => video.preload === "auto")).toBe(true);
+ await vi.advanceTimersByTimeAsync(5100);
+ expect(window.__player!.isPlaying()).toBe(false);
+ expect(main.time?.()).toBe(1);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+ window.__player!.seek(1);
+ for (const video of old) video.land();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
+});
+
+it("settles a stalled paused fallback while retaining the last complete picture", async () => {
+ resetSeekDispatchState();
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ const { main, old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(1);
+ pending.shift()!();
+ await initial;
+ vi.mocked(provider.decode).mockRejectedValueOnce(new Error("decoder unavailable"));
+ const stalled = window.__player!.seek(6);
+ const retargeted = window.__player!.seek(7);
+ const wait = window.__hfWaitForSeekCompletion!();
+ await vi.advanceTimersByTimeAsync(5100);
+ await stalled;
+ await retargeted;
+ await wait;
+ expect(main.time?.()).toBe(1);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+ expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
+ const retry = window.__player!.seek(8);
+ pending.shift()!();
+ await retry;
+ expect(main.time?.()).toBe(8);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+ expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
+});
+
+it.each([
+ ["already metadata-only, idle", false, 1],
+ ["ready, then metadata-only idle", true, 1],
+ ["ready, then loading forever", true, 2],
+ ["metadata-only, loading forever", false, 2],
+] as const)(
+ "bounds Play for %s while retaining its full-quality picture",
+ async (_, initiallyReady, networkState) => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+ const { main, old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(3);
+ pending.shift()!();
+ await initial;
+ const stalled = old[0]!;
+ if (initiallyReady) stalled.land();
+ else stalled.metadataAt(3);
+ Object.defineProperty(stalled.video, "networkState", {
+ value: networkState,
+ configurable: true,
+ });
+ window.__player!.play();
+ old[1]!.land();
+ await vi.advanceTimersByTimeAsync(5100);
+ if (initiallyReady || networkState === 2) expect(window.__player!.isPlaying()).toBe(false);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(
+ window.__player!.isPlaying() ? 1 : 2,
+ );
+ stalled.video.dispatchEvent(new Event("progress"));
+ await vi.advanceTimersByTimeAsync(4900);
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect(main.time?.()).toBeGreaterThanOrEqual(3);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(1);
+ const held = document.querySelector("[data-hf-preview-video-canvas]")!;
+ expect([held.getAttribute("width"), held.getAttribute("height")]).toEqual(["64", "36"]);
+ },
+);
+
+it("keeps every unready video picture at the hard deadline", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+ const { old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(3);
+ pending.shift()!();
+ await initial;
+ const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
+ window.__player!.play();
+ await vi.advanceTimersByTimeAsync(10000);
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
+ window.__player!.pause();
+ pending.shift()!();
+ await Promise.resolve();
+ await Promise.resolve();
+ window.__player!.play();
+ await Promise.resolve();
+ await Promise.resolve();
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+});
+
+it("reveals a delayed playing frame without chasing the moving clock with more seeks", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+ const { old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(3);
+ pending.shift()!();
+ await initial;
+ const stalled = old[0]!;
+ stalled.metadataAt(3);
+ let error: { code: number } | null = { code: 4 };
+ Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
+ window.__player!.play();
+ old[1]!.land();
+ await vi.advanceTimersByTimeAsync(0);
+ error = null;
+ const seeks = stalled.seeks.length;
+ Object.defineProperty(stalled.video, "paused", { configurable: true, value: false });
+ for (let i = 0; i < 3; i++) {
+ await vi.advanceTimersByTimeAsync(50);
+ stalled.land();
+ if (i === 0) stalled.video.dispatchEvent(new Event("playing"));
+ stalled.presentAt(3 + (i + 1) / 30);
+ }
+ expect(stalled.seeks).toHaveLength(seeks);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
+ expect(window.__player!.isPlaying()).toBe(true);
+});
+
+it("keeps a paused recovery covered until its static target seek lands", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+ const { old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(3);
+ pending.shift()!();
+ await initial;
+ const stalled = old[0]!;
+ stalled.metadataAt(3);
+ let error: { code: number } | null = { code: 4 };
+ Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
+ window.__player!.play();
+ old[1]!.land();
+ await vi.advanceTimersByTimeAsync(500);
+ expect(window.__player!.isPlaying()).toBe(true);
+ window.__hf!.setPausedVideoFrameProvider!(null);
+ window.__player!.pause();
+ const picture = document.querySelector("[data-hf-preview-video-canvas]");
+ expect(picture).not.toBeNull();
+ error = null;
+ stalled.metadataAt(3);
+ stalled.land();
+ expect(stalled.video.previousElementSibling).toBe(picture);
+ expect(stalled.video.currentTime).toBeCloseTo(3.5, 2);
+ stalled.land();
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
+});
+
+it("publishes a late native recovery after a paused fallback wait has settled", async () => {
+ resetSeekDispatchState();
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ const { main, old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(1);
+ pending.shift()!();
+ await initial;
+ vi.mocked(provider.decode).mockRejectedValueOnce(new Error("decoder unavailable"));
+ const stalled = window.__player!.seek(6);
+ await vi.advanceTimersByTimeAsync(5100);
+ await stalled;
+ expect(main.time?.()).toBe(1);
+ expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
+ expect(window.__player!.getTime()).toBe(1);
+ const priorSeeks = old[0]!.seeks.length;
+ const destinationSeeks = next.map((video) => video.seeks.length);
+ old[0]!.reload();
+ expect(old[0]!.seeks.length).toBeGreaterThan(priorSeeks);
+ expect(next.map((video) => video.seeks.length)).toEqual(destinationSeeks);
+ for (const video of next) video.land();
+ await Promise.resolve();
+ expect(main.time?.()).toBe(6);
+ expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
+ expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
+});
+
+it("keeps a hold-only snapshot visible while Play awaits decoder suspension", async () => {
+ const { next, old } = film();
+ const { provider } = frameProvider([...old, ...next]);
+ let suspended = () => {};
+ provider.suspend = () =>
+ new Promise((resolve) => {
+ suspended = resolve;
+ });
+ const seek = window.__player!.seek(6);
+ const pictures = [...document.querySelectorAll("canvas")];
+ expect(pictures).toHaveLength(2);
+ window.__player!.play();
+ await seek;
+ expect([...document.querySelectorAll("canvas")]).toEqual(pictures);
+ suspended();
+ await Promise.resolve();
+ for (const video of next) video.land();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
+});
+
+it("resumes a pending Play after its frame provider is replaced", async () => {
+ const { old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(1);
+ pending.shift()!();
+ await initial;
+ window.__player!.play();
+ window.__hf!.setPausedVideoFrameProvider!({ decode: provider.decode, dispose: vi.fn() });
+ pending.shift()!();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ for (const video of old) video.land();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
+});
+
+it("cancels the hard Play deadline when Pause restores provider ownership", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+ const { old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(3);
+ pending.shift()!();
+ await initial;
+ window.__player!.play();
+ await vi.advanceTimersByTimeAsync(9000);
+ window.__player!.pause();
+ pending.shift()!();
+ await vi.advanceTimersByTimeAsync(20000);
+ expect(window.__player!.isPlaying()).toBe(false);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+});
+
+it("bounds a deferred decoder release and ignores its late completion", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+ const { old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ let released = () => {};
+ provider.suspend = () =>
+ new Promise((resolve) => {
+ released = resolve;
+ });
+ const initial = window.__player!.seek(3);
+ pending.shift()!();
+ await initial;
+ window.__player!.play();
+ await vi.advanceTimersByTimeAsync(10000);
+ expect(window.__player!.isPlaying()).toBe(true);
+ const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
+ expect(pictures).toHaveLength(2);
+ released();
+ await Promise.resolve();
+ await Promise.resolve();
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
+});
+
+it("ignores decoder release after Pause cancels a pending Play", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+ const { old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ let released = () => {};
+ provider.suspend = () =>
+ new Promise((resolve) => {
+ released = resolve;
+ });
+ const initial = window.__player!.seek(3);
+ pending.shift()!();
+ await initial;
+ window.__player!.play();
+ await vi.advanceTimersByTimeAsync(9000);
+ window.__player!.pause();
+ pending.shift()!();
+ await Promise.resolve();
+ await Promise.resolve();
+ const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
+ const seeks = old.map((video) => video.seeks.length);
+ released();
+ await Promise.resolve();
+ await Promise.resolve();
+ expect(old.map((video) => video.seeks.length)).toEqual(seeks);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(window.__player!.isPlaying()).toBe(false);
+ expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
+});
+
+it("does not wait for a video frame from an audio-only video element", async () => {
+ const { main, next } = film();
+ for (const { video } of next)
+ Object.defineProperties(video, {
+ readyState: { value: 1, configurable: true },
+ videoWidth: { value: 0, configurable: true },
+ error: { value: null, configurable: true },
+ });
+ expect(window.__player!.seek(6)).toBeUndefined();
+ expect(main.time?.()).toBe(6);
+ window.__player!.play();
+ expect(window.__player!.isPlaying()).toBe(true);
+});
+
+it("waits for paused decoder suspension before preparing native playback", async () => {
+ const { old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ let releaseDecoders = () => {};
+ provider.suspend = vi.fn(
+ () =>
+ new Promise((resolve) => {
+ releaseDecoders = resolve;
+ }),
+ );
+ const initial = window.__player!.seek(1);
+ pending.shift()!();
+ await initial;
+ const seeks = old.map((v) => v.seeks.length);
+ window.__player!.play();
+ expect(provider.suspend).toHaveBeenCalledOnce();
+ expect(old.map((v) => v.seeks.length)).toEqual(seeks);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+ releaseDecoders();
+ await Promise.resolve();
+ for (const video of old) video.land();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(window.__player!.isPlaying()).toBe(true);
+});
+
+it("restores paused ownership when Pause cancels an unfinished decoder suspension", async () => {
+ const { old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ let releaseDecoders = () => {};
+ provider.suspend = () =>
+ new Promise((resolve) => {
+ releaseDecoders = resolve;
+ });
+ const initial = window.__player!.seek(1);
+ pending.shift()!();
+ await initial;
+ window.__player!.play();
+ window.__player!.pause();
+ expect(pending).toHaveLength(1);
+ pending.shift()!();
+ releaseDecoders();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(window.__player!.isPlaying()).toBe(false);
+ expect(
+ old.every(({ video }) => Object.hasOwn(video, "currentTime") && video.currentTime === 1),
+ ).toBe(true);
+});
+
+it("warms the restart destination before Play when paused at the end", async () => {
+ const { main, old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const landed = window.__player!.seek(10, { settled: true });
+ pending.shift()!();
+ await landed;
+ expect(main.time?.()).toBe(10);
+ for (const video of old) video.metadataAt(0);
+ const seeks = old.map((v) => v.seeks.length);
+ window.__player!.play();
+ expect(old.map((v) => v.seeks.length)).toEqual(seeks.map((n) => n + 1));
+ expect(window.__player!.isPlaying()).toBe(false);
+ for (const video of old) video.land();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(window.__player!.isPlaying()).toBe(true);
+ expect(window.__player!.getTime()).toBeLessThan(1);
+ expect(old.every(({ video }) => video.style.visibility === "visible")).toBe(true);
+});
+
+it("retains a complete paused picture while replacing or removing its frame provider", async () => {
+ const { main, old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ const initial = window.__player!.seek(1);
+ pending.shift()!();
+ await initial;
+ const replacement = { decode: provider.decode, dispose: vi.fn() };
+ window.__hf!.setPausedVideoFrameProvider!(replacement);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+ pending.shift()!();
+ await Promise.resolve();
+ await Promise.resolve();
+ expect(main.time?.()).toBe(1);
+ window.__hf!.setPausedVideoFrameProvider!(null);
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
+ for (const video of old) video.land();
+ await Promise.resolve();
+ expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
+});
+
+it("retains an in-flight destination when the host replaces its frame provider", async () => {
+ const { main, old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ const first = window.__player!.seek(6);
+ const picture = document.querySelector("[data-hf-preview-video-canvas]");
+ expect(picture).not.toBeNull();
+ const replacement = { decode: provider.decode, dispose: vi.fn() };
+ window.__hf!.setPausedVideoFrameProvider!(replacement);
+ expect(document.querySelector("[data-hf-preview-video-canvas]")).toBe(picture);
+ expect(provider.dispose).toHaveBeenCalledTimes(1);
+ expect(window.__player!.getTime()).toBe(6);
+ pending.shift()!();
+ pending.shift()!();
+ await first;
+ await Promise.resolve();
+ expect(main.time?.()).toBe(6);
+});
+
+it("loads original source metadata before publishing provider frames used by intrinsic-size effects", async () => {
+ const { main, old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ for (const { video } of next)
+ Object.defineProperty(video, "videoWidth", { value: 0, configurable: true });
+ const landed = window.__player!.seek(6);
+ pending.shift()!();
+ await Promise.resolve();
+ expect(main.time?.()).toBe(0);
+ for (const { video } of next) {
+ Object.defineProperty(video, "videoWidth", { value: 64, configurable: true });
+ video.dispatchEvent(new Event("loadedmetadata"));
+ }
+ await landed;
+ expect(main.time?.()).toBe(6);
+});
+
+it("refines the paused frame once after input settles, then warms native playback", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ const { old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ const landed = window.__player!.seek(1);
+ pending.shift()!();
+ await landed;
+ await vi.advanceTimersByTimeAsync(120);
+ expect(provider.decode).toHaveBeenCalledTimes(2);
+ expect(vi.mocked(provider.decode).mock.calls[1]![0].every((request) => request.settled)).toBe(
+ true,
+ );
+ pending.shift()!();
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(120);
+ expect(old.every(({ video }) => !Object.hasOwn(video, "currentTime"))).toBe(true);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(provider.decode).toHaveBeenCalledTimes(2);
+ expect(window.__player!.isPlaying()).toBe(false);
+});
+
+it("decodes frame-step seeks at settled quality immediately without a second refinement", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ const { old, next } = film();
+ const { provider, pending } = frameProvider([...old, ...next]);
+ const landed = window.__player!.seek(1, { settled: true });
+ expect(vi.mocked(provider.decode).mock.calls[0]![0].every((request) => request.settled)).toBe(
+ true,
+ );
+ pending.shift()!();
+ await landed;
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(provider.decode).toHaveBeenCalledTimes(1);
+ expect(window.__player!.isPlaying()).toBe(false);
+});
+
+it("notifies video-driven effects on the committed clock before animation redraw", async () => {
+ const proto = HTMLVideoElement.prototype;
+ const request = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
+ const cancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
+ Object.defineProperties(proto, {
+ requestVideoFrameCallback: { value: vi.fn(() => 1), configurable: true, writable: true },
+ cancelVideoFrameCallback: { value: vi.fn(), configurable: true, writable: true },
+ });
+ try {
+ const { main, old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const notifications: number[] = [];
+ old[0]!.video.requestVideoFrameCallback(() => notifications.push(window.__player!.getTime()));
+ const original = main.totalTime!;
+ main.totalTime = (time?: number) => {
+ if (time === 1) expect(notifications).toEqual([1]);
+ return original(time);
+ };
+ const landed = window.__player!.seek(1);
+ pending.shift()!();
+ await landed;
+ expect(notifications).toEqual([1]);
+ } finally {
+ if (request) Object.defineProperty(proto, "requestVideoFrameCallback", request);
+ else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
+ if (cancel) Object.defineProperty(proto, "cancelVideoFrameCallback", cancel);
+ else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
+ }
+});
+
afterEach(() => {
window.__hfRuntimeTeardown?.();
document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`)?.remove();
@@ -281,3 +883,21 @@ afterEach(() => {
vi.useRealTimers();
resetRuntimeFixtureDom();
});
+
+it("releases a native hold when stop-media cancels its first provider seek", async () => {
+ const { old, next } = film();
+ const { pending } = frameProvider([...old, ...next]);
+ const seek = window.__player!.seek(3);
+ expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(2);
+ window.dispatchEvent(
+ new MessageEvent("message", {
+ source: window,
+ data: { source: "hf-parent", type: "control", action: "stop-media" },
+ }),
+ );
+ await seek;
+ expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
+ pending.shift()!();
+ await Promise.resolve();
+ expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
+});
diff --git a/packages/core/src/runtime/init.swapScenes.test.ts b/packages/core/src/runtime/init.swapScenes.test.ts
index 47bb27a8d2..f4654db628 100644
--- a/packages/core/src/runtime/init.swapScenes.test.ts
+++ b/packages/core/src/runtime/init.swapScenes.test.ts
@@ -1879,6 +1879,8 @@ describe("the preview's scene window", () => {
return ready.has(this) ? 4 : 1;
});
vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
+ vi.spyOn(HTMLVideoElement.prototype, "videoWidth", "get").mockReturnValue(64);
+ vi.spyOn(HTMLVideoElement.prototype, "videoHeight", "get").mockReturnValue(36);
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
const { root } = trackingRoot();
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index ad2e8b23a6..af3d73800f 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -29,6 +29,7 @@ import {
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
import { hasFrozenPicture } from "./frozenPicture";
import { createPreviewSeek } from "./previewSeek";
+import { createPausedVideoFrames, releasePausedVideoFrame } from "./pausedVideoFrames";
import { copiesReady } from "./previewCopies";
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
import { instantTolerance } from "../clipFacts";
@@ -77,6 +78,8 @@ import { createFrameSourceAdapter } from "./frameSources";
import {
patchVideoTextureCompat,
patchWebGLVideoTextureCompat,
+ patchCanvasVideoFrameCompat,
+ patchExternalVideoTextureCompat,
} from "./adapters/video-texture-compat";
import {
forceDispatchSeekEvent,
@@ -530,6 +533,10 @@ function loadPreviewImagesIn(root: Element): void {
export function initSandboxRuntimeModular(): void {
const state = createRuntimeState();
let previewSeek: ReturnType | undefined;
+ const pausedFrames = createPausedVideoFrames();
+ let nativePlayGeneration = 0;
+ let nativePlayTime: number | undefined;
+ let nativeWarmTimer: ReturnType | undefined;
const stampsForStudio = window.parent !== window;
// Studio's discovery stamps time a clip for the timeline only; nothing hides it at their end.
const isDiscoveryStamped = (el: Element) =>
@@ -740,6 +747,13 @@ export function initSandboxRuntimeModular(): void {
const registerRuntimeCleanup = (callback: () => void) => {
runtimeCleanupCallbacks.push(callback);
};
+ registerRuntimeCleanup(() => {
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ wallClearTimeout(nativeWarmTimer);
+ pausedFrames.destroy();
+ delete window.__hf?.setPausedVideoFrameProvider;
+ });
const postRuntimeDiagnosticOnce = (
code: string,
details: Record,
@@ -761,7 +775,7 @@ export function initSandboxRuntimeModular(): void {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
- seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
+ seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => HeldSeek;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
@@ -3687,6 +3701,7 @@ export function initSandboxRuntimeModular(): void {
* exists for. On `window.__hf` because the Studio is across the iframe boundary. */
const pausedMediaLeases = new WeakSet();
const leasePausedMedia = (el: HTMLMediaElement): void => {
+ if (isVideoElement(el)) releasePausedVideoFrame(el);
pausedMediaLeases.add(el);
};
const releasePausedMedia = (el: HTMLMediaElement): void => {
@@ -3695,10 +3710,33 @@ export function initSandboxRuntimeModular(): void {
};
window.__hf.leasePausedMedia = leasePausedMedia;
window.__hf.releasePausedMedia = releasePausedMedia;
+ window.__hf.setPausedVideoFrameProvider = (provider) => {
+ const resumePlay = nativePlayTime !== undefined;
+ const destination =
+ previewSeek?.targetTime ?? (pausedFrames.active ? state.currentTime : undefined);
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ wallClearTimeout(nativeWarmTimer);
+ previewSeek?.cancel(true);
+ pausedFrames.setProvider(provider);
+ if (!provider) pausedFrames.prepareNative();
+ if (destination !== undefined) {
+ const landed = seekWhenReady(destination);
+ const generation = nativePlayGeneration;
+ if (resumePlay)
+ void Promise.resolve(landed).then(() => {
+ if (!state.tornDown && generation === nativePlayGeneration) transport.play();
+ });
+ }
+ };
// Clips the transport may move now (not a paused lease); null while the host drives media.
const transportMovableClips = (clips: RuntimeMediaClip[]): RuntimeMediaClip[] | null => {
if (state.nativeMediaSyncDisabled) return null;
- return state.isPlaying ? clips : clips.filter((clip) => !pausedMediaLeases.has(clip.el));
+ return state.isPlaying
+ ? clips
+ : clips.filter(
+ (clip) => !pausedMediaLeases.has(clip.el) && !previewSeek?.ownsRecovery(clip.el),
+ );
};
window.__hf.setLoopStart = (seconds) => {
loopStartSeconds = seconds != null && Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
@@ -3784,7 +3822,7 @@ export function initSandboxRuntimeModular(): void {
let lastVisibilityPassSeconds: number | null = null;
const syncMediaForCurrentState = (timingRevision?: number) => {
- if (previewSeek?.time !== undefined) return;
+ if (previewSeek?.time !== undefined && !previewSeek.committing) return;
// Scope 1 of 3 (see `withTimingResolver`). Closes before `syncRuntimeMedia`,
// which may call `el.load()` and invalidate every cached duration.
//
@@ -4493,7 +4531,7 @@ export function initSandboxRuntimeModular(): void {
// Redraw the current frame as a seek does; a playing film cut to end at or before it stops there, as at its end.
const cutShort = clock.isPlaying() && duration > 0 && state.currentTime >= duration;
if (joining && clock.isPlaying() && !cutShort) joinPlayback(swappedHosts);
- else if (previewSeek?.time !== undefined) previewSeek.refresh();
+ else if (previewSeek?.targetTime !== undefined) previewSeek.refresh();
else transport.seek(cutShort ? duration : state.currentTime, { keepPlaying: !cutShort });
if (previewSeek?.time === undefined) syncTimedElementVisibility(state.currentTime);
postTimeline();
@@ -4803,7 +4841,7 @@ export function initSandboxRuntimeModular(): void {
.sort((a, b) => Math.abs(a.start - time) - Math.abs(b.start - time))[0];
const toMount = landed ?? neighbour;
const toRetire =
- toMount || previewSeek?.time !== undefined
+ toMount || previewSeek?.targetTime !== undefined
? []
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
if (!toMount && toRetire.length === 0) {
@@ -4812,9 +4850,11 @@ export function initSandboxRuntimeModular(): void {
}
const wantedNow = (name: string) =>
heldScenes.has(name) ||
- scenesWanted(windowScenes, previewSeek?.time ?? state.currentTime, clock.isPlaying()).has(
- name,
- );
+ scenesWanted(
+ windowScenes,
+ previewSeek?.targetTime ?? state.currentTime,
+ clock.isPlaying(),
+ ).has(name);
const parkedBeforeStep = parkedNames.size;
windowWork = (async () => {
if (landed) await mountScene(landed.name);
@@ -4827,7 +4867,7 @@ export function initSandboxRuntimeModular(): void {
windowWork = null;
const stuckStepKeepsItsPlan = parkedNames.size === parkedBeforeStep;
windowNear = stuckStepKeepsItsPlan ? key : "";
- moveSceneWindow(previewSeek?.time ?? state.currentTime);
+ moveSceneWindow(previewSeek?.targetTime ?? state.currentTime);
});
// A seek is not done until the scene it lands in is mounted.
if (landed) registerSeekCompletion(windowWork, "scene-window");
@@ -4934,6 +4974,7 @@ export function initSandboxRuntimeModular(): void {
if (!keepPlaying) syncMediaForCurrentState();
const tl = state.capturedTimeline;
pauseTimelineIfPossible(tl);
+ pausedFrames.notify();
const pageAnimations = seekTimelineAndAdapters(state.currentTime);
if (!keepPlaying) decideClipsUnderScriptHidden(state.currentTime);
runAdapters("pause", 0, pageAnimations);
@@ -4968,7 +5009,65 @@ export function initSandboxRuntimeModular(): void {
postState(true);
};
+ const previewVideoClipsAt = (time: number) =>
+ withTimingResolver(() => {
+ const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
+ const visibleAt = timedVisibilityAt();
+ const root = resolveRootCompositionElement();
+ return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
+ (clip) =>
+ isVideoElement(clip.el) &&
+ !(clip.el.readyState >= 1 && clip.el.videoWidth === 0) &&
+ !isSilencedByHidden(clip.el) &&
+ (clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
+ isClipVisibleAt(time, clip.start, clip.end, duration) &&
+ clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
+ );
+ });
+
previewSeek = createPreviewSeek({
+ frames: {
+ available: pausedFrames.available,
+ active: () => pausedFrames.active,
+ skipped: pausedFrames.skipped,
+ hold: (videos) => {
+ if (pausedFrames.available()) return pausedFrames.holdNative(videos);
+ },
+ prepare: (clips, time, signal, settled) => {
+ moveSceneWindow(time);
+ for (const { el } of clips) {
+ if (!isVideoElement(el) || el.videoWidth > 0) continue;
+ if (el.preload === "none") el.preload = "metadata";
+ if (el.networkState !== 2) el.load();
+ }
+ const requests = clips.flatMap((clip) =>
+ isVideoElement(clip.el)
+ ? [
+ {
+ video: clip.el,
+ source: clip.el.src || clip.el.currentSrc,
+ time: mediaSourceTimeAt(clip, time),
+ settled,
+ },
+ ]
+ : [],
+ );
+ return pausedFrames.prepare(requests, signal);
+ },
+ suspend: pausedFrames.suspendProvider,
+ prepareNative: pausedFrames.prepareNative,
+ clear: (keep) =>
+ pausedFrames.clear(keep, (video, presented) => {
+ if (clock.isPlaying()) return presented;
+ const time = clock.now();
+ const clip = previewVideoClipsAt(time).find(({ el }) => el === video);
+ if (!clip) return true;
+ const expected = mediaSourceTimeAt(clip, time);
+ if (Math.abs(video.currentTime - expected) < 0.02) return true;
+ video.currentTime = mediaElementTime(expected);
+ return false;
+ }),
+ },
videosToHold: () =>
withTimingResolver(() =>
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? [])
@@ -4978,23 +5077,19 @@ export function initSandboxRuntimeModular(): void {
scenesReadyAt: (time) =>
!windowScenes.some(
({ name, start, end }) => start <= time && time <= end && parkedNames.has(name),
+ ) &&
+ previewVideoClipsAt(time).every(
+ ({ el }) => !isVideoElement(el) || el.videoWidth > 0 || isUnplayable(el),
),
- clipsAt: (time) =>
- withTimingResolver(() => {
- const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
- const visibleAt = timedVisibilityAt();
- const root = resolveRootCompositionElement();
- return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
- (clip) =>
- isVideoElement(clip.el) &&
- !isSilencedByHidden(clip.el) &&
- (clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
- isClipVisibleAt(time, clip.start, clip.end, duration) &&
- clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
- );
- }),
+ clipsAt: previewVideoClipsAt,
prepare: (clips, time) => {
moveSceneWindow(time);
+ for (const clip of clips) {
+ const { el } = clip;
+ if (el.preload !== "auto") el.preload = "auto";
+ if (el.readyState === 1 && !el.seeking)
+ el.currentTime = mediaElementTime(mediaSourceTimeAt(clip, time));
+ }
syncRuntimeMedia({
clips,
timeSeconds: time,
@@ -5004,10 +5099,30 @@ export function initSandboxRuntimeModular(): void {
getCompositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
});
},
- commit: (time) => applySeek(time),
+ commit: (time) => {
+ applySeek(time);
+ wallClearTimeout(nativeWarmTimer);
+ const refined = previewSeek?.refining;
+ if (pausedFrames.active)
+ nativeWarmTimer = wallSetTimeout(() => {
+ if (state.tornDown || clock.isPlaying() || previewSeek?.time !== undefined) return;
+ if (!refined) {
+ const landed = previewSeek?.refine(time);
+ if (landed) registerSeekCompletion(landed, "video-frame");
+ return;
+ }
+ pausedFrames.prepareNative();
+ state.mediaForceSyncNextTick = true;
+ syncMediaForCurrentState();
+ }, 120);
+ },
});
- const seekWhenReady = (time: number, options?: { keepPlaying?: boolean }) => {
+ const seekWhenReady = (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => {
+ if (nativePlayTime !== undefined && Math.abs(time - nativePlayTime) < 1e-5) return;
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ wallClearTimeout(nativeWarmTimer);
if (
!mediaDeferred ||
state.nativeMediaSyncDisabled ||
@@ -5016,6 +5131,7 @@ export function initSandboxRuntimeModular(): void {
(options?.keepPlaying && clock.isPlaying())
) {
previewSeek?.cancel();
+ pausedFrames.clear();
applySeek(time, options);
return;
}
@@ -5030,7 +5146,7 @@ export function initSandboxRuntimeModular(): void {
for (const { el } of transportMovableClips(clips) ?? []) {
if (!el.paused) el.pause();
}
- const landed = previewSeek?.seek(time);
+ const landed = options?.settled ? previewSeek?.refine(time) : previewSeek?.seek(time);
if (landed) registerSeekCompletion(landed, "video-frame");
postState(true);
return landed;
@@ -5039,6 +5155,33 @@ export function initSandboxRuntimeModular(): void {
const transport: RuntimePlayerTransport = {
play: () => {
flushHeldSeek();
+ if (
+ (pausedFrames.active || (pausedFrames.available() && previewSeek?.time !== undefined)) &&
+ !clock.isPlaying()
+ ) {
+ wallClearTimeout(nativeWarmTimer);
+ const generation = ++nativePlayGeneration;
+ const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
+ if (duration > 0) clock.setDuration(duration);
+ nativePlayTime =
+ previewSeek?.targetTime ??
+ (clock.reachedEnd() ? clock.getPlayStart() : state.currentTime);
+ const landed = previewSeek?.handoff(nativePlayTime);
+ if (landed) {
+ bufferingFor = withTimingResolver(() =>
+ (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).map(({ el }) => el),
+ );
+ postState(true);
+ void landed.then(() => {
+ if (state.tornDown || generation !== nativePlayGeneration) return;
+ bufferingFor = [];
+ nativePlayTime = undefined;
+ transport.play();
+ });
+ return;
+ }
+ nativePlayTime = undefined;
+ }
const destination = previewSeek?.time;
previewSeek?.cancel();
if (destination !== undefined) applySeek(destination);
@@ -5076,9 +5219,22 @@ export function initSandboxRuntimeModular(): void {
postState(true);
},
pause: () => {
+ const destination = previewSeek?.targetTime ?? nativePlayTime;
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ previewSeek?.cancel(true);
const wasBuffering = bufferingFor.length > 0;
bufferingFor = [];
- if (clock.isPlaying()) haltPlayback();
+ if (clock.isPlaying()) {
+ if (
+ mediaDeferred &&
+ pausedFrames.available() &&
+ !state.nativeMediaSyncDisabled &&
+ !isExportRenderDrivingFrames()
+ )
+ void seekWhenReady(clock.now());
+ else haltPlayback();
+ } else if (destination !== undefined) void seekWhenReady(destination);
else if (wasBuffering) postState(true);
},
seek: (timeSeconds, options) => {
@@ -5086,6 +5242,10 @@ export function initSandboxRuntimeModular(): void {
Math.max(0, Number(timeSeconds) || 0),
state.canonicalFps,
);
+ if (nativePlayTime !== undefined && Math.abs(quantized - nativePlayTime) < 1e-5) return;
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ wallClearTimeout(nativeWarmTimer);
heldSeek = null;
const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
if (undecoded.length === 0) {
@@ -5116,6 +5276,10 @@ export function initSandboxRuntimeModular(): void {
return landed;
},
renderSeek: (timeSeconds, options) => {
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ wallClearTimeout(nativeWarmTimer);
+ pausedFrames.clear();
heldSeek = null;
previewSeek?.cancel();
renderCaptureSeekStarted = true;
@@ -5142,7 +5306,8 @@ export function initSandboxRuntimeModular(): void {
if (blurred) registerSeekCompletion(blurred, "motion-blur");
postState(true);
},
- getTime: () => heldSeek?.time ?? previewSeek?.time ?? clock.now(),
+ getTime: () =>
+ previewSeek?.committing ? clock.now() : (heldSeek?.time ?? previewSeek?.time ?? clock.now()),
getDuration: () => {
const dur = clock.getDuration();
return Number.isFinite(dur) ? dur : 0;
@@ -5239,6 +5404,8 @@ export function initSandboxRuntimeModular(): void {
] as RuntimeDeterministicAdapter[];
patchVideoTextureCompat();
patchWebGLVideoTextureCompat();
+ patchCanvasVideoFrameCompat();
+ patchExternalVideoTextureCompat();
// Lets the engine re-render GPU compositions after it injects decoded video
// frames, so video-textured WebGL/WebGPU scenes sample the correct frame.
window.__hfReseekGpu = (time: number) => {
@@ -5969,6 +6136,7 @@ export function initSandboxRuntimeModular(): void {
!ready(el) &&
isPlayableTrack(el) &&
!stillStandsIn(el) &&
+ !pausedFrames.retained(el) &&
!webAudio.ownsElement(el),
)
.map(({ el }) => el);
@@ -6388,7 +6556,11 @@ export function initSandboxRuntimeModular(): void {
emitAnalyticsEvent("composition_paused", { time: player.getTime() });
},
onStopMedia: () => {
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ wallClearTimeout(nativeWarmTimer);
previewSeek?.cancel();
+ bufferingFor = [];
webAudio.stopAll();
const mediaEls = document.querySelectorAll("video, audio");
for (const el of mediaEls) {
@@ -6440,6 +6612,14 @@ export function initSandboxRuntimeModular(): void {
}
},
onSetNativeMediaSyncDisabled: (disabled) => {
+ if (disabled) {
+ nativePlayGeneration++;
+ nativePlayTime = undefined;
+ wallClearTimeout(nativeWarmTimer);
+ previewSeek?.cancel();
+ pausedFrames.clear();
+ bufferingFor = [];
+ }
if (state.nativeMediaSyncDisabled === disabled) return;
state.nativeMediaSyncDisabled = disabled;
state.mediaForceSyncNextTick = true;
diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts
index 9ce52a3d64..253b2dc6ac 100644
--- a/packages/core/src/runtime/media.ts
+++ b/packages/core/src/runtime/media.ts
@@ -12,6 +12,7 @@ import type { RuntimeMediaClip } from "./types.js";
import { clampAudioGain, clipGain, gainTimesLane } from "../audioGain.js";
import { isMemberGroupHidden } from "../audioGroups.js";
import { findInjectedRenderFrame } from "./renderFrameSibling.js";
+import { ownsPausedVideoTime } from "./pausedVideoFrames";
import { isRenderFrameVideo } from "./preloadMedia.js";
import { registerSeekCompletion } from "./adapters/seek-dispatch.js";
import { cutToHostSlots, type HostSlot } from "../mediaTiming.js";
@@ -683,6 +684,10 @@ export function syncRuntimeMedia(params: {
// (no-op when already "auto") and catches elements whose preload
// was overridden after init.ts set it.
if (el.preload !== "auto" && !isRenderFrameVideo(el)) el.preload = "auto";
+ if (!params.playing && ownsPausedVideoTime(el)) {
+ if (!el.paused) el.pause();
+ continue;
+ }
// Per-element rate × global transport rate
const baseRate = rateAt(clipRate, params.timeSeconds - origin) * params.playbackRate;
// Too slow for the browser to play: kept paused and stepped onto the playhead by seeks.
diff --git a/packages/core/src/runtime/pausedVideoCallbacks.test.ts b/packages/core/src/runtime/pausedVideoCallbacks.test.ts
new file mode 100644
index 0000000000..df15852e4a
--- /dev/null
+++ b/packages/core/src/runtime/pausedVideoCallbacks.test.ts
@@ -0,0 +1,329 @@
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import {
+ flushPausedVideoCallbacks,
+ installPausedVideoCallbacks,
+ observeNativeVideoFrames,
+ releasePausedVideoCallbacks,
+ stagePausedVideoCallbacks,
+} from "./pausedVideoCallbacks";
+
+const proto = HTMLVideoElement.prototype;
+const originalRequest = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
+const originalCancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
+const pending = new Map();
+let nextHandle = 100;
+const nativeRequest = vi.fn(function (this: HTMLVideoElement, callback: VideoFrameRequestCallback) {
+ const handle = ++nextHandle;
+ pending.set(handle, { video: this, callback });
+ return handle;
+});
+const nativeCancel = vi.fn(function (this: HTMLVideoElement, handle: number) {
+ if (pending.get(handle)?.video === this) pending.delete(handle);
+});
+
+beforeEach(() => {
+ pending.clear();
+ nextHandle = 100;
+ nativeRequest.mockClear();
+ nativeCancel.mockClear();
+ Object.defineProperty(proto, "requestVideoFrameCallback", {
+ configurable: true,
+ writable: true,
+ value: nativeRequest,
+ });
+ Object.defineProperty(proto, "cancelVideoFrameCallback", {
+ configurable: true,
+ writable: true,
+ value: nativeCancel,
+ });
+ installPausedVideoCallbacks();
+});
+afterEach(() => {
+ if (originalRequest) Object.defineProperty(proto, "requestVideoFrameCallback", originalRequest);
+ else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
+ if (originalCancel) Object.defineProperty(proto, "cancelVideoFrameCallback", originalCancel);
+ else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
+ pending.clear();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+});
+
+function metadata(mediaTime: number, presentedFrames: number): VideoFrameCallbackMetadata {
+ return {
+ mediaTime,
+ presentedFrames,
+ width: 1920,
+ height: 1080,
+ presentationTime: 123,
+ expectedDisplayTime: 125,
+ processingDuration: 0.003,
+ };
+}
+function deliverNative(handle: number, frame: VideoFrameCallbackMetadata) {
+ const request = pending.get(handle);
+ expect(request).toBeDefined();
+ pending.delete(handle);
+ request!.callback(120, frame);
+}
+const frame = (mediaTime: number) => ({ mediaTime, width: 1920, height: 1080 });
+
+it("forwards native timing and cancellation when no paused frame owns the video", () => {
+ const video = document.createElement("video");
+ const callback = vi.fn();
+ const id = video.requestVideoFrameCallback(callback);
+ expect(nativeRequest.mock.contexts[0]).toBe(video);
+ expect(id).toBe(101);
+ const nativeFrame = metadata(0.5, 15);
+ deliverNative(id, nativeFrame);
+ expect(callback).toHaveBeenCalledExactlyOnceWith(120, nativeFrame);
+ expect(callback.mock.lastCall?.[1]).toBe(nativeFrame);
+ const cancelled = video.requestVideoFrameCallback(callback);
+ video.cancelVideoFrameCallback(cancelled);
+ expect(nativeCancel).toHaveBeenLastCalledWith(cancelled);
+ expect(nativeCancel.mock.contexts.at(-1)).toBe(video);
+ expect(pending.has(cancelled)).toBe(false);
+ video.cancelVideoFrameCallback(999);
+ expect(nativeCancel).toHaveBeenLastCalledWith(999);
+});
+
+it("reports one presented frame to multiple native observers without counting each callback as a frame", () => {
+ const video = document.createElement("video");
+ const first = vi.fn();
+ const second = vi.fn();
+ const firstId = video.requestVideoFrameCallback(first);
+ const secondId = video.requestVideoFrameCallback(second);
+ deliverNative(firstId, metadata(0.2, 6));
+ deliverNative(secondId, metadata(0.2, 6));
+ expect(first.mock.lastCall?.[1].presentedFrames).toBe(6);
+ expect(second.mock.lastCall?.[1].presentedFrames).toBe(6);
+ const afterReload = vi.fn();
+ const next = video.requestVideoFrameCallback(afterReload);
+ deliverNative(next, metadata(0, 1));
+ expect(afterReload.mock.lastCall?.[1].presentedFrames).toBe(1);
+});
+
+it("preserves native callback receiver and reports authored failures without skipping other observers", () => {
+ const video = document.createElement("video");
+ const callback = vi.fn();
+ const id = video.requestVideoFrameCallback(callback);
+ const receiver = {};
+ const native = pending.get(id)!;
+ native.callback.call(receiver, 120, metadata(0.2, 6));
+ expect(callback.mock.contexts[0]).toBe(receiver);
+ const report = vi.fn();
+ vi.stubGlobal("reportError", report);
+ const failure = new Error("authored effect failed");
+ video.requestVideoFrameCallback(() => {
+ throw failure;
+ });
+ const later = vi.fn();
+ video.requestVideoFrameCallback(later);
+ stagePausedVideoCallbacks(video, frame(1));
+ flushPausedVideoCallbacks(video);
+ expect(report).toHaveBeenCalledExactlyOnceWith(failure);
+ expect(later).toHaveBeenCalledTimes(1);
+});
+
+it("publishes one-shot synthetic callbacks with the staged dimensions and source clock", () => {
+ vi.spyOn(performance, "now").mockReturnValue(300);
+ const video = document.createElement("video");
+ const callback = vi.fn();
+ const id = video.requestVideoFrameCallback(callback);
+ stagePausedVideoCallbacks(video, frame(3.5));
+ expect(callback).not.toHaveBeenCalled();
+ flushPausedVideoCallbacks(video);
+ expect(callback).toHaveBeenCalledExactlyOnceWith(300, {
+ ...frame(3.5),
+ presentedFrames: 1,
+ presentationTime: 300,
+ expectedDisplayTime: 300,
+ processingDuration: 0,
+ });
+ expect(nativeCancel).toHaveBeenCalledWith(id);
+ expect(pending.has(id)).toBe(false);
+ flushPausedVideoCallbacks(video);
+ stagePausedVideoCallbacks(video, frame(4));
+ flushPausedVideoCallbacks(video);
+ expect(callback).toHaveBeenCalledTimes(1);
+});
+
+it("defers a callback registered by a callback until the next published frame", () => {
+ const video = document.createElement("video");
+ const observed: number[] = [];
+ const callback: VideoFrameRequestCallback = (_now, info) => {
+ observed.push(info.mediaTime);
+ video.requestVideoFrameCallback(callback);
+ };
+ video.requestVideoFrameCallback(callback);
+ stagePausedVideoCallbacks(video, frame(1));
+ flushPausedVideoCallbacks(video);
+ flushPausedVideoCallbacks(video);
+ expect(observed).toEqual([1]);
+ stagePausedVideoCallbacks(video, frame(2));
+ flushPausedVideoCallbacks(video);
+ expect(observed).toEqual([1, 2]);
+ expect(pending.size).toBe(1);
+});
+
+it("honors cancellation of a later observer while a synthetic notification batch is running", () => {
+ const video = document.createElement("video");
+ const cancelled = vi.fn();
+ video.requestVideoFrameCallback(() => video.cancelVideoFrameCallback(later));
+ const later = video.requestVideoFrameCallback(cancelled);
+ stagePausedVideoCallbacks(video, frame(1));
+ flushPausedVideoCallbacks(video);
+ expect(cancelled).not.toHaveBeenCalled();
+ expect(nativeCancel).toHaveBeenCalledWith(later);
+ expect(pending.size).toBe(0);
+});
+
+it("suppresses native metadata while a paused frame owns the pixels and publishes the staged frame instead", () => {
+ const video = document.createElement("video");
+ const callback = vi.fn();
+ const id = video.requestVideoFrameCallback(callback);
+ stagePausedVideoCallbacks(video, frame(4));
+ deliverNative(id, metadata(0.5, 15));
+ expect(callback).not.toHaveBeenCalled();
+ flushPausedVideoCallbacks(video);
+ expect(callback).toHaveBeenCalledTimes(1);
+ expect(callback.mock.lastCall?.[1]).toMatchObject({ mediaTime: 4, presentedFrames: 1 });
+ expect(nativeCancel).not.toHaveBeenCalled();
+});
+
+it("rearms a native callback consumed during pause and still cancels it using the original public handle", () => {
+ const video = document.createElement("video");
+ const callback = vi.fn();
+ const publicId = video.requestVideoFrameCallback(callback);
+ stagePausedVideoCallbacks(video, frame(3));
+ deliverNative(publicId, metadata(1, 30));
+ releasePausedVideoCallbacks(video);
+ expect(callback).not.toHaveBeenCalled();
+ expect(nativeRequest).toHaveBeenCalledTimes(2);
+ const rearmedId = nativeRequest.mock.results.at(-1)!.value;
+ expect(rearmedId).not.toBe(publicId);
+ expect(pending.has(rearmedId)).toBe(true);
+ releasePausedVideoCallbacks(video);
+ expect(nativeRequest).toHaveBeenCalledTimes(2);
+ video.cancelVideoFrameCallback(publicId);
+ expect(nativeCancel).toHaveBeenLastCalledWith(rearmedId);
+ expect(pending.size).toBe(0);
+});
+
+it("resumes the pending observer on native frames after handoff", () => {
+ const video = document.createElement("video");
+ const callback = vi.fn();
+ const publicId = video.requestVideoFrameCallback(callback);
+ stagePausedVideoCallbacks(video, frame(3));
+ deliverNative(publicId, metadata(1, 30));
+ releasePausedVideoCallbacks(video);
+ const rearmedId = nativeRequest.mock.results.at(-1)!.value;
+ deliverNative(rearmedId, metadata(3.033, 31));
+ expect(callback).toHaveBeenCalledExactlyOnceWith(120, metadata(3.033, 31));
+ expect(pending.size).toBe(0);
+});
+
+it("installs once without double-wrapping request or cancellation", () => {
+ const request = proto.requestVideoFrameCallback;
+ const cancel = proto.cancelVideoFrameCallback;
+ installPausedVideoCallbacks();
+ expect(proto.requestVideoFrameCallback).toBe(request);
+ expect(proto.cancelVideoFrameCallback).toBe(cancel);
+ const callback = vi.fn();
+ const video = document.createElement("video");
+ video.requestVideoFrameCallback(callback);
+ stagePausedVideoCallbacks(video, frame(1));
+ flushPausedVideoCallbacks(video);
+ expect(callback).toHaveBeenCalledTimes(1);
+ expect(nativeRequest).toHaveBeenCalledTimes(1);
+});
+
+it("invalidates the actual vendored Three VideoTexture before redraw and continues natively after release", async () => {
+ const { VideoTexture } =
+ await import("../../../../registry/examples/path-traced-3d/assets/vendor/three-path-tracer.min.js");
+ const video = document.createElement("video");
+ Object.defineProperty(video, "readyState", { value: 4 });
+ const texture = new VideoTexture(video);
+ texture.update();
+ expect(texture.version).toBe(0);
+ stagePausedVideoCallbacks(video, frame(3));
+ flushPausedVideoCallbacks(video);
+ expect(texture.version).toBe(1);
+ texture.update();
+ expect(texture.version).toBe(1);
+ stagePausedVideoCallbacks(video, frame(4));
+ flushPausedVideoCallbacks(video);
+ expect(texture.version).toBe(2);
+ // The native decoder can finish a warm seek while provider pixels remain displayed.
+ const waitingId = [...pending.keys()][0]!;
+ deliverNative(waitingId, metadata(4, 120));
+ expect(texture.version).toBe(2);
+ releasePausedVideoCallbacks(video);
+ const resumedId = [...pending.keys()][0]!;
+ deliverNative(resumedId, metadata(4.033, 121));
+ expect(texture.version).toBe(3);
+ texture.dispose();
+ expect(pending.size).toBe(0);
+});
+
+it("observes native frames independently of staged authored callbacks", () => {
+ const video = document.createElement("video");
+ const authored = vi.fn();
+ video.requestVideoFrameCallback(authored);
+ stagePausedVideoCallbacks(video, frame(3));
+ const recovered = vi.fn(() => true);
+ const stop = observeNativeVideoFrames(video, recovered);
+ deliverNative(102, metadata(3.05, 2));
+ expect(recovered).toHaveBeenCalledOnce();
+ expect(authored).not.toHaveBeenCalled();
+ flushPausedVideoCallbacks(video);
+ expect(authored).toHaveBeenCalledOnce();
+ expect(authored.mock.lastCall?.[1].mediaTime).toBe(3);
+ stop();
+});
+
+it("cancels a re-armed native observer and ignores an already queued callback", () => {
+ const video = document.createElement("video");
+ const recovered = vi.fn(() => false);
+ const stop = observeNativeVideoFrames(video, recovered);
+ deliverNative(101, metadata(3, 1));
+ const queued = pending.get(102)!;
+ stop();
+ expect(pending.has(102)).toBe(false);
+ queued.callback(130, metadata(3.05, 2));
+ expect(recovered).toHaveBeenCalledOnce();
+ expect(pending.size).toBe(0);
+});
+
+it("uses playing and progressing time when opacity defers a native callback", () => {
+ const video = document.createElement("video");
+ let paused = true;
+ let time = 3;
+ Object.defineProperties(video, {
+ paused: { get: () => paused },
+ currentTime: { get: () => time },
+ });
+ const recovered = vi.fn(() => true);
+ const stop = observeNativeVideoFrames(video, recovered);
+ video.dispatchEvent(new Event("timeupdate"));
+ expect(recovered).not.toHaveBeenCalled();
+ paused = false;
+ video.dispatchEvent(new Event("playing"));
+ video.dispatchEvent(new Event("timeupdate"));
+ expect(recovered).not.toHaveBeenCalled();
+ time += 1 / 30;
+ video.dispatchEvent(new Event("timeupdate"));
+ expect(recovered).toHaveBeenCalledOnce();
+ expect(pending.size).toBe(0);
+ stop();
+});
+
+it("keeps waiting when no native frame or playback progress arrives", () => {
+ const video = document.createElement("video");
+ const recovered = vi.fn(() => true);
+ const stop = observeNativeVideoFrames(video, recovered);
+ for (let i = 0; i < 3; i++) video.dispatchEvent(new Event("timeupdate"));
+ expect(recovered).not.toHaveBeenCalled();
+ expect(pending.size).toBe(1);
+ stop();
+ expect(pending.size).toBe(0);
+});
diff --git a/packages/core/src/runtime/pausedVideoCallbacks.ts b/packages/core/src/runtime/pausedVideoCallbacks.ts
new file mode 100644
index 0000000000..f3515bd68d
--- /dev/null
+++ b/packages/core/src/runtime/pausedVideoCallbacks.ts
@@ -0,0 +1,171 @@
+type Frame = Pick;
+type Pending = { callback: VideoFrameRequestCallback; native: number | null };
+type State = {
+ pending: Map;
+ frame: Frame | null;
+ count: number;
+ nativeCount: number;
+ synthetic: boolean;
+ dirty: boolean;
+};
+type Request = typeof HTMLVideoElement.prototype.requestVideoFrameCallback;
+type Cancel = typeof HTMLVideoElement.prototype.cancelVideoFrameCallback;
+type Registry = { states: WeakMap; request: Request; cancel: Cancel };
+type TrackedRequest = Request & { __hfPausedCallbacks?: Registry };
+
+const registry = () =>
+ (HTMLVideoElement.prototype.requestVideoFrameCallback as TrackedRequest)?.__hfPausedCallbacks;
+
+function stateFor(owner: Registry, video: HTMLVideoElement): State {
+ let state = owner.states.get(video);
+ if (!state) {
+ state = {
+ pending: new Map(),
+ frame: null,
+ count: 0,
+ nativeCount: 0,
+ synthetic: false,
+ dirty: false,
+ };
+ owner.states.set(video, state);
+ }
+ return state;
+}
+
+function requestNative(
+ owner: Registry,
+ video: HTMLVideoElement,
+ state: State,
+ pending: Pending,
+ id: () => number,
+): number {
+ return owner.request.call(video, function (this: unknown, now, metadata) {
+ pending.native = null;
+ const advance = Math.max(0, metadata.presentedFrames - state.nativeCount);
+ state.nativeCount = metadata.presentedFrames;
+ if (state.frame) return;
+ state.pending.delete(id());
+ if (!state.synthetic) {
+ state.count = metadata.presentedFrames;
+ pending.callback.call(this, now, metadata);
+ return;
+ }
+ state.count += advance;
+ pending.callback.call(this, now, { ...metadata, presentedFrames: state.count });
+ });
+}
+
+/** Install before authored scripts: Three's VideoTexture registers its first callback in its constructor. */
+export function installPausedVideoCallbacks(): void {
+ const proto = HTMLVideoElement.prototype;
+ const request = proto.requestVideoFrameCallback as TrackedRequest;
+ if (typeof request !== "function" || request.__hfPausedCallbacks) return;
+ const owner: Registry = {
+ states: new WeakMap(),
+ request,
+ cancel: proto.cancelVideoFrameCallback,
+ };
+ const patched: TrackedRequest = function (this: HTMLVideoElement, callback) {
+ const state = stateFor(owner, this);
+ const pending: Pending = { callback, native: null };
+ const id = requestNative(owner, this, state, pending, () => id);
+ pending.native = id;
+ state.pending.set(id, pending);
+ return id;
+ };
+ patched.__hfPausedCallbacks = owner;
+ proto.requestVideoFrameCallback = patched;
+ proto.cancelVideoFrameCallback = function (id) {
+ const pending = owner.states.get(this)?.pending.get(id);
+ if (!pending) return owner.cancel.call(this, id);
+ owner.states.get(this)?.pending.delete(id);
+ if (pending.native !== null) owner.cancel.call(this, pending.native);
+ };
+}
+
+export function stagePausedVideoCallbacks(video: HTMLVideoElement, frame: Frame): void {
+ const owner = registry();
+ if (!owner) return;
+ const state = stateFor(owner, video);
+ state.frame = frame;
+ state.dirty = true;
+}
+
+/** Notify once before composition redraw; callbacks registered from a callback wait for the next frame. */
+export function flushPausedVideoCallbacks(video: HTMLVideoElement): void {
+ const owner = registry();
+ const state = owner?.states.get(video);
+ if (!owner || !state?.frame || !state.dirty) return;
+ state.dirty = false;
+ state.synthetic = true;
+ state.count++;
+ const now = performance.now();
+ const metadata: VideoFrameCallbackMetadata = {
+ ...state.frame,
+ presentedFrames: state.count,
+ presentationTime: now,
+ expectedDisplayTime: now,
+ processingDuration: 0,
+ };
+ for (const [id, pending] of [...state.pending]) {
+ if (!state.pending.delete(id)) continue;
+ if (pending.native !== null) owner.cancel.call(video, pending.native);
+ try {
+ pending.callback.call(undefined, now, metadata);
+ } catch (error) {
+ reportError(error);
+ }
+ }
+}
+
+export function releasePausedVideoCallbacks(video: HTMLVideoElement): void {
+ const owner = registry();
+ const state = owner?.states.get(video);
+ if (!owner || !state) return;
+ state.frame = null;
+ state.dirty = false;
+ for (const [id, pending] of state.pending) {
+ if (pending.native === null)
+ pending.native = requestNative(owner, video, state, pending, () => id);
+ }
+}
+
+/** Recovery must observe native frames even while authored callbacks still see staged pixels. */
+export function observeNativeVideoFrames(
+ video: HTMLVideoElement,
+ onFrame: () => boolean,
+): () => void {
+ const owner = registry();
+ const request = owner?.request ?? video.requestVideoFrameCallback;
+ const cancel = owner?.cancel ?? video.cancelVideoFrameCallback;
+ let handle: number | null = null;
+ let stopped = false;
+ let playingAt: number | null = video.paused ? null : video.currentTime;
+ const stop = () => {
+ if (stopped) return;
+ stopped = true;
+ if (handle !== null && typeof cancel === "function") cancel.call(video, handle);
+ video.removeEventListener("playing", playing);
+ video.removeEventListener("timeupdate", progress);
+ };
+ const arm = () => {
+ if (!stopped && typeof request === "function") handle = request.call(video, presented);
+ };
+ const presented = () => {
+ handle = null;
+ if (stopped) return;
+ if (onFrame()) stop();
+ else arm();
+ };
+ const playing = () => {
+ playingAt = video.currentTime;
+ };
+ const progress = () => {
+ if (stopped || playingAt === null || video.paused || video.currentTime === playingAt) return;
+ if (onFrame()) stop();
+ };
+ video.addEventListener("playing", playing);
+ video.addEventListener("timeupdate", progress);
+ arm();
+ return stop;
+}
diff --git a/packages/core/src/runtime/pausedVideoFrames.test.ts b/packages/core/src/runtime/pausedVideoFrames.test.ts
new file mode 100644
index 0000000000..74cc645b51
--- /dev/null
+++ b/packages/core/src/runtime/pausedVideoFrames.test.ts
@@ -0,0 +1,256 @@
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import {
+ createPausedVideoFrames,
+ getPausedVideoFrame,
+ ownsPausedVideoTime,
+ type PausedVideoFrameProvider,
+} from "./pausedVideoFrames";
+
+const sessions: ReturnType[] = [];
+beforeEach(() => {
+ vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
+ clearRect: vi.fn(),
+ drawImage: vi.fn(),
+ } as unknown as CanvasRenderingContext2D);
+});
+afterEach(() => {
+ for (const session of sessions.splice(0)) session.destroy();
+ document.body.innerHTML = "";
+ vi.restoreAllMocks();
+});
+
+function fixture() {
+ const session = createPausedVideoFrames();
+ sessions.push(session);
+ const video = document.createElement("video");
+ video.src = "/original.webm";
+ video.style.cssText = "position:absolute;width:64px;height:36px;transform:translateX(10px)";
+ document.body.append(video);
+ Object.defineProperty(video, "videoWidth", { value: 64 });
+ Object.defineProperty(video, "videoHeight", { value: 36 });
+ const release = vi.fn();
+ const provider: PausedVideoFrameProvider = {
+ decode: vi.fn(async () => [
+ {
+ source: document.createElement("canvas"),
+ timestamp: 3,
+ duration: 1 / 30,
+ width: 64,
+ height: 36,
+ release,
+ },
+ ]),
+ dispose: vi.fn(),
+ };
+ session.setProvider(provider);
+ const request = [{ video, source: video.src, time: 3 }];
+ return { session, video, provider, request, release };
+}
+
+it("presents decoded pixels with the authored source, dimensions, layout and displayed source clock", async () => {
+ const { session, video, provider, request, release } = fixture();
+ const originalSrc = video.src;
+ const batch = await session.prepare(request, new AbortController().signal);
+ batch!.present();
+ video.style.transform = "translateX(25px)";
+ batch!.refresh();
+ const canvas = getPausedVideoFrame(video)!;
+ expect([video.src, video.videoWidth, video.videoHeight, video.currentTime]).toEqual([
+ originalSrc,
+ 64,
+ 36,
+ 3,
+ ]);
+ expect(canvas.style.transform).toBe("translateX(25px)");
+ expect(canvas.nextElementSibling).toBe(video);
+ expect(ownsPausedVideoTime(video)).toBe(true);
+ expect(release).toHaveBeenCalledTimes(1);
+ session.prepareNative();
+ expect(Object.hasOwn(video, "currentTime")).toBe(false);
+ expect(ownsPausedVideoTime(video)).toBe(false);
+ expect(getPausedVideoFrame(video)).toBe(canvas);
+ session.clear();
+ expect(getPausedVideoFrame(video)).toBeNull();
+ expect(canvas.width).toBe(0);
+ expect(video.hasAttribute("data-hf-preview-video-frame")).toBe(false);
+ session.destroy();
+ expect(provider.dispose).toHaveBeenCalledTimes(1);
+});
+
+it("releases a late decode when its request is aborted or its source changes", async () => {
+ const { session, video, request, release } = fixture();
+ const controller = new AbortController();
+ const batch = session.prepare(request, controller.signal);
+ controller.abort();
+ expect(await batch).toBeNull();
+ expect(release).toHaveBeenCalledTimes(1);
+ const changed = session.prepare(request, new AbortController().signal);
+ video.src = "/replacement.webm";
+ expect(await changed).toBeNull();
+ expect(release).toHaveBeenCalledTimes(2);
+ expect(getPausedVideoFrame(video)).toBeNull();
+});
+
+it("does not surrender decoded presentation for the runtime's native seek padding", async () => {
+ const { session, video, request } = fixture();
+ const batch = await session.prepare(request, new AbortController().signal);
+ batch!.present();
+ batch!.refresh();
+ const canvas = getPausedVideoFrame(video);
+ video.currentTime = 3.00001;
+ expect(ownsPausedVideoTime(video)).toBe(true);
+ expect(getPausedVideoFrame(video)).toBe(canvas);
+ expect(video.currentTime).toBe(3);
+});
+
+it("gives an authored seek back to native media and preserves an existing currentTime override", async () => {
+ const { session, video, request } = fixture();
+ const batch = await session.prepare(request, new AbortController().signal);
+ batch!.present();
+ video.currentTime = 7;
+ expect(video.currentTime).toBe(7);
+ expect(getPausedVideoFrame(video)).toBeNull();
+ expect(await session.prepare(request, new AbortController().signal)).toBeNull();
+ const custom = document.createElement("video");
+ Object.defineProperty(custom, "currentTime", { value: 2, configurable: true });
+ expect(
+ await session.prepare([{ ...request[0]!, video: custom }], new AbortController().signal),
+ ).toBeNull();
+ expect(custom.currentTime).toBe(2);
+});
+
+it("rejects a neighbouring frame that does not cover the requested presentation time", async () => {
+ const { session, provider, request, release } = fixture();
+ vi.mocked(provider.decode).mockResolvedValue([
+ {
+ source: document.createElement("canvas"),
+ timestamp: 2,
+ duration: 1 / 30,
+ width: 64,
+ height: 36,
+ release,
+ },
+ ]);
+ expect(await session.prepare(request, new AbortController().signal)).toBeNull();
+ expect(release).toHaveBeenCalledTimes(1);
+});
+
+it("retains the whole previous presentation when staging a new layer fails", async () => {
+ const { session, video, request, release } = fixture();
+ const first = await session.prepare(request, new AbortController().signal);
+ first!.present();
+ first!.refresh();
+ const previous = getPausedVideoFrame(video);
+ const second = await session.prepare(request, new AbortController().signal);
+ vi.mocked(HTMLCanvasElement.prototype.getContext).mockReturnValueOnce(null);
+ expect(second!.present()).toBe(false);
+ expect(getPausedVideoFrame(video)).toBe(previous);
+ expect(video.currentTime).toBe(3);
+ expect(release).toHaveBeenCalledTimes(2);
+});
+
+it("revalidates source identity after preparing a frame and before publishing it", async () => {
+ const { session, video, request, release } = fixture();
+ const prepared = await session.prepare(request, new AbortController().signal);
+ video.src = "/edited.mp4";
+ expect(prepared!.valid()).toBe(false);
+ expect(prepared!.present()).toBe(false);
+ expect(getPausedVideoFrame(video)).toBeNull();
+ expect(release).toHaveBeenCalledTimes(1);
+});
+
+it("reuses display surfaces across updates and removes them when source children are replaced", async () => {
+ const { session, video, request } = fixture();
+ const first = await session.prepare(request, new AbortController().signal);
+ first!.present();
+ first!.refresh();
+ const canvas = getPausedVideoFrame(video);
+ const next = await session.prepare(request, new AbortController().signal);
+ next!.present();
+ next!.refresh();
+ expect(getPausedVideoFrame(video)).toBe(canvas);
+ video.append(document.createElement("source"));
+ await Promise.resolve();
+ expect(getPausedVideoFrame(video)).toBeNull();
+ expect(Object.hasOwn(video, "currentTime")).toBe(false);
+});
+
+it("keeps the displayed surface in sync with paused editing without releasing decoded pixels", async () => {
+ const { session, video, request } = fixture();
+ const batch = await session.prepare(request, new AbortController().signal);
+ batch!.present();
+ batch!.refresh();
+ const canvas = getPausedVideoFrame(video)!;
+ video.style.transform = "translateX(80px)";
+ video.style.opacity = "0.4";
+ video.style.width = "80px";
+ await Promise.resolve();
+ expect(getPausedVideoFrame(video)).toBe(canvas);
+ expect(canvas.style.transform).toBe("translateX(80px)");
+ expect(canvas.style.opacity).toBe("0.4");
+ expect(canvas.style.width).toBe("80px");
+ expect(video.currentTime).toBe(3);
+});
+
+it("holds a full-size native snapshot through original-source provider replacement", async () => {
+ const { session, video, request } = fixture();
+ Object.defineProperty(video, "readyState", { value: 4 });
+ session.holdNative([video]);
+ const snapshot = getPausedVideoFrame(video)!;
+ expect([snapshot.width, snapshot.height]).toEqual([64, 36]);
+ expect(snapshot.hasAttribute("data-hyperframes-picker-ignore")).toBe(true);
+ expect(ownsPausedVideoTime(video)).toBe(false);
+ const decoded = await session.prepare(request, new AbortController().signal);
+ expect(decoded!.present()).toBe(true);
+ expect(getPausedVideoFrame(video)).toBe(snapshot);
+ expect(ownsPausedVideoTime(video)).toBe(true);
+ expect(video.currentTime).toBe(3);
+});
+
+it("cancels a retained native observer when a new paused picture replaces it", async () => {
+ const proto = HTMLVideoElement.prototype;
+ const original = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
+ const originalCancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
+ let queued: VideoFrameRequestCallback = () => {};
+ const cancel = vi.fn();
+ Object.defineProperty(proto, "requestVideoFrameCallback", {
+ configurable: true,
+ writable: true,
+ value: (callback: VideoFrameRequestCallback) => {
+ queued = callback;
+ return 123;
+ },
+ });
+ Object.defineProperty(proto, "cancelVideoFrameCallback", {
+ configurable: true,
+ writable: true,
+ value: cancel,
+ });
+ try {
+ const { session, video, request } = fixture();
+ const first = await session.prepare(request, new AbortController().signal);
+ first!.present();
+ session.prepareNative();
+ session.clear(new Set([video]), () => false);
+ const replacement = await session.prepare(request, new AbortController().signal);
+ replacement!.present();
+ const picture = getPausedVideoFrame(video);
+ expect(cancel).toHaveBeenCalledWith(123);
+ queued(0, {
+ mediaTime: 3,
+ width: 64,
+ height: 36,
+ presentedFrames: 1,
+ presentationTime: 0,
+ expectedDisplayTime: 0,
+ processingDuration: 0,
+ });
+ expect(getPausedVideoFrame(video)).toBe(picture);
+ expect(ownsPausedVideoTime(video)).toBe(true);
+ } finally {
+ if (original) Object.defineProperty(proto, "requestVideoFrameCallback", original);
+ else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
+ if (originalCancel) Object.defineProperty(proto, "cancelVideoFrameCallback", originalCancel);
+ else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
+ }
+});
diff --git a/packages/core/src/runtime/pausedVideoFrames.ts b/packages/core/src/runtime/pausedVideoFrames.ts
new file mode 100644
index 0000000000..7b1ac2f311
--- /dev/null
+++ b/packages/core/src/runtime/pausedVideoFrames.ts
@@ -0,0 +1,423 @@
+import { MEDIA_ELEMENT_LEAD_SECONDS } from "./mediaClock";
+import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
+import {
+ installPausedVideoCallbacks,
+ stagePausedVideoCallbacks,
+ flushPausedVideoCallbacks,
+ releasePausedVideoCallbacks,
+ observeNativeVideoFrames,
+} from "./pausedVideoCallbacks";
+import {
+ PREVIEW_VIDEO_FRAME_ATTR as FRAME_ATTR,
+ PREVIEW_VIDEO_SOURCE_ATTR,
+ PREVIEW_VIDEO_CANVAS_ATTR,
+} from "../studioPreviewMark";
+
+export interface PausedVideoFrameRequest {
+ video: HTMLVideoElement;
+ source: string;
+ time: number;
+ settled?: boolean;
+}
+
+export interface PausedVideoFrame {
+ source: CanvasImageSource;
+ timestamp: number;
+ duration: number;
+ width: number;
+ height: number;
+ release(): void;
+}
+
+/** Decoding lives in the host; core owns composition timing and presentation. */
+export interface PausedVideoFrameProvider {
+ decode(requests: PausedVideoFrameRequest[], signal: AbortSignal): Promise;
+ suspend?(): Promise;
+ dispose(): void;
+}
+
+export interface PreparedPausedVideoFrames {
+ valid(): boolean;
+ present(): boolean;
+ refresh(): void;
+ release(): void;
+}
+
+const SURFACE_BUDGET = 128 * 1024 * 1024;
+type Presented = {
+ video: HTMLVideoElement;
+ canvas: HTMLCanvasElement;
+ time: number;
+ driving: boolean;
+ restoreTime: () => void;
+ gpuFrame?: VideoFrame;
+ stopWaiting?: () => void;
+ // Paused frame replacement does not retry a source native playback already skipped.
+ nativeSkipped?: boolean;
+};
+const presented = new Map();
+const authoredSeeks = new WeakSet();
+
+export function getPausedVideoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
+ return presented.get(video)?.canvas ?? null;
+}
+
+export function ownsPausedVideoTime(video: HTMLMediaElement): boolean {
+ return presented.get(video)?.driving === true;
+}
+
+export function getPausedExternalVideoFrame(video: HTMLVideoElement): VideoFrame | null {
+ const entry = presented.get(video);
+ if (!entry || typeof VideoFrame === "undefined") return null;
+ entry.gpuFrame ??= new VideoFrame(entry.canvas, { timestamp: Math.round(entry.time * 1e6) });
+ return entry.gpuFrame;
+}
+
+export function captureNativeVideoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
+ const canvas = document.createElement("canvas");
+ canvas.width = video.videoWidth;
+ canvas.height = video.videoHeight;
+ const context = canvas.getContext("2d");
+ if (!context) return null;
+ try {
+ context.drawImage(video, 0, 0);
+ return canvas;
+ } catch {
+ return null;
+ }
+}
+
+export function nativeSnapshotStyle(video: HTMLVideoElement): CSSStyleDeclaration | null {
+ if (getPausedVideoFrame(video)) return null;
+ // Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
+ if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return null;
+ const css = getComputedStyle(video);
+ if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return null;
+ if (!video.videoWidth || !video.videoHeight) return null;
+ return css;
+}
+
+function virtualTime(entry: Presented): void {
+ let proto = Object.getPrototypeOf(entry.video);
+ let descriptor: PropertyDescriptor | undefined;
+ while (proto && !descriptor) {
+ descriptor = Object.getOwnPropertyDescriptor(proto, "currentTime");
+ proto = Object.getPrototypeOf(proto);
+ }
+ const nativeSet = descriptor?.set;
+ Object.defineProperty(entry.video, "currentTime", {
+ configurable: true,
+ get: () => entry.time,
+ set: (value: number) => {
+ if (Math.abs(value - entry.time) <= MEDIA_ELEMENT_LEAD_SECONDS * 2) return;
+ authoredSeeks.add(entry.video);
+ remove(entry);
+ nativeSet?.call(entry.video, value);
+ },
+ });
+ entry.restoreTime = () => {
+ Reflect.deleteProperty(entry.video, "currentTime");
+ };
+}
+
+function remove(entry: Presented): void {
+ entry.stopWaiting?.();
+ entry.restoreTime();
+ releasePausedVideoCallbacks(entry.video);
+ entry.gpuFrame?.close();
+ entry.video.removeAttribute(FRAME_ATTR);
+ entry.video.removeAttribute(PREVIEW_VIDEO_SOURCE_ATTR);
+ entry.canvas.remove();
+ entry.canvas.width = entry.canvas.height = 0;
+ presented.delete(entry.video);
+}
+
+export function releasePausedVideoFrame(video: HTMLVideoElement): void {
+ const entry = presented.get(video);
+ if (entry) remove(entry);
+}
+
+function refresh(entry: Presented): void {
+ const { video, canvas } = entry;
+ video.removeAttribute(FRAME_ATTR);
+ if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) {
+ canvas.remove();
+ return;
+ }
+ const css = getComputedStyle(video);
+ for (const key of Array.from(css)) {
+ const value = css.getPropertyValue(key);
+ if (canvas.style.getPropertyValue(key) !== value) canvas.style.setProperty(key, value);
+ }
+ if (css.position !== "absolute" && css.position !== "fixed") {
+ canvas.style.position = "absolute";
+ canvas.style.inset = "auto";
+ canvas.style.left = `${video.offsetLeft}px`;
+ canvas.style.top = `${video.offsetTop}px`;
+ canvas.style.margin = "0";
+ }
+ canvas.style.setProperty("pointer-events", "none", "important");
+ canvas.style.setProperty("animation", "none", "important");
+ canvas.style.setProperty("transition", "none", "important");
+ if (video.previousElementSibling !== canvas) video.before(canvas);
+ video.setAttribute(FRAME_ATTR, "");
+}
+
+type ReusablePresentation = { entry: Presented; context: CanvasRenderingContext2D };
+
+function markFrameCanvas(canvas: HTMLCanvasElement): void {
+ canvas.setAttribute(PREVIEW_VIDEO_CANVAS_ATTR, "");
+ canvas.setAttribute("data-layout-ignore", "");
+ canvas.setAttribute("data-hyperframes-ignore", "");
+ canvas.setAttribute("data-hyperframes-picker-ignore", "");
+ canvas.setAttribute("data-hf-ignore", "");
+ canvas.setAttribute("aria-hidden", "true");
+}
+
+function adoptStagedFrame(next: Presented, existing?: ReusablePresentation): void {
+ if (existing) {
+ const { entry, context } = existing;
+ entry.stopWaiting?.();
+ entry.stopWaiting = undefined;
+ entry.gpuFrame?.close();
+ entry.gpuFrame = undefined;
+ context.clearRect(0, 0, entry.canvas.width, entry.canvas.height);
+ context.drawImage(next.canvas, 0, 0);
+ next.canvas.width = next.canvas.height = 0;
+ entry.time = next.time;
+ if (!entry.driving) virtualTime(entry);
+ entry.driving = true;
+ return;
+ }
+ const { video, canvas } = next;
+ releasePausedVideoFrame(video);
+ markFrameCanvas(canvas);
+ presented.set(video, next);
+ video.setAttribute(PREVIEW_VIDEO_SOURCE_ATTR, "");
+ virtualTime(next);
+}
+
+/** Keeps original nodes, URLs, intrinsic sizes and the displayed source clock intact. */
+export function createPausedVideoFrames() {
+ installPausedVideoCallbacks();
+ let provider: PausedVideoFrameProvider | null = null;
+ const style = document.createElement("style");
+ style.textContent = `[${FRAME_ATTR}] { opacity: 0 !important; }`;
+ const observer = new MutationObserver((records) => {
+ for (const entry of [...presented.values()]) {
+ if (
+ !entry.video.isConnected ||
+ records.some(
+ (record) =>
+ (record.type === "childList" && record.target === entry.video) ||
+ (record.type === "attributes" &&
+ record.attributeName === "src" &&
+ (record.target === entry.video || record.target.parentElement === entry.video)),
+ )
+ )
+ remove(entry);
+ else if (records.some((record) => record.target.contains(entry.video))) refresh(entry);
+ }
+ });
+ observer.observe(document.body, {
+ childList: true,
+ subtree: true,
+ attributes: true,
+ attributeFilter: ["src", "style", "class", "hidden", "data-hidden"],
+ });
+
+ const clear = (
+ keep?: Set,
+ current?: (video: HTMLVideoElement, presented: boolean) => boolean,
+ ) => {
+ for (const entry of [...presented.values()]) {
+ if (!keep?.has(entry.video)) {
+ remove(entry);
+ continue;
+ }
+ entry.nativeSkipped = true;
+ const reveal = (nativeFrame: boolean) => {
+ if (presented.get(entry.video) !== entry) return true;
+ if (
+ entry.video.readyState < 2 ||
+ entry.video.seeking ||
+ (current && !current(entry.video, nativeFrame))
+ )
+ return false;
+ remove(entry);
+ return true;
+ };
+ const ready = () => {
+ reveal(false);
+ };
+ const events = ["loadeddata", "canplay", "seeked"];
+ entry.stopWaiting?.();
+ for (const event of events) entry.video.addEventListener(event, ready);
+ const stopFrames = observeNativeVideoFrames(entry.video, () => reveal(true));
+ entry.stopWaiting = () => {
+ for (const event of events) entry.video.removeEventListener(event, ready);
+ stopFrames();
+ };
+ }
+ if (presented.size === 0) style.remove();
+ };
+ return {
+ get active() {
+ // A skipped native clip may keep a fallback picture without owning paused playback.
+ return [...presented.values()].some((entry) => !entry.stopWaiting);
+ },
+ holdNative(videos: HTMLVideoElement[]) {
+ const held: Presented[] = [];
+ for (const video of videos) {
+ if (!nativeSnapshotStyle(video)) continue;
+ const canvas = captureNativeVideoFrame(video);
+ if (!canvas) continue;
+ const entry: Presented = {
+ video,
+ canvas,
+ time: video.currentTime,
+ driving: false,
+ restoreTime: () => {},
+ };
+ held.push(entry);
+ markFrameCanvas(canvas);
+ presented.set(video, entry);
+ video.setAttribute(PREVIEW_VIDEO_SOURCE_ATTR, "");
+ if (!style.isConnected) document.head.append(style);
+ refresh(entry);
+ }
+ return () => {
+ for (const entry of held)
+ if (presented.get(entry.video) === entry && !entry.driving) remove(entry);
+ };
+ },
+ skipped: (el: HTMLMediaElement) => presented.get(el)?.nativeSkipped === true,
+ retained: (el: HTMLMediaElement) => presented.get(el)?.stopWaiting !== undefined,
+ available: () => provider !== null,
+ setProvider(next: PausedVideoFrameProvider | null) {
+ if (provider === next) return;
+ provider?.dispose();
+ provider = next;
+ },
+ async prepare(requests: PausedVideoFrameRequest[], signal: AbortSignal) {
+ if (
+ !provider ||
+ requests.length === 0 ||
+ requests.some(
+ ({ video }) =>
+ authoredSeeks.has(video) ||
+ (!presented.has(video) && Object.hasOwn(video, "currentTime")),
+ )
+ )
+ return null;
+ const owner = provider;
+ const identity = (video: HTMLVideoElement) =>
+ JSON.stringify([
+ video.src,
+ ...Array.from(video.querySelectorAll("source"), (source) => source.src),
+ ]);
+ const sources = requests.map(({ video }) => identity(video));
+ const frames = await owner.decode(requests, signal);
+ let released = false;
+ const release = () => {
+ if (released) return;
+ released = true;
+ for (const frame of frames) frame.release();
+ };
+ const current = () =>
+ !signal.aborted &&
+ provider === owner &&
+ requests.every(
+ ({ video }, index) => video.isConnected && identity(video) === sources[index],
+ );
+ if (!current()) {
+ release();
+ return null;
+ }
+ const valid =
+ frames.reduce((sum, frame) => sum + frame.width * frame.height * 4, 0) <= SURFACE_BUDGET &&
+ frames.length === requests.length &&
+ frames.every((frame, index) => {
+ const time = requests[index]!.time;
+ return (
+ frame.width > 0 &&
+ frame.height > 0 &&
+ frame.timestamp <= time + 1e-5 &&
+ time < frame.timestamp + frame.duration + 1e-5
+ );
+ });
+ if (!valid) {
+ release();
+ return null;
+ }
+ return {
+ valid: current,
+ present() {
+ if (!current()) {
+ release();
+ return false;
+ }
+ const staged: Presented[] = [];
+ const reusable = new Map();
+ try {
+ frames.forEach((frame, index) => {
+ const { video, time } = requests[index]!;
+ const canvas = document.createElement("canvas");
+ canvas.width = frame.width;
+ canvas.height = frame.height;
+ staged.push({ video, canvas, time, driving: true, restoreTime: () => {} });
+ const context = canvas.getContext("2d");
+ if (!context) throw new Error("Preview canvas unavailable");
+ context.drawImage(frame.source, 0, 0, canvas.width, canvas.height);
+ const prior = presented.get(video);
+ if (
+ prior &&
+ prior.canvas.width === frame.width &&
+ prior.canvas.height === frame.height
+ ) {
+ const destination = prior.canvas.getContext("2d");
+ if (!destination) throw new Error("Preview canvas unavailable");
+ reusable.set(staged[staged.length - 1]!, { entry: prior, context: destination });
+ }
+ });
+ } catch {
+ for (const { canvas } of staged) canvas.width = canvas.height = 0;
+ return false;
+ } finally {
+ release();
+ }
+ const wanted = new Set(requests.map(({ video }) => video));
+ for (const entry of [...presented.values()]) if (!wanted.has(entry.video)) remove(entry);
+ if (!style.isConnected) document.head.append(style);
+ for (const [index, next] of staged.entries()) {
+ adoptStagedFrame(next, reusable.get(next));
+ const { timestamp, width, height } = frames[index]!;
+ stagePausedVideoCallbacks(next.video, { mediaTime: timestamp, width, height });
+ }
+ return true;
+ },
+ refresh() {
+ for (const entry of presented.values()) refresh(entry);
+ },
+ release,
+ };
+ },
+ suspendProvider: () => provider?.suspend?.(),
+ prepareNative() {
+ for (const entry of presented.values()) {
+ entry.driving = false;
+ entry.restoreTime();
+ }
+ },
+ notify() {
+ for (const { video } of presented.values()) flushPausedVideoCallbacks(video);
+ },
+ clear,
+ destroy() {
+ observer.disconnect();
+ clear();
+ provider?.dispose();
+ provider = null;
+ },
+ };
+}
diff --git a/packages/core/src/runtime/picker.test.ts b/packages/core/src/runtime/picker.test.ts
index cf27cbecb0..af1d49b92f 100644
--- a/packages/core/src/runtime/picker.test.ts
+++ b/packages/core/src/runtime/picker.test.ts
@@ -302,6 +302,17 @@ describe("createPickerModule", () => {
}
});
+ it("picks a video whose decoded preview surface hides the native pixels", () => {
+ const picker = createPickerModule({ postMessage: createMockPostMessage() });
+ picker.installPickerApi();
+ document.body.innerHTML =
+ '';
+ const video = document.querySelector("video")!;
+ expect(candidatesUnder(() => [video])).toContain("#video");
+ video.removeAttribute("data-hf-preview-video-frame");
+ expect(candidatesUnder(() => [video])).not.toContain("#video");
+ });
+
it("still picks an image inside a faint texture wrapper", () => {
const picker = createPickerModule({ postMessage: createMockPostMessage() });
picker.installPickerApi();
diff --git a/packages/core/src/runtime/picker.ts b/packages/core/src/runtime/picker.ts
index 584eb421b5..5e658425c0 100644
--- a/packages/core/src/runtime/picker.ts
+++ b/packages/core/src/runtime/picker.ts
@@ -1,4 +1,5 @@
import type { RuntimeJson, RuntimeOutboundMessage, RuntimePickerElementInfo } from "./types";
+import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { swallow } from "./diagnostics";
import { isElementNode } from "./domRealm";
@@ -191,7 +192,8 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule {
if (
Number.isFinite(opacity) &&
opacity <= 0.01 &&
- !current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)
+ !current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR) &&
+ !current.hasAttribute(PREVIEW_VIDEO_FRAME_ATTR)
)
return true;
current = current.parentElement;
diff --git a/packages/core/src/runtime/player.ts b/packages/core/src/runtime/player.ts
index 89a13e665d..bcd17a2c5a 100644
--- a/packages/core/src/runtime/player.ts
+++ b/packages/core/src/runtime/player.ts
@@ -211,7 +211,7 @@ export function createRuntimePlayer(deps: PlayerDeps): RuntimePlayer {
deps.onRenderFrameSeek(time);
deps.onStatePost(true);
},
- seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => {
+ seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => {
const timeline = deps.getTimeline();
if (!timeline) return;
const safeTime = Math.max(0, Number(timeSeconds) || 0);
diff --git a/packages/core/src/runtime/previewSeek.frames.test.ts b/packages/core/src/runtime/previewSeek.frames.test.ts
new file mode 100644
index 0000000000..fe886d9499
--- /dev/null
+++ b/packages/core/src/runtime/previewSeek.frames.test.ts
@@ -0,0 +1,226 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { createPreviewSeek } from "./previewSeek";
+import type { PreparedPausedVideoFrames } from "./pausedVideoFrames";
+import { isUnplayable, syncRuntimeMedia } from "./media";
+import type { RuntimeMediaClip } from "./types";
+
+const jobs: ReturnType[] = [];
+afterEach(() => {
+ for (const job of jobs.splice(0)) job.cancel();
+ document.body.innerHTML = "";
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+});
+
+const batch = (): PreparedPausedVideoFrames => ({
+ valid: () => true,
+ present: vi.fn(() => true),
+ refresh: vi.fn(),
+ release: vi.fn(),
+});
+function fixture() {
+ const video = document.createElement("video");
+ document.body.append(video);
+ const clip: RuntimeMediaClip = {
+ el: video,
+ start: 0,
+ duration: 10,
+ end: 10,
+ mediaStart: 0,
+ playbackRate: 1,
+ volume: null,
+ loop: false,
+ sourceDuration: 10,
+ };
+ const pending: Array<{
+ signal: AbortSignal;
+ resolve: (result: PreparedPausedVideoFrames | null) => void;
+ }> = [];
+ const commit = vi.fn();
+ const native = vi.fn();
+ const prepare = vi.fn(
+ (_clips: RuntimeMediaClip[], _time: number, signal: AbortSignal) =>
+ new Promise((resolve) => pending.push({ signal, resolve })),
+ );
+ const host = {
+ clipsAt: () => [clip],
+ videosToHold: (): HTMLVideoElement[] => [],
+ scenesReadyAt: () => true,
+ prepare: native,
+ commit,
+ frames: {
+ active: () => false,
+ available: () => true,
+ prepare,
+ prepareNative: vi.fn(),
+ clear: vi.fn(),
+ },
+ };
+ const job = createPreviewSeek(host);
+ jobs.push(job);
+ return { job, commit, native, prepare, pending, video, clip, host };
+}
+
+it("presents coherent intermediate work while retaining the latest requested destination", async () => {
+ const { job, commit, pending } = fixture();
+ const done = job.seek(1);
+ expect(job.seek(2)).toBe(done);
+ const first = batch();
+ pending[0]!.resolve(first);
+ await Promise.resolve();
+ expect(commit).toHaveBeenCalledWith(1);
+ expect(job.time).toBe(2);
+ expect(first.present).toHaveBeenCalledTimes(1);
+ const last = batch();
+ pending[1]!.resolve(last);
+ await done;
+ expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 2]);
+ expect(job.time).toBeUndefined();
+});
+
+it("does not rebuild document structure when no native snapshots need hiding", async () => {
+ const { job, pending } = fixture();
+ const mutations: MutationRecord[] = [];
+ const observer = new MutationObserver((records) => mutations.push(...records));
+ observer.observe(document.head, { childList: true });
+ const done = job.seek(1);
+ job.seek(2);
+ pending[0]!.resolve(batch());
+ await Promise.resolve();
+ pending[1]!.resolve(batch());
+ await done;
+ expect([...mutations, ...observer.takeRecords()]).toEqual([]);
+ observer.disconnect();
+});
+
+it("resumes provider decoding when a scrub interrupts a native play handoff", async () => {
+ const { job, commit, prepare, native, pending } = fixture();
+ const initial = job.seek(1);
+ pending[0]!.resolve(batch());
+ await initial;
+ const play = job.handoff(2);
+ expect(native).toHaveBeenCalledTimes(1);
+ const scrub = job.seek(3);
+ await play;
+ expect(prepare).toHaveBeenLastCalledWith(expect.any(Array), 3, expect.any(AbortSignal), false);
+ pending[1]!.resolve(batch());
+ await scrub;
+ expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 3]);
+});
+
+it("releases a decode that arrives after cancellation without presenting it", async () => {
+ const { job, commit, pending } = fixture();
+ const done = job.seek(1);
+ job.cancel();
+ expect(pending[0]!.signal.aborted).toBe(true);
+ const late = batch();
+ pending[0]!.resolve(late);
+ await Promise.resolve();
+ await done;
+ expect(late.release).toHaveBeenCalledTimes(1);
+ expect(late.present).not.toHaveBeenCalled();
+ expect(commit).not.toHaveBeenCalled();
+});
+
+it("lets a new gesture supersede a full-quality refinement without changing its destination", async () => {
+ const { job, prepare, pending, commit } = fixture();
+ const refining = job.refine(1);
+ expect(prepare).toHaveBeenLastCalledWith(expect.any(Array), 1, expect.any(AbortSignal), true);
+ const next = job.seek(2);
+ expect(pending[0]!.signal.aborted).toBe(true);
+ pending[0]!.resolve(batch());
+ pending[1]!.resolve(batch());
+ await Promise.all([refining, next]);
+ expect(commit.mock.calls.map(([time]) => time)).toEqual([2]);
+});
+
+it("does not erase a seek requested synchronously while publishing the previous one", async () => {
+ const { job, commit, pending } = fixture();
+ commit.mockImplementationOnce(() => {
+ job.seek(4);
+ });
+ const first = job.seek(1);
+ pending[0]!.resolve(batch());
+ await first;
+ expect(job.time).toBe(4);
+ pending[1]!.resolve(batch());
+ await Promise.resolve();
+ expect(commit.mock.calls.map(([time]) => time)).toEqual([1, 4]);
+});
+
+it("keeps a request-wide hard deadline through refresh and same-clip retarget", async () => {
+ vi.useFakeTimers();
+ const { job, commit } = fixture();
+ const done = job.handoff(3);
+ await vi.advanceTimersByTimeAsync(4900);
+ job.refresh();
+ await vi.advanceTimersByTimeAsync(4100);
+ job.seek(4, true);
+ expect(commit).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(1000);
+ await done;
+ expect(commit.mock.calls.map(([time]) => time)).toEqual([4]);
+ expect(job.time).toBeUndefined();
+});
+
+it("does not reset a source already known to be unplayable on another Play", async () => {
+ vi.useFakeTimers();
+ const { job, commit, video, clip } = fixture();
+ video.src = "/stopped.mp4";
+ Object.defineProperties(video, {
+ readyState: { value: 1, configurable: true },
+ networkState: { value: 1, configurable: true },
+ });
+ video.play = vi.fn(async () => {});
+ video.pause = vi.fn();
+ syncRuntimeMedia({
+ clips: [clip],
+ timeSeconds: 0,
+ playing: true,
+ playbackRate: 1,
+ getCompositionDuration: () => 10,
+ });
+ await vi.advanceTimersByTimeAsync(5000);
+ expect(isUnplayable(video)).toBe(true);
+ const done = job.handoff(3);
+ await Promise.resolve();
+ await done;
+ expect(commit).toHaveBeenLastCalledWith(3);
+ expect(job.time).toBeUndefined();
+});
+
+it("preserves the held native picture when a new gesture cancels expired background work", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
+ drawImage: vi.fn(),
+ } as unknown as CanvasRenderingContext2D);
+ const { job, video, native, pending, prepare, host } = fixture();
+ let ready = 4;
+ let time = 0;
+ Object.defineProperties(video, {
+ readyState: { get: () => ready },
+ networkState: { value: 2 },
+ videoWidth: { value: 64 },
+ videoHeight: { value: 36 },
+ currentTime: { get: () => time },
+ });
+ host.videosToHold = () => [video];
+ host.frames.active = () => true;
+ native.mockImplementation((_clips, target) => {
+ ready = 1;
+ time = target;
+ });
+ prepare.mockResolvedValueOnce(null);
+ const expired = job.seek(3);
+ await vi.advanceTimersByTimeAsync(5000);
+ await expired;
+ expect(job.time).toBeUndefined();
+ expect(job.targetTime).toBe(3);
+ const picture = document.querySelector("canvas");
+ expect(picture).not.toBeNull();
+ const next = job.seek(4);
+ expect(document.querySelector("canvas")).toBe(picture);
+ pending[0]!.resolve(batch());
+ await next;
+ expect(document.querySelector("canvas")).toBeNull();
+});
diff --git a/packages/core/src/runtime/previewSeek.ts b/packages/core/src/runtime/previewSeek.ts
index f56f5944c1..fa4cadcb4f 100644
--- a/packages/core/src/runtime/previewSeek.ts
+++ b/packages/core/src/runtime/previewSeek.ts
@@ -1,35 +1,23 @@
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
-import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { isUnplayable, mediaSourceTimeAt } from "./media";
import type { RuntimeMediaClip } from "./types";
+import { isVideoElement } from "./domRealm";
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
+import {
+ nativeSnapshotStyle,
+ captureNativeVideoFrame,
+ type PreparedPausedVideoFrames,
+} from "./pausedVideoFrames";
const HELD_VIDEO_ATTR = "data-hf-seek-video";
const SEEK_CAP_MS = 5000;
const EVENTS = ["loadedmetadata", "loadeddata", "canplay", "seeked", "error"] as const;
-function videoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
- const canvas = document.createElement("canvas");
- canvas.width = video.videoWidth;
- canvas.height = video.videoHeight;
- const context = canvas.getContext("2d");
- if (!context) return null;
- try {
- context.drawImage(video, 0, 0);
- return canvas;
- } catch {
- return null;
- }
-}
-
/** Keep a displayed video in its layout slot while its decoder moves independently of the page. */
function holdVideoFrame(video: HTMLVideoElement): (() => void) | undefined {
- // Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
- if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return;
- const css = getComputedStyle(video);
- if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return;
- if (!video.videoWidth || !video.videoHeight) return;
- const canvas = videoFrame(video);
+ const css = nativeSnapshotStyle(video);
+ if (!css) return;
+ const canvas = captureNativeVideoFrame(video);
if (!canvas) return;
for (const name of Array.from(css)) canvas.style.setProperty(name, css.getPropertyValue(name));
canvas.style.setProperty("animation", "none", "important");
@@ -49,7 +37,7 @@ function holdVideoFrames(videos: HTMLVideoElement[]): () => void {
const restore = videos.map(holdVideoFrame);
const style = document.createElement("style");
style.textContent = `[${HELD_VIDEO_ATTR}] { display: none !important; }`;
- document.head.appendChild(style);
+ if (restore.some(Boolean)) document.head.appendChild(style);
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
return () => {
for (const putBack of restore) putBack?.();
@@ -64,72 +52,226 @@ type SeekHost = {
scenesReadyAt: (time: number) => boolean;
prepare: (clips: RuntimeMediaClip[], time: number) => void;
commit: (time: number) => void;
+ frames?: {
+ available(): boolean;
+ active?(): boolean;
+ skipped?(el: HTMLMediaElement): boolean;
+ hold?(videos: HTMLVideoElement[]): (() => void) | undefined;
+ prepare(
+ clips: RuntimeMediaClip[],
+ time: number,
+ signal: AbortSignal,
+ settled?: boolean,
+ ): Promise;
+ suspend?(): Promise | undefined;
+ prepareNative(): void;
+ clear(keep?: Set): void;
+ };
};
-/** One decoder job at a time; requests arriving during it replace the queued destination. */
+/** One preparation at a time; publish coherent progress and coalesce newer destinations. */
export function createPreviewSeek(host: SeekHost) {
- let pending: { time: number; done: Promise; resolve: () => void } | null = null;
- let releasePicture = () => {};
+ type SeekRequest = {
+ time: number;
+ native: boolean;
+ settled: boolean;
+ deadline?: ReturnType;
+ decoderRelease?: Promise | null;
+ force: () => void;
+ done: Promise;
+ resolve: () => void;
+ };
+ let pending: SeekRequest | null = null;
+ // Recovery keeps its listeners without blocking paused transport work.
+ let recovery: SeekRequest | null = null;
+ let releasePicture: (() => void) | null = null;
+ const frameHolds: Array<() => void> = [];
let stopWaiting = () => {};
let runningClips: RuntimeMediaClip[] = [];
+ let cancelWork = () => {};
+ let committing = false;
+ let committingSettled = false;
let runId = 0;
+ const needsPreparation = (native: boolean) => native || host.frames?.available();
+
+ const nativeReady = ({ el, ...clip }: RuntimeMediaClip, time: number) =>
+ !el.isConnected ||
+ (el.readyState >= 2 &&
+ !el.seeking &&
+ Math.abs(el.currentTime - mediaSourceTimeAt({ el, ...clip }, time)) < 0.02);
const ready = (clips: RuntimeMediaClip[], time: number) =>
host.scenesReadyAt(time) &&
- clips.every((clip) => {
- const { el } = clip;
- return (
- !el.isConnected ||
- isUnplayable(el) ||
- (el.readyState >= 2 &&
- !el.seeking &&
- Math.abs(el.currentTime - mediaSourceTimeAt(clip, time)) < 0.02)
- );
- });
+ clips.every(
+ (clip) => isUnplayable(clip.el) || host.frames?.skipped?.(clip.el) || nativeReady(clip, time),
+ );
- const cancel = () => {
- const request = pending;
- pending = null;
+ const releaseHeldPicture = () => {
+ releasePicture?.();
+ releasePicture = null;
+ };
+ const clearFrames = (clips: RuntimeMediaClip[], native: boolean, time: number) =>
+ host.frames?.clear(
+ native
+ ? new Set(
+ clips.flatMap((clip) =>
+ isVideoElement(clip.el) && !nativeReady(clip, time) ? [clip.el] : [],
+ ),
+ )
+ : undefined,
+ );
+
+ const retireRequest = (request: SeekRequest | null) => {
+ if (pending === request) pending = null;
+ if (recovery === request) recovery = null;
+ wallClearTimeout(request?.deadline);
+ };
+ const holdPicture = () => {
+ const videos = host.videosToHold();
+ const release = host.frames?.hold?.(videos);
+ if (release) frameHolds.push(release);
+ releasePicture ??= holdVideoFrames(videos);
+ };
+ const cancel = (keepPicture = false) => {
+ const request = pending ?? recovery;
+ retireRequest(request);
runId++;
stopWaiting();
stopWaiting = () => {};
- releasePicture();
- releasePicture = () => {};
+ cancelWork();
+ cancelWork = () => {};
+ if (!keepPicture) {
+ releaseHeldPicture();
+ for (const release of frameHolds.splice(0)) release();
+ }
request?.resolve();
};
+ const afterDecoderRelease = (request: SeekRequest, start: () => void) => {
+ if (request.decoderRelease === undefined) {
+ try {
+ request.decoderRelease = host.frames?.suspend?.() ?? null;
+ } catch {
+ request.decoderRelease = null;
+ }
+ }
+ if (request.decoderRelease) void request.decoderRelease.then(start, start);
+ else start();
+ };
+
const run = () => {
- const request = pending;
+ const request = pending ?? recovery;
if (!request) return;
+ cancelWork();
const id = ++runId;
const time = request.time;
const clips = host.clipsAt(time);
runningClips = clips;
+ const selection = clips.map((clip) => ({ el: clip.el, time: mediaSourceTimeAt(clip, time) }));
+ const controller = new AbortController();
+ let frame: PreparedPausedVideoFrames | null = null;
let preparing = true;
+ let nativePhase = false;
let finished = false;
+ const live = () => (pending === request || recovery === request) && runId === id && !finished;
+ cancelWork = () => {
+ controller.abort();
+ frame?.release();
+ };
+
+ const publish = () => {
+ frameHolds.length = 0;
+ releaseHeldPicture();
+ if (!frame) clearFrames(host.clipsAt(time), request.native, time);
+ committing = true;
+ committingSettled = request.settled;
+ try {
+ host.commit(time);
+ frame?.refresh();
+ } finally {
+ committing = false;
+ committingSettled = false;
+ }
+ };
+ const selectionChanged = () => {
+ const current = host.clipsAt(time);
+ return (
+ current.length !== selection.length ||
+ current.some(
+ (clip, index) =>
+ clip.el !== selection[index]?.el ||
+ Math.abs(mediaSourceTimeAt(clip, time) - selection[index]!.time) > 1e-8,
+ )
+ );
+ };
+ const frameReadyToPublish = () => {
+ if (!frame) return true;
+ if (selectionChanged()) {
+ stopWaiting();
+ run();
+ return false;
+ }
+ if (!frame.valid() || !frame.present()) {
+ native();
+ return false;
+ }
+ return true;
+ };
const finish = () => {
- if (finished || pending !== request || id !== runId) return;
+ if (!live() || !frameReadyToPublish()) return;
finished = true;
stopWaiting();
stopWaiting = () => {};
if (request.time !== time) {
- if (ready(host.clipsAt(time), time)) {
- releasePicture();
- host.commit(time);
- releasePicture = holdVideoFrames(host.videosToHold());
+ if (frame || ready(host.clipsAt(time), time)) {
+ publish();
+ if (pending !== request && recovery !== request) return;
+ holdPicture();
}
run();
return;
}
- cancel();
- host.commit(time);
+ retireRequest(request);
+ runId++;
+ const cleanup = cancelWork;
+ cancelWork = () => {};
+ try {
+ publish();
+ } finally {
+ cleanup();
+ request.resolve();
+ }
+ };
+ request.force = () => {
+ if (!live()) return;
+ if (request.time === time) finish();
+ else {
+ stopWaiting();
+ run();
+ request.force();
+ }
};
const check = () => {
- if (!preparing && ready(host.clipsAt(time), time)) finish();
+ if (preparing || !live()) return;
+ if (frame ? host.scenesReadyAt(time) : ready(host.clipsAt(time), time)) finish();
+ };
+ const native = () => {
+ if (!live() || nativePhase) return;
+ nativePhase = true;
+ controller.abort();
+ frame?.release();
+ frame = null;
+ host.frames?.prepareNative();
+ host.prepare(host.clipsAt(time), time);
+ preparing = false;
+ check();
};
const metadata = () => {
- if (pending !== request || finished || id !== runId) return;
- // A source/proxy change resets currentTime; the parked clock still describes the old picture.
+ if (!live()) return;
+ if (!nativePhase) {
+ check();
+ return;
+ }
preparing = true;
host.prepare(host.clipsAt(time), time);
preparing = false;
@@ -147,51 +289,106 @@ export function createPreviewSeek(host: SeekHost) {
);
};
});
- const cap = wallSetTimeout(finish, SEEK_CAP_MS);
+ const expire = () => {
+ if (!live()) return;
+ if (request.native) check();
+ else if (!nativePhase && !frame) {
+ native();
+ if (live()) cap = wallSetTimeout(expire, SEEK_CAP_MS);
+ } else if (host.frames?.active?.()) {
+ pending = null;
+ recovery = request;
+ request.resolve();
+ } else finish();
+ };
+ let cap = wallSetTimeout(expire, SEEK_CAP_MS);
stopWaiting = () => {
wallClearTimeout(cap);
for (const remove of listeners) remove();
};
- host.prepare(clips, time);
- preparing = false;
- check();
+ if (request.native) afterDecoderRelease(request, native);
+ else if (recovery !== request && host.frames && needsPreparation(request.native)) {
+ void host.frames
+ .prepare(clips, time, controller.signal, request.settled)
+ .then((decoded) => {
+ if (!live() || controller.signal.aborted) {
+ decoded?.release();
+ return;
+ }
+ if (!decoded) return native();
+ frame = decoded;
+ preparing = false;
+ check();
+ })
+ .catch(native);
+ } else native();
};
+ const retarget = (time: number) => {
+ const request = pending!;
+ request.time = time;
+ const clips = host.clipsAt(time);
+ if (
+ clips.length !== runningClips.length ||
+ clips.some((clip, index) => clip.el !== runningClips[index]?.el)
+ ) {
+ stopWaiting();
+ run();
+ }
+ return request.done;
+ };
+ const seek = (time: number, native = false, settled = false): Promise | void => {
+ if (recovery) cancel(true);
+ if (pending && (pending.native !== native || pending.settled !== settled)) cancel(true);
+ if (pending) return retarget(time);
+ if (!needsPreparation(native) && ready(host.clipsAt(time), time)) {
+ releaseHeldPicture();
+ clearFrames(host.clipsAt(time), native, time);
+ host.commit(time);
+ return;
+ }
+ let resolve = () => {};
+ const done = new Promise((settle) => {
+ resolve = settle;
+ });
+ const request = { time, native, settled, done, resolve, force: () => {} };
+ pending = request;
+ // Refreshing a run must not restart the wall-clock budget for Play.
+ if (native) pending.deadline = wallSetTimeout(() => request.force(), SEEK_CAP_MS * 2);
+ holdPicture();
+ run();
+ return done;
+ };
return {
+ get committing() {
+ return committing;
+ },
+ get refining() {
+ return committingSettled;
+ },
get time() {
return pending?.time;
},
- seek(time: number): Promise | void {
- if (pending) {
- const request = pending;
- request.time = time;
- const clips = host.clipsAt(time);
- if (
- clips.length !== runningClips.length ||
- clips.some((clip, index) => clip.el !== runningClips[index]?.el)
- ) {
- stopWaiting();
- run();
- }
- return request.done;
- }
- if (ready(host.clipsAt(time), time)) {
- host.commit(time);
- return;
- }
- let resolve = () => {};
- const done = new Promise((settle) => {
- resolve = settle;
- });
- pending = { time, done, resolve };
- releasePicture = holdVideoFrames(host.videosToHold());
- run();
- return done;
+ get targetTime() {
+ return (pending ?? recovery)?.time;
+ },
+ ownsRecovery(el: HTMLMediaElement) {
+ return recovery !== null && runningClips.some((clip) => clip.el === el);
+ },
+ seek,
+ refine(time: number) {
+ return seek(time, false, true);
},
refresh() {
- if (!pending) return;
- stopWaiting();
- run();
+ if (pending || recovery) {
+ stopWaiting();
+ run();
+ }
+ },
+ handoff(time: number) {
+ cancel(true);
+ host.frames?.prepareNative();
+ return seek(time, true);
},
cancel,
};
diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.ts
index ab139168db..18eebbeb17 100644
--- a/packages/core/src/runtime/types.ts
+++ b/packages/core/src/runtime/types.ts
@@ -337,7 +337,7 @@ export type RuntimePlayer = {
_timeline: RuntimeTimelineLike | null;
play: () => void;
pause: () => void;
- seek: (timeSeconds: number, options?: { keepPlaying?: boolean }) => HeldSeek;
+ seek: (timeSeconds: number, options?: { keepPlaying?: boolean; settled?: boolean }) => HeldSeek;
renderSeek: (timeSeconds: number, options?: RuntimeSeekOptions) => void;
getTime: () => number;
getDuration: () => number;
diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts
index 68fbf584cd..6d828dba35 100644
--- a/packages/core/src/runtime/window.d.ts
+++ b/packages/core/src/runtime/window.d.ts
@@ -11,6 +11,7 @@ import type { PlayerAPI } from "../core.types";
import type { ClipTree } from "./clipTree";
import type { CompositionLengthSource } from "./compositionLength";
import type { SvgSelectorAliases } from "../compiler/svgSelectorAliases";
+import type { PausedVideoFrameProvider } from "./pausedVideoFrames";
type ThreeClockLike = {
elapsedTime: number;
@@ -79,6 +80,7 @@ declare global {
* runtime's paused-side enforcement leaves it alone. Always release. */
leasePausedMedia?: (el: HTMLMediaElement) => void;
releasePausedMedia?: (el: HTMLMediaElement) => void;
+ setPausedVideoFrameProvider?: (provider: PausedVideoFrameProvider | null) => void;
/** Where Studio's loop wraps to while it plays, or null: the clips due there stay loaded. */
setLoopStart?: (seconds: number | null) => void;
/** Read-only level taps for the Studio meters: nothing exists until
diff --git a/packages/core/src/studioPreviewMark.ts b/packages/core/src/studioPreviewMark.ts
index 94c57ef617..f32716d740 100644
--- a/packages/core/src/studioPreviewMark.ts
+++ b/packages/core/src/studioPreviewMark.ts
@@ -2,6 +2,9 @@ import { isWebKit } from "./runtime/mediaClock.js";
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
export const PREVIEW_SEEK_HOLD_ATTR = "data-hf-preview-seek-hold";
+export const PREVIEW_VIDEO_FRAME_ATTR = "data-hf-preview-video-frame";
+export const PREVIEW_VIDEO_SOURCE_ATTR = "data-hf-preview-video-source";
+export const PREVIEW_VIDEO_CANVAS_ATTR = "data-hf-preview-video-canvas";
export const STUDIO_PREVIEW_CAPTURE_MODE = "capture";
export const PLAYER_FRAME_ATTR = "data-hf-player-frame";
@@ -31,6 +34,9 @@ export const CLIP_HIDDEN_ATTR = "data-hf-clip-hidden";
export const STUDIO_PREVIEW_FAR_SCENE_ATTR = "data-hf-far-scene";
export const STUDIO_PREVIEW_PARKED_SCENE_ATTR = "data-hf-parked-scene";
export const PREVIEW_ONLY_ATTRS = [
+ PREVIEW_VIDEO_FRAME_ATTR,
+ PREVIEW_VIDEO_SOURCE_ATTR,
+ PREVIEW_VIDEO_CANVAS_ATTR,
STUDIO_PREVIEW_UPCOMING_ATTR,
STUDIO_PREVIEW_FAR_SCENE_ATTR,
STUDIO_PREVIEW_PARKED_SCENE_ATTR,
diff --git a/packages/studio-server/src/helpers/mediaMetadata.test.ts b/packages/studio-server/src/helpers/mediaMetadata.test.ts
index 965c2b030f..11ef87356e 100644
--- a/packages/studio-server/src/helpers/mediaMetadata.test.ts
+++ b/packages/studio-server/src/helpers/mediaMetadata.test.ts
@@ -168,6 +168,20 @@ describe("probeMediaMetadata", () => {
.source?.bitDepth,
).toBe(12);
expect((await probe({ pix_fmt: "yuv420p" })).source).toBeUndefined();
+ expect(
+ (
+ await probe({
+ width: 1920,
+ height: 1080,
+ r_frame_rate: "30000/1001",
+ avg_frame_rate: "30000/1001",
+ })
+ ).source,
+ ).toMatchObject({ frameRate: 30000 / 1001, variableFrameRate: false });
+ expect(
+ (await probe({ width: 1920, height: 1080, r_frame_rate: "60/1", avg_frame_rate: "24/1" }))
+ .source,
+ ).toMatchObject({ frameRate: 60, variableFrameRate: true });
});
it("lists every stream with its flags, for a failure report", async () => {
diff --git a/packages/studio-server/src/helpers/mediaMetadata.ts b/packages/studio-server/src/helpers/mediaMetadata.ts
index a093fe9f27..6381f88b4b 100644
--- a/packages/studio-server/src/helpers/mediaMetadata.ts
+++ b/packages/studio-server/src/helpers/mediaMetadata.ts
@@ -101,6 +101,8 @@ interface FfprobeStream {
width?: number;
height?: number;
bit_rate?: string;
+ r_frame_rate?: string;
+ avg_frame_rate?: string;
disposition?: Record;
tags?: Record;
}
@@ -226,11 +228,25 @@ function sourceFactsOf(
): PreviewSourceFacts | undefined {
if (!stream?.width || !stream.height) return undefined;
const bitrate = Number(stream.bit_rate ?? formatBitrate);
+ const rate = (raw: string | undefined) => {
+ const [numerator, denominator = "1"] = (raw ?? "0").split("/");
+ const value = Number(numerator) / Number(denominator);
+ return Number.isFinite(value) && value > 0 ? value : 0;
+ };
+ const nominal = rate(stream.r_frame_rate);
+ const average = rate(stream.avg_frame_rate);
return {
width: stream.width,
height: stream.height,
bitDepth: bitDepthOf(stream),
bitrate: Number.isFinite(bitrate) ? bitrate : 0,
+ ...(nominal > 0
+ ? {
+ frameRate: nominal,
+ variableFrameRate:
+ average > 0 && Math.abs(nominal - average) / Math.max(nominal, average) > 0.1,
+ }
+ : {}),
};
}
@@ -317,7 +333,7 @@ export async function probeMediaMetadata(
"-v",
"error",
"-show_entries",
- "stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample,width,height,bit_rate:stream_disposition:stream_tags=alpha_mode:format=duration,bit_rate",
+ "stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample,width,height,bit_rate,r_frame_rate,avg_frame_rate:stream_disposition:stream_tags=alpha_mode:format=duration,bit_rate",
"-of",
"json",
"--",
diff --git a/packages/studio-server/src/helpers/mediaProxyPreview.test.ts b/packages/studio-server/src/helpers/mediaProxyPreview.test.ts
index bb12771fb3..9cfa78ae84 100644
--- a/packages/studio-server/src/helpers/mediaProxyPreview.test.ts
+++ b/packages/studio-server/src/helpers/mediaProxyPreview.test.ts
@@ -33,7 +33,10 @@ async function loadHelper(
}> {
vi.resetModules();
const resolveProxy = vi.fn(async () => "/unused-prewarm-proxy-path");
- vi.doMock("./proxyTranscoder.js", () => ({ resolveProxy, PROXY_PARAMS_VERSION: "v1" }));
+ vi.doMock("./proxyTranscoder.js", () => ({
+ resolveProxy,
+ PROXY_PARAMS_VERSION: "v1",
+ }));
// The real codec table and pre-warm gate; only the ffprobe-backed scan is
// replaced, so this asserts the shipped policy rather than a copy of it.
vi.doMock("./mediaCodecMap.js", async () => ({
@@ -172,19 +175,20 @@ describe("injectMediaCodecMapIntoHtml pre-warm", () => {
});
describe("injectMediaCodecMapIntoHtml placement", () => {
- it("puts the map before the document's own , not inside an inlined script that prints one", async () => {
+ it("puts facts before runtime and authored head scripts without confusing script contents for document tags", async () => {
const { injectMediaCodecMapIntoHtml } = await loadHelper({
"/audio/seg01.webm": hostile("vp9", 'video/webm; codecs="vp09.00.10.08"'),
});
const vendor = 'p.print(""),p.print(""),p.print("")';
const html = await injectMediaCodecMapIntoHtml(
- ``,
+ ``,
tmpProject(),
[{ html: "" }],
);
expect(html).toContain(``);
- expect(html.indexOf("data-hf-media-codec-map")).toBeGreaterThan(html.indexOf(vendor));
+ expect(html.indexOf("data-hf-media-codec-map")).toBeLessThan(html.indexOf("/api/runtime.js"));
+ expect(html.indexOf("data-hf-media-codec-map")).toBeLessThan(html.indexOf(vendor));
});
});
diff --git a/packages/studio-server/src/helpers/mediaProxyPreview.ts b/packages/studio-server/src/helpers/mediaProxyPreview.ts
index 7487ad1f0a..d5e9a98472 100644
--- a/packages/studio-server/src/helpers/mediaProxyPreview.ts
+++ b/packages/studio-server/src/helpers/mediaProxyPreview.ts
@@ -1,6 +1,6 @@
import { resolve } from "node:path";
import { formatPreviewProxyBox, type PreviewProxyBox } from "@hyperframes/core";
-import { insertBeforeCloseTag } from "@hyperframes/core/compiler/html-document";
+import { injectTagsAtHeadStart } from "@hyperframes/core/compiler/html-document";
import type { StudioApiAdapter } from "../types.js";
import {
createMediaCodecProbeCache,
@@ -134,7 +134,7 @@ export async function injectMediaCodecMapIntoHtml(
`window.__HF_MEDIA_SOURCE_FACTS__=${safeJson(sourceFacts)};`,
];
const tag = ``;
- return insertBeforeCloseTag(html, "head", `${tag}\n`) ?? `${tag}\n${html}`;
+ return injectTagsAtHeadStart(html, tag);
}
/**
diff --git a/packages/studio-server/src/routes/preview.test.ts b/packages/studio-server/src/routes/preview.test.ts
index 0faf30e605..116979a0c1 100644
--- a/packages/studio-server/src/routes/preview.test.ts
+++ b/packages/studio-server/src/routes/preview.test.ts
@@ -282,6 +282,24 @@ describe("registerPreviewRoutes", () => {
expect(html).toContain("window.AUTHOR_SEEN = 1");
});
+ it("keeps the accelerated-canvas tracker before authored scripts in blur previews", async () => {
+ const projectDir = createProjectDir();
+ writeFileSync(
+ join(projectDir, "index.html"),
+ '',
+ );
+ const app = new Hono();
+ registerPreviewRoutes(app, createAdapter(projectDir));
+ const html = await (await app.request("http://localhost/projects/demo/preview")).text();
+ expect(html.indexOf("data-hf-accel-tracker")).toBeGreaterThan(-1);
+ expect(html.indexOf("data-hf-accel-tracker")).toBeLessThan(
+ html.indexOf("window.AUTHOR_CANVAS"),
+ );
+ expect(html.indexOf('src="/api/runtime.js"')).toBeLessThan(
+ html.indexOf("window.AUTHOR_CANVAS"),
+ );
+ });
+
it("serves one preview runtime when the page on disk already links one", async () => {
const projectDir = createProjectDir();
writeFileSync(
@@ -1626,6 +1644,71 @@ describe("hf-proxy negotiation and media codec map injection (U3)", () => {
return import("./preview.js");
}
+ it.each(["bundled", "disk after null bundle", "disk after failed bundle"])(
+ "installs preview hooks before authored head consumers with flags, facts and error capture already present (%s)",
+ async (path) => {
+ const projectDir = createProjectDir();
+ const authored = `
+
+
+
+
+
+
+ `;
+ writeFileSync(join(projectDir, "index.html"), authored);
+ writeFileSync(
+ join(projectDir, "scene.html"),
+ ``,
+ );
+ const { registerPreviewRoutes: register } = await loadPreviewModule({
+ scanMapImpl: async () => ({
+ "/clip.webm": {
+ codecName: "vp9",
+ browserHostile: true,
+ representativeMime: 'video/webm; codecs="vp09.00.10.08"',
+ },
+ }),
+ });
+ const app = new Hono();
+ register(
+ app,
+ createAdapter(projectDir, {
+ bundle: async () => {
+ if (path === "disk after failed bundle") throw new Error("bundler unavailable");
+ return path === "bundled" ? authored : null;
+ },
+ }),
+ );
+ for (const suffix of ["", "/comp/scene.html"]) {
+ const response = await app.request("http://localhost/projects/demo/preview" + suffix);
+ expect(response.status).toBe(200);
+ const html = await response.text();
+ const runtime = html.indexOf('src="/api/runtime.js"');
+ expect(runtime).toBeGreaterThan(-1);
+ for (const early of [
+ `name="${STUDIO_PREVIEW_MARK_META}"`,
+ `window.${STUDIO_PREVIEW_ERRORS}=`,
+ "window.__HF_MEDIA_CODEC_MAP__=",
+ "window.__HF_MEDIA_SOURCE_FACTS__=",
+ ]) {
+ expect(html.indexOf(early)).toBeGreaterThan(-1);
+ expect(html.indexOf(early)).toBeLessThan(runtime);
+ }
+ for (const authoredId of ["head-consumer", "head-library", "body-consumer"])
+ expect(runtime).toBeLessThan(html.indexOf(`id="${authoredId}"`));
+ expect(html).not.toContain("stale-runtime.js");
+ expect(
+ parseHTML(html).document.querySelectorAll("script[data-hyperframes-preview-runtime]"),
+ ).toHaveLength(1);
+ expect(
+ parseHTML(html).document.querySelectorAll("script[data-hf-accel-tracker]"),
+ ).toHaveLength(1);
+ expect(html).toContain("window.TRACKER_KEPT = true;");
+ }
+ },
+ );
+
describe("?hf-proxy=h264 on the static asset route", () => {
it("serves proxy bytes with Accept-Ranges on a full request, and a 206 range slice on a Range request", async () => {
const projectDir = createProjectDir();
diff --git a/packages/studio-server/src/routes/preview.ts b/packages/studio-server/src/routes/preview.ts
index 8a2cf91bad..1f334cdcab 100644
--- a/packages/studio-server/src/routes/preview.ts
+++ b/packages/studio-server/src/routes/preview.ts
@@ -150,10 +150,10 @@ function parseStudioMotionManifestContent(content: string): {
}
}
-/** Swaps any runtime already in the page for the preview runtime, placed as the bundler places it. */
+/** Preview hooks must install before authored head scripts register media consumers. */
function withPreviewRuntime(html: string, runtimeUrl: string): string {
const tag = ``;
- return insertRuntimeTag(stripEmbeddedRuntimeScripts(html), tag);
+ return insertRuntimeTag(stripEmbeddedRuntimeScripts(html), tag, true);
}
function injectScriptTagIntoHead(html: string, scriptTag: string): string {
@@ -685,7 +685,7 @@ export function registerPreviewRoutes(api: Hono, adapter: PreviewApiAdapter): vo
gsapBaseUrl,
);
if (!html) return c.text("not found", 404);
- html = withRootImportMap(html, baseHref);
+ html = withPreviewRuntime(withRootImportMap(html, baseHref), adapter.runtimeUrl);
recordPreviewReferences(project.dir, html);
html = ensureHfIds(await transformPreviewHtml(html, adapter, project, compPath));
html = injectStudioPreviewAugmentations(html, adapter, signature, project.dir, compPath, {
diff --git a/packages/studio/package.json b/packages/studio/package.json
index 458f21f793..ba6d9d96eb 100644
--- a/packages/studio/package.json
+++ b/packages/studio/package.json
@@ -114,7 +114,7 @@
"cn": "0.2.4",
"dockview-react": "8.3.1",
"gsap": "^3.13.0",
- "mediabunny": "^1.45.3"
+ "mediabunny": "^1.65.2"
},
"devDependencies": {
"@hyperframes/producer": "workspace:*",
diff --git a/packages/studio/src/components/editor/domEditingDom.preview.test.ts b/packages/studio/src/components/editor/domEditingDom.preview.test.ts
new file mode 100644
index 0000000000..edf1ee2409
--- /dev/null
+++ b/packages/studio/src/components/editor/domEditingDom.preview.test.ts
@@ -0,0 +1,75 @@
+// @vitest-environment jsdom
+import { afterEach, expect, it } from "vitest";
+import {
+ PREVIEW_VIDEO_FRAME_ATTR,
+ PREVIEW_VIDEO_CANVAS_ATTR,
+} from "@hyperframes/core/studio-preview-mark";
+import { getCuratedComputedStyles, isElementVisibleThroughAncestors } from "./domEditingDom";
+import {
+ collectDomEditLayerItems,
+ resolveDomEditSelection,
+ resolveVisualDomEditSelectionTarget,
+} from "./domEditing";
+
+const options = { activeCompositionPath: "index.html", isMasterView: true, skipSourceProbe: true };
+afterEach(() => {
+ document.body.innerHTML = "";
+ document.head.innerHTML = "";
+});
+
+function preview(opacity: string) {
+ document.head.innerHTML = ``;
+ document.body.innerHTML = ``;
+ const video = document.querySelector("video")!;
+ const canvas = document.querySelector("canvas")!;
+ const root = document.querySelector("main")!;
+ Object.defineProperty(video, "getBoundingClientRect", {
+ value: () => ({
+ x: 10,
+ y: 20,
+ left: 10,
+ top: 20,
+ right: 210,
+ bottom: 120,
+ width: 200,
+ height: 100,
+ }),
+ });
+ return { video, canvas, root };
+}
+
+it("keeps a provider-backed video selectable and listed with its logical opacity", async () => {
+ const { video, canvas, root } = preview("0.65");
+ expect(getComputedStyle(video).opacity).toBe("0");
+ expect(isElementVisibleThroughAncestors(video)).toBe(true);
+ expect(resolveVisualDomEditSelectionTarget([canvas, video], options)).toBe(video);
+ expect(collectDomEditLayerItems(root, options).map((item) => item.element)).toEqual([video]);
+ const selected = await resolveDomEditSelection(video, options);
+ expect(selected?.computedStyles.opacity).toBe("0.65");
+ expect(selected?.inlineStyles.opacity).toBe("0.65");
+ expect(getComputedStyle(video).opacity).toBe("0");
+});
+
+it("keeps authored zero opacity invisible and reports zero to the inspector", async () => {
+ const { video, canvas } = preview("0");
+ expect(isElementVisibleThroughAncestors(video)).toBe(false);
+ expect(resolveVisualDomEditSelectionTarget([canvas, video], options)).toBeNull();
+ expect(getCuratedComputedStyles(video).opacity).toBe("0");
+ const selected = await resolveDomEditSelection(video, options);
+ expect(selected?.computedStyles.opacity).toBe("0");
+ expect(video.style.opacity).toBe("0");
+});
+
+it("still respects a hidden ancestor when the preview frame itself is visible", () => {
+ const { video, root } = preview("0.65");
+ root.style.opacity = "0";
+ expect(isElementVisibleThroughAncestors(video)).toBe(false);
+ expect(resolveVisualDomEditSelectionTarget([video], options)).toBeNull();
+});
+
+it("does not treat an unrelated preceding canvas as the video's visible frame", () => {
+ const { video, canvas } = preview("0.65");
+ canvas.removeAttribute(PREVIEW_VIDEO_CANVAS_ATTR);
+ expect(isElementVisibleThroughAncestors(video)).toBe(false);
+ expect(getCuratedComputedStyles(video).opacity).toBe("0");
+});
diff --git a/packages/studio/src/components/editor/domEditingDom.ts b/packages/studio/src/components/editor/domEditingDom.ts
index 2b69b77fb2..29a5f435cd 100644
--- a/packages/studio/src/components/editor/domEditingDom.ts
+++ b/packages/studio/src/components/editor/domEditingDom.ts
@@ -3,6 +3,10 @@
* selector utilities, and composition source resolution.
* No imports from other domEditing* modules — safe to import from anywhere.
*/
+import {
+ PREVIEW_VIDEO_FRAME_ATTR,
+ PREVIEW_VIDEO_CANVAS_ATTR,
+} from "@hyperframes/core/studio-preview-mark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading";
import { getSourceScopedSelectorIndex } from "../../utils/sourceScopedSelectorIndex";
import { CURATED_STYLE_PROPERTIES } from "./domEditingTypes";
@@ -34,12 +38,24 @@ export function isEditableTextLeaf(el: HTMLElement): boolean {
return isTextBearingTag(el.tagName.toLowerCase()) && el.children.length === 0;
}
+/** The preview canvas keeps the source's authored opacity while its native pixels are hidden. */
+export function getLogicalOpacity(el: Element, computed: CSSStyleDeclaration): string {
+ const overlay = el.previousElementSibling;
+ if (
+ el.hasAttribute(PREVIEW_VIDEO_FRAME_ATTR) &&
+ overlay?.localName === "canvas" &&
+ overlay.hasAttribute(PREVIEW_VIDEO_CANVAS_ATTR)
+ )
+ return el.ownerDocument.defaultView?.getComputedStyle(overlay).opacity ?? computed.opacity;
+ return computed.opacity;
+}
+
/** Does this node render AT ALL, ignoring what it inherits? Sole owner of the
* rule; the walk below only decides which nodes to ask it about. */
function elementRendersItself(win: Window, el: HTMLElement): boolean {
const computed = win.getComputedStyle(el);
if (computed.display === "none" || computed.visibility === "hidden") return false;
- const opacity = Number.parseFloat(computed.opacity);
+ const opacity = Number.parseFloat(getLogicalOpacity(el, computed));
return !(
Number.isFinite(opacity) &&
opacity <= 0.01 &&
@@ -101,7 +117,8 @@ export function getCuratedComputedStyles(el: HTMLElement): Record void;
pause: () => void;
togglePlay: () => void;
- seek: (time: number, options?: { keepPlaying?: boolean; follow?: boolean }) => boolean;
+ seek: (
+ time: number,
+ options?: { keepPlaying?: boolean; follow?: boolean; settled?: boolean },
+ ) => boolean;
refreshPlayer: () => void;
onIframeLoad: () => void;
// The hidden reload iframe NLEPreview renders next to the live one during a full reload.
diff --git a/packages/studio/src/hooks/useElementPicker.test.ts b/packages/studio/src/hooks/useElementPicker.test.ts
index 8ccc2a4da5..bd39acf466 100644
--- a/packages/studio/src/hooks/useElementPicker.test.ts
+++ b/packages/studio/src/hooks/useElementPicker.test.ts
@@ -115,6 +115,19 @@ function mountPicker(
};
}
+it.each(["0.65", "0"])(
+ "reads provider-backed video opacity %s in the picked-element inspector",
+ (opacity) => {
+ const mounted = ``;
+ const { iframe, picker } = mountPicker({ "index.html": SAVED }, "#provider-video", mounted);
+ expect(
+ iframe.contentWindow!.getComputedStyle(iframe.contentDocument!.querySelector("video")!)
+ .opacity,
+ ).toBe("0");
+ expect(picker().pickedElement?.computedStyles.opacity).toBe(opacity);
+ },
+);
+
describe("pick mode cancellation and hover feedback", () => {
it("cancels pick mode when Escape is pressed with Studio focus", () => {
const { picker } = mountPicker({ "index.html": SAVED });
diff --git a/packages/studio/src/hooks/useElementPicker.ts b/packages/studio/src/hooks/useElementPicker.ts
index ef55823dee..c054c34943 100644
--- a/packages/studio/src/hooks/useElementPicker.ts
+++ b/packages/studio/src/hooks/useElementPicker.ts
@@ -12,7 +12,7 @@ import {
postRuntimeControlMessage,
} from "../player/lib/runtimeProtocol";
import { compositionPathOfPreviewUrl } from "../player/components/CompositionThumbnail";
-import { getSourceFileForElement } from "../components/editor/domEditingDom";
+import { getLogicalOpacity, getSourceFileForElement } from "../components/editor/domEditingDom";
import {
createElementPickerHoverLabel,
type ElementPickerHoverInfo,
@@ -589,7 +589,8 @@ function readComputedStyles(iframe: HTMLIFrameElement, selector: string): Record
];
for (const prop of props) {
- const val = computed.getPropertyValue(prop);
+ const val =
+ prop === "opacity" ? getLogicalOpacity(el, computed) : computed.getPropertyValue(prop);
if (val) styles[prop] = val;
}
} catch {
diff --git a/packages/studio/src/player/components/PlayerControls.test.tsx b/packages/studio/src/player/components/PlayerControls.test.tsx
index ab68c79a68..1ae33fded1 100644
--- a/packages/studio/src/player/components/PlayerControls.test.tsx
+++ b/packages/studio/src/player/components/PlayerControls.test.tsx
@@ -57,7 +57,7 @@ describe("PlayerControls preview seeking", () => {
setValue.call(input, String(time));
input.dispatchEvent(new Event("input", { bubbles: true }));
});
- expect(onSeek).toHaveBeenLastCalledWith(time);
+ expect(onSeek).toHaveBeenLastCalledWith(time, { settled: true });
}
expect(onSeek).toHaveBeenCalledTimes(4);
});
diff --git a/packages/studio/src/player/components/PlayerControls.tsx b/packages/studio/src/player/components/PlayerControls.tsx
index 3b9883e8f8..a21ba3a0a1 100644
--- a/packages/studio/src/player/components/PlayerControls.tsx
+++ b/packages/studio/src/player/components/PlayerControls.tsx
@@ -161,7 +161,7 @@ const FullscreenButton = memo(function FullscreenButton({
interface PlayerControlsProps {
onTogglePlay: () => void;
- onSeek: (time: number) => void;
+ onSeek: (time: number, options?: { settled?: boolean }) => void;
disabled?: boolean;
isFullscreen?: boolean;
onToggleFullscreen?: () => void;
diff --git a/packages/studio/src/player/components/PreviewSeekBar.test.tsx b/packages/studio/src/player/components/PreviewSeekBar.test.tsx
new file mode 100644
index 0000000000..697e090f83
--- /dev/null
+++ b/packages/studio/src/player/components/PreviewSeekBar.test.tsx
@@ -0,0 +1,42 @@
+// @vitest-environment happy-dom
+import React, { act } from "react";
+import { expect, it, vi } from "vitest";
+import { PreviewSeekBar } from "./PreviewSeekBar";
+import { createHappyDomRootHarness } from "./testRootHarness";
+
+Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
+const { mount } = createHappyDomRootHarness();
+
+it("marks keyboard and released seeks as settled and pointer scrubbing as interactive", () => {
+ const host = document.createElement("div");
+ document.body.append(host);
+ const onSeek = vi.fn();
+ act(() => mount(host).render());
+ const input = host.querySelector("input")!;
+ const change = (value: number) => {
+ Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(
+ input,
+ String(value),
+ );
+ input.dispatchEvent(new Event("input", { bubbles: true }));
+ };
+ act(() => {
+ input.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
+ change(1);
+ });
+ expect(onSeek).toHaveBeenLastCalledWith(1, { settled: true });
+ act(() => {
+ input.dispatchEvent(new PointerEvent("pointerdown", { bubbles: true }));
+ change(2);
+ });
+ expect(onSeek).toHaveBeenLastCalledWith(2, { settled: false });
+ act(() => {
+ input.dispatchEvent(new PointerEvent("pointerup", { bubbles: true }));
+ });
+ expect(onSeek).toHaveBeenLastCalledWith(2, { settled: true });
+ act(() => {
+ input.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
+ change(3);
+ });
+ expect(onSeek).toHaveBeenLastCalledWith(3, { settled: true });
+});
diff --git a/packages/studio/src/player/components/PreviewSeekBar.tsx b/packages/studio/src/player/components/PreviewSeekBar.tsx
index 272e66ee43..093aceaabf 100644
--- a/packages/studio/src/player/components/PreviewSeekBar.tsx
+++ b/packages/studio/src/player/components/PreviewSeekBar.tsx
@@ -1,4 +1,4 @@
-import { memo, useEffect, useRef } from "react";
+import { memo, useEffect, useRef, type PointerEvent } from "react";
import { formatTime } from "../lib/time";
import { liveTime, usePlayerStore } from "../store/playerStore";
@@ -9,9 +9,16 @@ export const PreviewSeekBar = memo(function PreviewSeekBar({
}: {
duration: number;
disabled: boolean;
- onSeek: (time: number) => void;
+ onSeek: (time: number, options?: { settled?: boolean }) => void;
}) {
const inputRef = useRef(null);
+ const settledRef = useRef(true);
+
+ const finishPointerSeek = (event: PointerEvent) => {
+ if (settledRef.current) return;
+ settledRef.current = true;
+ onSeek(event.currentTarget.valueAsNumber, { settled: true });
+ };
useEffect(() => {
const input = inputRef.current;
@@ -39,7 +46,17 @@ export const PreviewSeekBar = memo(function PreviewSeekBar({
defaultValue={0}
disabled={disabled || duration <= 0}
aria-label="Seek preview"
- onChange={(event) => onSeek(event.currentTarget.valueAsNumber)}
+ onPointerDown={() => {
+ settledRef.current = false;
+ }}
+ onPointerUp={finishPointerSeek}
+ onPointerCancel={finishPointerSeek}
+ onKeyDown={() => {
+ settledRef.current = true;
+ }}
+ onChange={(event) =>
+ onSeek(event.currentTarget.valueAsNumber, { settled: settledRef.current })
+ }
className="h-6 w-full cursor-pointer accent-neutral-300 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-accent-ink disabled:cursor-default disabled:opacity-30"
/>
diff --git a/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts b/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts
index 8f1ce4b30b..b66c49b158 100644
--- a/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts
+++ b/packages/studio/src/player/hooks/usePlaybackKeyboard.test.ts
@@ -331,7 +331,7 @@ describe("usePlaybackKeyboard — frame step", () => {
dispatch(keydown({ code: "ArrowRight", key: "ArrowRight" }));
});
- expect(spies.seek).toHaveBeenCalledWith((frame + 1) / fps);
+ expect(spies.seek).toHaveBeenCalledWith((frame + 1) / fps, { settled: true });
});
});
diff --git a/packages/studio/src/player/hooks/usePlaybackKeyboard.ts b/packages/studio/src/player/hooks/usePlaybackKeyboard.ts
index 99a59a2dac..39cda4b457 100644
--- a/packages/studio/src/player/hooks/usePlaybackKeyboard.ts
+++ b/packages/studio/src/player/hooks/usePlaybackKeyboard.ts
@@ -30,7 +30,7 @@ interface UsePlaybackKeyboardParams {
play: () => void;
playBackward: (rate: number) => void;
pause: () => void;
- seek: (time: number, options?: { keepPlaying?: boolean }) => void;
+ seek: (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => void;
}
const FRAME_STEPS = {
@@ -112,6 +112,7 @@ export function usePlaybackKeyboard({
const currentTime = adapter?.getTime() ?? usePlayerStore.getState().currentTime;
seek(
stepFrameTime(currentTime, deltaFrames, useLinkedClipPreferences.getState().compositionFps),
+ { settled: true },
);
},
[getAdapter, seek],
diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.ts b/packages/studio/src/player/hooks/useTimelinePlayer.ts
index 4a398ca0cf..14fc807220 100644
--- a/packages/studio/src/player/hooks/useTimelinePlayer.ts
+++ b/packages/studio/src/player/hooks/useTimelinePlayer.ts
@@ -28,6 +28,7 @@ export {
} from "../lib/playbackShortcuts";
import type { PlaybackAdapter, IframeWindow } from "../lib/playbackTypes";
+import { attachPausedVideoProvider } from "../lib/pausedVideoProvider";
import { releaseStaticSeekCache, type StaticSeekCacheEntry } from "../lib/playbackAdapter";
import { mergeTimelineElementsPreservingDowngrades } from "../lib/timelineDOM";
import { findTimelineElementInIframe } from "../../hooks/timelineEditingHelpers";
@@ -150,6 +151,7 @@ export function useTimelinePlayer({
const iframe = overrideIframe !== undefined ? overrideIframe : iframeRef.current;
const win = iframe?.contentWindow as IframeWindow | null;
if (!iframe || !win) return null;
+ attachPausedVideoProvider(iframe, win);
const adapter = resolvePlaybackAdapter(iframe, win, {
cache: staticSeekAdapterRef,
warned: staticSeekWarnedRef,
@@ -295,7 +297,7 @@ export function useTimelinePlayer({
stopRAFLoop();
}, [getAdapter, setCurrentTime, setIsPlaying, stopRAFLoop, stopReverseLoop]);
const seek = useCallback(
- (time: number, options?: { keepPlaying?: boolean; follow?: boolean }) => {
+ (time: number, options?: { keepPlaying?: boolean; follow?: boolean; settled?: boolean }) => {
const wasReverseShuttle = shuttleDirectionRef.current === "backward";
stopReverseLoop();
const adapter = getAdapter();
diff --git a/packages/studio/src/player/lib/mediaFetch.test.ts b/packages/studio/src/player/lib/mediaFetch.test.ts
index a80d87553c..70b8205582 100644
--- a/packages/studio/src/player/lib/mediaFetch.test.ts
+++ b/packages/studio/src/player/lib/mediaFetch.test.ts
@@ -611,8 +611,7 @@ describe("mediaUrlSourceOptions", () => {
expect(failure.retries).toBe(0);
});
- // mediabunny rethrows a failed read-ahead nobody awaits "so it doesn't get swallowed": that copy is typed too.
- it("fails a real mediabunny read, and its unawaited read-ahead, with the typed error", async () => {
+ it("fails a real mediabunny read with the typed error without a duplicate unhandled rejection", async () => {
vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("", { status: 404 })));
const mb = await import("mediabunny");
const runnerListeners = process.listeners("unhandledRejection");
@@ -629,8 +628,8 @@ describe("mediaUrlSourceOptions", () => {
await new Promise(setImmediate);
expect(error).toMatchObject({ name: "MediaFetchError", status: 404, retries: 0 });
- expect(unhandled.length).toBeGreaterThan(0);
- for (const reason of unhandled) expect(reason).toBeInstanceOf(MediaFetchError);
+ expect(error).toBeInstanceOf(MediaFetchError);
+ expect(unhandled).toEqual([]);
} finally {
process.removeAllListeners("unhandledRejection");
for (const listener of runnerListeners) process.on("unhandledRejection", listener);
diff --git a/packages/studio/src/player/lib/pausedVideoDecoder.test.ts b/packages/studio/src/player/lib/pausedVideoDecoder.test.ts
new file mode 100644
index 0000000000..caa28ee5ef
--- /dev/null
+++ b/packages/studio/src/player/lib/pausedVideoDecoder.test.ts
@@ -0,0 +1,644 @@
+// @vitest-environment happy-dom
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { createPausedVideoDecoder } from "./pausedVideoDecoder";
+
+const inputs: { source: string; dispose: ReturnType