diff --git a/bun.lock b/bun.lock index faecd8248e..7477213746 100644 --- a/bun.lock +++ b/bun.lock @@ -389,7 +389,7 @@ "cn": "0.2.4", "dockview-react": "8.3.1", "gsap": "^3.13.0", - "mediabunny": "^1.45.3", + "mediabunny": "^1.65.2", }, "devDependencies": { "@hyperframes/producer": "workspace:*", @@ -1856,7 +1856,7 @@ "mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="], - "mediabunny": ["mediabunny@1.45.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-GUCPYjR+5olLM7DRmupCXCmZkkSrKHVl1gyW2RztpObqLfrix19kWGf/9WgWzDW0g49DvfGXpl+zfArCx5HDMQ=="], + "mediabunny": ["mediabunny@1.65.2", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-goZpibvo/5hkc/56WVNm93Ini+o7K4y8aXA20c/BN1B+m3L5aHQa8MTvO+S1APg/YI8kMILtrX6QCqUtxLW7bA=="], "meow": ["meow@13.2.0", "", {}, "sha512-pxQJQzB6djGPXh08dacEloMFopsOqGVRKFPYvPOt9XDZ1HasbgDZA74CJGreSU4G3Ak7EFJGoiH2auq+yXISgA=="], diff --git a/docs/packages/studio.mdx b/docs/packages/studio.mdx index 77ba6b2ac2..a5eed86bea 100644 --- a/docs/packages/studio.mdx +++ b/docs/packages/studio.mdx @@ -106,6 +106,31 @@ depends on the browser playing in real time, while render captures frames one at time. A heavy project can therefore stutter during preview and still render every frame. +Paused video scrubbing decodes and stages the visible video layers before +advancing the composition's animations. Completed frames appear during a drag, +with newer pointer positions coalesced into the next decode. Canvas and GPU +effects consume the same staged pixels. Continuous playback uses native video; +Studio keeps the paused picture visible while those videos catch up before Play. +A native handoff waits for ready frames for up to ten seconds; Pause or a new +seek cancels it. At the deadline, the rest plays while each unready video keeps +its last full-quality picture. During playback, native frame presentation or +playback progress releases that picture; normal transport sync corrects drift. +Paused recovery still waits for its requested frame. A skipped source keeps its +picture on subsequent Play requests until it recovers. +A paused fallback that outlasts its wait retains the presented clock and picture, +while paused sync continues. It can still publish when the video becomes ready; +a new gesture cancels that recovery. Studio requests paused-decoder release +before preparing native playback and reopens readers when paused scrubbing resumes. +Transparent video queries drain their alpha merges before input disposal. If one +never settles, its cleanup can outlast the Play deadline; the remaining playback +still starts while that video's last picture stays visible. +Scrubbing and paused frame steps decode the original video at source resolution +throughout. Automatic playback quality reductions do not reduce these frames, +and the frame decoder does not substitute re-encoded seek copies. Playback +quality settings do not reduce these paused frames. Browsers or +media that cannot use the frame decoder fall back to native seeking. Long-GOP +sources can take longer to seek, especially backward or across distant cuts. + ## Tailwind preset ```typescript diff --git a/packages/core/src/compiler/htmlDocument.test.ts b/packages/core/src/compiler/htmlDocument.test.ts index 466d993e02..b5bb0ebbf9 100644 --- a/packages/core/src/compiler/htmlDocument.test.ts +++ b/packages/core/src/compiler/htmlDocument.test.ts @@ -469,6 +469,14 @@ describe("insertRuntimeTag", () => { expect(out.indexOf("window.author")).toBeLessThan(out.indexOf(runtime)); }); + it("keeps the tracker and runtime ahead of author scripts for Studio previews", () => { + const out = insertRuntimeTag(page('data-motion-blur="on"'), runtime, true); + const tracker = out.indexOf("data-hf-accel-tracker"); + expect(tracker).toBeGreaterThan(-1); + expect(tracker).toBeLessThan(out.indexOf(runtime)); + expect(out.indexOf(runtime)).toBeLessThan(out.indexOf("window.author")); + }); + it("adds the tracker once, however many times the runtime is placed", () => { const once = insertRuntimeTag(page('data-motion-blur="on"'), runtime); const twice = insertRuntimeTag(once, runtime); diff --git a/packages/core/src/compiler/htmlDocument.ts b/packages/core/src/compiler/htmlDocument.ts index 09d4a6c182..45fe3fdca9 100644 --- a/packages/core/src/compiler/htmlDocument.ts +++ b/packages/core/src/compiler/htmlDocument.ts @@ -393,8 +393,8 @@ function insertTagInHead(html: string, tag: string): string { const ACCEL_TRACKER_ATTR = "data-hf-accel-tracker"; const OPTS_INTO_MOTION_BLUR = /\sdata-motion-blur\s*=\s*["']?on[\s"'>]/i; -export function insertRuntimeTag(html: string, tag: string): string { - const withRuntime = insertTagInHead(html, tag); +export function insertRuntimeTag(html: string, tag: string, atHeadStart = false): string { + const withRuntime = atHeadStart ? injectTagsAtHeadStart(html, tag) : insertTagInHead(html, tag); if (!OPTS_INTO_MOTION_BLUR.test(html) || html.includes(ACCEL_TRACKER_ATTR)) return withRuntime; const tracker = ``; return injectTagsAtHeadStart(withRuntime, tracker); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e0381edf31..fd13af9f2a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -308,6 +308,11 @@ export { type PreviewQuality, type PreviewSourceFacts, } from "./previewProxyBox"; +export type { + PausedVideoFrameProvider, + PausedVideoFrameRequest, + PausedVideoFrame, +} from "./runtime/pausedVideoFrames"; // Inline scripts export { diff --git a/packages/core/src/previewProxyBox.ts b/packages/core/src/previewProxyBox.ts index 6190eb193f..be011b27cc 100644 --- a/packages/core/src/previewProxyBox.ts +++ b/packages/core/src/previewProxyBox.ts @@ -55,6 +55,8 @@ export type PreviewSourceFacts = { bitDepth: number; bitrate: number; codec?: string; + frameRate?: number; + variableFrameRate?: boolean; }; export const HEAVY_PIXEL_SHARE = 0.5; diff --git a/packages/core/src/runtime/adapters/video-texture-compat.paused.test.ts b/packages/core/src/runtime/adapters/video-texture-compat.paused.test.ts new file mode 100644 index 0000000000..4c4bd69007 --- /dev/null +++ b/packages/core/src/runtime/adapters/video-texture-compat.paused.test.ts @@ -0,0 +1,238 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { createPausedVideoFrames, getPausedVideoFrame } from "../pausedVideoFrames"; +import { + patchCanvasVideoFrameCompat, + patchExternalVideoTextureCompat, + patchVideoTextureCompat, + patchWebGLVideoTextureCompat, +} from "./video-texture-compat"; + +const sessions: ReturnType[] = []; +afterEach(() => { + for (const session of sessions.splice(0)) session.destroy(); + document.body.innerHTML = ""; + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +function surfaces() { + const draw = vi.fn(); + const offscreenDraw = vi.fn(); + const image = vi.fn(); + const subImage = vi.fn(); + const copy = vi.fn(); + const imported = vi.fn((_descriptor: unknown) => ({ texture: true })); + const constructed = vi.fn(); + class Context2D { + drawImage(...args: unknown[]) { + draw.apply(this, args); + } + } + class Offscreen2D { + drawImage(...args: unknown[]) { + offscreenDraw.apply(this, args); + } + } + class GL { + texImage2D(...args: unknown[]) { + image.apply(this, args); + } + texSubImage2D(...args: unknown[]) { + subImage.apply(this, args); + } + } + class GL2 extends GL {} + class Queue { + copyExternalImageToTexture(...args: unknown[]) { + copy.apply(this, args); + } + } + class Device { + importExternalTexture(descriptor: unknown) { + return imported.call(this, descriptor); + } + } + class Frame { + close = vi.fn(); + constructor(source: unknown, options: unknown) { + constructed(source, options); + } + } + vi.stubGlobal("CanvasRenderingContext2D", Context2D); + vi.stubGlobal("OffscreenCanvasRenderingContext2D", Offscreen2D); + vi.stubGlobal("WebGLRenderingContext", GL); + vi.stubGlobal("WebGL2RenderingContext", GL2); + vi.stubGlobal("GPUQueue", Queue); + vi.stubGlobal("GPUDevice", Device); + vi.stubGlobal("VideoFrame", Frame); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + clearRect: vi.fn(), + drawImage: vi.fn(), + } as unknown as CanvasRenderingContext2D); + return { + Context2D, + Offscreen2D, + GL, + GL2, + Queue, + Device, + draw, + offscreenDraw, + image, + subImage, + copy, + imported, + constructed, + }; +} + +async function presented() { + const session = createPausedVideoFrames(); + sessions.push(session); + const video = document.createElement("video"); + video.src = "/original.mp4"; + document.body.append(video); + Object.defineProperties(video, { + videoWidth: { value: 1920 }, + videoHeight: { value: 1080 }, + }); + const source = document.createElement("canvas"); + source.width = 320; + source.height = 180; + session.setProvider({ + decode: async (requests) => + requests.map(({ time }) => ({ + source, + timestamp: time, + duration: 1 / 30, + width: 1920, + height: 1080, + release: vi.fn(), + })), + dispose: vi.fn(), + }); + const show = async (time: number) => { + const batch = await session.prepare( + [{ video, source: video.src, time }], + new AbortController().signal, + ); + expect(batch?.present()).toBe(true); + batch!.refresh(); + }; + await show(3); + const canvas = getPausedVideoFrame(video)!; + expect([canvas.width, canvas.height]).toEqual([video.videoWidth, video.videoHeight]); + return { session, video, canvas, show }; +} + +it.each([ + ["three arguments", [11, 17]], + ["five arguments", [11, 17, 640, 360]], + ["nine arguments", [120, 60, 960, 540, 11, 17, 640, 360]], +])("preserves %s and source coordinates for both canvas surfaces", async (_name, coordinates) => { + const { Context2D, Offscreen2D, draw, offscreenDraw } = surfaces(); + const { session, video, canvas } = await presented(); + patchCanvasVideoFrameCompat(); + patchCanvasVideoFrameCompat(); + const context = new Context2D(); + const offscreen = new Offscreen2D(); + context.drawImage(video, ...coordinates); + offscreen.drawImage(video, ...coordinates); + expect(draw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates); + expect(offscreenDraw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates); + expect(draw.mock.contexts[0]).toBe(context); + expect(offscreenDraw.mock.contexts[0]).toBe(offscreen); + session.clear(); + context.drawImage(video, ...coordinates); + expect(draw).toHaveBeenLastCalledWith(video, ...coordinates); + const image = document.createElement("img"); + offscreen.drawImage(image, ...coordinates); + expect(offscreenDraw).toHaveBeenLastCalledWith(image, ...coordinates); +}); + +it("uploads paused pixels through both WebGL versions without changing crop or upload parameters", async () => { + const { GL, GL2, image, subImage } = surfaces(); + const { session, video, canvas } = await presented(); + patchWebGLVideoTextureCompat(); + for (const Context of [GL, GL2]) { + const gl = new Context(); + const imageArgs = [0x0de1, 2, 0x1908, 0x1908, 0x1401]; + const subArgs = [0x0de1, 2, 13, 29, 0x1908, 0x1401]; + gl.texImage2D(...imageArgs, video); + gl.texSubImage2D(...subArgs, video); + expect(image).toHaveBeenLastCalledWith(...imageArgs, canvas); + expect(subImage).toHaveBeenLastCalledWith(...subArgs, canvas); + expect(image.mock.contexts.at(-1)).toBe(gl); + const pixels = new Uint8Array([10, 20, 30, 255]); + gl.texImage2D(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels); + expect(image).toHaveBeenLastCalledWith(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels); + } + session.clear(); + new GL().texImage2D(0x0de1, 0, 0x1908, 0x1908, 0x1401, video); + expect(image.mock.lastCall?.at(-1)).toBe(video); +}); + +it("preserves WebGPU crop, orientation, destination and full intrinsic copy extent", async () => { + const { Queue, copy } = surfaces(); + const { session, video, canvas } = await presented(); + patchVideoTextureCompat(); + const queue = new Queue(); + const source = { source: video, origin: { x: 37, y: 41 }, flipY: true }; + const destination = { texture: {}, origin: [5, 7], mipLevel: 1, premultipliedAlpha: true }; + const extent = { + width: video.videoWidth - 37, + height: video.videoHeight - 41, + depthOrArrayLayers: 1, + }; + queue.copyExternalImageToTexture(source, destination, extent); + expect(copy).toHaveBeenCalledExactlyOnceWith({ ...source, source: canvas }, destination, extent); + expect(copy.mock.contexts[0]).toBe(queue); + expect(source.source).toBe(video); + expect(copy.mock.calls[0]?.[1]).toBe(destination); + expect(copy.mock.calls[0]?.[2]).toBe(extent); + session.clear(); + queue.copyExternalImageToTexture(source, destination, extent); + expect(copy).toHaveBeenLastCalledWith(source, destination, extent); + expect(copy.mock.lastCall?.[0]).toBe(source); +}); + +it("reuses an external VideoFrame until presentation changes, then closes it on replacement and clear", async () => { + const { Device, imported, constructed } = surfaces(); + const { session, video, canvas, show } = await presented(); + patchExternalVideoTextureCompat(); + const once = Device.prototype.importExternalTexture; + patchExternalVideoTextureCompat(); + expect(Device.prototype.importExternalTexture).toBe(once); + const device = new Device(); + const descriptor = { source: video, colorSpace: "srgb", label: "authored texture" }; + const texture = device.importExternalTexture(descriptor); + const first = imported.mock.lastCall?.[0] as { + source: { close: ReturnType }; + colorSpace: string; + label: string; + }; + expect(texture).toEqual({ texture: true }); + expect(constructed).toHaveBeenCalledExactlyOnceWith(canvas, { timestamp: 3_000_000 }); + expect(first).toEqual({ ...descriptor, source: first.source }); + expect(first.source).not.toBe(video); + expect(imported.mock.contexts[0]).toBe(device); + expect(descriptor.source).toBe(video); + device.importExternalTexture(descriptor); + expect(imported.mock.lastCall?.[0]).toEqual(first); + expect(constructed).toHaveBeenCalledTimes(1); + expect(first.source.close).not.toHaveBeenCalled(); + await show(4); + expect(first.source.close).toHaveBeenCalledTimes(1); + device.importExternalTexture(descriptor); + const second = imported.mock.lastCall?.[0] as typeof first; + expect(second.source).not.toBe(first.source); + expect(constructed).toHaveBeenLastCalledWith(getPausedVideoFrame(video), { + timestamp: 4_000_000, + }); + session.clear(); + expect(second.source.close).toHaveBeenCalledTimes(1); + device.importExternalTexture(descriptor); + expect(imported).toHaveBeenLastCalledWith(descriptor); + expect(imported.mock.lastCall?.[0]).toBe(descriptor); + expect(constructed).toHaveBeenCalledTimes(2); +}); diff --git a/packages/core/src/runtime/adapters/video-texture-compat.ts b/packages/core/src/runtime/adapters/video-texture-compat.ts index 785d648066..4844b841a7 100644 --- a/packages/core/src/runtime/adapters/video-texture-compat.ts +++ b/packages/core/src/runtime/adapters/video-texture-compat.ts @@ -15,6 +15,7 @@ import { findInjectedRenderFrame } from "../renderFrameSibling.js"; import { isImageElement, isVideoElement } from "../domRealm"; +import { getPausedExternalVideoFrame, getPausedVideoFrame } from "../pausedVideoFrames"; /** * Resolve the decoded render-frame `` for a source `