From b87a01f6a851b0ca02bfad3acfd0c56398024267 Mon Sep 17 00:00:00 2001 From: James Russo Date: Sat, 7 Nov 2026 23:29:21 +0000 Subject: [PATCH] fix(core): retain complete preview pictures during native seeks (#7378) * fix(core): retain complete preview pictures during native seeks * fix: settle failed seeks and preserve coherent progress --- docs/packages/player.mdx | 6 + .../core/src/runtime/colorGrading.test.ts | 21 ++ packages/core/src/runtime/colorGrading.ts | 2 + packages/core/src/runtime/entry.test.ts | 31 ++ .../core/src/runtime/init.previewSeek.test.ts | 283 ++++++++++++++++++ .../core/src/runtime/init.swapScenes.test.ts | 39 ++- packages/core/src/runtime/init.ts | 126 ++++++-- packages/core/src/runtime/previewSeek.ts | 198 ++++++++++++ packages/core/src/studioPreviewMark.ts | 1 + 9 files changed, 689 insertions(+), 18 deletions(-) create mode 100644 packages/core/src/runtime/init.previewSeek.test.ts create mode 100644 packages/core/src/runtime/previewSeek.ts diff --git a/docs/packages/player.mdx b/docs/packages/player.mdx index 93815d401f..b5af8c08bc 100644 --- a/docs/packages/player.mdx +++ b/docs/packages/player.mdx @@ -100,6 +100,12 @@ console.log(player.duration, player.paused, player.ready); console.log(player.compositionWidth, player.compositionHeight); ``` +In Studio previews, scrubbing keeps the last complete picture visible while video +layers decode at the requested time. Rapid seeks keep only the latest queued +destination, and the captions and video layers appear together once it is ready. +Complete intermediate pictures can appear during a drag. Jumping to a different +clip supersedes an earlier pending seek; failed sources do not hold up the preview. + ## Events ```js diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.ts index 6d9348ff23..6a59ba09b5 100644 --- a/packages/core/src/runtime/colorGrading.test.ts +++ b/packages/core/src/runtime/colorGrading.test.ts @@ -1,6 +1,7 @@ // fallow-ignore-file code-duplication import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { mediaElementTime } from "./mediaClock"; +import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark"; import { HF_COLOR_GRADING_ATTR, HF_COLOR_GRADING_EFFECT_KEYS, @@ -352,6 +353,26 @@ describe("createColorGradingRuntime", () => { expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore); }); + it("keeps graded layers on the committed picture until a preview seek lands", () => { + const { video } = startRuntimeWithVideo(); + const drawsBefore = texImage2DCalls.length; + const canvas = document.querySelector("[data-hf-color-grading-canvas]"); + document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, ""); + try { + video.dispatchEvent(new Event("seeked")); + video.style.setProperty("--hf-color-grading-exposure", "1"); + runtime?.refresh(); + expect(runtime?.redraw()).toBe(0); + expect(texImage2DCalls).toHaveLength(drawsBefore); + expect(document.querySelector("[data-hf-color-grading-canvas]")).toBe(canvas); + expect(canvas?.isConnected).toBe(true); + } finally { + document.documentElement.removeAttribute(PREVIEW_SEEK_HOLD_ATTR); + } + expect(runtime?.redraw()).toBe(1); + expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore); + }); + it("does not redraw-loop on its own hide writes (opacity/visibility only)", async () => { const { video } = startRuntimeWithVideo(); await new Promise((resolve) => requestAnimationFrame(() => resolve(null))); diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.ts index bc8133395d..daa9170cd7 100644 --- a/packages/core/src/runtime/colorGrading.ts +++ b/packages/core/src/runtime/colorGrading.ts @@ -1,4 +1,5 @@ import { wallClearTimeout, wallSetTimeout } from "./wallClock"; +import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark"; import { HF_COLOR_GRADING_ADJUST_KEYS, HF_COLOR_GRADING_ANIMATABLE_PROPERTIES, @@ -3419,6 +3420,7 @@ type DrawableFrame = { }; function drawableFrame(entry: ColorGradingEntry): DrawableFrame | null { + if (document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)) return null; const source = getDrawableSource(entry.element); const pixels = source && readablePixels(source, entry); if (!source || !pixels) { diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts index 1cdb2c00da..29134b704e 100644 --- a/packages/core/src/runtime/entry.test.ts +++ b/packages/core/src/runtime/entry.test.ts @@ -316,6 +316,34 @@ describe("runtime entry", () => { await window.__hfWaitForSeekCompletion?.(); }); + it("resolves a mixed image/video seek only after both have decoded", async () => { + servePreview(); + const root = mountRoot(); + const current = timed(root, "div", "0"); + const later = timed(root, "div", "5"); + const image = later.appendChild(document.createElement("img")); + let decodeImage = () => {}; + image.decode = () => new Promise((resolve) => (decodeImage = resolve)); + const video = timed(later, "video", "5"); + video.src = "/movie.mp4"; + let ready = 1; + Object.defineProperty(video, "readyState", { configurable: true, get: () => ready }); + Object.defineProperty(video, "networkState", { configurable: true, value: 2 }); + await evaluateRuntime(); + let settled = false; + const landed = Promise.resolve(window.__player!.seek(5.5)).then(() => { + settled = true; + }); + decodeImage(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(settled).toBe(false); + expect(visibility(current, later)).toEqual(["visible", "hidden"]); + ready = 4; + video.dispatchEvent(new Event("seeked")); + await landed; + expect(visibility(current, later)).toEqual(["hidden", "visible"]); + }); + it("applies a held jump after the cap when an image never decodes", async () => { servePreview(); const root = mountRoot(); @@ -495,6 +523,9 @@ describe("runtime entry", () => { ]); window.__player?.seek(3.5); + const video = second![0]!; + Object.defineProperty(video, "readyState", { value: 4, configurable: true }); + video.dispatchEvent(new Event("seeked")); expect([first, second, third].map((take) => armed(...take))).toEqual([ [false, false], [true, true], diff --git a/packages/core/src/runtime/init.previewSeek.test.ts b/packages/core/src/runtime/init.previewSeek.test.ts new file mode 100644 index 0000000000..7f22243ba3 --- /dev/null +++ b/packages/core/src/runtime/init.previewSeek.test.ts @@ -0,0 +1,283 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { initSandboxRuntimeModular } from "./init"; +import { hideTimedClipsUntilFirstPass } from "./timedClipHide"; +import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers"; +import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark"; + +function decoder(video: HTMLVideoElement, initiallyReady: boolean) { + let time = 0; + let ready = initiallyReady ? 4 : 0; + let seeking = false; + const seeks: number[] = []; + Object.defineProperties(video, { + duration: { value: 10, configurable: true }, + videoWidth: { value: 64, configurable: true }, + videoHeight: { value: 36, configurable: true }, + readyState: { get: () => ready, configurable: true }, + networkState: { value: 2, configurable: true }, + paused: { value: true, configurable: true }, + seeking: { get: () => seeking, configurable: true }, + currentTime: { + get: () => time, + set: (next: number) => { + time = next; + seeks.push(next); + seeking = true; + ready = 1; + }, + configurable: true, + }, + }); + video.pause = vi.fn(); + video.play = vi.fn(() => Promise.resolve()); + video.load = vi.fn(); + return { + video, + seeks, + land: () => { + ready = 4; + seeking = false; + video.dispatchEvent(new Event("seeked")); + }, + reload: () => { + time = 0; + ready = 1; + seeking = false; + video.dispatchEvent(new Event("loadedmetadata")); + }, + }; +} + +function film(withImage = false) { + resetRuntimeFixtureDom(); + const meta = document.createElement("meta"); + meta.name = STUDIO_PREVIEW_MARK_META; + document.head.append(meta); + vi.spyOn(window, "requestAnimationFrame").mockReturnValue(0); + const drawImage = vi.fn(); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage, + } as unknown as CanvasRenderingContext2D); + document.body.innerHTML = + '
' + + [0, 5] + .flatMap((start) => + ["plate", "matte"].map( + (name) => + ``, + ), + ) + .join("") + + (withImage + ? '
' + : "") + + "
"; + const videos = [...document.querySelectorAll("video")].map((v) => + decoder(v, v.dataset.start === "0"), + ); + const main = createMockTimeline(10); + window.__timelines = { main }; + if (withImage) hideTimedClipsUntilFirstPass(); + initSandboxRuntimeModular(); + for (const video of videos.slice(0, 2)) video.land(); + return { main, old: videos.slice(0, 2), next: videos.slice(2), drawImage }; +} + +it("keeps the old picture and page until both destination videos have decoded", async () => { + const { main, old, next, drawImage } = film(); + const landed = window.__player!.seek(5.5); + expect(window.__player!.getTime()).toBe(5.5); + expect(main.time?.()).toBe(0); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true); + expect(drawImage).toHaveBeenCalledTimes(2); + expect(document.querySelectorAll("canvas")).toHaveLength(2); + expect(old.every(({ video }) => getComputedStyle(video).display === "none")).toBe(true); + next[0]!.land(); + expect(main.time?.()).toBe(0); + next[1]!.land(); + await landed; + expect(main.time?.()).toBe(5.5); + expect(document.querySelectorAll("canvas")).toHaveLength(0); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false); + expect(old.every(({ video }) => video.style.visibility === "hidden")).toBe(true); + expect(next.every(({ video }) => video.style.visibility === "visible")).toBe(true); +}); + +it("coalesces same-clip requests and publishes complete intermediate pictures", async () => { + const { main, next } = film(); + const landed = window.__player!.seek(5.5); + const initialSeeks = next.map((v) => v.seeks.length); + window.__player!.seek(6); + window.__player!.seek(7.5); + expect(next.map((v) => v.seeks.length)).toEqual(initialSeeks); + for (const video of next) video.land(); + expect(main.time?.()).toBe(5.5); + expect(next.every((v) => Math.abs(v.video.currentTime - 2.5) < 0.001)).toBe(true); + for (const video of next) video.land(); + await landed; + expect(main.time?.()).toBe(7.5); +}); + +it("supersedes a stuck run immediately when the destination uses different clips", async () => { + const { main, old } = film(); + const landed = window.__player!.seek(6); + window.__player!.seek(1); + expect(old.every(({ video }) => Math.abs(video.currentTime - 1) < 0.001)).toBe(true); + for (const video of old) video.land(); + await landed; + expect(main.time?.()).toBe(1); + expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0); +}); + +it("settles a failed source without waiting for the decoder cap", async () => { + const { main, next } = film(); + const landed = window.__player!.seek(6); + for (const { video } of next) { + Object.defineProperties(video, { + error: { value: null, configurable: true }, + networkState: { value: 3, configurable: true }, + }); + const source = document.createElement("source"); + video.append(source); + source.dispatchEvent(new Event("error")); + } + await landed; + expect(main.time?.()).toBe(6); + expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0); +}); + +it("keeps a pending video's complete-picture hold while waiting for a later image", async () => { + const { main, old, next } = film(true); + const image = document.querySelector("#later-image")!; + let decodeImage = () => {}; + const decode = vi.fn(() => new Promise((resolve) => (decodeImage = resolve))); + image.decode = decode; + window.__player!.seek(6); + const landed = window.__player!.seek(8.5); + expect(decode).toHaveBeenCalledOnce(); + expect(window.__player!.getTime()).toBe(8.5); + expect(main.time?.()).toBe(0); + expect(old.every(({ video }) => getComputedStyle(video).display === "none")).toBe(true); + expect(document.querySelectorAll("canvas")).toHaveLength(2); + Object.defineProperty(image, "complete", { value: true, configurable: true }); + decodeImage(); + await new Promise((resolve) => setTimeout(resolve, 0)); + for (const video of next) video.land(); + for (const video of next) video.land(); + await landed; + expect(main.time?.()).toBe(8.5); +}); + +it("holds overlapping same-clip seeks too, and repositions a reloaded source at the requested time", async () => { + const { main, old } = film(); + const landed = window.__player!.seek(2); + old[0]!.reload(); + expect(old[0]!.video.currentTime).toBeCloseTo(2, 3); + old[0]!.land(); + expect(main.time?.()).toBe(0); + old[1]!.land(); + await landed; + expect(main.time?.()).toBe(2); +}); + +it("holds a paused keepPlaying seek and normal seeks after a non-export renderSeek", async () => { + const { main, old, next } = film(); + window.__player!.renderSeek(0.2); + for (const video of old) video.land(); + const landed = window.__player!.seek(6, { keepPlaying: true }); + expect(main.time?.()).toBe(0.2); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true); + for (const video of next) video.land(); + await landed; + expect(main.time?.()).toBe(6); +}); + +it("does not pause, seek or cover a video leased by another preview control", async () => { + const { old, next } = film(); + const leased = old[0]!; + window.__hf!.leasePausedMedia!(leased.video); + Object.defineProperty(leased.video, "paused", { value: false, configurable: true }); + const seeks = leased.seeks.length; + const landed = window.__player!.seek(6); + expect(leased.video.pause).not.toHaveBeenCalled(); + expect(leased.seeks).toHaveLength(seeks); + expect(leased.video.hasAttribute("data-hf-seek-video")).toBe(false); + for (const video of next) video.land(); + await landed; + expect(leased.video.pause).not.toHaveBeenCalled(); +}); + +it("leaves host-controlled native media to the host", () => { + const { old } = film(); + window.dispatchEvent( + new MessageEvent("message", { + source: window, + data: { + source: "hf-parent", + type: "control", + action: "set-native-media-sync-disabled", + disabled: true, + }, + }), + ); + Object.defineProperty(old[0]!.video, "paused", { value: false, configurable: true }); + const seeks = old[0]!.seeks.length; + window.__player!.seek(2); + expect(old[0]!.seeks).toHaveLength(seeks); + expect(old[0]!.video.pause).not.toHaveBeenCalled(); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false); +}); + +it("leaves a grading canvas's hidden source in its layout during the hold", async () => { + const { old, next, drawImage } = film(); + old[0]!.video.setAttribute("data-hf-color-grading-source-hidden", "true"); + old[0]!.video.style.setProperty("opacity", "0", "important"); + const landed = window.__player!.seek(6); + expect(old[0]!.video.hasAttribute("data-hf-seek-video")).toBe(false); + expect(drawImage).toHaveBeenCalledTimes(1); + for (const video of next) video.land(); + await landed; +}); + +it.each(["play", "render", "teardown"])( + "cleans up a pending seek on %s and ignores late completions", + async (action) => { + const { main, next } = film(); + const player = window.__player!; + const landed = player.seek(6); + if (action === "play") player.play(); + else if (action === "render") player.renderSeek(2); + else window.__hfRuntimeTeardown?.(); + const time = main.time?.(); + for (const video of next) video.land(); + await landed; + expect(main.time?.()).toBe(time); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false); + expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0); + if (action === "play") expect(player.getTime()).toBeGreaterThanOrEqual(6); + if (action === "render") expect(time).toBe(2); + }, +); + +it("caps a stalled decoder without leaving the picture hidden", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const { main } = film(); + const landed = window.__player!.seek(6); + await vi.advanceTimersByTimeAsync(5000); + await landed; + expect(main.time?.()).toBe(6); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false); + expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0); +}); + +afterEach(() => { + window.__hfRuntimeTeardown?.(); + document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`)?.remove(); + for (const style of document.querySelectorAll( + "style[data-hf-skip-hidden-images], style[data-hf-hidden-clip-contents]", + )) + style.remove(); + vi.restoreAllMocks(); + vi.useRealTimers(); + resetRuntimeFixtureDom(); +}); diff --git a/packages/core/src/runtime/init.swapScenes.test.ts b/packages/core/src/runtime/init.swapScenes.test.ts index e628c98de5..47bb27a8d2 100644 --- a/packages/core/src/runtime/init.swapScenes.test.ts +++ b/packages/core/src/runtime/init.swapScenes.test.ts @@ -9,6 +9,7 @@ import { probeAndCacheElementVolume } from "./mediaVolumeEnvelope.js"; import { wrapScopedCompositionScript } from "../compiler/compositionScoping"; import { AFTER_FONTS_SCRIPT_TYPE } from "../compiler/scriptRuns"; import { parkFarScenesAtBoot } from "./sceneWindow"; +import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark"; vi.mock("./mediaVolumeEnvelope.js", async (importOriginal) => { const actual = await importOriginal(); @@ -1767,9 +1768,13 @@ describe("the preview's scene window", () => { el.getAttribute("data-hf-scene"), ); // As the runtime's entry does: park before any scene script runs, then run the rest. - function bootWindowed(scenes: Scene[], root: Tl, withManifest = true) { + function bootWindowed(scenes: Scene[], root: Tl, withManifest = true, studioPreview = false) { const { head, body } = preview(scenes); document.head.innerHTML = withManifest ? head : head.replace(/]*>/, ""); + if (studioPreview) + document.head.append( + Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }), + ); document.body.innerHTML = body; window.__timelines = { main: root }; parkFarScenesAtBoot(); @@ -1868,6 +1873,38 @@ describe("the preview's scene window", () => { expect(parked()).toEqual(["s2"]); }); + it("holds a parked destination until its videos mount and decode without retiring the old picture", async () => { + const ready = new WeakSet(); + vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockImplementation(function () { + return ready.has(this) ? 4 : 1; + }); + vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2); + vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {}); + vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {}); + const { root } = trackingRoot(); + bootWindowed( + [S0, S1, { ...S2, body: '' }], + root, + true, + true, + ); + await tick(); + const landed = window.__player!.seek(5); + await vi.waitFor(() => { + expect(sceneHost("s2").querySelector("video")?.currentTime).toBeCloseTo(1, 3); + }); + expect(root.time?.()).toBe(0); + expect(window.__player!.getTime()).toBe(5); + expect(parked()).not.toContain("s0"); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true); + const video = sceneHost("s2").querySelector("video")!; + ready.add(video); + video.dispatchEvent(new Event("seeked")); + await landed; + expect(root.time?.()).toBe(5); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false); + }); + it("waits for the scene a seek lands in while its mount as a neighbour is still running", async () => { bootWindowed([S0, S1, S2], trackingRoot().root); await tick(); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 614cb777a3..ad2e8b23a6 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -28,10 +28,11 @@ import { } from "./preloadMedia"; import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth"; import { hasFrozenPicture } from "./frozenPicture"; +import { createPreviewSeek } from "./previewSeek"; import { copiesReady } from "./previewCopies"; import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge"; import { instantTolerance } from "../clipFacts"; -import { isInClipWindow } from "./clipWindow"; +import { isClipVisibleAt, isInClipWindow } from "./clipWindow"; import { isWebKit, mediaElementTime } from "./mediaClock"; import { CLIP_HIDDEN_ATTR, @@ -528,6 +529,7 @@ function loadPreviewImagesIn(root: Element): void { export function initSandboxRuntimeModular(): void { const state = createRuntimeState(); + let previewSeek: ReturnType | undefined; const stampsForStudio = window.parent !== window; // Studio's discovery stamps time a clip for the timeline only; nothing hides it at their end. const isDiscoveryStamped = (el: Element) => @@ -2687,7 +2689,14 @@ export function initSandboxRuntimeModular(): void { }; const onMediaLoadedMetadataForPlayhead = (event: Event) => { const el = event.currentTarget; - if (state.tornDown || clock.isPlaying() || !isMediaElement(el) || isRenderMode(el)) return; + if ( + state.tornDown || + previewSeek?.time !== undefined || + clock.isPlaying() || + !isMediaElement(el) || + isRenderMode(el) + ) + return; lastSyncedMediaTimeSeconds = null; state.mediaForceSyncNextTick = true; syncMediaForCurrentState(); @@ -3775,6 +3784,7 @@ export function initSandboxRuntimeModular(): void { let lastVisibilityPassSeconds: number | null = null; const syncMediaForCurrentState = (timingRevision?: number) => { + if (previewSeek?.time !== undefined) return; // Scope 1 of 3 (see `withTimingResolver`). Closes before `syncRuntimeMedia`, // which may call `el.load()` and invalidate every cached duration. // @@ -3871,7 +3881,8 @@ export function initSandboxRuntimeModular(): void { // wake call bolted onto each mutator (and forgotten on the next one). if (force) wakeTransport(); if (force && timelinePostHeld && !clock.isPlaying()) postTimeline(); - const frame = Math.max(0, Math.round((state.currentTime || 0) * state.canonicalFps)); + const currentTime = previewSeek?.time ?? state.currentTime ?? 0; + const frame = Math.max(0, Math.round(currentTime * state.canonicalFps)); const now = Date.now(); const shouldPost = force || @@ -3888,7 +3899,7 @@ export function initSandboxRuntimeModular(): void { source: "hf-preview", type: "state", frame, - currentTime: state.currentTime || 0, + currentTime, ended: clock.reachedEnd(), isPlaying: state.bridgeLastPostedPlaying, buffering: bufferingFor.length > 0, @@ -4482,8 +4493,9 @@ export function initSandboxRuntimeModular(): void { // Redraw the current frame as a seek does; a playing film cut to end at or before it stops there, as at its end. const cutShort = clock.isPlaying() && duration > 0 && state.currentTime >= duration; if (joining && clock.isPlaying() && !cutShort) joinPlayback(swappedHosts); + else if (previewSeek?.time !== undefined) previewSeek.refresh(); else transport.seek(cutShort ? duration : state.currentTime, { keepPlaying: !cutShort }); - syncTimedElementVisibility(state.currentTime); + if (previewSeek?.time === undefined) syncTimedElementVisibility(state.currentTime); postTimeline(); }; // Only the rules not here yet: rewriting a loaded rule's style would load its font again. @@ -4790,16 +4802,19 @@ export function initSandboxRuntimeModular(): void { .filter(({ name }) => near.has(name) && parked(name)) .sort((a, b) => Math.abs(a.start - time) - Math.abs(b.start - time))[0]; const toMount = landed ?? neighbour; - const toRetire = toMount - ? [] - : windowScenes.filter(({ name }) => !near.has(name) && !parked(name)); + const toRetire = + toMount || previewSeek?.time !== undefined + ? [] + : windowScenes.filter(({ name }) => !near.has(name) && !parked(name)); if (!toMount && toRetire.length === 0) { windowNear = key; return false; } const wantedNow = (name: string) => heldScenes.has(name) || - scenesWanted(windowScenes, state.currentTime, clock.isPlaying()).has(name); + scenesWanted(windowScenes, previewSeek?.time ?? state.currentTime, clock.isPlaying()).has( + name, + ); const parkedBeforeStep = parkedNames.size; windowWork = (async () => { if (landed) await mountScene(landed.name); @@ -4812,7 +4827,7 @@ export function initSandboxRuntimeModular(): void { windowWork = null; const stuckStepKeepsItsPlan = parkedNames.size === parkedBeforeStep; windowNear = stuckStepKeepsItsPlan ? key : ""; - moveSceneWindow(state.currentTime); + moveSceneWindow(previewSeek?.time ?? state.currentTime); }); // A seek is not done until the scene it lands in is mounted. if (landed) registerSeekCompletion(windowWork, "scene-window"); @@ -4898,11 +4913,11 @@ export function initSandboxRuntimeModular(): void { // A paused seek that would reveal undecoded images holds the previous picture until they decode. // Capped: a request that never settles must not freeze the preview; measured jumps settle in ~200 ms. const SEEK_HOLD_CAP_MS = 1000; - let heldSeek: { time: number; apply: () => void } | null = null; + let heldSeek: { time: number; apply: () => HeldSeek } | null = null; const flushHeldSeek = () => { const held = heldSeek; heldSeek = null; - held?.apply(); + return held?.apply(); }; const applySeek = (quantized: number, options?: { keepPlaying?: boolean }) => { webAudio.stopAll(); @@ -4953,9 +4968,80 @@ export function initSandboxRuntimeModular(): void { postState(true); }; + previewSeek = createPreviewSeek({ + videosToHold: () => + withTimingResolver(() => + (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []) + .map(({ el }) => el) + .filter(isVideoElement), + ), + scenesReadyAt: (time) => + !windowScenes.some( + ({ name, start, end }) => start <= time && time <= end && parkedNames.has(name), + ), + clipsAt: (time) => + withTimingResolver(() => { + const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0); + const visibleAt = timedVisibilityAt(); + const root = resolveRootCompositionElement(); + return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter( + (clip) => + isVideoElement(clip.el) && + !isSilencedByHidden(clip.el) && + (clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) && + isClipVisibleAt(time, clip.start, clip.end, duration) && + clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root), + ); + }), + prepare: (clips, time) => { + moveSceneWindow(time); + syncRuntimeMedia({ + clips, + timeSeconds: time, + playing: false, + playbackRate: state.playbackRate, + forceSync: true, + getCompositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0), + }); + }, + commit: (time) => applySeek(time), + }); + + const seekWhenReady = (time: number, options?: { keepPlaying?: boolean }) => { + if ( + !mediaDeferred || + state.nativeMediaSyncDisabled || + isExportRenderDrivingFrames() || + window.__HF_EXPORT_RENDER_SEEK_CONFIG || + (options?.keepPlaying && clock.isPlaying()) + ) { + previewSeek?.cancel(); + applySeek(time, options); + return; + } + webAudio.stopAll(); + clock.detachAudioSource(); + clock.pause(); + state.isPlaying = false; + bufferingFor = []; + pauseTimelineIfPossible(state.capturedTimeline); + runAdapters("pause"); + const clips = withTimingResolver(() => buildRuntimeMediaCache().mediaClips); + for (const { el } of transportMovableClips(clips) ?? []) { + if (!el.paused) el.pause(); + } + const landed = previewSeek?.seek(time); + if (landed) registerSeekCompletion(landed, "video-frame"); + postState(true); + return landed; + }; + const transport: RuntimePlayerTransport = { play: () => { flushHeldSeek(); + const destination = previewSeek?.time; + previewSeek?.cancel(); + if (destination !== undefined) applySeek(destination); const resumingStarved = bufferingFor.length > 0; bufferingFor = []; const tl = state.capturedTimeline; @@ -5003,10 +5089,9 @@ export function initSandboxRuntimeModular(): void { heldSeek = null; const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized); if (undecoded.length === 0) { - applySeek(quantized, options); - return; + return seekWhenReady(quantized, options); } - const held = { time: quantized, apply: () => applySeek(quantized, options) }; + const held = { time: quantized, apply: () => seekWhenReady(quantized, options) }; heldSeek = held; for (const img of undecoded) { loadPreviewImagesIn(img); @@ -5025,13 +5110,14 @@ export function initSandboxRuntimeModular(): void { ); const landed = Promise.race([decoded, capped]).then(() => { wallClearTimeout(capTimer); - if (heldSeek === held) flushHeldSeek(); + if (heldSeek === held) return flushHeldSeek(); }); registerSeekCompletion(landed, "image-decode"); return landed; }, renderSeek: (timeSeconds, options) => { heldSeek = null; + previewSeek?.cancel(); renderCaptureSeekStarted = true; const seekTime = resolveRenderSeekTime( Math.max(0, Number(timeSeconds) || 0), @@ -5056,7 +5142,7 @@ export function initSandboxRuntimeModular(): void { if (blurred) registerSeekCompletion(blurred, "motion-blur"); postState(true); }, - getTime: () => heldSeek?.time ?? clock.now(), + getTime: () => heldSeek?.time ?? previewSeek?.time ?? clock.now(), getDuration: () => { const dur = clock.getDuration(); return Number.isFinite(dur) ? dur : 0; @@ -5891,6 +5977,10 @@ export function initSandboxRuntimeModular(): void { if (state.tornDown || inTransportTick) return; inTransportTick = true; try { + if (previewSeek?.time !== undefined) { + postState(false); + return; + } transportTickCount += 1; // The jobs below run when the composition revision moves (rate-limited). What no @@ -6298,6 +6388,7 @@ export function initSandboxRuntimeModular(): void { emitAnalyticsEvent("composition_paused", { time: player.getTime() }); }, onStopMedia: () => { + previewSeek?.cancel(); webAudio.stopAll(); const mediaEls = document.querySelectorAll("video, audio"); for (const el of mediaEls) { @@ -6451,6 +6542,7 @@ export function initSandboxRuntimeModular(): void { const teardown = () => { if (state.tornDown) return; state.tornDown = true; + previewSeek?.cancel(); cancelTransportTick(); document.removeEventListener("visibilitychange", onTransportVisibilityChange); if (transportParkTimerId != null) { diff --git a/packages/core/src/runtime/previewSeek.ts b/packages/core/src/runtime/previewSeek.ts new file mode 100644 index 0000000000..f56f5944c1 --- /dev/null +++ b/packages/core/src/runtime/previewSeek.ts @@ -0,0 +1,198 @@ +import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark"; +import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading"; +import { isUnplayable, mediaSourceTimeAt } from "./media"; +import type { RuntimeMediaClip } from "./types"; +import { wallClearTimeout, wallSetTimeout } from "./wallClock"; + +const HELD_VIDEO_ATTR = "data-hf-seek-video"; +const SEEK_CAP_MS = 5000; +const EVENTS = ["loadedmetadata", "loadeddata", "canplay", "seeked", "error"] as const; + +function videoFrame(video: HTMLVideoElement): HTMLCanvasElement | null { + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const context = canvas.getContext("2d"); + if (!context) return null; + try { + context.drawImage(video, 0, 0); + return canvas; + } catch { + return null; + } +} + +/** Keep a displayed video in its layout slot while its decoder moves independently of the page. */ +function holdVideoFrame(video: HTMLVideoElement): (() => void) | undefined { + // Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh. + if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return; + const css = getComputedStyle(video); + if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return; + if (!video.videoWidth || !video.videoHeight) return; + const canvas = videoFrame(video); + if (!canvas) return; + for (const name of Array.from(css)) canvas.style.setProperty(name, css.getPropertyValue(name)); + canvas.style.setProperty("animation", "none", "important"); + canvas.style.setProperty("transition", "none", "important"); + canvas.style.setProperty("pointer-events", "none", "important"); + canvas.setAttribute("data-layout-ignore", ""); + canvas.setAttribute("aria-hidden", "true"); + video.before(canvas); + video.setAttribute(HELD_VIDEO_ATTR, ""); + return () => { + video.removeAttribute(HELD_VIDEO_ATTR); + canvas.remove(); + }; +} + +function holdVideoFrames(videos: HTMLVideoElement[]): () => void { + const restore = videos.map(holdVideoFrame); + const style = document.createElement("style"); + style.textContent = `[${HELD_VIDEO_ATTR}] { display: none !important; }`; + document.head.appendChild(style); + document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, ""); + return () => { + for (const putBack of restore) putBack?.(); + style.remove(); + document.documentElement.removeAttribute(PREVIEW_SEEK_HOLD_ATTR); + }; +} + +type SeekHost = { + clipsAt: (time: number) => RuntimeMediaClip[]; + videosToHold: () => HTMLVideoElement[]; + scenesReadyAt: (time: number) => boolean; + prepare: (clips: RuntimeMediaClip[], time: number) => void; + commit: (time: number) => void; +}; + +/** One decoder job at a time; requests arriving during it replace the queued destination. */ +export function createPreviewSeek(host: SeekHost) { + let pending: { time: number; done: Promise; resolve: () => void } | null = null; + let releasePicture = () => {}; + let stopWaiting = () => {}; + let runningClips: RuntimeMediaClip[] = []; + let runId = 0; + + const ready = (clips: RuntimeMediaClip[], time: number) => + host.scenesReadyAt(time) && + clips.every((clip) => { + const { el } = clip; + return ( + !el.isConnected || + isUnplayable(el) || + (el.readyState >= 2 && + !el.seeking && + Math.abs(el.currentTime - mediaSourceTimeAt(clip, time)) < 0.02) + ); + }); + + const cancel = () => { + const request = pending; + pending = null; + runId++; + stopWaiting(); + stopWaiting = () => {}; + releasePicture(); + releasePicture = () => {}; + request?.resolve(); + }; + + const run = () => { + const request = pending; + if (!request) return; + const id = ++runId; + const time = request.time; + const clips = host.clipsAt(time); + runningClips = clips; + let preparing = true; + let finished = false; + const finish = () => { + if (finished || pending !== request || id !== runId) return; + finished = true; + stopWaiting(); + stopWaiting = () => {}; + if (request.time !== time) { + if (ready(host.clipsAt(time), time)) { + releasePicture(); + host.commit(time); + releasePicture = holdVideoFrames(host.videosToHold()); + } + run(); + return; + } + cancel(); + host.commit(time); + }; + const check = () => { + if (!preparing && ready(host.clipsAt(time), time)) finish(); + }; + const metadata = () => { + if (pending !== request || finished || id !== runId) return; + // A source/proxy change resets currentTime; the parked clock still describes the old picture. + preparing = true; + host.prepare(host.clipsAt(time), time); + preparing = false; + check(); + }; + const listeners = clips.map(({ el }) => { + for (const type of EVENTS) + el.addEventListener(type, type === "loadedmetadata" ? metadata : check, type === "error"); + return () => { + for (const type of EVENTS) + el.removeEventListener( + type, + type === "loadedmetadata" ? metadata : check, + type === "error", + ); + }; + }); + const cap = wallSetTimeout(finish, SEEK_CAP_MS); + stopWaiting = () => { + wallClearTimeout(cap); + for (const remove of listeners) remove(); + }; + host.prepare(clips, time); + preparing = false; + check(); + }; + + return { + get time() { + return pending?.time; + }, + seek(time: number): Promise | void { + if (pending) { + const request = pending; + request.time = time; + const clips = host.clipsAt(time); + if ( + clips.length !== runningClips.length || + clips.some((clip, index) => clip.el !== runningClips[index]?.el) + ) { + stopWaiting(); + run(); + } + return request.done; + } + if (ready(host.clipsAt(time), time)) { + host.commit(time); + return; + } + let resolve = () => {}; + const done = new Promise((settle) => { + resolve = settle; + }); + pending = { time, done, resolve }; + releasePicture = holdVideoFrames(host.videosToHold()); + run(); + return done; + }, + refresh() { + if (!pending) return; + stopWaiting(); + run(); + }, + cancel, + }; +} diff --git a/packages/core/src/studioPreviewMark.ts b/packages/core/src/studioPreviewMark.ts index f291aa83fa..94c57ef617 100644 --- a/packages/core/src/studioPreviewMark.ts +++ b/packages/core/src/studioPreviewMark.ts @@ -1,6 +1,7 @@ import { isWebKit } from "./runtime/mediaClock.js"; export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview"; +export const PREVIEW_SEEK_HOLD_ATTR = "data-hf-preview-seek-hold"; export const STUDIO_PREVIEW_CAPTURE_MODE = "capture"; export const PLAYER_FRAME_ATTR = "data-hf-player-frame";