fix(core): retain complete preview pictures during native seeks (#7378)

* fix(core): retain complete preview pictures during native seeks

* fix: settle failed seeks and preserve coherent progress
This commit is contained in:
James Russo
2026-11-07 23:29:21 +00:00
committed by GitHub
parent 34118d8f20
commit b87a01f6a8
9 changed files with 689 additions and 18 deletions
+6
View File
@@ -100,6 +100,12 @@ console.log(player.duration, player.paused, player.ready);
console.log(player.compositionWidth, player.compositionHeight);
```
In Studio previews, scrubbing keeps the last complete picture visible while video
layers decode at the requested time. Rapid seeks keep only the latest queued
destination, and the captions and video layers appear together once it is ready.
Complete intermediate pictures can appear during a drag. Jumping to a different
clip supersedes an earlier pending seek; failed sources do not hold up the preview.
## Events
```js
@@ -1,6 +1,7 @@
// fallow-ignore-file code-duplication
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mediaElementTime } from "./mediaClock";
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
import {
HF_COLOR_GRADING_ATTR,
HF_COLOR_GRADING_EFFECT_KEYS,
@@ -352,6 +353,26 @@ describe("createColorGradingRuntime", () => {
expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
});
it("keeps graded layers on the committed picture until a preview seek lands", () => {
const { video } = startRuntimeWithVideo();
const drawsBefore = texImage2DCalls.length;
const canvas = document.querySelector("[data-hf-color-grading-canvas]");
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
try {
video.dispatchEvent(new Event("seeked"));
video.style.setProperty("--hf-color-grading-exposure", "1");
runtime?.refresh();
expect(runtime?.redraw()).toBe(0);
expect(texImage2DCalls).toHaveLength(drawsBefore);
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBe(canvas);
expect(canvas?.isConnected).toBe(true);
} finally {
document.documentElement.removeAttribute(PREVIEW_SEEK_HOLD_ATTR);
}
expect(runtime?.redraw()).toBe(1);
expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
});
it("does not redraw-loop on its own hide writes (opacity/visibility only)", async () => {
const { video } = startRuntimeWithVideo();
await new Promise((resolve) => requestAnimationFrame(() => resolve(null)));
@@ -1,4 +1,5 @@
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
import {
HF_COLOR_GRADING_ADJUST_KEYS,
HF_COLOR_GRADING_ANIMATABLE_PROPERTIES,
@@ -3419,6 +3420,7 @@ type DrawableFrame = {
};
function drawableFrame(entry: ColorGradingEntry): DrawableFrame | null {
if (document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)) return null;
const source = getDrawableSource(entry.element);
const pixels = source && readablePixels(source, entry);
if (!source || !pixels) {
+31
View File
@@ -316,6 +316,34 @@ describe("runtime entry", () => {
await window.__hfWaitForSeekCompletion?.();
});
it("resolves a mixed image/video seek only after both have decoded", async () => {
servePreview();
const root = mountRoot();
const current = timed(root, "div", "0");
const later = timed(root, "div", "5");
const image = later.appendChild(document.createElement("img"));
let decodeImage = () => {};
image.decode = () => new Promise<void>((resolve) => (decodeImage = resolve));
const video = timed(later, "video", "5");
video.src = "/movie.mp4";
let ready = 1;
Object.defineProperty(video, "readyState", { configurable: true, get: () => ready });
Object.defineProperty(video, "networkState", { configurable: true, value: 2 });
await evaluateRuntime();
let settled = false;
const landed = Promise.resolve(window.__player!.seek(5.5)).then(() => {
settled = true;
});
decodeImage();
await new Promise((resolve) => setTimeout(resolve, 0));
expect(settled).toBe(false);
expect(visibility(current, later)).toEqual(["visible", "hidden"]);
ready = 4;
video.dispatchEvent(new Event("seeked"));
await landed;
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
});
it("applies a held jump after the cap when an image never decodes", async () => {
servePreview();
const root = mountRoot();
@@ -495,6 +523,9 @@ describe("runtime entry", () => {
]);
window.__player?.seek(3.5);
const video = second![0]!;
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
video.dispatchEvent(new Event("seeked"));
expect([first, second, third].map((take) => armed(...take))).toEqual([
[false, false],
[true, true],
@@ -0,0 +1,283 @@
import { afterEach, expect, it, vi } from "vitest";
import { initSandboxRuntimeModular } from "./init";
import { hideTimedClipsUntilFirstPass } from "./timedClipHide";
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
let time = 0;
let ready = initiallyReady ? 4 : 0;
let seeking = false;
const seeks: number[] = [];
Object.defineProperties(video, {
duration: { value: 10, configurable: true },
videoWidth: { value: 64, configurable: true },
videoHeight: { value: 36, configurable: true },
readyState: { get: () => ready, configurable: true },
networkState: { value: 2, configurable: true },
paused: { value: true, configurable: true },
seeking: { get: () => seeking, configurable: true },
currentTime: {
get: () => time,
set: (next: number) => {
time = next;
seeks.push(next);
seeking = true;
ready = 1;
},
configurable: true,
},
});
video.pause = vi.fn();
video.play = vi.fn(() => Promise.resolve());
video.load = vi.fn();
return {
video,
seeks,
land: () => {
ready = 4;
seeking = false;
video.dispatchEvent(new Event("seeked"));
},
reload: () => {
time = 0;
ready = 1;
seeking = false;
video.dispatchEvent(new Event("loadedmetadata"));
},
};
}
function film(withImage = false) {
resetRuntimeFixtureDom();
const meta = document.createElement("meta");
meta.name = STUDIO_PREVIEW_MARK_META;
document.head.append(meta);
vi.spyOn(window, "requestAnimationFrame").mockReturnValue(0);
const drawImage = vi.fn();
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
drawImage,
} as unknown as CanvasRenderingContext2D);
document.body.innerHTML =
'<div data-composition-id="main" data-root="true" data-duration="10">' +
[0, 5]
.flatMap((start) =>
["plate", "matte"].map(
(name) =>
`<video id="${name}-${start}" style="width:64px;height:36px" data-start="${start}" data-duration="5" src="/${name}.mp4" muted></video>`,
),
)
.join("") +
(withImage
? '<div data-start="8" data-duration="2"><img id="later-image" src="/later.png"></div>'
: "") +
"</div>";
const videos = [...document.querySelectorAll("video")].map((v) =>
decoder(v, v.dataset.start === "0"),
);
const main = createMockTimeline(10);
window.__timelines = { main };
if (withImage) hideTimedClipsUntilFirstPass();
initSandboxRuntimeModular();
for (const video of videos.slice(0, 2)) video.land();
return { main, old: videos.slice(0, 2), next: videos.slice(2), drawImage };
}
it("keeps the old picture and page until both destination videos have decoded", async () => {
const { main, old, next, drawImage } = film();
const landed = window.__player!.seek(5.5);
expect(window.__player!.getTime()).toBe(5.5);
expect(main.time?.()).toBe(0);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
expect(drawImage).toHaveBeenCalledTimes(2);
expect(document.querySelectorAll("canvas")).toHaveLength(2);
expect(old.every(({ video }) => getComputedStyle(video).display === "none")).toBe(true);
next[0]!.land();
expect(main.time?.()).toBe(0);
next[1]!.land();
await landed;
expect(main.time?.()).toBe(5.5);
expect(document.querySelectorAll("canvas")).toHaveLength(0);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
expect(old.every(({ video }) => video.style.visibility === "hidden")).toBe(true);
expect(next.every(({ video }) => video.style.visibility === "visible")).toBe(true);
});
it("coalesces same-clip requests and publishes complete intermediate pictures", async () => {
const { main, next } = film();
const landed = window.__player!.seek(5.5);
const initialSeeks = next.map((v) => v.seeks.length);
window.__player!.seek(6);
window.__player!.seek(7.5);
expect(next.map((v) => v.seeks.length)).toEqual(initialSeeks);
for (const video of next) video.land();
expect(main.time?.()).toBe(5.5);
expect(next.every((v) => Math.abs(v.video.currentTime - 2.5) < 0.001)).toBe(true);
for (const video of next) video.land();
await landed;
expect(main.time?.()).toBe(7.5);
});
it("supersedes a stuck run immediately when the destination uses different clips", async () => {
const { main, old } = film();
const landed = window.__player!.seek(6);
window.__player!.seek(1);
expect(old.every(({ video }) => Math.abs(video.currentTime - 1) < 0.001)).toBe(true);
for (const video of old) video.land();
await landed;
expect(main.time?.()).toBe(1);
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
});
it("settles a failed source without waiting for the decoder cap", async () => {
const { main, next } = film();
const landed = window.__player!.seek(6);
for (const { video } of next) {
Object.defineProperties(video, {
error: { value: null, configurable: true },
networkState: { value: 3, configurable: true },
});
const source = document.createElement("source");
video.append(source);
source.dispatchEvent(new Event("error"));
}
await landed;
expect(main.time?.()).toBe(6);
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
});
it("keeps a pending video's complete-picture hold while waiting for a later image", async () => {
const { main, old, next } = film(true);
const image = document.querySelector<HTMLImageElement>("#later-image")!;
let decodeImage = () => {};
const decode = vi.fn(() => new Promise<void>((resolve) => (decodeImage = resolve)));
image.decode = decode;
window.__player!.seek(6);
const landed = window.__player!.seek(8.5);
expect(decode).toHaveBeenCalledOnce();
expect(window.__player!.getTime()).toBe(8.5);
expect(main.time?.()).toBe(0);
expect(old.every(({ video }) => getComputedStyle(video).display === "none")).toBe(true);
expect(document.querySelectorAll("canvas")).toHaveLength(2);
Object.defineProperty(image, "complete", { value: true, configurable: true });
decodeImage();
await new Promise((resolve) => setTimeout(resolve, 0));
for (const video of next) video.land();
for (const video of next) video.land();
await landed;
expect(main.time?.()).toBe(8.5);
});
it("holds overlapping same-clip seeks too, and repositions a reloaded source at the requested time", async () => {
const { main, old } = film();
const landed = window.__player!.seek(2);
old[0]!.reload();
expect(old[0]!.video.currentTime).toBeCloseTo(2, 3);
old[0]!.land();
expect(main.time?.()).toBe(0);
old[1]!.land();
await landed;
expect(main.time?.()).toBe(2);
});
it("holds a paused keepPlaying seek and normal seeks after a non-export renderSeek", async () => {
const { main, old, next } = film();
window.__player!.renderSeek(0.2);
for (const video of old) video.land();
const landed = window.__player!.seek(6, { keepPlaying: true });
expect(main.time?.()).toBe(0.2);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
for (const video of next) video.land();
await landed;
expect(main.time?.()).toBe(6);
});
it("does not pause, seek or cover a video leased by another preview control", async () => {
const { old, next } = film();
const leased = old[0]!;
window.__hf!.leasePausedMedia!(leased.video);
Object.defineProperty(leased.video, "paused", { value: false, configurable: true });
const seeks = leased.seeks.length;
const landed = window.__player!.seek(6);
expect(leased.video.pause).not.toHaveBeenCalled();
expect(leased.seeks).toHaveLength(seeks);
expect(leased.video.hasAttribute("data-hf-seek-video")).toBe(false);
for (const video of next) video.land();
await landed;
expect(leased.video.pause).not.toHaveBeenCalled();
});
it("leaves host-controlled native media to the host", () => {
const { old } = film();
window.dispatchEvent(
new MessageEvent("message", {
source: window,
data: {
source: "hf-parent",
type: "control",
action: "set-native-media-sync-disabled",
disabled: true,
},
}),
);
Object.defineProperty(old[0]!.video, "paused", { value: false, configurable: true });
const seeks = old[0]!.seeks.length;
window.__player!.seek(2);
expect(old[0]!.seeks).toHaveLength(seeks);
expect(old[0]!.video.pause).not.toHaveBeenCalled();
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
});
it("leaves a grading canvas's hidden source in its layout during the hold", async () => {
const { old, next, drawImage } = film();
old[0]!.video.setAttribute("data-hf-color-grading-source-hidden", "true");
old[0]!.video.style.setProperty("opacity", "0", "important");
const landed = window.__player!.seek(6);
expect(old[0]!.video.hasAttribute("data-hf-seek-video")).toBe(false);
expect(drawImage).toHaveBeenCalledTimes(1);
for (const video of next) video.land();
await landed;
});
it.each(["play", "render", "teardown"])(
"cleans up a pending seek on %s and ignores late completions",
async (action) => {
const { main, next } = film();
const player = window.__player!;
const landed = player.seek(6);
if (action === "play") player.play();
else if (action === "render") player.renderSeek(2);
else window.__hfRuntimeTeardown?.();
const time = main.time?.();
for (const video of next) video.land();
await landed;
expect(main.time?.()).toBe(time);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
if (action === "play") expect(player.getTime()).toBeGreaterThanOrEqual(6);
if (action === "render") expect(time).toBe(2);
},
);
it("caps a stalled decoder without leaving the picture hidden", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const { main } = film();
const landed = window.__player!.seek(6);
await vi.advanceTimersByTimeAsync(5000);
await landed;
expect(main.time?.()).toBe(6);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
});
afterEach(() => {
window.__hfRuntimeTeardown?.();
document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`)?.remove();
for (const style of document.querySelectorAll(
"style[data-hf-skip-hidden-images], style[data-hf-hidden-clip-contents]",
))
style.remove();
vi.restoreAllMocks();
vi.useRealTimers();
resetRuntimeFixtureDom();
});
@@ -9,6 +9,7 @@ import { probeAndCacheElementVolume } from "./mediaVolumeEnvelope.js";
import { wrapScopedCompositionScript } from "../compiler/compositionScoping";
import { AFTER_FONTS_SCRIPT_TYPE } from "../compiler/scriptRuns";
import { parkFarScenesAtBoot } from "./sceneWindow";
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
vi.mock("./mediaVolumeEnvelope.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("./mediaVolumeEnvelope.js")>();
@@ -1767,9 +1768,13 @@ describe("the preview's scene window", () => {
el.getAttribute("data-hf-scene"),
);
// As the runtime's entry does: park before any scene script runs, then run the rest.
function bootWindowed(scenes: Scene[], root: Tl, withManifest = true) {
function bootWindowed(scenes: Scene[], root: Tl, withManifest = true, studioPreview = false) {
const { head, body } = preview(scenes);
document.head.innerHTML = withManifest ? head : head.replace(/<meta[^>]*>/, "");
if (studioPreview)
document.head.append(
Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
);
document.body.innerHTML = body;
window.__timelines = { main: root };
parkFarScenesAtBoot();
@@ -1868,6 +1873,38 @@ describe("the preview's scene window", () => {
expect(parked()).toEqual(["s2"]);
});
it("holds a parked destination until its videos mount and decode without retiring the old picture", async () => {
const ready = new WeakSet<HTMLMediaElement>();
vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockImplementation(function () {
return ready.has(this) ? 4 : 1;
});
vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
const { root } = trackingRoot();
bootWindowed(
[S0, S1, { ...S2, body: '<video src="/late.mp4" data-start="0" data-duration="2"></video>' }],
root,
true,
true,
);
await tick();
const landed = window.__player!.seek(5);
await vi.waitFor(() => {
expect(sceneHost("s2").querySelector("video")?.currentTime).toBeCloseTo(1, 3);
});
expect(root.time?.()).toBe(0);
expect(window.__player!.getTime()).toBe(5);
expect(parked()).not.toContain("s0");
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
const video = sceneHost("s2").querySelector("video")!;
ready.add(video);
video.dispatchEvent(new Event("seeked"));
await landed;
expect(root.time?.()).toBe(5);
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
});
it("waits for the scene a seek lands in while its mount as a neighbour is still running", async () => {
bootWindowed([S0, S1, S2], trackingRoot().root);
await tick();
+109 -17
View File
@@ -28,10 +28,11 @@ import {
} from "./preloadMedia";
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
import { hasFrozenPicture } from "./frozenPicture";
import { createPreviewSeek } from "./previewSeek";
import { copiesReady } from "./previewCopies";
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
import { instantTolerance } from "../clipFacts";
import { isInClipWindow } from "./clipWindow";
import { isClipVisibleAt, isInClipWindow } from "./clipWindow";
import { isWebKit, mediaElementTime } from "./mediaClock";
import {
CLIP_HIDDEN_ATTR,
@@ -528,6 +529,7 @@ function loadPreviewImagesIn(root: Element): void {
export function initSandboxRuntimeModular(): void {
const state = createRuntimeState();
let previewSeek: ReturnType<typeof createPreviewSeek> | undefined;
const stampsForStudio = window.parent !== window;
// Studio's discovery stamps time a clip for the timeline only; nothing hides it at their end.
const isDiscoveryStamped = (el: Element) =>
@@ -2687,7 +2689,14 @@ export function initSandboxRuntimeModular(): void {
};
const onMediaLoadedMetadataForPlayhead = (event: Event) => {
const el = event.currentTarget;
if (state.tornDown || clock.isPlaying() || !isMediaElement(el) || isRenderMode(el)) return;
if (
state.tornDown ||
previewSeek?.time !== undefined ||
clock.isPlaying() ||
!isMediaElement(el) ||
isRenderMode(el)
)
return;
lastSyncedMediaTimeSeconds = null;
state.mediaForceSyncNextTick = true;
syncMediaForCurrentState();
@@ -3775,6 +3784,7 @@ export function initSandboxRuntimeModular(): void {
let lastVisibilityPassSeconds: number | null = null;
const syncMediaForCurrentState = (timingRevision?: number) => {
if (previewSeek?.time !== undefined) return;
// Scope 1 of 3 (see `withTimingResolver`). Closes before `syncRuntimeMedia`,
// which may call `el.load()` and invalidate every cached duration.
//
@@ -3871,7 +3881,8 @@ export function initSandboxRuntimeModular(): void {
// wake call bolted onto each mutator (and forgotten on the next one).
if (force) wakeTransport();
if (force && timelinePostHeld && !clock.isPlaying()) postTimeline();
const frame = Math.max(0, Math.round((state.currentTime || 0) * state.canonicalFps));
const currentTime = previewSeek?.time ?? state.currentTime ?? 0;
const frame = Math.max(0, Math.round(currentTime * state.canonicalFps));
const now = Date.now();
const shouldPost =
force ||
@@ -3888,7 +3899,7 @@ export function initSandboxRuntimeModular(): void {
source: "hf-preview",
type: "state",
frame,
currentTime: state.currentTime || 0,
currentTime,
ended: clock.reachedEnd(),
isPlaying: state.bridgeLastPostedPlaying,
buffering: bufferingFor.length > 0,
@@ -4482,8 +4493,9 @@ export function initSandboxRuntimeModular(): void {
// Redraw the current frame as a seek does; a playing film cut to end at or before it stops there, as at its end.
const cutShort = clock.isPlaying() && duration > 0 && state.currentTime >= duration;
if (joining && clock.isPlaying() && !cutShort) joinPlayback(swappedHosts);
else if (previewSeek?.time !== undefined) previewSeek.refresh();
else transport.seek(cutShort ? duration : state.currentTime, { keepPlaying: !cutShort });
syncTimedElementVisibility(state.currentTime);
if (previewSeek?.time === undefined) syncTimedElementVisibility(state.currentTime);
postTimeline();
};
// Only the rules not here yet: rewriting a loaded rule's style would load its font again.
@@ -4790,16 +4802,19 @@ export function initSandboxRuntimeModular(): void {
.filter(({ name }) => near.has(name) && parked(name))
.sort((a, b) => Math.abs(a.start - time) - Math.abs(b.start - time))[0];
const toMount = landed ?? neighbour;
const toRetire = toMount
? []
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
const toRetire =
toMount || previewSeek?.time !== undefined
? []
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
if (!toMount && toRetire.length === 0) {
windowNear = key;
return false;
}
const wantedNow = (name: string) =>
heldScenes.has(name) ||
scenesWanted(windowScenes, state.currentTime, clock.isPlaying()).has(name);
scenesWanted(windowScenes, previewSeek?.time ?? state.currentTime, clock.isPlaying()).has(
name,
);
const parkedBeforeStep = parkedNames.size;
windowWork = (async () => {
if (landed) await mountScene(landed.name);
@@ -4812,7 +4827,7 @@ export function initSandboxRuntimeModular(): void {
windowWork = null;
const stuckStepKeepsItsPlan = parkedNames.size === parkedBeforeStep;
windowNear = stuckStepKeepsItsPlan ? key : "";
moveSceneWindow(state.currentTime);
moveSceneWindow(previewSeek?.time ?? state.currentTime);
});
// A seek is not done until the scene it lands in is mounted.
if (landed) registerSeekCompletion(windowWork, "scene-window");
@@ -4898,11 +4913,11 @@ export function initSandboxRuntimeModular(): void {
// A paused seek that would reveal undecoded images holds the previous picture until they decode.
// Capped: a request that never settles must not freeze the preview; measured jumps settle in ~200 ms.
const SEEK_HOLD_CAP_MS = 1000;
let heldSeek: { time: number; apply: () => void } | null = null;
let heldSeek: { time: number; apply: () => HeldSeek } | null = null;
const flushHeldSeek = () => {
const held = heldSeek;
heldSeek = null;
held?.apply();
return held?.apply();
};
const applySeek = (quantized: number, options?: { keepPlaying?: boolean }) => {
webAudio.stopAll();
@@ -4953,9 +4968,80 @@ export function initSandboxRuntimeModular(): void {
postState(true);
};
previewSeek = createPreviewSeek({
videosToHold: () =>
withTimingResolver(() =>
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? [])
.map(({ el }) => el)
.filter(isVideoElement),
),
scenesReadyAt: (time) =>
!windowScenes.some(
({ name, start, end }) => start <= time && time <= end && parkedNames.has(name),
),
clipsAt: (time) =>
withTimingResolver(() => {
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
const visibleAt = timedVisibilityAt();
const root = resolveRootCompositionElement();
return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
(clip) =>
isVideoElement(clip.el) &&
!isSilencedByHidden(clip.el) &&
(clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
isClipVisibleAt(time, clip.start, clip.end, duration) &&
clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
);
}),
prepare: (clips, time) => {
moveSceneWindow(time);
syncRuntimeMedia({
clips,
timeSeconds: time,
playing: false,
playbackRate: state.playbackRate,
forceSync: true,
getCompositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
});
},
commit: (time) => applySeek(time),
});
const seekWhenReady = (time: number, options?: { keepPlaying?: boolean }) => {
if (
!mediaDeferred ||
state.nativeMediaSyncDisabled ||
isExportRenderDrivingFrames() ||
window.__HF_EXPORT_RENDER_SEEK_CONFIG ||
(options?.keepPlaying && clock.isPlaying())
) {
previewSeek?.cancel();
applySeek(time, options);
return;
}
webAudio.stopAll();
clock.detachAudioSource();
clock.pause();
state.isPlaying = false;
bufferingFor = [];
pauseTimelineIfPossible(state.capturedTimeline);
runAdapters("pause");
const clips = withTimingResolver(() => buildRuntimeMediaCache().mediaClips);
for (const { el } of transportMovableClips(clips) ?? []) {
if (!el.paused) el.pause();
}
const landed = previewSeek?.seek(time);
if (landed) registerSeekCompletion(landed, "video-frame");
postState(true);
return landed;
};
const transport: RuntimePlayerTransport = {
play: () => {
flushHeldSeek();
const destination = previewSeek?.time;
previewSeek?.cancel();
if (destination !== undefined) applySeek(destination);
const resumingStarved = bufferingFor.length > 0;
bufferingFor = [];
const tl = state.capturedTimeline;
@@ -5003,10 +5089,9 @@ export function initSandboxRuntimeModular(): void {
heldSeek = null;
const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
if (undecoded.length === 0) {
applySeek(quantized, options);
return;
return seekWhenReady(quantized, options);
}
const held = { time: quantized, apply: () => applySeek(quantized, options) };
const held = { time: quantized, apply: () => seekWhenReady(quantized, options) };
heldSeek = held;
for (const img of undecoded) {
loadPreviewImagesIn(img);
@@ -5025,13 +5110,14 @@ export function initSandboxRuntimeModular(): void {
);
const landed = Promise.race([decoded, capped]).then(() => {
wallClearTimeout(capTimer);
if (heldSeek === held) flushHeldSeek();
if (heldSeek === held) return flushHeldSeek();
});
registerSeekCompletion(landed, "image-decode");
return landed;
},
renderSeek: (timeSeconds, options) => {
heldSeek = null;
previewSeek?.cancel();
renderCaptureSeekStarted = true;
const seekTime = resolveRenderSeekTime(
Math.max(0, Number(timeSeconds) || 0),
@@ -5056,7 +5142,7 @@ export function initSandboxRuntimeModular(): void {
if (blurred) registerSeekCompletion(blurred, "motion-blur");
postState(true);
},
getTime: () => heldSeek?.time ?? clock.now(),
getTime: () => heldSeek?.time ?? previewSeek?.time ?? clock.now(),
getDuration: () => {
const dur = clock.getDuration();
return Number.isFinite(dur) ? dur : 0;
@@ -5891,6 +5977,10 @@ export function initSandboxRuntimeModular(): void {
if (state.tornDown || inTransportTick) return;
inTransportTick = true;
try {
if (previewSeek?.time !== undefined) {
postState(false);
return;
}
transportTickCount += 1;
// The jobs below run when the composition revision moves (rate-limited). What no
@@ -6298,6 +6388,7 @@ export function initSandboxRuntimeModular(): void {
emitAnalyticsEvent("composition_paused", { time: player.getTime() });
},
onStopMedia: () => {
previewSeek?.cancel();
webAudio.stopAll();
const mediaEls = document.querySelectorAll("video, audio");
for (const el of mediaEls) {
@@ -6451,6 +6542,7 @@ export function initSandboxRuntimeModular(): void {
const teardown = () => {
if (state.tornDown) return;
state.tornDown = true;
previewSeek?.cancel();
cancelTransportTick();
document.removeEventListener("visibilitychange", onTransportVisibilityChange);
if (transportParkTimerId != null) {
+198
View File
@@ -0,0 +1,198 @@
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
import { isUnplayable, mediaSourceTimeAt } from "./media";
import type { RuntimeMediaClip } from "./types";
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
const HELD_VIDEO_ATTR = "data-hf-seek-video";
const SEEK_CAP_MS = 5000;
const EVENTS = ["loadedmetadata", "loadeddata", "canplay", "seeked", "error"] as const;
function videoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const context = canvas.getContext("2d");
if (!context) return null;
try {
context.drawImage(video, 0, 0);
return canvas;
} catch {
return null;
}
}
/** Keep a displayed video in its layout slot while its decoder moves independently of the page. */
function holdVideoFrame(video: HTMLVideoElement): (() => void) | undefined {
// Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return;
const css = getComputedStyle(video);
if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return;
if (!video.videoWidth || !video.videoHeight) return;
const canvas = videoFrame(video);
if (!canvas) return;
for (const name of Array.from(css)) canvas.style.setProperty(name, css.getPropertyValue(name));
canvas.style.setProperty("animation", "none", "important");
canvas.style.setProperty("transition", "none", "important");
canvas.style.setProperty("pointer-events", "none", "important");
canvas.setAttribute("data-layout-ignore", "");
canvas.setAttribute("aria-hidden", "true");
video.before(canvas);
video.setAttribute(HELD_VIDEO_ATTR, "");
return () => {
video.removeAttribute(HELD_VIDEO_ATTR);
canvas.remove();
};
}
function holdVideoFrames(videos: HTMLVideoElement[]): () => void {
const restore = videos.map(holdVideoFrame);
const style = document.createElement("style");
style.textContent = `[${HELD_VIDEO_ATTR}] { display: none !important; }`;
document.head.appendChild(style);
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
return () => {
for (const putBack of restore) putBack?.();
style.remove();
document.documentElement.removeAttribute(PREVIEW_SEEK_HOLD_ATTR);
};
}
type SeekHost = {
clipsAt: (time: number) => RuntimeMediaClip[];
videosToHold: () => HTMLVideoElement[];
scenesReadyAt: (time: number) => boolean;
prepare: (clips: RuntimeMediaClip[], time: number) => void;
commit: (time: number) => void;
};
/** One decoder job at a time; requests arriving during it replace the queued destination. */
export function createPreviewSeek(host: SeekHost) {
let pending: { time: number; done: Promise<void>; resolve: () => void } | null = null;
let releasePicture = () => {};
let stopWaiting = () => {};
let runningClips: RuntimeMediaClip[] = [];
let runId = 0;
const ready = (clips: RuntimeMediaClip[], time: number) =>
host.scenesReadyAt(time) &&
clips.every((clip) => {
const { el } = clip;
return (
!el.isConnected ||
isUnplayable(el) ||
(el.readyState >= 2 &&
!el.seeking &&
Math.abs(el.currentTime - mediaSourceTimeAt(clip, time)) < 0.02)
);
});
const cancel = () => {
const request = pending;
pending = null;
runId++;
stopWaiting();
stopWaiting = () => {};
releasePicture();
releasePicture = () => {};
request?.resolve();
};
const run = () => {
const request = pending;
if (!request) return;
const id = ++runId;
const time = request.time;
const clips = host.clipsAt(time);
runningClips = clips;
let preparing = true;
let finished = false;
const finish = () => {
if (finished || pending !== request || id !== runId) return;
finished = true;
stopWaiting();
stopWaiting = () => {};
if (request.time !== time) {
if (ready(host.clipsAt(time), time)) {
releasePicture();
host.commit(time);
releasePicture = holdVideoFrames(host.videosToHold());
}
run();
return;
}
cancel();
host.commit(time);
};
const check = () => {
if (!preparing && ready(host.clipsAt(time), time)) finish();
};
const metadata = () => {
if (pending !== request || finished || id !== runId) return;
// A source/proxy change resets currentTime; the parked clock still describes the old picture.
preparing = true;
host.prepare(host.clipsAt(time), time);
preparing = false;
check();
};
const listeners = clips.map(({ el }) => {
for (const type of EVENTS)
el.addEventListener(type, type === "loadedmetadata" ? metadata : check, type === "error");
return () => {
for (const type of EVENTS)
el.removeEventListener(
type,
type === "loadedmetadata" ? metadata : check,
type === "error",
);
};
});
const cap = wallSetTimeout(finish, SEEK_CAP_MS);
stopWaiting = () => {
wallClearTimeout(cap);
for (const remove of listeners) remove();
};
host.prepare(clips, time);
preparing = false;
check();
};
return {
get time() {
return pending?.time;
},
seek(time: number): Promise<void> | void {
if (pending) {
const request = pending;
request.time = time;
const clips = host.clipsAt(time);
if (
clips.length !== runningClips.length ||
clips.some((clip, index) => clip.el !== runningClips[index]?.el)
) {
stopWaiting();
run();
}
return request.done;
}
if (ready(host.clipsAt(time), time)) {
host.commit(time);
return;
}
let resolve = () => {};
const done = new Promise<void>((settle) => {
resolve = settle;
});
pending = { time, done, resolve };
releasePicture = holdVideoFrames(host.videosToHold());
run();
return done;
},
refresh() {
if (!pending) return;
stopWaiting();
run();
},
cancel,
};
}
+1
View File
@@ -1,6 +1,7 @@
import { isWebKit } from "./runtime/mediaClock.js";
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
export const PREVIEW_SEEK_HOLD_ATTR = "data-hf-preview-seek-hold";
export const STUDIO_PREVIEW_CAPTURE_MODE = "capture";
export const PLAYER_FRAME_ATTR = "data-hf-player-frame";