mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(core): retain complete preview pictures during native seeks (#7378)
* fix(core): retain complete preview pictures during native seeks * fix: settle failed seeks and preserve coherent progress
This commit is contained in:
@@ -100,6 +100,12 @@ console.log(player.duration, player.paused, player.ready);
|
||||
console.log(player.compositionWidth, player.compositionHeight);
|
||||
```
|
||||
|
||||
In Studio previews, scrubbing keeps the last complete picture visible while video
|
||||
layers decode at the requested time. Rapid seeks keep only the latest queued
|
||||
destination, and the captions and video layers appear together once it is ready.
|
||||
Complete intermediate pictures can appear during a drag. Jumping to a different
|
||||
clip supersedes an earlier pending seek; failed sources do not hold up the preview.
|
||||
|
||||
## Events
|
||||
|
||||
```js
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mediaElementTime } from "./mediaClock";
|
||||
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
|
||||
import {
|
||||
HF_COLOR_GRADING_ATTR,
|
||||
HF_COLOR_GRADING_EFFECT_KEYS,
|
||||
@@ -352,6 +353,26 @@ describe("createColorGradingRuntime", () => {
|
||||
expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
|
||||
});
|
||||
|
||||
it("keeps graded layers on the committed picture until a preview seek lands", () => {
|
||||
const { video } = startRuntimeWithVideo();
|
||||
const drawsBefore = texImage2DCalls.length;
|
||||
const canvas = document.querySelector("[data-hf-color-grading-canvas]");
|
||||
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
|
||||
try {
|
||||
video.dispatchEvent(new Event("seeked"));
|
||||
video.style.setProperty("--hf-color-grading-exposure", "1");
|
||||
runtime?.refresh();
|
||||
expect(runtime?.redraw()).toBe(0);
|
||||
expect(texImage2DCalls).toHaveLength(drawsBefore);
|
||||
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBe(canvas);
|
||||
expect(canvas?.isConnected).toBe(true);
|
||||
} finally {
|
||||
document.documentElement.removeAttribute(PREVIEW_SEEK_HOLD_ATTR);
|
||||
}
|
||||
expect(runtime?.redraw()).toBe(1);
|
||||
expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
|
||||
});
|
||||
|
||||
it("does not redraw-loop on its own hide writes (opacity/visibility only)", async () => {
|
||||
const { video } = startRuntimeWithVideo();
|
||||
await new Promise((resolve) => requestAnimationFrame(() => resolve(null)));
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
|
||||
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
|
||||
import {
|
||||
HF_COLOR_GRADING_ADJUST_KEYS,
|
||||
HF_COLOR_GRADING_ANIMATABLE_PROPERTIES,
|
||||
@@ -3419,6 +3420,7 @@ type DrawableFrame = {
|
||||
};
|
||||
|
||||
function drawableFrame(entry: ColorGradingEntry): DrawableFrame | null {
|
||||
if (document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)) return null;
|
||||
const source = getDrawableSource(entry.element);
|
||||
const pixels = source && readablePixels(source, entry);
|
||||
if (!source || !pixels) {
|
||||
|
||||
@@ -316,6 +316,34 @@ describe("runtime entry", () => {
|
||||
await window.__hfWaitForSeekCompletion?.();
|
||||
});
|
||||
|
||||
it("resolves a mixed image/video seek only after both have decoded", async () => {
|
||||
servePreview();
|
||||
const root = mountRoot();
|
||||
const current = timed(root, "div", "0");
|
||||
const later = timed(root, "div", "5");
|
||||
const image = later.appendChild(document.createElement("img"));
|
||||
let decodeImage = () => {};
|
||||
image.decode = () => new Promise<void>((resolve) => (decodeImage = resolve));
|
||||
const video = timed(later, "video", "5");
|
||||
video.src = "/movie.mp4";
|
||||
let ready = 1;
|
||||
Object.defineProperty(video, "readyState", { configurable: true, get: () => ready });
|
||||
Object.defineProperty(video, "networkState", { configurable: true, value: 2 });
|
||||
await evaluateRuntime();
|
||||
let settled = false;
|
||||
const landed = Promise.resolve(window.__player!.seek(5.5)).then(() => {
|
||||
settled = true;
|
||||
});
|
||||
decodeImage();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(settled).toBe(false);
|
||||
expect(visibility(current, later)).toEqual(["visible", "hidden"]);
|
||||
ready = 4;
|
||||
video.dispatchEvent(new Event("seeked"));
|
||||
await landed;
|
||||
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
|
||||
});
|
||||
|
||||
it("applies a held jump after the cap when an image never decodes", async () => {
|
||||
servePreview();
|
||||
const root = mountRoot();
|
||||
@@ -495,6 +523,9 @@ describe("runtime entry", () => {
|
||||
]);
|
||||
|
||||
window.__player?.seek(3.5);
|
||||
const video = second![0]!;
|
||||
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
|
||||
video.dispatchEvent(new Event("seeked"));
|
||||
expect([first, second, third].map((take) => armed(...take))).toEqual([
|
||||
[false, false],
|
||||
[true, true],
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { initSandboxRuntimeModular } from "./init";
|
||||
import { hideTimedClipsUntilFirstPass } from "./timedClipHide";
|
||||
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
|
||||
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
|
||||
|
||||
function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
|
||||
let time = 0;
|
||||
let ready = initiallyReady ? 4 : 0;
|
||||
let seeking = false;
|
||||
const seeks: number[] = [];
|
||||
Object.defineProperties(video, {
|
||||
duration: { value: 10, configurable: true },
|
||||
videoWidth: { value: 64, configurable: true },
|
||||
videoHeight: { value: 36, configurable: true },
|
||||
readyState: { get: () => ready, configurable: true },
|
||||
networkState: { value: 2, configurable: true },
|
||||
paused: { value: true, configurable: true },
|
||||
seeking: { get: () => seeking, configurable: true },
|
||||
currentTime: {
|
||||
get: () => time,
|
||||
set: (next: number) => {
|
||||
time = next;
|
||||
seeks.push(next);
|
||||
seeking = true;
|
||||
ready = 1;
|
||||
},
|
||||
configurable: true,
|
||||
},
|
||||
});
|
||||
video.pause = vi.fn();
|
||||
video.play = vi.fn(() => Promise.resolve());
|
||||
video.load = vi.fn();
|
||||
return {
|
||||
video,
|
||||
seeks,
|
||||
land: () => {
|
||||
ready = 4;
|
||||
seeking = false;
|
||||
video.dispatchEvent(new Event("seeked"));
|
||||
},
|
||||
reload: () => {
|
||||
time = 0;
|
||||
ready = 1;
|
||||
seeking = false;
|
||||
video.dispatchEvent(new Event("loadedmetadata"));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function film(withImage = false) {
|
||||
resetRuntimeFixtureDom();
|
||||
const meta = document.createElement("meta");
|
||||
meta.name = STUDIO_PREVIEW_MARK_META;
|
||||
document.head.append(meta);
|
||||
vi.spyOn(window, "requestAnimationFrame").mockReturnValue(0);
|
||||
const drawImage = vi.fn();
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
drawImage,
|
||||
} as unknown as CanvasRenderingContext2D);
|
||||
document.body.innerHTML =
|
||||
'<div data-composition-id="main" data-root="true" data-duration="10">' +
|
||||
[0, 5]
|
||||
.flatMap((start) =>
|
||||
["plate", "matte"].map(
|
||||
(name) =>
|
||||
`<video id="${name}-${start}" style="width:64px;height:36px" data-start="${start}" data-duration="5" src="/${name}.mp4" muted></video>`,
|
||||
),
|
||||
)
|
||||
.join("") +
|
||||
(withImage
|
||||
? '<div data-start="8" data-duration="2"><img id="later-image" src="/later.png"></div>'
|
||||
: "") +
|
||||
"</div>";
|
||||
const videos = [...document.querySelectorAll("video")].map((v) =>
|
||||
decoder(v, v.dataset.start === "0"),
|
||||
);
|
||||
const main = createMockTimeline(10);
|
||||
window.__timelines = { main };
|
||||
if (withImage) hideTimedClipsUntilFirstPass();
|
||||
initSandboxRuntimeModular();
|
||||
for (const video of videos.slice(0, 2)) video.land();
|
||||
return { main, old: videos.slice(0, 2), next: videos.slice(2), drawImage };
|
||||
}
|
||||
|
||||
it("keeps the old picture and page until both destination videos have decoded", async () => {
|
||||
const { main, old, next, drawImage } = film();
|
||||
const landed = window.__player!.seek(5.5);
|
||||
expect(window.__player!.getTime()).toBe(5.5);
|
||||
expect(main.time?.()).toBe(0);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
|
||||
expect(drawImage).toHaveBeenCalledTimes(2);
|
||||
expect(document.querySelectorAll("canvas")).toHaveLength(2);
|
||||
expect(old.every(({ video }) => getComputedStyle(video).display === "none")).toBe(true);
|
||||
next[0]!.land();
|
||||
expect(main.time?.()).toBe(0);
|
||||
next[1]!.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(5.5);
|
||||
expect(document.querySelectorAll("canvas")).toHaveLength(0);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
expect(old.every(({ video }) => video.style.visibility === "hidden")).toBe(true);
|
||||
expect(next.every(({ video }) => video.style.visibility === "visible")).toBe(true);
|
||||
});
|
||||
|
||||
it("coalesces same-clip requests and publishes complete intermediate pictures", async () => {
|
||||
const { main, next } = film();
|
||||
const landed = window.__player!.seek(5.5);
|
||||
const initialSeeks = next.map((v) => v.seeks.length);
|
||||
window.__player!.seek(6);
|
||||
window.__player!.seek(7.5);
|
||||
expect(next.map((v) => v.seeks.length)).toEqual(initialSeeks);
|
||||
for (const video of next) video.land();
|
||||
expect(main.time?.()).toBe(5.5);
|
||||
expect(next.every((v) => Math.abs(v.video.currentTime - 2.5) < 0.001)).toBe(true);
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(7.5);
|
||||
});
|
||||
|
||||
it("supersedes a stuck run immediately when the destination uses different clips", async () => {
|
||||
const { main, old } = film();
|
||||
const landed = window.__player!.seek(6);
|
||||
window.__player!.seek(1);
|
||||
expect(old.every(({ video }) => Math.abs(video.currentTime - 1) < 0.001)).toBe(true);
|
||||
for (const video of old) video.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(1);
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("settles a failed source without waiting for the decoder cap", async () => {
|
||||
const { main, next } = film();
|
||||
const landed = window.__player!.seek(6);
|
||||
for (const { video } of next) {
|
||||
Object.defineProperties(video, {
|
||||
error: { value: null, configurable: true },
|
||||
networkState: { value: 3, configurable: true },
|
||||
});
|
||||
const source = document.createElement("source");
|
||||
video.append(source);
|
||||
source.dispatchEvent(new Event("error"));
|
||||
}
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(6);
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps a pending video's complete-picture hold while waiting for a later image", async () => {
|
||||
const { main, old, next } = film(true);
|
||||
const image = document.querySelector<HTMLImageElement>("#later-image")!;
|
||||
let decodeImage = () => {};
|
||||
const decode = vi.fn(() => new Promise<void>((resolve) => (decodeImage = resolve)));
|
||||
image.decode = decode;
|
||||
window.__player!.seek(6);
|
||||
const landed = window.__player!.seek(8.5);
|
||||
expect(decode).toHaveBeenCalledOnce();
|
||||
expect(window.__player!.getTime()).toBe(8.5);
|
||||
expect(main.time?.()).toBe(0);
|
||||
expect(old.every(({ video }) => getComputedStyle(video).display === "none")).toBe(true);
|
||||
expect(document.querySelectorAll("canvas")).toHaveLength(2);
|
||||
Object.defineProperty(image, "complete", { value: true, configurable: true });
|
||||
decodeImage();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
for (const video of next) video.land();
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(8.5);
|
||||
});
|
||||
|
||||
it("holds overlapping same-clip seeks too, and repositions a reloaded source at the requested time", async () => {
|
||||
const { main, old } = film();
|
||||
const landed = window.__player!.seek(2);
|
||||
old[0]!.reload();
|
||||
expect(old[0]!.video.currentTime).toBeCloseTo(2, 3);
|
||||
old[0]!.land();
|
||||
expect(main.time?.()).toBe(0);
|
||||
old[1]!.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(2);
|
||||
});
|
||||
|
||||
it("holds a paused keepPlaying seek and normal seeks after a non-export renderSeek", async () => {
|
||||
const { main, old, next } = film();
|
||||
window.__player!.renderSeek(0.2);
|
||||
for (const video of old) video.land();
|
||||
const landed = window.__player!.seek(6, { keepPlaying: true });
|
||||
expect(main.time?.()).toBe(0.2);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(6);
|
||||
});
|
||||
|
||||
it("does not pause, seek or cover a video leased by another preview control", async () => {
|
||||
const { old, next } = film();
|
||||
const leased = old[0]!;
|
||||
window.__hf!.leasePausedMedia!(leased.video);
|
||||
Object.defineProperty(leased.video, "paused", { value: false, configurable: true });
|
||||
const seeks = leased.seeks.length;
|
||||
const landed = window.__player!.seek(6);
|
||||
expect(leased.video.pause).not.toHaveBeenCalled();
|
||||
expect(leased.seeks).toHaveLength(seeks);
|
||||
expect(leased.video.hasAttribute("data-hf-seek-video")).toBe(false);
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
expect(leased.video.pause).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves host-controlled native media to the host", () => {
|
||||
const { old } = film();
|
||||
window.dispatchEvent(
|
||||
new MessageEvent("message", {
|
||||
source: window,
|
||||
data: {
|
||||
source: "hf-parent",
|
||||
type: "control",
|
||||
action: "set-native-media-sync-disabled",
|
||||
disabled: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
Object.defineProperty(old[0]!.video, "paused", { value: false, configurable: true });
|
||||
const seeks = old[0]!.seeks.length;
|
||||
window.__player!.seek(2);
|
||||
expect(old[0]!.seeks).toHaveLength(seeks);
|
||||
expect(old[0]!.video.pause).not.toHaveBeenCalled();
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
});
|
||||
|
||||
it("leaves a grading canvas's hidden source in its layout during the hold", async () => {
|
||||
const { old, next, drawImage } = film();
|
||||
old[0]!.video.setAttribute("data-hf-color-grading-source-hidden", "true");
|
||||
old[0]!.video.style.setProperty("opacity", "0", "important");
|
||||
const landed = window.__player!.seek(6);
|
||||
expect(old[0]!.video.hasAttribute("data-hf-seek-video")).toBe(false);
|
||||
expect(drawImage).toHaveBeenCalledTimes(1);
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
});
|
||||
|
||||
it.each(["play", "render", "teardown"])(
|
||||
"cleans up a pending seek on %s and ignores late completions",
|
||||
async (action) => {
|
||||
const { main, next } = film();
|
||||
const player = window.__player!;
|
||||
const landed = player.seek(6);
|
||||
if (action === "play") player.play();
|
||||
else if (action === "render") player.renderSeek(2);
|
||||
else window.__hfRuntimeTeardown?.();
|
||||
const time = main.time?.();
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(time);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
if (action === "play") expect(player.getTime()).toBeGreaterThanOrEqual(6);
|
||||
if (action === "render") expect(time).toBe(2);
|
||||
},
|
||||
);
|
||||
|
||||
it("caps a stalled decoder without leaving the picture hidden", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
const { main } = film();
|
||||
const landed = window.__player!.seek(6);
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(6);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
window.__hfRuntimeTeardown?.();
|
||||
document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`)?.remove();
|
||||
for (const style of document.querySelectorAll(
|
||||
"style[data-hf-skip-hidden-images], style[data-hf-hidden-clip-contents]",
|
||||
))
|
||||
style.remove();
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
resetRuntimeFixtureDom();
|
||||
});
|
||||
@@ -9,6 +9,7 @@ import { probeAndCacheElementVolume } from "./mediaVolumeEnvelope.js";
|
||||
import { wrapScopedCompositionScript } from "../compiler/compositionScoping";
|
||||
import { AFTER_FONTS_SCRIPT_TYPE } from "../compiler/scriptRuns";
|
||||
import { parkFarScenesAtBoot } from "./sceneWindow";
|
||||
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
|
||||
|
||||
vi.mock("./mediaVolumeEnvelope.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("./mediaVolumeEnvelope.js")>();
|
||||
@@ -1767,9 +1768,13 @@ describe("the preview's scene window", () => {
|
||||
el.getAttribute("data-hf-scene"),
|
||||
);
|
||||
// As the runtime's entry does: park before any scene script runs, then run the rest.
|
||||
function bootWindowed(scenes: Scene[], root: Tl, withManifest = true) {
|
||||
function bootWindowed(scenes: Scene[], root: Tl, withManifest = true, studioPreview = false) {
|
||||
const { head, body } = preview(scenes);
|
||||
document.head.innerHTML = withManifest ? head : head.replace(/<meta[^>]*>/, "");
|
||||
if (studioPreview)
|
||||
document.head.append(
|
||||
Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
|
||||
);
|
||||
document.body.innerHTML = body;
|
||||
window.__timelines = { main: root };
|
||||
parkFarScenesAtBoot();
|
||||
@@ -1868,6 +1873,38 @@ describe("the preview's scene window", () => {
|
||||
expect(parked()).toEqual(["s2"]);
|
||||
});
|
||||
|
||||
it("holds a parked destination until its videos mount and decode without retiring the old picture", async () => {
|
||||
const ready = new WeakSet<HTMLMediaElement>();
|
||||
vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockImplementation(function () {
|
||||
return ready.has(this) ? 4 : 1;
|
||||
});
|
||||
vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
|
||||
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
|
||||
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
|
||||
const { root } = trackingRoot();
|
||||
bootWindowed(
|
||||
[S0, S1, { ...S2, body: '<video src="/late.mp4" data-start="0" data-duration="2"></video>' }],
|
||||
root,
|
||||
true,
|
||||
true,
|
||||
);
|
||||
await tick();
|
||||
const landed = window.__player!.seek(5);
|
||||
await vi.waitFor(() => {
|
||||
expect(sceneHost("s2").querySelector("video")?.currentTime).toBeCloseTo(1, 3);
|
||||
});
|
||||
expect(root.time?.()).toBe(0);
|
||||
expect(window.__player!.getTime()).toBe(5);
|
||||
expect(parked()).not.toContain("s0");
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
|
||||
const video = sceneHost("s2").querySelector("video")!;
|
||||
ready.add(video);
|
||||
video.dispatchEvent(new Event("seeked"));
|
||||
await landed;
|
||||
expect(root.time?.()).toBe(5);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
});
|
||||
|
||||
it("waits for the scene a seek lands in while its mount as a neighbour is still running", async () => {
|
||||
bootWindowed([S0, S1, S2], trackingRoot().root);
|
||||
await tick();
|
||||
|
||||
@@ -28,10 +28,11 @@ import {
|
||||
} from "./preloadMedia";
|
||||
import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
|
||||
import { hasFrozenPicture } from "./frozenPicture";
|
||||
import { createPreviewSeek } from "./previewSeek";
|
||||
import { copiesReady } from "./previewCopies";
|
||||
import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
|
||||
import { instantTolerance } from "../clipFacts";
|
||||
import { isInClipWindow } from "./clipWindow";
|
||||
import { isClipVisibleAt, isInClipWindow } from "./clipWindow";
|
||||
import { isWebKit, mediaElementTime } from "./mediaClock";
|
||||
import {
|
||||
CLIP_HIDDEN_ATTR,
|
||||
@@ -528,6 +529,7 @@ function loadPreviewImagesIn(root: Element): void {
|
||||
|
||||
export function initSandboxRuntimeModular(): void {
|
||||
const state = createRuntimeState();
|
||||
let previewSeek: ReturnType<typeof createPreviewSeek> | undefined;
|
||||
const stampsForStudio = window.parent !== window;
|
||||
// Studio's discovery stamps time a clip for the timeline only; nothing hides it at their end.
|
||||
const isDiscoveryStamped = (el: Element) =>
|
||||
@@ -2687,7 +2689,14 @@ export function initSandboxRuntimeModular(): void {
|
||||
};
|
||||
const onMediaLoadedMetadataForPlayhead = (event: Event) => {
|
||||
const el = event.currentTarget;
|
||||
if (state.tornDown || clock.isPlaying() || !isMediaElement(el) || isRenderMode(el)) return;
|
||||
if (
|
||||
state.tornDown ||
|
||||
previewSeek?.time !== undefined ||
|
||||
clock.isPlaying() ||
|
||||
!isMediaElement(el) ||
|
||||
isRenderMode(el)
|
||||
)
|
||||
return;
|
||||
lastSyncedMediaTimeSeconds = null;
|
||||
state.mediaForceSyncNextTick = true;
|
||||
syncMediaForCurrentState();
|
||||
@@ -3775,6 +3784,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
|
||||
let lastVisibilityPassSeconds: number | null = null;
|
||||
const syncMediaForCurrentState = (timingRevision?: number) => {
|
||||
if (previewSeek?.time !== undefined) return;
|
||||
// Scope 1 of 3 (see `withTimingResolver`). Closes before `syncRuntimeMedia`,
|
||||
// which may call `el.load()` and invalidate every cached duration.
|
||||
//
|
||||
@@ -3871,7 +3881,8 @@ export function initSandboxRuntimeModular(): void {
|
||||
// wake call bolted onto each mutator (and forgotten on the next one).
|
||||
if (force) wakeTransport();
|
||||
if (force && timelinePostHeld && !clock.isPlaying()) postTimeline();
|
||||
const frame = Math.max(0, Math.round((state.currentTime || 0) * state.canonicalFps));
|
||||
const currentTime = previewSeek?.time ?? state.currentTime ?? 0;
|
||||
const frame = Math.max(0, Math.round(currentTime * state.canonicalFps));
|
||||
const now = Date.now();
|
||||
const shouldPost =
|
||||
force ||
|
||||
@@ -3888,7 +3899,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
source: "hf-preview",
|
||||
type: "state",
|
||||
frame,
|
||||
currentTime: state.currentTime || 0,
|
||||
currentTime,
|
||||
ended: clock.reachedEnd(),
|
||||
isPlaying: state.bridgeLastPostedPlaying,
|
||||
buffering: bufferingFor.length > 0,
|
||||
@@ -4482,8 +4493,9 @@ export function initSandboxRuntimeModular(): void {
|
||||
// Redraw the current frame as a seek does; a playing film cut to end at or before it stops there, as at its end.
|
||||
const cutShort = clock.isPlaying() && duration > 0 && state.currentTime >= duration;
|
||||
if (joining && clock.isPlaying() && !cutShort) joinPlayback(swappedHosts);
|
||||
else if (previewSeek?.time !== undefined) previewSeek.refresh();
|
||||
else transport.seek(cutShort ? duration : state.currentTime, { keepPlaying: !cutShort });
|
||||
syncTimedElementVisibility(state.currentTime);
|
||||
if (previewSeek?.time === undefined) syncTimedElementVisibility(state.currentTime);
|
||||
postTimeline();
|
||||
};
|
||||
// Only the rules not here yet: rewriting a loaded rule's style would load its font again.
|
||||
@@ -4790,16 +4802,19 @@ export function initSandboxRuntimeModular(): void {
|
||||
.filter(({ name }) => near.has(name) && parked(name))
|
||||
.sort((a, b) => Math.abs(a.start - time) - Math.abs(b.start - time))[0];
|
||||
const toMount = landed ?? neighbour;
|
||||
const toRetire = toMount
|
||||
? []
|
||||
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
|
||||
const toRetire =
|
||||
toMount || previewSeek?.time !== undefined
|
||||
? []
|
||||
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
|
||||
if (!toMount && toRetire.length === 0) {
|
||||
windowNear = key;
|
||||
return false;
|
||||
}
|
||||
const wantedNow = (name: string) =>
|
||||
heldScenes.has(name) ||
|
||||
scenesWanted(windowScenes, state.currentTime, clock.isPlaying()).has(name);
|
||||
scenesWanted(windowScenes, previewSeek?.time ?? state.currentTime, clock.isPlaying()).has(
|
||||
name,
|
||||
);
|
||||
const parkedBeforeStep = parkedNames.size;
|
||||
windowWork = (async () => {
|
||||
if (landed) await mountScene(landed.name);
|
||||
@@ -4812,7 +4827,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
windowWork = null;
|
||||
const stuckStepKeepsItsPlan = parkedNames.size === parkedBeforeStep;
|
||||
windowNear = stuckStepKeepsItsPlan ? key : "";
|
||||
moveSceneWindow(state.currentTime);
|
||||
moveSceneWindow(previewSeek?.time ?? state.currentTime);
|
||||
});
|
||||
// A seek is not done until the scene it lands in is mounted.
|
||||
if (landed) registerSeekCompletion(windowWork, "scene-window");
|
||||
@@ -4898,11 +4913,11 @@ export function initSandboxRuntimeModular(): void {
|
||||
// A paused seek that would reveal undecoded images holds the previous picture until they decode.
|
||||
// Capped: a request that never settles must not freeze the preview; measured jumps settle in ~200 ms.
|
||||
const SEEK_HOLD_CAP_MS = 1000;
|
||||
let heldSeek: { time: number; apply: () => void } | null = null;
|
||||
let heldSeek: { time: number; apply: () => HeldSeek } | null = null;
|
||||
const flushHeldSeek = () => {
|
||||
const held = heldSeek;
|
||||
heldSeek = null;
|
||||
held?.apply();
|
||||
return held?.apply();
|
||||
};
|
||||
const applySeek = (quantized: number, options?: { keepPlaying?: boolean }) => {
|
||||
webAudio.stopAll();
|
||||
@@ -4953,9 +4968,80 @@ export function initSandboxRuntimeModular(): void {
|
||||
postState(true);
|
||||
};
|
||||
|
||||
previewSeek = createPreviewSeek({
|
||||
videosToHold: () =>
|
||||
withTimingResolver(() =>
|
||||
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? [])
|
||||
.map(({ el }) => el)
|
||||
.filter(isVideoElement),
|
||||
),
|
||||
scenesReadyAt: (time) =>
|
||||
!windowScenes.some(
|
||||
({ name, start, end }) => start <= time && time <= end && parkedNames.has(name),
|
||||
),
|
||||
clipsAt: (time) =>
|
||||
withTimingResolver(() => {
|
||||
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
|
||||
const visibleAt = timedVisibilityAt();
|
||||
const root = resolveRootCompositionElement();
|
||||
return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
|
||||
(clip) =>
|
||||
isVideoElement(clip.el) &&
|
||||
!isSilencedByHidden(clip.el) &&
|
||||
(clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
|
||||
isClipVisibleAt(time, clip.start, clip.end, duration) &&
|
||||
clipChainVisibleAt(clip.el.parentElement, time, visibleAt, root),
|
||||
);
|
||||
}),
|
||||
prepare: (clips, time) => {
|
||||
moveSceneWindow(time);
|
||||
syncRuntimeMedia({
|
||||
clips,
|
||||
timeSeconds: time,
|
||||
playing: false,
|
||||
playbackRate: state.playbackRate,
|
||||
forceSync: true,
|
||||
getCompositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
|
||||
});
|
||||
},
|
||||
commit: (time) => applySeek(time),
|
||||
});
|
||||
|
||||
const seekWhenReady = (time: number, options?: { keepPlaying?: boolean }) => {
|
||||
if (
|
||||
!mediaDeferred ||
|
||||
state.nativeMediaSyncDisabled ||
|
||||
isExportRenderDrivingFrames() ||
|
||||
window.__HF_EXPORT_RENDER_SEEK_CONFIG ||
|
||||
(options?.keepPlaying && clock.isPlaying())
|
||||
) {
|
||||
previewSeek?.cancel();
|
||||
applySeek(time, options);
|
||||
return;
|
||||
}
|
||||
webAudio.stopAll();
|
||||
clock.detachAudioSource();
|
||||
clock.pause();
|
||||
state.isPlaying = false;
|
||||
bufferingFor = [];
|
||||
pauseTimelineIfPossible(state.capturedTimeline);
|
||||
runAdapters("pause");
|
||||
const clips = withTimingResolver(() => buildRuntimeMediaCache().mediaClips);
|
||||
for (const { el } of transportMovableClips(clips) ?? []) {
|
||||
if (!el.paused) el.pause();
|
||||
}
|
||||
const landed = previewSeek?.seek(time);
|
||||
if (landed) registerSeekCompletion(landed, "video-frame");
|
||||
postState(true);
|
||||
return landed;
|
||||
};
|
||||
|
||||
const transport: RuntimePlayerTransport = {
|
||||
play: () => {
|
||||
flushHeldSeek();
|
||||
const destination = previewSeek?.time;
|
||||
previewSeek?.cancel();
|
||||
if (destination !== undefined) applySeek(destination);
|
||||
const resumingStarved = bufferingFor.length > 0;
|
||||
bufferingFor = [];
|
||||
const tl = state.capturedTimeline;
|
||||
@@ -5003,10 +5089,9 @@ export function initSandboxRuntimeModular(): void {
|
||||
heldSeek = null;
|
||||
const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
|
||||
if (undecoded.length === 0) {
|
||||
applySeek(quantized, options);
|
||||
return;
|
||||
return seekWhenReady(quantized, options);
|
||||
}
|
||||
const held = { time: quantized, apply: () => applySeek(quantized, options) };
|
||||
const held = { time: quantized, apply: () => seekWhenReady(quantized, options) };
|
||||
heldSeek = held;
|
||||
for (const img of undecoded) {
|
||||
loadPreviewImagesIn(img);
|
||||
@@ -5025,13 +5110,14 @@ export function initSandboxRuntimeModular(): void {
|
||||
);
|
||||
const landed = Promise.race([decoded, capped]).then(() => {
|
||||
wallClearTimeout(capTimer);
|
||||
if (heldSeek === held) flushHeldSeek();
|
||||
if (heldSeek === held) return flushHeldSeek();
|
||||
});
|
||||
registerSeekCompletion(landed, "image-decode");
|
||||
return landed;
|
||||
},
|
||||
renderSeek: (timeSeconds, options) => {
|
||||
heldSeek = null;
|
||||
previewSeek?.cancel();
|
||||
renderCaptureSeekStarted = true;
|
||||
const seekTime = resolveRenderSeekTime(
|
||||
Math.max(0, Number(timeSeconds) || 0),
|
||||
@@ -5056,7 +5142,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
if (blurred) registerSeekCompletion(blurred, "motion-blur");
|
||||
postState(true);
|
||||
},
|
||||
getTime: () => heldSeek?.time ?? clock.now(),
|
||||
getTime: () => heldSeek?.time ?? previewSeek?.time ?? clock.now(),
|
||||
getDuration: () => {
|
||||
const dur = clock.getDuration();
|
||||
return Number.isFinite(dur) ? dur : 0;
|
||||
@@ -5891,6 +5977,10 @@ export function initSandboxRuntimeModular(): void {
|
||||
if (state.tornDown || inTransportTick) return;
|
||||
inTransportTick = true;
|
||||
try {
|
||||
if (previewSeek?.time !== undefined) {
|
||||
postState(false);
|
||||
return;
|
||||
}
|
||||
transportTickCount += 1;
|
||||
|
||||
// The jobs below run when the composition revision moves (rate-limited). What no
|
||||
@@ -6298,6 +6388,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
emitAnalyticsEvent("composition_paused", { time: player.getTime() });
|
||||
},
|
||||
onStopMedia: () => {
|
||||
previewSeek?.cancel();
|
||||
webAudio.stopAll();
|
||||
const mediaEls = document.querySelectorAll("video, audio");
|
||||
for (const el of mediaEls) {
|
||||
@@ -6451,6 +6542,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
const teardown = () => {
|
||||
if (state.tornDown) return;
|
||||
state.tornDown = true;
|
||||
previewSeek?.cancel();
|
||||
cancelTransportTick();
|
||||
document.removeEventListener("visibilitychange", onTransportVisibilityChange);
|
||||
if (transportParkTimerId != null) {
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
|
||||
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
|
||||
import { isUnplayable, mediaSourceTimeAt } from "./media";
|
||||
import type { RuntimeMediaClip } from "./types";
|
||||
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
|
||||
|
||||
const HELD_VIDEO_ATTR = "data-hf-seek-video";
|
||||
const SEEK_CAP_MS = 5000;
|
||||
const EVENTS = ["loadedmetadata", "loadeddata", "canplay", "seeked", "error"] as const;
|
||||
|
||||
function videoFrame(video: HTMLVideoElement): HTMLCanvasElement | null {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = video.videoWidth;
|
||||
canvas.height = video.videoHeight;
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) return null;
|
||||
try {
|
||||
context.drawImage(video, 0, 0);
|
||||
return canvas;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Keep a displayed video in its layout slot while its decoder moves independently of the page. */
|
||||
function holdVideoFrame(video: HTMLVideoElement): (() => void) | undefined {
|
||||
// Its grading canvas already holds the picture; hiding the source would retire that canvas on refresh.
|
||||
if (video.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR)) return;
|
||||
const css = getComputedStyle(video);
|
||||
if (css.display === "none" || css.visibility === "hidden" || video.readyState < 2) return;
|
||||
if (!video.videoWidth || !video.videoHeight) return;
|
||||
const canvas = videoFrame(video);
|
||||
if (!canvas) return;
|
||||
for (const name of Array.from(css)) canvas.style.setProperty(name, css.getPropertyValue(name));
|
||||
canvas.style.setProperty("animation", "none", "important");
|
||||
canvas.style.setProperty("transition", "none", "important");
|
||||
canvas.style.setProperty("pointer-events", "none", "important");
|
||||
canvas.setAttribute("data-layout-ignore", "");
|
||||
canvas.setAttribute("aria-hidden", "true");
|
||||
video.before(canvas);
|
||||
video.setAttribute(HELD_VIDEO_ATTR, "");
|
||||
return () => {
|
||||
video.removeAttribute(HELD_VIDEO_ATTR);
|
||||
canvas.remove();
|
||||
};
|
||||
}
|
||||
|
||||
function holdVideoFrames(videos: HTMLVideoElement[]): () => void {
|
||||
const restore = videos.map(holdVideoFrame);
|
||||
const style = document.createElement("style");
|
||||
style.textContent = `[${HELD_VIDEO_ATTR}] { display: none !important; }`;
|
||||
document.head.appendChild(style);
|
||||
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
|
||||
return () => {
|
||||
for (const putBack of restore) putBack?.();
|
||||
style.remove();
|
||||
document.documentElement.removeAttribute(PREVIEW_SEEK_HOLD_ATTR);
|
||||
};
|
||||
}
|
||||
|
||||
type SeekHost = {
|
||||
clipsAt: (time: number) => RuntimeMediaClip[];
|
||||
videosToHold: () => HTMLVideoElement[];
|
||||
scenesReadyAt: (time: number) => boolean;
|
||||
prepare: (clips: RuntimeMediaClip[], time: number) => void;
|
||||
commit: (time: number) => void;
|
||||
};
|
||||
|
||||
/** One decoder job at a time; requests arriving during it replace the queued destination. */
|
||||
export function createPreviewSeek(host: SeekHost) {
|
||||
let pending: { time: number; done: Promise<void>; resolve: () => void } | null = null;
|
||||
let releasePicture = () => {};
|
||||
let stopWaiting = () => {};
|
||||
let runningClips: RuntimeMediaClip[] = [];
|
||||
let runId = 0;
|
||||
|
||||
const ready = (clips: RuntimeMediaClip[], time: number) =>
|
||||
host.scenesReadyAt(time) &&
|
||||
clips.every((clip) => {
|
||||
const { el } = clip;
|
||||
return (
|
||||
!el.isConnected ||
|
||||
isUnplayable(el) ||
|
||||
(el.readyState >= 2 &&
|
||||
!el.seeking &&
|
||||
Math.abs(el.currentTime - mediaSourceTimeAt(clip, time)) < 0.02)
|
||||
);
|
||||
});
|
||||
|
||||
const cancel = () => {
|
||||
const request = pending;
|
||||
pending = null;
|
||||
runId++;
|
||||
stopWaiting();
|
||||
stopWaiting = () => {};
|
||||
releasePicture();
|
||||
releasePicture = () => {};
|
||||
request?.resolve();
|
||||
};
|
||||
|
||||
const run = () => {
|
||||
const request = pending;
|
||||
if (!request) return;
|
||||
const id = ++runId;
|
||||
const time = request.time;
|
||||
const clips = host.clipsAt(time);
|
||||
runningClips = clips;
|
||||
let preparing = true;
|
||||
let finished = false;
|
||||
const finish = () => {
|
||||
if (finished || pending !== request || id !== runId) return;
|
||||
finished = true;
|
||||
stopWaiting();
|
||||
stopWaiting = () => {};
|
||||
if (request.time !== time) {
|
||||
if (ready(host.clipsAt(time), time)) {
|
||||
releasePicture();
|
||||
host.commit(time);
|
||||
releasePicture = holdVideoFrames(host.videosToHold());
|
||||
}
|
||||
run();
|
||||
return;
|
||||
}
|
||||
cancel();
|
||||
host.commit(time);
|
||||
};
|
||||
const check = () => {
|
||||
if (!preparing && ready(host.clipsAt(time), time)) finish();
|
||||
};
|
||||
const metadata = () => {
|
||||
if (pending !== request || finished || id !== runId) return;
|
||||
// A source/proxy change resets currentTime; the parked clock still describes the old picture.
|
||||
preparing = true;
|
||||
host.prepare(host.clipsAt(time), time);
|
||||
preparing = false;
|
||||
check();
|
||||
};
|
||||
const listeners = clips.map(({ el }) => {
|
||||
for (const type of EVENTS)
|
||||
el.addEventListener(type, type === "loadedmetadata" ? metadata : check, type === "error");
|
||||
return () => {
|
||||
for (const type of EVENTS)
|
||||
el.removeEventListener(
|
||||
type,
|
||||
type === "loadedmetadata" ? metadata : check,
|
||||
type === "error",
|
||||
);
|
||||
};
|
||||
});
|
||||
const cap = wallSetTimeout(finish, SEEK_CAP_MS);
|
||||
stopWaiting = () => {
|
||||
wallClearTimeout(cap);
|
||||
for (const remove of listeners) remove();
|
||||
};
|
||||
host.prepare(clips, time);
|
||||
preparing = false;
|
||||
check();
|
||||
};
|
||||
|
||||
return {
|
||||
get time() {
|
||||
return pending?.time;
|
||||
},
|
||||
seek(time: number): Promise<void> | void {
|
||||
if (pending) {
|
||||
const request = pending;
|
||||
request.time = time;
|
||||
const clips = host.clipsAt(time);
|
||||
if (
|
||||
clips.length !== runningClips.length ||
|
||||
clips.some((clip, index) => clip.el !== runningClips[index]?.el)
|
||||
) {
|
||||
stopWaiting();
|
||||
run();
|
||||
}
|
||||
return request.done;
|
||||
}
|
||||
if (ready(host.clipsAt(time), time)) {
|
||||
host.commit(time);
|
||||
return;
|
||||
}
|
||||
let resolve = () => {};
|
||||
const done = new Promise<void>((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
pending = { time, done, resolve };
|
||||
releasePicture = holdVideoFrames(host.videosToHold());
|
||||
run();
|
||||
return done;
|
||||
},
|
||||
refresh() {
|
||||
if (!pending) return;
|
||||
stopWaiting();
|
||||
run();
|
||||
},
|
||||
cancel,
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { isWebKit } from "./runtime/mediaClock.js";
|
||||
|
||||
export const STUDIO_PREVIEW_MARK_META = "hyperframes-studio-preview";
|
||||
export const PREVIEW_SEEK_HOLD_ATTR = "data-hf-preview-seek-hold";
|
||||
export const STUDIO_PREVIEW_CAPTURE_MODE = "capture";
|
||||
export const PLAYER_FRAME_ATTR = "data-hf-player-frame";
|
||||
|
||||
|
||||
Reference in New Issue
Block a user