fix(runtime): prepare trimmed video frames before preview cuts (#5633)

packages/core/src/runtime/init.ts

@@ -53,6 +53,7 @@ import {
   resolveNaturalMediaTimelineDuration,
   resolveRuntimeMediaClipDuration,
   syncRuntimeMedia,
+  VIDEO_PRESEEK_SECONDS,
   type RuntimeMediaClip,
 } from "./media";
 import {
@@ -3041,6 +3042,17 @@ export function initSandboxRuntimeModular(): void {
     for (const clip of clipsWithEndpointBetween(index.byEnd, (c) => c.end, lo, hi)) {
       visiting.add(clip.el);
     }
+    if (state.isPlaying && !window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
+      // An endpoint-only pass reaches trimmed video too late: its first visible tick starts a seek.
+      for (const clip of clipsWithEndpointBetween(
+        index.byStart,
+        (c) => c.start,
+        toSeconds,
+        toSeconds + VIDEO_PRESEEK_SECONDS * state.playbackRate,
+      )) {
+        if (clip.el.tagName === "VIDEO") visiting.add(clip.el);
+      }
+    }
     return [...visiting];
   };
 

packages/core/src/runtime/init.videoPreseek.test.ts

@@ -0,0 +1,60 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { initSandboxRuntimeModular } from "./init";
+import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
+
+it("prepares a hidden video when playback enters its lookahead, after metadata arrives", () => {
+  resetRuntimeFixtureDom();
+  let now = 0;
+  let id = 0;
+  const callbacks = new Map<number, FrameRequestCallback>();
+  vi.spyOn(performance, "now").mockImplementation(() => now);
+  vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) => {
+    callbacks.set(++id, callback);
+    return id;
+  });
+  vi.spyOn(window, "cancelAnimationFrame").mockImplementation((key) => {
+    callbacks.delete(key);
+  });
+  const step = () => {
+    now += 100;
+    const pending = [...callbacks.values()];
+    callbacks.clear();
+    for (const callback of pending) callback(now);
+  };
+  document.body.innerHTML =
+    '<div data-composition-id="main" data-root="true" data-duration="6"><video id="incoming" data-start="4" data-duration="2" data-media-start="8" src="/movie.mp4"></video></div>';
+  const video = document.querySelector("video")!;
+  Object.defineProperty(video, "duration", { value: 20, configurable: true });
+  Object.defineProperty(video, "readyState", { value: 0, configurable: true });
+  Object.defineProperty(video, "paused", { value: true, writable: true, configurable: true });
+  const play = vi.fn(() => {
+    Object.assign(video, { paused: false });
+    return Promise.resolve();
+  });
+  video.play = play;
+  video.pause = vi.fn(() => {
+    Object.assign(video, { paused: true });
+  });
+  video.load = vi.fn();
+  const seek = vi.spyOn(video, "currentTime", "set");
+  window.__timelines = { main: createMockTimeline(6) };
+  initSandboxRuntimeModular();
+  window.__player!.play();
+  for (let i = 0; i < 28; i++) step();
+  expect(seek).not.toHaveBeenCalled();
+  Object.defineProperty(video, "readyState", { value: 4, configurable: true });
+  step();
+  expect(video.style.visibility).toBe("hidden");
+  expect(video.currentTime).toBe(8);
+  expect(play).not.toHaveBeenCalled();
+  for (let i = 0; i < 11; i++) step();
+  expect(video.style.visibility).toBe("visible");
+  expect(seek).toHaveBeenCalledTimes(1);
+  expect(play).toHaveBeenCalledTimes(1);
+});
+
+afterEach(() => {
+  window.__hfRuntimeTeardown?.();
+  vi.restoreAllMocks();
+  resetRuntimeFixtureDom();
+});

packages/core/src/runtime/media.preseek.test.ts

@@ -0,0 +1,101 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";
+
+function fixture(overrides: Partial<RuntimeMediaClip> = {}, tag = "video") {
+  const el = document.createElement(tag) as HTMLMediaElement;
+  document.body.appendChild(el);
+  Object.defineProperty(el, "readyState", { value: 4, configurable: true });
+  el.play = vi.fn(() => Promise.resolve());
+  el.pause = vi.fn();
+  const seek = vi.spyOn(el, "currentTime", "set");
+  const clip: RuntimeMediaClip = {
+    el,
+    start: 4,
+    end: 6,
+    duration: 2,
+    mediaStart: 8,
+    playbackRate: 1,
+    volume: null,
+    loop: false,
+    sourceDuration: 20,
+    ...overrides,
+  };
+  const tick = (timeSeconds: number, playing = true, playbackRate = 1) =>
+    syncRuntimeMedia({
+      clips: [clip],
+      timeSeconds,
+      playing,
+      playbackRate,
+      getCompositionDuration: () => 10,
+    });
+  return { el, clip, seek, tick };
+}
+
+afterEach(() => {
+  document.body.innerHTML = "";
+  delete window.__HF_EXPORT_RENDER_SEEK_CONFIG;
+  vi.restoreAllMocks();
+});
+
+describe("hidden incoming video preparation", () => {
+  it("lands the in-point before the cut without playing or seeking it again at activation", () => {
+    const { el, seek, tick } = fixture();
+    tick(0);
+    expect(seek).not.toHaveBeenCalled();
+    tick(2.5);
+    expect(el.currentTime).toBe(8);
+    expect(el.play).not.toHaveBeenCalled();
+    tick(3);
+    tick(4);
+    expect(seek).toHaveBeenCalledTimes(1);
+    expect(el.play).toHaveBeenCalledTimes(1);
+  });
+
+  it("retries after metadata arrives without interrupting an outstanding seek", () => {
+    const { el, seek, tick } = fixture();
+    Object.defineProperty(el, "readyState", { value: 0, configurable: true });
+    tick(2.5);
+    expect(seek).not.toHaveBeenCalled();
+    Object.defineProperty(el, "readyState", { value: 1, configurable: true });
+    Object.defineProperty(el, "seeking", { value: true, configurable: true });
+    tick(2.6);
+    expect(seek).not.toHaveBeenCalled();
+    Object.defineProperty(el, "seeking", { value: false, configurable: true });
+    tick(2.7);
+    expect(el.currentTime).toBe(8);
+  });
+
+  it.each([
+    { origin: 2, playbackRate: 2, expected: 12 },
+    { origin: 2, playbackRate: 2, loop: true, sourceDuration: 10, expected: 8 },
+    { origin: 2, playbackRate: 2, sourceDuration: 10, expected: 10 },
+  ])("prepares the activation source position for %j", ({ expected, ...fields }) => {
+    const { el, seek, tick } = fixture(fields);
+    tick(3);
+    expect(el.currentTime).toBe(expected);
+    tick(4);
+    expect(seek).toHaveBeenCalledTimes(1);
+  });
+
+  it("uses wall-time lookahead at a faster transport rate", () => {
+    const { el, tick } = fixture();
+    tick(1, true, 2);
+    expect(el.currentTime).toBe(8);
+  });
+
+  it.each(["audio", "paused", "past", "export", "injected", "playing", "disconnected"])(
+    "leaves %s media untouched",
+    (kind) => {
+      const { el, tick, seek } = fixture({}, kind === "audio" ? "audio" : "video");
+      if (kind === "export") window.__HF_EXPORT_RENDER_SEEK_CONFIG = {};
+      if (kind === "injected") {
+        el.id = "incoming";
+        el.insertAdjacentHTML("afterend", '<img id="__render_frame_incoming__">');
+      }
+      if (kind === "playing") Object.defineProperty(el, "paused", { value: false });
+      if (kind === "disconnected") el.remove();
+      tick(kind === "past" ? 7 : 3, kind !== "paused");
+      expect(seek).not.toHaveBeenCalled();
+    },
+  );
+});

packages/core/src/runtime/media.ts

@@ -263,6 +263,46 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
 }
 
 export const MEDIA_HARD_SYNC_SECONDS = 0.5;
+export const VIDEO_PRESEEK_SECONDS = 1.5;
+
+function prepareUpcomingVideo(
+  clip: RuntimeMediaClip,
+  time: number,
+  playing: boolean,
+  rate: number,
+): void {
+  const { el } = clip;
+  const dueIn = clip.start - time;
+  if (
+    !playing ||
+    el.tagName !== "VIDEO" ||
+    dueIn <= 0 ||
+    dueIn > VIDEO_PRESEEK_SECONDS * rate ||
+    !el.paused ||
+    el.seeking ||
+    el.readyState < el.HAVE_METADATA ||
+    findInjectedRenderFrame(el) ||
+    ("__HF_EXPORT_RENDER_SEEK_CONFIG" in window && window.__HF_EXPORT_RENDER_SEEK_CONFIG)
+  )
+    return;
+  let target =
+    clip.mediaStart +
+    sourceTimeAt(clip.rate ?? clip.playbackRate, clip.start - (clip.origin ?? clip.start));
+  if (clip.sourceDuration != null) {
+    const loopLength = clip.sourceDuration - clip.mediaStart;
+    if (clip.loop && loopLength > 0 && target >= clip.sourceDuration)
+      target = clip.mediaStart + ((target - clip.mediaStart) % loopLength);
+    else if (!clip.loop) target = Math.min(target, clip.sourceDuration);
+  }
+  if (!Number.isFinite(target) || target < 0 || Math.abs(el.currentTime - target) <= 0.001) return;
+  try {
+    // A trimmed video exposed during its in-point seek has no current frame at the cut.
+    // Hidden preparation must not hold the visible-frame seek barrier.
+    el.currentTime = target;
+  } catch (err) {
+    swallow("runtime.media.preseek", err);
+  }
+}
 
 /** Drift a playing audio element may carry before sync pulls it back onto the playhead. */
 export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
@@ -657,5 +697,6 @@ export function syncRuntimeMedia(params: {
     evictMediaSyncState(el);
     if (remainsInsideAuthoredWindow) lastRelativeTime.set(el, relTime);
     if (!el.paused) el.pause();
+    prepareUpcomingVideo(clip, params.timeSeconds, params.playing, params.playbackRate);
   }
 }