diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad48d6b88..fcb599c921 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1009,6 +1009,8 @@ jobs: packages/studio/data/projects/undo-socket-pool cp -R packages/studio/tests/e2e/fixtures/aroll-backdrop \ packages/studio/data/projects/aroll-backdrop + cp -R packages/studio/tests/e2e/fixtures/preview-video-regression \ + packages/studio/data/projects/preview-video-regression - name: Build the base the viewport gate compares against # The first parent of the checked-out commit, as in base-file.sh. Measured only when the head misses a budget. run: | @@ -1181,7 +1183,7 @@ jobs: - name: Install ffmpeg for the paused-video fixture if: success() || failure() uses: ./.github/actions/install-ffmpeg-linux - - name: Undo while paused videos hold the preview's sockets + - name: Verify paused-video sockets, picture quality and native recovery # Counts the undo's wait for a socket, not its total time: unpinned it is 0 ms. if: success() || failure() run: | @@ -1200,6 +1202,12 @@ jobs: STUDIO_URL="http://localhost:5318/#project/undo-socket-pool" \ STUDIO_PROJECT_DIR="$PWD/packages/studio/data/projects/undo-socket-pool" \ node packages/studio/tests/e2e/undo-socket-pool.mjs | tee /tmp/undo-socket-pool.json + node packages/studio/tests/e2e/fixtures/preview-video-regression/generate.mjs \ + packages/studio/data/projects/preview-video-regression + STUDIO_URL="http://localhost:5318/#project/preview-video-regression" \ + STUDIO_PROJECT_DIR="$PWD/packages/studio/data/projects/preview-video-regression" \ + OUT_DIR=/tmp/preview-video-regression \ + node packages/studio/tests/e2e/preview-video-regression.mjs - name: Upload gate evidence # The gate's whole output is machine-readable evidence, and a red run is # exactly when someone needs to read it. Keep it on failure too. @@ -1212,6 +1220,7 @@ jobs: /tmp/studio-open-counts.json /tmp/inline-text-caret/*.png /tmp/undo-socket-pool.json + /tmp/preview-video-regression/ if-no-files-found: error # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json. A pull request diff --git a/docs/packages/studio.mdx b/docs/packages/studio.mdx index a5eed86bea..561b4298f9 100644 --- a/docs/packages/studio.mdx +++ b/docs/packages/studio.mdx @@ -121,9 +121,11 @@ A paused fallback that outlasts its wait retains the presented clock and picture while paused sync continues. It can still publish when the video becomes ready; a new gesture cancels that recovery. Studio requests paused-decoder release before preparing native playback and reopens readers when paused scrubbing resumes. -Transparent video queries drain their alpha merges before input disposal. If one -never settles, its cleanup can outlast the Play deadline; the remaining playback -still starts while that video's last picture stays visible. +Paused-reader release waits up to two seconds for transparent video queries to +drain. After that it cancels their downloads and lets playback continue while +unfinished merges close in the background; input disposal still waits for the +merge to finish. A recovering video more than half a second out of sync with the transport +gets at most one corrective seek per recovery registration before its retained picture is removed. Scrubbing and paused frame steps decode the original video at source resolution throughout. Automatic playback quality reductions do not reduce these frames, and the frame decoder does not substitute re-encoded seek copies. Playback diff --git a/packages/core/src/runtime/init.previewSeek.test.ts b/packages/core/src/runtime/init.previewSeek.test.ts index 47bf154215..f8071342e1 100644 --- a/packages/core/src/runtime/init.previewSeek.test.ts +++ b/packages/core/src/runtime/init.previewSeek.test.ts @@ -3,7 +3,11 @@ import { initSandboxRuntimeModular } from "./init"; import { hideTimedClipsUntilFirstPass } from "./timedClipHide"; import { resetSeekDispatchState } from "./adapters/seek-dispatch"; import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers"; -import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark"; +import { + PREVIEW_SEEK_HOLD_ATTR, + STUDIO_PREVIEW_MARK_META, + PLAYER_FRAME_ATTR, +} from "../studioPreviewMark"; import type { PausedVideoFrame, PausedVideoFrameProvider, @@ -65,11 +69,11 @@ function decoder(video: HTMLVideoElement, initiallyReady: boolean) { }; } -function film(withImage = false) { +function film(withImage = false, studio = true) { resetRuntimeFixtureDom(); const meta = document.createElement("meta"); meta.name = STUDIO_PREVIEW_MARK_META; - document.head.append(meta); + if (studio) document.head.append(meta); vi.spyOn(window, "requestAnimationFrame").mockReturnValue(0); const drawImage = vi.fn(); vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ @@ -301,22 +305,24 @@ it("leaves a grading canvas's hidden source in its layout during the hold", asyn }); it.each(["play", "render", "teardown"])( - "cleans up a pending seek on %s and ignores late completions", + "finishes native handoff on Play and ignores retired completions on render or teardown (%s)", async (action) => { const { main, next } = film(); const player = window.__player!; const landed = player.seek(6); - if (action === "play") player.play(); - else if (action === "render") player.renderSeek(2); - else window.__hfRuntimeTeardown?.(); - const time = main.time?.(); + const cases = { + play: { run: () => player.play(), before: 0, after: 6 }, + render: { run: () => player.renderSeek(2), before: 2, after: 2 }, + teardown: { run: () => window.__hfRuntimeTeardown?.(), before: 0, after: 0 }, + }; + const expected = cases[action as keyof typeof cases]; + expected.run(); + expect(main.time!()).toBe(expected.before); for (const video of next) video.land(); await landed; - expect(main.time?.()).toBe(time); + expect(main.time!()).toBe(expected.after); expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false); expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0); - if (action === "play") expect(player.getTime()).toBeGreaterThanOrEqual(6); - if (action === "render") expect(time).toBe(2); }, ); @@ -508,6 +514,53 @@ it("reveals a delayed playing frame without chasing the moving clock with more s expect(window.__player!.isPlaying()).toBe(true); }); +it.each([1500, 2500].flatMap((lateMs) => [25, 40, 100, 600].map((seekMs) => ({ lateMs, seekMs }))))( + "aligns a recovered picture once after $lateMs ms with $seekMs ms seek latency", + async ({ lateMs, seekMs }) => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] }); + const { old, next } = film(); + const { pending } = frameProvider([...old, ...next]); + const initial = window.__player!.seek(1); + pending.shift()!(); + await initial; + const stalled = old[0]!; + stalled.metadataAt(1); + let error: { code: number } | null = { code: 4 }; + Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error }); + window.__player!.play(); + old[1]!.land(); + await vi.advanceTimersByTimeAsync(0); + error = null; + const seeks = stalled.seeks.length; + for (let elapsed = 0; elapsed < lateMs; elapsed += 25) { + await vi.advanceTimersByTimeAsync(25); + window.__player!.getTime(); + } + expect(window.__player!.getTime()).toBeCloseTo(1 + lateMs / 1000, 2); + Object.defineProperty(stalled.video, "paused", { configurable: true, value: false }); + stalled.land(); + stalled.video.dispatchEvent(new Event("playing")); + stalled.presentAt(1 + 1 / 30); + expect(stalled.seeks).toHaveLength(seeks + 1); + const target = window.__player!.getTime(); + expect(stalled.seeks.at(-1)).toBeCloseTo(target, 2); + expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(1); + for (let elapsed = 0; elapsed < seekMs; elapsed += 25) { + await vi.advanceTimersByTimeAsync(Math.min(25, seekMs - elapsed)); + window.__player!.getTime(); + } + expect(window.__player!.getTime() - target).toBeCloseTo(seekMs / 1000, 2); + stalled.land(); + stalled.presentAt(target + 1 / 30); + expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0); + expect(stalled.seeks).toHaveLength(seeks + 1); + expect(Math.abs(stalled.video.currentTime - window.__player!.getTime())).toBeLessThan( + Math.max(0.1, seekMs / 1000), + ); + expect(window.__player!.isPlaying()).toBe(true); + }, +); + it("keeps a paused recovery covered until its static target seek lands", async () => { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] }); const { old, next } = film(); @@ -901,3 +954,58 @@ it("releases a native hold when stop-media cancels its first provider seek", asy await Promise.resolve(); expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0); }); + +it("keeps the native-only complete picture when Play interrupts an unresolved scrub", async () => { + const { main, next } = film(); + const seek = window.__player!.seek(6); + const pictures = [...document.querySelectorAll("canvas")]; + expect(pictures).toHaveLength(2); + window.__player!.play(); + await seek; + expect(main.time?.()).toBe(0); + expect(pictures.every((picture) => picture.isConnected)).toBe(true); + expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true); + next[0]!.land(); + expect(main.time?.()).toBe(0); + next[1]!.land(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(main.time?.()).toBe(6); + expect(window.__player!.isPlaying()).toBe(true); + expect(window.__player!.getTime()).toBeGreaterThanOrEqual(6); + expect(document.querySelectorAll("canvas")).toHaveLength(0); +}); + +function safariPlayerFilm() { + vi.spyOn(navigator, "userAgent", "get").mockReturnValue( + "AppleWebKit/605.1.15 Version/18.0 Safari/605.1.15", + ); + const frame = document.createElement("iframe"); + frame.setAttribute(PLAYER_FRAME_ATTR, ""); + vi.spyOn(window, "frameElement", "get").mockReturnValue(frame); + return film(false, false); +} + +it("keeps pending-seek Play synchronous in an unmarked Safari player frame", () => { + const { main, next } = safariPlayerFilm(); + void window.__player!.seek(6); + expect(main.time?.()).toBe(0); + window.__player!.play(); + expect(main.time?.()).toBe(6); + expect(next.some(({ video }) => vi.mocked(video.play).mock.calls.length > 0)).toBe(true); +}); + +it("retains an explicit unmarked provider handoff before its first picture", async () => { + const { main, old, next } = safariPlayerFilm(); + for (const video of old) video.metadataAt(0); + const { pending } = frameProvider([...old, ...next]); + const seeking = window.__player!.seek(6); + window.__player!.play(); + expect(main.time?.()).toBe(0); + expect(window.__player!.isPlaying()).toBe(false); + for (const video of next) video.land(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(window.__player!.isPlaying()).toBe(true); + pending.shift()!(); + await seeking; + expect(main.time?.()).toBe(6); +}); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 85238cef45..35d28eb7ab 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -51,6 +51,7 @@ import { } from "./timedClipHide"; import { inPlayerFrame, + STUDIO_PREVIEW_MARK_META, PREVIEW_ONLY_ATTRS, RUNTIME_STAMPED_ATTR, STUDIO_PREVIEW_FAR_SCENE_ATTR as FAR_SCENE_ATTR, @@ -2997,6 +2998,7 @@ export function initSandboxRuntimeModular(): void { }); const hiddenImagesSkipped = skipsHiddenImages(); const mediaDeferred = defersMedia(); + const studioPreview = document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null; const skipFarScenes = createFarSceneSkip((host) => runtimeElementWindow(host, { timelineRegistry: window.__timelines ?? {}, @@ -5062,17 +5064,22 @@ export function initSandboxRuntimeModular(): void { }, suspend: pausedFrames.suspendProvider, prepareNative: pausedFrames.prepareNative, - clear: (keep) => + clear: (keep) => { + const corrected = new WeakSet(); pausedFrames.clear(keep, (video, presented) => { - if (clock.isPlaying()) return presented; + const playing = clock.isPlaying(); + if (playing && !presented) return false; const time = clock.now(); const clip = previewVideoClipsAt(time).find(({ el }) => el === video); if (!clip) return true; const expected = mediaSourceTimeAt(clip, time); - if (Math.abs(video.currentTime - expected) < 0.02) return true; + if (playing && corrected.has(video)) return true; + if (Math.abs(video.currentTime - expected) < (playing ? 0.5 : 0.02)) return true; + if (playing) corrected.add(video); video.currentTime = mediaElementTime(expected); return false; - }), + }); + }, }, videosToHold: () => withTimingResolver(() => @@ -5162,7 +5169,8 @@ export function initSandboxRuntimeModular(): void { play: () => { flushHeldSeek(); if ( - (pausedFrames.active || (pausedFrames.available() && previewSeek?.time !== undefined)) && + (pausedFrames.active || + ((pausedFrames.available() || studioPreview) && previewSeek?.time !== undefined)) && !clock.isPlaying() ) { wallClearTimeout(nativeWarmTimer); diff --git a/packages/core/src/runtime/pausedVideoCallbacks.test.ts b/packages/core/src/runtime/pausedVideoCallbacks.test.ts index df15852e4a..2ff6931565 100644 --- a/packages/core/src/runtime/pausedVideoCallbacks.test.ts +++ b/packages/core/src/runtime/pausedVideoCallbacks.test.ts @@ -327,3 +327,25 @@ it("keeps waiting when no native frame or playback progress arrives", () => { stop(); expect(pending.size).toBe(0); }); + +it("does not reveal a paused seek from timeupdate after playback had started", () => { + const video = document.createElement("video"); + let paused = false; + let time = 3; + Object.defineProperties(video, { + paused: { get: () => paused }, + currentTime: { get: () => time }, + }); + const recovered = vi.fn(() => true); + const stop = observeNativeVideoFrames(video, recovered); + paused = true; + time = 4; + video.dispatchEvent(new Event("timeupdate")); + expect(recovered).not.toHaveBeenCalled(); + paused = false; + video.dispatchEvent(new Event("playing")); + time += 1 / 30; + video.dispatchEvent(new Event("timeupdate")); + expect(recovered).toHaveBeenCalledOnce(); + stop(); +}); diff --git a/packages/core/src/runtime/pausedVideoFrames.test.ts b/packages/core/src/runtime/pausedVideoFrames.test.ts index 74cc645b51..8cf7bfd888 100644 --- a/packages/core/src/runtime/pausedVideoFrames.test.ts +++ b/packages/core/src/runtime/pausedVideoFrames.test.ts @@ -254,3 +254,16 @@ it("cancels a retained native observer when a new paused picture replaces it", a else Reflect.deleteProperty(proto, "cancelVideoFrameCallback"); } }); + +it("does not retire an adopted decoded picture when its native snapshot lease ends", async () => { + const { session, video, request } = fixture(); + Object.defineProperty(video, "readyState", { value: 4 }); + const stopHold = session.holdNative([video]); + const snapshot = getPausedVideoFrame(video); + const decoded = await session.prepare(request, new AbortController().signal); + expect(decoded!.present()).toBe(true); + stopHold(); + expect(getPausedVideoFrame(video)).toBe(snapshot); + expect(ownsPausedVideoTime(video)).toBe(true); + expect(video.currentTime).toBe(3); +}); diff --git a/packages/core/src/runtime/previewSeek.frames.test.ts b/packages/core/src/runtime/previewSeek.frames.test.ts index fe886d9499..db11b1e8d1 100644 --- a/packages/core/src/runtime/previewSeek.frames.test.ts +++ b/packages/core/src/runtime/previewSeek.frames.test.ts @@ -224,3 +224,16 @@ it("preserves the held native picture when a new gesture cancels expired backgro await next; expect(document.querySelector("canvas")).toBeNull(); }); + +it("retires the hard Play timer after native readiness publishes", async () => { + vi.useFakeTimers(); + const { job, video, commit } = fixture(); + Object.defineProperties(video, { + readyState: { value: 4 }, + seeking: { value: false }, + currentTime: { value: 3 }, + }); + await job.handoff(3); + expect(commit).toHaveBeenCalledWith(3); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/packages/studio/src/player/lib/pausedVideoDecoder.test.ts b/packages/studio/src/player/lib/pausedVideoDecoder.test.ts index caa28ee5ef..dd180b4594 100644 --- a/packages/studio/src/player/lib/pausedVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/pausedVideoDecoder.test.ts @@ -113,7 +113,7 @@ beforeEach(() => { canvasContext.mockReset().mockReturnValue({ clearRect: vi.fn(), imageSmoothingQuality: "low" }); vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockImplementation(canvasContext); createBitmap.mockReset().mockImplementation(async () => bitmap()); - cancelReads.mockClear(); + cancelReads.mockReset(); qualityChosenBy = "person"; previewQuality = 1; displayScale = 1; @@ -642,3 +642,76 @@ it("closes the selected alpha sample when the finite iterator fails while draini await expect(owner.decode([request("a")], signal())).rejects.toThrow("Alpha drain failed"); expect(selected.close).toHaveBeenCalledOnce(); }); + +it("bounds alpha suspension without disposing an unfinished merger", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + getTrack.mockImplementation(async (source: string) => ({ + ...track(source), + canBeTransparent: async () => true, + })); + let drain: () => void = () => {}; + const drained = new Promise((resolve) => { + drain = resolve; + }); + const selected = sample(); + finiteSamples.mockImplementationOnce(async function* () { + yield selected; + await drained; + }); + const owner = provider(); + const pending = owner.decode([request("a")], signal()).catch((error: unknown) => error); + await vi.waitFor(() => expect(finiteSamples).toHaveBeenCalledOnce()); + let released = false; + const suspended = owner.suspend!().then(() => { + released = true; + }); + try { + await vi.advanceTimersByTimeAsync(2000); + expect(released).toBe(true); + expect(cancelReads).toHaveBeenCalledOnce(); + expect(inputs[0]!.dispose).not.toHaveBeenCalled(); + expect(selected.close).not.toHaveBeenCalled(); + const [next] = await owner.decode([request("a")], signal()); + expect(inputs).toHaveLength(2); + next!.release(); + } finally { + drain(); + await pending; + await suspended; + } + expect(selected.close).toHaveBeenCalledOnce(); + expect(inputs[0]!.dispose).toHaveBeenCalledOnce(); + expect(inputs[1]!.dispose).not.toHaveBeenCalled(); +}); + +it("closes the selected alpha sample when suspension cancels an in-flight read", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + getTrack.mockImplementation(async (source: string) => ({ + ...track(source), + canBeTransparent: async () => true, + })); + const selected = sample(); + let rejectRead: (error: Error) => void = () => {}; + const reading = new Promise((_resolve, reject) => { + rejectRead = reject; + }); + finiteSamples.mockImplementationOnce(async function* () { + yield selected; + await reading; + }); + cancelReads.mockImplementation(() => + rejectRead(new DOMException("Network read cancelled", "AbortError")), + ); + const owner = provider(); + const decoding = owner.decode([request("a")], signal()).catch((error: unknown) => error); + await vi.waitFor(() => expect(finiteSamples).toHaveBeenCalledOnce()); + const suspended = owner.suspend!(); + await vi.advanceTimersByTimeAsync(1999); + expect(cancelReads).not.toHaveBeenCalled(); + expect(inputs[0]!.dispose).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + await suspended; + expect(await decoding).toMatchObject({ name: "AbortError" }); + expect(selected.close).toHaveBeenCalledOnce(); + expect(inputs[0]!.dispose).toHaveBeenCalledOnce(); +}); diff --git a/packages/studio/src/player/lib/pausedVideoDecoder.ts b/packages/studio/src/player/lib/pausedVideoDecoder.ts index c79a5e5901..735cc9d229 100644 --- a/packages/studio/src/player/lib/pausedVideoDecoder.ts +++ b/packages/studio/src/player/lib/pausedVideoDecoder.ts @@ -22,6 +22,7 @@ import { const FRAME_BUDGET = 64 * 1024 * 1024; const MAX_READERS = 4; const NEARBY_FORWARD_SECONDS = 0.5; +const ALPHA_DRAIN_WAIT_MS = 2000; type Cached = { image: ImageBitmap; @@ -104,18 +105,28 @@ export function createPausedVideoDecoder(iframe: HTMLIFrameElement): PausedVideo reader.disposed = true; clearTimeout(reader.idleTimer); if (opened.get(reader.key) === reader) opened.delete(reader.key); - // Input disposal must not cut off an in-flight alpha merger. - if (reader.transparent) await reader.tail; - reader.cancelReads(); - reader.input.dispose(); - await closeIterator(reader).catch(() => {}); - for (const entry of reader.cache.values()) { - cacheOrder.delete(entry); - bytes -= entry.bytes; - entry.image.close(); - } - reader.cache.clear(); - reader.canvas.width = reader.canvas.height = 0; + const closing = (async () => { + // Input disposal must not cut off an in-flight alpha merger. + if (reader.transparent) await reader.tail; + reader.cancelReads(); + reader.input.dispose(); + await closeIterator(reader).catch(() => {}); + for (const entry of reader.cache.values()) { + cacheOrder.delete(entry); + bytes -= entry.bytes; + entry.image.close(); + } + reader.cache.clear(); + reader.canvas.width = reader.canvas.height = 0; + })(); + if (!reader.transparent) return closing; + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reader.cancelReads(); + resolve(); + }, ALPHA_DRAIN_WAIT_MS); + void closing.then(resolve, reject).finally(() => clearTimeout(timeout)); + }); }; const scheduleIdle = (reader: Reader) => { clearTimeout(reader.idleTimer); diff --git a/packages/studio/tests/e2e/fixtures/preview-video-regression/.gitignore b/packages/studio/tests/e2e/fixtures/preview-video-regression/.gitignore new file mode 100644 index 0000000000..ee889666ef --- /dev/null +++ b/packages/studio/tests/e2e/fixtures/preview-video-regression/.gitignore @@ -0,0 +1 @@ +assets/ diff --git a/packages/studio/tests/e2e/fixtures/preview-video-regression/generate.mjs b/packages/studio/tests/e2e/fixtures/preview-video-regression/generate.mjs new file mode 100644 index 0000000000..8aee8f6c3f --- /dev/null +++ b/packages/studio/tests/e2e/fixtures/preview-video-regression/generate.mjs @@ -0,0 +1,164 @@ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdir, copyFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { resolve, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = resolve(process.argv[2] ?? fileURLToPath(new URL(".", import.meta.url))); +const assets = join(root, "assets"); +const width = 640; +const height = 360; +await mkdir(assets, { recursive: true }); +await copyFile( + createRequire(import.meta.url).resolve("gsap/dist/gsap.min.js"), + join(assets, "gsap.min.js"), +); + +function codecOptions(alpha) { + return alpha + ? [ + "-c:v", + "libvpx-vp9", + "-lossless", + "1", + "-pix_fmt", + "yuva420p", + "-auto-alt-ref", + "0", + "-deadline", + "realtime", + "-cpu-used", + "8", + ] + : [ + "-c:v", + "libx264", + "-crf", + "10", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + ]; +} +function opaqueBase(base, color) { + for (let i = 0; i < base.length; i += 4) { + base[i] = color[0]; + base[i + 1] = color[1]; + base[i + 2] = color[2]; + base[i + 3] = 255; + } +} +function alphaBase(base) { + for (let y = 100; y < 220; y++) + for (let x = 80; x < 200; x++) { + const i = (y * width + x) * 4; + base[i + 1] = 208; + base[i + 2] = 96; + base[i + 3] = 255; + } +} +function baseFrame(color) { + const base = Buffer.alloc(width * height * 4); + if (color) opaqueBase(base, color); + else alphaBase(base); + return base; +} +function checker(pixels, frame) { + for (let y = 20; y < 52; y++) + for (let x = 300; x < 428; x++) { + const i = (y * width + x) * 4; + pixels[i] = pixels[i + 1] = pixels[i + 2] = (x + frame) % 2 ? 255 : 0; + pixels[i + 3] = 255; + } +} +function timeMarker(pixels, frame) { + for (let y = 300; y < 315; y++) { + const left = 20 + Math.floor(frame / 30) * 50; + for (let x = left; x < left + 20; x++) { + const i = (y * width + x) * 4; + pixels[i] = pixels[i + 1] = pixels[i + 2] = 255; + } + } +} +function texture(pixels, frame) { + // A deterministic texture keeps native range reads from buffering the entire fixture at once. + let seed = frame + 1; + for (let y = 0; y < 64; y++) + for (let x = 544; x < 640; x++) { + const i = (y * width + x) * 4; + for (let channel = 0; channel < 3; channel++) { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + pixels[i + channel] = seed >>> 24; + } + } +} +function framePixels(base, alpha, frame) { + const pixels = Buffer.from(base); + if (alpha) checker(pixels, frame); + else { + timeMarker(pixels, frame); + texture(pixels, frame); + } + return pixels; +} +async function writeFrames(stream, base, alpha, frames) { + for (let frame = 0; frame < frames; frame++) { + if (!stream.write(framePixels(base, alpha, frame))) await once(stream, "drain"); + } + stream.end(); +} +async function encode(name, frames, color) { + const alpha = color === null; + const child = spawn( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "rawvideo", + "-pixel_format", + "rgba", + "-video_size", + `${width}x${height}`, + "-framerate", + "30", + "-i", + "pipe:0", + ...codecOptions(alpha), + "-g", + "30", + "-an", + join(assets, name), + ], + { stdio: ["pipe", "ignore", "pipe"] }, + ); + let errors = ""; + child.stderr.on("data", (data) => { + errors += data; + }); + const finished = new Promise((resolveDone, reject) => { + child.once("error", reject); + child.once("exit", (code) => + code === 0 ? resolveDone() : reject(new Error(`ffmpeg ${name}: ${errors}`)), + ); + }); + try { + await writeFrames(child.stdin, baseFrame(color), alpha, frames); + await finished; + } catch (error) { + child.kill("SIGTERM"); + await finished.catch(() => {}); + throw error; + } +} + +await encode("background-a.mp4", 180, [204, 48, 48]); +await encode("background-b.mp4", 180, [40, 102, 221]); +await encode("foreground.webm", 360, null); +console.log(JSON.stringify({ assets, width, height, fps: 30, duration: 12, synthetic: true })); diff --git a/packages/studio/tests/e2e/fixtures/preview-video-regression/hyperframes.json b/packages/studio/tests/e2e/fixtures/preview-video-regression/hyperframes.json new file mode 100644 index 0000000000..5fb1d6d872 --- /dev/null +++ b/packages/studio/tests/e2e/fixtures/preview-video-regression/hyperframes.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://hyperframes.heygen.com/schema/hyperframes.json", + "registry": "https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry", + "paths": { + "blocks": "compositions", + "components": "compositions/components", + "assets": "assets" + } +} diff --git a/packages/studio/tests/e2e/fixtures/preview-video-regression/index.html b/packages/studio/tests/e2e/fixtures/preview-video-regression/index.html new file mode 100644 index 0000000000..4cea63b07b --- /dev/null +++ b/packages/studio/tests/e2e/fixtures/preview-video-regression/index.html @@ -0,0 +1,134 @@ + + + + + Original-source preview regression + + + + +
+ + + +
+ A · 0.000 + +
+
+ B · 0.000 + +
+
+ + + diff --git a/packages/studio/tests/e2e/preview-video-capture-witness.mjs b/packages/studio/tests/e2e/preview-video-capture-witness.mjs new file mode 100644 index 0000000000..98f40835f1 --- /dev/null +++ b/packages/studio/tests/e2e/preview-video-capture-witness.mjs @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; + +async function replaceVisibleHtml(host, scene, time) { + return host.frame.evaluate( + ({ scene, time }) => { + const label = document.getElementById("label-a"); + const strip = label.querySelector(".clock-bits"); + const clock = label.querySelector(".clock"); + const saved = { + text: clock.textContent, + prefix: label.firstChild.textContent, + bits: strip.innerHTML, + }; + label.firstChild.textContent = `${scene} · `; + clock.textContent = time.toFixed(3); + strip.children[2].style.background = scene === "B" ? "#fff" : "#000"; + for (let i = 3; i < strip.children.length; i++) + strip.children[i].style.background = (time >> (i - 3)) & 1 ? "#fff" : "#000"; + return saved; + }, + { scene, time }, + ); +} + +async function rejectMixedPicture(host, picture, scene, time, expected) { + const saved = await replaceVisibleHtml(host, scene, time); + try { + await assert.rejects(picture(host, `witness-mixed-${scene}`), expected); + } finally { + await host.frame.evaluate((saved) => { + const label = document.getElementById("label-a"); + label.firstChild.textContent = saved.prefix; + label.querySelector(".clock").textContent = saved.text; + label.querySelector(".clock-bits").innerHTML = saved.bits; + }, saved); + } +} + +export async function verifyCaptureIsolation(host, { picture, seek }) { + const original = host.page.screenshot.bind(host.page); + let published = false; + host.page.screenshot = async (options) => { + const png = await original(options); + if (options.encoding === "base64" && !published) { + published = true; + await seek(host, 7); + } + return png; + }; + let captured; + try { + captured = await picture(host, "witness-publish-after-capture"); + } finally { + host.page.screenshot = original; + } + assert(published, "The capture witness did not interleave a publication"); + assert.equal(captured.scene, "A"); + assert.equal(captured.capturedHtml.shownTime, 2); + assert(captured.labels[0].startsWith("B"), "The post-capture DOM did not publish scene B"); + await seek(host, 2); + await rejectMixedPicture(host, picture, "B", 8, /background disagrees with B/); + await rejectMixedPicture(host, picture, "A", 4, /video pixels disagree with the HTML clock/); + return { publishedAfterCapture: true, mixedSceneRejected: true, mixedClockRejected: true }; +} diff --git a/packages/studio/tests/e2e/preview-video-picture.mjs b/packages/studio/tests/e2e/preview-video-picture.mjs new file mode 100644 index 0000000000..7334ab6331 --- /dev/null +++ b/packages/studio/tests/e2e/preview-video-picture.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +async function capturePicture(host, name, output) { + let png; + let attempts = 0; + const bounds = async () => { + const outer = await host.iframe.boundingBox(); + const inner = await host.frame.evaluate(() => { + const box = document.getElementById("film").getBoundingClientRect(); + return { + x: box.x, + y: box.y, + width: box.width, + height: box.height, + viewportWidth: innerWidth, + viewportHeight: innerHeight, + }; + }); + return { + x: outer.x + (inner.x * outer.width) / inner.viewportWidth, + y: outer.y + (inner.y * outer.height) / inner.viewportHeight, + width: (inner.width * outer.width) / inner.viewportWidth, + height: (inner.height * outer.height) / inner.viewportHeight, + }; + }; + for (; attempts < 3; attempts++) { + const before = await bounds(); + const full = await host.page.screenshot({ encoding: "base64", captureBeyondViewport: false }); + png = await host.page.evaluate( + async ({ full, box }) => { + const image = new Image(); + image.src = `data:image/png;base64,${full}`; + await image.decode(); + const canvas = document.createElement("canvas"); + canvas.width = Math.round(box.width); + canvas.height = Math.round(box.height); + canvas + .getContext("2d") + .drawImage(image, box.x, box.y, box.width, box.height, 0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/png").split(",")[1]; + }, + { full, box: before }, + ); + const after = await bounds(); + if (Object.keys(before).every((key) => Math.abs(before[key] - after[key]) < 0.5)) break; + } + assert(attempts < 3, `${name}: composition kept re-fitting during capture`); + await writeFile(join(output, `${name}.png`), Buffer.from(png, "base64")); + return { png, captureAttempts: attempts + 1 }; +} + +async function samplePicture(host, png) { + return host.page.evaluate(async (png) => { + const image = new Image(); + image.src = `data:image/png;base64,${png}`; + await image.decode(); + const canvas = new OffscreenCanvas(image.width, image.height); + const context = canvas.getContext("2d"); + context.drawImage(image, 0, 0); + const sample = (x, y) => { + const data = context.getImageData( + Math.round(image.width * x), + Math.round(image.height * y), + 4, + 4, + ).data; + const sums = [0, 0, 0]; + for (let i = 0; i < data.length; i += 4) for (let c = 0; c < 3; c++) sums[c] += data[i + c]; + return sums.map((sum) => Math.round(sum / 16)); + }; + return { + reference: Array.from({ length: 7 }, (_, i) => sample((406 + i * 32) / 640, 322 / 360)), + background: sample(500 / 640, 200 / 360), + foreground: sample(140 / 640, 160 / 360), + markers: Array.from({ length: 6 }, (_, i) => sample((30 + i * 50) / 640, 307 / 360)), + }; + }, png); +} + +async function pictureState(host) { + return host.frame.evaluate(() => { + const visible = [...document.querySelectorAll(".label")].filter((el) => { + const style = getComputedStyle(el); + return ( + style.display !== "none" && + style.visibility !== "hidden" && + el.getBoundingClientRect().width > 0 + ); + }); + return { + time: window.__player.getTime(), + playing: window.__player.isPlaying(), + shownTime: Number(document.querySelector(".clock").textContent), + labels: visible.map((el) => el.textContent.trim()), + }; + }); +} + +function assertBackground(name, scene, rgb) { + const [red, green, blue] = rgb; + const matches = scene === "A" ? [red > 160, green < 100, blue < 100] : [blue > 170, red < 100]; + assert(matches.every(Boolean), `${name}: background disagrees with ${scene}: ${rgb}`); +} +function assertMarker(name, scene, state, pixels) { + const sourceSecond = Math.min(5, Math.floor(state.shownTime - (scene === "B" ? 6 : 0))); + const visible = pixels.markers.flatMap((rgb, i) => (Math.min(...rgb) > 210 ? [i] : [])); + assert.deepEqual(visible, [sourceSecond], `${name}: video pixels disagree with the HTML clock`); +} +export async function observePicture(host, name, { output, retained = false }) { + const { png, captureAttempts } = await capturePicture(host, name, output); + const pixels = await samplePicture(host, png); + const state = await pictureState(host); + assert.equal(state.labels.length, 1, `${name}: expected exactly one scene label`); + const bits = pixels.reference.map((rgb) => { + assert( + Math.max(...rgb) < 40 || Math.min(...rgb) > 210, + `${name}: HTML clock strip is obscured`, + ); + return Math.min(...rgb) > 210 ? 1 : 0; + }); + assert.deepEqual(bits.slice(0, 2), [1, 0], `${name}: HTML clock strip guards are absent`); + const scene = bits[2] ? "B" : "A"; + const shownSecond = bits.slice(3).reduce((value, bit, index) => value + bit * 2 ** index, 0); + const capturedHtml = { scene, shownSecond, shownTime: shownSecond }; + assertBackground(name, scene, pixels.background); + assert(pixels.foreground[1] > 150, `${name}: foreground green is missing: ${pixels.foreground}`); + assert( + pixels.foreground[0] < 80, + `${name}: foreground contains background red: ${pixels.foreground}`, + ); + if (!retained) assertMarker(name, scene, capturedHtml, pixels); + return { + name, + ...state, + capturedHtml, + scene, + pixels, + markerValidated: !retained, + captureAttempts, + }; +} diff --git a/packages/studio/tests/e2e/preview-video-regression.mjs b/packages/studio/tests/e2e/preview-video-regression.mjs new file mode 100644 index 0000000000..993e17707f --- /dev/null +++ b/packages/studio/tests/e2e/preview-video-regression.mjs @@ -0,0 +1,449 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { observePicture } from "./preview-video-picture.mjs"; +import { verifyCaptureIsolation } from "./preview-video-capture-witness.mjs"; +import { performance } from "node:perf_hooks"; +import { launchStudioChrome } from "./chrome-executable.mjs"; + +const url = process.env.STUDIO_URL; +assert(url, "STUDIO_URL must point at the preview-video-regression fixture"); +const project = process.env.STUDIO_PROJECT_DIR; +assert(project, "STUDIO_PROJECT_DIR must be the generated preview-video-regression project"); +const outputRoot = process.env.OUT_DIR; +if (outputRoot) await mkdir(outputRoot, { recursive: true }); +const output = await mkdtemp( + join(outputRoot ?? tmpdir(), outputRoot ? "run-" : "preview-video-regression-"), +); +const { browser, executablePath } = await launchStudioChrome(); +const evidence = { + executablePath, + browser: await browser.version(), + gpuDisabled: true, + output, + pictures: [], + timings: [], +}; +const errors = []; +const unavailableCliEndpoints = []; +const runtimeLoads = []; +const runtimeHashes = new Set(); + +function optionalCliError(item) { + const path = item.url ? new URL(item.url).pathname : ""; + return ( + item.message.includes("status of 404") && + ["/api/environment/ffmpeg", "/api/open-in-desktop"].includes(path) + ); +} +function recordConsole(message) { + if (message.type() !== "error") return; + const item = { message: message.text(), url: message.location().url }; + if (optionalCliError(item)) unavailableCliEndpoints.push(item); + else errors.push(item); +} + +async function open(beforeNavigate) { + const page = await browser.newPage(); + await page.setViewport({ width: 1440, height: 900 }); + await page.setCacheEnabled(false); + page.on("response", (response) => { + if (new URL(response.url()).pathname === "/api/runtime.js") { + runtimeLoads.push( + response.buffer().then((bytes) => { + runtimeHashes.add(createHash("sha256").update(bytes).digest("hex")); + }), + ); + } + }); + page.on("pageerror", (error) => errors.push(error.message)); + page.on("console", recordConsole); + if (beforeNavigate) await beforeNavigate(page); + await page.goto(url, { waitUntil: "domcontentloaded" }); + await page.waitForSelector('button[aria-label="Play"]:not([disabled])', { timeout: 60_000 }); + const handle = await page.waitForFunction(() => + document.querySelector("hyperframes-player")?.shadowRoot?.querySelector("iframe"), + ); + const iframe = handle.asElement(); + assert(iframe, "Composition iframe is absent"); + const frame = await iframe.contentFrame(); + assert(frame, "Composition document is absent"); + await frame.waitForFunction(() => window.__playerReady, { timeout: 30_000 }); + assert(await frame.$("#film"), "Synthetic composition is absent"); + return { page, iframe, frame }; +} + +async function seek(host, time) { + const start = performance.now(); + await host.frame.evaluate(async (time) => { + await window.__player.seek(time, { settled: true }); + }, time); + await host.frame + .waitForFunction( + (time) => { + const label = document.querySelector(".clock"); + return Math.abs(Number(label.textContent) - time) < 0.002; + }, + {}, + time, + ) + .catch(async (error) => { + const state = await host.frame.evaluate(() => ({ + time: window.__player.getTime(), + labels: [...document.querySelectorAll(".clock")].map((el) => el.textContent), + canvases: document.querySelectorAll("canvas").length, + ready: window.__playerReady, + })); + await host.page.screenshot({ path: join(output, "seek-failed.png") }); + throw new Error(`Seek to ${time} did not publish: ${JSON.stringify(state)}`, { + cause: error, + }); + }); + evidence.timings.push({ + action: "seek", + target: time, + automationObservedMs: performance.now() - start, + }); +} + +async function play(host) { + await host.page.click('button[aria-label="Play"]'); + await host.frame.waitForFunction(() => window.__player.isPlaying(), { timeout: 15_000 }); +} + +async function pause(host) { + await host.page.click('button[aria-label="Pause"]'); + await host.frame.waitForFunction(() => !window.__player.isPlaying()); +} + +async function picture(host, name, retained = false) { + const result = await observePicture(host, name, { output, retained }); + evidence.pictures.push(result); + return result; +} + +async function detail(host) { + return host.frame.evaluate(() => { + const video = document.getElementById("foreground"); + const canvas = video.previousElementSibling; + if (!(canvas instanceof HTMLCanvasElement)) return null; + const data = canvas.getContext("2d").getImageData(300, 20, 128, 1).data; + let contrast = 0; + for (let x = 1; x < 128; x++) contrast += Math.abs(data[x * 4] - data[(x - 1) * 4]); + return { + width: canvas.width, + height: canvas.height, + contrast: contrast / 127, + firstPixel: data[0], + mediaTime: video.currentTime, + }; + }); +} + +async function collectGarbage(host) { + const session = await host.page.createCDPSession(); + for (let i = 0; i < 2; i++) { + await session.send("HeapProfiler.collectGarbage"); + await host.page.evaluate( + () => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))), + ); + } + await session.detach(); +} + +async function withBlockedNative(test) { + const assets = new Map( + await Promise.all( + ["background-a.mp4", "background-b.mp4"].map(async (name) => [ + name, + await readFile(join(project, "assets", name)), + ]), + ), + ); + const prefixed = new Set(); + const held = new Set(); + let blocked = false; + const host = await open(async (page) => { + await page.setRequestInterception(true); + page.on("requestfailed", (request) => held.delete(request)); + const respondPrefix = (request, name, data) => { + if (prefixed.has(name)) { + held.add(request); + return; + } + prefixed.add(name); + const prefix = data.subarray(0, 16 * 1024); + void request.respond({ + status: 206, + contentType: "video/mp4", + headers: { + "Content-Range": `bytes 0-${prefix.length - 1}/${data.length}`, + "Content-Length": String(prefix.length), + "Accept-Ranges": "bytes", + }, + body: prefix, + }); + }; + page.on("request", (request) => { + const name = new URL(request.url()).pathname.split("/").at(-1); + const data = assets.get(name); + if (request.resourceType() === "media" && data && blocked) respondPrefix(request, name, data); + else void request.continue(); + }); + }); + const release = async () => { + blocked = false; + for (const request of [...held]) { + held.delete(request); + if (!request.failure() && !request.isInterceptResolutionHandled()) await request.continue(); + } + }; + const arm = async () => { + blocked = true; + await host.frame.evaluate(() => { + window.__previewTestSeeks = []; + const video = document.getElementById("background-b"); + let proto = Object.getPrototypeOf(video); + while (!Object.hasOwn(proto, "currentTime")) proto = Object.getPrototypeOf(proto); + const descriptor = Object.getOwnPropertyDescriptor(proto, "currentTime"); + Object.defineProperty(proto, "currentTime", { + ...descriptor, + set(time) { + window.__previewTestSeeks.push({ id: this.id, time }); + descriptor.set.call(this, time); + }, + }); + video.load(); + }); + const heldAt = performance.now(); + while (held.size === 0 && performance.now() - heldAt < 5000) { + await host.page.evaluate(() => new Promise(requestAnimationFrame)); + } + assert(held.size > 0, "Native delivery was not actually held"); + }; + try { + await test({ host, arm, release }); + } finally { + await release(); + await host.page.close(); + } +} +async function nativeOnlyHandoff() { + await withBlockedNative(async ({ host, arm, release }) => { + await host.frame.evaluate(() => window.__hf.setPausedVideoFrameProvider(null)); + await seek(host, 0); + await host.frame.waitForFunction( + () => !document.querySelector("[data-hf-preview-video-canvas]"), + ); + const baseline = await picture(host, "native-only-paused"); + assert.equal(baseline.scene, "A"); + assert.equal(baseline.capturedHtml.shownTime, 0); + evidence.nativeOnlyBaseline = { + scene: baseline.scene, + shownTime: baseline.capturedHtml.shownTime, + }; + await arm(); + await host.frame.evaluate(() => { + void window.__player.seek(7, { settled: true }); + }); + await host.page.click('button[aria-label="Play"]'); + const retained = await picture(host, "native-only-handoff", true); + assert.equal( + retained.scene, + "A", + "Native-only Play published before the destination was ready", + ); + assert.equal(retained.playing, false); + assert.equal( + await host.frame.evaluate(() => document.querySelectorAll("[data-hf-seek-video]").length), + 2, + ); + await release(); + await host.frame.waitForFunction(() => window.__player.isPlaying()); + await host.frame.waitForFunction(() => window.__player.getTime() > 7.1); + const next = await picture(host, "native-only-recovered"); + assert.equal(next.scene, "B"); + assert.equal( + await host.frame.evaluate(() => document.querySelectorAll("[data-hf-seek-video]").length), + 0, + ); + evidence.nativeOnlyHandoff = { retained, recovered: next }; + await collectGarbage(host); + }); +} +async function delayedNativeRecovery() { + await withBlockedNative(async ({ host, arm, release }) => { + await seek(host, 7); + await picture(host, "delayed-paused"); + await arm(); + const start = performance.now(); + await play(host); + const handoffMs = performance.now() - start; + assert(handoffMs >= 9000 && handoffMs < 14_000, `Play deadline not exercised: ${handoffMs}ms`); + const stalled = await host.frame.evaluate(() => { + const video = document.getElementById("background-b"); + return { + readyState: video.readyState, + seeking: video.seeking, + cover: video.previousElementSibling?.hasAttribute("data-hf-preview-video-canvas"), + }; + }); + assert( + stalled.readyState < 2 || stalled.seeking, + "Native video was ready; the delay probe is invalid", + ); + assert(stalled.cover, "Full-quality cover disappeared at the deadline"); + await picture(host, "delayed-deadline", true); + await host.frame.waitForFunction(() => window.__player.getTime() > 8.5); + evidence.beforeNativeRelease = await host.frame.evaluate(() => { + const video = document.getElementById("background-b"); + const state = { + time: window.__player.getTime(), + native: video.currentTime, + readyState: video.readyState, + seeking: video.seeking, + seeks: [...window.__previewTestSeeks], + }; + window.__previewTestSeeks.length = 0; + return state; + }); + const deliveredAt = performance.now(); + await release(); + await host.frame.waitForFunction( + () => !document.querySelector("[data-hf-preview-video-canvas]"), + { timeout: 5000 }, + ); + const recovery = await host.frame.evaluate(() => ({ + time: window.__player.getTime(), + native: document.getElementById("background-b").currentTime, + seeks: window.__previewTestSeeks.filter((seek) => seek.id === "background-b"), + })); + assert.equal( + recovery.seeks.length, + 1, + `Recovery must correct once, not chase the clock: ${JSON.stringify(recovery)}`, + ); + assert( + Math.abs(recovery.native - (recovery.time - 6)) < 0.3, + `Recovered video remained behind: ${JSON.stringify(recovery)}`, + ); + const recoveredPicture = await picture(host, "delayed-recovered"); + assert( + recoveredPicture.markerValidated, + "Recovery picture was not checked against its burnt-in time marker", + ); + evidence.delayedNative = { + handoffMs, + stalled, + recoveredAfterDeliveryMs: performance.now() - deliveredAt, + ...recovery, + }; + await collectGarbage(host); + }); +} + +try { + const host = await open(); + await seek(host, 2); + await picture(host, "paused-a"); + evidence.captureIsolation = await verifyCaptureIsolation(host, { picture, seek }); + const before = await detail(host); + assert(before, "Paused foreground has no decoded canvas"); + assert.deepEqual([before.width, before.height], [640, 360]); + assert(before.contrast > 180, `Source detail was blurred: ${before.contrast}`); + await host.page.click("::-p-text(preview-video-regression)"); + await host.page.keyboard.press("ArrowRight"); + await host.frame.waitForFunction( + () => Math.abs(Number(document.querySelector(".clock").textContent) - (2 + 1 / 30)) < 0.002, + ); + await host.frame.waitForFunction( + (previous) => { + const canvas = document.getElementById("foreground").previousElementSibling; + return ( + canvas instanceof HTMLCanvasElement && + Math.abs(canvas.getContext("2d").getImageData(300, 20, 1, 1).data[0] - previous) > 180 + ); + }, + {}, + before.firstPixel, + ); + const after = await detail(host); + assert.deepEqual([after.width, after.height], [640, 360]); + assert(after.contrast > 180); + evidence.frameStep = { before, after }; + + const point = await host.frame.evaluate(() => { + const video = document.getElementById("foreground").getBoundingClientRect(); + return { x: video.x + 140, y: video.y + 160, width: innerWidth, height: innerHeight }; + }); + const box = await host.iframe.boundingBox(); + assert(box); + await host.page.mouse.click( + box.x + (point.x * box.width) / point.width, + box.y + (point.y * box.height) / point.height, + ); + await host.page.waitForFunction(() => document.body.textContent.includes("#foreground")); + evidence.picking = "foreground source video"; + + const slider = await host.page.$('input[aria-label="Seek preview"]'); + const track = await slider.boundingBox(); + assert(track); + await host.page.mouse.move(track.x + track.width * (2 / 12), track.y + track.height / 2); + await host.page.mouse.down(); + for (const fraction of [0.2, 0.7, 0.3, 0.8]) { + await host.page.mouse.move(track.x + track.width * fraction, track.y + track.height / 2, { + steps: 5, + }); + await picture(host, `drag-${fraction}`); + } + await host.page.mouse.up(); + await seek(host, 7); + await picture(host, "paused-b"); + await seek(host, 5.5); + await play(host); + await host.frame.waitForFunction(() => window.__player.getTime() > 6.25); + const cut = await picture(host, "native-cut"); + assert.equal(cut.scene, "B"); + await pause(host); + + for (const time of [1, 7, 3, 9]) { + await seek(host, time); + await picture(host, `jump-${time}`); + await play(host); + await host.frame.waitForFunction((time) => window.__player.getTime() > time + 0.1, {}, time); + await picture(host, `play-${time}`); + await pause(host); + } + await play(host); + await pause(host); + await play(host); + await pause(host); + assert.equal(await host.frame.evaluate(() => window.__player.isPlaying()), false); + await play(host); + await host.frame.waitForFunction( + () => document.querySelectorAll("[data-hf-preview-video-canvas]").length === 0, + ); + await collectGarbage(host); + assert.deepEqual(errors, [], "Browser logged an error or unclosed VideoSample"); + await host.page.close(); + await delayedNativeRecovery(); + await nativeOnlyHandoff(); + assert.deepEqual(errors, [], "Delayed native recovery logged an error or unclosed VideoSample"); + await Promise.all(runtimeLoads); + evidence.runtimeSha256 = [...runtimeHashes]; + assert.equal(runtimeHashes.size, 1, "All test documents must load the same runtime bytes"); + evidence.ok = true; +} catch (error) { + evidence.ok = false; + evidence.failure = error.stack; + await (await browser.pages()).at(-1)?.screenshot({ path: join(output, "failed.png") }); + process.exitCode = 1; +} finally { + evidence.errors = errors; + evidence.unavailableCliEndpoints = unavailableCliEndpoints; + await writeFile(join(output, "results.json"), JSON.stringify(evidence, null, 2)); + console.log(JSON.stringify(evidence, null, 2)); + await browser.close(); +}