mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(studio): bound recovery and verify full-quality playback (#7393)
* fix(studio): bound recovery and verify full-quality playback * fix(studio): compare preview pixels with captured HTML clock
This commit is contained in:
@@ -1009,6 +1009,8 @@ jobs:
|
||||
packages/studio/data/projects/undo-socket-pool
|
||||
cp -R packages/studio/tests/e2e/fixtures/aroll-backdrop \
|
||||
packages/studio/data/projects/aroll-backdrop
|
||||
cp -R packages/studio/tests/e2e/fixtures/preview-video-regression \
|
||||
packages/studio/data/projects/preview-video-regression
|
||||
- name: Build the base the viewport gate compares against
|
||||
# The first parent of the checked-out commit, as in base-file.sh. Measured only when the head misses a budget.
|
||||
run: |
|
||||
@@ -1181,7 +1183,7 @@ jobs:
|
||||
- name: Install ffmpeg for the paused-video fixture
|
||||
if: success() || failure()
|
||||
uses: ./.github/actions/install-ffmpeg-linux
|
||||
- name: Undo while paused videos hold the preview's sockets
|
||||
- name: Verify paused-video sockets, picture quality and native recovery
|
||||
# Counts the undo's wait for a socket, not its total time: unpinned it is 0 ms.
|
||||
if: success() || failure()
|
||||
run: |
|
||||
@@ -1200,6 +1202,12 @@ jobs:
|
||||
STUDIO_URL="http://localhost:5318/#project/undo-socket-pool" \
|
||||
STUDIO_PROJECT_DIR="$PWD/packages/studio/data/projects/undo-socket-pool" \
|
||||
node packages/studio/tests/e2e/undo-socket-pool.mjs | tee /tmp/undo-socket-pool.json
|
||||
node packages/studio/tests/e2e/fixtures/preview-video-regression/generate.mjs \
|
||||
packages/studio/data/projects/preview-video-regression
|
||||
STUDIO_URL="http://localhost:5318/#project/preview-video-regression" \
|
||||
STUDIO_PROJECT_DIR="$PWD/packages/studio/data/projects/preview-video-regression" \
|
||||
OUT_DIR=/tmp/preview-video-regression \
|
||||
node packages/studio/tests/e2e/preview-video-regression.mjs
|
||||
- name: Upload gate evidence
|
||||
# The gate's whole output is machine-readable evidence, and a red run is
|
||||
# exactly when someone needs to read it. Keep it on failure too.
|
||||
@@ -1212,6 +1220,7 @@ jobs:
|
||||
/tmp/studio-open-counts.json
|
||||
/tmp/inline-text-caret/*.png
|
||||
/tmp/undo-socket-pool.json
|
||||
/tmp/preview-video-regression/
|
||||
if-no-files-found: error
|
||||
|
||||
# Manual editing accuracy in the built Studio, gated against the base branch's baseline.json. A pull request
|
||||
|
||||
@@ -121,9 +121,11 @@ A paused fallback that outlasts its wait retains the presented clock and picture
|
||||
while paused sync continues. It can still publish when the video becomes ready;
|
||||
a new gesture cancels that recovery. Studio requests paused-decoder release
|
||||
before preparing native playback and reopens readers when paused scrubbing resumes.
|
||||
Transparent video queries drain their alpha merges before input disposal. If one
|
||||
never settles, its cleanup can outlast the Play deadline; the remaining playback
|
||||
still starts while that video's last picture stays visible.
|
||||
Paused-reader release waits up to two seconds for transparent video queries to
|
||||
drain. After that it cancels their downloads and lets playback continue while
|
||||
unfinished merges close in the background; input disposal still waits for the
|
||||
merge to finish. A recovering video more than half a second out of sync with the transport
|
||||
gets at most one corrective seek per recovery registration before its retained picture is removed.
|
||||
Scrubbing and paused frame steps decode the original video at source resolution
|
||||
throughout. Automatic playback quality reductions do not reduce these frames,
|
||||
and the frame decoder does not substitute re-encoded seek copies. Playback
|
||||
|
||||
@@ -3,7 +3,11 @@ import { initSandboxRuntimeModular } from "./init";
|
||||
import { hideTimedClipsUntilFirstPass } from "./timedClipHide";
|
||||
import { resetSeekDispatchState } from "./adapters/seek-dispatch";
|
||||
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
|
||||
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
|
||||
import {
|
||||
PREVIEW_SEEK_HOLD_ATTR,
|
||||
STUDIO_PREVIEW_MARK_META,
|
||||
PLAYER_FRAME_ATTR,
|
||||
} from "../studioPreviewMark";
|
||||
import type {
|
||||
PausedVideoFrame,
|
||||
PausedVideoFrameProvider,
|
||||
@@ -65,11 +69,11 @@ function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
|
||||
};
|
||||
}
|
||||
|
||||
function film(withImage = false) {
|
||||
function film(withImage = false, studio = true) {
|
||||
resetRuntimeFixtureDom();
|
||||
const meta = document.createElement("meta");
|
||||
meta.name = STUDIO_PREVIEW_MARK_META;
|
||||
document.head.append(meta);
|
||||
if (studio) document.head.append(meta);
|
||||
vi.spyOn(window, "requestAnimationFrame").mockReturnValue(0);
|
||||
const drawImage = vi.fn();
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
|
||||
@@ -301,22 +305,24 @@ it("leaves a grading canvas's hidden source in its layout during the hold", asyn
|
||||
});
|
||||
|
||||
it.each(["play", "render", "teardown"])(
|
||||
"cleans up a pending seek on %s and ignores late completions",
|
||||
"finishes native handoff on Play and ignores retired completions on render or teardown (%s)",
|
||||
async (action) => {
|
||||
const { main, next } = film();
|
||||
const player = window.__player!;
|
||||
const landed = player.seek(6);
|
||||
if (action === "play") player.play();
|
||||
else if (action === "render") player.renderSeek(2);
|
||||
else window.__hfRuntimeTeardown?.();
|
||||
const time = main.time?.();
|
||||
const cases = {
|
||||
play: { run: () => player.play(), before: 0, after: 6 },
|
||||
render: { run: () => player.renderSeek(2), before: 2, after: 2 },
|
||||
teardown: { run: () => window.__hfRuntimeTeardown?.(), before: 0, after: 0 },
|
||||
};
|
||||
const expected = cases[action as keyof typeof cases];
|
||||
expected.run();
|
||||
expect(main.time!()).toBe(expected.before);
|
||||
for (const video of next) video.land();
|
||||
await landed;
|
||||
expect(main.time?.()).toBe(time);
|
||||
expect(main.time!()).toBe(expected.after);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
|
||||
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
|
||||
if (action === "play") expect(player.getTime()).toBeGreaterThanOrEqual(6);
|
||||
if (action === "render") expect(time).toBe(2);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -508,6 +514,53 @@ it("reveals a delayed playing frame without chasing the moving clock with more s
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([1500, 2500].flatMap((lateMs) => [25, 40, 100, 600].map((seekMs) => ({ lateMs, seekMs }))))(
|
||||
"aligns a recovered picture once after $lateMs ms with $seekMs ms seek latency",
|
||||
async ({ lateMs, seekMs }) => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const initial = window.__player!.seek(1);
|
||||
pending.shift()!();
|
||||
await initial;
|
||||
const stalled = old[0]!;
|
||||
stalled.metadataAt(1);
|
||||
let error: { code: number } | null = { code: 4 };
|
||||
Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
|
||||
window.__player!.play();
|
||||
old[1]!.land();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
error = null;
|
||||
const seeks = stalled.seeks.length;
|
||||
for (let elapsed = 0; elapsed < lateMs; elapsed += 25) {
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
window.__player!.getTime();
|
||||
}
|
||||
expect(window.__player!.getTime()).toBeCloseTo(1 + lateMs / 1000, 2);
|
||||
Object.defineProperty(stalled.video, "paused", { configurable: true, value: false });
|
||||
stalled.land();
|
||||
stalled.video.dispatchEvent(new Event("playing"));
|
||||
stalled.presentAt(1 + 1 / 30);
|
||||
expect(stalled.seeks).toHaveLength(seeks + 1);
|
||||
const target = window.__player!.getTime();
|
||||
expect(stalled.seeks.at(-1)).toBeCloseTo(target, 2);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(1);
|
||||
for (let elapsed = 0; elapsed < seekMs; elapsed += 25) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(25, seekMs - elapsed));
|
||||
window.__player!.getTime();
|
||||
}
|
||||
expect(window.__player!.getTime() - target).toBeCloseTo(seekMs / 1000, 2);
|
||||
stalled.land();
|
||||
stalled.presentAt(target + 1 / 30);
|
||||
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
expect(stalled.seeks).toHaveLength(seeks + 1);
|
||||
expect(Math.abs(stalled.video.currentTime - window.__player!.getTime())).toBeLessThan(
|
||||
Math.max(0.1, seekMs / 1000),
|
||||
);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a paused recovery covered until its static target seek lands", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
|
||||
const { old, next } = film();
|
||||
@@ -901,3 +954,58 @@ it("releases a native hold when stop-media cancels its first provider seek", asy
|
||||
await Promise.resolve();
|
||||
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps the native-only complete picture when Play interrupts an unresolved scrub", async () => {
|
||||
const { main, next } = film();
|
||||
const seek = window.__player!.seek(6);
|
||||
const pictures = [...document.querySelectorAll("canvas")];
|
||||
expect(pictures).toHaveLength(2);
|
||||
window.__player!.play();
|
||||
await seek;
|
||||
expect(main.time?.()).toBe(0);
|
||||
expect(pictures.every((picture) => picture.isConnected)).toBe(true);
|
||||
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
|
||||
next[0]!.land();
|
||||
expect(main.time?.()).toBe(0);
|
||||
next[1]!.land();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(main.time?.()).toBe(6);
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
expect(window.__player!.getTime()).toBeGreaterThanOrEqual(6);
|
||||
expect(document.querySelectorAll("canvas")).toHaveLength(0);
|
||||
});
|
||||
|
||||
function safariPlayerFilm() {
|
||||
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
|
||||
"AppleWebKit/605.1.15 Version/18.0 Safari/605.1.15",
|
||||
);
|
||||
const frame = document.createElement("iframe");
|
||||
frame.setAttribute(PLAYER_FRAME_ATTR, "");
|
||||
vi.spyOn(window, "frameElement", "get").mockReturnValue(frame);
|
||||
return film(false, false);
|
||||
}
|
||||
|
||||
it("keeps pending-seek Play synchronous in an unmarked Safari player frame", () => {
|
||||
const { main, next } = safariPlayerFilm();
|
||||
void window.__player!.seek(6);
|
||||
expect(main.time?.()).toBe(0);
|
||||
window.__player!.play();
|
||||
expect(main.time?.()).toBe(6);
|
||||
expect(next.some(({ video }) => vi.mocked(video.play).mock.calls.length > 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("retains an explicit unmarked provider handoff before its first picture", async () => {
|
||||
const { main, old, next } = safariPlayerFilm();
|
||||
for (const video of old) video.metadataAt(0);
|
||||
const { pending } = frameProvider([...old, ...next]);
|
||||
const seeking = window.__player!.seek(6);
|
||||
window.__player!.play();
|
||||
expect(main.time?.()).toBe(0);
|
||||
expect(window.__player!.isPlaying()).toBe(false);
|
||||
for (const video of next) video.land();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(window.__player!.isPlaying()).toBe(true);
|
||||
pending.shift()!();
|
||||
await seeking;
|
||||
expect(main.time?.()).toBe(6);
|
||||
});
|
||||
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
} from "./timedClipHide";
|
||||
import {
|
||||
inPlayerFrame,
|
||||
STUDIO_PREVIEW_MARK_META,
|
||||
PREVIEW_ONLY_ATTRS,
|
||||
RUNTIME_STAMPED_ATTR,
|
||||
STUDIO_PREVIEW_FAR_SCENE_ATTR as FAR_SCENE_ATTR,
|
||||
@@ -2997,6 +2998,7 @@ export function initSandboxRuntimeModular(): void {
|
||||
});
|
||||
const hiddenImagesSkipped = skipsHiddenImages();
|
||||
const mediaDeferred = defersMedia();
|
||||
const studioPreview = document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`) !== null;
|
||||
const skipFarScenes = createFarSceneSkip((host) =>
|
||||
runtimeElementWindow(host, {
|
||||
timelineRegistry: window.__timelines ?? {},
|
||||
@@ -5062,17 +5064,22 @@ export function initSandboxRuntimeModular(): void {
|
||||
},
|
||||
suspend: pausedFrames.suspendProvider,
|
||||
prepareNative: pausedFrames.prepareNative,
|
||||
clear: (keep) =>
|
||||
clear: (keep) => {
|
||||
const corrected = new WeakSet<HTMLVideoElement>();
|
||||
pausedFrames.clear(keep, (video, presented) => {
|
||||
if (clock.isPlaying()) return presented;
|
||||
const playing = clock.isPlaying();
|
||||
if (playing && !presented) return false;
|
||||
const time = clock.now();
|
||||
const clip = previewVideoClipsAt(time).find(({ el }) => el === video);
|
||||
if (!clip) return true;
|
||||
const expected = mediaSourceTimeAt(clip, time);
|
||||
if (Math.abs(video.currentTime - expected) < 0.02) return true;
|
||||
if (playing && corrected.has(video)) return true;
|
||||
if (Math.abs(video.currentTime - expected) < (playing ? 0.5 : 0.02)) return true;
|
||||
if (playing) corrected.add(video);
|
||||
video.currentTime = mediaElementTime(expected);
|
||||
return false;
|
||||
}),
|
||||
});
|
||||
},
|
||||
},
|
||||
videosToHold: () =>
|
||||
withTimingResolver(() =>
|
||||
@@ -5162,7 +5169,8 @@ export function initSandboxRuntimeModular(): void {
|
||||
play: () => {
|
||||
flushHeldSeek();
|
||||
if (
|
||||
(pausedFrames.active || (pausedFrames.available() && previewSeek?.time !== undefined)) &&
|
||||
(pausedFrames.active ||
|
||||
((pausedFrames.available() || studioPreview) && previewSeek?.time !== undefined)) &&
|
||||
!clock.isPlaying()
|
||||
) {
|
||||
wallClearTimeout(nativeWarmTimer);
|
||||
|
||||
@@ -327,3 +327,25 @@ it("keeps waiting when no native frame or playback progress arrives", () => {
|
||||
stop();
|
||||
expect(pending.size).toBe(0);
|
||||
});
|
||||
|
||||
it("does not reveal a paused seek from timeupdate after playback had started", () => {
|
||||
const video = document.createElement("video");
|
||||
let paused = false;
|
||||
let time = 3;
|
||||
Object.defineProperties(video, {
|
||||
paused: { get: () => paused },
|
||||
currentTime: { get: () => time },
|
||||
});
|
||||
const recovered = vi.fn(() => true);
|
||||
const stop = observeNativeVideoFrames(video, recovered);
|
||||
paused = true;
|
||||
time = 4;
|
||||
video.dispatchEvent(new Event("timeupdate"));
|
||||
expect(recovered).not.toHaveBeenCalled();
|
||||
paused = false;
|
||||
video.dispatchEvent(new Event("playing"));
|
||||
time += 1 / 30;
|
||||
video.dispatchEvent(new Event("timeupdate"));
|
||||
expect(recovered).toHaveBeenCalledOnce();
|
||||
stop();
|
||||
});
|
||||
|
||||
@@ -254,3 +254,16 @@ it("cancels a retained native observer when a new paused picture replaces it", a
|
||||
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retire an adopted decoded picture when its native snapshot lease ends", async () => {
|
||||
const { session, video, request } = fixture();
|
||||
Object.defineProperty(video, "readyState", { value: 4 });
|
||||
const stopHold = session.holdNative([video]);
|
||||
const snapshot = getPausedVideoFrame(video);
|
||||
const decoded = await session.prepare(request, new AbortController().signal);
|
||||
expect(decoded!.present()).toBe(true);
|
||||
stopHold();
|
||||
expect(getPausedVideoFrame(video)).toBe(snapshot);
|
||||
expect(ownsPausedVideoTime(video)).toBe(true);
|
||||
expect(video.currentTime).toBe(3);
|
||||
});
|
||||
|
||||
@@ -224,3 +224,16 @@ it("preserves the held native picture when a new gesture cancels expired backgro
|
||||
await next;
|
||||
expect(document.querySelector("canvas")).toBeNull();
|
||||
});
|
||||
|
||||
it("retires the hard Play timer after native readiness publishes", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { job, video, commit } = fixture();
|
||||
Object.defineProperties(video, {
|
||||
readyState: { value: 4 },
|
||||
seeking: { value: false },
|
||||
currentTime: { value: 3 },
|
||||
});
|
||||
await job.handoff(3);
|
||||
expect(commit).toHaveBeenCalledWith(3);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
@@ -113,7 +113,7 @@ beforeEach(() => {
|
||||
canvasContext.mockReset().mockReturnValue({ clearRect: vi.fn(), imageSmoothingQuality: "low" });
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockImplementation(canvasContext);
|
||||
createBitmap.mockReset().mockImplementation(async () => bitmap());
|
||||
cancelReads.mockClear();
|
||||
cancelReads.mockReset();
|
||||
qualityChosenBy = "person";
|
||||
previewQuality = 1;
|
||||
displayScale = 1;
|
||||
@@ -642,3 +642,76 @@ it("closes the selected alpha sample when the finite iterator fails while draini
|
||||
await expect(owner.decode([request("a")], signal())).rejects.toThrow("Alpha drain failed");
|
||||
expect(selected.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("bounds alpha suspension without disposing an unfinished merger", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
canBeTransparent: async () => true,
|
||||
}));
|
||||
let drain: () => void = () => {};
|
||||
const drained = new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
const selected = sample();
|
||||
finiteSamples.mockImplementationOnce(async function* () {
|
||||
yield selected;
|
||||
await drained;
|
||||
});
|
||||
const owner = provider();
|
||||
const pending = owner.decode([request("a")], signal()).catch((error: unknown) => error);
|
||||
await vi.waitFor(() => expect(finiteSamples).toHaveBeenCalledOnce());
|
||||
let released = false;
|
||||
const suspended = owner.suspend!().then(() => {
|
||||
released = true;
|
||||
});
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(2000);
|
||||
expect(released).toBe(true);
|
||||
expect(cancelReads).toHaveBeenCalledOnce();
|
||||
expect(inputs[0]!.dispose).not.toHaveBeenCalled();
|
||||
expect(selected.close).not.toHaveBeenCalled();
|
||||
const [next] = await owner.decode([request("a")], signal());
|
||||
expect(inputs).toHaveLength(2);
|
||||
next!.release();
|
||||
} finally {
|
||||
drain();
|
||||
await pending;
|
||||
await suspended;
|
||||
}
|
||||
expect(selected.close).toHaveBeenCalledOnce();
|
||||
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
|
||||
expect(inputs[1]!.dispose).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("closes the selected alpha sample when suspension cancels an in-flight read", async () => {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
getTrack.mockImplementation(async (source: string) => ({
|
||||
...track(source),
|
||||
canBeTransparent: async () => true,
|
||||
}));
|
||||
const selected = sample();
|
||||
let rejectRead: (error: Error) => void = () => {};
|
||||
const reading = new Promise<void>((_resolve, reject) => {
|
||||
rejectRead = reject;
|
||||
});
|
||||
finiteSamples.mockImplementationOnce(async function* () {
|
||||
yield selected;
|
||||
await reading;
|
||||
});
|
||||
cancelReads.mockImplementation(() =>
|
||||
rejectRead(new DOMException("Network read cancelled", "AbortError")),
|
||||
);
|
||||
const owner = provider();
|
||||
const decoding = owner.decode([request("a")], signal()).catch((error: unknown) => error);
|
||||
await vi.waitFor(() => expect(finiteSamples).toHaveBeenCalledOnce());
|
||||
const suspended = owner.suspend!();
|
||||
await vi.advanceTimersByTimeAsync(1999);
|
||||
expect(cancelReads).not.toHaveBeenCalled();
|
||||
expect(inputs[0]!.dispose).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await suspended;
|
||||
expect(await decoding).toMatchObject({ name: "AbortError" });
|
||||
expect(selected.close).toHaveBeenCalledOnce();
|
||||
expect(inputs[0]!.dispose).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
const FRAME_BUDGET = 64 * 1024 * 1024;
|
||||
const MAX_READERS = 4;
|
||||
const NEARBY_FORWARD_SECONDS = 0.5;
|
||||
const ALPHA_DRAIN_WAIT_MS = 2000;
|
||||
|
||||
type Cached = {
|
||||
image: ImageBitmap;
|
||||
@@ -104,18 +105,28 @@ export function createPausedVideoDecoder(iframe: HTMLIFrameElement): PausedVideo
|
||||
reader.disposed = true;
|
||||
clearTimeout(reader.idleTimer);
|
||||
if (opened.get(reader.key) === reader) opened.delete(reader.key);
|
||||
// Input disposal must not cut off an in-flight alpha merger.
|
||||
if (reader.transparent) await reader.tail;
|
||||
reader.cancelReads();
|
||||
reader.input.dispose();
|
||||
await closeIterator(reader).catch(() => {});
|
||||
for (const entry of reader.cache.values()) {
|
||||
cacheOrder.delete(entry);
|
||||
bytes -= entry.bytes;
|
||||
entry.image.close();
|
||||
}
|
||||
reader.cache.clear();
|
||||
reader.canvas.width = reader.canvas.height = 0;
|
||||
const closing = (async () => {
|
||||
// Input disposal must not cut off an in-flight alpha merger.
|
||||
if (reader.transparent) await reader.tail;
|
||||
reader.cancelReads();
|
||||
reader.input.dispose();
|
||||
await closeIterator(reader).catch(() => {});
|
||||
for (const entry of reader.cache.values()) {
|
||||
cacheOrder.delete(entry);
|
||||
bytes -= entry.bytes;
|
||||
entry.image.close();
|
||||
}
|
||||
reader.cache.clear();
|
||||
reader.canvas.width = reader.canvas.height = 0;
|
||||
})();
|
||||
if (!reader.transparent) return closing;
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
reader.cancelReads();
|
||||
resolve();
|
||||
}, ALPHA_DRAIN_WAIT_MS);
|
||||
void closing.then(resolve, reject).finally(() => clearTimeout(timeout));
|
||||
});
|
||||
};
|
||||
const scheduleIdle = (reader: Reader) => {
|
||||
clearTimeout(reader.idleTimer);
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
assets/
|
||||
@@ -0,0 +1,164 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { mkdir, copyFile } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import { resolve, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = resolve(process.argv[2] ?? fileURLToPath(new URL(".", import.meta.url)));
|
||||
const assets = join(root, "assets");
|
||||
const width = 640;
|
||||
const height = 360;
|
||||
await mkdir(assets, { recursive: true });
|
||||
await copyFile(
|
||||
createRequire(import.meta.url).resolve("gsap/dist/gsap.min.js"),
|
||||
join(assets, "gsap.min.js"),
|
||||
);
|
||||
|
||||
function codecOptions(alpha) {
|
||||
return alpha
|
||||
? [
|
||||
"-c:v",
|
||||
"libvpx-vp9",
|
||||
"-lossless",
|
||||
"1",
|
||||
"-pix_fmt",
|
||||
"yuva420p",
|
||||
"-auto-alt-ref",
|
||||
"0",
|
||||
"-deadline",
|
||||
"realtime",
|
||||
"-cpu-used",
|
||||
"8",
|
||||
]
|
||||
: [
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-crf",
|
||||
"10",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
];
|
||||
}
|
||||
function opaqueBase(base, color) {
|
||||
for (let i = 0; i < base.length; i += 4) {
|
||||
base[i] = color[0];
|
||||
base[i + 1] = color[1];
|
||||
base[i + 2] = color[2];
|
||||
base[i + 3] = 255;
|
||||
}
|
||||
}
|
||||
function alphaBase(base) {
|
||||
for (let y = 100; y < 220; y++)
|
||||
for (let x = 80; x < 200; x++) {
|
||||
const i = (y * width + x) * 4;
|
||||
base[i + 1] = 208;
|
||||
base[i + 2] = 96;
|
||||
base[i + 3] = 255;
|
||||
}
|
||||
}
|
||||
function baseFrame(color) {
|
||||
const base = Buffer.alloc(width * height * 4);
|
||||
if (color) opaqueBase(base, color);
|
||||
else alphaBase(base);
|
||||
return base;
|
||||
}
|
||||
function checker(pixels, frame) {
|
||||
for (let y = 20; y < 52; y++)
|
||||
for (let x = 300; x < 428; x++) {
|
||||
const i = (y * width + x) * 4;
|
||||
pixels[i] = pixels[i + 1] = pixels[i + 2] = (x + frame) % 2 ? 255 : 0;
|
||||
pixels[i + 3] = 255;
|
||||
}
|
||||
}
|
||||
function timeMarker(pixels, frame) {
|
||||
for (let y = 300; y < 315; y++) {
|
||||
const left = 20 + Math.floor(frame / 30) * 50;
|
||||
for (let x = left; x < left + 20; x++) {
|
||||
const i = (y * width + x) * 4;
|
||||
pixels[i] = pixels[i + 1] = pixels[i + 2] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
function texture(pixels, frame) {
|
||||
// A deterministic texture keeps native range reads from buffering the entire fixture at once.
|
||||
let seed = frame + 1;
|
||||
for (let y = 0; y < 64; y++)
|
||||
for (let x = 544; x < 640; x++) {
|
||||
const i = (y * width + x) * 4;
|
||||
for (let channel = 0; channel < 3; channel++) {
|
||||
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
|
||||
pixels[i + channel] = seed >>> 24;
|
||||
}
|
||||
}
|
||||
}
|
||||
function framePixels(base, alpha, frame) {
|
||||
const pixels = Buffer.from(base);
|
||||
if (alpha) checker(pixels, frame);
|
||||
else {
|
||||
timeMarker(pixels, frame);
|
||||
texture(pixels, frame);
|
||||
}
|
||||
return pixels;
|
||||
}
|
||||
async function writeFrames(stream, base, alpha, frames) {
|
||||
for (let frame = 0; frame < frames; frame++) {
|
||||
if (!stream.write(framePixels(base, alpha, frame))) await once(stream, "drain");
|
||||
}
|
||||
stream.end();
|
||||
}
|
||||
async function encode(name, frames, color) {
|
||||
const alpha = color === null;
|
||||
const child = spawn(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pixel_format",
|
||||
"rgba",
|
||||
"-video_size",
|
||||
`${width}x${height}`,
|
||||
"-framerate",
|
||||
"30",
|
||||
"-i",
|
||||
"pipe:0",
|
||||
...codecOptions(alpha),
|
||||
"-g",
|
||||
"30",
|
||||
"-an",
|
||||
join(assets, name),
|
||||
],
|
||||
{ stdio: ["pipe", "ignore", "pipe"] },
|
||||
);
|
||||
let errors = "";
|
||||
child.stderr.on("data", (data) => {
|
||||
errors += data;
|
||||
});
|
||||
const finished = new Promise((resolveDone, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("exit", (code) =>
|
||||
code === 0 ? resolveDone() : reject(new Error(`ffmpeg ${name}: ${errors}`)),
|
||||
);
|
||||
});
|
||||
try {
|
||||
await writeFrames(child.stdin, baseFrame(color), alpha, frames);
|
||||
await finished;
|
||||
} catch (error) {
|
||||
child.kill("SIGTERM");
|
||||
await finished.catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
await encode("background-a.mp4", 180, [204, 48, 48]);
|
||||
await encode("background-b.mp4", 180, [40, 102, 221]);
|
||||
await encode("foreground.webm", 360, null);
|
||||
console.log(JSON.stringify({ assets, width, height, fps: 30, duration: 12, synthetic: true }));
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"$schema": "https://hyperframes.heygen.com/schema/hyperframes.json",
|
||||
"registry": "https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry",
|
||||
"paths": {
|
||||
"blocks": "compositions",
|
||||
"components": "compositions/components",
|
||||
"assets": "assets"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Original-source preview regression</title>
|
||||
<script src="assets/gsap.min.js"></script>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
margin: 0;
|
||||
width: 640px;
|
||||
height: 360px;
|
||||
overflow: hidden;
|
||||
background: #000;
|
||||
}
|
||||
#film {
|
||||
position: relative;
|
||||
width: 640px;
|
||||
height: 360px;
|
||||
overflow: hidden;
|
||||
}
|
||||
video {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 640px;
|
||||
height: 360px;
|
||||
}
|
||||
#foreground {
|
||||
z-index: 2;
|
||||
}
|
||||
.label {
|
||||
position: absolute;
|
||||
bottom: 16px;
|
||||
right: 20px;
|
||||
z-index: 3;
|
||||
padding: 8px;
|
||||
background: #111;
|
||||
color: #fff;
|
||||
font: 20px/24px monospace;
|
||||
box-sizing: border-box;
|
||||
width: 232px;
|
||||
}
|
||||
.clock-bits {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
height: 24px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
.clock-bits i {
|
||||
flex: 0 0 24px;
|
||||
height: 24px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div
|
||||
id="film"
|
||||
data-composition-id="preview-film"
|
||||
data-width="640"
|
||||
data-height="360"
|
||||
data-duration="12"
|
||||
data-fps="30"
|
||||
data-layout-allow-overlap
|
||||
>
|
||||
<video
|
||||
id="background-a"
|
||||
class="clip"
|
||||
src="assets/background-a.mp4"
|
||||
muted
|
||||
playsinline
|
||||
preload="metadata"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
></video>
|
||||
<video
|
||||
id="background-b"
|
||||
class="clip"
|
||||
src="assets/background-b.mp4"
|
||||
muted
|
||||
playsinline
|
||||
preload="metadata"
|
||||
data-start="6"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
></video>
|
||||
<video
|
||||
id="foreground"
|
||||
class="clip"
|
||||
src="assets/foreground.webm"
|
||||
muted
|
||||
playsinline
|
||||
preload="metadata"
|
||||
data-start="0"
|
||||
data-duration="12"
|
||||
data-track-index="1"
|
||||
></video>
|
||||
<div id="label-a" class="clip label" data-start="0" data-duration="6" data-track-index="2">
|
||||
A · <span class="clock">0.000</span>
|
||||
<span class="clock-bits" data-scene="0" aria-hidden="true"></span>
|
||||
</div>
|
||||
<div id="label-b" class="clip label" data-start="6" data-duration="6" data-track-index="2">
|
||||
B · <span class="clock">0.000</span>
|
||||
<span class="clock-bits" data-scene="1" aria-hidden="true"></span>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const clock = { time: 0 };
|
||||
const strips = [...document.querySelectorAll(".clock-bits")];
|
||||
for (const strip of strips)
|
||||
for (let i = 0; i < 7; i++) strip.append(document.createElement("i"));
|
||||
const paintClock = () => {
|
||||
for (const label of document.querySelectorAll(".clock"))
|
||||
label.textContent = clock.time.toFixed(3);
|
||||
const second = Math.floor(clock.time);
|
||||
for (const strip of strips) {
|
||||
const bits = [
|
||||
1,
|
||||
0,
|
||||
Number(strip.dataset.scene),
|
||||
...Array.from({ length: 4 }, (_, i) => (second >> i) & 1),
|
||||
];
|
||||
[...strip.children].forEach(
|
||||
(cell, i) => (cell.style.background = bits[i] ? "#fff" : "#000"),
|
||||
);
|
||||
}
|
||||
};
|
||||
const timeline = gsap.timeline({ paused: true });
|
||||
timeline.to(clock, { time: 12, duration: 12, ease: "none", onUpdate: paintClock });
|
||||
paintClock();
|
||||
window.__timelines = { "preview-film": timeline };
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
async function replaceVisibleHtml(host, scene, time) {
|
||||
return host.frame.evaluate(
|
||||
({ scene, time }) => {
|
||||
const label = document.getElementById("label-a");
|
||||
const strip = label.querySelector(".clock-bits");
|
||||
const clock = label.querySelector(".clock");
|
||||
const saved = {
|
||||
text: clock.textContent,
|
||||
prefix: label.firstChild.textContent,
|
||||
bits: strip.innerHTML,
|
||||
};
|
||||
label.firstChild.textContent = `${scene} · `;
|
||||
clock.textContent = time.toFixed(3);
|
||||
strip.children[2].style.background = scene === "B" ? "#fff" : "#000";
|
||||
for (let i = 3; i < strip.children.length; i++)
|
||||
strip.children[i].style.background = (time >> (i - 3)) & 1 ? "#fff" : "#000";
|
||||
return saved;
|
||||
},
|
||||
{ scene, time },
|
||||
);
|
||||
}
|
||||
|
||||
async function rejectMixedPicture(host, picture, scene, time, expected) {
|
||||
const saved = await replaceVisibleHtml(host, scene, time);
|
||||
try {
|
||||
await assert.rejects(picture(host, `witness-mixed-${scene}`), expected);
|
||||
} finally {
|
||||
await host.frame.evaluate((saved) => {
|
||||
const label = document.getElementById("label-a");
|
||||
label.firstChild.textContent = saved.prefix;
|
||||
label.querySelector(".clock").textContent = saved.text;
|
||||
label.querySelector(".clock-bits").innerHTML = saved.bits;
|
||||
}, saved);
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyCaptureIsolation(host, { picture, seek }) {
|
||||
const original = host.page.screenshot.bind(host.page);
|
||||
let published = false;
|
||||
host.page.screenshot = async (options) => {
|
||||
const png = await original(options);
|
||||
if (options.encoding === "base64" && !published) {
|
||||
published = true;
|
||||
await seek(host, 7);
|
||||
}
|
||||
return png;
|
||||
};
|
||||
let captured;
|
||||
try {
|
||||
captured = await picture(host, "witness-publish-after-capture");
|
||||
} finally {
|
||||
host.page.screenshot = original;
|
||||
}
|
||||
assert(published, "The capture witness did not interleave a publication");
|
||||
assert.equal(captured.scene, "A");
|
||||
assert.equal(captured.capturedHtml.shownTime, 2);
|
||||
assert(captured.labels[0].startsWith("B"), "The post-capture DOM did not publish scene B");
|
||||
await seek(host, 2);
|
||||
await rejectMixedPicture(host, picture, "B", 8, /background disagrees with B/);
|
||||
await rejectMixedPicture(host, picture, "A", 4, /video pixels disagree with the HTML clock/);
|
||||
return { publishedAfterCapture: true, mixedSceneRejected: true, mixedClockRejected: true };
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
async function capturePicture(host, name, output) {
|
||||
let png;
|
||||
let attempts = 0;
|
||||
const bounds = async () => {
|
||||
const outer = await host.iframe.boundingBox();
|
||||
const inner = await host.frame.evaluate(() => {
|
||||
const box = document.getElementById("film").getBoundingClientRect();
|
||||
return {
|
||||
x: box.x,
|
||||
y: box.y,
|
||||
width: box.width,
|
||||
height: box.height,
|
||||
viewportWidth: innerWidth,
|
||||
viewportHeight: innerHeight,
|
||||
};
|
||||
});
|
||||
return {
|
||||
x: outer.x + (inner.x * outer.width) / inner.viewportWidth,
|
||||
y: outer.y + (inner.y * outer.height) / inner.viewportHeight,
|
||||
width: (inner.width * outer.width) / inner.viewportWidth,
|
||||
height: (inner.height * outer.height) / inner.viewportHeight,
|
||||
};
|
||||
};
|
||||
for (; attempts < 3; attempts++) {
|
||||
const before = await bounds();
|
||||
const full = await host.page.screenshot({ encoding: "base64", captureBeyondViewport: false });
|
||||
png = await host.page.evaluate(
|
||||
async ({ full, box }) => {
|
||||
const image = new Image();
|
||||
image.src = `data:image/png;base64,${full}`;
|
||||
await image.decode();
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.round(box.width);
|
||||
canvas.height = Math.round(box.height);
|
||||
canvas
|
||||
.getContext("2d")
|
||||
.drawImage(image, box.x, box.y, box.width, box.height, 0, 0, canvas.width, canvas.height);
|
||||
return canvas.toDataURL("image/png").split(",")[1];
|
||||
},
|
||||
{ full, box: before },
|
||||
);
|
||||
const after = await bounds();
|
||||
if (Object.keys(before).every((key) => Math.abs(before[key] - after[key]) < 0.5)) break;
|
||||
}
|
||||
assert(attempts < 3, `${name}: composition kept re-fitting during capture`);
|
||||
await writeFile(join(output, `${name}.png`), Buffer.from(png, "base64"));
|
||||
return { png, captureAttempts: attempts + 1 };
|
||||
}
|
||||
|
||||
async function samplePicture(host, png) {
|
||||
return host.page.evaluate(async (png) => {
|
||||
const image = new Image();
|
||||
image.src = `data:image/png;base64,${png}`;
|
||||
await image.decode();
|
||||
const canvas = new OffscreenCanvas(image.width, image.height);
|
||||
const context = canvas.getContext("2d");
|
||||
context.drawImage(image, 0, 0);
|
||||
const sample = (x, y) => {
|
||||
const data = context.getImageData(
|
||||
Math.round(image.width * x),
|
||||
Math.round(image.height * y),
|
||||
4,
|
||||
4,
|
||||
).data;
|
||||
const sums = [0, 0, 0];
|
||||
for (let i = 0; i < data.length; i += 4) for (let c = 0; c < 3; c++) sums[c] += data[i + c];
|
||||
return sums.map((sum) => Math.round(sum / 16));
|
||||
};
|
||||
return {
|
||||
reference: Array.from({ length: 7 }, (_, i) => sample((406 + i * 32) / 640, 322 / 360)),
|
||||
background: sample(500 / 640, 200 / 360),
|
||||
foreground: sample(140 / 640, 160 / 360),
|
||||
markers: Array.from({ length: 6 }, (_, i) => sample((30 + i * 50) / 640, 307 / 360)),
|
||||
};
|
||||
}, png);
|
||||
}
|
||||
|
||||
async function pictureState(host) {
|
||||
return host.frame.evaluate(() => {
|
||||
const visible = [...document.querySelectorAll(".label")].filter((el) => {
|
||||
const style = getComputedStyle(el);
|
||||
return (
|
||||
style.display !== "none" &&
|
||||
style.visibility !== "hidden" &&
|
||||
el.getBoundingClientRect().width > 0
|
||||
);
|
||||
});
|
||||
return {
|
||||
time: window.__player.getTime(),
|
||||
playing: window.__player.isPlaying(),
|
||||
shownTime: Number(document.querySelector(".clock").textContent),
|
||||
labels: visible.map((el) => el.textContent.trim()),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function assertBackground(name, scene, rgb) {
|
||||
const [red, green, blue] = rgb;
|
||||
const matches = scene === "A" ? [red > 160, green < 100, blue < 100] : [blue > 170, red < 100];
|
||||
assert(matches.every(Boolean), `${name}: background disagrees with ${scene}: ${rgb}`);
|
||||
}
|
||||
function assertMarker(name, scene, state, pixels) {
|
||||
const sourceSecond = Math.min(5, Math.floor(state.shownTime - (scene === "B" ? 6 : 0)));
|
||||
const visible = pixels.markers.flatMap((rgb, i) => (Math.min(...rgb) > 210 ? [i] : []));
|
||||
assert.deepEqual(visible, [sourceSecond], `${name}: video pixels disagree with the HTML clock`);
|
||||
}
|
||||
export async function observePicture(host, name, { output, retained = false }) {
|
||||
const { png, captureAttempts } = await capturePicture(host, name, output);
|
||||
const pixels = await samplePicture(host, png);
|
||||
const state = await pictureState(host);
|
||||
assert.equal(state.labels.length, 1, `${name}: expected exactly one scene label`);
|
||||
const bits = pixels.reference.map((rgb) => {
|
||||
assert(
|
||||
Math.max(...rgb) < 40 || Math.min(...rgb) > 210,
|
||||
`${name}: HTML clock strip is obscured`,
|
||||
);
|
||||
return Math.min(...rgb) > 210 ? 1 : 0;
|
||||
});
|
||||
assert.deepEqual(bits.slice(0, 2), [1, 0], `${name}: HTML clock strip guards are absent`);
|
||||
const scene = bits[2] ? "B" : "A";
|
||||
const shownSecond = bits.slice(3).reduce((value, bit, index) => value + bit * 2 ** index, 0);
|
||||
const capturedHtml = { scene, shownSecond, shownTime: shownSecond };
|
||||
assertBackground(name, scene, pixels.background);
|
||||
assert(pixels.foreground[1] > 150, `${name}: foreground green is missing: ${pixels.foreground}`);
|
||||
assert(
|
||||
pixels.foreground[0] < 80,
|
||||
`${name}: foreground contains background red: ${pixels.foreground}`,
|
||||
);
|
||||
if (!retained) assertMarker(name, scene, capturedHtml, pixels);
|
||||
return {
|
||||
name,
|
||||
...state,
|
||||
capturedHtml,
|
||||
scene,
|
||||
pixels,
|
||||
markerValidated: !retained,
|
||||
captureAttempts,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { observePicture } from "./preview-video-picture.mjs";
|
||||
import { verifyCaptureIsolation } from "./preview-video-capture-witness.mjs";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { launchStudioChrome } from "./chrome-executable.mjs";
|
||||
|
||||
const url = process.env.STUDIO_URL;
|
||||
assert(url, "STUDIO_URL must point at the preview-video-regression fixture");
|
||||
const project = process.env.STUDIO_PROJECT_DIR;
|
||||
assert(project, "STUDIO_PROJECT_DIR must be the generated preview-video-regression project");
|
||||
const outputRoot = process.env.OUT_DIR;
|
||||
if (outputRoot) await mkdir(outputRoot, { recursive: true });
|
||||
const output = await mkdtemp(
|
||||
join(outputRoot ?? tmpdir(), outputRoot ? "run-" : "preview-video-regression-"),
|
||||
);
|
||||
const { browser, executablePath } = await launchStudioChrome();
|
||||
const evidence = {
|
||||
executablePath,
|
||||
browser: await browser.version(),
|
||||
gpuDisabled: true,
|
||||
output,
|
||||
pictures: [],
|
||||
timings: [],
|
||||
};
|
||||
const errors = [];
|
||||
const unavailableCliEndpoints = [];
|
||||
const runtimeLoads = [];
|
||||
const runtimeHashes = new Set();
|
||||
|
||||
function optionalCliError(item) {
|
||||
const path = item.url ? new URL(item.url).pathname : "";
|
||||
return (
|
||||
item.message.includes("status of 404") &&
|
||||
["/api/environment/ffmpeg", "/api/open-in-desktop"].includes(path)
|
||||
);
|
||||
}
|
||||
function recordConsole(message) {
|
||||
if (message.type() !== "error") return;
|
||||
const item = { message: message.text(), url: message.location().url };
|
||||
if (optionalCliError(item)) unavailableCliEndpoints.push(item);
|
||||
else errors.push(item);
|
||||
}
|
||||
|
||||
async function open(beforeNavigate) {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1440, height: 900 });
|
||||
await page.setCacheEnabled(false);
|
||||
page.on("response", (response) => {
|
||||
if (new URL(response.url()).pathname === "/api/runtime.js") {
|
||||
runtimeLoads.push(
|
||||
response.buffer().then((bytes) => {
|
||||
runtimeHashes.add(createHash("sha256").update(bytes).digest("hex"));
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
page.on("pageerror", (error) => errors.push(error.message));
|
||||
page.on("console", recordConsole);
|
||||
if (beforeNavigate) await beforeNavigate(page);
|
||||
await page.goto(url, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForSelector('button[aria-label="Play"]:not([disabled])', { timeout: 60_000 });
|
||||
const handle = await page.waitForFunction(() =>
|
||||
document.querySelector("hyperframes-player")?.shadowRoot?.querySelector("iframe"),
|
||||
);
|
||||
const iframe = handle.asElement();
|
||||
assert(iframe, "Composition iframe is absent");
|
||||
const frame = await iframe.contentFrame();
|
||||
assert(frame, "Composition document is absent");
|
||||
await frame.waitForFunction(() => window.__playerReady, { timeout: 30_000 });
|
||||
assert(await frame.$("#film"), "Synthetic composition is absent");
|
||||
return { page, iframe, frame };
|
||||
}
|
||||
|
||||
async function seek(host, time) {
|
||||
const start = performance.now();
|
||||
await host.frame.evaluate(async (time) => {
|
||||
await window.__player.seek(time, { settled: true });
|
||||
}, time);
|
||||
await host.frame
|
||||
.waitForFunction(
|
||||
(time) => {
|
||||
const label = document.querySelector(".clock");
|
||||
return Math.abs(Number(label.textContent) - time) < 0.002;
|
||||
},
|
||||
{},
|
||||
time,
|
||||
)
|
||||
.catch(async (error) => {
|
||||
const state = await host.frame.evaluate(() => ({
|
||||
time: window.__player.getTime(),
|
||||
labels: [...document.querySelectorAll(".clock")].map((el) => el.textContent),
|
||||
canvases: document.querySelectorAll("canvas").length,
|
||||
ready: window.__playerReady,
|
||||
}));
|
||||
await host.page.screenshot({ path: join(output, "seek-failed.png") });
|
||||
throw new Error(`Seek to ${time} did not publish: ${JSON.stringify(state)}`, {
|
||||
cause: error,
|
||||
});
|
||||
});
|
||||
evidence.timings.push({
|
||||
action: "seek",
|
||||
target: time,
|
||||
automationObservedMs: performance.now() - start,
|
||||
});
|
||||
}
|
||||
|
||||
async function play(host) {
|
||||
await host.page.click('button[aria-label="Play"]');
|
||||
await host.frame.waitForFunction(() => window.__player.isPlaying(), { timeout: 15_000 });
|
||||
}
|
||||
|
||||
async function pause(host) {
|
||||
await host.page.click('button[aria-label="Pause"]');
|
||||
await host.frame.waitForFunction(() => !window.__player.isPlaying());
|
||||
}
|
||||
|
||||
async function picture(host, name, retained = false) {
|
||||
const result = await observePicture(host, name, { output, retained });
|
||||
evidence.pictures.push(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function detail(host) {
|
||||
return host.frame.evaluate(() => {
|
||||
const video = document.getElementById("foreground");
|
||||
const canvas = video.previousElementSibling;
|
||||
if (!(canvas instanceof HTMLCanvasElement)) return null;
|
||||
const data = canvas.getContext("2d").getImageData(300, 20, 128, 1).data;
|
||||
let contrast = 0;
|
||||
for (let x = 1; x < 128; x++) contrast += Math.abs(data[x * 4] - data[(x - 1) * 4]);
|
||||
return {
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
contrast: contrast / 127,
|
||||
firstPixel: data[0],
|
||||
mediaTime: video.currentTime,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function collectGarbage(host) {
|
||||
const session = await host.page.createCDPSession();
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await session.send("HeapProfiler.collectGarbage");
|
||||
await host.page.evaluate(
|
||||
() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))),
|
||||
);
|
||||
}
|
||||
await session.detach();
|
||||
}
|
||||
|
||||
async function withBlockedNative(test) {
|
||||
const assets = new Map(
|
||||
await Promise.all(
|
||||
["background-a.mp4", "background-b.mp4"].map(async (name) => [
|
||||
name,
|
||||
await readFile(join(project, "assets", name)),
|
||||
]),
|
||||
),
|
||||
);
|
||||
const prefixed = new Set();
|
||||
const held = new Set();
|
||||
let blocked = false;
|
||||
const host = await open(async (page) => {
|
||||
await page.setRequestInterception(true);
|
||||
page.on("requestfailed", (request) => held.delete(request));
|
||||
const respondPrefix = (request, name, data) => {
|
||||
if (prefixed.has(name)) {
|
||||
held.add(request);
|
||||
return;
|
||||
}
|
||||
prefixed.add(name);
|
||||
const prefix = data.subarray(0, 16 * 1024);
|
||||
void request.respond({
|
||||
status: 206,
|
||||
contentType: "video/mp4",
|
||||
headers: {
|
||||
"Content-Range": `bytes 0-${prefix.length - 1}/${data.length}`,
|
||||
"Content-Length": String(prefix.length),
|
||||
"Accept-Ranges": "bytes",
|
||||
},
|
||||
body: prefix,
|
||||
});
|
||||
};
|
||||
page.on("request", (request) => {
|
||||
const name = new URL(request.url()).pathname.split("/").at(-1);
|
||||
const data = assets.get(name);
|
||||
if (request.resourceType() === "media" && data && blocked) respondPrefix(request, name, data);
|
||||
else void request.continue();
|
||||
});
|
||||
});
|
||||
const release = async () => {
|
||||
blocked = false;
|
||||
for (const request of [...held]) {
|
||||
held.delete(request);
|
||||
if (!request.failure() && !request.isInterceptResolutionHandled()) await request.continue();
|
||||
}
|
||||
};
|
||||
const arm = async () => {
|
||||
blocked = true;
|
||||
await host.frame.evaluate(() => {
|
||||
window.__previewTestSeeks = [];
|
||||
const video = document.getElementById("background-b");
|
||||
let proto = Object.getPrototypeOf(video);
|
||||
while (!Object.hasOwn(proto, "currentTime")) proto = Object.getPrototypeOf(proto);
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, "currentTime");
|
||||
Object.defineProperty(proto, "currentTime", {
|
||||
...descriptor,
|
||||
set(time) {
|
||||
window.__previewTestSeeks.push({ id: this.id, time });
|
||||
descriptor.set.call(this, time);
|
||||
},
|
||||
});
|
||||
video.load();
|
||||
});
|
||||
const heldAt = performance.now();
|
||||
while (held.size === 0 && performance.now() - heldAt < 5000) {
|
||||
await host.page.evaluate(() => new Promise(requestAnimationFrame));
|
||||
}
|
||||
assert(held.size > 0, "Native delivery was not actually held");
|
||||
};
|
||||
try {
|
||||
await test({ host, arm, release });
|
||||
} finally {
|
||||
await release();
|
||||
await host.page.close();
|
||||
}
|
||||
}
|
||||
async function nativeOnlyHandoff() {
|
||||
await withBlockedNative(async ({ host, arm, release }) => {
|
||||
await host.frame.evaluate(() => window.__hf.setPausedVideoFrameProvider(null));
|
||||
await seek(host, 0);
|
||||
await host.frame.waitForFunction(
|
||||
() => !document.querySelector("[data-hf-preview-video-canvas]"),
|
||||
);
|
||||
const baseline = await picture(host, "native-only-paused");
|
||||
assert.equal(baseline.scene, "A");
|
||||
assert.equal(baseline.capturedHtml.shownTime, 0);
|
||||
evidence.nativeOnlyBaseline = {
|
||||
scene: baseline.scene,
|
||||
shownTime: baseline.capturedHtml.shownTime,
|
||||
};
|
||||
await arm();
|
||||
await host.frame.evaluate(() => {
|
||||
void window.__player.seek(7, { settled: true });
|
||||
});
|
||||
await host.page.click('button[aria-label="Play"]');
|
||||
const retained = await picture(host, "native-only-handoff", true);
|
||||
assert.equal(
|
||||
retained.scene,
|
||||
"A",
|
||||
"Native-only Play published before the destination was ready",
|
||||
);
|
||||
assert.equal(retained.playing, false);
|
||||
assert.equal(
|
||||
await host.frame.evaluate(() => document.querySelectorAll("[data-hf-seek-video]").length),
|
||||
2,
|
||||
);
|
||||
await release();
|
||||
await host.frame.waitForFunction(() => window.__player.isPlaying());
|
||||
await host.frame.waitForFunction(() => window.__player.getTime() > 7.1);
|
||||
const next = await picture(host, "native-only-recovered");
|
||||
assert.equal(next.scene, "B");
|
||||
assert.equal(
|
||||
await host.frame.evaluate(() => document.querySelectorAll("[data-hf-seek-video]").length),
|
||||
0,
|
||||
);
|
||||
evidence.nativeOnlyHandoff = { retained, recovered: next };
|
||||
await collectGarbage(host);
|
||||
});
|
||||
}
|
||||
async function delayedNativeRecovery() {
|
||||
await withBlockedNative(async ({ host, arm, release }) => {
|
||||
await seek(host, 7);
|
||||
await picture(host, "delayed-paused");
|
||||
await arm();
|
||||
const start = performance.now();
|
||||
await play(host);
|
||||
const handoffMs = performance.now() - start;
|
||||
assert(handoffMs >= 9000 && handoffMs < 14_000, `Play deadline not exercised: ${handoffMs}ms`);
|
||||
const stalled = await host.frame.evaluate(() => {
|
||||
const video = document.getElementById("background-b");
|
||||
return {
|
||||
readyState: video.readyState,
|
||||
seeking: video.seeking,
|
||||
cover: video.previousElementSibling?.hasAttribute("data-hf-preview-video-canvas"),
|
||||
};
|
||||
});
|
||||
assert(
|
||||
stalled.readyState < 2 || stalled.seeking,
|
||||
"Native video was ready; the delay probe is invalid",
|
||||
);
|
||||
assert(stalled.cover, "Full-quality cover disappeared at the deadline");
|
||||
await picture(host, "delayed-deadline", true);
|
||||
await host.frame.waitForFunction(() => window.__player.getTime() > 8.5);
|
||||
evidence.beforeNativeRelease = await host.frame.evaluate(() => {
|
||||
const video = document.getElementById("background-b");
|
||||
const state = {
|
||||
time: window.__player.getTime(),
|
||||
native: video.currentTime,
|
||||
readyState: video.readyState,
|
||||
seeking: video.seeking,
|
||||
seeks: [...window.__previewTestSeeks],
|
||||
};
|
||||
window.__previewTestSeeks.length = 0;
|
||||
return state;
|
||||
});
|
||||
const deliveredAt = performance.now();
|
||||
await release();
|
||||
await host.frame.waitForFunction(
|
||||
() => !document.querySelector("[data-hf-preview-video-canvas]"),
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
const recovery = await host.frame.evaluate(() => ({
|
||||
time: window.__player.getTime(),
|
||||
native: document.getElementById("background-b").currentTime,
|
||||
seeks: window.__previewTestSeeks.filter((seek) => seek.id === "background-b"),
|
||||
}));
|
||||
assert.equal(
|
||||
recovery.seeks.length,
|
||||
1,
|
||||
`Recovery must correct once, not chase the clock: ${JSON.stringify(recovery)}`,
|
||||
);
|
||||
assert(
|
||||
Math.abs(recovery.native - (recovery.time - 6)) < 0.3,
|
||||
`Recovered video remained behind: ${JSON.stringify(recovery)}`,
|
||||
);
|
||||
const recoveredPicture = await picture(host, "delayed-recovered");
|
||||
assert(
|
||||
recoveredPicture.markerValidated,
|
||||
"Recovery picture was not checked against its burnt-in time marker",
|
||||
);
|
||||
evidence.delayedNative = {
|
||||
handoffMs,
|
||||
stalled,
|
||||
recoveredAfterDeliveryMs: performance.now() - deliveredAt,
|
||||
...recovery,
|
||||
};
|
||||
await collectGarbage(host);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const host = await open();
|
||||
await seek(host, 2);
|
||||
await picture(host, "paused-a");
|
||||
evidence.captureIsolation = await verifyCaptureIsolation(host, { picture, seek });
|
||||
const before = await detail(host);
|
||||
assert(before, "Paused foreground has no decoded canvas");
|
||||
assert.deepEqual([before.width, before.height], [640, 360]);
|
||||
assert(before.contrast > 180, `Source detail was blurred: ${before.contrast}`);
|
||||
await host.page.click("::-p-text(preview-video-regression)");
|
||||
await host.page.keyboard.press("ArrowRight");
|
||||
await host.frame.waitForFunction(
|
||||
() => Math.abs(Number(document.querySelector(".clock").textContent) - (2 + 1 / 30)) < 0.002,
|
||||
);
|
||||
await host.frame.waitForFunction(
|
||||
(previous) => {
|
||||
const canvas = document.getElementById("foreground").previousElementSibling;
|
||||
return (
|
||||
canvas instanceof HTMLCanvasElement &&
|
||||
Math.abs(canvas.getContext("2d").getImageData(300, 20, 1, 1).data[0] - previous) > 180
|
||||
);
|
||||
},
|
||||
{},
|
||||
before.firstPixel,
|
||||
);
|
||||
const after = await detail(host);
|
||||
assert.deepEqual([after.width, after.height], [640, 360]);
|
||||
assert(after.contrast > 180);
|
||||
evidence.frameStep = { before, after };
|
||||
|
||||
const point = await host.frame.evaluate(() => {
|
||||
const video = document.getElementById("foreground").getBoundingClientRect();
|
||||
return { x: video.x + 140, y: video.y + 160, width: innerWidth, height: innerHeight };
|
||||
});
|
||||
const box = await host.iframe.boundingBox();
|
||||
assert(box);
|
||||
await host.page.mouse.click(
|
||||
box.x + (point.x * box.width) / point.width,
|
||||
box.y + (point.y * box.height) / point.height,
|
||||
);
|
||||
await host.page.waitForFunction(() => document.body.textContent.includes("#foreground"));
|
||||
evidence.picking = "foreground source video";
|
||||
|
||||
const slider = await host.page.$('input[aria-label="Seek preview"]');
|
||||
const track = await slider.boundingBox();
|
||||
assert(track);
|
||||
await host.page.mouse.move(track.x + track.width * (2 / 12), track.y + track.height / 2);
|
||||
await host.page.mouse.down();
|
||||
for (const fraction of [0.2, 0.7, 0.3, 0.8]) {
|
||||
await host.page.mouse.move(track.x + track.width * fraction, track.y + track.height / 2, {
|
||||
steps: 5,
|
||||
});
|
||||
await picture(host, `drag-${fraction}`);
|
||||
}
|
||||
await host.page.mouse.up();
|
||||
await seek(host, 7);
|
||||
await picture(host, "paused-b");
|
||||
await seek(host, 5.5);
|
||||
await play(host);
|
||||
await host.frame.waitForFunction(() => window.__player.getTime() > 6.25);
|
||||
const cut = await picture(host, "native-cut");
|
||||
assert.equal(cut.scene, "B");
|
||||
await pause(host);
|
||||
|
||||
for (const time of [1, 7, 3, 9]) {
|
||||
await seek(host, time);
|
||||
await picture(host, `jump-${time}`);
|
||||
await play(host);
|
||||
await host.frame.waitForFunction((time) => window.__player.getTime() > time + 0.1, {}, time);
|
||||
await picture(host, `play-${time}`);
|
||||
await pause(host);
|
||||
}
|
||||
await play(host);
|
||||
await pause(host);
|
||||
await play(host);
|
||||
await pause(host);
|
||||
assert.equal(await host.frame.evaluate(() => window.__player.isPlaying()), false);
|
||||
await play(host);
|
||||
await host.frame.waitForFunction(
|
||||
() => document.querySelectorAll("[data-hf-preview-video-canvas]").length === 0,
|
||||
);
|
||||
await collectGarbage(host);
|
||||
assert.deepEqual(errors, [], "Browser logged an error or unclosed VideoSample");
|
||||
await host.page.close();
|
||||
await delayedNativeRecovery();
|
||||
await nativeOnlyHandoff();
|
||||
assert.deepEqual(errors, [], "Delayed native recovery logged an error or unclosed VideoSample");
|
||||
await Promise.all(runtimeLoads);
|
||||
evidence.runtimeSha256 = [...runtimeHashes];
|
||||
assert.equal(runtimeHashes.size, 1, "All test documents must load the same runtime bytes");
|
||||
evidence.ok = true;
|
||||
} catch (error) {
|
||||
evidence.ok = false;
|
||||
evidence.failure = error.stack;
|
||||
await (await browser.pages()).at(-1)?.screenshot({ path: join(output, "failed.png") });
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
evidence.errors = errors;
|
||||
evidence.unavailableCliEndpoints = unavailableCliEndpoints;
|
||||
await writeFile(join(output, "results.json"), JSON.stringify(evidence, null, 2));
|
||||
console.log(JSON.stringify(evidence, null, 2));
|
||||
await browser.close();
|
||||
}
|
||||
Reference in New Issue
Block a user