mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 00:18:27 +00:00
fix(core): snapshot and preview show a clip on the frame render shows it (#7238)
* fix(core): snapshot and preview show a clip on the frame render shows it * fix(cli): snapshot pulls a video's frame on the same on-screen rule * refactor(cli): drop two doc lines the names already carry
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import sharp from "sharp";
|
||||
import { afterAll, expect, it, vi } from "vitest";
|
||||
import snapshotCommand from "./snapshot.js";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-held-boundary-"));
|
||||
|
||||
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
// A title held across two back-to-back clips; the second starts a hair past 100 s, as a start chained after a media
|
||||
// clip's measured length lands.
|
||||
writeFileSync(
|
||||
join(dir, "index.html"),
|
||||
`<!doctype html><html><body style="margin:0;background:#000">
|
||||
<div id="root" data-composition-id="main" data-start="0" data-duration="110" data-width="160" data-height="90" style="position:relative;width:160px;height:90px;background:#000">
|
||||
<div id="held" data-start="90" data-duration="10" data-track-index="0" style="position:absolute;inset:0;background:#f00"></div>
|
||||
<div id="rest" data-start="100.00002" data-duration="10" data-track-index="0" style="position:absolute;inset:0;background:#f00"></div>
|
||||
</div>
|
||||
<script>
|
||||
window.__timelines = window.__timelines || {};
|
||||
window.__timelines["main"] = { pause() {}, time() { return 0; }, duration() { return 110; }, seek() {} };
|
||||
</script></body></html>`,
|
||||
);
|
||||
|
||||
const redAtCentre = async (path: string) => {
|
||||
const { data, info } = await sharp(path).raw().toBuffer({ resolveWithObject: true });
|
||||
const at =
|
||||
(Math.floor(info.height / 2) * info.width + Math.floor(info.width / 2)) * info.channels;
|
||||
return data[at]! > 200 && data[at + 1]! < 60;
|
||||
};
|
||||
|
||||
// fails without the fix: the still at exactly 100 s came out black, while 98 s, 102 s and the render show the title.
|
||||
it("keeps a title held across back-to-back clips on screen at the exact second the second clip takes over", async () => {
|
||||
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
|
||||
vi.spyOn(console, "error").mockImplementation(() => undefined);
|
||||
vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||
try {
|
||||
await snapshotCommand.run?.({
|
||||
args: {
|
||||
dir,
|
||||
at: "98,100,102",
|
||||
end: false,
|
||||
json: true,
|
||||
output: join(dir, "out"),
|
||||
describe: "false",
|
||||
"browser-gpu": false,
|
||||
},
|
||||
} as never);
|
||||
const { frames } = JSON.parse(String(log.mock.calls.at(-1)?.[0])) as {
|
||||
frames: { path: string; time: number }[];
|
||||
};
|
||||
const shown = await Promise.all(
|
||||
frames.map(async (frame) => [frame.time, await redAtCentre(frame.path)]),
|
||||
);
|
||||
expect(shown).toEqual([
|
||||
[98, true],
|
||||
[100, true],
|
||||
[102, true],
|
||||
]);
|
||||
} finally {
|
||||
vi.restoreAllMocks();
|
||||
}
|
||||
}, 120_000);
|
||||
@@ -322,6 +322,21 @@ describe("snapshot video overlays and the blurred frame", () => {
|
||||
});
|
||||
|
||||
describe("resolveSnapshotVideoFrameTime", () => {
|
||||
// fails without the fix: the element showed at 100 s on render's window, but no frame was pulled for its video.
|
||||
it("pulls a frame for a video starting a hair past 100 s at exactly 100 s, as render does", () => {
|
||||
expect(
|
||||
resolveSnapshotVideoFrameTime({
|
||||
globalTime: 100,
|
||||
clipStart: 100.00002,
|
||||
clipDuration: 10,
|
||||
relativeTime: -0.00002,
|
||||
sourceDuration: 10,
|
||||
compositionDuration: 110,
|
||||
fps: 30,
|
||||
}),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("holds a clip ending with the composition on its last decodable frame", () => {
|
||||
expect(
|
||||
resolveSnapshotVideoFrameTime({
|
||||
|
||||
@@ -70,12 +70,14 @@ import {
|
||||
} from "../capture/captureCompositionFrame.js";
|
||||
import {
|
||||
firstFrameColourArgs,
|
||||
exportClipWindow,
|
||||
hdrToSdrToneMapFilter,
|
||||
keyframeTransferArgs,
|
||||
parseKeyframeTransfers,
|
||||
type ToneMapFrameColour,
|
||||
isClipVisibleAt,
|
||||
isClipOnScreenAt,
|
||||
isInClipWindow,
|
||||
parseFpsWithDefault,
|
||||
MEDIA_RENDER_ID_ATTR,
|
||||
parseFirstFrameColour,
|
||||
readsKeyframeTransfers,
|
||||
@@ -84,6 +86,7 @@ import {
|
||||
timeAtSourceTime,
|
||||
type RateSpec,
|
||||
} from "@hyperframes/core";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { resolveProject } from "../utils/project.js";
|
||||
import {
|
||||
definitiveEntryMismatchComposition,
|
||||
@@ -250,15 +253,18 @@ export function resolveSnapshotVideoFrameTime(input: {
|
||||
relativeTime: number;
|
||||
sourceDuration: number;
|
||||
compositionDuration: number;
|
||||
fps?: number;
|
||||
}): number | null {
|
||||
const { globalTime, clipStart, clipDuration, sourceDuration, compositionDuration } = input;
|
||||
const fps = input.fps ?? 30;
|
||||
const clipEnd = clipStart + clipDuration;
|
||||
if (!isClipVisibleAt(globalTime, clipStart, clipEnd, compositionDuration)) return null;
|
||||
if (!isClipOnScreenAt(globalTime, clipStart, clipEnd, fps, compositionDuration)) return null;
|
||||
const onFrames = exportClipWindow(clipStart, clipEnd, fps);
|
||||
const relativeTime =
|
||||
globalTime < clipStart ? Math.max(0, input.relativeTime) : input.relativeTime;
|
||||
if (relativeTime < 0) return null;
|
||||
const pastSource = sourceDuration > 0 && relativeTime >= sourceDuration;
|
||||
if (isInClipWindow(globalTime, clipStart, clipEnd) && !pastSource) return relativeTime;
|
||||
if (isInClipWindow(globalTime, onFrames.start, onFrames.end) && !pastSource) return relativeTime;
|
||||
|
||||
const sourceEnd = sourceDuration > 0 ? sourceDuration : relativeTime;
|
||||
return Math.max(0, Math.min(relativeTime, sourceEnd - 1 / 30));
|
||||
@@ -312,6 +318,7 @@ export function activeVideoAt(
|
||||
candidate: SnapshotVideoCandidate,
|
||||
time: number,
|
||||
compositionDuration: number,
|
||||
fps: number,
|
||||
) {
|
||||
const start = resolveSnapshotVideoClipStart(candidate);
|
||||
const playbackRate = resolveSnapshotVideoRateSpec(candidate);
|
||||
@@ -337,6 +344,7 @@ export function activeVideoAt(
|
||||
relativeTime: relTime,
|
||||
sourceDuration: candidate.srcDuration,
|
||||
compositionDuration,
|
||||
fps,
|
||||
});
|
||||
return frameTime === null
|
||||
? null
|
||||
@@ -901,6 +909,12 @@ export function computeSnapshotTimes(
|
||||
return { times: times.map(round), appendedTail: false };
|
||||
}
|
||||
|
||||
function compositionFps(html: string): number {
|
||||
const root = parseHTML(html).document.querySelector("[data-composition-id]");
|
||||
const declared = parseFpsWithDefault(root?.getAttribute("data-fps") ?? undefined);
|
||||
return declared.ok ? declared.value.num / declared.value.den : 30;
|
||||
}
|
||||
|
||||
export function serveCapturePage(projectDir: string, html: string, proxy: boolean | undefined) {
|
||||
return serveStaticProjectHtml(projectDir, html, undefined, [], proxy === true);
|
||||
}
|
||||
@@ -971,6 +985,7 @@ export async function captureSnapshots(
|
||||
console.warn(` ${c.warn("⚠")} ${swallowed}`);
|
||||
}
|
||||
const server = await serveCapturePage(projectDir, html, opts.autoProxy);
|
||||
const fps = compositionFps(html);
|
||||
const heldVideoIds = localVideoRenderIds(html);
|
||||
const heldProbes = new Map<string, Promise<HeldVideoProbe>>();
|
||||
const heldHints = await heldVideoHints(projectDir, localVideos(html), heldProbes);
|
||||
@@ -1270,7 +1285,7 @@ export async function captureSnapshots(
|
||||
);
|
||||
let unreadable = 0;
|
||||
const active = candidates.flatMap((candidate) => {
|
||||
const video = activeVideoAt(candidate, time, duration);
|
||||
const video = activeVideoAt(candidate, time, duration, fps);
|
||||
if (!video) return [];
|
||||
if (!candidate.id || !candidate.src) {
|
||||
unreadable += 1;
|
||||
@@ -1314,7 +1329,7 @@ export async function captureSnapshots(
|
||||
(opts.zoom ? (opts.zoomScale ?? DEFAULT_ZOOM_SCALE) : v.devicePixelRatio),
|
||||
);
|
||||
const wantAt = (t: number): FrameWant | null => {
|
||||
const shown = activeVideoAt(v, t, duration);
|
||||
const shown = activeVideoAt(v, t, duration, fps);
|
||||
if (!shown) return null;
|
||||
const holdLastFrame =
|
||||
shown.srcDuration > 0 && shown.relTime >= shown.srcDuration - 1;
|
||||
|
||||
@@ -97,7 +97,6 @@ describe("scanPendingCompositionAssets", () => {
|
||||
currentTime: 0,
|
||||
compositionDuration: Number.POSITIVE_INFINITY,
|
||||
canonicalFps: 30,
|
||||
exportRenderSeek: false,
|
||||
timelineRegistry: {},
|
||||
resolver,
|
||||
});
|
||||
|
||||
@@ -67,7 +67,6 @@ function isActiveAtFirstFrame(
|
||||
currentTime: 0,
|
||||
compositionDuration: Number.POSITIVE_INFINITY,
|
||||
canonicalFps: 30,
|
||||
exportRenderSeek: false,
|
||||
timelineRegistry,
|
||||
resolver,
|
||||
})
|
||||
|
||||
@@ -397,7 +397,12 @@ export {
|
||||
// (verify:packed-manifests catches exactly that).
|
||||
export { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
|
||||
// Also exposed via the ./runtime/clip-window subpath; re-exported here for the same dist-emit reason.
|
||||
export { hasClipStarted, isClipVisibleAt, isInClipWindow } from "./runtime/clipWindow.js";
|
||||
export {
|
||||
hasClipStarted,
|
||||
isClipOnScreenAt,
|
||||
isClipVisibleAt,
|
||||
isInClipWindow,
|
||||
} from "./runtime/clipWindow.js";
|
||||
export { MEDIA_ELEMENT_LEAD_SECONDS, mediaElementTime } from "./runtime/mediaClock.js";
|
||||
export {
|
||||
normalizePlaybackRate,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { sameInstant } from "../clipFacts";
|
||||
import { exportClipWindow } from "../inline-scripts/parityContract";
|
||||
|
||||
export const hasClipStarted = (time: number, start: number) =>
|
||||
time >= start || sameInstant(time, start);
|
||||
@@ -23,3 +24,15 @@ export const isClipVisibleAt = (
|
||||
(hasClipStarted(time, start) &&
|
||||
compositionDuration > 0 &&
|
||||
end >= compositionDuration - TERMINAL_EPSILON_SECONDS);
|
||||
|
||||
/** The one on-screen rule render, snapshot and preview share: edges a hair off a frame count as on it. */
|
||||
export const isClipOnScreenAt = (
|
||||
time: number,
|
||||
start: number,
|
||||
end: number,
|
||||
fps: number,
|
||||
compositionDuration: number,
|
||||
): boolean => {
|
||||
const snapped = exportClipWindow(start, end, fps);
|
||||
return isClipVisibleAt(time, snapped.start, snapped.end, compositionDuration);
|
||||
};
|
||||
|
||||
@@ -14,13 +14,12 @@ function deferred() {
|
||||
}
|
||||
|
||||
const adapters: ReturnType<typeof createFrameSourceAdapter>[] = [];
|
||||
const adapter = (compositionDuration = 20, exportRenderSeek = false) => {
|
||||
const adapter = (compositionDuration = 20) => {
|
||||
const runtime = createFrameSourceAdapter({
|
||||
start: (element) => createRuntimeStartTimeResolver({}).resolveStartForElement(element, 0),
|
||||
duration: (element) => createRuntimeStartTimeResolver({}).resolveDurationForElement(element),
|
||||
compositionDuration: () => compositionDuration,
|
||||
canonicalFps: () => 30,
|
||||
exportRenderSeek: () => exportRenderSeek,
|
||||
});
|
||||
adapters.push(runtime);
|
||||
return runtime;
|
||||
@@ -293,10 +292,10 @@ describe("frame sources", () => {
|
||||
expect(render).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("matches snapped export visibility at near-frame starts and ends", async () => {
|
||||
it("shows a source on render's snapped window at near-frame starts and ends, in preview too", async () => {
|
||||
const render = vi.fn();
|
||||
disposers.push(registerFrameSource({ element: mount("1.00001", "1"), render }));
|
||||
const runtime = adapter(3, true);
|
||||
const runtime = adapter(3);
|
||||
runtime.seek({ time: 1 });
|
||||
await waitForSeekCompletion();
|
||||
expect(render).toHaveBeenLastCalledWith(0, expect.any(AbortSignal));
|
||||
@@ -308,18 +307,6 @@ describe("frame sources", () => {
|
||||
expect(render).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("retains unsnapped authored timing during interactive preview", async () => {
|
||||
const render = vi.fn();
|
||||
disposers.push(registerFrameSource({ element: mount("1.00001", "1"), render }));
|
||||
const runtime = adapter(3);
|
||||
runtime.seek({ time: 1 });
|
||||
await waitForSeekCompletion();
|
||||
expect(render).not.toHaveBeenCalled();
|
||||
runtime.seek({ time: 2 });
|
||||
await waitForSeekCompletion();
|
||||
expect(render).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects invalid original ranges before registering a source", () => {
|
||||
const element = mount();
|
||||
for (const sourceRange of [
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import { exportClipWindow } from "../inline-scripts/parityContract";
|
||||
import { sourceTimeAt } from "../speedRamp";
|
||||
import { readElementRateSpec, readMediaStart } from "./playbackRate";
|
||||
import { registerSeekCompletion } from "./adapters/seek-dispatch";
|
||||
import { isClipVisibleAt } from "./clipWindow";
|
||||
import { isClipOnScreenAt } from "./clipWindow";
|
||||
import type { RuntimeDeterministicAdapter } from "./types";
|
||||
|
||||
export interface FrameSource {
|
||||
@@ -93,7 +92,6 @@ export function createFrameSourceAdapter(timing: {
|
||||
duration: (element: Element) => number | null;
|
||||
compositionDuration: () => number;
|
||||
canonicalFps: () => number;
|
||||
exportRenderSeek: () => boolean;
|
||||
}): RuntimeDeterministicAdapter {
|
||||
const owned = new Set<RegisteredSource>();
|
||||
let readySources: RegisteredSource[] = [];
|
||||
@@ -132,11 +130,14 @@ export function createFrameSourceAdapter(timing: {
|
||||
const start = timing.start(element);
|
||||
const duration = timing.duration(element);
|
||||
const end = start + (duration ?? Infinity);
|
||||
const clipWindow = timing.exportRenderSeek()
|
||||
? exportClipWindow(start, end, timing.canonicalFps())
|
||||
: { start, end };
|
||||
if (!isClipVisibleAt(time, clipWindow.start, clipWindow.end, timing.compositionDuration()))
|
||||
continue;
|
||||
const onScreen = isClipOnScreenAt(
|
||||
time,
|
||||
start,
|
||||
end,
|
||||
timing.canonicalFps(),
|
||||
timing.compositionDuration(),
|
||||
);
|
||||
if (!onScreen) continue;
|
||||
const localTime = Math.max(0, time - start);
|
||||
const sourceTime =
|
||||
readMediaStart(element) + sourceTimeAt(readElementRateSpec(element), localTime);
|
||||
|
||||
@@ -2941,7 +2941,6 @@ export function initSandboxRuntimeModular(): void {
|
||||
currentTime: time,
|
||||
compositionDuration,
|
||||
canonicalFps: state.canonicalFps,
|
||||
exportRenderSeek: Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
|
||||
timelineRegistry: window.__timelines ?? {},
|
||||
resolver: timingResolverFor(true),
|
||||
});
|
||||
@@ -5091,7 +5090,6 @@ export function initSandboxRuntimeModular(): void {
|
||||
duration: (element) => resolveDurationForElement(element),
|
||||
compositionDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0),
|
||||
canonicalFps: () => state.canonicalFps,
|
||||
exportRenderSeek: () => Boolean(window.__HF_EXPORT_RENDER_SEEK_CONFIG),
|
||||
}),
|
||||
createWaapiAdapter(),
|
||||
createCssAdapter({
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import { DEFAULT_IMAGE_TIMELINE_DURATION_SECONDS } from "@hyperframes/parsers/media-duration";
|
||||
import { describe, it, expect, afterEach } from "vitest";
|
||||
import { collectRuntimeTimelinePayload, runtimeElementWindow } from "./timeline";
|
||||
import {
|
||||
collectRuntimeTimelinePayload,
|
||||
isRuntimeElementVisibleAt,
|
||||
runtimeElementWindow,
|
||||
} from "./timeline";
|
||||
import { createRuntimeStartTimeResolver } from "./startResolver";
|
||||
|
||||
type TimelineTestWindow = Window & {
|
||||
@@ -1134,3 +1138,30 @@ describe("runtimeElementWindow", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("isRuntimeElementVisibleAt", () => {
|
||||
afterEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
// fails without the fix: snapshot and preview read the authored seconds, so a title held across two clips whose
|
||||
// second start sits a hair past 100 s went blank at exactly 100 s, the frame render shows it on.
|
||||
it("puts a clip edge a hair off a frame on that frame for preview and snapshot, as render does", () => {
|
||||
document.body.innerHTML = `<div id="main" data-composition-id="main" data-start="0" data-duration="110">
|
||||
<div id="held" data-start="90" data-duration="10"></div>
|
||||
<div id="rest" data-start="100.00002" data-duration="10"></div></div>`;
|
||||
const shownAt = (currentTime: number) =>
|
||||
["held", "rest"].filter((id) =>
|
||||
isRuntimeElementVisibleAt(document.getElementById(id)!, {
|
||||
currentTime,
|
||||
compositionDuration: 110,
|
||||
canonicalFps: 30,
|
||||
timelineRegistry: {},
|
||||
resolver: createRuntimeStartTimeResolver({}),
|
||||
}),
|
||||
);
|
||||
expect(shownAt(99.875)).toEqual(["held"]);
|
||||
expect(shownAt(100)).toEqual(["rest"]);
|
||||
expect(shownAt(100.125)).toEqual(["rest"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -22,8 +22,7 @@ import {
|
||||
} from "./playbackRate";
|
||||
import { resolveCssStackingContextId } from "./stackingContext";
|
||||
import { createRuntimeStartTimeResolver } from "./startResolver";
|
||||
import { isClipVisibleAt } from "./clipWindow";
|
||||
import { exportClipWindow } from "../inline-scripts/parityContract";
|
||||
import { isClipOnScreenAt } from "./clipWindow";
|
||||
import { isSceneLikeCompositionId } from "../slideshow/index.js";
|
||||
import { COMPOSITION_CONTRACT_VERSION } from "../compositionContract.js";
|
||||
import { runtimeProtocolMetadata } from "./protocol.js";
|
||||
@@ -79,21 +78,17 @@ export function isRuntimeElementVisibleAt(
|
||||
currentTime: number;
|
||||
compositionDuration: number;
|
||||
canonicalFps: number;
|
||||
exportRenderSeek: boolean;
|
||||
timelineRegistry: Record<string, RuntimeTimelineLike | undefined>;
|
||||
resolver: ReturnType<typeof createRuntimeStartTimeResolver>;
|
||||
},
|
||||
): boolean {
|
||||
const span = runtimeElementWindow(rawNode, options);
|
||||
if (!span) return false;
|
||||
// Export seeks snap to frame boundaries; interactive visibility uses authored seconds.
|
||||
const clipWindow = options.exportRenderSeek
|
||||
? exportClipWindow(span.start, span.end, options.canonicalFps)
|
||||
: span;
|
||||
return isClipVisibleAt(
|
||||
return isClipOnScreenAt(
|
||||
options.currentTime,
|
||||
clipWindow.start,
|
||||
clipWindow.end,
|
||||
span.start,
|
||||
span.end,
|
||||
options.canonicalFps,
|
||||
options.compositionDuration,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user