diff --git a/docs/developers/harness-tool-context.mdx b/docs/developers/harness-tool-context.mdx index 115ab2aea2..80c93ae8f1 100644 --- a/docs/developers/harness-tool-context.mdx +++ b/docs/developers/harness-tool-context.mdx @@ -93,6 +93,18 @@ the same as in a terminal. | `findingFrames` | Saves one plain, unannotated full frame for each distinct `frameTime` of the error and warning findings in `layout` and `contrast` that have a bbox (not `motion` or `craft`), and adds two fields to each of those findings: `frameTime`, the middle sample of the longest held run in its `times` (the same gap rule as `heldMs`), and `frame`, the absolute path of that PNG, shared by every finding at that time. The `bbox` is measured at the finding's `time`, so an element that moves can sit elsewhere on the frame. Only finite, nonnegative sample times are eligible; a finding with no eligible time gets no frame. At most 60 distinct times are captured, errors first. The frames go in the run's snapshot folder but are not listed in `snapshots`, not recorded in project history and not reported as `events`. | | `inspect` | Only with `findingFrames` on; alone it does nothing. While each frame is on screen, runs the [component inspector](https://github.com/heygen-com/hyperframes/tree/main/packages/inspector) on every finding framed at that time and adds its report, unchanged, as the finding's `inspect` field: `results` (the element, how many GSAP animations target it), and `snippets`, each with its source `id` (`file:start-end`), `kind` (`html opening tag`, `matched CSS candidate`, `animation creation` and so on) and the exact source `code`. The target is the finding's `data-hf-id` when it has one, else its `selector`, without descendants, under the inspector's default 12,000-byte budget. A selector that matches multiple elements is treated as an inspection failure, so unrelated source snippets are not attached. A finding whose inspection fails gets `inspect: { "error": "" }` instead; the check itself never fails because of it. Inspection counts toward the `frames` phase of the check deadline. A timeout or stop ends capture without attaching frame or inspection fields. Only GSAP animations are seen, not CSS animations, Web Animations or canvas drawing. | +## Inspecting a live preview + +The `@hyperframes/inspector/live` API reads components, timelines and element maps +across every composition, including scenes parked away from the preview playhead. +Each query temporarily mounts those scenes without seeking and releases its own +hold when it finishes or fails. Concurrent queries keep their scenes mounted until +each finishes; Studio's selection hold remains independent. The preview then +restores the scene window for the current playhead. If scenes remain parked after +mounting, the query reports `scenes_not_mounted` with their names instead of +reporting their elements as missing. Mounting runs the scenes' scripts through the +preview's normal scene lifecycle. + ## Hooks that run code Running your own command before or after a CLI command is not a `HF_TOOL_CONTEXT` capability and diff --git a/packages/core/src/runtime/init.swapScenes.test.ts b/packages/core/src/runtime/init.swapScenes.test.ts index 1461ea5c39..3cd4a58d5d 100644 --- a/packages/core/src/runtime/init.swapScenes.test.ts +++ b/packages/core/src/runtime/init.swapScenes.test.ts @@ -1952,6 +1952,21 @@ describe("the preview's scene window", () => { expect(parked()).toEqual(["s2"]); }); + it("keeps each holder's scenes, so the inspector letting go leaves Studio's held scene mounted", async () => { + bootWindowed([S0, S1, S2], trackingRoot().root); + await tick(); + await window.__hfHoldScenes!(["s2"]); + await window.__hfHoldScenes!(["s0", "s1", "s2"], "inspection"); + expect(parked()).toEqual([]); + await window.__hfHoldScenes!([], "inspection"); + await seekAndSettle(0.5); + for (let i = 0; i < 5; i++) await tick(); + expect(parked()).toEqual([]); + await window.__hfHoldScenes!([]); + for (let i = 0; i < 5; i++) await tick(); + expect(parked()).toEqual(["s2"]); + }); + it("keeps a parked scene's clips on the timeline, at the time its scene puts them", async () => { bootWindowed([S0, S1, S2], trackingRoot().root); await tick(); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 6a0d8ddcbd..b02097e9ef 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -4511,7 +4511,8 @@ export function initSandboxRuntimeModular(): void { bindRootTimelineIfAvailable(); postTimeline(); }; - // Scenes Studio holds mounted for selecting or editing, whatever the playhead does. + // Scenes held mounted whatever the playhead does: Studio's for selecting or editing, the inspector's for a query. + const holds = new Map>(); let heldScenes = new Set(); moveSceneWindow = (time: number): boolean => { if (windowScenes.length === 0 || state.tornDown) return false; @@ -4558,9 +4559,12 @@ export function initSandboxRuntimeModular(): void { if (landed) registerSeekCompletion(windowWork); return landed !== undefined; }; - // Mounts the named scenes and keeps them mounted until the next call names others. - const holdScenes = async (names: string[]): Promise => { - heldScenes = new Set(names.filter((name) => windowScenes.some((s) => s.name === name))); + // Mounts the named scenes and keeps them mounted until the same holder's next call names others. + const holdScenes = async (names: string[], holder = "studio"): Promise => { + const scenes = new Set(names.filter((name) => windowScenes.some((s) => s.name === name))); + if (scenes.size) holds.set(holder, scenes); + else holds.delete(holder); + heldScenes = new Set([...holds.values()].flatMap((held) => [...held])); windowNear = ""; // Bounded: a scene that cannot mount must not keep this waiting. for (let turn = 0; turn <= windowScenes.length && !state.tornDown; turn++) { diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index f0ef770390..277672652f 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -99,7 +99,7 @@ declare global { __hfSceneAnimations?: Record | null; /** Swap edited scenes from a rebuilt preview document; refuses before changing anything when it cannot. */ __hfSwapScenes?: (html: string, signal?: AbortSignal) => Promise; - __hfHoldScenes?: (names: string[]) => Promise; + __hfHoldScenes?: (names: string[], holder?: string) => Promise; __hfParkedSceneMarkup?: (placeholder: Element) => Element | null; __HF_EXPORT_RENDER_SEEK_CONFIG?: { mode?: string; diff --git a/packages/inspector/package.json b/packages/inspector/package.json index cecefd1cc8..cf0b5b838d 100644 --- a/packages/inspector/package.json +++ b/packages/inspector/package.json @@ -61,6 +61,7 @@ "test:timeline": "bun tests/timeline.ts", "test:timeline-compositions": "bun tests/timelineCompositions.ts", "test:live": "bun run build && bun tests/live.ts", + "test:live-scenes": "bun run build && bun tests/liveScenes.ts", "test:source-release": "bun tests/sourceRelease.ts", "serve:node": "node dist/server.js" }, diff --git a/packages/inspector/src/live.ts b/packages/inspector/src/live.ts index c6b017e11c..22bb44e94f 100644 --- a/packages/inspector/src/live.ts +++ b/packages/inspector/src/live.ts @@ -12,7 +12,12 @@ import { type SourceContext, type TimelineReport, } from "./engine"; -import { liveGenerationPage, type PageRunner } from "./pageQueries"; +import { + holdEveryScenePage, + liveGenerationPage, + releaseEveryScenePage, + type PageRunner, +} from "./pageQueries"; import { assertGeneration, validateInspectRequest, validateTimelineRequest } from "./requests"; import { Sources, @@ -74,6 +79,17 @@ export function prepareLiveInspection( throw new Error("stale_preview: the preview reloaded; query again"); await view.assertCurrent(); } + async function withEveryScene(runner: PageRunner, query: () => Promise): Promise { + const holder = `inspection:${randomUUID()}`; + try { + const parked = await runner.run(holdEveryScenePage, holder); + if (parked.length) + throw new Error(`scenes_not_mounted: the preview could not mount ${parked.join(", ")}`); + return await query(); + } finally { + await runner.run(releaseEveryScenePage, holder).catch(() => {}); + } + } return { html: prepared, generation, @@ -86,21 +102,23 @@ export function prepareLiveInspection( if (request.time != null) throw new Error("live_query_does_not_seek"); assertGeneration(request.generation, generation); await current(runner); - return inspectComponents(runner, view, request, { - catalog, - reused: true, - start: performance.now(), - }); + return withEveryScene(runner, () => + inspectComponents(runner, view, request, { + catalog, + reused: true, + start: performance.now(), + }), + ); }, async timeline(runner, request) { validateTimelineRequest(request); assertGeneration(request.generation, generation); await current(runner); - return inspectTimeline(runner, view, request, true); + return withEveryScene(runner, () => inspectTimeline(runner, view, request, true)); }, async elementMap(runner) { await current(runner); - return inspectElementMap(runner, view); + return withEveryScene(runner, () => inspectElementMap(runner, view)); }, }; } diff --git a/packages/inspector/src/pageQueries.ts b/packages/inspector/src/pageQueries.ts index 56db6480a4..bceccb2dfb 100644 --- a/packages/inspector/src/pageQueries.ts +++ b/packages/inspector/src/pageQueries.ts @@ -25,6 +25,7 @@ type InspectedWindow = Window & { timeline(request: TimelineRequest): TimelineRuntimeResult; }; __hfLiveInspection?: { generation: string; project: string }; + __hfHoldScenes?: (names: string[], holder?: string) => Promise; __player?: { renderSeek(time: number, options: { exact: boolean }): void; getTime(): number; @@ -90,3 +91,16 @@ export function timelinePage(request: TimelineRequest): PageTimeline { export function liveGenerationPage(): string | null { return (window as InspectedWindow).__hfLiveInspection?.generation ?? null; } + +export async function holdEveryScenePage(holder: string): Promise { + const hold = (window as InspectedWindow).__hfHoldScenes; + if (!hold) return []; + const sceneOf = (el: Element) => el.getAttribute("data-hf-scene") ?? ""; + const names = new Set(Array.from(document.querySelectorAll("[data-hf-scene]"), sceneOf)); + await hold([...names], holder); + return Array.from(document.querySelectorAll("[data-hf-parked-scene]"), sceneOf); +} + +export async function releaseEveryScenePage(holder: string): Promise { + await (window as InspectedWindow).__hfHoldScenes?.([], holder); +} diff --git a/packages/inspector/tests/liveScenes.ts b/packages/inspector/tests/liveScenes.ts new file mode 100644 index 0000000000..3139b53725 --- /dev/null +++ b/packages/inspector/tests/liveScenes.ts @@ -0,0 +1,214 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createServer } from "node:http"; +import puppeteer from "puppeteer-core"; +import { bundleToSingleHtml } from "../../core/src/compiler/htmlBundler"; +import { addScenePartsManifest } from "../../core/src/compiler/scenePartsManifest"; +// Built Node entry, exactly the import consumed by Desktop. +const entry = "../dist/live.js"; +const { prepareLiveInspection, fingerprint }: typeof import("../src/live") = await import(entry); +type PageRunner = import("../src/live").PageRunner; + +const vendor = process.argv[2]; +if (!vendor) throw new Error("Pass gsap.min.js"); +const SCENES = ["one", "two", "three", "four"]; +const SCENE_SECONDS = 5; +const dir = mkdtempSync(join(tmpdir(), "hf-live-scenes-")); +mkdirSync(join(dir, "compositions")); +writeFileSync(join(dir, "gsap.min.js"), readFileSync(vendor)); +const hosts = SCENES.map( + (name, n) => + `
`, +).join("\n"); +writeFileSync( + join(dir, "index.html"), + ` +
+${hosts} +
+ +`, +); +for (const name of SCENES) + writeFileSync( + join(dir, "compositions", `${name}.html`), + ``, + ); + +let browser: Awaited> | undefined; +let server: ReturnType | undefined; +try { + const version = fingerprint(dir); + const provenance: import("../../core/src/compiler/htmlBundler").BundleScriptProvenance[] = []; + const bundled = await bundleToSingleHtml(dir, { + runtime: "inline", + sceneParts: true, + scriptProvenance: (p) => provenance.push(p), + }); + // What Studio's preview route serves: the scene manifest is what lets the runtime park far scenes. + const live = prepareLiveInspection(dir, addScenePartsManifest(bundled), provenance, version); + live.finalizeDocument(live.html); + server = createServer((req, res) => { + const gsap = req.url?.endsWith("/gsap.min.js"); + res.setHeader("Content-Type", gsap ? "text/javascript" : "text/html"); + res.end(gsap ? readFileSync(vendor) : live.html); + }); + await new Promise((r) => server!.listen(0, "127.0.0.1", r)); + browser = await puppeteer.launch({ + executablePath: + process.env.HF_INSPECT_CHROME ?? + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + headless: true, + }); + const page = await browser.newPage(); + const port = (server.address() as import("node:net").AddressInfo).port; + await page.goto(`http://127.0.0.1:${port}/api/projects/test/preview`); + await page.waitForFunction("!!window.__hfInspection && !!window.__hfHoldScenes", { + timeout: 5000, + }); + const runner: PageRunner = { + run: (fn: (arg: A) => R, arg: A) => + page.evaluate(`(${fn.toString()})(${JSON.stringify(arg)})`) as Promise, + }; + const parked = () => + page.evaluate(() => + Array.from(document.querySelectorAll("[data-hf-parked-scene]"), (el) => + el.getAttribute("data-hf-scene"), + ).sort(), + ); + const seek = async (time: number) => { + await page.evaluate((t) => (window as any).__player.seek(t), time); + await page.evaluate(() => (window as any).__hfWaitForSeekCompletion?.()); + await new Promise((r) => setTimeout(r, 200)); + }; + const words = SCENES.map((name) => ({ selector: `#${name}-word` })); + + for (const [n, scene] of SCENES.entries()) { + await seek(n * SCENE_SECONDS + 1); + const before = await parked(); + assert(before.length > 0, `at scene ${scene} the preview parks some scenes`); + const found = await live.inspect(runner, { targets: words }); + assert.deepEqual( + found.results.map((r) => r.status), + SCENES.map(() => "found"), + `inspect sees every composition with the playhead in ${scene}`, + ); + const timeline = await live.timeline(runner, {}); + for (const name of SCENES) + assert(timeline.text.includes(`#${name}-word`), `timeline lists ${name} from ${scene}`); + const map = await live.elementMap(runner); + for (const name of SCENES) + assert(map.text.includes(`#${name}-word`), `element map lists ${name} from ${scene}`); + for (let i = 0; i < 10; i++) await new Promise((r) => setTimeout(r, 50)); + assert.deepEqual(await parked(), before, `the query hands the scene window back at ${scene}`); + } + + await seek(1); + const beforeConcurrent = await parked(); + let queryReady = () => {}; + const ready = new Promise((resolve) => { + queryReady = resolve; + }); + let resumeQuery = () => {}; + const resume = new Promise((resolve) => { + resumeQuery = resolve; + }); + const delayedRunner: PageRunner = { + async run(fn, arg) { + if (fn.name === "queryPage") { + queryReady(); + await resume; + } + return runner.run(fn, arg); + }, + }; + const pending = live.inspect(delayedRunner, { targets: words }); + await ready; + try { + await live.elementMap(runner); + await seek(16); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.deepEqual( + await parked(), + [], + "another request cannot release an active inspection's scenes", + ); + } finally { + resumeQuery(); + await pending; + } + await seek(1); + assert.deepEqual(await parked(), beforeConcurrent, "concurrent queries release their holds"); + + const nestedRunner: PageRunner = { + async run(fn, arg) { + if (fn.name === "queryPage") { + await live.timeline(runner, {}); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.deepEqual(await parked(), [], "a nested query leaves its caller's scenes mounted"); + } + return runner.run(fn, arg); + }, + }; + const nested = await live.inspect(nestedRunner, { targets: words }); + assert(nested.results.every((result) => result.status === "found")); + + for (const failurePoint of ["queryPage", "holdEveryScenePage"]) { + const failedRunner: PageRunner = { + async run(fn, arg) { + if (fn.name === failurePoint) { + if (failurePoint === "holdEveryScenePage") await runner.run(fn, arg); + throw new DOMException("inspection aborted", "AbortError"); + } + return runner.run(fn, arg); + }, + }; + await assert.rejects(live.inspect(failedRunner, { targets: words }), /inspection aborted/); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.deepEqual(await parked(), beforeConcurrent, `${failurePoint} failure releases its hold`); + } + + await page.evaluate(() => { + const missing = document.createElement("div"); + missing.id = "unmountable"; + missing.setAttribute("data-hf-scene", "unmountable"); + missing.setAttribute("data-hf-parked-scene", ""); + document.body.append(missing); + }); + try { + await assert.rejects(live.elementMap(runner), /scenes_not_mounted:.*unmountable/); + } finally { + await page.evaluate(() => document.getElementById("unmountable")?.remove()); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.deepEqual( + await parked(), + beforeConcurrent, + "a mount failure releases the scenes it acquired", + ); + + // Studio's own hold survives a query: the inspector holds apart from it. + await seek(1); + await page.evaluate(() => (window as any).__hfHoldScenes(["four"])); + await live.inspect(runner, { targets: words }); + for (let i = 0; i < 10; i++) await new Promise((r) => setTimeout(r, 50)); + assert(!(await parked()).includes("four"), "Studio's held scene stays mounted after a query"); + + console.log( + "PASS live preview scenes: inspect, timeline and element map see every composition wherever the playhead is, and hand the scene window back", + ); +} finally { + await browser?.close(); + if (server) await new Promise((r) => server!.close(() => r())); + rmSync(dir, { recursive: true, force: true }); +}