fix(inspector): live queries see every composition, wherever the playhead is (#6651)

* fix(inspector): live queries see every composition, wherever the playhead is

The Studio preview parks scenes away from the playhead, so a live inspect, timeline or
element-map query answered missing for every element in a parked composition. Each live
query now holds every scene mounted under its own holder, then hands the scene window back.
__hfHoldScenes keeps holds per holder, so the inspector letting go never drops a scene
Studio holds.

* fix(inspector): isolate concurrent live query scene holds

docs/developers/harness-tool-context.mdx

@@ -93,6 +93,18 @@ the same as in a terminal.
 | `findingFrames` | Saves one plain, unannotated full frame for each distinct `frameTime` of the error and warning findings in `layout` and `contrast` that have a bbox (not `motion` or `craft`), and adds two fields to each of those findings: `frameTime`, the middle sample of the longest held run in its `times` (the same gap rule as `heldMs`), and `frame`, the absolute path of that PNG, shared by every finding at that time. The `bbox` is measured at the finding's `time`, so an element that moves can sit elsewhere on the frame. Only finite, nonnegative sample times are eligible; a finding with no eligible time gets no frame. At most 60 distinct times are captured, errors first. The frames go in the run's snapshot folder but are not listed in `snapshots`, not recorded in project history and not reported as `events`. |
 | `inspect` | Only with `findingFrames` on; alone it does nothing. While each frame is on screen, runs the [component inspector](https://github.com/heygen-com/hyperframes/tree/main/packages/inspector) on every finding framed at that time and adds its report, unchanged, as the finding's `inspect` field: `results` (the element, how many GSAP animations target it), and `snippets`, each with its source `id` (`file:start-end`), `kind` (`html opening tag`, `matched CSS candidate`, `animation creation` and so on) and the exact source `code`. The target is the finding's `data-hf-id` when it has one, else its `selector`, without descendants, under the inspector's default 12,000-byte budget. A selector that matches multiple elements is treated as an inspection failure, so unrelated source snippets are not attached. A finding whose inspection fails gets `inspect: { "error": "<message>" }` instead; the check itself never fails because of it. Inspection counts toward the `frames` phase of the check deadline. A timeout or stop ends capture without attaching frame or inspection fields. Only GSAP animations are seen, not CSS animations, Web Animations or canvas drawing. |
 
+## Inspecting a live preview
+
+The `@hyperframes/inspector/live` API reads components, timelines and element maps
+across every composition, including scenes parked away from the preview playhead.
+Each query temporarily mounts those scenes without seeking and releases its own
+hold when it finishes or fails. Concurrent queries keep their scenes mounted until
+each finishes; Studio's selection hold remains independent. The preview then
+restores the scene window for the current playhead. If scenes remain parked after
+mounting, the query reports `scenes_not_mounted` with their names instead of
+reporting their elements as missing. Mounting runs the scenes' scripts through the
+preview's normal scene lifecycle.
+
 ## Hooks that run code
 
 Running your own command before or after a CLI command is not a `HF_TOOL_CONTEXT` capability and

packages/core/src/runtime/init.swapScenes.test.ts

@@ -1952,6 +1952,21 @@ describe("the preview's scene window", () => {
     expect(parked()).toEqual(["s2"]);
   });
 
+  it("keeps each holder's scenes, so the inspector letting go leaves Studio's held scene mounted", async () => {
+    bootWindowed([S0, S1, S2], trackingRoot().root);
+    await tick();
+    await window.__hfHoldScenes!(["s2"]);
+    await window.__hfHoldScenes!(["s0", "s1", "s2"], "inspection");
+    expect(parked()).toEqual([]);
+    await window.__hfHoldScenes!([], "inspection");
+    await seekAndSettle(0.5);
+    for (let i = 0; i < 5; i++) await tick();
+    expect(parked()).toEqual([]);
+    await window.__hfHoldScenes!([]);
+    for (let i = 0; i < 5; i++) await tick();
+    expect(parked()).toEqual(["s2"]);
+  });
+
   it("keeps a parked scene's clips on the timeline, at the time its scene puts them", async () => {
     bootWindowed([S0, S1, S2], trackingRoot().root);
     await tick();

packages/core/src/runtime/init.ts

@@ -4511,7 +4511,8 @@ export function initSandboxRuntimeModular(): void {
     bindRootTimelineIfAvailable();
     postTimeline();
   };
-  // Scenes Studio holds mounted for selecting or editing, whatever the playhead does.
+  // Scenes held mounted whatever the playhead does: Studio's for selecting or editing, the inspector's for a query.
+  const holds = new Map<string, Set<string>>();
   let heldScenes = new Set<string>();
   moveSceneWindow = (time: number): boolean => {
     if (windowScenes.length === 0 || state.tornDown) return false;
@@ -4558,9 +4559,12 @@ export function initSandboxRuntimeModular(): void {
     if (landed) registerSeekCompletion(windowWork);
     return landed !== undefined;
   };
-  // Mounts the named scenes and keeps them mounted until the next call names others.
-  const holdScenes = async (names: string[]): Promise<void> => {
-    heldScenes = new Set(names.filter((name) => windowScenes.some((s) => s.name === name)));
+  // Mounts the named scenes and keeps them mounted until the same holder's next call names others.
+  const holdScenes = async (names: string[], holder = "studio"): Promise<void> => {
+    const scenes = new Set(names.filter((name) => windowScenes.some((s) => s.name === name)));
+    if (scenes.size) holds.set(holder, scenes);
+    else holds.delete(holder);
+    heldScenes = new Set([...holds.values()].flatMap((held) => [...held]));
     windowNear = "";
     // Bounded: a scene that cannot mount must not keep this waiting.
     for (let turn = 0; turn <= windowScenes.length && !state.tornDown; turn++) {

packages/core/src/runtime/window.d.ts

@@ -99,7 +99,7 @@ declare global {
     __hfSceneAnimations?: Record<string, SceneAnimation[]> | null;
     /** Swap edited scenes from a rebuilt preview document; refuses before changing anything when it cannot. */
     __hfSwapScenes?: (html: string, signal?: AbortSignal) => Promise<void>;
-    __hfHoldScenes?: (names: string[]) => Promise<void>;
+    __hfHoldScenes?: (names: string[], holder?: string) => Promise<void>;
     __hfParkedSceneMarkup?: (placeholder: Element) => Element | null;
     __HF_EXPORT_RENDER_SEEK_CONFIG?: {
       mode?: string;

packages/inspector/package.json

@@ -61,6 +61,7 @@
     "test:timeline": "bun tests/timeline.ts",
     "test:timeline-compositions": "bun tests/timelineCompositions.ts",
     "test:live": "bun run build && bun tests/live.ts",
+    "test:live-scenes": "bun run build && bun tests/liveScenes.ts",
     "test:source-release": "bun tests/sourceRelease.ts",
     "serve:node": "node dist/server.js"
   },

packages/inspector/src/live.ts

@@ -12,7 +12,12 @@ import {
   type SourceContext,
   type TimelineReport,
 } from "./engine";
-import { liveGenerationPage, type PageRunner } from "./pageQueries";
+import {
+  holdEveryScenePage,
+  liveGenerationPage,
+  releaseEveryScenePage,
+  type PageRunner,
+} from "./pageQueries";
 import { assertGeneration, validateInspectRequest, validateTimelineRequest } from "./requests";
 import {
   Sources,
@@ -74,6 +79,17 @@ export function prepareLiveInspection(
       throw new Error("stale_preview: the preview reloaded; query again");
     await view.assertCurrent();
   }
+  async function withEveryScene<R>(runner: PageRunner, query: () => Promise<R>): Promise<R> {
+    const holder = `inspection:${randomUUID()}`;
+    try {
+      const parked = await runner.run(holdEveryScenePage, holder);
+      if (parked.length)
+        throw new Error(`scenes_not_mounted: the preview could not mount ${parked.join(", ")}`);
+      return await query();
+    } finally {
+      await runner.run(releaseEveryScenePage, holder).catch(() => {});
+    }
+  }
   return {
     html: prepared,
     generation,
@@ -86,21 +102,23 @@ export function prepareLiveInspection(
       if (request.time != null) throw new Error("live_query_does_not_seek");
       assertGeneration(request.generation, generation);
       await current(runner);
-      return inspectComponents(runner, view, request, {
-        catalog,
-        reused: true,
-        start: performance.now(),
-      });
+      return withEveryScene(runner, () =>
+        inspectComponents(runner, view, request, {
+          catalog,
+          reused: true,
+          start: performance.now(),
+        }),
+      );
     },
     async timeline(runner, request) {
       validateTimelineRequest(request);
       assertGeneration(request.generation, generation);
       await current(runner);
-      return inspectTimeline(runner, view, request, true);
+      return withEveryScene(runner, () => inspectTimeline(runner, view, request, true));
     },
     async elementMap(runner) {
       await current(runner);
-      return inspectElementMap(runner, view);
+      return withEveryScene(runner, () => inspectElementMap(runner, view));
     },
   };
 }

packages/inspector/src/pageQueries.ts

@@ -25,6 +25,7 @@ type InspectedWindow = Window & {
     timeline(request: TimelineRequest): TimelineRuntimeResult;
   };
   __hfLiveInspection?: { generation: string; project: string };
+  __hfHoldScenes?: (names: string[], holder?: string) => Promise<void>;
   __player?: {
     renderSeek(time: number, options: { exact: boolean }): void;
     getTime(): number;
@@ -90,3 +91,16 @@ export function timelinePage(request: TimelineRequest): PageTimeline {
 export function liveGenerationPage(): string | null {
   return (window as InspectedWindow).__hfLiveInspection?.generation ?? null;
 }
+
+export async function holdEveryScenePage(holder: string): Promise<string[]> {
+  const hold = (window as InspectedWindow).__hfHoldScenes;
+  if (!hold) return [];
+  const sceneOf = (el: Element) => el.getAttribute("data-hf-scene") ?? "";
+  const names = new Set(Array.from(document.querySelectorAll("[data-hf-scene]"), sceneOf));
+  await hold([...names], holder);
+  return Array.from(document.querySelectorAll("[data-hf-parked-scene]"), sceneOf);
+}
+
+export async function releaseEveryScenePage(holder: string): Promise<void> {
+  await (window as InspectedWindow).__hfHoldScenes?.([], holder);
+}

packages/inspector/tests/liveScenes.ts

@@ -0,0 +1,214 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createServer } from "node:http";
+import puppeteer from "puppeteer-core";
+import { bundleToSingleHtml } from "../../core/src/compiler/htmlBundler";
+import { addScenePartsManifest } from "../../core/src/compiler/scenePartsManifest";
+// Built Node entry, exactly the import consumed by Desktop.
+const entry = "../dist/live.js";
+const { prepareLiveInspection, fingerprint }: typeof import("../src/live") = await import(entry);
+type PageRunner = import("../src/live").PageRunner;
+
+const vendor = process.argv[2];
+if (!vendor) throw new Error("Pass gsap.min.js");
+const SCENES = ["one", "two", "three", "four"];
+const SCENE_SECONDS = 5;
+const dir = mkdtempSync(join(tmpdir(), "hf-live-scenes-"));
+mkdirSync(join(dir, "compositions"));
+writeFileSync(join(dir, "gsap.min.js"), readFileSync(vendor));
+const hosts = SCENES.map(
+  (name, n) =>
+    `<div id="${name}" data-composition-id="${name}" data-composition-src="compositions/${name}.html" data-start="${n * SCENE_SECONDS}" data-duration="${SCENE_SECONDS}" data-track-index="1" data-width="1920" data-height="1080"></div>`,
+).join("\n");
+writeFileSync(
+  join(dir, "index.html"),
+  `<!doctype html><html><head><script src="gsap.min.js"></script></head><body>
+<div id="root" data-composition-id="main" data-start="0" data-duration="${SCENES.length * SCENE_SECONDS}" data-width="1920" data-height="1080">
+${hosts}
+</div>
+<script>const tl=gsap.timeline({paused:true});window.__timelines=window.__timelines||{};window.__timelines.main=tl;</script>
+</body></html>`,
+);
+for (const name of SCENES)
+  writeFileSync(
+    join(dir, "compositions", `${name}.html`),
+    `<template>
+<div id="root" data-composition-id="${name}" data-width="1920" data-height="1080"><p id="${name}-word">${name}</p></div>
+<script>
+const tl=gsap.timeline({paused:true});
+tl.to('#${name}-word',{x:120,duration:1},1);
+window.__timelines=window.__timelines||{};
+window.__timelines["${name}"]=tl;
+</script>
+</template>`,
+  );
+
+let browser: Awaited<ReturnType<typeof puppeteer.launch>> | undefined;
+let server: ReturnType<typeof createServer> | undefined;
+try {
+  const version = fingerprint(dir);
+  const provenance: import("../../core/src/compiler/htmlBundler").BundleScriptProvenance[] = [];
+  const bundled = await bundleToSingleHtml(dir, {
+    runtime: "inline",
+    sceneParts: true,
+    scriptProvenance: (p) => provenance.push(p),
+  });
+  // What Studio's preview route serves: the scene manifest is what lets the runtime park far scenes.
+  const live = prepareLiveInspection(dir, addScenePartsManifest(bundled), provenance, version);
+  live.finalizeDocument(live.html);
+  server = createServer((req, res) => {
+    const gsap = req.url?.endsWith("/gsap.min.js");
+    res.setHeader("Content-Type", gsap ? "text/javascript" : "text/html");
+    res.end(gsap ? readFileSync(vendor) : live.html);
+  });
+  await new Promise<void>((r) => server!.listen(0, "127.0.0.1", r));
+  browser = await puppeteer.launch({
+    executablePath:
+      process.env.HF_INSPECT_CHROME ??
+      "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+    headless: true,
+  });
+  const page = await browser.newPage();
+  const port = (server.address() as import("node:net").AddressInfo).port;
+  await page.goto(`http://127.0.0.1:${port}/api/projects/test/preview`);
+  await page.waitForFunction("!!window.__hfInspection && !!window.__hfHoldScenes", {
+    timeout: 5000,
+  });
+  const runner: PageRunner = {
+    run: <A, R>(fn: (arg: A) => R, arg: A) =>
+      page.evaluate(`(${fn.toString()})(${JSON.stringify(arg)})`) as Promise<R>,
+  };
+  const parked = () =>
+    page.evaluate(() =>
+      Array.from(document.querySelectorAll("[data-hf-parked-scene]"), (el) =>
+        el.getAttribute("data-hf-scene"),
+      ).sort(),
+    );
+  const seek = async (time: number) => {
+    await page.evaluate((t) => (window as any).__player.seek(t), time);
+    await page.evaluate(() => (window as any).__hfWaitForSeekCompletion?.());
+    await new Promise((r) => setTimeout(r, 200));
+  };
+  const words = SCENES.map((name) => ({ selector: `#${name}-word` }));
+
+  for (const [n, scene] of SCENES.entries()) {
+    await seek(n * SCENE_SECONDS + 1);
+    const before = await parked();
+    assert(before.length > 0, `at scene ${scene} the preview parks some scenes`);
+    const found = await live.inspect(runner, { targets: words });
+    assert.deepEqual(
+      found.results.map((r) => r.status),
+      SCENES.map(() => "found"),
+      `inspect sees every composition with the playhead in ${scene}`,
+    );
+    const timeline = await live.timeline(runner, {});
+    for (const name of SCENES)
+      assert(timeline.text.includes(`#${name}-word`), `timeline lists ${name} from ${scene}`);
+    const map = await live.elementMap(runner);
+    for (const name of SCENES)
+      assert(map.text.includes(`#${name}-word`), `element map lists ${name} from ${scene}`);
+    for (let i = 0; i < 10; i++) await new Promise((r) => setTimeout(r, 50));
+    assert.deepEqual(await parked(), before, `the query hands the scene window back at ${scene}`);
+  }
+
+  await seek(1);
+  const beforeConcurrent = await parked();
+  let queryReady = () => {};
+  const ready = new Promise<void>((resolve) => {
+    queryReady = resolve;
+  });
+  let resumeQuery = () => {};
+  const resume = new Promise<void>((resolve) => {
+    resumeQuery = resolve;
+  });
+  const delayedRunner: PageRunner = {
+    async run(fn, arg) {
+      if (fn.name === "queryPage") {
+        queryReady();
+        await resume;
+      }
+      return runner.run(fn, arg);
+    },
+  };
+  const pending = live.inspect(delayedRunner, { targets: words });
+  await ready;
+  try {
+    await live.elementMap(runner);
+    await seek(16);
+    await new Promise((resolve) => setTimeout(resolve, 100));
+    assert.deepEqual(
+      await parked(),
+      [],
+      "another request cannot release an active inspection's scenes",
+    );
+  } finally {
+    resumeQuery();
+    await pending;
+  }
+  await seek(1);
+  assert.deepEqual(await parked(), beforeConcurrent, "concurrent queries release their holds");
+
+  const nestedRunner: PageRunner = {
+    async run(fn, arg) {
+      if (fn.name === "queryPage") {
+        await live.timeline(runner, {});
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        assert.deepEqual(await parked(), [], "a nested query leaves its caller's scenes mounted");
+      }
+      return runner.run(fn, arg);
+    },
+  };
+  const nested = await live.inspect(nestedRunner, { targets: words });
+  assert(nested.results.every((result) => result.status === "found"));
+
+  for (const failurePoint of ["queryPage", "holdEveryScenePage"]) {
+    const failedRunner: PageRunner = {
+      async run(fn, arg) {
+        if (fn.name === failurePoint) {
+          if (failurePoint === "holdEveryScenePage") await runner.run(fn, arg);
+          throw new DOMException("inspection aborted", "AbortError");
+        }
+        return runner.run(fn, arg);
+      },
+    };
+    await assert.rejects(live.inspect(failedRunner, { targets: words }), /inspection aborted/);
+    await new Promise((resolve) => setTimeout(resolve, 100));
+    assert.deepEqual(await parked(), beforeConcurrent, `${failurePoint} failure releases its hold`);
+  }
+
+  await page.evaluate(() => {
+    const missing = document.createElement("div");
+    missing.id = "unmountable";
+    missing.setAttribute("data-hf-scene", "unmountable");
+    missing.setAttribute("data-hf-parked-scene", "");
+    document.body.append(missing);
+  });
+  try {
+    await assert.rejects(live.elementMap(runner), /scenes_not_mounted:.*unmountable/);
+  } finally {
+    await page.evaluate(() => document.getElementById("unmountable")?.remove());
+  }
+  await new Promise((resolve) => setTimeout(resolve, 100));
+  assert.deepEqual(
+    await parked(),
+    beforeConcurrent,
+    "a mount failure releases the scenes it acquired",
+  );
+
+  // Studio's own hold survives a query: the inspector holds apart from it.
+  await seek(1);
+  await page.evaluate(() => (window as any).__hfHoldScenes(["four"]));
+  await live.inspect(runner, { targets: words });
+  for (let i = 0; i < 10; i++) await new Promise((r) => setTimeout(r, 50));
+  assert(!(await parked()).includes("four"), "Studio's held scene stays mounted after a query");
+
+  console.log(
+    "PASS live preview scenes: inspect, timeline and element map see every composition wherever the playhead is, and hand the scene window back",
+  );
+} finally {
+  await browser?.close();
+  if (server) await new Promise<void>((r) => server!.close(() => r()));
+  rmSync(dir, { recursive: true, force: true });
+}