From befdc993c0f75c88a0f943ba9cc76ad28ab2eb9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 26 Sep 2026 23:01:17 -0400 Subject: [PATCH] test(studio): gate playback cost with the Catalog open A new step in the Studio timeline viewport job plays a small GSAP film with the Catalog showing. It holds three work counts exactly through perf-ceilings.json (Catalog elements, composited layers, eventful timeline seeks per frame), and times main-thread CPU per frame against the base branch's Studio built on the same runner, in alternating runs, failing above 1.15x. Chrome runs without vsync or a frame cap, so a 60 Hz runner measures what a frame costs, not the display's pace. The catalog, its posters and GSAP are served by the script, so the counts never follow the live registry. --- .github/workflows/ci.yml | 63 ++++- .../e2e/fixtures/studio-playback/README.md | 4 + .../fixtures/studio-playback/hyperframes.json | 8 + .../e2e/fixtures/studio-playback/index.html | 75 +++++ packages/studio/tests/e2e/perf-ceilings.json | 8 + packages/studio/tests/e2e/studio-playback.mjs | 257 ++++++++++++++++++ 6 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 packages/studio/tests/e2e/fixtures/studio-playback/README.md create mode 100644 packages/studio/tests/e2e/fixtures/studio-playback/hyperframes.json create mode 100644 packages/studio/tests/e2e/fixtures/studio-playback/index.html create mode 100644 packages/studio/tests/e2e/studio-playback.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20f1a9c4e4..933b86bb30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,6 +144,7 @@ jobs: - "packages/studio/**" - "packages/core/**" - "packages/studio-server/**" + - "packages/player/**" - "bun.lock" - ".github/workflows/ci.yml" gcp_beginframe: @@ -763,7 +764,7 @@ jobs: needs: [changes] if: needs.changes.outputs.studio == 'true' runs-on: ubuntu-latest - timeout-minutes: 18 + timeout-minutes: 25 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -786,6 +787,8 @@ jobs: packages/studio/data/projects/timeline-virtualization cp -R packages/studio/tests/e2e/fixtures/studio-open \ packages/studio/data/projects/studio-open + cp -R packages/studio/tests/e2e/fixtures/studio-playback \ + packages/studio/data/projects/studio-playback - name: Read the base branch's work-count ceilings # A ceiling may not disappear, or rise without a new reason, against the branch this PR merges into. if: github.event_name == 'pull_request' @@ -797,6 +800,21 @@ jobs: rm -f /tmp/base-perf-ceilings.json echo "The base branch has no perf-ceilings.json yet" fi + - name: Build the base branch's Studio for the playback timing + # The playback gate times this Studio against its base on the same runner, in + # alternating runs, so the runner's speed cancels out of the ratio. + if: github.event_name == 'pull_request' + run: | + set -euo pipefail + git worktree add --detach /tmp/hf-base FETCH_HEAD + cd /tmp/hf-base + bash scripts/ci/install-workspace-dependencies.sh + bun run --filter '@hyperframes/{parsers,lint,studio-server}' build + bun run --cwd packages/core build + bun run --cwd packages/core build:hyperframes-runtime + mkdir -p packages/studio/data/projects + cp -R "$GITHUB_WORKSPACE/packages/studio/tests/e2e/fixtures/studio-playback" \ + packages/studio/data/projects/studio-playback - name: Run both arms of the timeline viewport gate run: | set -euo pipefail @@ -893,6 +911,48 @@ jobs: node packages/studio/tests/e2e/perf-ratchet.mjs check \ packages/studio/tests/e2e/perf-ceilings.json studio-open /tmp/studio-open-counts.json \ /tmp/base-perf-ceilings.json + - name: Count and time playback with the Catalog open + # Counts go through perf-ceilings.json like studio-open. Time is only ever a ratio + # against the base branch's Studio on this runner, so a slow runner cannot fail it. + # Both servers stay up; they sit idle while the other one is measured. + if: success() || failure() + run: | + set -uo pipefail + PIDS=() + trap 'kill "${PIDS[@]}" 2>/dev/null || true' EXIT + NODE_ENV=production bun run --cwd packages/studio dev -- --port 5316 --strictPort & + PIDS+=($!) + PORTS=(5316) + BASE_URL="" + if [[ -d /tmp/hf-base ]]; then + (cd /tmp/hf-base && NODE_ENV=production bun run --cwd packages/studio dev -- --port 5317 --strictPort) & + PIDS+=($!) + PORTS+=(5317) + BASE_URL="http://localhost:5317/#project/studio-playback" + fi + for port in "${PORTS[@]}"; do + for i in $(seq 1 60); do + if curl -sf "http://localhost:${port}/" >/dev/null 2>&1; then break; fi + sleep 1 + done + if ! curl -sf "http://localhost:${port}/" >/dev/null 2>&1; then + echo "FAIL: studio dev server did not start on port ${port}" + exit 1 + fi + done + STUDIO_URL="http://localhost:5316/#project/studio-playback" \ + STUDIO_BASE_URL="$BASE_URL" \ + node packages/studio/tests/e2e/studio-playback.mjs \ + | tee /tmp/studio-playback.json + TIMING_STATUS=$? + node packages/studio/tests/e2e/perf-ratchet.mjs check \ + packages/studio/tests/e2e/perf-ceilings.json studio-playback /tmp/studio-playback.json \ + /tmp/base-perf-ceilings.json + COUNTS_STATUS=$? + if (( TIMING_STATUS != 0 || COUNTS_STATUS != 0 )); then + echo "FAIL: playback timing=${TIMING_STATUS}, counts=${COUNTS_STATUS}" + exit 1 + fi - name: Upload gate evidence # The gate's whole output is machine-readable evidence, and a red run is # exactly when someone needs to read it. Keep it on failure too. @@ -903,6 +963,7 @@ jobs: path: | /tmp/timeline-gate-*.json /tmp/studio-open-counts.json + /tmp/studio-playback.json if-no-files-found: error smoke-global-install: diff --git a/packages/studio/tests/e2e/fixtures/studio-playback/README.md b/packages/studio/tests/e2e/fixtures/studio-playback/README.md new file mode 100644 index 0000000000..8bb9a181af --- /dev/null +++ b/packages/studio/tests/e2e/fixtures/studio-playback/README.md @@ -0,0 +1,4 @@ +# Studio playback fixture + +A 12 s film of 24 GSAP-animated tiles and a title, with no media. `studio-playback.mjs` opens it, +closes the Compositions tab so the Catalog shows, plays it and records what each frame costs. diff --git a/packages/studio/tests/e2e/fixtures/studio-playback/hyperframes.json b/packages/studio/tests/e2e/fixtures/studio-playback/hyperframes.json new file mode 100644 index 0000000000..c4816b5eaa --- /dev/null +++ b/packages/studio/tests/e2e/fixtures/studio-playback/hyperframes.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://hyperframes.heygen.com/schema/hyperframes.json", + "paths": { + "blocks": "compositions", + "components": "compositions/components", + "assets": "assets" + } +} diff --git a/packages/studio/tests/e2e/fixtures/studio-playback/index.html b/packages/studio/tests/e2e/fixtures/studio-playback/index.html new file mode 100644 index 0000000000..0919514ba2 --- /dev/null +++ b/packages/studio/tests/e2e/fixtures/studio-playback/index.html @@ -0,0 +1,75 @@ + + + + + Studio playback fixture + + + + +
+

Playback

+
+ + + diff --git a/packages/studio/tests/e2e/perf-ceilings.json b/packages/studio/tests/e2e/perf-ceilings.json index 2bf6b4fa36..5b41c1425b 100644 --- a/packages/studio/tests/e2e/perf-ceilings.json +++ b/packages/studio/tests/e2e/perf-ceilings.json @@ -10,5 +10,13 @@ "raised": { "total.requests.thumbnail": "#4401 shows a cached frame-0 poster while the preview boots: a cache-only ask that misses on a fresh open, then a render of that poster for the next open (+2 requests)" } + }, + "studio-playback": { + "browser": "154", + "counts": { + "playing.catalogElements": 6878, + "playing.compositedLayers": 172, + "playing.maxSeeksPerFrame": 1 + } } } diff --git a/packages/studio/tests/e2e/studio-playback.mjs b/packages/studio/tests/e2e/studio-playback.mjs new file mode 100644 index 0000000000..f14bef59cc --- /dev/null +++ b/packages/studio/tests/e2e/studio-playback.mjs @@ -0,0 +1,257 @@ +#!/usr/bin/env node +// Plays the studio-playback fixture with the Catalog open and prints evidence: exact work counts for +// perf-ratchet.mjs and main-thread CPU per frame. With STUDIO_BASE_URL (the base branch's Studio on this runner) +// it alternates runs of both and exits 1 when this build's median CPU per frame exceeds MAX_CPU_RATIO of the base's. +// Chrome runs without vsync or a frame cap, so a frame costs what the page makes it cost, even on a 60 Hz runner. +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import puppeteer from "puppeteer-core"; +import { resolveChromeExecutable } from "./chrome-executable.mjs"; + +const HEAD_URL = process.env.STUDIO_URL; +const BASE_URL = process.env.STUDIO_BASE_URL; +const ROUNDS = Number(process.env.STUDIO_PLAYBACK_ROUNDS || 3); +const PLAY_MS = 4_000; +const MAX_CPU_RATIO = 1.15; +// A catalog the size of the real registry, served by this script so the counts never follow it. +const CATALOG_SIZE = 400; +const CATEGORIES = ["transitions", "vfx", "social", "data", "scenes", "captions", "effects"]; + +if (!HEAD_URL) { + console.error("STUDIO_URL is required and must point at the studio-playback fixture"); + process.exit(2); +} +const executablePath = resolveChromeExecutable(); +if (!executablePath) { + console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH"); + process.exit(2); +} +const gsapSource = readFileSync( + createRequire(import.meta.url).resolve("gsap/dist/gsap.min.js"), + "utf8", +); + +const catalog = Array.from({ length: CATALOG_SIZE }, (_, i) => ({ + name: `perf-block-${i}`, + type: "hyperframes:block", + title: `Block ${i}`, + description: `Catalog entry ${i} for the playback gate`, + tags: [CATEGORIES[i % CATEGORIES.length]], + duration: 5, + preview: { poster: `/__studio-playback/poster/${i}.svg` }, +})); +const poster = (i) => + `${i}`; + +/** A response for each fixture this script serves in place of the network, by URL. */ +// fallow-ignore-next-line complexity +function fixtureResponse(url) { + if (url.pathname.endsWith("/api/registry/blocks")) { + return { contentType: "application/json", body: JSON.stringify(catalog) }; + } + const posterMatch = /^\/__studio-playback\/poster\/(\d+)\.svg$/.exec(url.pathname); + if (posterMatch) return { contentType: "image/svg+xml", body: poster(Number(posterMatch[1])) }; + const isGsap = url.hostname === "cdn.jsdelivr.net" && url.pathname.endsWith("/gsap.min.js"); + return isGsap ? { contentType: "application/javascript", body: gsapSource } : null; +} + +/** Serves the catalog, its posters and GSAP locally, and refuses every other off-origin request. */ +async function serveFixtures(page, origin) { + await page.setRequestInterception(true); + page.on("request", (request) => { + const url = new URL(request.url()); + const fixture = fixtureResponse(url); + if (fixture) return request.respond(fixture); + return url.origin === origin ? request.continue() : request.abort(); + }); +} + +/** Counts, per composition animation frame, the eventful seeks of GSAP timelines. */ +function instrumentPlayback() { + const player = document.querySelector("hyperframes-player"); + const win = player.shadowRoot.querySelector("iframe").contentWindow; + const timelines = Object.values(win.__timelines).filter( + (tl) => typeof tl?.totalTime === "function", + ); + if (timelines.length === 0) throw new Error("the composition registered no timeline to watch"); + const state = (window.__studioPlayback = { on: false, frames: [], current: 0 }); + // Studio wraps each timeline's own totalTime, so count on the instance, outside those wrappers. + for (const tl of timelines) { + const totalTime = tl.totalTime; + tl.totalTime = function (time, suppressEvents) { + const eventfulSeek = arguments.length > 0 && !suppressEvents; + if (eventfulSeek && state.on) state.current += 1; + return totalTime.apply(this, arguments); + }; + } + const frame = () => { + if (state.on) state.frames.push(state.current); + state.current = 0; + win.requestAnimationFrame(frame); + }; + win.requestAnimationFrame(frame); +} + +/** The renderer main thread's complete events, in time order. */ +function mainThreadEvents(traceEvents) { + const main = traceEvents.find( + (e) => e.ph === "M" && e.name === "thread_name" && e.args.name === "CrRendererMain", + ); + return traceEvents + .filter((e) => e.pid === main.pid && e.tid === main.tid && e.ph === "X") + .sort((a, b) => a.ts - b.ts); +} + +/** Main-thread CPU per produced frame: top-level task CPU between consecutive BeginMainFrames. */ +// fallow-ignore-next-line complexity +function cpuPerFrameMs(traceEvents) { + const events = mainThreadEvents(traceEvents); + const frameStarts = events.filter((e) => e.name === "ProxyMain::BeginMainFrame").map((e) => e.ts); + const cpu = new Array(Math.max(0, frameStarts.length - 1)).fill(0); + let taskEnd = -1; + for (const event of events) { + if (event.ts < taskEnd) continue; // nested inside the previous top-level task + taskEnd = event.ts + event.dur; + const frame = frameStarts.findLastIndex((start) => start <= event.ts); + if (frame >= 0 && frame < cpu.length) cpu[frame] += (event.tdur ?? event.dur) / 1000; + } + return cpu; +} + +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b); + return sorted.length ? sorted[Math.floor(sorted.length / 2)] : Number.NaN; +}; +const round = (value) => Math.round(value * 100) / 100; + +async function measure(url) { + const browser = await puppeteer.launch({ + executablePath, + headless: true, + args: [ + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu-vsync", + "--disable-frame-rate-limit", + "--window-size=1600,900", + ], + }); + try { + const page = await browser.newPage(); + await page.setViewport({ width: 1600, height: 900 }); + await serveFixtures(page, new URL(url).origin); + await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60_000 }); + await page.waitForFunction( + () => { + const play = document.querySelector('button[aria-label="Play"]'); + return play instanceof HTMLButtonElement && !play.disabled; + }, + { timeout: 120_000, polling: 50 }, + ); + // #4529's setup: closing the Compositions tab leaves the Catalog showing. + await page.click('[aria-label="Close Compositions"]'); + await page.waitForFunction( + () => document.querySelectorAll('img[src*="/__studio-playback/poster/"]').length > 0, + { timeout: 30_000, polling: 100 }, + ); + // Posters below the panel's fold are lazy and never load, so wait for the first rows only. + await page.waitForFunction( + () => + [...document.querySelectorAll('img[src*="/__studio-playback/poster/"]')] + .slice(0, 4) + .every((img) => img.complete), + { timeout: 30_000, polling: 100 }, + ); + await page.evaluate(instrumentPlayback); + await page.mouse.move(0, 0); + await page.click('button[aria-label="Play"]'); + await page.mouse.move(0, 0); + await page.waitForFunction(() => document.querySelector('button[aria-label="Pause"]'), { + timeout: 10_000, + }); + await new Promise((resolve) => setTimeout(resolve, 500)); + + await page.tracing.start({ categories: ["devtools.timeline", "toplevel", "cc"] }); + await page.evaluate(() => { + window.__studioPlayback.on = true; + }); + await new Promise((resolve) => setTimeout(resolve, PLAY_MS)); + const counts = await page.evaluate(() => { + window.__studioPlayback.on = false; + // The Catalog's own subtree: the nearest ancestor of its search box that holds its posters. + let panel = document.querySelector('input[aria-label="Search blocks"]'); + while (panel && !panel.querySelector('img[src*="/__studio-playback/poster/"]')) { + panel = panel.parentElement; + } + return { + catalogElements: panel ? panel.querySelectorAll("*").length : -1, + seekFrames: window.__studioPlayback.frames, + }; + }); + const trace = JSON.parse(Buffer.from(await page.tracing.stop()).toString("utf8")); + // Layers are read paused: while playing, the film's own layers come and go with its tweens. + await page.click('button[aria-label="Pause"]'); + await new Promise((resolve) => setTimeout(resolve, 500)); + const cdp = await page.createCDPSession(); + const layers = new Promise((resolve) => + cdp.on("LayerTree.layerTreeDidChange", (event) => event.layers && resolve(event.layers)), + ); + await cdp.send("LayerTree.enable"); + const compositedLayers = (await layers).length; + const cpu = cpuPerFrameMs(trace.traceEvents ?? trace); + const seekFrames = counts.seekFrames.slice(1); + return { + browser: await browser.version(), + frames: cpu.length, + cpuMedianMs: round(median(cpu)), + workCounts: { + "playing.catalogElements": counts.catalogElements, + "playing.compositedLayers": compositedLayers, + "playing.maxSeeksPerFrame": seekFrames.length ? Math.max(...seekFrames) : -1, + }, + }; + } finally { + await browser.close(); + } +} + +const head = []; +const base = []; +for (let round_ = 0; round_ < ROUNDS; round_ += 1) { + head.push(await measure(HEAD_URL)); + if (BASE_URL) base.push(await measure(BASE_URL)); +} +// Counts must repeat exactly across runs of one build, or the ratchet would fail at random. +const countSets = new Set(head.map((run) => JSON.stringify(run.workCounts))); +if (countSets.size !== 1) { + console.error(`work counts differ between runs of one build: ${[...countSets].join(" vs ")}`); + process.exit(1); +} +const evidence = { + journey: "studio-playback", + browser: head[0].browser, + workCounts: head[0].workCounts, + cpuMedianMsPerRun: head.map((run) => run.cpuMedianMs), + framesPerRun: head.map((run) => run.frames), +}; +let failed = false; +if (BASE_URL) { + const headMs = median(head.map((run) => run.cpuMedianMs)); + const baseMs = median(base.map((run) => run.cpuMedianMs)); + evidence.ab = { + headCpuMedianMs: headMs, + baseCpuMedianMs: baseMs, + baseCpuMedianMsPerRun: base.map((run) => run.cpuMedianMs), + baseWorkCounts: base[0].workCounts, + ratio: round(headMs / baseMs), + maxRatio: MAX_CPU_RATIO, + }; + failed = headMs > baseMs * MAX_CPU_RATIO; +} +console.log(JSON.stringify(evidence, null, 2)); +if (failed) { + console.error( + `FAIL main-thread CPU per frame ${evidence.ab.headCpuMedianMs} ms against the base's ${evidence.ab.baseCpuMedianMs} ms (x${evidence.ab.ratio}, limit x${MAX_CPU_RATIO})`, + ); + process.exit(1); +}