From 480ac6106733c3172d6301ac7d9859d3792ddf57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 27 Sep 2026 03:57:32 -0400 Subject: [PATCH] test(studio): the playback gate counts on CI's Chrome and cannot miss a pause Review findings on the playback gate: - Ceilings are recorded on Chrome 153, the runner's browser. The script now stops with a clear message when it runs another Chrome major. - Frames were read from the first renderer main thread in the trace; on Chrome 153 that is not the page's, so no frames came out. The thread producing frames is used now, and zero frames fail loudly instead of passing the CPU check with a NaN median. - Play goes through Studio's Space shortcut and pause through the runtime's player, which both builds share, so a stopped film or a renamed control cannot throw before the JSON is written. - Every timeline seek is counted, silent ones included (3 per frame on main), and the layer count is named for when it is read (paused). - Rounds alternate which build goes first, and a failed base build no longer skips the timeline gate; playback is then counted, not timed. --- .github/workflows/ci.yml | 9 ++- packages/studio/tests/e2e/perf-ceilings.json | 6 +- packages/studio/tests/e2e/studio-playback.mjs | 65 ++++++++++++------- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 933b86bb30..f0014e7a05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -802,8 +802,10 @@ jobs: fi - name: Build the base branch's Studio for the playback timing # The playback gate times this Studio against its base on the same runner, in - # alternating runs, so the runner's speed cancels out of the ratio. + # alternating runs, so the runner's speed cancels out of the ratio. A failed base + # build must not skip the timeline gate below; the playback step then counts only. if: github.event_name == 'pull_request' + continue-on-error: true run: | set -euo pipefail git worktree add --detach /tmp/hf-base FETCH_HEAD @@ -815,6 +817,7 @@ jobs: mkdir -p packages/studio/data/projects cp -R "$GITHUB_WORKSPACE/packages/studio/tests/e2e/fixtures/studio-playback" \ packages/studio/data/projects/studio-playback + touch /tmp/hf-base/.playback-base-built - name: Run both arms of the timeline viewport gate run: | set -euo pipefail @@ -924,11 +927,13 @@ jobs: PIDS+=($!) PORTS=(5316) BASE_URL="" - if [[ -d /tmp/hf-base ]]; then + if [[ -f /tmp/hf-base/.playback-base-built ]]; then (cd /tmp/hf-base && NODE_ENV=production bun run --cwd packages/studio dev -- --port 5317 --strictPort) & PIDS+=($!) PORTS+=(5317) BASE_URL="http://localhost:5317/#project/studio-playback" + elif [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "::warning::The base branch's Studio did not build, so playback is counted but not timed" fi for port in "${PORTS[@]}"; do for i in $(seq 1 60); do diff --git a/packages/studio/tests/e2e/perf-ceilings.json b/packages/studio/tests/e2e/perf-ceilings.json index c7d3137a57..e263fc65e2 100644 --- a/packages/studio/tests/e2e/perf-ceilings.json +++ b/packages/studio/tests/e2e/perf-ceilings.json @@ -12,11 +12,11 @@ } }, "studio-playback": { - "browser": "154", + "browser": "153", "counts": { "playing.catalogElements": 6878, - "playing.compositedLayers": 171, - "playing.maxSeeksPerFrame": 1 + "paused.compositedLayers": 171, + "playing.maxSeeksPerFrame": 3 } } } diff --git a/packages/studio/tests/e2e/studio-playback.mjs b/packages/studio/tests/e2e/studio-playback.mjs index b1d8afa001..dc2061fb27 100644 --- a/packages/studio/tests/e2e/studio-playback.mjs +++ b/packages/studio/tests/e2e/studio-playback.mjs @@ -1,9 +1,9 @@ #!/usr/bin/env node -// Plays the studio-playback fixture with the Catalog open and prints evidence: exact work counts for -// perf-ratchet.mjs and main-thread CPU per frame. With STUDIO_BASE_URL (the base branch's Studio on this runner) -// it alternates runs of both and exits 1 when this build's median CPU per frame exceeds MAX_CPU_RATIO of the base's. -// Chrome runs without vsync or a frame cap, so a frame costs what the page makes it cost, even on a 60 Hz runner. +// Plays the studio-playback fixture with the Catalog open: exact work counts for perf-ratchet.mjs, and, against +// STUDIO_BASE_URL (the base branch's Studio on this runner, alternating runs), exits 1 when median main-thread +// CPU per frame exceeds MAX_CPU_RATIO of the base's. Chrome runs uncapped, so a frame costs what the page costs. import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; import { createRequire } from "node:module"; import puppeteer from "puppeteer-core"; import { resolveChromeExecutable } from "./chrome-executable.mjs"; @@ -67,10 +67,11 @@ async function serveFixtures(page, origin) { }); } -/** Counts, per composition animation frame, the eventful seeks of GSAP timelines. */ +/** Counts, per composition animation frame, every seek of a GSAP timeline, silent ones included. */ function instrumentPlayback() { - const player = document.querySelector("hyperframes-player"); - const win = player.shadowRoot.querySelector("iframe").contentWindow; + window.previewWindow = () => + document.querySelector("hyperframes-player").shadowRoot.querySelector("iframe").contentWindow; + const win = window.previewWindow(); const timelines = Object.values(win.__timelines).filter( (tl) => typeof tl?.totalTime === "function", ); @@ -79,9 +80,8 @@ function instrumentPlayback() { // Studio wraps each timeline's own totalTime, so count on the instance, outside those wrappers. for (const tl of timelines) { const totalTime = tl.totalTime; - tl.totalTime = function (time, suppressEvents) { - const eventfulSeek = arguments.length > 0 && !suppressEvents; - if (eventfulSeek && state.on) state.current += 1; + tl.totalTime = function () { + if (arguments.length > 0 && state.on) state.current += 1; return totalTime.apply(this, arguments); }; } @@ -93,11 +93,19 @@ function instrumentPlayback() { win.requestAnimationFrame(frame); } -/** The renderer main thread's complete events, in time order. */ +/** The page's renderer main thread (the one producing frames; Chrome may run more than one), in time order. */ function mainThreadEvents(traceEvents) { - const main = traceEvents.find( + const frames = new Map(); + for (const e of traceEvents) { + if (e.name !== "ProxyMain::BeginMainFrame") continue; + const key = `${e.pid}:${e.tid}`; + frames.set(key, (frames.get(key) ?? 0) + 1); + } + const mains = traceEvents.filter( (e) => e.ph === "M" && e.name === "thread_name" && e.args.name === "CrRendererMain", ); + const count = (e) => frames.get(`${e.pid}:${e.tid}`) ?? 0; + const main = mains.reduce((best, e) => (count(e) > count(best) ? e : best), mains[0]); return traceEvents .filter((e) => e.pid === main.pid && e.tid === main.tid && e.ph === "X") .sort((a, b) => a.ts - b.ts); @@ -165,9 +173,10 @@ async function measure(url) { ); await page.evaluate(instrumentPlayback); await page.mouse.move(0, 0); - await page.click('button[aria-label="Play"]'); - await page.mouse.move(0, 0); - await page.waitForFunction(() => document.querySelector('button[aria-label="Pause"]'), { + // Play and pause go through Studio's Space shortcut and the runtime's player, which both builds share. + await page.evaluate(() => document.activeElement?.blur()); + await page.keyboard.press("Space"); + await page.waitForFunction(() => previewWindow().__player?.isPlaying?.() === true, { timeout: 10_000, }); await new Promise((resolve) => setTimeout(resolve, 500)); @@ -191,17 +200,13 @@ async function measure(url) { }); const trace = JSON.parse(Buffer.from(await page.tracing.stop()).toString("utf8")); // The film's own layers come and go with its tweens, so layers are read paused at one film time. - await page.click('button[aria-label="Pause"]'); + await page.evaluate(() => previewWindow().__player.pause()); await page.evaluate( (t) => document.querySelector("hyperframes-player").seek(t), LAYER_READ_TIME_S, ); await page.waitForFunction( - (t) => { - const player = document.querySelector("hyperframes-player"); - const time = player.shadowRoot.querySelector("iframe").contentWindow.__player?.getTime?.(); - return Math.abs(time - t) < 0.01; - }, + (t) => Math.abs(previewWindow().__player?.getTime?.() - t) < 0.01, { timeout: 10_000 }, LAYER_READ_TIME_S, ); @@ -213,6 +218,7 @@ async function measure(url) { await cdp.send("LayerTree.enable"); const compositedLayers = (await layers).length; const cpu = cpuPerFrameMs(trace.traceEvents ?? trace); + if (cpu.length === 0) throw new Error(`no frames captured while playing ${url}`); const seekFrames = counts.seekFrames.slice(1); return { browser: await browser.version(), @@ -220,7 +226,7 @@ async function measure(url) { cpuMedianMs: round(median(cpu)), workCounts: { "playing.catalogElements": counts.catalogElements, - "playing.compositedLayers": compositedLayers, + "paused.compositedLayers": compositedLayers, "playing.maxSeeksPerFrame": seekFrames.length ? Math.max(...seekFrames) : -1, }, }; @@ -232,8 +238,10 @@ async function measure(url) { const head = []; const base = []; for (let round_ = 0; round_ < ROUNDS; round_ += 1) { + // Alternate which build goes first, so a runner that slows over the job does not favour one. + if (BASE_URL && round_ % 2 === 1) base.push(await measure(BASE_URL)); head.push(await measure(HEAD_URL)); - if (BASE_URL) base.push(await measure(BASE_URL)); + if (BASE_URL && round_ % 2 === 0) base.push(await measure(BASE_URL)); } // Counts must repeat exactly across runs of one build, or the ratchet would fail at random. const countSets = new Set(head.map((run) => JSON.stringify(run.workCounts))); @@ -241,6 +249,17 @@ if (countSets.size !== 1) { console.error(`work counts differ between runs of one build: ${[...countSets].join(" vs ")}`); process.exit(1); } +// Layer and element counts differ between Chrome releases, so they are only comparable on the recorded one. +const ceilingsPath = fileURLToPath(new URL("./perf-ceilings.json", import.meta.url)); +const recordedBrowser = JSON.parse(readFileSync(ceilingsPath, "utf8"))["studio-playback"]?.browser; +const measuredBrowser = /\/(\d+)\./.exec(head[0].browser)?.[1]; +if (recordedBrowser && measuredBrowser !== recordedBrowser) { + console.error( + `studio-playback counts are recorded on Chrome ${recordedBrowser} but this ran Chrome ${measuredBrowser}: ` + + "run it in the CI job, or take new ceilings from that job's evidence", + ); + process.exit(1); +} const evidence = { journey: "studio-playback", browser: head[0].browser,