From 5aed93b5c6faeef00c7327dcb94325dbeb7dd8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 27 Sep 2026 16:55:55 -0400 Subject: [PATCH] test(studio): the playback gate reports timing and fails only on counts Timing was meant to be reporting-only, but a ratio over 1.15x still exited 1 and turned the job red; identical builds cross it in 31% of draws. It now prints a warning (a workflow warning in CI) and exits 0. Only the counts, a Chrome other than the ceilings', or a broken run fail the step. On a Chrome mismatch the evidence JSON is printed before the exit, so new ceilings can be taken from it. The mismatch rule has one owner, perf-ratchet.mjs browserMismatch, which lower, the check notes and the script all call. playbackVerdict is exported and tested: a 1.3x ratio exits 0 with a warning, a mismatch exits 1, and a count rise fails the ratchet. --- .github/workflows/ci.yml | 11 +- packages/studio/tests/e2e/perf-ratchet.mjs | 27 ++-- packages/studio/tests/e2e/studio-playback.mjs | 140 +++++++++++------- .../studio/tests/e2e/studio-playback.test.mjs | 39 +++++ 4 files changed, 145 insertions(+), 72 deletions(-) create mode 100644 packages/studio/tests/e2e/studio-playback.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0014e7a05..c52ecd509e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -915,8 +915,8 @@ jobs: packages/studio/tests/e2e/perf-ceilings.json studio-open /tmp/studio-open-counts.json \ /tmp/base-perf-ceilings.json - name: Count and time playback with the Catalog open - # Counts go through perf-ceilings.json like studio-open. Time is only ever a ratio - # against the base branch's Studio on this runner, so a slow runner cannot fail it. + # Counts go through perf-ceilings.json like studio-open and can fail the step. Time is a + # ratio against the base branch's Studio on this runner, reported as a warning only. # Both servers stay up; they sit idle while the other one is measured. if: success() || failure() run: | @@ -949,13 +949,14 @@ jobs: STUDIO_BASE_URL="$BASE_URL" \ node packages/studio/tests/e2e/studio-playback.mjs \ | tee /tmp/studio-playback.json - TIMING_STATUS=$? + PLAYBACK_STATUS=$? + node -e 'const w = require("/tmp/studio-playback.json").ab?.warning; if (w) console.log(`::warning::${w}`)' || true node packages/studio/tests/e2e/perf-ratchet.mjs check \ packages/studio/tests/e2e/perf-ceilings.json studio-playback /tmp/studio-playback.json \ /tmp/base-perf-ceilings.json COUNTS_STATUS=$? - if (( TIMING_STATUS != 0 || COUNTS_STATUS != 0 )); then - echo "FAIL: playback timing=${TIMING_STATUS}, counts=${COUNTS_STATUS}" + if (( PLAYBACK_STATUS != 0 || COUNTS_STATUS != 0 )); then + echo "FAIL: playback run=${PLAYBACK_STATUS}, counts=${COUNTS_STATUS}" exit 1 fi - name: Upload gate evidence diff --git a/packages/studio/tests/e2e/perf-ratchet.mjs b/packages/studio/tests/e2e/perf-ratchet.mjs index 9c8f5b7b8b..ac8e43aa0c 100644 --- a/packages/studio/tests/e2e/perf-ratchet.mjs +++ b/packages/studio/tests/e2e/perf-ratchet.mjs @@ -139,6 +139,16 @@ export function browserMajor(evidence) { return match ? match[1] : null; } +/** Why counts in `evidence` cannot be compared with ceilings recorded on Chrome `recorded`, or null. */ +export function browserMismatch(recorded, evidence) { + const measuredOn = browserMajor(evidence); + if (measuredOn === recorded) return null; + return ( + `evidence is from Chrome ${measuredOn} but the ceilings were recorded on Chrome ${recorded}: ` + + "take the evidence from the CI job, which runs that browser" + ); +} + /** The whole ceilings file, the journey named in it, and that journey's evidence. */ function journeyInputs([ceilingsPath, journey, evidencePath]) { const all = readJson(ceilingsPath); @@ -188,12 +198,8 @@ function loadBase(path, all, journey) { } function printNotes(recorded, evidence, rows) { - const measuredOn = browserMajor(evidence); - if (measuredOn !== recorded.browser) { - console.log( - `[perf-ratchet] note: measured on Chrome ${measuredOn}, ceilings recorded on Chrome ${recorded.browser}`, - ); - } + const mismatch = browserMismatch(recorded.browser, evidence); + if (mismatch) console.log(`[perf-ratchet] note: ${mismatch}`); if (rows.some((row) => row.status === "rose")) { console.log( "[perf-ratchet] If this change should not add that work, confirm the counter still repeats " + @@ -230,13 +236,8 @@ function runCheck(args) { function runLower(args) { const { all, ceilingsPath, journey, evidence, counts } = journeyInputs(args); - const measuredOn = browserMajor(evidence); - if (measuredOn !== all[journey].browser) { - throw new Error( - `evidence is from Chrome ${measuredOn} but ${journey}'s ceilings were recorded on Chrome ` + - `${all[journey].browser}: take the evidence from the CI job, which runs that browser`, - ); - } + const mismatch = browserMismatch(all[journey].browser, evidence); + if (mismatch) throw new Error(`${journey}: ${mismatch}`); all[journey].counts = lowerCeilings(all[journey].counts, counts); writeFileSync(ceilingsPath, `${JSON.stringify(all, null, 2)}\n`); return 0; diff --git a/packages/studio/tests/e2e/studio-playback.mjs b/packages/studio/tests/e2e/studio-playback.mjs index dc2061fb27..0d92731f59 100644 --- a/packages/studio/tests/e2e/studio-playback.mjs +++ b/packages/studio/tests/e2e/studio-playback.mjs @@ -1,12 +1,13 @@ #!/usr/bin/env node // Plays the studio-playback fixture with the Catalog open: exact work counts for perf-ratchet.mjs, and, against -// STUDIO_BASE_URL (the base branch's Studio on this runner, alternating runs), exits 1 when median main-thread +// STUDIO_BASE_URL (the base branch's Studio on this runner, alternating runs), a warning when median main-thread // CPU per frame exceeds MAX_CPU_RATIO of the base's. Chrome runs uncapped, so a frame costs what the page costs. -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; +import { readFileSync, realpathSync } from "node:fs"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { createRequire } from "node:module"; import puppeteer from "puppeteer-core"; import { resolveChromeExecutable } from "./chrome-executable.mjs"; +import { browserMismatch } from "./perf-ratchet.mjs"; const HEAD_URL = process.env.STUDIO_URL; const BASE_URL = process.env.STUDIO_BASE_URL; @@ -18,19 +19,8 @@ const LAYER_READ_TIME_S = 5; const CATALOG_SIZE = 400; const CATEGORIES = ["transitions", "vfx", "social", "data", "scenes", "captions", "effects"]; -if (!HEAD_URL) { - console.error("STUDIO_URL is required and must point at the studio-playback fixture"); - process.exit(2); -} -const executablePath = resolveChromeExecutable(); -if (!executablePath) { - console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH"); - process.exit(2); -} -const gsapSource = readFileSync( - createRequire(import.meta.url).resolve("gsap/dist/gsap.min.js"), - "utf8", -); +let executablePath; +let gsapSource; const catalog = Array.from({ length: CATALOG_SIZE }, (_, i) => ({ name: `perf-block-${i}`, @@ -235,40 +225,45 @@ async function measure(url) { } } -const head = []; -const base = []; -for (let round_ = 0; round_ < ROUNDS; round_ += 1) { - // Alternate which build goes first, so a runner that slows over the job does not favour one. - if (BASE_URL && round_ % 2 === 1) base.push(await measure(BASE_URL)); - head.push(await measure(HEAD_URL)); - if (BASE_URL && round_ % 2 === 0) base.push(await measure(BASE_URL)); -} -// Counts must repeat exactly across runs of one build, or the ratchet would fail at random. -const countSets = new Set(head.map((run) => JSON.stringify(run.workCounts))); -if (countSets.size !== 1) { - console.error(`work counts differ between runs of one build: ${[...countSets].join(" vs ")}`); - process.exit(1); -} -// Layer and element counts differ between Chrome releases, so they are only comparable on the recorded one. -const ceilingsPath = fileURLToPath(new URL("./perf-ceilings.json", import.meta.url)); -const recordedBrowser = JSON.parse(readFileSync(ceilingsPath, "utf8"))["studio-playback"]?.browser; -const measuredBrowser = /\/(\d+)\./.exec(head[0].browser)?.[1]; -if (recordedBrowser && measuredBrowser !== recordedBrowser) { - console.error( - `studio-playback counts are recorded on Chrome ${recordedBrowser} but this ran Chrome ${measuredBrowser}: ` + - "run it in the CI job, or take new ceilings from that job's evidence", +function timingWarning(ab) { + if (!ab || ab.headCpuMedianMs <= ab.baseCpuMedianMs * ab.maxRatio) return null; + return ( + `main-thread CPU per frame ${ab.headCpuMedianMs} ms against the base's ${ab.baseCpuMedianMs} ms ` + + `(x${ab.ratio}, limit x${ab.maxRatio}); reported, not failed` ); - process.exit(1); } -const evidence = { - journey: "studio-playback", - browser: head[0].browser, - workCounts: head[0].workCounts, - cpuMedianMsPerRun: head.map((run) => run.cpuMedianMs), - framesPerRun: head.map((run) => run.frames), -}; -let failed = false; -if (BASE_URL) { + +/** + * Timing is reporting-only (a warning, never a failure), and only counts from the ceilings' Chrome are + * comparable. Returns the exit code and what to print, so the rule can be tested without a browser. + */ +export function playbackVerdict(evidence, recordedBrowser) { + const error = recordedBrowser ? browserMismatch(recordedBrowser, evidence) : null; + return { exitCode: error ? 1 : 0, warning: timingWarning(evidence.ab), error }; +} + +/** Alternates which build goes first each round, so a runner that slows over the job favours neither. */ +// fallow-ignore-next-line complexity +async function measureRounds() { + const head = []; + const base = []; + for (let round_ = 0; round_ < ROUNDS; round_ += 1) { + if (BASE_URL && round_ % 2 === 1) base.push(await measure(BASE_URL)); + head.push(await measure(HEAD_URL)); + if (BASE_URL && round_ % 2 === 0) base.push(await measure(BASE_URL)); + } + return { head, base }; +} + +function buildEvidence(head, base) { + const evidence = { + journey: "studio-playback", + browser: head[0].browser, + workCounts: head[0].workCounts, + cpuMedianMsPerRun: head.map((run) => run.cpuMedianMs), + framesPerRun: head.map((run) => run.frames), + }; + if (base.length === 0) return evidence; const headMs = median(head.map((run) => run.cpuMedianMs)); const baseMs = median(base.map((run) => run.cpuMedianMs)); evidence.ab = { @@ -279,12 +274,49 @@ if (BASE_URL) { ratio: round(headMs / baseMs), maxRatio: MAX_CPU_RATIO, }; - failed = headMs > baseMs * MAX_CPU_RATIO; + return evidence; } -console.log(JSON.stringify(evidence, null, 2)); -if (failed) { - console.error( - `FAIL main-thread CPU per frame ${evidence.ab.headCpuMedianMs} ms against the base's ${evidence.ab.baseCpuMedianMs} ms (x${evidence.ab.ratio}, limit x${MAX_CPU_RATIO})`, + +/** Exit code 2 for a missing URL or Chrome, else null once the browser and GSAP are ready. */ +function prepare() { + if (!HEAD_URL) { + console.error("STUDIO_URL is required and must point at the studio-playback fixture"); + return 2; + } + executablePath = resolveChromeExecutable(); + if (!executablePath) { + console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH"); + return 2; + } + gsapSource = readFileSync( + createRequire(import.meta.url).resolve("gsap/dist/gsap.min.js"), + "utf8", ); - process.exit(1); + return null; +} + +// fallow-ignore-next-line complexity +async function main() { + const setupFailure = prepare(); + if (setupFailure !== null) return setupFailure; + const { head, base } = await measureRounds(); + // Counts must repeat exactly across runs of one build, or the ratchet would fail at random. + const countSets = new Set(head.map((run) => JSON.stringify(run.workCounts))); + if (countSets.size !== 1) { + console.error(`work counts differ between runs of one build: ${[...countSets].join(" vs ")}`); + return 1; + } + const evidence = buildEvidence(head, base); + const ceilingsPath = fileURLToPath(new URL("./perf-ceilings.json", import.meta.url)); + const recorded = JSON.parse(readFileSync(ceilingsPath, "utf8"))["studio-playback"]?.browser; + const verdict = playbackVerdict(evidence, recorded); + if (verdict.warning) evidence.ab.warning = verdict.warning; + // Evidence first, even on a browser mismatch: it is what new ceilings are taken from. + console.log(JSON.stringify(evidence, null, 2)); + if (verdict.error) console.error(`studio-playback: ${verdict.error}`); + return verdict.exitCode; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) { + process.exitCode = await main(); } diff --git a/packages/studio/tests/e2e/studio-playback.test.mjs b/packages/studio/tests/e2e/studio-playback.test.mjs new file mode 100644 index 0000000000..8a0d258850 --- /dev/null +++ b/packages/studio/tests/e2e/studio-playback.test.mjs @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { checkCeilings } from "./perf-ratchet.mjs"; +import { playbackVerdict } from "./studio-playback.mjs"; + +const evidence = (headMs, baseMs, browser = "HeadlessChrome/153.0.8010.52") => ({ + browser, + workCounts: { "playing.catalogElements": 176 }, + ab: { headCpuMedianMs: headMs, baseCpuMedianMs: baseMs, ratio: headMs / baseMs, maxRatio: 1.15 }, +}); + +describe("playbackVerdict", () => { + it("reports a 1.3x timing ratio as a warning and exits 0", () => { + const verdict = playbackVerdict(evidence(6.5, 5), "153"); + expect(verdict.exitCode).toBe(0); + expect(verdict.warning).toMatch(/reported, not failed/); + }); + + it("has nothing to say about timing within the limit", () => { + expect(playbackVerdict(evidence(5.5, 5), "153")).toEqual({ + exitCode: 0, + warning: null, + error: null, + }); + }); + + it("exits 1 on a Chrome other than the ceilings', naming both", () => { + const verdict = playbackVerdict(evidence(5, 5, "HeadlessChrome/154.0.1.2"), "153"); + expect(verdict.exitCode).toBe(1); + expect(verdict.error).toMatch(/Chrome 154 .* Chrome 153/); + }); + + it("leaves a count rise to the ratchet, which fails it", () => { + const rise = checkCeilings( + { "playing.catalogElements": 176 }, + { "playing.catalogElements": 177 }, + ); + expect(rise.passed).toBe(false); + }); +});