From d218cec091a7b362eacf884f8b566afcd7f62bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 1 Oct 2026 16:35:49 -0700 Subject: [PATCH] ci(studio): gates read the base the run measured, and drag frames on a quiet runner (#4872) * ci(studio): measure drag smoothness on a quiet runner, with a blank-page control * ci(studio): gates read the base the run measured, not the base branch tip * style(ci): format the base-file test --- .github/workflows/ci.yml | 27 ++------- .github/workflows/studio-drag-frames.yml | 76 ++++++++++++++++++++++++ package.json | 2 +- scripts/ci/base-file.sh | 14 +++++ scripts/ci/base-file.test.mjs | 68 +++++++++++++++++++++ 5 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/studio-drag-frames.yml create mode 100755 scripts/ci/base-file.sh create mode 100644 scripts/ci/base-file.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a72ef42c90..28fb97023a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -851,14 +851,7 @@ jobs: - name: Read the base branch's work-count ceilings # A ceiling may not disappear, or rise without a new reason, against the branch this PR merges into. if: github.event_name == 'pull_request' - env: - BASE_REF: ${{ github.base_ref }} - run: | - git fetch --no-tags --depth=1 origin "$BASE_REF" - if ! git show FETCH_HEAD:packages/studio/tests/e2e/perf-ceilings.json > /tmp/base-perf-ceilings.json; then - rm -f /tmp/base-perf-ceilings.json - echo "The base branch has no perf-ceilings.json yet" - fi + run: bash scripts/ci/base-file.sh packages/studio/tests/e2e/perf-ceilings.json /tmp/base-perf-ceilings.json - name: Run both arms of the timeline viewport gate run: | set -euo pipefail @@ -1020,14 +1013,9 @@ jobs: # Studio and the producer both run in the engine's pinned headless shell. - run: node packages/cli/dist/cli.js browser ensure - name: Read the base branch's edit accuracy baseline - env: - BASE_REF: ${{ github.base_ref }} run: | - git fetch --no-tags --depth=1 origin "$BASE_REF" - if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then - echo '{"cases":{}}' > /tmp/base-edit-accuracy.json - echo "The base branch has no edit accuracy baseline yet" - fi + bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json + [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json - name: Run this shard, then re-run twice each case whose verdict differs from the base branch run: | set -euo pipefail @@ -1062,14 +1050,9 @@ jobs: with: node-version: 22 - name: Read the base branch's edit accuracy baseline - env: - BASE_REF: ${{ github.base_ref }} run: | - git fetch --no-tags --depth=1 origin "$BASE_REF" - if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then - echo '{"cases":{}}' > /tmp/base-edit-accuracy.json - echo "The base branch has no edit accuracy baseline yet" - fi + bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json + [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: pattern: edit-accuracy-shard-* diff --git a/.github/workflows/studio-drag-frames.yml b/.github/workflows/studio-drag-frames.yml new file mode 100644 index 0000000000..9c95f2cd75 --- /dev/null +++ b/.github/workflows/studio-drag-frames.yml @@ -0,0 +1,76 @@ +name: Studio drag frames + +permissions: + contents: read + +env: + HYPERFRAMES_NO_TELEMETRY: "1" + +# Measures drag smoothness on a runner with nothing else on it. The bench measures a blank page per case +# in the same run, and the box only counts as quiet if that control dropped no frame anywhere. +on: + workflow_dispatch: + inputs: + ref: + description: Commit, branch or tag to measure + default: main + pull_request: + paths: + - ".github/workflows/studio-drag-frames.yml" + +jobs: + drag-frames: + name: "Studio: drag frames" + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + BENCH_REF: ${{ inputs.ref || 'main' }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ env.BENCH_REF }} + lfs: true + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - run: corepack enable + - run: corepack prepare pnpm@10.17.1 --activate + - uses: ./.github/actions/prepare-ffmpeg-bin + - run: bash scripts/ci/install-workspace-dependencies.sh + - run: bun run build + - run: node packages/cli/dist/cli.js browser ensure + - name: Run the drag cases five times + working-directory: packages/studio + run: | + set -euo pipefail + echo "Measuring $(git rev-parse HEAD)" + for n in 1 2 3 4 5; do + bun tests/e2e/edit-accuracy/run.mjs --grid pr \ + --filter '^(move|resize|rotate|crop)-(none|tween)-px-r0-root-z100$' \ + --jobs 1 --port 5961 --cli "$GITHUB_WORKSPACE/packages/cli/dist/cli.js" \ + --out "$RUNNER_TEMP/drag-frames/run-$n" + done + - name: Require a quiet box (the blank page dropped no frame) + run: | + # shellcheck disable=SC2016 # the backticks are JavaScript template strings + node -e ' + const fs = require("fs"), dir = process.env.RUNNER_TEMP + "/drag-frames"; + let noisy = 0; + for (const run of fs.readdirSync(dir)) { + const { cases } = JSON.parse(fs.readFileSync(`${dir}/${run}/results.json`, "utf8")); + for (const c of cases) { + const dropped = c.smooth?.control?.dropped; + if (dropped !== 0) { noisy++; console.log(`${run} ${c.id} control dropped=${dropped}`); } + } + } + if (noisy) { console.log(`::error::the blank page dropped frames, or went unmeasured, in ${noisy} case(s); this box is not quiet`); process.exit(1); } + console.log("the blank page dropped 0 frames in every case of every run"); + ' + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: studio-drag-frames + path: ${{ runner.temp }}/drag-frames + if-no-files-found: error + retention-days: 30 diff --git a/package.json b/package.json index dafda9bd95..642414e008 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", + "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", "test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", diff --git a/scripts/ci/base-file.sh b/scripts/ci/base-file.sh new file mode 100755 index 0000000000..0d3c280cf1 --- /dev/null +++ b/scripts/ci/base-file.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# base-file.sh : writes as it was on the base this pull_request run measured, the first parent of +# the checked-out merge commit. The base branch tip moves while a run is in flight, so it is never read. A base without +# the file removes ; a failed fetch fails the step. +set -euo pipefail +base="$(git cat-file -p HEAD | awk '/^parent /{print $2; exit}')" +echo "Reading $1 from the base this run measured: $base" +git fetch --no-tags --depth=1 origin "$base" +if git cat-file -e "$base:$1" 2>/dev/null; then + git show "$base:$1" > "$2" +else + rm -f "$2" + echo "The base has no $1 yet" +fi diff --git a/scripts/ci/base-file.test.mjs b/scripts/ci/base-file.test.mjs new file mode 100644 index 0000000000..a0e0df7936 --- /dev/null +++ b/scripts/ci/base-file.test.mjs @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +const script = join(import.meta.dirname, "base-file.sh"); +const git = (cwd, ...args) => + execFileSync( + "git", + ["-c", "user.email=t@t", "-c", "user.name=t", "-c", "commit.gpgsign=false", ...args], + { + cwd, + encoding: "utf8", + }, + ).trim(); + +// A PR run checks out the merge commit shallowly; the base branch then moves before the gate reads its baseline. +function raceFixture(baseline) { + const root = mkdtempSync(join(tmpdir(), "base-file-")); + const origin = join(root, "origin.git"); + const work = join(root, "work"); + git(root, "init", "-q", "--bare", origin); + git(root, "init", "-q", "-b", "main", work); + git(work, "remote", "add", "origin", origin); + if (baseline) writeFileSync(join(work, "baseline.json"), baseline); + writeFileSync(join(work, "readme"), "base"); + git(work, "add", "."); + git(work, "commit", "-qm", "base"); + const base = git(work, "rev-parse", "HEAD"); + git(work, "switch", "-qc", "pr"); + writeFileSync(join(work, "pr"), "change"); + git(work, "add", "."); + git(work, "commit", "-qm", "pr"); + git(work, "switch", "-q", "main"); + git(work, "merge", "-q", "--no-ff", "-m", "merge", "pr"); + const merge = git(work, "rev-parse", "HEAD"); + git(work, "push", "-q", "origin", `${merge}:refs/pull/1/merge`); + git(work, "reset", "-q", "--hard", base); + writeFileSync(join(work, "baseline.json"), '{"total":852}'); + git(work, "add", "."); + git(work, "commit", "-qm", "base moves on"); + git(work, "push", "-q", "origin", "HEAD:main"); + const checkout = join(root, "checkout"); + git(root, "init", "-q", checkout); + git(checkout, "remote", "add", "origin", `file://${origin}`); + git(checkout, "fetch", "-q", "--depth=1", "origin", "refs/pull/1/merge"); + git(checkout, "checkout", "-q", "FETCH_HEAD"); + return { checkout, base, out: join(root, "out.json") }; +} + +test("reads the base the run measured, not the base branch tip that moved since", () => { + const { checkout, base, out } = raceFixture('{"total":846}'); + const log = execFileSync("bash", [script, "baseline.json", out], { + cwd: checkout, + encoding: "utf8", + }); + assert.equal(readFileSync(out, "utf8"), '{"total":846}'); + assert.match(log, new RegExp(`measured: ${base}`)); +}); + +test("a base without the file leaves no output", () => { + const { checkout, out } = raceFixture(null); + writeFileSync(out, "stale"); + execFileSync("bash", [script, "baseline.json", out], { cwd: checkout, encoding: "utf8" }); + assert.equal(existsSync(out), false); +});