perf(studio): speed up the edit accuracy bench and trim its grid (#5569)

* perf(studio): cut fixed waits from the edit accuracy bench - Settle window 1000 ms -> 500 ms (EDIT_ACCURACY_STILL_MS overrides it). - Drop the fixed 1 s sleep in openStudio; the settle that follows is the real readiness wait. - Mark composition hosts data-no-timeline in the producer's page when the saved project has no GSAP timeline, so the render skips the 10 s playerReadyTimeout wait. - Run the blank-page smoothness control once per gesture per browser. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * ci(studio): trim the edit accuracy grid to 741 cases Drops zoom 200, the pct and center placements, and nested keyframed cases from the grid (2061 -> 741 cases; the pr slice 435 -> 265). Drag paths and sequences are unchanged. baseline.json loses the 1320 removed cases. The ratchet no longer counts a base case that the head's baseline.json does not hold, and lists such cases as retired, so removing cases from the grid and baseline.json together does not fail the gate. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * fix(studio): measure the blank-page control again after it fails The control is cached per browser and gesture, so a rejected measurement stayed cached and failed every later case with that gesture before Studio was tested. Drop a rejected entry so the next case measures again. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * ci(studio): keep the nested keyframed resize cases in the edit accuracy PR slice Two historical gate runs failed only on resize-scale-px-r0-nested-z100-on, which the trim removed, so the PR slice would have passed that build. Keep the nested keyframed resize on a keyframe at zoom 100 without rotation, one case per GSAP mode, with their baseline entries. PR slice 265 -> 271 cases, full grid 741 -> 747. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-30 09:16:55 +00:00 · 2026-10-14 03:46:17 +00:00
parent 95837a3df7
commit a59a422195
10 changed files with 142 additions and 1351 deletions
@@ -231,7 +231,8 @@ export async function measure(ctx) {
  return { map, quad, size, visible: visibleQuad(quad, size, parseInset(box.clip)) };
 }

-const STILL_MS = 1000;
+// 500 ms, from 1000: the window restarts on any change, so a shorter one still waits out a slow save or swap.
+const STILL_MS = Number(process.env.EDIT_ACCURACY_STILL_MS ?? 500);

 // Studio reloads an edited preview in a shadow iframe (`_t` in its URL) and swaps it in when painted.
 const previewFrames = (page) =>
@@ -303,7 +304,6 @@ export async function openStudio(ctx) {
    seek = null;
  }
  if (!seek) throw new Error("studio never reported a seekable composition");
-  await sleep(1000);
  return settled(ctx);
 }

@@ -675,8 +675,23 @@ export function smoothness(rec) {
 const CONTROL_PAGE = `data:text/html,<body style="margin:0;background:%23202020"><div id="box"
  style="position:absolute;left:600px;top:300px;width:240px;height:160px;background:%23f0c020"></div>`;

+// Reported, never gated, so it runs once per gesture per browser rather than once per case.
+const controls = new WeakMap();
+
 /** The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops. */
-export async function controlDrag(browser, gesture) {
+export function controlDrag(browser, gesture) {
+  const byGesture = controls.get(browser) ?? new Map();
+  controls.set(browser, byGesture);
+  if (!byGesture.has(gesture)) {
+    const measured = measureControl(browser, gesture);
+    // A failed control is not cached, so one transient failure does not fail every later case.
+    measured.catch(() => byGesture.get(gesture) === measured && byGesture.delete(gesture));
+    byGesture.set(gesture, measured);
+  }
+  return byGesture.get(gesture);
+}
+
+async function measureControl(browser, gesture) {
  const context = await browser.createBrowserContext();
  try {
    const page = await context.newPage();
@@ -27,13 +27,16 @@ export function localAsset(url) {

 // Studio has corner handles only (ResizeHandle is nw|ne|sw|se); its edge strips crop, so there is no edge resize.
 const GESTURES = ["move", "resize", "rotate", "crop", "nudge"];
+// Trimmed from 2061 cases: across 444 historical result snapshots, zoom 200, the pct and center placements and most nested
+// keyframed cases never failed where a kept case with the same gesture, GSAP mode and playhead did not. Drag paths and
+// sequences still cover pct, center and zoom 200.
 const AXES = {
  // idle: GSAP loaded and a paused timeline tweens another element; the target itself is plain CSS.
  gsap: ["none", "idle", "tween", "hold"],
-  placement: ["px", "pct", "center", "xpercent"],
+  placement: ["px", "xpercent"],
  rotation: [0, 30],
  nesting: ["root", "nested"],
-  zoom: [50, 100, 200],
+  zoom: [50, 100],
 };

 // Timelines with keyframes at `times` (0, 2 and 3 s unless set) on the property each animates; `css` is it in CSS.
@@ -105,23 +108,30 @@ function keyframedCases() {
    nesting: AXES.nesting,
    zoom: AXES.zoom,
    gesture: GESTURES,
-  }).flatMap((c) => {
-    const times = KEYFRAMED[c.gsap].times ?? KEY_TIMES;
-    return playheadsFor(times).map((at) => ({
-      id: `${caseId(c)}-${at}`,
-      ...c,
-      at,
-      playhead: AT[at],
-      keys: {
-        times: times.filter((t) => t !== AT[at]),
-        props: KEYFRAMED[c.gsap].props,
-        css: KEYFRAMED[c.gsap].css,
-        render: times.at(-1),
-      },
-    }));
-  });
+  })
+    .flatMap((c) => {
+      const times = KEYFRAMED[c.gsap].times ?? KEY_TIMES;
+      return playheadsFor(times).map((at) => ({
+        id: `${caseId(c)}-${at}`,
+        ...c,
+        at,
+        playhead: AT[at],
+        keys: {
+          times: times.filter((t) => t !== AT[at]),
+          props: KEYFRAMED[c.gsap].props,
+          css: KEYFRAMED[c.gsap].css,
+          render: times.at(-1),
+        },
+      }));
+    })
+    .filter((c) => c.nesting === "root" || NESTED_KEYFRAMED(c));
 }

+// Nested keyframed cases are kept only for a resize on a keyframe at zoom 100 without rotation, so the PR slice
+// still catches the nested resize regression two historical gate runs failed on (resize-scale-px-r0-nested-z100-on).
+const NESTED_KEYFRAMED = (c) =>
+  c.gesture === "resize" && c.at === "on" && c.zoom === 100 && c.rotation === 0;
+
 const product = (axes) =>
  Object.entries(axes).reduce(
    (rows, [key, values]) => rows.flatMap((row) => values.map((v) => ({ ...row, [key]: v }))),
@@ -28,3 +28,12 @@ describe("grid fixtures", () => {
    expect([...new Set(unserved)]).toEqual([]);
  }, 30_000);
 });
+
+describe("grid size", () => {
+  it("holds 747 cases, 271 of them in the pull request slice, each id once", () => {
+    const full = buildGrid("full");
+    expect(full).toHaveLength(747);
+    expect(new Set(full.map((c) => c.id)).size).toBe(747);
+    expect(buildGrid("pr")).toHaveLength(271);
+  });
+});
@@ -81,8 +81,16 @@ describe("strayCss", () => {
 describe("the keyframed grid", () => {
  const grid = buildGrid("keyframes");

-  it("crosses 6 variants, 5 gestures and 3 playheads each with rotation, nesting and zoom", () => {
-    expect(grid).toHaveLength(6 * 5 * 3 * 2 * 2 * 3);
+  it("crosses 6 variants, 5 gestures and 3 playheads each with rotation and zoom at the root, plus a nested resize per variant", () => {
+    expect(grid).toHaveLength(6 * 5 * 3 * 2 * 2 + 6);
+    const nested = grid.filter((c) => c.nesting === "nested");
+    expect(nested.map((c) => c.id)).toContain("resize-scale-px-r0-nested-z100-on");
+    expect(nested).toHaveLength(6);
+    expect(
+      nested.every(
+        (c) => c.gesture === "resize" && c.at === "on" && c.zoom === 100 && c.rotation === 0,
+      ),
+    ).toBe(true);
    expect(new Set(grid.map((c) => c.id)).size).toBe(grid.length);
    expect(new Set(grid.map((c) => c.gsap))).toEqual(
      new Set(["size", "scale", "spin", "keys", "fromto", "late"]),
@@ -108,7 +116,11 @@ describe("the keyframed grid", () => {
  });

  it("writes each variant's timeline into the file that holds the target", () => {
-    const spec = grid.find((c) => c.id === "move-fromto-px-r30-nested-z100-on");
+    // Nested keyframed cases left the grid; the fixture still writes them.
+    const spec = {
+      ...grid.find((c) => c.id === "move-fromto-px-r30-root-z100-on"),
+      nesting: "nested",
+    };
    const dir = mkdtempSync(join(tmpdir(), "edit-bench-keys-"));
    writeFixture(spec, dir);
    const sub = readFileSync(join(dir, "compositions/sub.html"), "utf8");
@@ -59,10 +59,12 @@ export const QUARANTINED = {};

 /**
 * Every run of every case: each shard's run plus the re-runs of the cases it flipped. `scope`, for a run of
- * part of the grid, limits the base branch's passing count to the cases that ran.
+ * part of the grid, limits the base branch's passing count to the cases that ran. A case removed from the grid
+ * on purpose leaves the head's baseline.json in the same change: it is listed as retired and no longer counted.
 */
 // fallow-ignore-next-line complexity
 export function gate(base, head, runs, quarantine = QUARANTINED, scope = null) {
+  const counted = (id) => Object.hasOwn(head.cases, id) && (!scope || scope.has(id));
  const seen = new Map();
  for (const r of runs) seen.set(r.id, [...(seen.get(r.id) ?? []), entry(r)]);
  const all = [...seen].map(([id, entries]) => {
@@ -81,7 +83,7 @@ export function gate(base, head, runs, quarantine = QUARANTINED, scope = null) {
      return { id, fixer, passed: c?.passed ?? null, runs: c?.entries.map(accurate) ?? [] };
    }),
    basePassing: Object.entries(base.cases).filter(
-      ([id, e]) => !Object.hasOwn(quarantine, id) && (!scope || scope.has(id)) && accurate(e),
+      ([id, e]) => !Object.hasOwn(quarantine, id) && counted(id) && accurate(e),
    ).length,
    headPassing: passing.length,
    headSmooth: passing.filter((c) => mostlySmooth(c.entries)).length,
@@ -92,7 +94,8 @@ export function gate(base, head, runs, quarantine = QUARANTINED, scope = null) {
    newlyPassing: passing.filter((c) => !c.basePassed).map((c) => c.id),
    unbanked: passing.filter((c) => !accurate(head.cases[c.id])).map((c) => c.id),
    overclaimed: cases.filter((c) => !c.passed && accurate(head.cases[c.id])).map((c) => c.id),
-    missing: Object.keys(base.cases).filter((id) => (!scope || scope.has(id)) && !seen.has(id)),
+    missing: Object.keys(base.cases).filter((id) => counted(id) && !seen.has(id)),
+    retired: Object.keys(base.cases).filter((id) => !Object.hasOwn(head.cases, id)),
  };
  const reasons = [
    result.regressed.length &&
@@ -131,6 +134,7 @@ export function comment(g) {
    ),
    ...list("Marked passing in baseline.json but failing", g.overclaimed),
    ...list("In the base grid but not run", g.missing),
+    ...list("Retired: removed from baseline.json with the grid, not counted", g.retired),
    `**Quarantined, measured but not gated** (${g.quarantined.length})`,
    ...g.quarantined.map(
      (q) =>
@@ -158,6 +158,12 @@ describe("gate", () => {
    expect(g.reasons.join()).toContain("fell from 2 to 1");
  });

+  it("does not count a case retired from the grid with its baseline.json entry, and lists it", () => {
+    const g = gate(base, baseline({ a: good }), [run("a")]);
+    expect(g).toMatchObject({ basePassing: 1, missing: [], retired: ["b"], ok: true });
+    expect(comment(g)).toContain("Retired");
+  });
+
  it("counts only the cases that ran when a run covers part of the grid, still failing a regression", () => {
    const slice = new Set(["a"]);
    expect(gate(base, base, [run("a")], {}, slice)).toMatchObject({ missing: [], ok: true });
@@ -1,5 +1,5 @@
 /** Render drift: the target's box in a frame the producer captures, found by its flat colour. */
-import { mkdirSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 // Source import, as scripts/preview-capture.ts does: bun runs the producer's TypeScript directly.
 import {
@@ -63,6 +63,22 @@ const luminance = (hex) => {
  return 0.299 * r + 0.587 * g + 0.114 * b;
 };

+/** A saved project with no GSAP timeline; a keyframe added in Studio writes one, so the files decide, not the spec. */
+const noTimeline = (dir) =>
+  !["index.html", "compositions/sub.html"].some(
+    (f) => existsSync(join(dir, f)) && readFileSync(join(dir, f), "utf8").includes("__timelines"),
+  );
+
+/** Runs in the producer's page: marks every composition host `data-no-timeline`, the engine's opt-out of the wait. */
+function markNoTimeline() {
+  const mark = (e) => e.matches("[data-composition-id]") && e.setAttribute("data-no-timeline", "");
+  new MutationObserver((records) => {
+    for (const r of records)
+      for (const n of r.addedNodes)
+        if (n.nodeType === 1) [n, ...n.querySelectorAll("[data-composition-id]")].forEach(mark);
+  }).observe(document, { childList: true, subtree: true });
+}
+
 /** Renders `dir` at `time` and measures the target's box in the frame. */
 export async function renderBox(dir, decoder, time = PLAYHEAD) {
  const framesDir = join(dir, ".bench-frames");
@@ -87,6 +103,8 @@ export async function renderBox(dir, decoder, time = PLAYHEAD) {
    );
    // The producer's own browser would fetch GSAP from the CDN; a stalled fetch renders the 0 s frame.
    await serveFixtureAssetsLocally(session.page);
+    // Without the opt-out, a project with no timeline waits out playerReadyTimeout on every render.
+    if (noTimeline(dir)) await session.page.evaluateOnNewDocument(markNoTimeline);
    await initializeSession(session);
    const { buffer } = await captureFrameToBuffer(session, 0, time);
    // Diagnostic only: a DOM rect ignores clip-path, so it cannot score a crop.
@@ -1,4 +1,4 @@
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -18,7 +18,9 @@ const chromeUrlPattern = (pattern) =>
 const sent = [];
 const handlers = {};
 const sessionArgs = [];
+const onNewDocument = [];
 const page = {
+  evaluateOnNewDocument: async (fn) => void onNewDocument.push(fn),
  createCDPSession: async () => ({
    on: (event, handler) => (handlers[event] = handler),
    send: async (method, params) => void sent.push({ method, params }),
@@ -36,16 +38,25 @@ vi.mock("../../../../producer/src/index.js", () => ({
  closeCaptureSession: async () => undefined,
 }));

+const decoder = { evaluate: async () => ({ left: 0, right: 1, top: 0, bottom: 1, area: 1 }) };
+
+async function render(html) {
+  const { renderBox } = await import("./render.mjs");
+  const dir = mkdtempSync(join(tmpdir(), "edit-bench-render-"));
+  if (html) writeFileSync(join(dir, "index.html"), html);
+  onNewDocument.length = 0;
+  try {
+    await renderBox(dir, decoder);
+  } finally {
+    rmSync(dir, { recursive: true });
+  }
+  return onNewDocument.map((fn) => fn.name);
+}
+
 describe("renderBox", () => {
  it("serves the fixture's GSAP from the repo and opens no second page that would fetch it", async () => {
-    const { renderBox } = await import("./render.mjs");
-    const dir = mkdtempSync(join(tmpdir(), "edit-bench-render-"));
-    const decoder = { evaluate: async () => ({ left: 0, right: 1, top: 0, bottom: 1, area: 1 }) };
-    try {
-      await renderBox(dir, decoder);
-    } finally {
-      rmSync(dir, { recursive: true });
-    }
+    sent.length = 0;
+    await render();
    expect(sent.map((s) => s.method)).toEqual(["Fetch.enable", "Fetch.fulfillRequest"]);
    const { patterns } = sent[0].params;
    expect(patterns).toEqual([{ urlPattern: patterns[0].urlPattern }]);
@@ -54,4 +65,9 @@ describe("renderBox", () => {
    expect(sent[1].params.body).toBe(gsap);
    expect(sessionArgs[0][4]).toMatchObject({ staticFrameDedup: false });
  });
+
+  it("opts a project with no timeline out of the engine's timeline wait, and only such a project", async () => {
+    expect(await render("<div data-composition-id='main'></div>")).toEqual(["markNoTimeline"]);
+    expect(await render("<script>window.__timelines = {};</script>")).toEqual([]);
+  });
 });
@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { smoothness } from "./case.mjs";
+import { controlDrag, smoothness } from "./case.mjs";

 const thread = { pid: 1, tid: 2 };
 const mark = { name: "edit-bench-end", cat: "blink.user_timing", ts: 1_000_000, ...thread };
@@ -26,3 +26,18 @@ describe("smoothness work per frame", () => {
    expect(s.unknown).toBe("no end mark in the trace");
  });
 });
+
+describe("controlDrag", () => {
+  it("measures again after a failed control instead of failing every later case", async () => {
+    let calls = 0;
+    const browser = {
+      createBrowserContext: async () => {
+        calls += 1;
+        throw new Error(`blank page ${calls} failed`);
+      },
+    };
+    await expect(controlDrag(browser, "move")).rejects.toThrow("blank page 1 failed");
+    await expect(controlDrag(browser, "move")).rejects.toThrow("blank page 2 failed");
+    expect(calls).toBe(2);
+  });
+});