test(studio): measure render drift against a producer frame in the edit accuracy bench (#4764)

* test(studio): measure render drift against a producer frame in the edit accuracy bench

Finds the target's box in a producer-captured frame by its flat colour, with sub-pixel edges from area moments, and compares it with the reloaded preview. The baseline gains the render column from the same full run on main.

* test(studio): mark the pixel box scan as intentionally complex for fallow

* test(studio): regenerate the edit accuracy baseline with render drift on main

Same full run on main 14e87005d as #4760's baseline, with the render column: render passes 618/660, 70/660 pass, 314/660 pass everything but smoothness.

* test(studio): fail only render drift when the producer cannot render a case

A producer capture failure used to turn the case into a harness error that wiped every accuracy metric. It now sets render to null with renderError, fails render alone, and is counted in the table.

* test(studio): shorten the render drift box comment to two lines

packages/studio/tests/e2e/edit-accuracy/case.mjs

@@ -688,6 +688,7 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
       undoTimeout: saved && !undo.reached ? "undo" : saved && !redo.reached ? "redo" : null,
       smooth: { ...drive.smooth, control },
       unsettled: Object.keys(quads).filter((k) => quads[k].unsettled),
+      reloaded,
       diag: {
         ...drive.diag,
         consoleErrors: consoleErrors.slice(0, 5),

packages/studio/tests/e2e/edit-accuracy/geometry.mjs

@@ -53,6 +53,20 @@ export const quadDistance = (a, b) => Math.max(...a.map((p, i) => dist(p, b[i]))
 export const angleOf = (q) => Math.atan2(q[1][1] - q[0][1], q[1][0] - q[0][0]);
 export const normalizeAngle = (a) => Math.atan2(Math.sin(a), Math.cos(a));
 
+export function aabb(q) {
+  const xs = q.map((p) => p[0]);
+  const ys = q.map((p) => p[1]);
+  return {
+    left: Math.min(...xs),
+    top: Math.min(...ys),
+    right: Math.max(...xs),
+    bottom: Math.max(...ys),
+  };
+}
+
+export const boxDistance = (a, b) =>
+  Math.max(...["left", "top", "right", "bottom"].map((k) => Math.abs(a[k] - b[k])));
+
 /** `inset(t r b l [round ...])` in px, as getComputedStyle reports it; none means no crop. */
 export function parseInset(clipPath) {
   const zero = { top: 0, right: 0, bottom: 0, left: 0 };

packages/studio/tests/e2e/edit-accuracy/grid.mjs

@@ -6,8 +6,8 @@ import { join } from "node:path";
 export const COMPOSITION = { width: 1920, height: 1080 };
 /** Frame-aligned at 30 fps, inside every tween, so preview and producer sample the same instant. */
 export const PLAYHEAD = 1;
-const TARGET = { width: 240, height: 160, color: "#f0c020" };
-const BACKGROUND = "#202020";
+export const TARGET = { width: 240, height: 160, color: "#f0c020" };
+export const BACKGROUND = "#202020";
 const NESTED_HOST = { left: 160, top: 90, width: 1600, height: 900 };
 const GSAP_CDN = "https://cdn.jsdelivr.net/npm/gsap@3.14.2/dist/gsap.min.js";
 

packages/studio/tests/e2e/edit-accuracy/render.mjs

@@ -0,0 +1,105 @@
+/** Render drift: the target's box in a frame the producer captures, found by its flat colour. */
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+// Source import, as scripts/preview-capture.ts does: bun runs the producer's TypeScript directly.
+import {
+  captureFrameToBuffer,
+  closeCaptureSession,
+  createCaptureSession,
+  createFileServer,
+  initializeSession,
+} from "../../../../producer/src/index.js";
+import { BACKGROUND, COMPOSITION, PLAYHEAD, TARGET } from "./grid.mjs";
+
+const FPS = { num: 30, den: 1 };
+
+// Sub-pixel box from luminance coverage between the two fixture colours (JPEG keeps full-res luminance).
+// Area moments give the centre and sides, so rotated and cropped rectangles measure the same way.
+function pixelBox(b64, bgY, fgY) {
+  // fallow-ignore-next-line complexity
+  return (async () => {
+    const img = new Image();
+    img.src = `data:image/jpeg;base64,${b64}`;
+    await img.decode();
+    const [w, h] = [img.width, img.height];
+    const g = new OffscreenCanvas(w, h).getContext("2d");
+    g.drawImage(img, 0, 0);
+    const d = g.getImageData(0, 0, w, h).data;
+    let [a, sx, sy, sxx, syy, sxy] = [0, 0, 0, 0, 0, 0];
+    for (let y = 0; y < h; y++)
+      for (let x = 0; x < w; x++) {
+        const i = (y * w + x) * 4;
+        let k = (0.299 * d[i] + 0.587 * d[i + 1] + 0.114 * d[i + 2] - bgY) / (fgY - bgY);
+        if (k < 0.02) continue;
+        k = k > 0.98 ? 1 : k;
+        const [px, py] = [x + 0.5, y + 0.5];
+        a += k;
+        sx += k * px;
+        sy += k * py;
+        sxx += k * px * px;
+        syy += k * py * py;
+        sxy += k * px * py;
+      }
+    if (a === 0) return null;
+    const [mx, my] = [sx / a, sy / a];
+    // Pixel coverage is the shape blurred by a 1 px box, which adds 1/12 px² of variance per axis.
+    const cxx = sxx / a - mx * mx - 1 / 12;
+    const cyy = syy / a - my * my - 1 / 12;
+    const cxy = sxy / a - mx * my;
+    const theta = 0.5 * Math.atan2(2 * cxy, cxx - cyy);
+    const spread = Math.hypot((cxx - cyy) / 2, cxy);
+    const side1 = Math.sqrt(12 * ((cxx + cyy) / 2 + spread));
+    const side2 = Math.sqrt(12 * Math.max(0, (cxx + cyy) / 2 - spread));
+    const [c, s] = [Math.abs(Math.cos(theta)), Math.abs(Math.sin(theta))];
+    const hx = (side1 * c + side2 * s) / 2;
+    const hy = (side1 * s + side2 * c) / 2;
+    return { left: mx - hx, right: mx + hx, top: my - hy, bottom: my + hy, area: a };
+  })();
+}
+
+const luminance = (hex) => {
+  const [r, g, b] = [1, 3, 5].map((i) => Number.parseInt(hex.slice(i, i + 2), 16));
+  return 0.299 * r + 0.587 * g + 0.114 * b;
+};
+
+/** Renders `dir` at the playhead and measures the target's box in the frame. */
+export async function renderBox(dir, decoder) {
+  const framesDir = join(dir, ".bench-frames");
+  mkdirSync(framesDir, { recursive: true });
+  const server = await createFileServer({ projectDir: dir, port: 0, fps: FPS });
+  let session;
+  try {
+    session = await createCaptureSession(
+      server.url,
+      framesDir,
+      {
+        width: COMPOSITION.width,
+        height: COMPOSITION.height,
+        fps: FPS,
+        format: "jpeg",
+        quality: 100,
+      },
+      null,
+      // A composition with no GSAP never registers a timeline; the default 45 s wait for one changes no pixel.
+      { playerReadyTimeout: 10_000 },
+    );
+    await initializeSession(session);
+    const { buffer } = await captureFrameToBuffer(session, 0, PLAYHEAD);
+    // Diagnostic only: a DOM rect ignores clip-path, so it cannot score a crop.
+    const domRect = await session.page.evaluate(() => {
+      const r = document.querySelector("#target")?.getBoundingClientRect();
+      return r ? { left: r.left, top: r.top, right: r.right, bottom: r.bottom } : null;
+    });
+    const box = await decoder.evaluate(
+      pixelBox,
+      buffer.toString("base64"),
+      luminance(BACKGROUND),
+      luminance(TARGET.color),
+    );
+    if (!box) throw new Error("target colour not found in the producer frame");
+    return { box, domRect, jpeg: buffer };
+  } finally {
+    server.close();
+    if (session) await closeCaptureSession(session).catch(() => undefined);
+  }
+}

packages/studio/tests/e2e/edit-accuracy/report.mjs

@@ -7,7 +7,7 @@ const LIMIT_PX = 0.5;
 // A frame over 1.5 vsyncs is dropped; raw rAF p95 stays reported so a different rule re-scores without a re-run.
 const DROPPED_FRAME_MS = 25;
 const WORK_MS = 8;
-export const METRICS = ["tracking", "press", "drop", "reload", "undo", "smooth"];
+export const METRICS = ["tracking", "press", "drop", "reload", "render", "undo", "smooth"];
 
 /** Worst-first value per metric; undo ranks by box distance, and its byte failures are counted apart. */
 const worstValue = {
@@ -15,6 +15,7 @@ const worstValue = {
   press: (r) => r.pressJump ?? 0,
   drop: (r) => r.drop,
   reload: (r) => r.reload,
+  render: (r) => r.render ?? 0,
   undo: (r) => Math.max(r.undo.box, r.undo.redoBox ?? 0),
   smooth: (r) => r.smooth.dropped - r.smooth.control.dropped,
 };
@@ -32,7 +33,7 @@ const FED_BY = {
   committed: ["drop", "reload", "undo"],
   undone: ["undo"],
   redone: ["undo"],
-  reloaded: ["reload"],
+  reloaded: ["reload", "render"],
 };
 const unsettledMetrics = (r) => new Set(r.unsettled.flatMap((k) => FED_BY[k]));
 
@@ -51,6 +52,7 @@ export function score(spec, r) {
     press: r.pressJump === null || r.pressJump <= LIMIT_PX,
     drop: r.drop <= LIMIT_PX,
     reload: r.reload <= LIMIT_PX,
+    render: r.render !== null && r.render <= LIMIT_PX,
     undo: r.undo.bytes && r.undo.redoBytes && Math.max(r.undo.box, r.undo.redoBox) <= LIMIT_PX,
     // Only drops beyond the blank page's, driven the same way in the same Chrome, are the edit's.
     smooth:
@@ -114,6 +116,7 @@ function summarize(results, seconds) {
     perMetric,
     unsettled: measured.filter((r) => r.unsettled.length).length,
     undoTimeouts: measured.filter((r) => r.undoTimeout).length,
+    renderErrors: measured.filter((r) => r.renderError).length,
     smooth: smoothSummary(measured),
     seconds: Math.round(seconds),
   };
@@ -130,11 +133,12 @@ function table(summary, meta, results) {
     `Every metric counts except smoothness, which is reported against the blank-page control: ${summary.perMetric.find((m) => m.metric === "smooth").pass}/${summary.total} pass it, and ${summary.passing}/${summary.total} pass everything including it.`,
     "",
     `Studio ${meta.studio} (build ${meta.build}), bench ${meta.bench}, grid \`${meta.grid}\`, ${meta.date}, ${summary.seconds}s with ${meta.jobs} jobs, ${summary.errors} harness errors, load ${meta.load}.`,
-    `Pass: tracking, press jump, drop and reload ≤ ${LIMIT_PX} px; undo and redo byte-identical with the box ≤ ${LIMIT_PX} px; no more frames over ${DROPPED_FRAME_MS} ms than the blank-page control, and main-thread work ≤ ${WORK_MS} ms per frame at p95.`,
+    `Pass: tracking, press jump, drop, reload and render ≤ ${LIMIT_PX} px; undo and redo byte-identical with the box ≤ ${LIMIT_PX} px; no more frames over ${DROPPED_FRAME_MS} ms than the blank-page control, and main-thread work ≤ ${WORK_MS} ms per frame at p95.`,
     "",
     `Undo or redo left different bytes in ${summary.bytesDiffer.undo} undo and ${summary.bytesDiffer.redo} redo cases.`,
     `The preview never held still for 1 s within 15 s in ${summary.unsettled} cases; the metrics that snapshot feeds fail.`,
     `An undo or redo write never landed within 15 s in ${summary.undoTimeouts} cases; undo fails there.`,
+    `The producer failed to render ${summary.renderErrors} cases; render fails there.`,
     `Smoothness: ${summary.smooth.unknown} cases with unknown work; dropped frames per case (median/max) ${summary.smooth.dropped}, blank-page control ${summary.smooth.control}; raw rAF p95 (median/max) ${summary.smooth.p95} ms, control ${summary.smooth.controlP95} ms.`,
     "",
     "| Metric | Pass | Worst | Worst case |",
@@ -157,6 +161,7 @@ function table(summary, meta, results) {
 function baseline(meta, results) {
   const entries = [...results]
     .sort((a, b) => a.id.localeCompare(b.id))
+    // fallow-ignore-next-line complexity
     .map((r) => {
       const v = r.error
         ? { pass: false, error: true }
@@ -166,6 +171,7 @@ function baseline(meta, results) {
             pressJump: roundUp(r.pressJump),
             drop: roundUp(r.drop),
             reload: roundUp(r.reload),
+            render: roundUp(r.render),
             undo: r.checks.undo,
             dropped: r.smooth.dropped,
             controlDropped: r.smooth.control.dropped,
@@ -173,6 +179,7 @@ function baseline(meta, results) {
             frameP95: roundUp(r.smooth.p95),
             ...(r.unsettled.length && { unsettled: r.unsettled }),
             ...(r.undoTimeout && { undoTimeout: r.undoTimeout }),
+            ...(r.renderError && { renderError: true }),
           };
       return `    ${JSON.stringify(r.id)}: ${JSON.stringify(v)}`;
     });

packages/studio/tests/e2e/edit-accuracy/run.mjs

@@ -13,6 +13,8 @@ import { resolveHeadlessShellPath } from "../../../../engine/src/index.ts";
 import { buildGrid, writeFixture } from "./grid.mjs";
 import { killServers, runCase, startServer, stopServer } from "./case.mjs";
 import { METRICS, score, writeReport } from "./report.mjs";
+import { renderBox } from "./render.mjs";
+import { aabb, boxDistance } from "./geometry.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const REPO = resolve(HERE, "../../../../..");
@@ -47,6 +49,8 @@ function saveEvidence(id, evidence) {
     writeFileSync(join(caseDir, `${name}.jpg`), jpeg);
   for (const [name, text] of Object.entries(evidence.files ?? {}))
     writeFileSync(join(caseDir, `saved-${name.replace("/", "-")}`), text);
+  if (evidence.renderFailed && evidence.frame)
+    writeFileSync(join(caseDir, "producer.jpg"), evidence.frame);
 }
 
 function verdict(r) {
@@ -62,7 +66,31 @@ const errorResult = (error, log) => ({
 
 const liveRoots = new Set();
 
-async function runOne(spec, browser, port) {
+/** Render drift: the reloaded preview's visible box against the target's pixel box in a producer frame. */
+async function withRender(dir, decoder, { reloaded, ...measured }, evidence) {
+  const expected = aabb(reloaded.visible);
+  const render = await renderBox(dir, decoder).catch((error) => ({ error }));
+  // A producer failure fails render alone; the case's other metrics still count.
+  if (render.error)
+    return {
+      ...measured,
+      render: null,
+      renderError: `${render.error?.message ?? render.error}`.slice(0, 600),
+    };
+  evidence.frame = render.jpeg;
+  return {
+    ...measured,
+    render: boxDistance(render.box, expected),
+    diag: {
+      ...measured.diag,
+      previewBox: expected,
+      pixelBox: render.box,
+      producerDomRect: render.domRect,
+    },
+  };
+}
+
+async function runOne(spec, browser, decoder, port) {
   const started = Date.now();
   const root = mkdtempSync(join(tmpdir(), "hf-edit-accuracy-"));
   liveRoots.add(root);
@@ -74,7 +102,7 @@ async function runOne(spec, browser, port) {
   let server;
   try {
     server = await startServer(opt.cli, dir, port, log, join(root, "home"));
-    result = await runCase({
+    const measured = await runCase({
       browser,
       spec,
       dir,
@@ -82,13 +110,16 @@ async function runOne(spec, browser, port) {
       url: `http://127.0.0.1:${port}/#project/case`,
       evidence,
     });
+    await stopServer(server);
+    server = null;
+    result = await withRender(dir, decoder, measured, evidence);
   } catch (error) {
     result = errorResult(error, log);
   } finally {
     if (server) await stopServer(server);
   }
   const scored = { ...score(spec, result), seconds: (Date.now() - started) / 1000 };
-  if (!scored.pass) saveEvidence(spec.id, evidence);
+  if (!scored.pass) saveEvidence(spec.id, { ...evidence, renderFailed: !scored.checks.render });
   rmSync(root, { recursive: true, force: true });
   liveRoots.delete(root);
   console.log(verdict(scored));
@@ -111,7 +142,7 @@ function acquireLock(path) {
 const LOCK_CHUNK = 8;
 
 // fallow-ignore-next-line complexity
-async function runChunks(queue, browsers, results) {
+async function runChunks(queue, browsers, decoders, results) {
   for (let start = 0; start < queue.length; start += opt.lock ? LOCK_CHUNK : queue.length) {
     const chunk = queue.slice(start, opt.lock ? start + LOCK_CHUNK : queue.length);
     const release = opt.lock ? await acquireLock(opt.lock) : () => undefined;
@@ -119,7 +150,7 @@ async function runChunks(queue, browsers, results) {
       await Promise.all(
         browsers.map(async (browser, i) => {
           for (let spec = chunk.shift(); spec; spec = chunk.shift())
-            results.push(await runOne(spec, browser, Number(opt.port) + i));
+            results.push(await runOne(spec, browser, decoders[i], Number(opt.port) + i));
         }),
       );
     } finally {
@@ -174,7 +205,8 @@ const browsers = await Promise.all(
   ),
 );
 try {
-  await runChunks([...cases], browsers, results);
+  const decoders = await Promise.all(browsers.map((b) => b.newPage()));
+  await runChunks([...cases], browsers, decoders, results);
 } finally {
   await Promise.all(browsers.map((b) => b.close()));
 }