test(studio): record each edit-accuracy shard's file writes and reloads when HF_WRITE_TRACE is set (#6532)

* test(studio): record each edit-accuracy shard's file writes and reloads when HF_WRITE_TRACE is set

* style(studio): format the write trace

* fix(core): hash the write trace with sha256

* style(studio): wrap the trace hash helper

* style(studio): format the trace hash helper

* fix(studio): turn the reload log on in the Studio page the trace watches

packages/core/src/atomicFile.test.ts

@@ -510,3 +510,31 @@ describe.skipIf(process.platform === "win32")("resolveWritePath", () => {
     );
   });
 });
+
+describe("HF_WRITE_TRACE", () => {
+  afterEach(() => vi.unstubAllEnvs());
+
+  it("records one line per replaced .html when it names a file, and nothing when unset", () => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-write-trace-"));
+    const trace = join(dir, "trace.log");
+    try {
+      replaceFileAtomically(join(dir, "a.html"), "<p>one</p>");
+      expect(fs.existsSync(trace)).toBe(false);
+
+      vi.stubEnv("HF_WRITE_TRACE", trace);
+      replaceFileAtomically(join(dir, "a.html"), "<p>two</p>");
+      replaceFileAtomically(join(dir, "data.json"), "{}");
+
+      const lines = readFileSync(trace, "utf8").trim().split("\n");
+      expect(lines).toHaveLength(1);
+      const line = JSON.parse(lines[0]!);
+      expect(line).toMatchObject({ kind: "write", file: "a.html", size: 10 });
+      expect(line.hash).toBe(
+        crypto.createHash("sha256").update("<p>two</p>").digest("hex").slice(0, 8),
+      );
+      expect(line.stack[0]).toContain("replaceFileAtomically");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});

packages/core/src/atomicFile.ts

@@ -1,5 +1,5 @@
 import * as fs from "node:fs";
-import { randomBytes } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
 import { realpath } from "./safePath.js";
 
@@ -25,6 +25,23 @@ export function isAtomicTempPath(path: string): boolean {
   return TEMP_SUFFIX.test(path);
 }
 
+/** With HF_WRITE_TRACE naming a file, one JSON line per replaced .html: its hash, size and caller. */
+function traceWrite(filePath: string, content: string | Uint8Array): void {
+  const out = process.env.HF_WRITE_TRACE;
+  if (!out || !filePath.endsWith(".html")) return;
+  const text = typeof content === "string" ? content : Buffer.from(content).toString("utf8");
+  const hash = createHash("sha256").update(text).digest("hex").slice(0, 8);
+  const stack = (new Error().stack ?? "")
+    .split("\n")
+    .slice(2, 7)
+    .map((line) => line.trim());
+  const file = basename(filePath);
+  fs.appendFileSync(
+    out,
+    `${JSON.stringify({ t: Date.now(), kind: "write", file, hash, size: text.length, stack })}\n`,
+  );
+}
+
 /** Replace a file only after the complete sibling temp file is written. No mode: the default one. */
 export function replaceFileAtomically(
   filePath: string,
@@ -32,6 +49,7 @@ export function replaceFileAtomically(
   mode?: number,
   operations: SiblingFileSystem & Pick<typeof fs, "renameSync"> = fs,
 ): void {
+  traceWrite(filePath, content);
   // Node fs.rename uses libuv uv_fs_rename; win32 calls MoveFileExW with MOVEFILE_REPLACE_EXISTING.
   publishSibling(filePath, content, mode, operations, (tempPath) =>
     renameWithRetry(operations, tempPath, filePath),

packages/studio/tests/e2e/edit-accuracy/case.mjs

@@ -5,6 +5,7 @@ import { parseGsapScript } from "../../../../parsers/src/gsapParser.ts";
 import { readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { COMPOSITION, FIXTURE_CDN, PLAYHEAD, localAsset } from "./grid.mjs";
+import { traceBench } from "./trace.mjs";
 import {
   angleOf,
   centre,
@@ -886,9 +887,11 @@ export async function pointerGesture(ctx, gesture, pre, route) {
   await ctx.page.mouse.move(press[0], press[1]);
   await startFrames(ctx.page, ctx.selector ?? "#target");
   await nextFrame(ctx.page, 2);
+  traceBench("press", { gesture });
   await ctx.page.mouse.down();
   await nextFrame(ctx.page);
   const s0 = await sample(ctx, gesture, g.point, press);
+  traceBench("pressed", { gesture, jump: pre && quadDistance(s0.m.visible, pre.visible) });
   await recording(ctx.page, true);
   const errors = [];
   let last = s0;
@@ -908,6 +911,7 @@ export async function pointerGesture(ctx, gesture, pre, route) {
   }
   const rec = await recording(ctx.page, false);
   const smooth = smoothness(rec);
+  traceBench("release", { gesture, worst: Math.max(...errors) });
   await ctx.page.mouse.up();
   const lastQuad = gesture === "crop" ? last.m.outline : last.m.visible;
   return {
@@ -993,11 +997,18 @@ export async function inStudio({ browser, spec, dir, files, url, evidence }, dri
   };
   const consoleErrors = [];
   page.on("pageerror", (e) => consoleErrors.push(e.message));
+  page.on(
+    "console",
+    (m) =>
+      m.text().startsWith("[hf-reload]") && traceBench("reload", { line: m.text().slice(0, 400) }),
+  );
   evidence.shots = {};
   const shoot = async (name) =>
     (evidence.shots[name] = await page.screenshot({ type: "jpeg", quality: 70 }));
   try {
     await page.setViewport(VIEWPORT);
+    if (process.env.HF_WRITE_TRACE)
+      await page.evaluateOnNewDocument(() => localStorage.setItem("hf-reload-debug", "1"));
     await page.evaluateOnNewDocument(installWebMcpHost, "__editBench");
     await page.evaluateOnNewDocument(instrumentPage);
     if (spec.slowSaves) await page.evaluateOnNewDocument(slowSaves, spec.slowSaves);

packages/studio/tests/e2e/edit-accuracy/sequences.mjs

@@ -2,6 +2,7 @@
 import { watch } from "node:fs";
 import { dirname, join } from "node:path";
 import { COMPOSITION, PLAYHEAD } from "./grid.mjs";
+import { hashOf, traceBench } from "./trace.mjs";
 import { centre, dist, percentile, quadDistance } from "./geometry.mjs";
 import {
   VIEWPORT,
@@ -109,7 +110,10 @@ export function watchVersions(dir, files) {
   const versions = [readFiles(dir, files)];
   const record = () => {
     const now = readFiles(dir, files);
-    if (!sameFiles(now, versions.at(-1))) versions.push(now);
+    if (!sameFiles(now, versions.at(-1))) {
+      versions.push(now);
+      traceBench("version", { n: versions.length - 1, hash: hashOf(JSON.stringify(now)) });
+    }
   };
   const watchers = [...new Set(files.map((f) => dirname(join(dir, f))))].map((d) =>
     watch(d, record),

packages/studio/tests/e2e/edit-accuracy/trace.mjs

@@ -0,0 +1,12 @@
+// Bench-side lines for the write trace (HF_WRITE_TRACE), in the same file as the server's.
+import { appendFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+
+export const traceBench = (kind, data = {}) => {
+  if (process.env.HF_WRITE_TRACE)
+    appendFileSync(
+      process.env.HF_WRITE_TRACE,
+      JSON.stringify({ t: Date.now(), kind, ...data }) + "\n",
+    );
+};
+export const hashOf = (text) => createHash("sha256").update(String(text)).digest("hex").slice(0, 8);

scripts/ci/edit-accuracy-shard.sh

@@ -2,6 +2,8 @@
 # edit-accuracy-shard.sh <i/n>: runs one edit accuracy shard into /tmp/edit-accuracy, then re-runs twice each case
 # whose verdict differs from the base branch (/tmp/base-edit-accuracy.json). A slow runner makes the bench exit 75.
 set -euo pipefail
+export HF_WRITE_TRACE="/tmp/edit-accuracy/write-trace-${1//\//-}.log"
+mkdir -p /tmp/edit-accuracy
 bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
 bench --shard "$1" --out /tmp/edit-accuracy/run1
 mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \