From 58552ede4bfff5366fa535551845d20cda3f2875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 27 Oct 2026 11:50:20 +0000 Subject: [PATCH] test(studio): record each edit-accuracy shard's file writes and reloads when HF_WRITE_TRACE is set (#6532) * test(studio): record each edit-accuracy shard's file writes and reloads when HF_WRITE_TRACE is set * style(studio): format the write trace * fix(core): hash the write trace with sha256 * style(studio): wrap the trace hash helper * style(studio): format the trace hash helper * fix(studio): turn the reload log on in the Studio page the trace watches --- packages/core/src/atomicFile.test.ts | 28 +++++++++++++++++++ packages/core/src/atomicFile.ts | 20 ++++++++++++- .../studio/tests/e2e/edit-accuracy/case.mjs | 11 ++++++++ .../tests/e2e/edit-accuracy/sequences.mjs | 6 +++- .../studio/tests/e2e/edit-accuracy/trace.mjs | 12 ++++++++ scripts/ci/edit-accuracy-shard.sh | 2 ++ 6 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 packages/studio/tests/e2e/edit-accuracy/trace.mjs diff --git a/packages/core/src/atomicFile.test.ts b/packages/core/src/atomicFile.test.ts index 3e419d61f1..e5b26c0a8e 100644 --- a/packages/core/src/atomicFile.test.ts +++ b/packages/core/src/atomicFile.test.ts @@ -510,3 +510,31 @@ describe.skipIf(process.platform === "win32")("resolveWritePath", () => { ); }); }); + +describe("HF_WRITE_TRACE", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("records one line per replaced .html when it names a file, and nothing when unset", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-write-trace-")); + const trace = join(dir, "trace.log"); + try { + replaceFileAtomically(join(dir, "a.html"), "

one

"); + expect(fs.existsSync(trace)).toBe(false); + + vi.stubEnv("HF_WRITE_TRACE", trace); + replaceFileAtomically(join(dir, "a.html"), "

two

"); + replaceFileAtomically(join(dir, "data.json"), "{}"); + + const lines = readFileSync(trace, "utf8").trim().split("\n"); + expect(lines).toHaveLength(1); + const line = JSON.parse(lines[0]!); + expect(line).toMatchObject({ kind: "write", file: "a.html", size: 10 }); + expect(line.hash).toBe( + crypto.createHash("sha256").update("

two

").digest("hex").slice(0, 8), + ); + expect(line.stack[0]).toContain("replaceFileAtomically"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/core/src/atomicFile.ts b/packages/core/src/atomicFile.ts index 3366e3cc21..0abb733f49 100644 --- a/packages/core/src/atomicFile.ts +++ b/packages/core/src/atomicFile.ts @@ -1,5 +1,5 @@ import * as fs from "node:fs"; -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path"; import { realpath } from "./safePath.js"; @@ -25,6 +25,23 @@ export function isAtomicTempPath(path: string): boolean { return TEMP_SUFFIX.test(path); } +/** With HF_WRITE_TRACE naming a file, one JSON line per replaced .html: its hash, size and caller. */ +function traceWrite(filePath: string, content: string | Uint8Array): void { + const out = process.env.HF_WRITE_TRACE; + if (!out || !filePath.endsWith(".html")) return; + const text = typeof content === "string" ? content : Buffer.from(content).toString("utf8"); + const hash = createHash("sha256").update(text).digest("hex").slice(0, 8); + const stack = (new Error().stack ?? "") + .split("\n") + .slice(2, 7) + .map((line) => line.trim()); + const file = basename(filePath); + fs.appendFileSync( + out, + `${JSON.stringify({ t: Date.now(), kind: "write", file, hash, size: text.length, stack })}\n`, + ); +} + /** Replace a file only after the complete sibling temp file is written. No mode: the default one. */ export function replaceFileAtomically( filePath: string, @@ -32,6 +49,7 @@ export function replaceFileAtomically( mode?: number, operations: SiblingFileSystem & Pick = fs, ): void { + traceWrite(filePath, content); // Node fs.rename uses libuv uv_fs_rename; win32 calls MoveFileExW with MOVEFILE_REPLACE_EXISTING. publishSibling(filePath, content, mode, operations, (tempPath) => renameWithRetry(operations, tempPath, filePath), diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs index 0f91e62350..e0a5f67b8a 100644 --- a/packages/studio/tests/e2e/edit-accuracy/case.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs @@ -5,6 +5,7 @@ import { parseGsapScript } from "../../../../parsers/src/gsapParser.ts"; import { readFileSync, statSync } from "node:fs"; import { join } from "node:path"; import { COMPOSITION, FIXTURE_CDN, PLAYHEAD, localAsset } from "./grid.mjs"; +import { traceBench } from "./trace.mjs"; import { angleOf, centre, @@ -886,9 +887,11 @@ export async function pointerGesture(ctx, gesture, pre, route) { await ctx.page.mouse.move(press[0], press[1]); await startFrames(ctx.page, ctx.selector ?? "#target"); await nextFrame(ctx.page, 2); + traceBench("press", { gesture }); await ctx.page.mouse.down(); await nextFrame(ctx.page); const s0 = await sample(ctx, gesture, g.point, press); + traceBench("pressed", { gesture, jump: pre && quadDistance(s0.m.visible, pre.visible) }); await recording(ctx.page, true); const errors = []; let last = s0; @@ -908,6 +911,7 @@ export async function pointerGesture(ctx, gesture, pre, route) { } const rec = await recording(ctx.page, false); const smooth = smoothness(rec); + traceBench("release", { gesture, worst: Math.max(...errors) }); await ctx.page.mouse.up(); const lastQuad = gesture === "crop" ? last.m.outline : last.m.visible; return { @@ -993,11 +997,18 @@ export async function inStudio({ browser, spec, dir, files, url, evidence }, dri }; const consoleErrors = []; page.on("pageerror", (e) => consoleErrors.push(e.message)); + page.on( + "console", + (m) => + m.text().startsWith("[hf-reload]") && traceBench("reload", { line: m.text().slice(0, 400) }), + ); evidence.shots = {}; const shoot = async (name) => (evidence.shots[name] = await page.screenshot({ type: "jpeg", quality: 70 })); try { await page.setViewport(VIEWPORT); + if (process.env.HF_WRITE_TRACE) + await page.evaluateOnNewDocument(() => localStorage.setItem("hf-reload-debug", "1")); await page.evaluateOnNewDocument(installWebMcpHost, "__editBench"); await page.evaluateOnNewDocument(instrumentPage); if (spec.slowSaves) await page.evaluateOnNewDocument(slowSaves, spec.slowSaves); diff --git a/packages/studio/tests/e2e/edit-accuracy/sequences.mjs b/packages/studio/tests/e2e/edit-accuracy/sequences.mjs index 4f20abb06d..bf721f16be 100644 --- a/packages/studio/tests/e2e/edit-accuracy/sequences.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/sequences.mjs @@ -2,6 +2,7 @@ import { watch } from "node:fs"; import { dirname, join } from "node:path"; import { COMPOSITION, PLAYHEAD } from "./grid.mjs"; +import { hashOf, traceBench } from "./trace.mjs"; import { centre, dist, percentile, quadDistance } from "./geometry.mjs"; import { VIEWPORT, @@ -109,7 +110,10 @@ export function watchVersions(dir, files) { const versions = [readFiles(dir, files)]; const record = () => { const now = readFiles(dir, files); - if (!sameFiles(now, versions.at(-1))) versions.push(now); + if (!sameFiles(now, versions.at(-1))) { + versions.push(now); + traceBench("version", { n: versions.length - 1, hash: hashOf(JSON.stringify(now)) }); + } }; const watchers = [...new Set(files.map((f) => dirname(join(dir, f))))].map((d) => watch(d, record), diff --git a/packages/studio/tests/e2e/edit-accuracy/trace.mjs b/packages/studio/tests/e2e/edit-accuracy/trace.mjs new file mode 100644 index 0000000000..d9e79f5fbe --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/trace.mjs @@ -0,0 +1,12 @@ +// Bench-side lines for the write trace (HF_WRITE_TRACE), in the same file as the server's. +import { appendFileSync } from "node:fs"; +import { createHash } from "node:crypto"; + +export const traceBench = (kind, data = {}) => { + if (process.env.HF_WRITE_TRACE) + appendFileSync( + process.env.HF_WRITE_TRACE, + JSON.stringify({ t: Date.now(), kind, ...data }) + "\n", + ); +}; +export const hashOf = (text) => createHash("sha256").update(String(text)).digest("hex").slice(0, 8); diff --git a/scripts/ci/edit-accuracy-shard.sh b/scripts/ci/edit-accuracy-shard.sh index a2c8fdb74a..5ba17d7e47 100755 --- a/scripts/ci/edit-accuracy-shard.sh +++ b/scripts/ci/edit-accuracy-shard.sh @@ -2,6 +2,8 @@ # edit-accuracy-shard.sh : runs one edit accuracy shard into /tmp/edit-accuracy, then re-runs twice each case # whose verdict differs from the base branch (/tmp/base-edit-accuracy.json). A slow runner makes the bench exit 75. set -euo pipefail +export HF_WRITE_TRACE="/tmp/edit-accuracy/write-trace-${1//\//-}.log" +mkdir -p /tmp/edit-accuracy bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; } bench --shard "$1" --out /tmp/edit-accuracy/run1 mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \