From d36cad79853c62845b15d2e4236a5b0efd716392 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Wed, 4 Nov 2026 17:55:07 +0000 Subject: [PATCH] test(engine): pixel invariants and fixtures that guard default-on capture blur (#7108) * test(engine): pixel invariants for capture blur against blur off Co-Authored-By: Claude Sonnet 5.5 * test(engine): fixtures for every capture blur failure class seen so far Co-Authored-By: Claude Sonnet 5.5 * test(engine): nested canvas fixture reproduces #6975, sharp-tile fixtures have no moving times Co-Authored-By: Claude Sonnet 5.5 * test(engine): capture blur must match blur off outside the trail and the shutter average inside it Co-Authored-By: Claude Sonnet 5.5 * test(engine): differential tests pin known bugs by symptom, history test checks the reference frame Co-Authored-By: Claude Sonnet 5.5 * test(engine): a playing preview keeps changing with blur on Co-Authored-By: Claude Sonnet 5.5 * test(producer): a multi-worker blur render must match blur off outside the trail Renders five blur fixtures through the real producer as a PNG sequence with 4 workers (contiguous blocks, three of them starting mid-film, the #6967 condition): blur off twice, blur on once. Outside the trail of everything that moved, blur on must equal blur off in every frame with two neighbours. The callbacks and scroll-list known differences are pinned by symptom. Import route: a relative import of blurDifferential.ts, fixtures.ts and thresholds.ts from packages/engine/src/services/__fixtures__. The producer tsconfig excludes test files, so I type-checked the file with a temporary tsconfig (rootDir widened); no error. No package subpath was needed. RENDER_OUTSIDE_MAX_MAE: render noise is 0 (blur off, twice, bit-identical), the rewind mutation measured 111.085 when caught, so the 1.0 floor applies. Co-Authored-By: Claude Sonnet 5.5 * test(producer): render callbacks with workers that start between its calls With 4 workers the blocks started at 0, 15, 30, 45, each on a call, so the rewind mutation showed in a render only 3 times in 5. Seven workers over 60 frames start at 0, 9, 18, 27, 36, 45 and 54, two of them strictly between the calls at frames 16 and 30; the mutation then fails callbacks in 8 of 8 runs (worst MAE 97.02), the clean build in none. A new case checks the partition with distributeFrames. The #list and #label rects now come from the fixture's markup, and MEASURED_RENDER_REWIND_MUTATION is 97.02. Co-Authored-By: Claude Sonnet 5.5 * test(engine): blur pins bound the symptom, liveness detector reports blindness, fixtures time out Pinned known bugs now require every bad tile inside the pinned region and a worst MAE within half to double of the measured one. tileTouches, Rect and the band check move into blurDifferential.ts with a dpr 2 unit test. The blank-frame census counts canvases it cannot read and asserts none, and names both readings when no blank frame is seen. No console.info on passing runs. Fixtures cases get a 30 s timeout. Co-Authored-By: Claude Sonnet 5.5 * test(producer): scroll-list render case bounds the race, pins share the engine helpers The scroll-list loss depends on timing per worker block, so the case now requires frames 2..27 clean and any later bad tile inside #list, without requiring the loss. The callbacks pin keeps requiring its symptom, bounds it to the label and to half to double of its measured size, and its message names the entry to delete. Wording says multi-worker, not 4-worker. Co-Authored-By: Claude Sonnet 5.5 * build(engine): keep the blur test helpers out of the published tarball Co-Authored-By: Claude Sonnet 5.5 * test(engine): print the worst outside tile of each arm to explain the CI text diff Co-Authored-By: Claude Sonnet 5.5 * test: both arms draw grayscale text so host subpixel settings are not read as blur Co-Authored-By: Claude Sonnet 5.5 * test(producer): run the multi-worker blur render alone so it cannot skew other Chrome tests Co-Authored-By: Claude Sonnet 5.5 * test: re-pin the producer lane runner after splitting out the heavy blur render Co-Authored-By: Claude Sonnet 5.5 * test(engine): read the blank-frame pin from a wrap observer, not an animation frame poll The pin sampled in requestAnimationFrame, so it could miss the gap between the wrap and the first draw. A MutationObserver now reads the output canvas the moment the root is wrapped, which is deterministic, and every fixture must report a wrapped root. Co-Authored-By: Claude Sonnet 5.5 * test(engine): drop the blurs field no fixture could set to false Co-Authored-By: Claude Sonnet 5.5 * ci: give the flake gate 60 minutes, it budgets runs by file count not time Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 4 +- packages/engine/package.json | 2 + .../__fixtures__/blurDifferential.test.ts | 210 +++++++++++ .../services/__fixtures__/blurDifferential.ts | 241 ++++++++++++ .../__fixtures__/motionBlur/blurPage.ts | 111 ++++++ .../__fixtures__/motionBlur/fixtures.ts | 254 +++++++++++++ .../__fixtures__/motionBlur/thresholds.ts | 46 +++ ...ureMotionBlur.differential.browser.test.ts | 294 +++++++++++++++ ...captureMotionBlur.fixtures.browser.test.ts | 95 +++++ ...captureMotionBlur.liveness.browser.test.ts | 230 ++++++++++++ .../captureMotionBlur.pixels.browser.test.ts | 91 +---- .../producer/scripts/integration-tests.json | 1 + packages/producer/scripts/run-test-lane.mjs | 8 +- .../blurDifferential.integration.test.ts | 350 ++++++++++++++++++ scripts/test-reachability.json | 2 +- 15 files changed, 1860 insertions(+), 79 deletions(-) create mode 100644 packages/engine/src/services/__fixtures__/blurDifferential.test.ts create mode 100644 packages/engine/src/services/__fixtures__/blurDifferential.ts create mode 100644 packages/engine/src/services/__fixtures__/motionBlur/blurPage.ts create mode 100644 packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts create mode 100644 packages/engine/src/services/__fixtures__/motionBlur/thresholds.ts create mode 100644 packages/engine/src/services/captureMotionBlur.differential.browser.test.ts create mode 100644 packages/engine/src/services/captureMotionBlur.fixtures.browser.test.ts create mode 100644 packages/engine/src/services/captureMotionBlur.liveness.browser.test.ts create mode 100644 packages/producer/src/services/render/blurDifferential.integration.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96781a4be7..f5de16d6d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -623,7 +623,9 @@ jobs: name: Flake gate if: github.event_name == 'pull_request' runs-on: ubuntu-latest - timeout-minutes: 30 + # The gate budgets by file count, not elapsed time (flake-gate.mjs runsFor), so a PR that + # changes heavy browser or render tests needs the time: 5 engine blur files x 10 runs take ~26 min. + timeout-minutes: 60 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: diff --git a/packages/engine/package.json b/packages/engine/package.json index 24db4fafce..f568538aa9 100644 --- a/packages/engine/package.json +++ b/packages/engine/package.json @@ -9,6 +9,8 @@ }, "files": [ "dist", + "!dist/services/__fixtures__/blurDifferential.*", + "!dist/services/__fixtures__/motionBlur", "README.md" ], "type": "module", diff --git a/packages/engine/src/services/__fixtures__/blurDifferential.test.ts b/packages/engine/src/services/__fixtures__/blurDifferential.test.ts new file mode 100644 index 0000000000..9e992cfdac --- /dev/null +++ b/packages/engine/src/services/__fixtures__/blurDifferential.test.ts @@ -0,0 +1,210 @@ +import { describe, expect, it } from "vitest"; +import { + compareTiles, + meanFrame, + tileTouches, + tilesOutsideRegion, + trailMask, + withinSymptomBand, + type Rgba, +} from "./blurDifferential.js"; + +const SIZE = 64; + +function frame(gray = 0): Rgba { + const data = new Uint8Array(SIZE * SIZE * 4); + for (let i = 0; i < data.length; i += 4) { + data[i] = gray; + data[i + 1] = gray; + data[i + 2] = gray; + data[i + 3] = 255; + } + return { width: SIZE, height: SIZE, data }; +} + +function withRect(base: Rgba, x: number, y: number, w: number, h: number, gray = 255): Rgba { + const out = { ...base, data: base.data.slice() }; + for (let row = y; row < y + h; row++) { + for (let col = x; col < x + w; col++) { + const i = (row * SIZE + col) * 4; + out.data[i] = gray; + out.data[i + 1] = gray; + out.data[i + 2] = gray; + } + } + return out; +} + +function maskAt(mask: Uint8Array, x: number, y: number): number { + return mask[y * SIZE + x] ?? -1; +} + +function leftHalfMask(): Uint8Array { + const mask = new Uint8Array(SIZE * SIZE); + for (let y = 0; y < SIZE; y++) mask.fill(1, y * SIZE, y * SIZE + SIZE / 2); + return mask; +} + +describe("trailMask", () => { + it("marks the pixels that change across the frames, dilated", () => { + const a = withRect(frame(), 10, 20, 4, 4); + const b = withRect(frame(), 18, 20, 4, 4); + const mask = trailMask([a, b], 3); + expect(maskAt(mask, 10, 20)).toBe(1); + expect(maskAt(mask, 21, 23)).toBe(1); + expect(maskAt(mask, 7, 17)).toBe(1); + expect(maskAt(mask, 24, 26)).toBe(1); + expect(maskAt(mask, 6, 20)).toBe(0); + expect(maskAt(mask, 25, 20)).toBe(0); + expect(maskAt(mask, 15, 27)).toBe(0); + expect(maskAt(mask, 15, 16)).toBe(0); + expect(maskAt(mask, 60, 60)).toBe(0); + }); + + it("does not mark differences of 2 or less", () => { + const a = frame(100); + const b = frame(102); + expect(trailMask([a, b], 0).every((v) => v === 0)).toBe(true); + expect(trailMask([a, frame(103)], 0).every((v) => v === 1)).toBe(true); + }); + + it("of identical frames is empty", () => { + const a = withRect(frame(), 10, 10, 8, 8); + expect(trailMask([a, a, a], 4).every((v) => v === 0)).toBe(true); + }); + + it("throws on mismatched sizes", () => { + const small = { width: 32, height: 32, data: new Uint8Array(32 * 32 * 4) }; + expect(() => trailMask([frame(), small], 2)).toThrow(/size/i); + }); +}); + +describe("meanFrame", () => { + it("averages per channel", () => { + const mean = meanFrame([frame(0), frame(255)]); + expect(mean.width).toBe(SIZE); + expect(mean.height).toBe(SIZE); + expect(mean.data[0]).toBe(128); + expect(mean.data[1]).toBe(128); + expect(mean.data[2]).toBe(128); + expect(mean.data[3]).toBe(255); + }); + + it("throws on mismatched sizes", () => { + const small = { width: 32, height: 32, data: new Uint8Array(32 * 32 * 4) }; + expect(() => meanFrame([frame(), small])).toThrow(/size/i); + }); +}); + +describe("compareTiles", () => { + it("outside ignores a difference inside the mask", () => { + const reference = frame(); + const actual = withRect(frame(), 4, 4, 8, 8); + const mask = new Uint8Array(SIZE * SIZE); + for (let y = 0; y < 16; y++) mask.fill(1, y * SIZE, y * SIZE + 16); + const result = compareTiles(reference, actual, mask, "outside"); + expect(result.bad).toEqual([]); + expect(result.worstMae).toBe(0); + expect(result.checkedBlocks).toBe(15); + expect(result.blocks).toBe(16); + }); + + it("outside flags a displaced block outside the mask", () => { + const reference = withRect(frame(), 0, 8, 24, 24); + const actual = withRect(frame(), 40, 8, 24, 24); + const result = compareTiles(reference, actual, new Uint8Array(SIZE * SIZE), "outside"); + expect(result.bad.length).toBeGreaterThan(0); + const block = result.bad.find((b) => b.x === 0 && b.y === 16); + expect(block).toBeDefined(); + expect(block?.mae).toBeGreaterThan(8); + expect(result.bad.some((b) => b.x === 48 && b.y === 16)).toBe(true); + expect(result.worstMae).toBe(255); + }); + + it("inside flags a missing half of a moving element", () => { + const first = withRect(frame(), 8, 24, 32, 16); + const second = withRect(frame(), 24, 24, 32, 16); + const reference = meanFrame([first, second]); + const mask = trailMask([first, second], 2); + const fullActual = meanFrame([first, second]); + expect(compareTiles(reference, fullActual, mask, "inside").bad).toEqual([]); + + const halfActual = withRect(fullActual, 32, 24, 24, 16, 0); + const result = compareTiles(reference, halfActual, mask, "inside"); + expect(result.bad.length).toBeGreaterThanOrEqual(1); + expect(result.bad.every((b) => b.mae > 8)).toBe(true); + }); + + it("tolerates anti-aliasing-sized noise", () => { + const reference = withRect(frame(100), 20, 20, 16, 16, 200); + const actual = { ...reference, data: reference.data.slice() }; + for (let i = 0; i < actual.data.length; i += 4) { + const delta = (i / 4) % 2 === 0 ? 3 : -3; + for (let c = 0; c < 3; c++) actual.data[i + c] = (actual.data[i + c] ?? 0) + delta; + } + const mask = new Uint8Array(SIZE * SIZE); + for (let y = 16; y < 40; y++) mask.fill(1, y * SIZE + 16, y * SIZE + 40); + for (const region of ["outside", "inside"] as const) { + const result = compareTiles(reference, actual, mask, region); + expect(result.bad).toEqual([]); + expect(result.checkedBlocks).toBeGreaterThan(0); + expect(result.worstMae).toBeLessThanOrEqual(3); + } + }); + + it("reports checkedFraction as the share of pixels in checked tiles", () => { + const mask = leftHalfMask(); + const outside = compareTiles(frame(), frame(), mask, "outside"); + const inside = compareTiles(frame(), frame(), mask, "inside"); + expect(outside.checkedFraction).toBe(0.5); + expect(inside.checkedFraction).toBe(0.5); + expect(outside.checkedBlocks + inside.checkedBlocks).toBe(outside.blocks); + }); + + it("honours block and maxMae options", () => { + const reference = frame(); + const actual = withRect(frame(), 0, 0, 8, 8, 20); + const mask = new Uint8Array(SIZE * SIZE); + expect(compareTiles(reference, actual, mask, "outside", { block: 8 }).bad).toEqual([ + { x: 0, y: 0, mae: 20 }, + ]); + expect(compareTiles(reference, actual, mask, "outside", { block: 8, maxMae: 20 }).bad).toEqual( + [], + ); + }); + + it("throws on mismatched sizes", () => { + const small = { width: 32, height: 32, data: new Uint8Array(32 * 32 * 4) }; + const mask = new Uint8Array(SIZE * SIZE); + expect(() => compareTiles(frame(), small, mask, "outside")).toThrow(/size/i); + expect(() => compareTiles(frame(), frame(), new Uint8Array(10), "outside")).toThrow(/mask/i); + }); +}); + +describe("pinned-symptom helpers", () => { + const rect = { x: 10, y: 10, width: 20, height: 20 }; + + it("scales the CSS-pixel rect and its tolerance by the device pixel ratio", () => { + // dpr 2: the rect spans device px 20..60 and the 2 px tolerance becomes 4 px, so 16..64. + expect(tileTouches({ x: 64, y: 20 }, 16, rect, 2)).toBe(false); + expect(tileTouches({ x: 48, y: 20 }, 16, rect, 2)).toBe(true); + expect(tileTouches({ x: 1, y: 20 }, 16, rect, 2)).toBe(true); + expect(tileTouches({ x: 0, y: 20 }, 16, rect, 2)).toBe(false); + // The same device-pixel tile at dpr 1 sits far outside the unscaled rect. + expect(tileTouches({ x: 48, y: 20 }, 16, rect, 1)).toBe(false); + }); + + it("lists the bad tiles that miss the rect", () => { + const touching = { x: 0, y: 0, mae: 5 }; + const far = { x: 160, y: 160, mae: 9 }; + expect(tilesOutsideRegion([touching, far], 16, rect, 1)).toEqual([far]); + expect(tilesOutsideRegion([touching], 16, rect, 1)).toEqual([]); + }); + + it("accepts a worst MAE from half to double the measured one", () => { + expect(withinSymptomBand(5, 10)).toBe(true); + expect(withinSymptomBand(20, 10)).toBe(true); + expect(withinSymptomBand(4.9, 10)).toBe(false); + expect(withinSymptomBand(20.1, 10)).toBe(false); + }); +}); diff --git a/packages/engine/src/services/__fixtures__/blurDifferential.ts b/packages/engine/src/services/__fixtures__/blurDifferential.ts new file mode 100644 index 0000000000..8aeb0da7c9 --- /dev/null +++ b/packages/engine/src/services/__fixtures__/blurDifferential.ts @@ -0,0 +1,241 @@ +// Two pixel invariants for checking motion-blurred capture against a blur-off render. +// Outside the trail of everything that moved, blur on must equal blur off. Inside it, the +// blur-off render's own sub-frames averaged together are the answer blur on must reproduce. + +export interface Rgba { + width: number; + height: number; + data: Uint8Array; +} + +export interface BadBlock { + x: number; + y: number; + mae: number; +} + +export interface Differential { + blocks: number; + checkedBlocks: number; + checkedFraction: number; + worstMae: number; + bad: BadBlock[]; +} + +const CHANGED_ABOVE = 2; +const DEFAULT_BLOCK = 16; +const DEFAULT_MAX_MAE = 8; + +function assertSameSize(frames: readonly Rgba[]): Rgba { + const first = frames[0]; + if (!first) throw new Error("expected at least one frame"); + for (const frame of frames) { + if (frame.width !== first.width || frame.height !== first.height) { + throw new Error( + `frame size mismatch: ${frame.width}x${frame.height} vs ${first.width}x${first.height}`, + ); + } + } + return first; +} + +function cellAt( + source: Uint8Array, + base: number, + step: number, + index: number, + length: number, +): number { + if (index < 0 || index >= length) return 0; + return source[base + index * step] ?? 0; +} + +// For every line, 1 where any source value within `radius` along the line is set. +function boxMaxAlongAxis( + source: Uint8Array, + target: Uint8Array, + lines: number, + lineLength: number, + step: number, + lineStep: number, + radius: number, +): void { + for (let line = 0; line < lines; line++) { + const base = line * lineStep; + let inWindow = 0; + for (let i = 0; i < radius; i++) inWindow += cellAt(source, base, step, i, lineLength); + for (let i = 0; i < lineLength; i++) { + inWindow += cellAt(source, base, step, i + radius, lineLength); + inWindow -= cellAt(source, base, step, i - radius - 1, lineLength); + target[base + i * step] = inWindow > 0 ? 1 : 0; + } + } +} + +function channelRange(frames: readonly Rgba[], first: Rgba): { low: Uint8Array; high: Uint8Array } { + const low = first.data.slice(); + const high = first.data.slice(); + for (const { data } of frames) { + for (let i = 0; i < data.length; i++) { + const v = data[i] ?? 0; + low[i] = Math.min(low[i] ?? v, v); + high[i] = Math.max(high[i] ?? v, v); + } + } + return { low, high }; +} + +function pixelChanged(low: Uint8Array, high: Uint8Array, pixel: number): boolean { + for (let c = pixel * 4; c < pixel * 4 + 4; c++) { + if ((high[c] ?? 0) - (low[c] ?? 0) > CHANGED_ABOVE) return true; + } + return false; +} + +export function trailMask(frames: readonly Rgba[], radiusPx: number): Uint8Array { + const first = assertSameSize(frames); + if (!Number.isInteger(radiusPx) || radiusPx < 0) { + throw new RangeError(`radiusPx must be a non-negative integer, got ${radiusPx}`); + } + const { width, height } = first; + const { low, high } = channelRange(frames, first); + const changed = new Uint8Array(width * height); + for (let p = 0; p < changed.length; p++) changed[p] = pixelChanged(low, high, p) ? 1 : 0; + if (radiusPx === 0) return changed; + const horizontal = new Uint8Array(changed.length); + boxMaxAlongAxis(changed, horizontal, height, width, 1, width, radiusPx); + const dilated = new Uint8Array(changed.length); + boxMaxAlongAxis(horizontal, dilated, width, height, width, 1, radiusPx); + return dilated; +} + +export function meanFrame(frames: readonly Rgba[]): Rgba { + const first = assertSameSize(frames); + const sums = new Uint32Array(first.data.length); + for (const frame of frames) { + const { data } = frame; + for (let i = 0; i < data.length; i++) sums[i] = (sums[i] ?? 0) + (data[i] ?? 0); + } + const data = new Uint8Array(sums.length); + for (let i = 0; i < sums.length; i++) data[i] = Math.round((sums[i] ?? 0) / frames.length); + return { width: first.width, height: first.height, data }; +} + +interface TileStats { + maskedPixels: number; + pixels: number; + absDiffSum: number; +} + +function tileStats( + reference: Rgba, + actual: Rgba, + mask: Uint8Array, + x0: number, + y0: number, + block: number, +): TileStats { + const { width, height } = reference; + const x1 = Math.min(x0 + block, width); + const y1 = Math.min(y0 + block, height); + let maskedPixels = 0; + let absDiffSum = 0; + for (let y = y0; y < y1; y++) { + for (let x = x0; x < x1; x++) { + const p = y * width + x; + maskedPixels += mask[p] ? 1 : 0; + const i = p * 4; + for (let c = 0; c < 3; c++) { + absDiffSum += Math.abs((reference.data[i + c] ?? 0) - (actual.data[i + c] ?? 0)); + } + } + } + return { maskedPixels, pixels: (x1 - x0) * (y1 - y0), absDiffSum }; +} + +function isChecked(stats: TileStats, region: "outside" | "inside"): boolean { + return region === "inside" ? stats.maskedPixels > 0 : stats.maskedPixels === 0; +} + +export function compareTiles( + reference: Rgba, + actual: Rgba, + mask: Uint8Array, + region: "outside" | "inside", + options: { block?: number; maxMae?: number } = {}, +): Differential { + const block = options.block ?? DEFAULT_BLOCK; + const maxMae = options.maxMae ?? DEFAULT_MAX_MAE; + assertSameSize([reference, actual]); + const { width, height } = reference; + if (mask.length !== width * height) { + throw new Error(`mask length ${mask.length} does not match ${width}x${height}`); + } + if (!Number.isInteger(block) || block < 1) { + throw new RangeError(`block must be a positive integer, got ${block}`); + } + let blocks = 0; + let checkedBlocks = 0; + let checkedPixels = 0; + let worstMae = 0; + const bad: BadBlock[] = []; + for (let y = 0; y < height; y += block) { + for (let x = 0; x < width; x += block) { + blocks++; + const stats = tileStats(reference, actual, mask, x, y, block); + if (!isChecked(stats, region)) continue; + checkedBlocks++; + checkedPixels += stats.pixels; + const mae = stats.absDiffSum / (stats.pixels * 3); + worstMae = Math.max(worstMae, mae); + if (mae > maxMae) bad.push({ x, y, mae }); + } + } + return { + blocks, + checkedBlocks, + checkedFraction: checkedPixels / (width * height), + worstMae, + bad, + }; +} + +export interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +const REGION_TOLERANCE_PX = 2; +const SYMPTOM_BAND = { low: 0.5, high: 2 }; + +/** True when the tile, in device pixels, overlaps the CSS-pixel rect widened by the tolerance. */ +export function tileTouches( + tile: { x: number; y: number }, + block: number, + rect: Rect, + dpr: number, +): boolean { + const pad = REGION_TOLERANCE_PX * dpr; + const left = rect.x * dpr - pad; + const top = rect.y * dpr - pad; + const right = (rect.x + rect.width) * dpr + pad; + const bottom = (rect.y + rect.height) * dpr + pad; + return tile.x < right && tile.x + block > left && tile.y < bottom && tile.y + block > top; +} + +/** The bad tiles that do not touch the rect: a defect somewhere the pinned symptom does not explain. */ +export function tilesOutsideRegion( + bad: readonly BadBlock[], + block: number, + rect: Rect, + dpr: number, +): BadBlock[] { + return bad.filter((tile) => !tileTouches(tile, block, rect, dpr)); +} + +/** True when a pinned symptom's worst MAE is still within half to double of the value measured when it was pinned. */ +export function withinSymptomBand(worstMae: number, measured: number): boolean { + return worstMae >= SYMPTOM_BAND.low * measured && worstMae <= SYMPTOM_BAND.high * measured; +} diff --git a/packages/engine/src/services/__fixtures__/motionBlur/blurPage.ts b/packages/engine/src/services/__fixtures__/motionBlur/blurPage.ts new file mode 100644 index 0000000000..c00288fd05 --- /dev/null +++ b/packages/engine/src/services/__fixtures__/motionBlur/blurPage.ts @@ -0,0 +1,111 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import puppeteer, { type Browser, type Page } from "puppeteer-core"; +import { acceleratedCanvasTrackerSource } from "@hyperframes/core"; +import { decodePng } from "../../../utils/alphaBlit.js"; +import { resolveHeadlessShellPath } from "../../browserManager.js"; +import { waitForPendingSeekCompletion } from "../../frameCapture.js"; +import type { Rgba } from "../blurDifferential.js"; + +export const RUNTIME = resolve( + import.meta.dirname, + "../../../../../core/dist/hyperframe.runtime.iife.js", +); +export const chromePath = resolveHeadlessShellPath(); +export const hasBrowser = + chromePath !== undefined && + existsSync(RUNTIME) && + spawnSync(chromePath, ["--version"]).status === 0; + +export interface BlurReport { + state: string; + tier: string | null; + reasonCode: string | null; + movers: number; + failedFrames: number; +} + +export type BlurWindow = Window & { + __player?: { renderSeek?: (t: number, options?: { subFrameDivisions?: number }) => void }; + __hf_page_composite_pending?: boolean; + __hf_page_composite_prepare?: () => Promise; + __hf_page_composite_resolve?: () => boolean; + __hfMotionBlur?: { report: () => BlurReport; probe: () => BlurReport }; +}; + +export async function launch(): Promise<{ browser: Browser; supported: boolean }> { + const browser = await puppeteer.launch({ + executablePath: chromePath, + headless: true, + // Windows ClearType and Linux fontconfig subpixel text colour DOM glyphs but not the capture canvas; + // that is a host setting, not blur, so both arms draw grayscale text. + args: [ + "--no-sandbox", + "--disable-dev-shm-usage", + "--enable-features=CanvasDrawElement", + "--disable-lcd-text", + ], + }); + const probe = await browser.newPage(); + const supported = await probe.evaluate( + () => typeof Reflect.get(CanvasRenderingContext2D.prototype, "drawElementImage") === "function", + ); + await probe.close(); + return { browser, supported }; +} + +/** A render-mode page: the engine's seek config and the canvas tracker are in before the runtime boots. */ +export async function openFixture( + browser: Browser, + html: string, + dpr: number, + size: { width: number; height: number }, +): Promise { + const page = await browser.newPage(); + await page.setViewport({ ...size, deviceScaleFactor: dpr }); + await page.evaluate(acceleratedCanvasTrackerSource()); + await page.setContent(html); + await page.addScriptTag({ content: `window.__HF_EXPORT_RENDER_SEEK_CONFIG={fps:30};` }); + await page.addScriptTag({ content: readFileSync(RUNTIME, "utf8") }); + await page.waitForFunction(() => { + const w = window as BlurWindow & { __playerReady?: boolean; __renderReady?: boolean }; + return w.__playerReady === true && w.__renderReady === true; + }); + return page; +} + +/** One frame the way the engine takes it: seek, wait for the page, composite, screenshot. */ +export async function captureFrame( + page: Page, + t: number, + options?: { subFrameDivisions?: number }, +): Promise { + const pending = await page.evaluate( + (time, seekOptions) => { + const w = window as BlurWindow; + w.__player?.renderSeek?.(time, seekOptions); + return w.__hf_page_composite_pending === true; + }, + t, + options, + ); + await waitForPendingSeekCompletion(page); + if (pending) { + await page.evaluate(async () => { + const w = window as BlurWindow; + await w.__hf_page_composite_prepare?.(); + }); + const cdp = await page.createCDPSession(); + await cdp.send("Page.captureScreenshot", { + format: "jpeg", + quality: 1, + clip: { x: 0, y: 0, width: 1, height: 1, scale: 1 }, + }); + await cdp.detach(); + await page.evaluate(() => (window as BlurWindow).__hf_page_composite_resolve?.()); + } + const png = Buffer.from(await page.screenshot({ type: "png" })); + const { width, data } = decodePng(png); + return { width, height: data.length / 4 / width, data }; +} diff --git a/packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts b/packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts new file mode 100644 index 0000000000..10c4e4dcfe --- /dev/null +++ b/packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts @@ -0,0 +1,254 @@ +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { resolve } from "node:path"; + +export interface BlurFixture { + name: string; + width: number; + height: number; + dpr: number; + durationSeconds: number; + fps: 30; + /** Times inside holds: nothing moves within three frames of them. */ + checkpoints: readonly number[]; + /** Times while a mover moves. */ + moving: readonly number[]; + html(blur: "on" | "off"): string; + expectOn: { + state: "on" | "off"; + tier: "capture" | "filter" | null; + reasonCode: string | null; + minMovers?: number; + }; +} + +const repoRoot = resolve(import.meta.dirname, "../../../../../.."); +const GSAP = readFileSync( + createRequire(resolve(repoRoot, "packages/core/package.json")).resolve("gsap/dist/gsap.min.js"), + "utf8", +); + +const CARD_WIDTH = 80; +const CARD_HEIGHT = 40; +const SLIDE = 0.4; +const frame = (k: number): number => k / 30; +const frames = (...ks: number[]): number[] => ks.map(frame); + +interface Spec { + name: string; + width?: number; + height?: number; + dpr?: number; + duration?: number; + checkpoints?: readonly number[]; + moving?: readonly number[]; + expectOn?: Partial; + /** Pins the capture tier; left out for fixtures that exist to be handed to the filter tier. */ + captureTier?: boolean; + cardLeft?: number; + cardTop?: number; + cardStyle?: string; + cardContent?: string; + /** Markup placed under the card, or over it when `over` is set. */ + extra?: string; + over?: boolean; + style?: string; + /** Page script run after the root timeline `tl` exists. */ + script?: string; + cssMotion?: boolean; +} + +function slideScript(width: number, duration: number): string { + return `tl.fromTo("#card",{x:-200},{x:0,duration:${SLIDE},ease:"power2.out"},0) + .to("#card",{x:${width},duration:${SLIDE},ease:"power2.in"},${duration - SLIDE});`; +} + +function cssSlide(width: number, duration: number): string { + return `@keyframes slide { + 0% { transform: translateX(-200px); animation-timing-function: ease-out } + ${(SLIDE / duration) * 100}% { transform: translateX(0) } + ${100 - (SLIDE / duration) * 100}% { transform: translateX(0); animation-timing-function: ease-in } + 100% { transform: translateX(${width}px) } + } + #card { animation: slide ${duration}s linear both }`; +} + +const rootAttrs = (spec: Spec, blur: "on" | "off"): string => { + if (blur === "off") return 'data-motion-blur="off"'; + const tier = spec.captureTier === false ? "" : ' data-motion-blur-tier="capture"'; + return `data-motion-blur="on" data-motion-blur-events="on" data-motion-blur-shutter="240"${tier}`; +}; + +const rootBody = (spec: Spec): string => { + const card = `
${spec.cardContent ?? ""}
`; + return spec.over ? card + (spec.extra ?? "") : (spec.extra ?? "") + card; +}; + +const pageScript = (spec: Spec, width: number, duration: number): string => + spec.cssMotion + ? "" + : ``; + +function compose(spec: Spec, blur: "on" | "off"): string { + const width = spec.width ?? 320; + const height = spec.height ?? 180; + const duration = spec.duration ?? 2; + const left = spec.cardLeft ?? (width - CARD_WIDTH) / 2; + const top = spec.cardTop ?? (height - CARD_HEIGHT) / 2; + return ` +
${rootBody(spec)}
+ ${pageScript(spec, width, duration)} + `; +} + +function fixture(spec: Spec): BlurFixture { + return { + name: spec.name, + width: spec.width ?? 320, + height: spec.height ?? 180, + dpr: spec.dpr ?? 1, + durationSeconds: spec.duration ?? 2, + fps: 30, + checkpoints: spec.checkpoints ?? frames(15, 30, 45), + moving: spec.moving ?? frames(6, 54), + html: (blur) => compose(spec, blur), + expectOn: { state: "on", tier: "capture", reasonCode: null, ...spec.expectOn }, + }; +} + +const rows = Array.from( + { length: 30 }, + (_, i) => + `
Row ${i}
`, +).join(""); + +const WEBGL_TINT = ` + `; + +const FLIP = `
`; + +const LOGO_CLIP = ``; + +const HOLD_AFTER_FLIP = frames(36, 45); + +export const FIXTURES: readonly BlurFixture[] = [ + fixture({ name: "held-card" }), + fixture({ + name: "nested-canvas-offset", + style: + ".clip { position: absolute; inset: 0; overflow: hidden; will-change: transform, filter, opacity }", + extra: `
`, + script: `const cvx = document.getElementById("hcanvas").getContext("2d"); + const draw = () => { + cvx.clearRect(0, 0, 320, 100); + cvx.strokeStyle = "#fff"; + cvx.strokeRect(2, 2, 316, 96); + cvx.fillStyle = "#fff"; + cvx.font = "700 48px monospace"; + cvx.fillText("OPEN", 20, 60); + }; + tl.to({}, { duration: 2, onUpdate: draw }, 0);`, + }), + fixture({ + name: "scroll-list", + cardTop: 130, + checkpoints: frames(30, 36, 45), + extra: `
${rows}
`, + script: `const list = document.getElementById("list"); + const scroll = { v: 0 }; + tl.to(scroll, { v: 400, duration: 0.4, ease: "none", onUpdate() { list.scrollTop = scroll.v } }, 0.5);`, + }), + fixture({ + name: "callbacks", + checkpoints: frames(21, 24, 36), + extra: `
A
`, + script: `const label = document.getElementById("label"); + const write = (text) => () => { label.textContent = text }; + tl.call(write("B"), null, 0.5).call(write("A"), null, 0.5333) + .call(write("first"), null, 1).call(write("second"), null, 1);`, + }), + fixture({ + name: "stateful-callback", + extra: `
0
`, + script: `const count = document.getElementById("count"); + let last = 0, crossings = 0, shown = 0; + tl.eventCallback("onUpdate", () => { + const t = tl.time(); + if (t < last) { last = 0; crossings = 0 } + if (last < 0.6 && t >= 0.6) crossings++; + last = t; + if (crossings !== shown) { shown = crossings; count.textContent = String(crossings) } + });`, + checkpoints: frames(24, 30, 45), + }), + fixture({ name: "css-keyframes", cssMotion: true }), + fixture({ + name: "webgl-canvas", + captureTier: false, + over: true, + extra: WEBGL_TINT, + checkpoints: frames(24, 30, 45), + expectOn: { tier: "filter", reasonCode: "accelerated-canvas" }, + }), + // The flip tile stays sharp beside the blurred card: no time in the trail matches an average of its poses. + fixture({ + name: "flip-card-3d", + moving: [], + captureTier: false, + extra: FLIP, + style: "@keyframes flip { to { transform: rotateY(180deg) } }", + checkpoints: HOLD_AFTER_FLIP, + expectOn: { tier: "filter", reasonCode: "backface-visibility" }, + }), + fixture({ + name: "overflow-wordmark", + cardLeft: 20, + cardStyle: "white-space:nowrap;overflow:visible;font:700 40px/40px monospace;color:#f5d76e", + cardContent: "HYPERFRAME", + }), + fixture({ + name: "svg-clip-logo", + extra: LOGO_CLIP, + cardStyle: "background:#1f2a44", + cardContent: `
`, + }), + // The iframe stays sharp beside the blurred card: no time in the trail matches an average of its poses. + fixture({ + name: "iframe-in-root", + moving: [], + captureTier: false, + extra: ``, + expectOn: { tier: "filter", reasonCode: "iframe-in-root" }, + }), + // The custom element stays sharp beside the blurred card: no time in the trail matches an average of its poses. + fixture({ + name: "custom-element-in-root", + moving: [], + captureTier: false, + extra: ` + `, + expectOn: { tier: "filter", reasonCode: "iframe-in-root" }, + }), + fixture({ name: "dpr2-mover", dpr: 2 }), + fixture({ name: "portrait-mover", width: 180, height: 320 }), +]; diff --git a/packages/engine/src/services/__fixtures__/motionBlur/thresholds.ts b/packages/engine/src/services/__fixtures__/motionBlur/thresholds.ts new file mode 100644 index 0000000000..32879e31b4 --- /dev/null +++ b/packages/engine/src/services/__fixtures__/motionBlur/thresholds.ts @@ -0,0 +1,46 @@ +export const MEASURED_OUTSIDE_NOISE = 0.0078125; +export const MEASURED_CANVAS_MUTATION = 169.741; +export const MEASURED_REWIND_MUTATION = 106.099; +export const OUTSIDE_FLOOR = 1; + +// Through the producer's multi-worker PNG render. Noise: blur off rendered twice, five fixtures, bit-identical. +export const MEASURED_RENDER_NOISE = 0; +// Worst-tile MAE of the rewind mutation through the render (`callbacks`, 7 workers; caught 8 of 8 runs, always 97.02). +export const MEASURED_RENDER_REWIND_MUTATION = 97.02; + +// Worst-tile MAE of each pinned known bug, where it was pinned. A pin passes only within half to double of it. +export const MEASURED_SCROLL_LOSS_FRESH_PAINT = 24.65; +export const MEASURED_LABEL_EARLY_FRESH_PAINT = 54.3; +export const MEASURED_SCROLL_LOSS_RENDER = 18.82; +export const MEASURED_LABEL_EARLY_RENDER = 53.87; + +export const MEASURED_INSIDE_NOISE = 10.993; +export const MEASURED_CONTENT_CUT_MUTATION = 130.129; +export const MEASURED_CLIP_PATH_MUTATION = 124.322; + +const geometricMean = (a: number, b: number): number => Math.sqrt(a * b); + +export const RENDER_OUTSIDE_MAX_MAE = Math.max( + OUTSIDE_FLOOR, + geometricMean(MEASURED_RENDER_NOISE, MEASURED_RENDER_REWIND_MUTATION), +); + +export const INVARIANTS = { + block: 16, + outsideMaxMae: Math.max( + OUTSIDE_FLOOR, + geometricMean( + MEASURED_OUTSIDE_NOISE, + Math.min(MEASURED_CANVAS_MUTATION, MEASURED_REWIND_MUTATION), + ), + ), + // Not reliably caught: no mover blurred at all. It shows only where a large mover fills whole 16 px tiles. + insideMaxMae: geometricMean( + MEASURED_INSIDE_NOISE, + Math.min(MEASURED_CONTENT_CUT_MUTATION, MEASURED_CLIP_PATH_MUTATION), + ), + // Any DOM change within radiusFrames of a checkpoint is masked by the trail mask: keep checkpoints 3 frames from one. + radiusFrames: 2, + minCheckedFraction: 0.5, + shutterSamples: 8, +}; diff --git a/packages/engine/src/services/captureMotionBlur.differential.browser.test.ts b/packages/engine/src/services/captureMotionBlur.differential.browser.test.ts new file mode 100644 index 0000000000..e9fdbecca7 --- /dev/null +++ b/packages/engine/src/services/captureMotionBlur.differential.browser.test.ts @@ -0,0 +1,294 @@ +import type { Browser, Page } from "puppeteer-core"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + compareTiles, + meanFrame, + tilesOutsideRegion, + trailMask, + withinSymptomBand, + type Differential, + type Rect, + type Rgba, +} from "./__fixtures__/blurDifferential.js"; +import { closeTestChrome } from "./__fixtures__/loopback.js"; +import { + captureFrame, + hasBrowser, + launch, + openFixture, +} from "./__fixtures__/motionBlur/blurPage.js"; +import { FIXTURES, type BlurFixture } from "./__fixtures__/motionBlur/fixtures.js"; +import { + INVARIANTS, + MEASURED_LABEL_EARLY_FRESH_PAINT, + MEASURED_SCROLL_LOSS_FRESH_PAINT, +} from "./__fixtures__/motionBlur/thresholds.js"; + +const FPS = 30; +const frameTime = (n: number): number => n / FPS; +const SUB_FRAME_DIVISIONS = 24; +const SHUTTER_SECONDS = 240 / 360 / FPS; +const CASE_TIMEOUT_MS = 60_000; + +interface KnownBug { + reason: string; + /** The element whose rect every bad tile must touch. */ + regionId: string; + /** Worst-tile MAE of the symptom where it was pinned. */ + measuredMae: number; +} + +const SCROLL_LOSS: KnownBug = { + reason: "scroll-list loses its scroll offset on a fresh first paint", + regionId: "list", + measuredMae: MEASURED_SCROLL_LOSS_FRESH_PAINT, +}; +const LABEL_EARLY: KnownBug = { + reason: "callbacks shows its label B before the call that sets it", + regionId: "label", + measuredMae: MEASURED_LABEL_EARLY_FRESH_PAINT, +}; + +/** Cases where blur on really differs from blur off today. They pin the symptom and fail once it is fixed. */ +const KNOWN_REAL_DIFFERENCES = new Map([ + ["outside:scroll-list@1", SCROLL_LOSS], + ["outside:scroll-list@1.2", SCROLL_LOSS], + ["outside:scroll-list@1.5", SCROLL_LOSS], + ["outside:callbacks@0.2", LABEL_EARLY], + ["outside:callbacks@0.3", LABEL_EARLY], + ["outside:callbacks@0.4", LABEL_EARLY], +]); + +/** Before the label's first call: blur on gets it wrong at these frames, so they are the known-bug cases above. */ +const EXTRA_OUTSIDE_TIMES = new Map([ + ["callbacks", [frameTime(6), frameTime(9), frameTime(12)]], +]); + +const radiusPx = (f: BlurFixture): number => Math.ceil(2 * f.dpr); +const sizeOf = (f: BlurFixture): { width: number; height: number } => ({ + width: f.width, + height: f.height, +}); + +async function onFreshPage( + browser: Browser, + f: BlurFixture, + blur: "on" | "off", + use: (page: Page) => Promise, +): Promise { + const page = await openFixture(browser, f.html(blur), f.dpr, sizeOf(f)); + try { + return await use(page); + } finally { + await page.close(); + } +} + +async function captureInOrder( + page: Page, + times: readonly number[], + options?: { subFrameDivisions?: number }, +): Promise { + const frames: Rgba[] = []; + for (const t of times) frames.push(await captureFrame(page, t, options)); + return frames; +} + +function neighbourTimes(t: number): number[] { + const n = Math.round(t * FPS); + const { radiusFrames } = INVARIANTS; + return Array.from({ length: 2 * radiusFrames + 1 }, (_, k) => frameTime(n - radiusFrames + k)); +} + +function shutterTimes(t: number): number[] { + const { shutterSamples } = INVARIANTS; + return Array.from( + { length: shutterSamples }, + (_, k) => t - SHUTTER_SECONDS / 2 + ((k + 0.5) * SHUTTER_SECONDS) / shutterSamples, + ); +} + +function firstFive(d: Differential): string { + return JSON.stringify(d.bad.slice(0, 5)); +} + +async function measureOutside( + browser: Browser, + f: BlurFixture, + t: number, + regionId?: string, +): Promise<{ d: Differential; region: Rect }> { + const { offFrames, region } = await onFreshPage(browser, f, "off", async (p) => ({ + offFrames: await captureInOrder(p, neighbourTimes(t)), + region: await elementRect(p, regionId), + })); + const onFrame = await onFreshPage(browser, f, "on", (p) => captureFrame(p, t)); + const reference = offFrames[INVARIANTS.radiusFrames]; + if (!reference) throw new Error("missing reference frame"); + const d = compareTiles(reference, onFrame, trailMask(offFrames, radiusPx(f)), "outside", { + block: INVARIANTS.block, + maxMae: INVARIANTS.outsideMaxMae, + }); + return { d, region }; +} + +async function elementRect(page: Page, id: string | undefined): Promise { + const box = await page.evaluate((elementId) => { + const el = elementId ? document.getElementById(elementId) : null; + return el ? el.getBoundingClientRect().toJSON() : null; + }, id); + return box ?? { x: 0, y: 0, width: 0, height: 0 }; +} + +const caseName = (what: string, f: BlurFixture, t: number): string => `${what}: ${f.name} at ${t}`; + +let launched: Browser | undefined; +let supported = false; + +function chrome(): Browser { + if (!launched) throw new Error("Chrome did not launch"); + return launched; +} + +function fixtureNamed(name: string): BlurFixture { + const f = FIXTURES.find((x) => x.name === name); + if (!f) throw new Error(`missing fixture ${name}`); + return f; +} + +function defineKnownBugCase(f: BlurFixture, t: number, known: KnownBug): void { + it( + `KNOWN BUG on main: ${known.reason} (outside, ${t} s)`, + async (ctx) => { + if (!supported) ctx.skip(); + const { d, region } = await measureOutside(chrome(), f, t, known.regionId); + const fixed = `${known.reason}: this case pins a known bug; if it now passes, delete its \`outside:${f.name}@${t}\` entry from KNOWN_REAL_DIFFERENCES, which makes it an ordinary outside case`; + expect(d.checkedFraction).toBeGreaterThanOrEqual(INVARIANTS.minCheckedFraction); + expect(d.bad.length, fixed).toBeGreaterThan(0); + expect( + tilesOutsideRegion(d.bad, INVARIANTS.block, region, f.dpr), + `a new defect outside the ${known.regionId} rect ${JSON.stringify(region)}: ${firstFive(d)}`, + ).toEqual([]); + expect( + withinSymptomBand(d.worstMae, known.measuredMae), + `the pinned symptom changed: re-measure (worst MAE ${d.worstMae.toFixed(2)}, pinned at ${known.measuredMae})`, + ).toBe(true); + }, + CASE_TIMEOUT_MS, + ); +} + +function defineOutsideCase(f: BlurFixture, t: number): void { + it( + caseName("outside the trail, blur on equals blur off", f, t), + async (ctx) => { + if (!supported) ctx.skip(); + const { d } = await measureOutside(chrome(), f, t); + expect(d.bad, `${f.name} ${firstFive(d)}`).toEqual([]); + expect(d.checkedFraction).toBeGreaterThanOrEqual(INVARIANTS.minCheckedFraction); + }, + CASE_TIMEOUT_MS, + ); +} + +function defineInsideCase(f: BlurFixture, t: number): void { + it( + caseName("inside the trail, blur on matches the shutter average", f, t), + async (ctx) => { + if (!supported) ctx.skip(); + const samples = await onFreshPage(chrome(), f, "off", (p) => + captureInOrder(p, shutterTimes(t), { subFrameDivisions: SUB_FRAME_DIVISIONS }), + ); + const onFrame = await onFreshPage(chrome(), f, "on", (p) => captureFrame(p, t)); + const d = compareTiles( + meanFrame(samples), + onFrame, + trailMask(samples, radiusPx(f)), + "inside", + { + block: INVARIANTS.block, + maxMae: INVARIANTS.insideMaxMae, + }, + ); + expect(d.checkedBlocks).toBeGreaterThan(0); + expect(d.bad, `${f.name} ${firstFive(d)}`).toEqual([]); + }, + CASE_TIMEOUT_MS, + ); +} + +function defineHistoryCase(f: BlurFixture, t: number): void { + it( + `blur off does not depend on seek history: ${f.name} at ${t}`, + async (ctx) => { + if (!supported) ctx.skip(); + const inSequence = await onFreshPage( + chrome(), + f, + "off", + async (p) => (await captureInOrder(p, neighbourTimes(t)))[INVARIANTS.radiusFrames], + ); + const firstPaint = await onFreshPage(chrome(), f, "off", (p) => captureFrame(p, t)); + expect(inSequence).toBeDefined(); + expect(Buffer.from(inSequence?.data ?? []).equals(Buffer.from(firstPaint.data))).toBe(true); + }, + CASE_TIMEOUT_MS, + ); +} + +function defineSubFrameCase(): void { + it( + "renderSeek honours sub-frame times", + async (ctx) => { + if (!supported) ctx.skip(); + const t = 0.2; + const [a, mid, b] = await onFreshPage(chrome(), fixtureNamed("held-card"), "off", (p) => + captureInOrder(p, [t, t + 0.5 / FPS, t + 1 / FPS], { + subFrameDivisions: SUB_FRAME_DIVISIONS, + }), + ); + const left = [a, mid, b].map((frame) => (frame ? cardLeftEdge(frame) : -1)); + expect(left[0]).toBeGreaterThanOrEqual(0); + expect(left[1]).toBeGreaterThan(left[0] ?? Infinity); + expect(left[1]).toBeLessThan(left[2] ?? -Infinity); + }, + CASE_TIMEOUT_MS, + ); +} + +function defineFixtureCases(f: BlurFixture): void { + for (const t of [...f.checkpoints, ...(EXTRA_OUTSIDE_TIMES.get(f.name) ?? [])]) { + const known = KNOWN_REAL_DIFFERENCES.get(`outside:${f.name}@${t}`); + if (known) defineKnownBugCase(f, t, known); + else defineOutsideCase(f, t); + } + for (const t of f.moving) defineInsideCase(f, t); +} + +describe.skipIf(!hasBrowser)("capture motion blur against blur off in Chrome", () => { + beforeAll(async () => { + ({ browser: launched, supported } = await launch()); + }, 30_000); + afterAll(() => closeTestChrome(launched)); + + for (const f of FIXTURES) defineFixtureCases(f); + + describe("the blur-off reference is a valid yardstick", () => { + for (const name of ["held-card", "callbacks"]) { + const f = fixtureNamed(name); + for (const t of f.checkpoints) defineHistoryCase(f, t); + } + defineSubFrameCase(); + }); +}); + +/** First column of the card's amber on the card's middle row; -1 when the card is off screen. */ +function cardLeftEdge(frame: Rgba): number { + const y = Math.floor(frame.height / 2); + for (let x = 0; x < frame.width; x++) { + const i = (y * frame.width + x) * 4; + const [r, g, b] = [frame.data[i] ?? 0, frame.data[i + 1] ?? 0, frame.data[i + 2] ?? 0]; + if (r > 200 && g > 130 && g < 190 && b < 100) return x; + } + return -1; +} diff --git a/packages/engine/src/services/captureMotionBlur.fixtures.browser.test.ts b/packages/engine/src/services/captureMotionBlur.fixtures.browser.test.ts new file mode 100644 index 0000000000..fb55f4e71f --- /dev/null +++ b/packages/engine/src/services/captureMotionBlur.fixtures.browser.test.ts @@ -0,0 +1,95 @@ +import type { Browser } from "puppeteer-core"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { closeTestChrome } from "./__fixtures__/loopback.js"; +import { + captureFrame, + hasBrowser, + launch, + openFixture, + type BlurWindow, +} from "./__fixtures__/motionBlur/blurPage.js"; +import { FIXTURES } from "./__fixtures__/motionBlur/fixtures.js"; +import type { Rgba } from "./__fixtures__/blurDifferential.js"; + +// A fixture with no `moving` times still slides its card in at 0.2 s: that is where blur on and blur off must differ. +const PROBE_WHILE_CARD_SLIDES = 6 / 30; +const CASE_TIMEOUT_MS = 30_000; + +describe.skipIf(!hasBrowser)("capture motion blur fixtures in Chrome", () => { + let browser: Browser; + let supported = false; + + beforeAll(async () => { + ({ browser, supported } = await launch()); + }, 30_000); + afterAll(() => closeTestChrome(browser)); + + it("covers every failure class with a distinct fixture", () => { + expect(new Set(FIXTURES.map((f) => f.name)).size).toBe(FIXTURES.length); + expect(FIXTURES.length).toBeGreaterThanOrEqual(13); + }); + + it.each(FIXTURES)( + "renders $name with blur off and on without page errors, and the on report matches expectOn", + async (fixture, ctx) => { + if (!supported) ctx.skip(); + const size = { width: fixture.width, height: fixture.height }; + const first = fixture.checkpoints[0] ?? 0; + const moving = fixture.moving[0] ?? PROBE_WHILE_CARD_SLIDES; + const errors: string[] = []; + const watch = (page: Awaited>): void => { + page.on("pageerror", (e) => errors.push(String(e))); + page.on("console", (m) => { + if (m.type() === "error") errors.push(m.text()); + }); + }; + + let sharpMoving: Rgba | undefined; + const off = await openFixture(browser, fixture.html("off"), fixture.dpr, size); + watch(off); + try { + sharpMoving = await captureFrame(off, moving); + const frame = await captureFrame(off, first); + expect(frame.width).toBe(fixture.width * fixture.dpr); + expect(frame.height).toBe(fixture.height * fixture.dpr); + const report = await off.evaluate( + () => (window as BlurWindow).__hfMotionBlur?.report() ?? null, + ); + if (report) expect(report.state).toBe("off"); + } finally { + await off.close(); + } + + const on = await openFixture(browser, fixture.html("on"), fixture.dpr, size); + watch(on); + try { + const blurredMoving = await captureFrame(on, moving); + await captureFrame(on, first); + const report = await on.evaluate(() => (window as BlurWindow).__hfMotionBlur?.probe()); + expect(errors).toEqual([]); + const { minMovers, ...expected } = fixture.expectOn; + expect(report).toMatchObject({ ...expected, failedFrames: 0 }); + if (expected.state === "on") { + expect(report?.movers ?? 0).toBeGreaterThanOrEqual(minMovers ?? 1); + } + const differs = !Buffer.from(blurredMoving.data).equals(Buffer.from(sharpMoving.data)); + expect(differs).toBe(true); + } finally { + await on.close(); + } + }, + CASE_TIMEOUT_MS, + ); + + it.each(FIXTURES)("places the checkpoints of $name inside its holds on frame boundaries", (f) => { + const onFrame = (t: number): boolean => Math.abs(t * 30 - Math.round(t * 30)) < 1e-9; + expect(f.checkpoints.length).toBeGreaterThan(0); + for (const t of [...f.checkpoints, ...f.moving]) { + expect(onFrame(t)).toBe(true); + expect(t).toBeLessThanOrEqual(f.durationSeconds); + } + for (const c of f.checkpoints) { + for (const m of f.moving) expect(Math.round(Math.abs(c - m) * 30)).toBeGreaterThanOrEqual(3); + } + }); +}); diff --git a/packages/engine/src/services/captureMotionBlur.liveness.browser.test.ts b/packages/engine/src/services/captureMotionBlur.liveness.browser.test.ts new file mode 100644 index 0000000000..cde0205ec1 --- /dev/null +++ b/packages/engine/src/services/captureMotionBlur.liveness.browser.test.ts @@ -0,0 +1,230 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { Browser, Frame, Page } from "puppeteer-core"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { acceleratedCanvasTrackerSource } from "@hyperframes/core"; +import { closeTestChrome } from "./__fixtures__/loopback.js"; +import { RUNTIME, hasBrowser, launch } from "./__fixtures__/motionBlur/blurPage.js"; +import { FIXTURES, type BlurFixture } from "./__fixtures__/motionBlur/fixtures.js"; + +const MIN_CLOCK_ADVANCE_S = 0.3; +const MIN_DISTINCT_FRAMES = 3; +const LIVENESS_DEADLINE_MS = 3_000; +const WRAP_DEADLINE_MS = 3_000; +const CASE_TIMEOUT_MS = 30_000; + +type PlayerWindow = Window & { + __playerReady?: boolean; + __player?: { play: () => void; pause: () => void; getTime: () => number }; +}; + +interface BlankCensus { + wrapped: number; + blank: number; + /** Wrapped callbacks whose output canvas has no 2d context, which the detector cannot read. */ + unreadable: number; +} + +const playable = FIXTURES.filter((f) => f.expectOn.state === "on"); +const captureTier = FIXTURES.filter( + (f) => f.expectOn.state === "on" && f.expectOn.tier === "capture", +); + +/** + * The preview as Studio builds it: the tracker in the host window, the composition in an iframe + * the host's MutationObserver already watches, the runtime booted inside it. No render-mode seek + * config, so the runtime takes its playing path. + */ +async function openPreview( + browser: Browser, + fixture: BlurFixture, +): Promise<{ page: Page; preview: Frame }> { + const page = await browser.newPage(); + await page.setViewport({ + width: fixture.width, + height: fixture.height, + deviceScaleFactor: fixture.dpr, + }); + await page.setContent( + ``, + ); + await page.$eval( + "#preview", + (el, html) => + new Promise((done) => { + const frame = el as HTMLIFrameElement; + frame.onload = done; + frame.srcdoc = html; + }), + fixture.html("on"), + ); + await page.evaluate(() => { + const frame = document.getElementById("preview") as HTMLIFrameElement; + new MutationObserver(() => {}).observe( + frame.contentDocument?.getElementById("root") as Element, + { + childList: true, + }, + ); + }); + const preview = page.frames().find((f) => f !== page.mainFrame()); + if (!preview) throw new Error("no preview frame"); + await preview.addScriptTag({ content: readFileSync(RUNTIME, "utf8") }); + await preview.waitForFunction(() => (window as PlayerWindow).__playerReady === true); + return { page, preview }; +} + +const clockOf = (preview: Frame): Promise => + preview.evaluate(() => (window as PlayerWindow).__player?.getTime() ?? -1); + +async function startPlaying(preview: Frame): Promise { + await preview.evaluate(() => (window as PlayerWindow).__player?.play()); +} + +async function stopPlaying(preview: Frame): Promise { + await preview.evaluate(() => (window as PlayerWindow).__player?.pause()); +} + +const hashOf = (png: Uint8Array): string => createHash("md5").update(png).digest("hex"); + +/** Screenshots of the preview while it plays, until the clock and the pixels have both moved or time runs out. */ +async function watchPreview( + page: Page, + preview: Frame, + fixture: BlurFixture, +): Promise<{ advanced: number; distinct: number }> { + const clip = { x: 0, y: 0, width: fixture.width, height: fixture.height }; + const seen = new Set(); + const clock0 = await clockOf(preview); + let advanced = 0; + const deadline = Date.now() + LIVENESS_DEADLINE_MS; + await startPlaying(preview); + while (Date.now() < deadline) { + seen.add(hashOf(await page.screenshot({ type: "png", clip }))); + advanced = (await clockOf(preview)) - clock0; + if (advanced >= MIN_CLOCK_ADVANCE_S && seen.size >= MIN_DISTINCT_FRAMES) break; + } + return { advanced, distinct: seen.size }; +} + +async function checkKeepsChanging(browser: Browser, fixture: BlurFixture): Promise { + const { page, preview } = await openPreview(browser, fixture); + try { + const { advanced, distinct } = await watchPreview(page, preview, fixture); + expect( + advanced, + `the preview never played: ${fixture.name}'s clock moved ${advanced.toFixed(3)} s in ${LIVENESS_DEADLINE_MS} ms`, + ).toBeGreaterThanOrEqual(MIN_CLOCK_ADVANCE_S); + expect( + distinct, + `the preview froze: ${fixture.name}'s clock advanced ${advanced.toFixed(3)} s but the pixels showed ${distinct} distinct frames`, + ).toBeGreaterThanOrEqual(MIN_DISTINCT_FRAMES); + } finally { + await page.close(); + } +} + +/** + * Plays until the root is wrapped and reads the output canvas the moment the wrap lands. A MutationObserver + * runs when the wrap's own task ends, so the read cannot fall between two polls. The wrap clears the output and + * moves the root into the capture canvas, which the page does not render; the first draw needs a later paint. + * An output with no ink at that moment is what the next presented frame shows. How many frames reach the + * screen before the first draw depends on where the wrap falls between two vsyncs (zero or one here), so that + * count is not asserted. + */ +async function censusOfBlankFrames(browser: Browser, fixture: BlurFixture): Promise { + const { page, preview } = await openPreview(browser, fixture); + try { + return await preview.evaluate( + (deadlineMs) => + new Promise((done) => { + const census = { wrapped: 0, blank: 0, unreadable: 0 }; + const root = document.getElementById("root") as HTMLElement; + const hasInk = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement): boolean => { + for (let i = 0; i < 8; i++) + for (let j = 0; j < 8; j++) { + const x = Math.floor(((i + 0.5) / 8) * canvas.width); + const y = Math.floor(((j + 0.5) / 8) * canvas.height); + if ((ctx.getImageData(x, y, 1, 1).data[3] ?? 0) > 0) return true; + } + return false; + }; + const observer = new MutationObserver(() => { + if (root.parentElement?.getAttribute("data-hf-motion-blur") !== "capture") return; + observer.disconnect(); + census.wrapped++; + const out = document.querySelector( + 'canvas[data-hf-motion-blur="output"]', + ); + const ctx = out?.getContext("2d"); + if (!out || !ctx) census.unreadable++; + else if (!hasInk(ctx, out)) census.blank++; + done(census); + }); + observer.observe(document, { subtree: true, childList: true }); + setTimeout(() => { + observer.disconnect(); + done(census); + }, deadlineMs); + (window as PlayerWindow).__player?.play(); + }), + WRAP_DEADLINE_MS, + ); + } finally { + await stopPlaying(preview).catch(() => {}); + await page.close(); + } +} + +async function censusOfAll(browser: Browser): Promise<{ name: string; census: BlankCensus }[]> { + const rows: { name: string; census: BlankCensus }[] = []; + for (const fixture of captureTier) + rows.push({ name: fixture.name, census: await censusOfBlankFrames(browser, fixture) }); + return rows; +} + +const describeRow = ({ name, census }: { name: string; census: BlankCensus }): string => + `${name}: ${census.wrapped} wrapped, ${census.blank} blank, ${census.unreadable} unreadable`; + +describe.skipIf(!hasBrowser)("a playing capture motion blur preview in Chrome", () => { + let browser: Browser; + let supported = false; + + beforeAll(async () => { + ({ browser, supported } = await launch()); + }, 30_000); + afterAll(() => closeTestChrome(browser)); + + it.each(playable)( + "keeps changing while a $name preview plays", + async (fixture, ctx) => { + if (!supported) ctx.skip(); + await checkKeepsChanging(browser, fixture); + }, + CASE_TIMEOUT_MS, + ); + + it( + "KNOWN BUG on main: preview shows a blank frame after play() (capture tier)", + async (ctx) => { + if (!supported) ctx.skip(); + const rows = await censusOfAll(browser); + const summary = rows.map(describeRow).join("; "); + const total = (pick: (census: BlankCensus) => number): number => + rows.reduce((sum, row) => sum + pick(row.census), 0); + expect( + total((c) => c.unreadable), + `the output canvas has no 2d context, so the detector is blind and cannot see a blank frame (${summary})`, + ).toBe(0); + expect( + total((c) => c.wrapped), + `the detector did not see every fixture's root wrapped, so it cannot see a blank frame (${summary})`, + ).toBe(rows.length); + expect( + total((c) => c.blank), + `no blank frame seen after play(). Either the bug is fixed (delete this case, keep the wrapped > 0 guard and assert blank == 0) or the wrap no longer leaves the output empty (${summary})`, + ).toBeGreaterThan(0); + }, + CASE_TIMEOUT_MS * 2, + ); +}); diff --git a/packages/engine/src/services/captureMotionBlur.pixels.browser.test.ts b/packages/engine/src/services/captureMotionBlur.pixels.browser.test.ts index 6ea0e8ab51..6b3d2fe22b 100644 --- a/packages/engine/src/services/captureMotionBlur.pixels.browser.test.ts +++ b/packages/engine/src/services/captureMotionBlur.pixels.browser.test.ts @@ -1,20 +1,16 @@ -import { spawnSync } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; -import { resolve } from "node:path"; -import puppeteer, { type Browser, type Page } from "puppeteer-core"; +import { readFileSync } from "node:fs"; +import type { Browser, Page } from "puppeteer-core"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { decodePng } from "../utils/alphaBlit.js"; -import { resolveHeadlessShellPath } from "./browserManager.js"; -import { closeTestChrome } from "./__fixtures__/loopback.js"; import { acceleratedCanvasTrackerSource } from "@hyperframes/core"; -import { waitForPendingSeekCompletion } from "./frameCapture.js"; - -const RUNTIME = resolve(import.meta.dirname, "../../../core/dist/hyperframe.runtime.iife.js"); -const chromePath = resolveHeadlessShellPath(); -const hasBrowser = - chromePath !== undefined && - existsSync(RUNTIME) && - spawnSync(chromePath, ["--version"]).status === 0; +import { closeTestChrome } from "./__fixtures__/loopback.js"; +import { + RUNTIME, + captureFrame, + hasBrowser, + launch, + openFixture, + type BlurWindow as Win, +} from "./__fixtures__/motionBlur/blurPage.js"; const W = 320; const H = 180; @@ -26,17 +22,6 @@ interface Frame { width: number; data: Uint8Array; } -type Win = Window & { - __player?: { renderSeek?: (t: number) => void }; - __hf_page_composite_pending?: boolean; - __hf_page_composite_prepare?: () => Promise; - __hf_page_composite_resolve?: () => boolean; - __hfMotionBlur?: { - report: () => { state: string; tier: string | null; reasonCode: string | null; movers: number }; - probe: () => { state: string; tier: string | null; reasonCode: string | null; movers: number }; - }; -}; - /** 40 px box moving 60 px per frame (1800 px over the 1 s composition). */ const STYLE = ` @keyframes run { to { transform: translateX(1800px) } } @@ -95,60 +80,14 @@ describe.skipIf(!hasBrowser)("capture motion blur pixels in Chrome", () => { let supported = false; beforeAll(async () => { - browser = await puppeteer.launch({ - executablePath: chromePath, - headless: true, - args: ["--no-sandbox", "--disable-dev-shm-usage", "--enable-features=CanvasDrawElement"], - }); - const probe = await browser.newPage(); - supported = await probe.evaluate( - () => - typeof Reflect.get(CanvasRenderingContext2D.prototype, "drawElementImage") === "function", - ); - await probe.close(); + ({ browser, supported } = await launch()); }, 30_000); afterAll(() => closeTestChrome(browser)); - async function open(body: string, blur: boolean, tier?: Tier): Promise { - const page = await browser.newPage(); - await page.setViewport({ width: W, height: H, deviceScaleFactor: 1 }); - await page.evaluate(acceleratedCanvasTrackerSource()); - await page.setContent(composition(body, blur, tier)); - await page.addScriptTag({ content: `window.__HF_EXPORT_RENDER_SEEK_CONFIG={fps:${FPS}};` }); - await page.addScriptTag({ content: readFileSync(RUNTIME, "utf8") }); - await page.waitForFunction(() => { - const w = window as Win & { __playerReady?: boolean; __renderReady?: boolean }; - return w.__playerReady === true && w.__renderReady === true; - }); - return page; - } + const open = (body: string, blur: boolean, tier?: Tier): Promise => + openFixture(browser, composition(body, blur, tier), 1, { width: W, height: H }); - /** One frame the way the engine takes it: seek, wait for the page, composite, screenshot. */ - async function capture(page: Page, t: number): Promise { - const pending = await page.evaluate((time) => { - const w = window as Win; - w.__player?.renderSeek?.(time); - return w.__hf_page_composite_pending === true; - }, t); - await waitForPendingSeekCompletion(page); - if (pending) { - await page.evaluate(async () => { - const w = window as Win; - await w.__hf_page_composite_prepare?.(); - }); - const cdp = await page.createCDPSession(); - await cdp.send("Page.captureScreenshot", { - format: "jpeg", - quality: 1, - clip: { x: 0, y: 0, width: 1, height: 1, scale: 1 }, - }); - await cdp.detach(); - await page.evaluate(() => (window as Win).__hf_page_composite_resolve?.()); - } - const png = Buffer.from(await page.screenshot({ type: "png" })); - const { width, data } = decodePng(png); - return { width, data }; - } + const capture = (page: Page, t: number): Promise => captureFrame(page, t); async function frameOf(body: string, blur: boolean, t: number, tier?: Tier) { const page = await open(body, blur, tier); diff --git a/packages/producer/scripts/integration-tests.json b/packages/producer/scripts/integration-tests.json index 44cffa2042..eab0415df0 100644 --- a/packages/producer/scripts/integration-tests.json +++ b/packages/producer/scripts/integration-tests.json @@ -30,6 +30,7 @@ "src/services/htmlCompiler.variableMedia.test.ts", "src/services/render/audioPadTrim.integration.test.ts", "src/services/render/audioTruePeak.test.ts", + "src/services/render/blurDifferential.integration.test.ts", "src/services/render/deliveryRender.integration.test.ts", "src/services/render/firstFrameStartingState.integration.test.ts", "src/services/render/hlsRender.integration.test.ts", diff --git a/packages/producer/scripts/run-test-lane.mjs b/packages/producer/scripts/run-test-lane.mjs index cf954b6121..48884213df 100644 --- a/packages/producer/scripts/run-test-lane.mjs +++ b/packages/producer/scripts/run-test-lane.mjs @@ -32,7 +32,13 @@ const vitestFiles = tests.filter((test) => test.runner === "vitest").map((test) const vitestEnv = unit ? { NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ""} --import ${noNetwork.href}`.trim() } : {}; -if (vitestFiles.length > 0) run(["x", "vitest", "run", ...vitestFiles], vitestEnv); +// Files that start many Chromes at once run alone, after the rest: next to them, the Chrome +// node-count assertions in coreRuntimeBrowser.test.ts see a different garbage-collection schedule. +const runsAlone = (file) => file.endsWith("blurDifferential.integration.test.ts"); +const shared = vitestFiles.filter((file) => !runsAlone(file)); +const alone = vitestFiles.filter(runsAlone); +if (shared.length > 0) run(["x", "vitest", "run", ...shared], vitestEnv); +if (alone.length > 0) run(["x", "vitest", "run", ...alone], vitestEnv); // Bun's mock.module registry is process-global. Run each file in a fresh // process so mocks from one source test cannot mutate another test's imports. diff --git a/packages/producer/src/services/render/blurDifferential.integration.test.ts b/packages/producer/src/services/render/blurDifferential.integration.test.ts new file mode 100644 index 0000000000..a2fec2d245 --- /dev/null +++ b/packages/producer/src/services/render/blurDifferential.integration.test.ts @@ -0,0 +1,350 @@ +// Page motion blur through a real multi-worker render (#6967): outside the trail of everything +// that moved, a blur-on render must equal the blur-off render frame by frame. A PNG sequence is +// lossless and does not encode while capturing, so each worker captures one contiguous +// block of frames and every block but the first starts mid-film. Needs Chrome + ffmpeg: integration lane. + +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { decodePng, distributeFrames, getFfmpegBinary, resolveConfig } from "@hyperframes/engine"; +import { + compareTiles, + tilesOutsideRegion, + trailMask, + withinSymptomBand, + type BadBlock, + type Differential, + type Rect, + type Rgba, +} from "../../../../engine/src/services/__fixtures__/blurDifferential.js"; +import { + FIXTURES, + type BlurFixture, +} from "../../../../engine/src/services/__fixtures__/motionBlur/fixtures.js"; +import { + INVARIANTS, + MEASURED_LABEL_EARLY_RENDER, + MEASURED_SCROLL_LOSS_RENDER, + RENDER_OUTSIDE_MAX_MAE, +} from "../../../../engine/src/services/__fixtures__/motionBlur/thresholds.js"; +import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js"; + +const FFMPEG = getFfmpegBinary(); +const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0; + +const FPS = 30; +const WORKERS = 4; +/** + * `callbacks` calls at frames 15, 16 and 30. A worker whose block starts on a call + * sees the bug only by timing; one starting strictly between the calls always does (#6967). + * 7 workers over 60 frames start at 0, 9, 18, 27, 36, 45 and 54. + */ +const WORKERS_BY_FIXTURE = new Map([["callbacks", 7]]); +/** Block starts the partition must have, at least `count` of them in [from, to]. */ +const REQUIRED_STARTS = new Map([ + ["callbacks", { from: 18, to: 28, count: 2 }], + ["scroll-list", { from: 28, to: Infinity, count: 2 }], +]); +const MIN_SECONDS = 1.2; +const RENDER_TIMEOUT_MS = 600_000; + +const FIXTURE_NAMES = [ + "callbacks", + "scroll-list", + "nested-canvas-offset", + "held-card", + "overflow-wordmark", +] as const; + +interface KnownBug { + title: string; + reason: string; + /** The element whose rect every bad tile must touch, in CSS px from the fixture's markup. */ + regionId: string; + region: (f: BlurFixture) => Rect; + /** Output frames in which blur on really differs from blur off today. */ + frames: (n: number) => boolean; + /** Worst-tile MAE of the symptom where it was pinned. */ + measuredMae: number; + /** Set when the symptom must appear in those frames: what to edit when it no longer does. */ + ifFixed?: string; +} + +const markupOf = (f: BlurFixture, id: string): string => + new RegExp(`<[^>]*id="${id}"[^>]*>`).exec(f.html("off"))?.[0] ?? ""; + +function cssPx(f: BlurFixture, id: string, property: string): number { + const px = new RegExp(`(?:^|[;"\\s])${property}:\\s*(-?[\\d.]+)px`).exec(markupOf(f, id))?.[1]; + if (px === undefined) throw new Error(`#${id} in ${f.name} has no ${property} in px`); + return Number(px); +} + +/** The element's box from its inline left, top, width and height. */ +const boxFromMarkup = (f: BlurFixture, id: string): Rect => ({ + x: cssPx(f, id, "left"), + y: cssPx(f, id, "top"), + width: cssPx(f, id, "width"), + height: cssPx(f, id, "height"), +}); + +/** One glyph of the element's text: left and top from its inline style; a monospace advance is 0.6 em, a line 1.2 em. */ +function glyphFromMarkup(f: BlurFixture, id: string): Rect { + const size = Number(/font:\s*\d+\s+([\d.]+)px/.exec(markupOf(f, id))?.[1]); + if (Number.isNaN(size)) throw new Error(`#${id} in ${f.name} has no font size in px`); + return { x: cssPx(f, id, "left"), y: cssPx(f, id, "top"), width: 0.6 * size, height: 1.2 * size }; +} + +/** + * Cases where blur on really differs from blur off today. Whatever differs must stay inside the + * pinned region and keep its measured size. A case with `ifFixed` also requires the symptom to + * appear and fails once it is fixed; the others only bound it, because it is a race. + */ +const KNOWN_REAL_DIFFERENCES = new Map([ + [ + "scroll-list", + { + title: + "scroll-list: any loss after a worker's first wrap stays inside #list (KNOWN BUG, flip lives in the engine differential)", + reason: "scroll-list loses its scroll offset after a worker's first wrap", + regionId: "list", + region: (f) => boxFromMarkup(f, "list"), + frames: (n) => n >= 28, + measuredMae: MEASURED_SCROLL_LOSS_RENDER, + }, + ], + [ + "callbacks", + { + title: "KNOWN BUG on main: callbacks shows its label B before the call that sets it", + reason: "callbacks shows its label B before the call that sets it", + regionId: "label", + region: (f) => glyphFromMarkup(f, "label"), + frames: (n) => n < 15, + measuredMae: MEASURED_LABEL_EARLY_RENDER, + ifFixed: + "delete the callbacks entry from KNOWN_REAL_DIFFERENCES; keep WORKERS_BY_FIXTURE and REQUIRED_STARTS, which make the rewind regression visible after the fix", + }, + ], +]); + +interface Renders { + off: Rgba[]; + off2: Rgba[]; + on: Rgba[]; +} + +interface FrameResult { + frame: number; + outside: Differential; + noise: Differential; + insideMae: number; +} + +let root = ""; + +const fixtureNamed = (name: string): BlurFixture => { + const f = FIXTURES.find((x) => x.name === name); + if (!f) throw new Error(`missing fixture ${name}`); + return f; +}; + +async function renderFixture(f: BlurFixture, blur: "on" | "off", tag: string): Promise { + const projectDir = join(root, `${f.name}-${tag}`); + const outputDir = join(projectDir, "frames"); + mkdirSync(projectDir, { recursive: true }); + writeFileSync(join(projectDir, "index.html"), f.html(blur)); + const job = createRenderJob({ + fps: FPS, + format: "png-sequence", + quality: "standard", + hdrMode: "force-sdr", + useGpu: false, + workers: workersFor(f), + logger: { info() {}, warn() {}, error() {}, debug() {} }, + producerConfig: resolveConfig({ browserGpuMode: "software" }), + }); + await executeRenderJob(job, projectDir, outputDir); + const frames = readdirSync(outputDir) + .filter((file) => file.endsWith(".png")) + .sort() + .map((file) => decodePng(readFileSync(join(outputDir, file)))); + rmSync(projectDir, { recursive: true, force: true }); + return frames; +} + +async function renderAll(f: BlurFixture): Promise { + const off = await renderFixture(f, "off", "off"); + const off2 = await renderFixture(f, "off", "off2"); + const on = await renderFixture(f, "on", "on"); + return { off, off2, on }; +} + +const workersFor = (f: BlurFixture): number => WORKERS_BY_FIXTURE.get(f.name) ?? WORKERS; + +function blockStarts(f: BlurFixture): number[] { + const frames = Math.round(f.durationSeconds * FPS); + return distributeFrames(frames, workersFor(f), "", 0).map((task) => task.startFrame); +} + +const radiusPx = (f: BlurFixture): number => Math.ceil(2 * f.dpr); + +/** Frames whose `radiusFrames` neighbours on both sides exist. */ +function comparedFrames(count: number): number[] { + const { radiusFrames } = INVARIANTS; + return Array.from({ length: Math.max(0, count - 2 * radiusFrames) }, (_, k) => k + radiusFrames); +} + +function at(frames: readonly Rgba[], n: number): Rgba { + const frame = frames[n]; + if (!frame) throw new Error(`missing frame ${n}`); + return frame; +} + +function compareFrame(f: BlurFixture, r: Renders, n: number): FrameResult { + const { radiusFrames, block } = INVARIANTS; + const neighbours = r.off.slice(n - radiusFrames, n + radiusFrames + 1); + const mask = trailMask(neighbours, radiusPx(f)); + const options = { block, maxMae: RENDER_OUTSIDE_MAX_MAE }; + return { + frame: n, + outside: compareTiles(at(r.off, n), at(r.on, n), mask, "outside", options), + noise: compareTiles(at(r.off, n), at(r.off2, n), mask, "outside", options), + insideMae: compareTiles(at(r.off, n), at(r.on, n), mask, "inside", options).worstMae, + }; +} + +function compareFixture(f: BlurFixture, r: Renders): FrameResult[] { + expect(r.off).toHaveLength(Math.round(f.durationSeconds * FPS)); + expect(r.off2).toHaveLength(r.off.length); + expect(r.on).toHaveLength(r.off.length); + return comparedFrames(r.off.length).map((n) => compareFrame(f, r, n)); +} + +function firstBad(results: readonly FrameResult[]): string { + return JSON.stringify( + results + .filter((x) => x.outside.bad.length > 0) + .slice(0, 5) + .map((x) => ({ frame: x.frame, bad: x.outside.bad.slice(0, 3) })), + ); +} + +function expectNoNewDefect(f: BlurFixture, known: KnownBug, bad: readonly BadBlock[]): void { + const region = known.region(f); + expect( + tilesOutsideRegion(bad, INVARIANTS.block, region, 1), + `a new defect outside the ${known.regionId} rect ${JSON.stringify(region)}`, + ).toEqual([]); +} + +function expectKnownBug(f: BlurFixture, known: KnownBug, results: readonly FrameResult[]): void { + const pinned = results.filter((x) => known.frames(x.frame)); + const rest = results.filter((x) => !known.frames(x.frame)); + const bad = pinned.flatMap((x) => x.outside.bad); + if (known.ifFixed) { + expect( + bad.length, + `${known.reason}: this case pins a known bug; if it now passes, ${known.ifFixed}`, + ).toBeGreaterThan(0); + } + expectNoNewDefect(f, known, bad); + if (bad.length > 0) { + const worstMae = Math.max(...pinned.map((x) => x.outside.worstMae)); + expect( + withinSymptomBand(worstMae, known.measuredMae), + `the pinned symptom changed: re-measure (worst MAE ${worstMae.toFixed(2)}, pinned at ${known.measuredMae})`, + ).toBe(true); + } + expect( + rest.flatMap((x) => x.outside.bad), + `${f.name} outside the pinned frames ${firstBad(rest)}`, + ).toEqual([]); +} + +function expectEqualOutsideTrail(f: BlurFixture, results: readonly FrameResult[]): void { + expect( + results.flatMap((x) => x.outside.bad), + `${f.name} ${firstBad(results)}`, + ).toEqual([]); +} + +function defineFixtureCases(f: BlurFixture): void { + describe(f.name, () => { + let results: FrameResult[] = []; + + beforeAll(async () => { + expect(f.durationSeconds).toBeGreaterThanOrEqual(MIN_SECONDS); + results = compareFixture(f, await renderAll(f)); + }, RENDER_TIMEOUT_MS); + + const required = REQUIRED_STARTS.get(f.name); + if (required) { + it("partitions the frames so workers start where the bug shows", () => { + const starts = blockStarts(f).filter((n) => n >= required.from && n <= required.to); + expect(starts.length, `block starts ${blockStarts(f)}`).toBeGreaterThanOrEqual( + required.count, + ); + }); + } + + it("renders blur off the same way twice", () => { + expect(Math.max(...results.map((x) => x.noise.worstMae))).toBeLessThanOrEqual( + RENDER_OUTSIDE_MAX_MAE, + ); + }); + + it("checks most of every frame", () => { + for (const x of results) { + expect(x.outside.checkedFraction, `frame ${x.frame}`).toBeGreaterThanOrEqual( + INVARIANTS.minCheckedFraction, + ); + } + }); + + const known = KNOWN_REAL_DIFFERENCES.get(f.name); + if (known) { + it(known.title, () => expectKnownBug(f, known, results)); + } else { + it("outside the trail, blur on equals blur off in every frame", () => + expectEqualOutsideTrail(f, results)); + } + + if (f.name === "held-card") { + it("blurs the card, so the comparison is not vacuous", () => { + expect(Math.max(...results.map((x) => x.insideMae))).toBeGreaterThan( + RENDER_OUTSIDE_MAX_MAE, + ); + }); + } + }); +} + +describe.skipIf(!HAS_FFMPEG)("page motion blur through a multi-worker render", () => { + let priorFontconfig: string | undefined; + + beforeAll(() => { + root = mkdtempSync(join(tmpdir(), "hf-blur-differential-")); + // A host with fontconfig subpixel text colours DOM glyphs but not the capture canvas. That is a + // host setting, not blur, so both arms draw grayscale text. + priorFontconfig = process.env.FONTCONFIG_FILE; + const conf = join(root, "fonts.conf"); + writeFileSync( + conf, + ` +/etc/fonts/fonts.conf +none +`, + ); + process.env.FONTCONFIG_FILE = conf; + }); + + afterAll(() => { + if (priorFontconfig === undefined) delete process.env.FONTCONFIG_FILE; + else process.env.FONTCONFIG_FILE = priorFontconfig; + if (root) rmSync(root, { recursive: true, force: true }); + }); + + for (const name of FIXTURE_NAMES) defineFixtureCases(fixtureNamed(name)); +}); diff --git a/scripts/test-reachability.json b/scripts/test-reachability.json index 7c36fef51d..529b18dc51 100644 --- a/scripts/test-reachability.json +++ b/scripts/test-reachability.json @@ -46,7 +46,7 @@ "packages/producer/tests/**/*.test.ts" ], "sources": { - "packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7", + "packages/producer/scripts/run-test-lane.mjs": "eaf6d1ee00375fbc0934bcdb2281657bb06dd3a794deb928ffde2dcce15b874c", "packages/producer/scripts/test-classification.mjs": "e0874fac79225203b38778b144adb753a656afa0f78e08758001ff32ba9a0c62", ".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864", "packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1",