diff --git a/.github/workflows/motion-blur-nightly.yml b/.github/workflows/motion-blur-nightly.yml new file mode 100644 index 0000000000..c856976481 --- /dev/null +++ b/.github/workflows/motion-blur-nightly.yml @@ -0,0 +1,117 @@ +name: motion-blur-nightly + +permissions: + contents: read + +env: + HYPERFRAMES_NO_TELEMETRY: "1" + +concurrency: + group: motion-blur-nightly + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + seeds: + description: "Seeds to run, a comma list or a-b range (default: today's 200)." + type: string + default: "" + schedule: + # Daily at 08:41 UTC, off the :00 pile and away from regression.yml's 06:17. + - cron: "41 8 * * *" + +jobs: + plan: + name: Plan seeds + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + chunks: ${{ steps.plan.outputs.chunks }} + steps: + # Today's 200 seeds are days-since-2026-01-01 * 200 onward, in 4 chunks of 50. + - id: plan + env: + SEEDS: ${{ inputs.seeds }} + run: | + if [ -n "$SEEDS" ]; then + chunks="$(jq -cn --arg seeds "$SEEDS" '[{id: "custom", seeds: $seeds}]')" + else + days=$(( ($(date -u +%s) - $(date -u -d 2026-01-01 +%s)) / 86400 )) + start=$(( days * 200 )) + chunks="$(jq -cn --argjson start "$start" '[range(0; 4) | ($start + . * 50) as $from | {id: ($from | tostring), seeds: "\($from)-\($from + 49)"}]')" + fi + echo "chunks=$chunks" >> "$GITHUB_OUTPUT" + echo "$chunks" + + generated: + name: Generated compositions (${{ matrix.flavour }}, ${{ matrix.chunk.seeds }}) + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + flavour: [pin, stable] + chunk: ${{ fromJSON(needs.plan.outputs.chunks) }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - id: workspace + uses: ./.github/actions/setup-test-workspace + with: + engine: "true" + headless-shell: "true" + - id: pin + if: matrix.flavour == 'pin' + shell: bash + run: | + dir="$RUNNER_TEMP/headless-shell-pin" + bunx @puppeteer/browsers@3.2.2 install chrome-headless-shell@153.0.8010.36 --path "$dir" + echo "path=$(find "$dir" -type f -name chrome-headless-shell | head -n 1)" >> "$GITHUB_OUTPUT" + - name: Run the generated compositions + working-directory: packages/engine + env: + HYPERFRAMES_BROWSER_PATH: ${{ matrix.flavour == 'pin' && steps.pin.outputs.path || steps.workspace.outputs.headless-shell-path }} + HF_BLUR_SEEDS: ${{ matrix.chunk.seeds }} + HF_BLUR_FAILURES_FILE: ${{ github.workspace }}/blur-nightly-failures.json + HF_BLUR_REQUIRED: "1" + run: bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts + - name: Keep the failing seeds + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: blur-nightly-failures-${{ matrix.flavour }}-${{ matrix.chunk.id }} + path: blur-nightly-failures.json + if-no-files-found: ignore + + report: + name: Report failures + needs: [plan, generated] + if: always() && needs.plan.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + continue-on-error: true + with: + pattern: blur-nightly-failures-* + path: failures + - name: Open issues, warn and summarize + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + GENERATED_RESULT: ${{ needs.generated.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: >- + node scripts/ci/motion-blur-nightly-report.mjs + --failures failures + --generated-result "$GENERATED_RESULT" + --date "$(date -u +%F)" + --run-url "$RUN_URL" + --summary "$GITHUB_STEP_SUMMARY" diff --git a/package.json b/package.json index 33c32da7a0..3f0f686860 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", + "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/motion-blur-nightly-report.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", "test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", diff --git a/packages/engine/src/services/__fixtures__/motionBlur/differentialPage.ts b/packages/engine/src/services/__fixtures__/motionBlur/differentialPage.ts index ecc6f535cf..7f4bcd7a31 100644 --- a/packages/engine/src/services/__fixtures__/motionBlur/differentialPage.ts +++ b/packages/engine/src/services/__fixtures__/motionBlur/differentialPage.ts @@ -69,20 +69,29 @@ export function firstFive(d: Differential): string { return JSON.stringify(d.bad.slice(0, 5)); } +export interface Measured { + d: Differential; + reference: Rgba; + onFrame: Rgba; + mask: Uint8Array; +} + export async function measureOutside( browser: Browser, f: BlurFixture, t: number, tier?: Tier, -): Promise { +): Promise { const offFrames = await captureInOrder(await referencePage(f), neighbourTimes(t)); const onFrame = await onFreshPage(browser, f, "on", (p) => captureFrame(p, t), tier); const reference = offFrames[INVARIANTS.radiusFrames]; if (!reference) throw new Error("missing reference frame"); - return compareTiles(reference, onFrame, trailMask(offFrames, radiusPx(f)), "outside", { + const mask = trailMask(offFrames, radiusPx(f)); + const d = compareTiles(reference, onFrame, mask, "outside", { block: INVARIANTS.block, maxMae: INVARIANTS.outsideMaxMae, }); + return { d, reference, onFrame, mask }; } export const caseName = (what: string, f: BlurFixture, t: number, tier?: Tier): string => @@ -114,8 +123,11 @@ async function referencePage(f: BlurFixture): Promise { } export async function stopChrome(): Promise { - await closeReference(); - await closeTestChrome(launched); + try { + await closeReference(); + } finally { + await closeTestChrome(launched); + } } export function chrome(): Browser { @@ -134,7 +146,7 @@ export function defineOutsideCase(f: BlurFixture, t: number, tier?: Tier): void caseName("outside the trail, blur on equals blur off", f, t, tier), async (ctx) => { if (!supported) ctx.skip(); - const d = await measureOutside(chrome(), f, t, tier); + const { d } = await measureOutside(chrome(), f, t, tier); expect(d.bad, `${f.name} ${firstFive(d)}`).toEqual([]); expect(d.checkedFraction).toBeGreaterThanOrEqual(INVARIANTS.minCheckedFraction); }, @@ -154,22 +166,60 @@ function expectBlurredAwayFromSharp( ); } +const shutterMean = async (page: Page, t: number): Promise => + meanFrame( + await captureInOrder(page, shutterTimes(t), { subFrameDivisions: SUB_FRAME_DIVISIONS }), + ); + +export async function measureInside( + browser: Browser, + f: BlurFixture, + t: number, + tier?: Tier, +): Promise { + const samples = await captureInOrder(await referencePage(f), shutterTimes(t), { + subFrameDivisions: SUB_FRAME_DIVISIONS, + }); + const onFrame = await onFreshPage(browser, f, "on", (p) => captureFrame(p, t), tier); + const reference = meanFrame(samples); + const mask = trailMask(samples, radiusPx(f)); + const d = compareTiles(reference, onFrame, mask, "inside", { + block: INVARIANTS.block, + maxMae: INVARIANTS.insideMaxMae, + }); + return { d, reference, onFrame, mask }; +} + +export const REFERENCE_RECAPTURES = 3; + +export async function blurOffDisagreement( + browser: Browser, + f: BlurFixture, + region: "outside" | "inside", + t: number, + first: Measured, +): Promise { + const maxMae = region === "outside" ? INVARIANTS.outsideMaxMae : INVARIANTS.insideMaxMae; + for (let i = 0; i < REFERENCE_RECAPTURES; i++) { + const again = await onFreshPage(browser, f, "off", (p) => + region === "outside" ? captureFrame(p, t) : shutterMean(p, t), + ); + const d = compareTiles(first.reference, again, first.mask, region, { + block: INVARIANTS.block, + maxMae, + }); + if (d.bad.length > 0) return d.worstMae; + } + return undefined; +} + export function defineInsideCase(f: BlurFixture, t: number, tier?: Tier): void { it( caseName("inside the trail, blur on matches the shutter average", f, t, tier), async (ctx) => { if (!supported) ctx.skip(); - const offPage = await referencePage(f); - const samples = await captureInOrder(offPage, shutterTimes(t), { - subFrameDivisions: SUB_FRAME_DIVISIONS, - }); - const sharp = await captureFrame(offPage, t); - const onFrame = await onFreshPage(chrome(), f, "on", (p) => captureFrame(p, t), tier); - const mask = trailMask(samples, radiusPx(f)); - const d = compareTiles(meanFrame(samples), onFrame, mask, "inside", { - block: INVARIANTS.block, - maxMae: INVARIANTS.insideMaxMae, - }); + const { d, onFrame, mask } = await measureInside(chrome(), f, t, tier); + const sharp = await captureFrame(await referencePage(f), t); expect(d.checkedBlocks).toBeGreaterThan(0); expect(d.bad, `${f.name} ${firstFive(d)}`).toEqual([]); expectBlurredAwayFromSharp(f, sharp, onFrame, mask); diff --git a/packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts b/packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts index b13f3cb367..9e2d07e19c 100644 --- a/packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts +++ b/packages/engine/src/services/__fixtures__/motionBlur/fixtures.ts @@ -34,7 +34,7 @@ const SLIDE = 0.4; const frame = (k: number): number => k / 30; const frames = (...ks: number[]): number[] => ks.map(frame); -interface Spec { +export interface Spec { name: string; width?: number; height?: number; @@ -114,7 +114,7 @@ function compose(spec: Spec, blur: "on" | "off", tier?: "capture" | "filter"): s `; } -function fixture(spec: Spec): BlurFixture { +export function fixture(spec: Spec): BlurFixture { return { name: spec.name, width: spec.width ?? 320, diff --git a/packages/engine/src/services/__fixtures__/motionBlur/generate.test.ts b/packages/engine/src/services/__fixtures__/motionBlur/generate.test.ts new file mode 100644 index 0000000000..b8efc4872d --- /dev/null +++ b/packages/engine/src/services/__fixtures__/motionBlur/generate.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; +import { + EXCLUDED_FEATURES, + FEATURES, + generateComposition, + parseSeeds, + type Feature, +} from "./generate.js"; + +const SEEDS = parseSeeds("1-200"); + +const GSAP_SCRIPT = /`, + }; + }, + svgClip: ({ slot, flip }) => { + const s = slot(); + const defs = ``; + const clip = "clip-path:url(#genClip)"; + if (flip()) { + const part = wrapped("sv", s, clip); + return { ...part, markup: defs + part.markup }; + } + return { + markup: + defs + + el( + "sibling", + `${at(s, 60, 44)};background:linear-gradient(135deg,#ff7a59,#ffd166);${clip}`, + ), + }; + }, + mask: ({ slot, flip }) => { + const s = slot(); + const mask = + "-webkit-mask-image:radial-gradient(circle,#000 40%,transparent 75%);mask-image:radial-gradient(circle,#000 40%,transparent 75%)"; + return flip() + ? wrapped("mk", s, mask) + : { + markup: el( + "masked", + `${at(s, SLOT_WIDTH, SLOT_HEIGHT)};background:linear-gradient(90deg,#ff7a59,#06d6a0);${mask}`, + ), + }; + }, + onUpdateMover: ({ slot }) => { + const s = slot(); + return { + markup: el("ou", `${at(s)};background:#9b5de5;transform:translate(${fromLeft(s)}px,0)`), + script: `const driven = { v: 0 }; + const place = () => { + const v = driven.v; + document.getElementById("ou").style.transform = "translate(" + (v <= 1 ? ${fromLeft(s)} * (1 - v) : ${toRight(s)} * (v - 1)) + "px,0)"; + }; + tl.to(driven,{ v: 1, duration: 0.4, ease: "power2.out", onUpdate: place },0) + .to(driven,{ v: 2, duration: 0.4, ease: "power2.in", onUpdate: place },1.6);`, + }; + }, + tlCall: ({ slot, flip }) => { + const s = slot(); + const byClass = flip(); + return { + css: byClass ? "#fl.on { background: #06d6a0 }" : "", + markup: el("fl", `${at(s)};background:#3a86ff`), + script: byClass + ? `const fl = document.getElementById("fl"); + tl.call(() => fl.classList.add("on"), null, 0.7).call(() => fl.classList.remove("on"), null, 1.3);` + : `const fl = document.getElementById("fl"); + tl.call(() => { fl.style.background = "#06d6a0" }, null, 0.7).call(() => { fl.style.background = "#3a86ff" }, null, 1.3);`, + }; + }, + onceGuard: ({ slot, pick }) => { + const s = slot(); + return { + markup: el("og", `${at(s)};background:#ef476f`), + script: `let fired = false; + tl.call(() => { if (fired) return; fired = true; document.getElementById("og").style.background = "#ffd166" }, null, ${frame(pick(ONCE_FRAMES))});`, + }; + }, + cssKeyframes: ({ slot }) => { + const s = slot(); + return { + css: `@keyframes genSlide { + 0% { transform: translateX(${fromLeft(s)}px); animation-timing-function: ease-out } + 20% { transform: translateX(0) } + 80% { transform: translateX(0); animation-timing-function: ease-in } + 100% { transform: translateX(${toRight(s)}px) } + }`, + markup: el("ck", `${at(s)};background:#118ab2;animation:genSlide 2s linear both`), + }; + }, + nesting: ({ slot }) => { + const s = slot(); + return { + markup: el( + "no", + `${at(s, SLOT_WIDTH, SLOT_HEIGHT)};background:#264653`, + `
`, + ), + script: `${inOut("#no", { x: fromLeft(s) }, { x: 0 }, { x: toRight(s) })} + tl.to("#ni",{ x: 50, duration: 0.4, ease: "power2.inOut" },0);`, + }; + }, + pseudoElement: ({ pick }) => { + const before = `#card::before { content: ""; position: absolute; left: -34px; top: 8px; width: 24px; height: 24px; border-radius: 50%; background: #ffd166 }`; + const after = `#card::after { content: ""; position: absolute; left: 90px; top: 8px; width: 24px; height: 24px; border-radius: 50%; background: #06d6a0 }`; + return { css: pick([before, after, `${before} ${after}`]) }; + }, + panelInFront: () => ({ + markup: `
`, + }), + blendMode: () => ({ + markup: `
`, + }), + fixedChild: () => ({ + content: `
`, + }), + lastChild: () => ({ + css: "#root > :last-child { outline: 3px solid #ffd166 }", + last: `
`, + }), +}; + +function chooseFeatures(next: () => number): Feature[] { + const count = 1 + Math.floor(next() * 4); + const chosen = new Set(shuffled(FEATURES, next).slice(0, count)); + return FEATURES.filter((feature) => chosen.has(feature)); +} + +export function generateComposition(seed: number): GeneratedFixture { + const next = lcg(seed); + const features = chooseFeatures(next); + const free = shuffled(SLOTS, next); + const draw: Draw = { + slot: () => { + const taken = free.pop(); + if (!taken) throw new Error("out of slots"); + return taken; + }, + pick: (items: readonly T[]): T => items[Math.floor(next() * items.length)] as T, + flip: () => next() < 0.5, + }; + const parts = features.map((feature) => BUILDERS[feature](draw)); + const join = (key: "css" | "markup" | "script" | "content" | "last"): string => + parts.map((part) => part[key] ?? "").join("\n"); + const checkpoints = shuffled(CHECKPOINT_FRAMES, next) + .slice(0, 2) + .sort((a, b) => a - b) + .map(frame); + const moving = [frame(draw.pick(MOVING_FRAMES))]; + const base = fixture({ + name: `seed ${seed} [${features.join(", ")}]`, + captureTier: false, + over: true, + cardStyle: `z-index:2;${parts.map((part) => part.card ?? "").join("")}`, + cardContent: join("content"), + extra: join("markup") + join("last"), + style: join("css"), + script: join("script"), + checkpoints, + moving, + expectOn: { tier: "filter", reasonCode: "tier-auto" }, + }); + return { ...base, seed, features }; +} + +export function parseSeeds(spec: string): number[] { + return spec.split(",").flatMap((part) => { + const [from, to = from] = part.split("-").map((n) => Number.parseInt(n, 10)); + if (from === undefined || to === undefined || Number.isNaN(from) || Number.isNaN(to)) { + throw new Error(`bad seed list: ${spec}`); + } + return Array.from({ length: to - from + 1 }, (_, i) => from + i); + }); +} diff --git a/packages/engine/src/services/captureMotionBlur.generated.browser.test.ts b/packages/engine/src/services/captureMotionBlur.generated.browser.test.ts new file mode 100644 index 0000000000..ad2459eca2 --- /dev/null +++ b/packages/engine/src/services/captureMotionBlur.generated.browser.test.ts @@ -0,0 +1,156 @@ +import { writeFileSync } from "node:fs"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { Differential } from "./__fixtures__/blurDifferential.js"; +import { hasBrowser } from "./__fixtures__/motionBlur/blurPage.js"; +import { + CASE_TIMEOUT_MS, + blurOffDisagreement, + chrome, + firstFive, + isSupported, + measureInside, + type Measured, + measureOutside, + startChrome, + stopChrome, +} from "./__fixtures__/motionBlur/differentialPage.js"; +import { + generateComposition, + parseSeeds, + type GeneratedFixture, +} from "./__fixtures__/motionBlur/generate.js"; +import { INVARIANTS } from "./__fixtures__/motionBlur/thresholds.js"; + +const SEEDS = parseSeeds(process.env.HF_BLUR_SEEDS ?? "1,2,3"); +const FAILURES_FILE = process.env.HF_BLUR_FAILURES_FILE; +const REQUIRED = process.env.HF_BLUR_REQUIRED === "1"; + +interface KnownBad { + finding: string; + reason: string; +} + +const KNOWN_BAD: Readonly> = {}; + +interface FailureRecord { + seed: number | null; + features: readonly string[]; + problems: readonly string[]; + unstableReference?: number; +} + +const records: FailureRecord[] = []; + +const worst = (d: Differential): string => + `${d.bad.length} bad tiles, worst MAE ${d.worstMae.toFixed(1)}, first ${firstFive(d)}`; + +interface Outcome { + problems: string[]; + unstable: number[]; +} + +async function judge( + f: GeneratedFixture, + region: "outside" | "inside", + t: number, + first: Measured, + outcome: Outcome, +): Promise { + if (first.d.bad.length === 0) return; + const disagreement = await blurOffDisagreement(chrome(), f, region, t, first); + if (disagreement === undefined) { + outcome.problems.push(`${region} the trail at ${t}: ${worst(first.d)}`); + } else { + outcome.unstable.push(disagreement); + } +} + +async function check(f: GeneratedFixture): Promise { + const outcome: Outcome = { problems: [], unstable: [] }; + const times = [...f.checkpoints, ...f.moving].sort((a, b) => a - b); + for (const t of times) { + const moving = f.moving.includes(t); + const outside = await measureOutside(chrome(), f, t); + await judge(f, "outside", t, outside, outcome); + if (!moving && outside.d.checkedFraction < INVARIANTS.minCheckedFraction) { + outcome.problems.push( + `outside the trail at ${t}: only ${outside.d.checkedFraction} of the frame was checked`, + ); + } + if (outside.d.checkedBlocks === 0) { + outcome.problems.push(`outside the trail at ${t}: no tile was checked`); + } + if (!moving) continue; + const inside = await measureInside(chrome(), f, t); + await judge(f, "inside", t, inside, outcome); + if (inside.d.checkedBlocks === 0) { + outcome.problems.push(`inside the trail at ${t}: no tile was checked`); + } + } + return outcome; +} + +function defineSeedCase(seed: number): void { + const f = generateComposition(seed); + const pinned = KNOWN_BAD[seed]; + it( + pinned ? `KNOWN BAD on main: ${pinned.finding}: ${pinned.reason}: ${f.name}` : f.name, + async (ctx) => { + if (!isSupported()) ctx.skip(); + let found: string[] = []; + ctx.onTestFailed(() => { + const thrown = ctx.task.result?.errors?.map((e) => String(e.message)) ?? []; + records.push({ seed, features: f.features, problems: found.length ? found : thrown }); + }); + const { problems, unstable } = await check(f); + if (pinned) { + expect( + problems.length, + `${pinned.finding} looks fixed: delete seed ${seed} from KNOWN_BAD`, + ).toBeGreaterThan(0); + return; + } + found = problems; + if (problems.length === 0 && unstable.length > 0) { + records.push({ + seed, + features: f.features, + problems: [], + unstableReference: Math.max(...unstable), + }); + } + expect(problems, `${f.name}: rerun with HF_BLUR_SEEDS=${seed}`).toEqual([]); + }, + CASE_TIMEOUT_MS * 2, + ); +} + +async function startRequiredChrome(): Promise { + if (!hasBrowser) throw new Error("HF_BLUR_REQUIRED is set but no Chrome was found"); + await startChrome(); + if (!isSupported()) + throw new Error("HF_BLUR_REQUIRED is set but this Chrome lacks drawElementImage"); +} + +describe.skipIf(!hasBrowser && !REQUIRED)( + "capture motion blur over generated compositions in Chrome", + () => { + beforeAll(async () => { + try { + await (REQUIRED ? startRequiredChrome() : startChrome()); + } catch (error) { + records.push({ seed: null, features: [], problems: [`Chrome did not start: ${error}`] }); + throw error; + } + }, 30_000); + afterAll(async () => { + try { + if (FAILURES_FILE) writeFileSync(FAILURES_FILE, JSON.stringify(records, null, 2)); + } finally { + await stopChrome(); + } + }); + + for (const seed of SEEDS) defineSeedCase(seed); + }, +); diff --git a/scripts/ci/motion-blur-nightly-report.mjs b/scripts/ci/motion-blur-nightly-report.mjs new file mode 100644 index 0000000000..38dfa8e471 --- /dev/null +++ b/scripts/ci/motion-blur-nightly-report.mjs @@ -0,0 +1,328 @@ +#!/usr/bin/env node +// motion-blur-nightly-report.mjs --failures --generated-result --date ... +// Reads the nightly's per-job failure files and opens the issues, warnings and summary they call for. +import { spawnSync } from "node:child_process"; +import { + appendFileSync, + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ARTIFACT_PREFIX = "blur-nightly-failures-"; +const FILE = "blur-nightly-failures.json"; +const LABELS = ["bug", "area/quality"]; +const DETAILED_SEEDS = 50; +const MAX_PROBLEM_CHARS = 400; + +export const DEFAULTS = { maxIssues: 25, systemicAbove: 5 }; + +const clip = (text, limit) => (text.length > limit ? `${text.slice(0, limit)}...` : text); + +const escapeData = (text) => + String(text).replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A"); +const inertColons = (text) => text.replace(/:(?=:)/g, ": "); +const escapeProperty = (text) => escapeData(text).replace(/:/g, "%3A").replace(/,/g, "%2C"); + +export function workflowCommand(name, message, properties = {}) { + const props = Object.entries(properties) + .map(([key, value]) => `${key}=${escapeProperty(value)}`) + .join(","); + return `::${name}${props ? ` ${props}` : ""}::${inertColons(escapeData(message))}`; +} + +const warning = (message) => workflowCommand("warning", message); + +const inline = (text) => + `\`${String(text) + .replace(/\\/g, "\\\\") + .replace(/`/g, "'") + .replace(/\r?\n|\r/g, " ") + .replace(/\|/g, "\\|")}\``; + +function fenced(text) { + const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length)); + const ticks = "`".repeat(Math.max(3, longest + 1)); + return [ticks, text, ticks].join("\n"); +} + +const flavourList = (flavours) => [...new Set(flavours)].map(inline).join(", "); + +function readFailureFile(file, flavour) { + try { + return JSON.parse(readFileSync(file, "utf8")).map((record) => ({ ...record, flavour })); + } catch (error) { + return [{ seed: null, features: [], problems: [`unreadable ${file}: ${error}`], flavour }]; + } +} + +export function readRecords(dir) { + if (!existsSync(dir)) return []; + return readdirSync(dir).flatMap((name) => { + const file = join(dir, name, FILE); + if (!name.startsWith(ARTIFACT_PREFIX) || !existsSync(file)) return []; + return readFailureFile(file, name.slice(ARTIFACT_PREFIX.length).split("-")[0]); + }); +} + +const isSeed = (seed) => Number.isSafeInteger(seed) && seed >= 0; + +export function classify(records) { + const failing = new Map(); + const unstable = []; + const setup = []; + for (const record of records) { + const { flavour } = record; + const problems = Array.isArray(record.problems) ? record.problems.map(String) : []; + const features = Array.isArray(record.features) ? record.features.map(String) : []; + if (record.seed === null || record.seed === undefined) { + setup.push({ flavour, problems: problems.length ? problems : ["record without a seed"] }); + } else if (!isSeed(record.seed)) { + setup.push({ + flavour, + problems: [`record without a usable seed (${record.seed})`, ...problems], + }); + } else if (problems.length > 0) { + const entry = failing.get(record.seed) ?? { + seed: record.seed, + features, + flavours: [], + problems: [], + }; + entry.flavours.push(flavour); + entry.problems.push(...problems.map((problem) => `[${flavour}] ${problem}`)); + failing.set(record.seed, entry); + } else if (typeof record.unstableReference === "number") { + unstable.push({ seed: record.seed, features, flavour, mae: record.unstableReference }); + } + } + return { + failing: [...failing.values()].sort((a, b) => a.seed - b.seed), + unstable: unstable.sort((a, b) => a.seed - b.seed), + setup, + }; +} + +const seedTitle = (seed) => `motion blur nightly: seed ${seed}`; +const systemicTitle = (date) => `motion blur nightly: systemic failure ${date}`; +const features = (entry) => entry.features.join(", ") || "none"; +const flavoursOf = (entries) => entries.flatMap((entry) => entry.flavours); + +export function seedBody(entry, runUrl) { + return [ + `Seed ${entry.seed} fails the pixel invariants on \`auto\`.`, + "", + `- Features: ${inline(features(entry))}`, + `- Chrome flavours that failed: ${flavourList(entry.flavours)}`, + `- Run: ${runUrl}`, + "", + "Reproduce from `packages/engine`:", + "", + "```sh", + `HF_BLUR_SEEDS=${entry.seed} bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts`, + "```", + "", + "Findings, by flavour:", + "", + fenced( + entry.problems + .slice(0, 10) + .map((problem) => clip(problem, MAX_PROBLEM_CHARS)) + .join("\n"), + ), + ].join("\n"); +} + +export function systemicBody(entries, date, runUrl) { + const shown = entries.slice(0, DETAILED_SEEDS); + const rest = entries.slice(DETAILED_SEEDS).map((entry) => entry.seed); + const lines = shown.map( + (entry) => + `${entry.seed} [${features(entry)}] (${[...new Set(entry.flavours)].join(", ")}): ${clip(entry.problems[0], 200)}`, + ); + return [ + `${entries.length} seeds failed the pixel invariants on \`auto\` in the ${date} run, which points at one cause rather than ${entries.length} defects.`, + "", + `- Run: ${runUrl}`, + `- Chrome flavours that failed: ${flavourList(flavoursOf(entries))}`, + "", + "Reproduce one from `packages/engine`:", + "", + "```sh", + `HF_BLUR_SEEDS=${entries[0].seed} bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts`, + "```", + "", + "Seeds:", + "", + fenced(lines.join("\n")), + ...(rest.length ? ["", `And ${rest.length} more: ${rest.join(", ")}`] : []), + ].join("\n"); +} + +function openTitles(gh) { + const result = gh([ + "issue", + "list", + "--state", + "open", + "--limit", + "1000", + "--json", + "title", + "--jq", + ".[].title", + ]); + if (result.status !== 0) throw new Error(`gh issue list failed: ${result.stderr}`); + return new Set(result.stdout.split("\n").filter(Boolean)); +} + +function openIssue(gh, titles, title, body, log) { + if (titles.has(title)) { + log(`skipping "${title}": an open issue has that title`); + return; + } + const file = join(mkdtempSync(join(tmpdir(), "blur-nightly-")), "body.md"); + writeFileSync(file, body); + const labels = LABELS.flatMap((label) => ["--label", label]); + const result = gh(["issue", "create", "--title", title, "--body-file", file, ...labels]); + if (result.status !== 0) + throw new Error(`gh issue create failed for "${title}": ${result.stderr}`); + titles.add(title); + log(`opened "${title}"`); +} + +function fileIssues({ failing, gh, date, runUrl, maxIssues, systemicAbove, log }) { + if (failing.length === 0) return; + const titles = openTitles(gh); + if (failing.length > systemicAbove) { + openIssue(gh, titles, systemicTitle(date), systemicBody(failing, date, runUrl), log); + return; + } + const entries = failing.slice(0, maxIssues); + if (failing.length > entries.length) + log(warning(`${failing.length} failing seeds; the first ${maxIssues} get an issue`)); + for (const entry of entries) + openIssue(gh, titles, seedTitle(entry.seed), seedBody(entry, runUrl), log); +} + +function summaryLines({ failing, unstable, setup, silentFailure }) { + const lines = ["## Motion blur nightly", ""]; + if (silentFailure) { + lines.push( + `A generated job ended ${inline(silentFailure)} without recording a failing seed (a crash, a kill or a setup step); see the job logs.`, + "", + ); + } + if (setup.length) { + const items = setup.map( + (item) => `${item.flavour}: ${item.problems.map((p) => clip(p, 300)).join("; ")}`, + ); + lines.push( + "### The run did not start or its record was unusable", + "", + fenced(items.join("\n")), + "", + ); + } + if (failing.length) { + const items = failing.map( + (entry) => `${entry.seed} [${features(entry)}] on ${[...new Set(entry.flavours)].join(", ")}`, + ); + lines.push("### Failing seeds", "", fenced(items.join("\n")), ""); + } + if (unstable.length) { + lines.push( + "### Seeds whose blur-off reference disagreed with itself (renderer noise, no issue opened)", + "", + "| Seed | Features | Flavour | Worst blur-off MAE |", + "| --- | --- | --- | --- |", + ...unstable.map( + (u) => + `| ${u.seed} | ${inline(u.features.join(", "))} | ${inline(u.flavour)} | ${u.mae.toFixed(2)} |`, + ), + "", + ); + } + if (lines.length === 2) lines.push("No failing and no unstable seeds.", ""); + return lines; +} + +export function report(options) { + const { dir, generatedResult, summaryFile, log } = options; + const { failing, unstable, setup } = classify(readRecords(dir)); + const silentFailure = + generatedResult !== "success" && failing.length === 0 && setup.length === 0 + ? generatedResult || "without a result" + : ""; + if (silentFailure) + log( + warning( + `A generated job ended ${silentFailure} without recording a failing seed; see the run logs.`, + ), + ); + if (setup.length) { + const causes = setup.map((s) => `${s.flavour}: ${clip(s.problems[0], 200)}`).join(" | "); + log(warning(`The nightly did not run cleanly: ${causes}`)); + } + if (unstable.length) { + const seeds = [...new Set(unstable.map((u) => u.seed))]; + log( + warning( + `${seeds.length} seed(s) had a blur-off reference that disagrees with itself and were not counted: ${seeds.join(", ")}`, + ), + ); + } + if (summaryFile) + appendFileSync( + summaryFile, + `${summaryLines({ failing, unstable, setup, silentFailure }).join("\n")}\n`, + ); + fileIssues({ ...options, failing }); +} + +function gh(args) { + const result = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 1 << 26 }); + return { status: result.status ?? 1, stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; +} + +function readFlags(argv) { + const values = {}; + for (let i = 0; i < argv.length; i += 2) { + if (!argv[i]?.startsWith("--") || argv[i + 1] === undefined) { + throw new Error(`bad argument ${argv[i]}`); + } + values[argv[i].slice(2)] = argv[i + 1]; + } + return values; +} + +function parseArgs(argv) { + const flags = { + failures: "failures", + "generated-result": "", + "run-url": "", + "max-issues": DEFAULTS.maxIssues, + "systemic-above": DEFAULTS.systemicAbove, + date: new Date().toISOString().slice(0, 10), + ...readFlags(argv), + }; + if (!/^\d{4}-\d{2}-\d{2}$/.test(flags.date)) throw new Error(`bad date ${flags.date}`); + return { + dir: flags.failures, + generatedResult: flags["generated-result"], + date: flags.date, + runUrl: flags["run-url"], + summaryFile: flags.summary, + maxIssues: Number(flags["max-issues"]), + systemicAbove: Number(flags["systemic-above"]), + gh, + log: (line) => console.log(line), + }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) report(parseArgs(process.argv.slice(2))); diff --git a/scripts/ci/motion-blur-nightly-report.test.mjs b/scripts/ci/motion-blur-nightly-report.test.mjs new file mode 100644 index 0000000000..3fcb250ef6 --- /dev/null +++ b/scripts/ci/motion-blur-nightly-report.test.mjs @@ -0,0 +1,363 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { classify, workflowCommand } from "./motion-blur-nightly-report.mjs"; + +const SCRIPT = join(import.meta.dirname, "motion-blur-nightly-report.mjs"); + +const STUB = `#!/bin/sh +printf '%s\\n' "$*" >> "$STUB_LOG" +if [ "$1 $2" = "issue list" ]; then + [ "$*" = "issue list --state open --limit 1000 --json title --jq .[].title" ] || { echo "unexpected gh arguments: $*" >&2; exit 2; } + [ -n "$STUB_LIST_FAIL" ] && { echo "boom" >&2; exit 1; } + cat "$STUB_TITLES" + exit 0 +fi +if [ "$1 $2" = "issue create" ]; then + while [ $# -gt 0 ]; do [ "$1" = "--body-file" ] && { cat "$2"; echo "====="; } >> "$STUB_LOG.bodies"; shift; done +fi +exit 0 +`; + +function night({ + files = {}, + titles = [], + generatedResult = "failure", + args = [], + listFails = false, +}) { + const dir = mkdtempSync(join(tmpdir(), "nightly-report-test-")); + const failures = join(dir, "failures"); + for (const [artifact, records] of Object.entries(files)) { + mkdirSync(join(failures, `blur-nightly-failures-${artifact}`), { recursive: true }); + writeFileSync( + join(failures, `blur-nightly-failures-${artifact}`, "blur-nightly-failures.json"), + JSON.stringify(records), + ); + } + mkdirSync(join(dir, "bin")); + writeFileSync(join(dir, "bin", "gh"), STUB); + chmodSync(join(dir, "bin", "gh"), 0o755); + writeFileSync(join(dir, "titles.txt"), titles.map((t) => `${t}\n`).join("")); + const summary = join(dir, "summary.md"); + const run = spawnSync( + process.execPath, + [ + SCRIPT, + "--failures", + failures, + "--generated-result", + generatedResult, + "--date", + "2026-11-06", + "--run-url", + "https://example.test/run/1", + "--summary", + summary, + ...args, + ], + { + encoding: "utf8", + env: { + ...process.env, + PATH: `${join(dir, "bin")}:${process.env.PATH}`, + STUB_LOG: join(dir, "gh.log"), + STUB_TITLES: join(dir, "titles.txt"), + STUB_LIST_FAIL: listFails ? "1" : "", + }, + }, + ); + const calls = existsSync(join(dir, "gh.log")) + ? readFileSync(join(dir, "gh.log"), "utf8").split("\n").filter(Boolean) + : []; + return { + run, + calls, + lists: calls.filter((c) => c.startsWith("issue list")), + creates: calls.filter((c) => c.startsWith("issue create")), + summary: existsSync(summary) ? readFileSync(summary, "utf8") : "", + dir, + }; +} + +const real = (seed, problems = ["outside the trail at 0.5: 3 bad tiles"], features = ["mask"]) => ({ + seed, + features, + problems, +}); + +test("one issue per failing seed, titled by seed, with the flavours that failed", () => { + const n = night({ files: { "pin-1": [real(7)], "stable-1": [real(7), real(9)] } }); + assert.equal(n.run.status, 0, n.run.stderr); + assert.equal(n.lists.length, 1); + assert.equal(n.creates.length, 2); + assert.match(n.creates[0], /--title motion blur nightly: seed 7 /); + assert.match(n.creates[1], /--title motion blur nightly: seed 9 /); + assert.match(n.creates[0], /--label bug --label area\/quality/); + const [first, second] = readFileSync(join(n.dir, "gh.log.bodies"), "utf8").split("=====\n"); + assert.match(first, /Chrome flavours that failed: `pin`, `stable`/); + assert.match(first, /^\[pin\] outside the trail at 0.5/m); + assert.match(first, /^\[stable\] outside the trail at 0.5/m); + assert.match(second, /Chrome flavours that failed: `stable`/); + assert.match(first, /HF_BLUR_SEEDS=7 /); + rmSync(n.dir, { recursive: true }); +}); + +test("the title match is exact: an open seed 129 issue does not hide seed 12", () => { + const n = night({ + files: { "pin-1": [real(12), real(129)] }, + titles: ["motion blur nightly: seed 129", "unrelated"], + }); + assert.equal(n.creates.length, 1); + assert.match(n.creates[0], /seed 12 /); + assert.match(n.run.stdout, /skipping "motion blur nightly: seed 129"/); + rmSync(n.dir, { recursive: true }); +}); + +test("open titles are fetched once, however many seeds fail", () => { + const n = night({ files: { "pin-1": [real(1), real(2), real(3), real(4), real(5)] } }); + assert.equal(n.lists.length, 1); + assert.equal(n.creates.length, 5); + rmSync(n.dir, { recursive: true }); +}); + +test("more than 5 failing seeds open one systemic issue and no per-seed issue", () => { + const seeds = [1, 2, 3, 4, 5, 6, 7].map((s) => real(s)); + const n = night({ files: { "pin-1": seeds, "stable-1": seeds } }); + assert.equal(n.creates.length, 1); + assert.match(n.creates[0], /--title motion blur nightly: systemic failure 2026-11-06 /); + assert.ok(!n.creates[0].includes("seed 1 ")); + rmSync(n.dir, { recursive: true }); +}); + +test("the systemic title is deduped like the others", () => { + const seeds = [1, 2, 3, 4, 5, 6].map((s) => real(s)); + const n = night({ + files: { "pin-1": seeds }, + titles: ["motion blur nightly: systemic failure 2026-11-06"], + }); + assert.equal(n.creates.length, 0); + assert.match(n.run.stdout, /skipping "motion blur nightly: systemic failure 2026-11-06"/); + rmSync(n.dir, { recursive: true }); +}); + +test("the cap applies before any per-seed work", () => { + const seeds = [1, 2, 3, 4].map((s) => real(s)); + const n = night({ + files: { "pin-1": seeds }, + args: ["--max-issues", "2", "--systemic-above", "10"], + }); + assert.equal(n.creates.length, 2); + assert.equal(n.lists.length, 1); + assert.match(n.run.stdout, /::warning::4 failing seeds; the first 2 get an issue/); + rmSync(n.dir, { recursive: true }); +}); + +test("a crash record becomes an issue", () => { + const n = night({ + files: { "pin-1": [real(33, ["Error: Test timed out in 120000ms."], ["webgl"])] }, + }); + assert.equal(n.creates.length, 1); + assert.match(n.creates[0], /seed 33 /); + rmSync(n.dir, { recursive: true }); +}); + +test("unstable seeds go to the summary and one warning, never to an issue", () => { + const n = night({ + files: { + "pin-1": [ + { seed: 129, features: ["canvas2d", "svgClip"], problems: [], unstableReference: 2.03 }, + ], + "stable-1": [{ seed: 130, features: ["mask"], problems: [], unstableReference: 1.5 }], + }, + generatedResult: "success", + }); + assert.equal(n.run.status, 0, n.run.stderr); + assert.equal(n.calls.length, 0); + assert.equal(n.run.stdout.match(/::warning::/g)?.length, 1); + assert.match( + n.run.stdout, + /::warning::2 seed\(s\) had a blur-off reference that disagrees with itself.*129, 130/, + ); + assert.match(n.summary, /\| 129 \| `canvas2d, svgClip` \| `pin` \| 2\.03 \|/); + rmSync(n.dir, { recursive: true }); +}); + +test("a failed job with no record warns and writes the summary", () => { + const n = night({ files: { "pin-1": [], "stable-1": [] }, generatedResult: "failure" }); + assert.equal(n.run.status, 0, n.run.stderr); + assert.equal(n.calls.length, 0); + assert.match( + n.run.stdout, + /::warning::A generated job ended failure without recording a failing seed/, + ); + assert.match(n.summary, /ended `failure` without recording a failing seed/); + rmSync(n.dir, { recursive: true }); +}); + +test("a failed job with no artifacts at all warns the same way", () => { + const n = night({ generatedResult: "failure" }); + assert.match(n.run.stdout, /::warning::A generated job ended failure without recording/); + rmSync(n.dir, { recursive: true }); +}); + +test("a launch failure record warns and goes to the summary", () => { + const n = night({ + files: { + "stable-1": [{ seed: null, features: [], problems: ["Chrome lacks drawElementImage"] }], + }, + }); + assert.equal(n.calls.length, 0); + assert.match( + n.run.stdout, + /::warning::The nightly did not run cleanly: stable: Chrome lacks drawElementImage/, + ); + assert.match(n.summary, /did not start/); + assert.ok(!n.run.stdout.includes("without recording")); + rmSync(n.dir, { recursive: true }); +}); + +test("a green night with nothing to say opens nothing and warns of nothing", () => { + const n = night({ files: { "pin-1": [], "stable-1": [] }, generatedResult: "success" }); + assert.equal(n.calls.length, 0); + assert.ok(!n.run.stdout.includes("::warning::")); + assert.match(n.summary, /No failing and no unstable seeds/); + rmSync(n.dir, { recursive: true }); +}); + +test("a failed open-title fetch fails the job instead of risking duplicates", () => { + const n = night({ files: { "pin-1": [real(5)] }, listFails: true }); + assert.notEqual(n.run.status, 0); + assert.equal(n.creates.length, 0); + rmSync(n.dir, { recursive: true }); +}); + +test("a seed that is not an integer never reaches a title", () => { + const n = night({ files: { "pin-1": [real("1; echo x"), real(4)] } }); + assert.equal(n.creates.length, 1); + assert.match(n.creates[0], /seed 4 /); + assert.match(n.summary, /record without a usable seed/); + rmSync(n.dir, { recursive: true }); +}); + +test("classify groups a seed across flavours and prefixes each finding with its flavour", () => { + const { failing } = classify([ + { ...real(8, ["a"]), flavour: "pin" }, + { ...real(8, ["b"]), flavour: "stable" }, + ]); + assert.deepEqual(failing, [ + { + seed: 8, + features: ["mask"], + flavours: ["pin", "stable"], + problems: ["[pin] a", "[stable] b"], + }, + ]); +}); + +const commandLines = (stdout) => stdout.split("\n").filter((line) => line.startsWith("::")); + +test("the open-title fetch uses exactly the flags the report relies on", () => { + const n = night({ files: { "pin-1": [real(5)] } }); + assert.equal(n.lists[0], "issue list --state open --limit 1000 --json title --jq .[].title"); + rmSync(n.dir, { recursive: true }); +}); + +test("a job that was cancelled or timed out warns like a failed one", () => { + for (const generatedResult of ["cancelled", "skipped"]) { + const n = night({ files: { "pin-1": [] }, generatedResult }); + assert.match( + n.run.stdout, + new RegExp(`::warning::A generated job ended ${generatedResult} without`), + ); + rmSync(n.dir, { recursive: true }); + } +}); + +test("workflowCommand escapes data and properties the way the runner does", () => { + assert.equal(workflowCommand("warning", "a%b\r\nc"), "::warning::a%25b%0D%0Ac"); + assert.equal( + workflowCommand("warning", "m", { file: "x:y,z\n" }), + "::warning file=x%3Ay%2Cz%0A::m", + ); +}); + +test("workflowCommand leaves no :: in its message, so text on one line cannot start a command", () => { + const line = workflowCommand("warning", "x ::add-mask::secret ::stop-commands::t :::y"); + assert.equal(line, "::warning::x : :add-mask: :secret : :stop-commands: :t : : :y"); + assert.equal(line.slice("::warning::".length).includes("::"), false); +}); + +test("untrusted newlines never start a second workflow command", () => { + const hostile = "1\n::error::pwned"; + const n = night({ + files: { + "stable-1": [ + { seed: hostile, features: [], problems: [] }, + { + seed: null, + features: [], + problems: ["Chrome died\n::error::pwned %0A\r::warning::again"], + }, + { seed: 3, features: [], problems: [], unstableReference: 1.5 }, + ], + }, + }); + assert.equal(n.run.status, 0, n.run.stderr); + const commands = commandLines(n.run.stdout); + assert.equal(commands.length, 2); + assert.ok(commands.every((line) => line.startsWith("::warning::"))); + assert.ok( + !n.run.stdout + .split("\n") + .some((line) => /^::error/.test(line) || line.startsWith("::warning::again")), + ); + const setup = commands.find((line) => line.includes("did not run cleanly")); + assert.ok(setup.includes("Chrome died%0A: :error: :pwned %250A%0D: :warning: :again"), setup); + assert.ok(setup.includes("record without a usable seed (1%0A: :error: :pwned)")); + rmSync(n.dir, { recursive: true }); +}); + +test("summary and issue text cannot break out of code, the table or mention anyone", () => { + const nasty = "bad | cell @someone ```` and ``` \n@other"; + const n = night({ + files: { + "pin-1": [ + { seed: 8, features: ["a|b", "@someone", "x", "`tick`"], problems: [nasty] }, + { + seed: 9, + features: ["m|n", "@someone", "x", "`tick`", "p\\|q"], + problems: [], + unstableReference: 2, + }, + { seed: null, features: [], problems: [nasty] }, + ], + }, + }); + const body = readFileSync(join(n.dir, "gh.log.bodies"), "utf8").split("=====\n")[0]; + const fence = body.match(/^(`{4,})$/m); + assert.ok(fence, "the findings sit in a fence longer than any backtick run in the text"); + assert.equal(fence[1].length, 5); + const [outside, inside, after] = body.split(fence[1]); + assert.ok(inside.includes("") && inside.includes("@other")); + assert.ok(!outside.replace(/`[^`\n]*`/g, "").includes("@someone")); + assert.ok(!outside.includes("x<\/b>, 'tick'`/); + const row = n.summary.split("\n").find((line) => line.startsWith("| 9 ")); + assert.equal(row, "| 9 | `m\\|n, @someone, x, 'tick', p\\\\\\|q` | `pin` | 2.00 |"); + const setupFence = n.summary.match(/^(`{4,})$/m); + assert.ok(setupFence); + assert.ok(!n.summary.replace(/^(`{5})$[\s\S]*?^\1$/gm, "").includes("