mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
test(engine): nightly motion blur run over generated compositions (#7437)
* test(engine): nightly motion blur run over generated compositions A seeded generator of text-free compositions combines 19 features (nested canvas, overflow, clip path, scroll container, callbacks, a last-child rule) and checks the two pixel invariants on auto: outside the trail at two holds and at a moving time, inside the trail at a moving time. The runtime lane runs seeds 1 to 3; motion-blur-nightly.yml runs 200 a day on the Chrome 153 pin and on stable, and opens one issue per failing seed. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * test(engine): the nightly records every failure, ignores noisy references, caps its issues A crash, a timeout or a launch failure is recorded before the job goes red; a failing check whose blur-off reference disagrees with itself is reported as unstable instead of failing; the report job moves into a tested script that fetches open titles once, caps first, and opens one systemic issue above 5 seeds. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * test(engine): the nightly report resists injected failure text and keeps a crashed leg's record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix(engine): nightly report escapes backslashes, test strips tags to a fixpoint Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
4b3f70fb1b
commit
7bf19ab5f9
@@ -0,0 +1,117 @@
|
||||
name: motion-blur-nightly
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
HYPERFRAMES_NO_TELEMETRY: "1"
|
||||
|
||||
concurrency:
|
||||
group: motion-blur-nightly
|
||||
cancel-in-progress: false
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
seeds:
|
||||
description: "Seeds to run, a comma list or a-b range (default: today's 200)."
|
||||
type: string
|
||||
default: ""
|
||||
schedule:
|
||||
# Daily at 08:41 UTC, off the :00 pile and away from regression.yml's 06:17.
|
||||
- cron: "41 8 * * *"
|
||||
|
||||
jobs:
|
||||
plan:
|
||||
name: Plan seeds
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
chunks: ${{ steps.plan.outputs.chunks }}
|
||||
steps:
|
||||
# Today's 200 seeds are days-since-2026-01-01 * 200 onward, in 4 chunks of 50.
|
||||
- id: plan
|
||||
env:
|
||||
SEEDS: ${{ inputs.seeds }}
|
||||
run: |
|
||||
if [ -n "$SEEDS" ]; then
|
||||
chunks="$(jq -cn --arg seeds "$SEEDS" '[{id: "custom", seeds: $seeds}]')"
|
||||
else
|
||||
days=$(( ($(date -u +%s) - $(date -u -d 2026-01-01 +%s)) / 86400 ))
|
||||
start=$(( days * 200 ))
|
||||
chunks="$(jq -cn --argjson start "$start" '[range(0; 4) | ($start + . * 50) as $from | {id: ($from | tostring), seeds: "\($from)-\($from + 49)"}]')"
|
||||
fi
|
||||
echo "chunks=$chunks" >> "$GITHUB_OUTPUT"
|
||||
echo "$chunks"
|
||||
|
||||
generated:
|
||||
name: Generated compositions (${{ matrix.flavour }}, ${{ matrix.chunk.seeds }})
|
||||
needs: plan
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
flavour: [pin, stable]
|
||||
chunk: ${{ fromJSON(needs.plan.outputs.chunks) }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- id: workspace
|
||||
uses: ./.github/actions/setup-test-workspace
|
||||
with:
|
||||
engine: "true"
|
||||
headless-shell: "true"
|
||||
- id: pin
|
||||
if: matrix.flavour == 'pin'
|
||||
shell: bash
|
||||
run: |
|
||||
dir="$RUNNER_TEMP/headless-shell-pin"
|
||||
bunx @puppeteer/browsers@3.2.2 install chrome-headless-shell@153.0.8010.36 --path "$dir"
|
||||
echo "path=$(find "$dir" -type f -name chrome-headless-shell | head -n 1)" >> "$GITHUB_OUTPUT"
|
||||
- name: Run the generated compositions
|
||||
working-directory: packages/engine
|
||||
env:
|
||||
HYPERFRAMES_BROWSER_PATH: ${{ matrix.flavour == 'pin' && steps.pin.outputs.path || steps.workspace.outputs.headless-shell-path }}
|
||||
HF_BLUR_SEEDS: ${{ matrix.chunk.seeds }}
|
||||
HF_BLUR_FAILURES_FILE: ${{ github.workspace }}/blur-nightly-failures.json
|
||||
HF_BLUR_REQUIRED: "1"
|
||||
run: bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts
|
||||
- name: Keep the failing seeds
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: blur-nightly-failures-${{ matrix.flavour }}-${{ matrix.chunk.id }}
|
||||
path: blur-nightly-failures.json
|
||||
if-no-files-found: ignore
|
||||
|
||||
report:
|
||||
name: Report failures
|
||||
needs: [plan, generated]
|
||||
if: always() && needs.plan.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
pattern: blur-nightly-failures-*
|
||||
path: failures
|
||||
- name: Open issues, warn and summarize
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
GENERATED_RESULT: ${{ needs.generated.result }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: >-
|
||||
node scripts/ci/motion-blur-nightly-report.mjs
|
||||
--failures failures
|
||||
--generated-result "$GENERATED_RESULT"
|
||||
--date "$(date -u +%F)"
|
||||
--run-url "$RUN_URL"
|
||||
--summary "$GITHUB_STEP_SUMMARY"
|
||||
+1
-1
@@ -54,7 +54,7 @@
|
||||
"player:perf": "bun run --filter @hyperframes/player perf",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-cli-lazy-deps.test.mjs scripts/file-url-paths.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/clawhub-unlist-retired.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/ci-cache-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/check-core-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/lefthook-order.test.mjs scripts/inline-gsap.test.mjs scripts/ci/flake-gate.test.mjs scripts/ci/motion-blur-nightly-report.test.mjs scripts/ci/base-file.test.mjs scripts/ci/release-only-diff.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-theme-fallbacks.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs registry/components/echo-smear/echo-smear.test.mjs registry/examples/path-traced-3d/source/path-traced-3d.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
|
||||
"test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
|
||||
@@ -69,20 +69,29 @@ export function firstFive(d: Differential): string {
|
||||
return JSON.stringify(d.bad.slice(0, 5));
|
||||
}
|
||||
|
||||
export interface Measured {
|
||||
d: Differential;
|
||||
reference: Rgba;
|
||||
onFrame: Rgba;
|
||||
mask: Uint8Array;
|
||||
}
|
||||
|
||||
export async function measureOutside(
|
||||
browser: Browser,
|
||||
f: BlurFixture,
|
||||
t: number,
|
||||
tier?: Tier,
|
||||
): Promise<Differential> {
|
||||
): Promise<Measured> {
|
||||
const offFrames = await captureInOrder(await referencePage(f), neighbourTimes(t));
|
||||
const onFrame = await onFreshPage(browser, f, "on", (p) => captureFrame(p, t), tier);
|
||||
const reference = offFrames[INVARIANTS.radiusFrames];
|
||||
if (!reference) throw new Error("missing reference frame");
|
||||
return compareTiles(reference, onFrame, trailMask(offFrames, radiusPx(f)), "outside", {
|
||||
const mask = trailMask(offFrames, radiusPx(f));
|
||||
const d = compareTiles(reference, onFrame, mask, "outside", {
|
||||
block: INVARIANTS.block,
|
||||
maxMae: INVARIANTS.outsideMaxMae,
|
||||
});
|
||||
return { d, reference, onFrame, mask };
|
||||
}
|
||||
|
||||
export const caseName = (what: string, f: BlurFixture, t: number, tier?: Tier): string =>
|
||||
@@ -114,8 +123,11 @@ async function referencePage(f: BlurFixture): Promise<Page> {
|
||||
}
|
||||
|
||||
export async function stopChrome(): Promise<void> {
|
||||
await closeReference();
|
||||
await closeTestChrome(launched);
|
||||
try {
|
||||
await closeReference();
|
||||
} finally {
|
||||
await closeTestChrome(launched);
|
||||
}
|
||||
}
|
||||
|
||||
export function chrome(): Browser {
|
||||
@@ -134,7 +146,7 @@ export function defineOutsideCase(f: BlurFixture, t: number, tier?: Tier): void
|
||||
caseName("outside the trail, blur on equals blur off", f, t, tier),
|
||||
async (ctx) => {
|
||||
if (!supported) ctx.skip();
|
||||
const d = await measureOutside(chrome(), f, t, tier);
|
||||
const { d } = await measureOutside(chrome(), f, t, tier);
|
||||
expect(d.bad, `${f.name} ${firstFive(d)}`).toEqual([]);
|
||||
expect(d.checkedFraction).toBeGreaterThanOrEqual(INVARIANTS.minCheckedFraction);
|
||||
},
|
||||
@@ -154,22 +166,60 @@ function expectBlurredAwayFromSharp(
|
||||
);
|
||||
}
|
||||
|
||||
const shutterMean = async (page: Page, t: number): Promise<Rgba> =>
|
||||
meanFrame(
|
||||
await captureInOrder(page, shutterTimes(t), { subFrameDivisions: SUB_FRAME_DIVISIONS }),
|
||||
);
|
||||
|
||||
export async function measureInside(
|
||||
browser: Browser,
|
||||
f: BlurFixture,
|
||||
t: number,
|
||||
tier?: Tier,
|
||||
): Promise<Measured> {
|
||||
const samples = await captureInOrder(await referencePage(f), shutterTimes(t), {
|
||||
subFrameDivisions: SUB_FRAME_DIVISIONS,
|
||||
});
|
||||
const onFrame = await onFreshPage(browser, f, "on", (p) => captureFrame(p, t), tier);
|
||||
const reference = meanFrame(samples);
|
||||
const mask = trailMask(samples, radiusPx(f));
|
||||
const d = compareTiles(reference, onFrame, mask, "inside", {
|
||||
block: INVARIANTS.block,
|
||||
maxMae: INVARIANTS.insideMaxMae,
|
||||
});
|
||||
return { d, reference, onFrame, mask };
|
||||
}
|
||||
|
||||
export const REFERENCE_RECAPTURES = 3;
|
||||
|
||||
export async function blurOffDisagreement(
|
||||
browser: Browser,
|
||||
f: BlurFixture,
|
||||
region: "outside" | "inside",
|
||||
t: number,
|
||||
first: Measured,
|
||||
): Promise<number | undefined> {
|
||||
const maxMae = region === "outside" ? INVARIANTS.outsideMaxMae : INVARIANTS.insideMaxMae;
|
||||
for (let i = 0; i < REFERENCE_RECAPTURES; i++) {
|
||||
const again = await onFreshPage(browser, f, "off", (p) =>
|
||||
region === "outside" ? captureFrame(p, t) : shutterMean(p, t),
|
||||
);
|
||||
const d = compareTiles(first.reference, again, first.mask, region, {
|
||||
block: INVARIANTS.block,
|
||||
maxMae,
|
||||
});
|
||||
if (d.bad.length > 0) return d.worstMae;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function defineInsideCase(f: BlurFixture, t: number, tier?: Tier): void {
|
||||
it(
|
||||
caseName("inside the trail, blur on matches the shutter average", f, t, tier),
|
||||
async (ctx) => {
|
||||
if (!supported) ctx.skip();
|
||||
const offPage = await referencePage(f);
|
||||
const samples = await captureInOrder(offPage, shutterTimes(t), {
|
||||
subFrameDivisions: SUB_FRAME_DIVISIONS,
|
||||
});
|
||||
const sharp = await captureFrame(offPage, t);
|
||||
const onFrame = await onFreshPage(chrome(), f, "on", (p) => captureFrame(p, t), tier);
|
||||
const mask = trailMask(samples, radiusPx(f));
|
||||
const d = compareTiles(meanFrame(samples), onFrame, mask, "inside", {
|
||||
block: INVARIANTS.block,
|
||||
maxMae: INVARIANTS.insideMaxMae,
|
||||
});
|
||||
const { d, onFrame, mask } = await measureInside(chrome(), f, t, tier);
|
||||
const sharp = await captureFrame(await referencePage(f), t);
|
||||
expect(d.checkedBlocks).toBeGreaterThan(0);
|
||||
expect(d.bad, `${f.name} ${firstFive(d)}`).toEqual([]);
|
||||
expectBlurredAwayFromSharp(f, sharp, onFrame, mask);
|
||||
|
||||
@@ -34,7 +34,7 @@ const SLIDE = 0.4;
|
||||
const frame = (k: number): number => k / 30;
|
||||
const frames = (...ks: number[]): number[] => ks.map(frame);
|
||||
|
||||
interface Spec {
|
||||
export interface Spec {
|
||||
name: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
@@ -114,7 +114,7 @@ function compose(spec: Spec, blur: "on" | "off", tier?: "capture" | "filter"): s
|
||||
</body></html>`;
|
||||
}
|
||||
|
||||
function fixture(spec: Spec): BlurFixture {
|
||||
export function fixture(spec: Spec): BlurFixture {
|
||||
return {
|
||||
name: spec.name,
|
||||
width: spec.width ?? 320,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
EXCLUDED_FEATURES,
|
||||
FEATURES,
|
||||
generateComposition,
|
||||
parseSeeds,
|
||||
type Feature,
|
||||
} from "./generate.js";
|
||||
|
||||
const SEEDS = parseSeeds("1-200");
|
||||
|
||||
const GSAP_SCRIPT = /<script>\/\*![\s\S]*?<\/script>/;
|
||||
|
||||
const withoutMatches = (text: string, pattern: RegExp): string => {
|
||||
let current = text;
|
||||
for (
|
||||
let next = current.replace(pattern, "");
|
||||
next !== current;
|
||||
next = current.replace(pattern, "")
|
||||
) {
|
||||
current = next;
|
||||
}
|
||||
return current;
|
||||
};
|
||||
|
||||
const visibleText = (html: string): string =>
|
||||
withoutMatches(withoutMatches(html, /<(script|style)[\s\S]*?<\/\1>/g), /<[^>]*>/g).trim();
|
||||
|
||||
describe("generateComposition", () => {
|
||||
it("is deterministic per seed", () => {
|
||||
for (const seed of [1, 17, 200]) {
|
||||
const a = generateComposition(seed);
|
||||
const b = generateComposition(seed);
|
||||
expect(b.features).toEqual(a.features);
|
||||
expect(b.name).toBe(a.name);
|
||||
expect(b.checkpoints).toEqual(a.checkpoints);
|
||||
expect(b.html("on")).toBe(a.html("on"));
|
||||
expect(b.html("off")).toBe(a.html("off"));
|
||||
}
|
||||
});
|
||||
|
||||
it("varies with the seed", () => {
|
||||
const documents = new Set(SEEDS.map((seed) => generateComposition(seed).html("on")));
|
||||
expect(documents.size).toBeGreaterThan(150);
|
||||
});
|
||||
|
||||
it("uses every feature within 200 seeds", () => {
|
||||
const seen = new Set<Feature>();
|
||||
for (const seed of SEEDS)
|
||||
for (const feature of generateComposition(seed).features) seen.add(feature);
|
||||
expect([...FEATURES].filter((feature) => !seen.has(feature))).toEqual([]);
|
||||
});
|
||||
|
||||
it("holds each composition to 1 to 4 features at 320x180, 2 s, 30 fps", () => {
|
||||
for (const seed of SEEDS) {
|
||||
const f = generateComposition(seed);
|
||||
expect(f.features.length).toBeGreaterThanOrEqual(1);
|
||||
expect(f.features.length).toBeLessThanOrEqual(4);
|
||||
expect(new Set(f.features).size).toBe(f.features.length);
|
||||
expect([f.width, f.height, f.durationSeconds, f.fps, f.dpr]).toEqual([320, 180, 2, 30, 1]);
|
||||
expect(f.checkpoints).toHaveLength(2);
|
||||
expect(f.moving).toHaveLength(1);
|
||||
expect(f.name).toBe(`seed ${seed} [${f.features.join(", ")}]`);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the features known wrong on the filter tier out", () => {
|
||||
expect(EXCLUDED_FEATURES).toEqual({ ownClipPath: "E", visibilityToggle: "O" });
|
||||
for (const seed of SEEDS) {
|
||||
const html = generateComposition(seed).html("on").replace(GSAP_SCRIPT, "");
|
||||
expect(html).not.toMatch(/#card[^{]*\{[^}]*(clip-path|mask)/);
|
||||
expect(html).not.toMatch(/visibility/);
|
||||
}
|
||||
});
|
||||
|
||||
it("draws no text", () => {
|
||||
for (const seed of SEEDS) expect(visibleText(generateComposition(seed).html("on"))).toBe("");
|
||||
});
|
||||
|
||||
it("pins no tier on the blur-on page", () => {
|
||||
for (const seed of [1, 2, 3]) {
|
||||
expect(generateComposition(seed).html("on")).not.toContain("data-motion-blur-tier");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseSeeds", () => {
|
||||
it("reads lists and ranges", () => {
|
||||
expect(parseSeeds("1,2,3")).toEqual([1, 2, 3]);
|
||||
expect(parseSeeds("5-8")).toEqual([5, 6, 7, 8]);
|
||||
expect(parseSeeds("4,10-11")).toEqual([4, 10, 11]);
|
||||
expect(() => parseSeeds("x")).toThrow("bad seed list");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,355 @@
|
||||
import { fixture, type BlurFixture } from "./fixtures.js";
|
||||
|
||||
export const FEATURES = [
|
||||
"transform",
|
||||
"leftTop",
|
||||
"rotation",
|
||||
"overflowParent",
|
||||
"scrollContainer",
|
||||
"canvas2d",
|
||||
"webgl",
|
||||
"svgClip",
|
||||
"mask",
|
||||
"onUpdateMover",
|
||||
"tlCall",
|
||||
"onceGuard",
|
||||
"cssKeyframes",
|
||||
"nesting",
|
||||
"pseudoElement",
|
||||
"panelInFront",
|
||||
"blendMode",
|
||||
"fixedChild",
|
||||
"lastChild",
|
||||
] as const;
|
||||
|
||||
export type Feature = (typeof FEATURES)[number];
|
||||
|
||||
export const EXCLUDED_FEATURES = { ownClipPath: "E", visibilityToggle: "O" } as const;
|
||||
|
||||
export type GeneratedFixture = BlurFixture & { seed: number; features: readonly Feature[] };
|
||||
|
||||
const WIDTH = 320;
|
||||
const SLOT_WIDTH = 96;
|
||||
const SLOT_HEIGHT = 52;
|
||||
const SLOTS = [8, 112, 216].flatMap((x) => [8, 120].map((y) => ({ x, y })));
|
||||
const CHECKPOINT_FRAMES = [15, 30, 45];
|
||||
const MOVING_FRAMES = [6, 54];
|
||||
const ONCE_FRAMES = [20, 23, 26, 35, 38, 41];
|
||||
|
||||
interface Slot {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
interface Part {
|
||||
css?: string;
|
||||
markup?: string;
|
||||
script?: string;
|
||||
card?: string;
|
||||
content?: string;
|
||||
last?: string;
|
||||
}
|
||||
|
||||
interface Draw {
|
||||
slot(): Slot;
|
||||
pick<T>(items: readonly T[]): T;
|
||||
flip(): boolean;
|
||||
}
|
||||
|
||||
const hash = (seed: number): number => Math.imul(seed + 0x9e3779b9, 0x85ebca6b) >>> 0;
|
||||
|
||||
function lcg(seed: number): () => number {
|
||||
let state = hash(seed);
|
||||
const next = (): number => {
|
||||
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
||||
return state / 2 ** 32;
|
||||
};
|
||||
for (let i = 0; i < 4; i++) next();
|
||||
return next;
|
||||
}
|
||||
|
||||
function shuffled<T>(items: readonly T[], next: () => number): T[] {
|
||||
const out = [...items];
|
||||
for (let i = out.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(next() * (i + 1));
|
||||
[out[i], out[j]] = [out[j] as T, out[i] as T];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const props = (values: Record<string, number | string>): string => JSON.stringify(values);
|
||||
|
||||
const tween = (sel: string, from: object, to: object, ease: string): string =>
|
||||
`tl.fromTo(${JSON.stringify(sel)},${JSON.stringify(from)},${JSON.stringify({ ...to, duration: 0.4, ease })},0);`;
|
||||
|
||||
function inOut(
|
||||
sel: string,
|
||||
from: Record<string, number | string>,
|
||||
rest: Record<string, number | string>,
|
||||
away: Record<string, number | string>,
|
||||
): string {
|
||||
return `${tween(sel, from, rest, "power2.out")}
|
||||
tl.to(${JSON.stringify(sel)},${props({ ...away, duration: 0.4, ease: "power2.in" })},1.6);`;
|
||||
}
|
||||
|
||||
const el = (id: string, style: string, inner = ""): string =>
|
||||
`<div id="${id}" style="position:absolute;z-index:1;${style}">${inner}</div>`;
|
||||
|
||||
const at = (s: Slot, width = 30, height = 30): string =>
|
||||
`left:${s.x}px;top:${s.y}px;width:${width}px;height:${height}px`;
|
||||
|
||||
const fromLeft = (s: Slot): number => -(s.x + 120);
|
||||
const toRight = (s: Slot): number => WIDTH - s.x + 40;
|
||||
|
||||
const frame = (k: number): number => k / 30;
|
||||
|
||||
const PALETTE = ["#1c2330", "#2b3a55", "#3d2b55", "#2b553d"];
|
||||
|
||||
const wrapped = (id: string, s: Slot, wrapperStyle: string): Part => ({
|
||||
markup: el(
|
||||
`${id}w`,
|
||||
`${at(s, SLOT_WIDTH, SLOT_HEIGHT)};background:#1f2a44;${wrapperStyle}`,
|
||||
el(id, "left:30px;top:10px;width:30px;height:30px;background:#ffd166"),
|
||||
),
|
||||
script: inOut(`#${id}`, { x: -150 }, { x: 0 }, { x: 150 }),
|
||||
});
|
||||
|
||||
const BUILDERS: Record<Feature, (draw: Draw) => Part> = {
|
||||
transform: () => ({
|
||||
script: `${tween("#card", { y: 30, scale: 0.6 }, { y: 0, scale: 1 }, "power2.out")}
|
||||
tl.to("#card",${props({ y: -30, scale: 0.6, duration: 0.4, ease: "power2.in" })},1.6);`,
|
||||
}),
|
||||
leftTop: ({ slot }) => {
|
||||
const s = slot();
|
||||
return {
|
||||
markup: el("lt", `${at(s)};background:#e63946`),
|
||||
script: inOut(
|
||||
"#lt",
|
||||
{ left: fromLeft(s), top: 130 },
|
||||
{ left: s.x, top: s.y },
|
||||
{ left: 340, top: 130 },
|
||||
),
|
||||
};
|
||||
},
|
||||
rotation: () => ({
|
||||
script: `${tween("#card", { rotation: -90 }, { rotation: 0 }, "power2.out")}
|
||||
tl.to("#card",${props({ rotation: 90, duration: 0.4, ease: "power2.in" })},1.6);`,
|
||||
}),
|
||||
overflowParent: ({ slot }) => {
|
||||
const s = slot();
|
||||
return {
|
||||
markup: el(
|
||||
"ovw",
|
||||
`${at(s, SLOT_WIDTH, SLOT_HEIGHT)};overflow:hidden;border-radius:12px;background:#223`,
|
||||
el("ov", "left:8px;top:10px;width:30px;height:30px;background:#e63946"),
|
||||
),
|
||||
script: inOut("#ov", { x: -60 }, { x: 20 }, { x: 140 }),
|
||||
};
|
||||
},
|
||||
scrollContainer: ({ slot }) => {
|
||||
const s = slot();
|
||||
const rows = Array.from(
|
||||
{ length: 12 },
|
||||
(_, i) => `<div style="height:20px;background:${PALETTE[i % PALETTE.length]}"></div>`,
|
||||
).join("");
|
||||
return {
|
||||
markup: el(
|
||||
"sc",
|
||||
`${at(s, SLOT_WIDTH, SLOT_HEIGHT)};overflow:auto;scrollbar-width:none`,
|
||||
rows,
|
||||
),
|
||||
script: `const scrolled = { v: 0 };
|
||||
tl.to(scrolled,{ v: 100, duration: 0.4, ease: "power2.out", onUpdate() { document.getElementById("sc").scrollTop = scrolled.v } },0);`,
|
||||
};
|
||||
},
|
||||
canvas2d: ({ slot, flip }) => {
|
||||
const s = slot();
|
||||
const canvas = `<canvas id="cv" width="${SLOT_WIDTH}" height="${SLOT_HEIGHT}" style="position:absolute;z-index:1;left:${s.x}px;top:${s.y}px"></canvas>`;
|
||||
return {
|
||||
markup: flip()
|
||||
? `<section style="position:absolute;inset:0;overflow:hidden;will-change:transform,filter,opacity;z-index:1">${canvas}</section>`
|
||||
: canvas,
|
||||
script: `const cvx = document.getElementById("cv").getContext("2d");
|
||||
const draw = () => {
|
||||
const p = Math.min(1, tl.time() / 0.4);
|
||||
cvx.fillStyle = "#2a9d8f";
|
||||
cvx.fillRect(0, 0, ${SLOT_WIDTH}, ${SLOT_HEIGHT});
|
||||
cvx.fillStyle = "#ffd166";
|
||||
cvx.fillRect(4, 4, 88 * p, 20);
|
||||
cvx.strokeStyle = "#fff";
|
||||
cvx.strokeRect(2, 28, 92, 22);
|
||||
};
|
||||
tl.to({}, { duration: 2, onUpdate: draw }, 0);`,
|
||||
};
|
||||
},
|
||||
webgl: ({ slot }) => {
|
||||
const s = slot();
|
||||
return {
|
||||
markup: `<canvas id="gl" width="${SLOT_WIDTH}" height="${SLOT_HEIGHT}" style="position:absolute;z-index:1;left:${s.x}px;top:${s.y}px"></canvas>
|
||||
<script>(() => {
|
||||
const g = document.getElementById("gl").getContext("webgl");
|
||||
g.clearColor(0.05, 0.2, 0.5, 0.6);
|
||||
g.clear(g.COLOR_BUFFER_BIT);
|
||||
})();</script>`,
|
||||
};
|
||||
},
|
||||
svgClip: ({ slot, flip }) => {
|
||||
const s = slot();
|
||||
const defs = `<svg width="0" height="0" style="position:absolute" aria-hidden="true"><defs><clipPath id="genClip" clipPathUnits="objectBoundingBox"><polygon points="0.5,0 1,0.5 0.5,1 0,0.5"/></clipPath></defs></svg>`;
|
||||
const clip = "clip-path:url(#genClip)";
|
||||
if (flip()) {
|
||||
const part = wrapped("sv", s, clip);
|
||||
return { ...part, markup: defs + part.markup };
|
||||
}
|
||||
return {
|
||||
markup:
|
||||
defs +
|
||||
el(
|
||||
"sibling",
|
||||
`${at(s, 60, 44)};background:linear-gradient(135deg,#ff7a59,#ffd166);${clip}`,
|
||||
),
|
||||
};
|
||||
},
|
||||
mask: ({ slot, flip }) => {
|
||||
const s = slot();
|
||||
const mask =
|
||||
"-webkit-mask-image:radial-gradient(circle,#000 40%,transparent 75%);mask-image:radial-gradient(circle,#000 40%,transparent 75%)";
|
||||
return flip()
|
||||
? wrapped("mk", s, mask)
|
||||
: {
|
||||
markup: el(
|
||||
"masked",
|
||||
`${at(s, SLOT_WIDTH, SLOT_HEIGHT)};background:linear-gradient(90deg,#ff7a59,#06d6a0);${mask}`,
|
||||
),
|
||||
};
|
||||
},
|
||||
onUpdateMover: ({ slot }) => {
|
||||
const s = slot();
|
||||
return {
|
||||
markup: el("ou", `${at(s)};background:#9b5de5;transform:translate(${fromLeft(s)}px,0)`),
|
||||
script: `const driven = { v: 0 };
|
||||
const place = () => {
|
||||
const v = driven.v;
|
||||
document.getElementById("ou").style.transform = "translate(" + (v <= 1 ? ${fromLeft(s)} * (1 - v) : ${toRight(s)} * (v - 1)) + "px,0)";
|
||||
};
|
||||
tl.to(driven,{ v: 1, duration: 0.4, ease: "power2.out", onUpdate: place },0)
|
||||
.to(driven,{ v: 2, duration: 0.4, ease: "power2.in", onUpdate: place },1.6);`,
|
||||
};
|
||||
},
|
||||
tlCall: ({ slot, flip }) => {
|
||||
const s = slot();
|
||||
const byClass = flip();
|
||||
return {
|
||||
css: byClass ? "#fl.on { background: #06d6a0 }" : "",
|
||||
markup: el("fl", `${at(s)};background:#3a86ff`),
|
||||
script: byClass
|
||||
? `const fl = document.getElementById("fl");
|
||||
tl.call(() => fl.classList.add("on"), null, 0.7).call(() => fl.classList.remove("on"), null, 1.3);`
|
||||
: `const fl = document.getElementById("fl");
|
||||
tl.call(() => { fl.style.background = "#06d6a0" }, null, 0.7).call(() => { fl.style.background = "#3a86ff" }, null, 1.3);`,
|
||||
};
|
||||
},
|
||||
onceGuard: ({ slot, pick }) => {
|
||||
const s = slot();
|
||||
return {
|
||||
markup: el("og", `${at(s)};background:#ef476f`),
|
||||
script: `let fired = false;
|
||||
tl.call(() => { if (fired) return; fired = true; document.getElementById("og").style.background = "#ffd166" }, null, ${frame(pick(ONCE_FRAMES))});`,
|
||||
};
|
||||
},
|
||||
cssKeyframes: ({ slot }) => {
|
||||
const s = slot();
|
||||
return {
|
||||
css: `@keyframes genSlide {
|
||||
0% { transform: translateX(${fromLeft(s)}px); animation-timing-function: ease-out }
|
||||
20% { transform: translateX(0) }
|
||||
80% { transform: translateX(0); animation-timing-function: ease-in }
|
||||
100% { transform: translateX(${toRight(s)}px) }
|
||||
}`,
|
||||
markup: el("ck", `${at(s)};background:#118ab2;animation:genSlide 2s linear both`),
|
||||
};
|
||||
},
|
||||
nesting: ({ slot }) => {
|
||||
const s = slot();
|
||||
return {
|
||||
markup: el(
|
||||
"no",
|
||||
`${at(s, SLOT_WIDTH, SLOT_HEIGHT)};background:#264653`,
|
||||
`<div id="ni" style="position:absolute;left:6px;top:10px;width:30px;height:30px;background:#e9c46a"></div>`,
|
||||
),
|
||||
script: `${inOut("#no", { x: fromLeft(s) }, { x: 0 }, { x: toRight(s) })}
|
||||
tl.to("#ni",{ x: 50, duration: 0.4, ease: "power2.inOut" },0);`,
|
||||
};
|
||||
},
|
||||
pseudoElement: ({ pick }) => {
|
||||
const before = `#card::before { content: ""; position: absolute; left: -34px; top: 8px; width: 24px; height: 24px; border-radius: 50%; background: #ffd166 }`;
|
||||
const after = `#card::after { content: ""; position: absolute; left: 90px; top: 8px; width: 24px; height: 24px; border-radius: 50%; background: #06d6a0 }`;
|
||||
return { css: pick([before, after, `${before} ${after}`]) };
|
||||
},
|
||||
panelInFront: () => ({
|
||||
markup: `<div style="position:absolute;left:150px;top:60px;width:34px;height:54px;background:#ffd166;border:3px solid #000;z-index:5"></div>`,
|
||||
}),
|
||||
blendMode: () => ({
|
||||
markup: `<div style="position:absolute;left:215px;top:60px;width:50px;height:60px;background:#fff;mix-blend-mode:difference;z-index:5"></div>`,
|
||||
}),
|
||||
fixedChild: () => ({
|
||||
content: `<div style="position:fixed;left:6px;top:6px;width:24px;height:12px;background:#ffd166"></div>`,
|
||||
}),
|
||||
lastChild: () => ({
|
||||
css: "#root > :last-child { outline: 3px solid #ffd166 }",
|
||||
last: `<div id="last" style="position:absolute;left:290px;top:150px;width:20px;height:20px;background:#000"></div>`,
|
||||
}),
|
||||
};
|
||||
|
||||
function chooseFeatures(next: () => number): Feature[] {
|
||||
const count = 1 + Math.floor(next() * 4);
|
||||
const chosen = new Set(shuffled(FEATURES, next).slice(0, count));
|
||||
return FEATURES.filter((feature) => chosen.has(feature));
|
||||
}
|
||||
|
||||
export function generateComposition(seed: number): GeneratedFixture {
|
||||
const next = lcg(seed);
|
||||
const features = chooseFeatures(next);
|
||||
const free = shuffled(SLOTS, next);
|
||||
const draw: Draw = {
|
||||
slot: () => {
|
||||
const taken = free.pop();
|
||||
if (!taken) throw new Error("out of slots");
|
||||
return taken;
|
||||
},
|
||||
pick: <T>(items: readonly T[]): T => items[Math.floor(next() * items.length)] as T,
|
||||
flip: () => next() < 0.5,
|
||||
};
|
||||
const parts = features.map((feature) => BUILDERS[feature](draw));
|
||||
const join = (key: "css" | "markup" | "script" | "content" | "last"): string =>
|
||||
parts.map((part) => part[key] ?? "").join("\n");
|
||||
const checkpoints = shuffled(CHECKPOINT_FRAMES, next)
|
||||
.slice(0, 2)
|
||||
.sort((a, b) => a - b)
|
||||
.map(frame);
|
||||
const moving = [frame(draw.pick(MOVING_FRAMES))];
|
||||
const base = fixture({
|
||||
name: `seed ${seed} [${features.join(", ")}]`,
|
||||
captureTier: false,
|
||||
over: true,
|
||||
cardStyle: `z-index:2;${parts.map((part) => part.card ?? "").join("")}`,
|
||||
cardContent: join("content"),
|
||||
extra: join("markup") + join("last"),
|
||||
style: join("css"),
|
||||
script: join("script"),
|
||||
checkpoints,
|
||||
moving,
|
||||
expectOn: { tier: "filter", reasonCode: "tier-auto" },
|
||||
});
|
||||
return { ...base, seed, features };
|
||||
}
|
||||
|
||||
export function parseSeeds(spec: string): number[] {
|
||||
return spec.split(",").flatMap((part) => {
|
||||
const [from, to = from] = part.split("-").map((n) => Number.parseInt(n, 10));
|
||||
if (from === undefined || to === undefined || Number.isNaN(from) || Number.isNaN(to)) {
|
||||
throw new Error(`bad seed list: ${spec}`);
|
||||
}
|
||||
return Array.from({ length: to - from + 1 }, (_, i) => from + i);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { Differential } from "./__fixtures__/blurDifferential.js";
|
||||
import { hasBrowser } from "./__fixtures__/motionBlur/blurPage.js";
|
||||
import {
|
||||
CASE_TIMEOUT_MS,
|
||||
blurOffDisagreement,
|
||||
chrome,
|
||||
firstFive,
|
||||
isSupported,
|
||||
measureInside,
|
||||
type Measured,
|
||||
measureOutside,
|
||||
startChrome,
|
||||
stopChrome,
|
||||
} from "./__fixtures__/motionBlur/differentialPage.js";
|
||||
import {
|
||||
generateComposition,
|
||||
parseSeeds,
|
||||
type GeneratedFixture,
|
||||
} from "./__fixtures__/motionBlur/generate.js";
|
||||
import { INVARIANTS } from "./__fixtures__/motionBlur/thresholds.js";
|
||||
|
||||
const SEEDS = parseSeeds(process.env.HF_BLUR_SEEDS ?? "1,2,3");
|
||||
const FAILURES_FILE = process.env.HF_BLUR_FAILURES_FILE;
|
||||
const REQUIRED = process.env.HF_BLUR_REQUIRED === "1";
|
||||
|
||||
interface KnownBad {
|
||||
finding: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
const KNOWN_BAD: Readonly<Record<number, KnownBad>> = {};
|
||||
|
||||
interface FailureRecord {
|
||||
seed: number | null;
|
||||
features: readonly string[];
|
||||
problems: readonly string[];
|
||||
unstableReference?: number;
|
||||
}
|
||||
|
||||
const records: FailureRecord[] = [];
|
||||
|
||||
const worst = (d: Differential): string =>
|
||||
`${d.bad.length} bad tiles, worst MAE ${d.worstMae.toFixed(1)}, first ${firstFive(d)}`;
|
||||
|
||||
interface Outcome {
|
||||
problems: string[];
|
||||
unstable: number[];
|
||||
}
|
||||
|
||||
async function judge(
|
||||
f: GeneratedFixture,
|
||||
region: "outside" | "inside",
|
||||
t: number,
|
||||
first: Measured,
|
||||
outcome: Outcome,
|
||||
): Promise<void> {
|
||||
if (first.d.bad.length === 0) return;
|
||||
const disagreement = await blurOffDisagreement(chrome(), f, region, t, first);
|
||||
if (disagreement === undefined) {
|
||||
outcome.problems.push(`${region} the trail at ${t}: ${worst(first.d)}`);
|
||||
} else {
|
||||
outcome.unstable.push(disagreement);
|
||||
}
|
||||
}
|
||||
|
||||
async function check(f: GeneratedFixture): Promise<Outcome> {
|
||||
const outcome: Outcome = { problems: [], unstable: [] };
|
||||
const times = [...f.checkpoints, ...f.moving].sort((a, b) => a - b);
|
||||
for (const t of times) {
|
||||
const moving = f.moving.includes(t);
|
||||
const outside = await measureOutside(chrome(), f, t);
|
||||
await judge(f, "outside", t, outside, outcome);
|
||||
if (!moving && outside.d.checkedFraction < INVARIANTS.minCheckedFraction) {
|
||||
outcome.problems.push(
|
||||
`outside the trail at ${t}: only ${outside.d.checkedFraction} of the frame was checked`,
|
||||
);
|
||||
}
|
||||
if (outside.d.checkedBlocks === 0) {
|
||||
outcome.problems.push(`outside the trail at ${t}: no tile was checked`);
|
||||
}
|
||||
if (!moving) continue;
|
||||
const inside = await measureInside(chrome(), f, t);
|
||||
await judge(f, "inside", t, inside, outcome);
|
||||
if (inside.d.checkedBlocks === 0) {
|
||||
outcome.problems.push(`inside the trail at ${t}: no tile was checked`);
|
||||
}
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
function defineSeedCase(seed: number): void {
|
||||
const f = generateComposition(seed);
|
||||
const pinned = KNOWN_BAD[seed];
|
||||
it(
|
||||
pinned ? `KNOWN BAD on main: ${pinned.finding}: ${pinned.reason}: ${f.name}` : f.name,
|
||||
async (ctx) => {
|
||||
if (!isSupported()) ctx.skip();
|
||||
let found: string[] = [];
|
||||
ctx.onTestFailed(() => {
|
||||
const thrown = ctx.task.result?.errors?.map((e) => String(e.message)) ?? [];
|
||||
records.push({ seed, features: f.features, problems: found.length ? found : thrown });
|
||||
});
|
||||
const { problems, unstable } = await check(f);
|
||||
if (pinned) {
|
||||
expect(
|
||||
problems.length,
|
||||
`${pinned.finding} looks fixed: delete seed ${seed} from KNOWN_BAD`,
|
||||
).toBeGreaterThan(0);
|
||||
return;
|
||||
}
|
||||
found = problems;
|
||||
if (problems.length === 0 && unstable.length > 0) {
|
||||
records.push({
|
||||
seed,
|
||||
features: f.features,
|
||||
problems: [],
|
||||
unstableReference: Math.max(...unstable),
|
||||
});
|
||||
}
|
||||
expect(problems, `${f.name}: rerun with HF_BLUR_SEEDS=${seed}`).toEqual([]);
|
||||
},
|
||||
CASE_TIMEOUT_MS * 2,
|
||||
);
|
||||
}
|
||||
|
||||
async function startRequiredChrome(): Promise<void> {
|
||||
if (!hasBrowser) throw new Error("HF_BLUR_REQUIRED is set but no Chrome was found");
|
||||
await startChrome();
|
||||
if (!isSupported())
|
||||
throw new Error("HF_BLUR_REQUIRED is set but this Chrome lacks drawElementImage");
|
||||
}
|
||||
|
||||
describe.skipIf(!hasBrowser && !REQUIRED)(
|
||||
"capture motion blur over generated compositions in Chrome",
|
||||
() => {
|
||||
beforeAll(async () => {
|
||||
try {
|
||||
await (REQUIRED ? startRequiredChrome() : startChrome());
|
||||
} catch (error) {
|
||||
records.push({ seed: null, features: [], problems: [`Chrome did not start: ${error}`] });
|
||||
throw error;
|
||||
}
|
||||
}, 30_000);
|
||||
afterAll(async () => {
|
||||
try {
|
||||
if (FAILURES_FILE) writeFileSync(FAILURES_FILE, JSON.stringify(records, null, 2));
|
||||
} finally {
|
||||
await stopChrome();
|
||||
}
|
||||
});
|
||||
|
||||
for (const seed of SEEDS) defineSeedCase(seed);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env node
|
||||
// motion-blur-nightly-report.mjs --failures <dir> --generated-result <result> --date <YYYY-MM-DD> ...
|
||||
// Reads the nightly's per-job failure files and opens the issues, warnings and summary they call for.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
appendFileSync,
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ARTIFACT_PREFIX = "blur-nightly-failures-";
|
||||
const FILE = "blur-nightly-failures.json";
|
||||
const LABELS = ["bug", "area/quality"];
|
||||
const DETAILED_SEEDS = 50;
|
||||
const MAX_PROBLEM_CHARS = 400;
|
||||
|
||||
export const DEFAULTS = { maxIssues: 25, systemicAbove: 5 };
|
||||
|
||||
const clip = (text, limit) => (text.length > limit ? `${text.slice(0, limit)}...` : text);
|
||||
|
||||
const escapeData = (text) =>
|
||||
String(text).replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
|
||||
const inertColons = (text) => text.replace(/:(?=:)/g, ": ");
|
||||
const escapeProperty = (text) => escapeData(text).replace(/:/g, "%3A").replace(/,/g, "%2C");
|
||||
|
||||
export function workflowCommand(name, message, properties = {}) {
|
||||
const props = Object.entries(properties)
|
||||
.map(([key, value]) => `${key}=${escapeProperty(value)}`)
|
||||
.join(",");
|
||||
return `::${name}${props ? ` ${props}` : ""}::${inertColons(escapeData(message))}`;
|
||||
}
|
||||
|
||||
const warning = (message) => workflowCommand("warning", message);
|
||||
|
||||
const inline = (text) =>
|
||||
`\`${String(text)
|
||||
.replace(/\\/g, "\\\\")
|
||||
.replace(/`/g, "'")
|
||||
.replace(/\r?\n|\r/g, " ")
|
||||
.replace(/\|/g, "\\|")}\``;
|
||||
|
||||
function fenced(text) {
|
||||
const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length));
|
||||
const ticks = "`".repeat(Math.max(3, longest + 1));
|
||||
return [ticks, text, ticks].join("\n");
|
||||
}
|
||||
|
||||
const flavourList = (flavours) => [...new Set(flavours)].map(inline).join(", ");
|
||||
|
||||
function readFailureFile(file, flavour) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(file, "utf8")).map((record) => ({ ...record, flavour }));
|
||||
} catch (error) {
|
||||
return [{ seed: null, features: [], problems: [`unreadable ${file}: ${error}`], flavour }];
|
||||
}
|
||||
}
|
||||
|
||||
export function readRecords(dir) {
|
||||
if (!existsSync(dir)) return [];
|
||||
return readdirSync(dir).flatMap((name) => {
|
||||
const file = join(dir, name, FILE);
|
||||
if (!name.startsWith(ARTIFACT_PREFIX) || !existsSync(file)) return [];
|
||||
return readFailureFile(file, name.slice(ARTIFACT_PREFIX.length).split("-")[0]);
|
||||
});
|
||||
}
|
||||
|
||||
const isSeed = (seed) => Number.isSafeInteger(seed) && seed >= 0;
|
||||
|
||||
export function classify(records) {
|
||||
const failing = new Map();
|
||||
const unstable = [];
|
||||
const setup = [];
|
||||
for (const record of records) {
|
||||
const { flavour } = record;
|
||||
const problems = Array.isArray(record.problems) ? record.problems.map(String) : [];
|
||||
const features = Array.isArray(record.features) ? record.features.map(String) : [];
|
||||
if (record.seed === null || record.seed === undefined) {
|
||||
setup.push({ flavour, problems: problems.length ? problems : ["record without a seed"] });
|
||||
} else if (!isSeed(record.seed)) {
|
||||
setup.push({
|
||||
flavour,
|
||||
problems: [`record without a usable seed (${record.seed})`, ...problems],
|
||||
});
|
||||
} else if (problems.length > 0) {
|
||||
const entry = failing.get(record.seed) ?? {
|
||||
seed: record.seed,
|
||||
features,
|
||||
flavours: [],
|
||||
problems: [],
|
||||
};
|
||||
entry.flavours.push(flavour);
|
||||
entry.problems.push(...problems.map((problem) => `[${flavour}] ${problem}`));
|
||||
failing.set(record.seed, entry);
|
||||
} else if (typeof record.unstableReference === "number") {
|
||||
unstable.push({ seed: record.seed, features, flavour, mae: record.unstableReference });
|
||||
}
|
||||
}
|
||||
return {
|
||||
failing: [...failing.values()].sort((a, b) => a.seed - b.seed),
|
||||
unstable: unstable.sort((a, b) => a.seed - b.seed),
|
||||
setup,
|
||||
};
|
||||
}
|
||||
|
||||
const seedTitle = (seed) => `motion blur nightly: seed ${seed}`;
|
||||
const systemicTitle = (date) => `motion blur nightly: systemic failure ${date}`;
|
||||
const features = (entry) => entry.features.join(", ") || "none";
|
||||
const flavoursOf = (entries) => entries.flatMap((entry) => entry.flavours);
|
||||
|
||||
export function seedBody(entry, runUrl) {
|
||||
return [
|
||||
`Seed ${entry.seed} fails the pixel invariants on \`auto\`.`,
|
||||
"",
|
||||
`- Features: ${inline(features(entry))}`,
|
||||
`- Chrome flavours that failed: ${flavourList(entry.flavours)}`,
|
||||
`- Run: ${runUrl}`,
|
||||
"",
|
||||
"Reproduce from `packages/engine`:",
|
||||
"",
|
||||
"```sh",
|
||||
`HF_BLUR_SEEDS=${entry.seed} bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts`,
|
||||
"```",
|
||||
"",
|
||||
"Findings, by flavour:",
|
||||
"",
|
||||
fenced(
|
||||
entry.problems
|
||||
.slice(0, 10)
|
||||
.map((problem) => clip(problem, MAX_PROBLEM_CHARS))
|
||||
.join("\n"),
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function systemicBody(entries, date, runUrl) {
|
||||
const shown = entries.slice(0, DETAILED_SEEDS);
|
||||
const rest = entries.slice(DETAILED_SEEDS).map((entry) => entry.seed);
|
||||
const lines = shown.map(
|
||||
(entry) =>
|
||||
`${entry.seed} [${features(entry)}] (${[...new Set(entry.flavours)].join(", ")}): ${clip(entry.problems[0], 200)}`,
|
||||
);
|
||||
return [
|
||||
`${entries.length} seeds failed the pixel invariants on \`auto\` in the ${date} run, which points at one cause rather than ${entries.length} defects.`,
|
||||
"",
|
||||
`- Run: ${runUrl}`,
|
||||
`- Chrome flavours that failed: ${flavourList(flavoursOf(entries))}`,
|
||||
"",
|
||||
"Reproduce one from `packages/engine`:",
|
||||
"",
|
||||
"```sh",
|
||||
`HF_BLUR_SEEDS=${entries[0].seed} bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts`,
|
||||
"```",
|
||||
"",
|
||||
"Seeds:",
|
||||
"",
|
||||
fenced(lines.join("\n")),
|
||||
...(rest.length ? ["", `And ${rest.length} more: ${rest.join(", ")}`] : []),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function openTitles(gh) {
|
||||
const result = gh([
|
||||
"issue",
|
||||
"list",
|
||||
"--state",
|
||||
"open",
|
||||
"--limit",
|
||||
"1000",
|
||||
"--json",
|
||||
"title",
|
||||
"--jq",
|
||||
".[].title",
|
||||
]);
|
||||
if (result.status !== 0) throw new Error(`gh issue list failed: ${result.stderr}`);
|
||||
return new Set(result.stdout.split("\n").filter(Boolean));
|
||||
}
|
||||
|
||||
function openIssue(gh, titles, title, body, log) {
|
||||
if (titles.has(title)) {
|
||||
log(`skipping "${title}": an open issue has that title`);
|
||||
return;
|
||||
}
|
||||
const file = join(mkdtempSync(join(tmpdir(), "blur-nightly-")), "body.md");
|
||||
writeFileSync(file, body);
|
||||
const labels = LABELS.flatMap((label) => ["--label", label]);
|
||||
const result = gh(["issue", "create", "--title", title, "--body-file", file, ...labels]);
|
||||
if (result.status !== 0)
|
||||
throw new Error(`gh issue create failed for "${title}": ${result.stderr}`);
|
||||
titles.add(title);
|
||||
log(`opened "${title}"`);
|
||||
}
|
||||
|
||||
function fileIssues({ failing, gh, date, runUrl, maxIssues, systemicAbove, log }) {
|
||||
if (failing.length === 0) return;
|
||||
const titles = openTitles(gh);
|
||||
if (failing.length > systemicAbove) {
|
||||
openIssue(gh, titles, systemicTitle(date), systemicBody(failing, date, runUrl), log);
|
||||
return;
|
||||
}
|
||||
const entries = failing.slice(0, maxIssues);
|
||||
if (failing.length > entries.length)
|
||||
log(warning(`${failing.length} failing seeds; the first ${maxIssues} get an issue`));
|
||||
for (const entry of entries)
|
||||
openIssue(gh, titles, seedTitle(entry.seed), seedBody(entry, runUrl), log);
|
||||
}
|
||||
|
||||
function summaryLines({ failing, unstable, setup, silentFailure }) {
|
||||
const lines = ["## Motion blur nightly", ""];
|
||||
if (silentFailure) {
|
||||
lines.push(
|
||||
`A generated job ended ${inline(silentFailure)} without recording a failing seed (a crash, a kill or a setup step); see the job logs.`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
if (setup.length) {
|
||||
const items = setup.map(
|
||||
(item) => `${item.flavour}: ${item.problems.map((p) => clip(p, 300)).join("; ")}`,
|
||||
);
|
||||
lines.push(
|
||||
"### The run did not start or its record was unusable",
|
||||
"",
|
||||
fenced(items.join("\n")),
|
||||
"",
|
||||
);
|
||||
}
|
||||
if (failing.length) {
|
||||
const items = failing.map(
|
||||
(entry) => `${entry.seed} [${features(entry)}] on ${[...new Set(entry.flavours)].join(", ")}`,
|
||||
);
|
||||
lines.push("### Failing seeds", "", fenced(items.join("\n")), "");
|
||||
}
|
||||
if (unstable.length) {
|
||||
lines.push(
|
||||
"### Seeds whose blur-off reference disagreed with itself (renderer noise, no issue opened)",
|
||||
"",
|
||||
"| Seed | Features | Flavour | Worst blur-off MAE |",
|
||||
"| --- | --- | --- | --- |",
|
||||
...unstable.map(
|
||||
(u) =>
|
||||
`| ${u.seed} | ${inline(u.features.join(", "))} | ${inline(u.flavour)} | ${u.mae.toFixed(2)} |`,
|
||||
),
|
||||
"",
|
||||
);
|
||||
}
|
||||
if (lines.length === 2) lines.push("No failing and no unstable seeds.", "");
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function report(options) {
|
||||
const { dir, generatedResult, summaryFile, log } = options;
|
||||
const { failing, unstable, setup } = classify(readRecords(dir));
|
||||
const silentFailure =
|
||||
generatedResult !== "success" && failing.length === 0 && setup.length === 0
|
||||
? generatedResult || "without a result"
|
||||
: "";
|
||||
if (silentFailure)
|
||||
log(
|
||||
warning(
|
||||
`A generated job ended ${silentFailure} without recording a failing seed; see the run logs.`,
|
||||
),
|
||||
);
|
||||
if (setup.length) {
|
||||
const causes = setup.map((s) => `${s.flavour}: ${clip(s.problems[0], 200)}`).join(" | ");
|
||||
log(warning(`The nightly did not run cleanly: ${causes}`));
|
||||
}
|
||||
if (unstable.length) {
|
||||
const seeds = [...new Set(unstable.map((u) => u.seed))];
|
||||
log(
|
||||
warning(
|
||||
`${seeds.length} seed(s) had a blur-off reference that disagrees with itself and were not counted: ${seeds.join(", ")}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (summaryFile)
|
||||
appendFileSync(
|
||||
summaryFile,
|
||||
`${summaryLines({ failing, unstable, setup, silentFailure }).join("\n")}\n`,
|
||||
);
|
||||
fileIssues({ ...options, failing });
|
||||
}
|
||||
|
||||
function gh(args) {
|
||||
const result = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 1 << 26 });
|
||||
return { status: result.status ?? 1, stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
|
||||
}
|
||||
|
||||
function readFlags(argv) {
|
||||
const values = {};
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
if (!argv[i]?.startsWith("--") || argv[i + 1] === undefined) {
|
||||
throw new Error(`bad argument ${argv[i]}`);
|
||||
}
|
||||
values[argv[i].slice(2)] = argv[i + 1];
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const flags = {
|
||||
failures: "failures",
|
||||
"generated-result": "",
|
||||
"run-url": "",
|
||||
"max-issues": DEFAULTS.maxIssues,
|
||||
"systemic-above": DEFAULTS.systemicAbove,
|
||||
date: new Date().toISOString().slice(0, 10),
|
||||
...readFlags(argv),
|
||||
};
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(flags.date)) throw new Error(`bad date ${flags.date}`);
|
||||
return {
|
||||
dir: flags.failures,
|
||||
generatedResult: flags["generated-result"],
|
||||
date: flags.date,
|
||||
runUrl: flags["run-url"],
|
||||
summaryFile: flags.summary,
|
||||
maxIssues: Number(flags["max-issues"]),
|
||||
systemicAbove: Number(flags["systemic-above"]),
|
||||
gh,
|
||||
log: (line) => console.log(line),
|
||||
};
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) report(parseArgs(process.argv.slice(2)));
|
||||
@@ -0,0 +1,363 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { classify, workflowCommand } from "./motion-blur-nightly-report.mjs";
|
||||
|
||||
const SCRIPT = join(import.meta.dirname, "motion-blur-nightly-report.mjs");
|
||||
|
||||
const STUB = `#!/bin/sh
|
||||
printf '%s\\n' "$*" >> "$STUB_LOG"
|
||||
if [ "$1 $2" = "issue list" ]; then
|
||||
[ "$*" = "issue list --state open --limit 1000 --json title --jq .[].title" ] || { echo "unexpected gh arguments: $*" >&2; exit 2; }
|
||||
[ -n "$STUB_LIST_FAIL" ] && { echo "boom" >&2; exit 1; }
|
||||
cat "$STUB_TITLES"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1 $2" = "issue create" ]; then
|
||||
while [ $# -gt 0 ]; do [ "$1" = "--body-file" ] && { cat "$2"; echo "====="; } >> "$STUB_LOG.bodies"; shift; done
|
||||
fi
|
||||
exit 0
|
||||
`;
|
||||
|
||||
function night({
|
||||
files = {},
|
||||
titles = [],
|
||||
generatedResult = "failure",
|
||||
args = [],
|
||||
listFails = false,
|
||||
}) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "nightly-report-test-"));
|
||||
const failures = join(dir, "failures");
|
||||
for (const [artifact, records] of Object.entries(files)) {
|
||||
mkdirSync(join(failures, `blur-nightly-failures-${artifact}`), { recursive: true });
|
||||
writeFileSync(
|
||||
join(failures, `blur-nightly-failures-${artifact}`, "blur-nightly-failures.json"),
|
||||
JSON.stringify(records),
|
||||
);
|
||||
}
|
||||
mkdirSync(join(dir, "bin"));
|
||||
writeFileSync(join(dir, "bin", "gh"), STUB);
|
||||
chmodSync(join(dir, "bin", "gh"), 0o755);
|
||||
writeFileSync(join(dir, "titles.txt"), titles.map((t) => `${t}\n`).join(""));
|
||||
const summary = join(dir, "summary.md");
|
||||
const run = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
SCRIPT,
|
||||
"--failures",
|
||||
failures,
|
||||
"--generated-result",
|
||||
generatedResult,
|
||||
"--date",
|
||||
"2026-11-06",
|
||||
"--run-url",
|
||||
"https://example.test/run/1",
|
||||
"--summary",
|
||||
summary,
|
||||
...args,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${join(dir, "bin")}:${process.env.PATH}`,
|
||||
STUB_LOG: join(dir, "gh.log"),
|
||||
STUB_TITLES: join(dir, "titles.txt"),
|
||||
STUB_LIST_FAIL: listFails ? "1" : "",
|
||||
},
|
||||
},
|
||||
);
|
||||
const calls = existsSync(join(dir, "gh.log"))
|
||||
? readFileSync(join(dir, "gh.log"), "utf8").split("\n").filter(Boolean)
|
||||
: [];
|
||||
return {
|
||||
run,
|
||||
calls,
|
||||
lists: calls.filter((c) => c.startsWith("issue list")),
|
||||
creates: calls.filter((c) => c.startsWith("issue create")),
|
||||
summary: existsSync(summary) ? readFileSync(summary, "utf8") : "",
|
||||
dir,
|
||||
};
|
||||
}
|
||||
|
||||
const real = (seed, problems = ["outside the trail at 0.5: 3 bad tiles"], features = ["mask"]) => ({
|
||||
seed,
|
||||
features,
|
||||
problems,
|
||||
});
|
||||
|
||||
test("one issue per failing seed, titled by seed, with the flavours that failed", () => {
|
||||
const n = night({ files: { "pin-1": [real(7)], "stable-1": [real(7), real(9)] } });
|
||||
assert.equal(n.run.status, 0, n.run.stderr);
|
||||
assert.equal(n.lists.length, 1);
|
||||
assert.equal(n.creates.length, 2);
|
||||
assert.match(n.creates[0], /--title motion blur nightly: seed 7 /);
|
||||
assert.match(n.creates[1], /--title motion blur nightly: seed 9 /);
|
||||
assert.match(n.creates[0], /--label bug --label area\/quality/);
|
||||
const [first, second] = readFileSync(join(n.dir, "gh.log.bodies"), "utf8").split("=====\n");
|
||||
assert.match(first, /Chrome flavours that failed: `pin`, `stable`/);
|
||||
assert.match(first, /^\[pin\] outside the trail at 0.5/m);
|
||||
assert.match(first, /^\[stable\] outside the trail at 0.5/m);
|
||||
assert.match(second, /Chrome flavours that failed: `stable`/);
|
||||
assert.match(first, /HF_BLUR_SEEDS=7 /);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("the title match is exact: an open seed 129 issue does not hide seed 12", () => {
|
||||
const n = night({
|
||||
files: { "pin-1": [real(12), real(129)] },
|
||||
titles: ["motion blur nightly: seed 129", "unrelated"],
|
||||
});
|
||||
assert.equal(n.creates.length, 1);
|
||||
assert.match(n.creates[0], /seed 12 /);
|
||||
assert.match(n.run.stdout, /skipping "motion blur nightly: seed 129"/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("open titles are fetched once, however many seeds fail", () => {
|
||||
const n = night({ files: { "pin-1": [real(1), real(2), real(3), real(4), real(5)] } });
|
||||
assert.equal(n.lists.length, 1);
|
||||
assert.equal(n.creates.length, 5);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("more than 5 failing seeds open one systemic issue and no per-seed issue", () => {
|
||||
const seeds = [1, 2, 3, 4, 5, 6, 7].map((s) => real(s));
|
||||
const n = night({ files: { "pin-1": seeds, "stable-1": seeds } });
|
||||
assert.equal(n.creates.length, 1);
|
||||
assert.match(n.creates[0], /--title motion blur nightly: systemic failure 2026-11-06 /);
|
||||
assert.ok(!n.creates[0].includes("seed 1 "));
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("the systemic title is deduped like the others", () => {
|
||||
const seeds = [1, 2, 3, 4, 5, 6].map((s) => real(s));
|
||||
const n = night({
|
||||
files: { "pin-1": seeds },
|
||||
titles: ["motion blur nightly: systemic failure 2026-11-06"],
|
||||
});
|
||||
assert.equal(n.creates.length, 0);
|
||||
assert.match(n.run.stdout, /skipping "motion blur nightly: systemic failure 2026-11-06"/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("the cap applies before any per-seed work", () => {
|
||||
const seeds = [1, 2, 3, 4].map((s) => real(s));
|
||||
const n = night({
|
||||
files: { "pin-1": seeds },
|
||||
args: ["--max-issues", "2", "--systemic-above", "10"],
|
||||
});
|
||||
assert.equal(n.creates.length, 2);
|
||||
assert.equal(n.lists.length, 1);
|
||||
assert.match(n.run.stdout, /::warning::4 failing seeds; the first 2 get an issue/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a crash record becomes an issue", () => {
|
||||
const n = night({
|
||||
files: { "pin-1": [real(33, ["Error: Test timed out in 120000ms."], ["webgl"])] },
|
||||
});
|
||||
assert.equal(n.creates.length, 1);
|
||||
assert.match(n.creates[0], /seed 33 /);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("unstable seeds go to the summary and one warning, never to an issue", () => {
|
||||
const n = night({
|
||||
files: {
|
||||
"pin-1": [
|
||||
{ seed: 129, features: ["canvas2d", "svgClip"], problems: [], unstableReference: 2.03 },
|
||||
],
|
||||
"stable-1": [{ seed: 130, features: ["mask"], problems: [], unstableReference: 1.5 }],
|
||||
},
|
||||
generatedResult: "success",
|
||||
});
|
||||
assert.equal(n.run.status, 0, n.run.stderr);
|
||||
assert.equal(n.calls.length, 0);
|
||||
assert.equal(n.run.stdout.match(/::warning::/g)?.length, 1);
|
||||
assert.match(
|
||||
n.run.stdout,
|
||||
/::warning::2 seed\(s\) had a blur-off reference that disagrees with itself.*129, 130/,
|
||||
);
|
||||
assert.match(n.summary, /\| 129 \| `canvas2d, svgClip` \| `pin` \| 2\.03 \|/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a failed job with no record warns and writes the summary", () => {
|
||||
const n = night({ files: { "pin-1": [], "stable-1": [] }, generatedResult: "failure" });
|
||||
assert.equal(n.run.status, 0, n.run.stderr);
|
||||
assert.equal(n.calls.length, 0);
|
||||
assert.match(
|
||||
n.run.stdout,
|
||||
/::warning::A generated job ended failure without recording a failing seed/,
|
||||
);
|
||||
assert.match(n.summary, /ended `failure` without recording a failing seed/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a failed job with no artifacts at all warns the same way", () => {
|
||||
const n = night({ generatedResult: "failure" });
|
||||
assert.match(n.run.stdout, /::warning::A generated job ended failure without recording/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a launch failure record warns and goes to the summary", () => {
|
||||
const n = night({
|
||||
files: {
|
||||
"stable-1": [{ seed: null, features: [], problems: ["Chrome lacks drawElementImage"] }],
|
||||
},
|
||||
});
|
||||
assert.equal(n.calls.length, 0);
|
||||
assert.match(
|
||||
n.run.stdout,
|
||||
/::warning::The nightly did not run cleanly: stable: Chrome lacks drawElementImage/,
|
||||
);
|
||||
assert.match(n.summary, /did not start/);
|
||||
assert.ok(!n.run.stdout.includes("without recording"));
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a green night with nothing to say opens nothing and warns of nothing", () => {
|
||||
const n = night({ files: { "pin-1": [], "stable-1": [] }, generatedResult: "success" });
|
||||
assert.equal(n.calls.length, 0);
|
||||
assert.ok(!n.run.stdout.includes("::warning::"));
|
||||
assert.match(n.summary, /No failing and no unstable seeds/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a failed open-title fetch fails the job instead of risking duplicates", () => {
|
||||
const n = night({ files: { "pin-1": [real(5)] }, listFails: true });
|
||||
assert.notEqual(n.run.status, 0);
|
||||
assert.equal(n.creates.length, 0);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a seed that is not an integer never reaches a title", () => {
|
||||
const n = night({ files: { "pin-1": [real("1; echo x"), real(4)] } });
|
||||
assert.equal(n.creates.length, 1);
|
||||
assert.match(n.creates[0], /seed 4 /);
|
||||
assert.match(n.summary, /record without a usable seed/);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("classify groups a seed across flavours and prefixes each finding with its flavour", () => {
|
||||
const { failing } = classify([
|
||||
{ ...real(8, ["a"]), flavour: "pin" },
|
||||
{ ...real(8, ["b"]), flavour: "stable" },
|
||||
]);
|
||||
assert.deepEqual(failing, [
|
||||
{
|
||||
seed: 8,
|
||||
features: ["mask"],
|
||||
flavours: ["pin", "stable"],
|
||||
problems: ["[pin] a", "[stable] b"],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
const commandLines = (stdout) => stdout.split("\n").filter((line) => line.startsWith("::"));
|
||||
|
||||
test("the open-title fetch uses exactly the flags the report relies on", () => {
|
||||
const n = night({ files: { "pin-1": [real(5)] } });
|
||||
assert.equal(n.lists[0], "issue list --state open --limit 1000 --json title --jq .[].title");
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("a job that was cancelled or timed out warns like a failed one", () => {
|
||||
for (const generatedResult of ["cancelled", "skipped"]) {
|
||||
const n = night({ files: { "pin-1": [] }, generatedResult });
|
||||
assert.match(
|
||||
n.run.stdout,
|
||||
new RegExp(`::warning::A generated job ended ${generatedResult} without`),
|
||||
);
|
||||
rmSync(n.dir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("workflowCommand escapes data and properties the way the runner does", () => {
|
||||
assert.equal(workflowCommand("warning", "a%b\r\nc"), "::warning::a%25b%0D%0Ac");
|
||||
assert.equal(
|
||||
workflowCommand("warning", "m", { file: "x:y,z\n" }),
|
||||
"::warning file=x%3Ay%2Cz%0A::m",
|
||||
);
|
||||
});
|
||||
|
||||
test("workflowCommand leaves no :: in its message, so text on one line cannot start a command", () => {
|
||||
const line = workflowCommand("warning", "x ::add-mask::secret ::stop-commands::t :::y");
|
||||
assert.equal(line, "::warning::x : :add-mask: :secret : :stop-commands: :t : : :y");
|
||||
assert.equal(line.slice("::warning::".length).includes("::"), false);
|
||||
});
|
||||
|
||||
test("untrusted newlines never start a second workflow command", () => {
|
||||
const hostile = "1\n::error::pwned";
|
||||
const n = night({
|
||||
files: {
|
||||
"stable-1": [
|
||||
{ seed: hostile, features: [], problems: [] },
|
||||
{
|
||||
seed: null,
|
||||
features: [],
|
||||
problems: ["Chrome died\n::error::pwned %0A\r::warning::again"],
|
||||
},
|
||||
{ seed: 3, features: [], problems: [], unstableReference: 1.5 },
|
||||
],
|
||||
},
|
||||
});
|
||||
assert.equal(n.run.status, 0, n.run.stderr);
|
||||
const commands = commandLines(n.run.stdout);
|
||||
assert.equal(commands.length, 2);
|
||||
assert.ok(commands.every((line) => line.startsWith("::warning::")));
|
||||
assert.ok(
|
||||
!n.run.stdout
|
||||
.split("\n")
|
||||
.some((line) => /^::error/.test(line) || line.startsWith("::warning::again")),
|
||||
);
|
||||
const setup = commands.find((line) => line.includes("did not run cleanly"));
|
||||
assert.ok(setup.includes("Chrome died%0A: :error: :pwned %250A%0D: :warning: :again"), setup);
|
||||
assert.ok(setup.includes("record without a usable seed (1%0A: :error: :pwned)"));
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
|
||||
test("summary and issue text cannot break out of code, the table or mention anyone", () => {
|
||||
const nasty = "bad | cell @someone ```` and ``` <img src=x onerror=1>\n@other";
|
||||
const n = night({
|
||||
files: {
|
||||
"pin-1": [
|
||||
{ seed: 8, features: ["a|b", "@someone", "<b>x</b>", "`tick`"], problems: [nasty] },
|
||||
{
|
||||
seed: 9,
|
||||
features: ["m|n", "@someone", "<b>x</b>", "`tick`", "p\\|q"],
|
||||
problems: [],
|
||||
unstableReference: 2,
|
||||
},
|
||||
{ seed: null, features: [], problems: [nasty] },
|
||||
],
|
||||
},
|
||||
});
|
||||
const body = readFileSync(join(n.dir, "gh.log.bodies"), "utf8").split("=====\n")[0];
|
||||
const fence = body.match(/^(`{4,})$/m);
|
||||
assert.ok(fence, "the findings sit in a fence longer than any backtick run in the text");
|
||||
assert.equal(fence[1].length, 5);
|
||||
const [outside, inside, after] = body.split(fence[1]);
|
||||
assert.ok(inside.includes("<img src=x onerror=1>") && inside.includes("@other"));
|
||||
assert.ok(!outside.replace(/`[^`\n]*`/g, "").includes("@someone"));
|
||||
assert.ok(!outside.includes("<img") && !after.includes("<img"));
|
||||
assert.match(outside, /- Features: `a\\\|b, @someone, <b>x<\/b>, 'tick'`/);
|
||||
const row = n.summary.split("\n").find((line) => line.startsWith("| 9 "));
|
||||
assert.equal(row, "| 9 | `m\\|n, @someone, <b>x</b>, 'tick', p\\\\\\|q` | `pin` | 2.00 |");
|
||||
const setupFence = n.summary.match(/^(`{4,})$/m);
|
||||
assert.ok(setupFence);
|
||||
assert.ok(!n.summary.replace(/^(`{5})$[\s\S]*?^\1$/gm, "").includes("<img"));
|
||||
rmSync(n.dir, { recursive: true });
|
||||
});
|
||||
Reference in New Issue
Block a user