test(engine): nightly motion blur run over generated compositions (#7437)

* test(engine): nightly motion blur run over generated compositions

A seeded generator of text-free compositions combines 19 features (nested
canvas, overflow, clip path, scroll container, callbacks, a last-child rule)
and checks the two pixel invariants on auto: outside the trail at two holds
and at a moving time, inside the trail at a moving time. The runtime lane
runs seeds 1 to 3; motion-blur-nightly.yml runs 200 a day on the Chrome 153
pin and on stable, and opens one issue per failing seed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* test(engine): the nightly records every failure, ignores noisy references, caps its issues

A crash, a timeout or a launch failure is recorded before the job goes red; a
failing check whose blur-off reference disagrees with itself is reported as
unstable instead of failing; the report job moves into a tested script that
fetches open titles once, caps first, and opens one systemic issue above 5 seeds.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* test(engine): the nightly report resists injected failure text and keeps a crashed leg's record

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* fix(engine): nightly report escapes backslashes, test strips tags to a fixpoint

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Vance Ingalls
2026-11-09 21:37:26 +00:00
committed by GitHub
co-authored by Claude Sonnet 5.5
parent 4b3f70fb1b
commit 7bf19ab5f9
9 changed files with 1482 additions and 19 deletions
+328
View File
@@ -0,0 +1,328 @@
#!/usr/bin/env node
// motion-blur-nightly-report.mjs --failures <dir> --generated-result <result> --date <YYYY-MM-DD> ...
// Reads the nightly's per-job failure files and opens the issues, warnings and summary they call for.
import { spawnSync } from "node:child_process";
import {
appendFileSync,
existsSync,
mkdtempSync,
readdirSync,
readFileSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
const ARTIFACT_PREFIX = "blur-nightly-failures-";
const FILE = "blur-nightly-failures.json";
const LABELS = ["bug", "area/quality"];
const DETAILED_SEEDS = 50;
const MAX_PROBLEM_CHARS = 400;
export const DEFAULTS = { maxIssues: 25, systemicAbove: 5 };
const clip = (text, limit) => (text.length > limit ? `${text.slice(0, limit)}...` : text);
const escapeData = (text) =>
String(text).replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
const inertColons = (text) => text.replace(/:(?=:)/g, ": ");
const escapeProperty = (text) => escapeData(text).replace(/:/g, "%3A").replace(/,/g, "%2C");
export function workflowCommand(name, message, properties = {}) {
const props = Object.entries(properties)
.map(([key, value]) => `${key}=${escapeProperty(value)}`)
.join(",");
return `::${name}${props ? ` ${props}` : ""}::${inertColons(escapeData(message))}`;
}
const warning = (message) => workflowCommand("warning", message);
const inline = (text) =>
`\`${String(text)
.replace(/\\/g, "\\\\")
.replace(/`/g, "'")
.replace(/\r?\n|\r/g, " ")
.replace(/\|/g, "\\|")}\``;
function fenced(text) {
const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length));
const ticks = "`".repeat(Math.max(3, longest + 1));
return [ticks, text, ticks].join("\n");
}
const flavourList = (flavours) => [...new Set(flavours)].map(inline).join(", ");
function readFailureFile(file, flavour) {
try {
return JSON.parse(readFileSync(file, "utf8")).map((record) => ({ ...record, flavour }));
} catch (error) {
return [{ seed: null, features: [], problems: [`unreadable ${file}: ${error}`], flavour }];
}
}
export function readRecords(dir) {
if (!existsSync(dir)) return [];
return readdirSync(dir).flatMap((name) => {
const file = join(dir, name, FILE);
if (!name.startsWith(ARTIFACT_PREFIX) || !existsSync(file)) return [];
return readFailureFile(file, name.slice(ARTIFACT_PREFIX.length).split("-")[0]);
});
}
const isSeed = (seed) => Number.isSafeInteger(seed) && seed >= 0;
export function classify(records) {
const failing = new Map();
const unstable = [];
const setup = [];
for (const record of records) {
const { flavour } = record;
const problems = Array.isArray(record.problems) ? record.problems.map(String) : [];
const features = Array.isArray(record.features) ? record.features.map(String) : [];
if (record.seed === null || record.seed === undefined) {
setup.push({ flavour, problems: problems.length ? problems : ["record without a seed"] });
} else if (!isSeed(record.seed)) {
setup.push({
flavour,
problems: [`record without a usable seed (${record.seed})`, ...problems],
});
} else if (problems.length > 0) {
const entry = failing.get(record.seed) ?? {
seed: record.seed,
features,
flavours: [],
problems: [],
};
entry.flavours.push(flavour);
entry.problems.push(...problems.map((problem) => `[${flavour}] ${problem}`));
failing.set(record.seed, entry);
} else if (typeof record.unstableReference === "number") {
unstable.push({ seed: record.seed, features, flavour, mae: record.unstableReference });
}
}
return {
failing: [...failing.values()].sort((a, b) => a.seed - b.seed),
unstable: unstable.sort((a, b) => a.seed - b.seed),
setup,
};
}
const seedTitle = (seed) => `motion blur nightly: seed ${seed}`;
const systemicTitle = (date) => `motion blur nightly: systemic failure ${date}`;
const features = (entry) => entry.features.join(", ") || "none";
const flavoursOf = (entries) => entries.flatMap((entry) => entry.flavours);
export function seedBody(entry, runUrl) {
return [
`Seed ${entry.seed} fails the pixel invariants on \`auto\`.`,
"",
`- Features: ${inline(features(entry))}`,
`- Chrome flavours that failed: ${flavourList(entry.flavours)}`,
`- Run: ${runUrl}`,
"",
"Reproduce from `packages/engine`:",
"",
"```sh",
`HF_BLUR_SEEDS=${entry.seed} bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts`,
"```",
"",
"Findings, by flavour:",
"",
fenced(
entry.problems
.slice(0, 10)
.map((problem) => clip(problem, MAX_PROBLEM_CHARS))
.join("\n"),
),
].join("\n");
}
export function systemicBody(entries, date, runUrl) {
const shown = entries.slice(0, DETAILED_SEEDS);
const rest = entries.slice(DETAILED_SEEDS).map((entry) => entry.seed);
const lines = shown.map(
(entry) =>
`${entry.seed} [${features(entry)}] (${[...new Set(entry.flavours)].join(", ")}): ${clip(entry.problems[0], 200)}`,
);
return [
`${entries.length} seeds failed the pixel invariants on \`auto\` in the ${date} run, which points at one cause rather than ${entries.length} defects.`,
"",
`- Run: ${runUrl}`,
`- Chrome flavours that failed: ${flavourList(flavoursOf(entries))}`,
"",
"Reproduce one from `packages/engine`:",
"",
"```sh",
`HF_BLUR_SEEDS=${entries[0].seed} bunx vitest run --project browser src/services/captureMotionBlur.generated.browser.test.ts`,
"```",
"",
"Seeds:",
"",
fenced(lines.join("\n")),
...(rest.length ? ["", `And ${rest.length} more: ${rest.join(", ")}`] : []),
].join("\n");
}
function openTitles(gh) {
const result = gh([
"issue",
"list",
"--state",
"open",
"--limit",
"1000",
"--json",
"title",
"--jq",
".[].title",
]);
if (result.status !== 0) throw new Error(`gh issue list failed: ${result.stderr}`);
return new Set(result.stdout.split("\n").filter(Boolean));
}
function openIssue(gh, titles, title, body, log) {
if (titles.has(title)) {
log(`skipping "${title}": an open issue has that title`);
return;
}
const file = join(mkdtempSync(join(tmpdir(), "blur-nightly-")), "body.md");
writeFileSync(file, body);
const labels = LABELS.flatMap((label) => ["--label", label]);
const result = gh(["issue", "create", "--title", title, "--body-file", file, ...labels]);
if (result.status !== 0)
throw new Error(`gh issue create failed for "${title}": ${result.stderr}`);
titles.add(title);
log(`opened "${title}"`);
}
function fileIssues({ failing, gh, date, runUrl, maxIssues, systemicAbove, log }) {
if (failing.length === 0) return;
const titles = openTitles(gh);
if (failing.length > systemicAbove) {
openIssue(gh, titles, systemicTitle(date), systemicBody(failing, date, runUrl), log);
return;
}
const entries = failing.slice(0, maxIssues);
if (failing.length > entries.length)
log(warning(`${failing.length} failing seeds; the first ${maxIssues} get an issue`));
for (const entry of entries)
openIssue(gh, titles, seedTitle(entry.seed), seedBody(entry, runUrl), log);
}
function summaryLines({ failing, unstable, setup, silentFailure }) {
const lines = ["## Motion blur nightly", ""];
if (silentFailure) {
lines.push(
`A generated job ended ${inline(silentFailure)} without recording a failing seed (a crash, a kill or a setup step); see the job logs.`,
"",
);
}
if (setup.length) {
const items = setup.map(
(item) => `${item.flavour}: ${item.problems.map((p) => clip(p, 300)).join("; ")}`,
);
lines.push(
"### The run did not start or its record was unusable",
"",
fenced(items.join("\n")),
"",
);
}
if (failing.length) {
const items = failing.map(
(entry) => `${entry.seed} [${features(entry)}] on ${[...new Set(entry.flavours)].join(", ")}`,
);
lines.push("### Failing seeds", "", fenced(items.join("\n")), "");
}
if (unstable.length) {
lines.push(
"### Seeds whose blur-off reference disagreed with itself (renderer noise, no issue opened)",
"",
"| Seed | Features | Flavour | Worst blur-off MAE |",
"| --- | --- | --- | --- |",
...unstable.map(
(u) =>
`| ${u.seed} | ${inline(u.features.join(", "))} | ${inline(u.flavour)} | ${u.mae.toFixed(2)} |`,
),
"",
);
}
if (lines.length === 2) lines.push("No failing and no unstable seeds.", "");
return lines;
}
export function report(options) {
const { dir, generatedResult, summaryFile, log } = options;
const { failing, unstable, setup } = classify(readRecords(dir));
const silentFailure =
generatedResult !== "success" && failing.length === 0 && setup.length === 0
? generatedResult || "without a result"
: "";
if (silentFailure)
log(
warning(
`A generated job ended ${silentFailure} without recording a failing seed; see the run logs.`,
),
);
if (setup.length) {
const causes = setup.map((s) => `${s.flavour}: ${clip(s.problems[0], 200)}`).join(" | ");
log(warning(`The nightly did not run cleanly: ${causes}`));
}
if (unstable.length) {
const seeds = [...new Set(unstable.map((u) => u.seed))];
log(
warning(
`${seeds.length} seed(s) had a blur-off reference that disagrees with itself and were not counted: ${seeds.join(", ")}`,
),
);
}
if (summaryFile)
appendFileSync(
summaryFile,
`${summaryLines({ failing, unstable, setup, silentFailure }).join("\n")}\n`,
);
fileIssues({ ...options, failing });
}
function gh(args) {
const result = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 1 << 26 });
return { status: result.status ?? 1, stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
}
function readFlags(argv) {
const values = {};
for (let i = 0; i < argv.length; i += 2) {
if (!argv[i]?.startsWith("--") || argv[i + 1] === undefined) {
throw new Error(`bad argument ${argv[i]}`);
}
values[argv[i].slice(2)] = argv[i + 1];
}
return values;
}
function parseArgs(argv) {
const flags = {
failures: "failures",
"generated-result": "",
"run-url": "",
"max-issues": DEFAULTS.maxIssues,
"systemic-above": DEFAULTS.systemicAbove,
date: new Date().toISOString().slice(0, 10),
...readFlags(argv),
};
if (!/^\d{4}-\d{2}-\d{2}$/.test(flags.date)) throw new Error(`bad date ${flags.date}`);
return {
dir: flags.failures,
generatedResult: flags["generated-result"],
date: flags.date,
runUrl: flags["run-url"],
summaryFile: flags.summary,
maxIssues: Number(flags["max-issues"]),
systemicAbove: Number(flags["systemic-above"]),
gh,
log: (line) => console.log(line),
};
}
if (process.argv[1] === fileURLToPath(import.meta.url)) report(parseArgs(process.argv.slice(2)));
@@ -0,0 +1,363 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import {
chmodSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { classify, workflowCommand } from "./motion-blur-nightly-report.mjs";
const SCRIPT = join(import.meta.dirname, "motion-blur-nightly-report.mjs");
const STUB = `#!/bin/sh
printf '%s\\n' "$*" >> "$STUB_LOG"
if [ "$1 $2" = "issue list" ]; then
[ "$*" = "issue list --state open --limit 1000 --json title --jq .[].title" ] || { echo "unexpected gh arguments: $*" >&2; exit 2; }
[ -n "$STUB_LIST_FAIL" ] && { echo "boom" >&2; exit 1; }
cat "$STUB_TITLES"
exit 0
fi
if [ "$1 $2" = "issue create" ]; then
while [ $# -gt 0 ]; do [ "$1" = "--body-file" ] && { cat "$2"; echo "====="; } >> "$STUB_LOG.bodies"; shift; done
fi
exit 0
`;
function night({
files = {},
titles = [],
generatedResult = "failure",
args = [],
listFails = false,
}) {
const dir = mkdtempSync(join(tmpdir(), "nightly-report-test-"));
const failures = join(dir, "failures");
for (const [artifact, records] of Object.entries(files)) {
mkdirSync(join(failures, `blur-nightly-failures-${artifact}`), { recursive: true });
writeFileSync(
join(failures, `blur-nightly-failures-${artifact}`, "blur-nightly-failures.json"),
JSON.stringify(records),
);
}
mkdirSync(join(dir, "bin"));
writeFileSync(join(dir, "bin", "gh"), STUB);
chmodSync(join(dir, "bin", "gh"), 0o755);
writeFileSync(join(dir, "titles.txt"), titles.map((t) => `${t}\n`).join(""));
const summary = join(dir, "summary.md");
const run = spawnSync(
process.execPath,
[
SCRIPT,
"--failures",
failures,
"--generated-result",
generatedResult,
"--date",
"2026-11-06",
"--run-url",
"https://example.test/run/1",
"--summary",
summary,
...args,
],
{
encoding: "utf8",
env: {
...process.env,
PATH: `${join(dir, "bin")}:${process.env.PATH}`,
STUB_LOG: join(dir, "gh.log"),
STUB_TITLES: join(dir, "titles.txt"),
STUB_LIST_FAIL: listFails ? "1" : "",
},
},
);
const calls = existsSync(join(dir, "gh.log"))
? readFileSync(join(dir, "gh.log"), "utf8").split("\n").filter(Boolean)
: [];
return {
run,
calls,
lists: calls.filter((c) => c.startsWith("issue list")),
creates: calls.filter((c) => c.startsWith("issue create")),
summary: existsSync(summary) ? readFileSync(summary, "utf8") : "",
dir,
};
}
const real = (seed, problems = ["outside the trail at 0.5: 3 bad tiles"], features = ["mask"]) => ({
seed,
features,
problems,
});
test("one issue per failing seed, titled by seed, with the flavours that failed", () => {
const n = night({ files: { "pin-1": [real(7)], "stable-1": [real(7), real(9)] } });
assert.equal(n.run.status, 0, n.run.stderr);
assert.equal(n.lists.length, 1);
assert.equal(n.creates.length, 2);
assert.match(n.creates[0], /--title motion blur nightly: seed 7 /);
assert.match(n.creates[1], /--title motion blur nightly: seed 9 /);
assert.match(n.creates[0], /--label bug --label area\/quality/);
const [first, second] = readFileSync(join(n.dir, "gh.log.bodies"), "utf8").split("=====\n");
assert.match(first, /Chrome flavours that failed: `pin`, `stable`/);
assert.match(first, /^\[pin\] outside the trail at 0.5/m);
assert.match(first, /^\[stable\] outside the trail at 0.5/m);
assert.match(second, /Chrome flavours that failed: `stable`/);
assert.match(first, /HF_BLUR_SEEDS=7 /);
rmSync(n.dir, { recursive: true });
});
test("the title match is exact: an open seed 129 issue does not hide seed 12", () => {
const n = night({
files: { "pin-1": [real(12), real(129)] },
titles: ["motion blur nightly: seed 129", "unrelated"],
});
assert.equal(n.creates.length, 1);
assert.match(n.creates[0], /seed 12 /);
assert.match(n.run.stdout, /skipping "motion blur nightly: seed 129"/);
rmSync(n.dir, { recursive: true });
});
test("open titles are fetched once, however many seeds fail", () => {
const n = night({ files: { "pin-1": [real(1), real(2), real(3), real(4), real(5)] } });
assert.equal(n.lists.length, 1);
assert.equal(n.creates.length, 5);
rmSync(n.dir, { recursive: true });
});
test("more than 5 failing seeds open one systemic issue and no per-seed issue", () => {
const seeds = [1, 2, 3, 4, 5, 6, 7].map((s) => real(s));
const n = night({ files: { "pin-1": seeds, "stable-1": seeds } });
assert.equal(n.creates.length, 1);
assert.match(n.creates[0], /--title motion blur nightly: systemic failure 2026-11-06 /);
assert.ok(!n.creates[0].includes("seed 1 "));
rmSync(n.dir, { recursive: true });
});
test("the systemic title is deduped like the others", () => {
const seeds = [1, 2, 3, 4, 5, 6].map((s) => real(s));
const n = night({
files: { "pin-1": seeds },
titles: ["motion blur nightly: systemic failure 2026-11-06"],
});
assert.equal(n.creates.length, 0);
assert.match(n.run.stdout, /skipping "motion blur nightly: systemic failure 2026-11-06"/);
rmSync(n.dir, { recursive: true });
});
test("the cap applies before any per-seed work", () => {
const seeds = [1, 2, 3, 4].map((s) => real(s));
const n = night({
files: { "pin-1": seeds },
args: ["--max-issues", "2", "--systemic-above", "10"],
});
assert.equal(n.creates.length, 2);
assert.equal(n.lists.length, 1);
assert.match(n.run.stdout, /::warning::4 failing seeds; the first 2 get an issue/);
rmSync(n.dir, { recursive: true });
});
test("a crash record becomes an issue", () => {
const n = night({
files: { "pin-1": [real(33, ["Error: Test timed out in 120000ms."], ["webgl"])] },
});
assert.equal(n.creates.length, 1);
assert.match(n.creates[0], /seed 33 /);
rmSync(n.dir, { recursive: true });
});
test("unstable seeds go to the summary and one warning, never to an issue", () => {
const n = night({
files: {
"pin-1": [
{ seed: 129, features: ["canvas2d", "svgClip"], problems: [], unstableReference: 2.03 },
],
"stable-1": [{ seed: 130, features: ["mask"], problems: [], unstableReference: 1.5 }],
},
generatedResult: "success",
});
assert.equal(n.run.status, 0, n.run.stderr);
assert.equal(n.calls.length, 0);
assert.equal(n.run.stdout.match(/::warning::/g)?.length, 1);
assert.match(
n.run.stdout,
/::warning::2 seed\(s\) had a blur-off reference that disagrees with itself.*129, 130/,
);
assert.match(n.summary, /\| 129 \| `canvas2d, svgClip` \| `pin` \| 2\.03 \|/);
rmSync(n.dir, { recursive: true });
});
test("a failed job with no record warns and writes the summary", () => {
const n = night({ files: { "pin-1": [], "stable-1": [] }, generatedResult: "failure" });
assert.equal(n.run.status, 0, n.run.stderr);
assert.equal(n.calls.length, 0);
assert.match(
n.run.stdout,
/::warning::A generated job ended failure without recording a failing seed/,
);
assert.match(n.summary, /ended `failure` without recording a failing seed/);
rmSync(n.dir, { recursive: true });
});
test("a failed job with no artifacts at all warns the same way", () => {
const n = night({ generatedResult: "failure" });
assert.match(n.run.stdout, /::warning::A generated job ended failure without recording/);
rmSync(n.dir, { recursive: true });
});
test("a launch failure record warns and goes to the summary", () => {
const n = night({
files: {
"stable-1": [{ seed: null, features: [], problems: ["Chrome lacks drawElementImage"] }],
},
});
assert.equal(n.calls.length, 0);
assert.match(
n.run.stdout,
/::warning::The nightly did not run cleanly: stable: Chrome lacks drawElementImage/,
);
assert.match(n.summary, /did not start/);
assert.ok(!n.run.stdout.includes("without recording"));
rmSync(n.dir, { recursive: true });
});
test("a green night with nothing to say opens nothing and warns of nothing", () => {
const n = night({ files: { "pin-1": [], "stable-1": [] }, generatedResult: "success" });
assert.equal(n.calls.length, 0);
assert.ok(!n.run.stdout.includes("::warning::"));
assert.match(n.summary, /No failing and no unstable seeds/);
rmSync(n.dir, { recursive: true });
});
test("a failed open-title fetch fails the job instead of risking duplicates", () => {
const n = night({ files: { "pin-1": [real(5)] }, listFails: true });
assert.notEqual(n.run.status, 0);
assert.equal(n.creates.length, 0);
rmSync(n.dir, { recursive: true });
});
test("a seed that is not an integer never reaches a title", () => {
const n = night({ files: { "pin-1": [real("1; echo x"), real(4)] } });
assert.equal(n.creates.length, 1);
assert.match(n.creates[0], /seed 4 /);
assert.match(n.summary, /record without a usable seed/);
rmSync(n.dir, { recursive: true });
});
test("classify groups a seed across flavours and prefixes each finding with its flavour", () => {
const { failing } = classify([
{ ...real(8, ["a"]), flavour: "pin" },
{ ...real(8, ["b"]), flavour: "stable" },
]);
assert.deepEqual(failing, [
{
seed: 8,
features: ["mask"],
flavours: ["pin", "stable"],
problems: ["[pin] a", "[stable] b"],
},
]);
});
const commandLines = (stdout) => stdout.split("\n").filter((line) => line.startsWith("::"));
test("the open-title fetch uses exactly the flags the report relies on", () => {
const n = night({ files: { "pin-1": [real(5)] } });
assert.equal(n.lists[0], "issue list --state open --limit 1000 --json title --jq .[].title");
rmSync(n.dir, { recursive: true });
});
test("a job that was cancelled or timed out warns like a failed one", () => {
for (const generatedResult of ["cancelled", "skipped"]) {
const n = night({ files: { "pin-1": [] }, generatedResult });
assert.match(
n.run.stdout,
new RegExp(`::warning::A generated job ended ${generatedResult} without`),
);
rmSync(n.dir, { recursive: true });
}
});
test("workflowCommand escapes data and properties the way the runner does", () => {
assert.equal(workflowCommand("warning", "a%b\r\nc"), "::warning::a%25b%0D%0Ac");
assert.equal(
workflowCommand("warning", "m", { file: "x:y,z\n" }),
"::warning file=x%3Ay%2Cz%0A::m",
);
});
test("workflowCommand leaves no :: in its message, so text on one line cannot start a command", () => {
const line = workflowCommand("warning", "x ::add-mask::secret ::stop-commands::t :::y");
assert.equal(line, "::warning::x : :add-mask: :secret : :stop-commands: :t : : :y");
assert.equal(line.slice("::warning::".length).includes("::"), false);
});
test("untrusted newlines never start a second workflow command", () => {
const hostile = "1\n::error::pwned";
const n = night({
files: {
"stable-1": [
{ seed: hostile, features: [], problems: [] },
{
seed: null,
features: [],
problems: ["Chrome died\n::error::pwned %0A\r::warning::again"],
},
{ seed: 3, features: [], problems: [], unstableReference: 1.5 },
],
},
});
assert.equal(n.run.status, 0, n.run.stderr);
const commands = commandLines(n.run.stdout);
assert.equal(commands.length, 2);
assert.ok(commands.every((line) => line.startsWith("::warning::")));
assert.ok(
!n.run.stdout
.split("\n")
.some((line) => /^::error/.test(line) || line.startsWith("::warning::again")),
);
const setup = commands.find((line) => line.includes("did not run cleanly"));
assert.ok(setup.includes("Chrome died%0A: :error: :pwned %250A%0D: :warning: :again"), setup);
assert.ok(setup.includes("record without a usable seed (1%0A: :error: :pwned)"));
rmSync(n.dir, { recursive: true });
});
test("summary and issue text cannot break out of code, the table or mention anyone", () => {
const nasty = "bad | cell @someone ```` and ``` <img src=x onerror=1>\n@other";
const n = night({
files: {
"pin-1": [
{ seed: 8, features: ["a|b", "@someone", "<b>x</b>", "`tick`"], problems: [nasty] },
{
seed: 9,
features: ["m|n", "@someone", "<b>x</b>", "`tick`", "p\\|q"],
problems: [],
unstableReference: 2,
},
{ seed: null, features: [], problems: [nasty] },
],
},
});
const body = readFileSync(join(n.dir, "gh.log.bodies"), "utf8").split("=====\n")[0];
const fence = body.match(/^(`{4,})$/m);
assert.ok(fence, "the findings sit in a fence longer than any backtick run in the text");
assert.equal(fence[1].length, 5);
const [outside, inside, after] = body.split(fence[1]);
assert.ok(inside.includes("<img src=x onerror=1>") && inside.includes("@other"));
assert.ok(!outside.replace(/`[^`\n]*`/g, "").includes("@someone"));
assert.ok(!outside.includes("<img") && !after.includes("<img"));
assert.match(outside, /- Features: `a\\\|b, @someone, <b>x<\/b>, 'tick'`/);
const row = n.summary.split("\n").find((line) => line.startsWith("| 9 "));
assert.equal(row, "| 9 | `m\\|n, @someone, <b>x</b>, 'tick', p\\\\\\|q` | `pin` | 2.00 |");
const setupFence = n.summary.match(/^(`{4,})$/m);
assert.ok(setupFence);
assert.ok(!n.summary.replace(/^(`{5})$[\s\S]*?^\1$/gm, "").includes("<img"));
rmSync(n.dir, { recursive: true });
});