mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
ci: ratchet test reachability across workflow filters (#4253)
* ci: ratchet test reachability across workflow filters * fix: fail closed on skipped detection and quoted selectors * refactor: separate reachability parsing decisions * fix: keep comments and runner prefixes out of reachability * refactor: bound reachability parser complexity * fix: reject conditional shell runner blocks * fix: reject conditional shell constructs in test routing
This commit is contained in:
Miguel Ángel
@@ -26,6 +26,20 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test-reachability:
|
||||
name: Test reachability
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
- run: node --test scripts/check-test-reachability.test.mjs
|
||||
- run: node scripts/check-test-reachability.mjs --base origin/main
|
||||
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -0,0 +1,463 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync, existsSync } from "node:fs";
|
||||
import { posix, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { readTrackedPaths } from "./check-tracked-artifacts.mjs";
|
||||
|
||||
const BASELINE = "scripts/test-reachability-baseline.json";
|
||||
const MANIFEST = "scripts/test-reachability.json";
|
||||
const TEST = /\.test\.(?:mjs|tsx?)$/;
|
||||
const strings = (text) => [...text.matchAll(/["']([^"']+)["']/g)].map((m) => m[1]);
|
||||
export const digest = (text) => createHash("sha256").update(text).digest("hex");
|
||||
|
||||
const patterns = new Map();
|
||||
export function matches(path, glob) {
|
||||
if (patterns.has(glob)) return patterns.get(glob).test(path);
|
||||
if (glob.includes("{")) {
|
||||
const [part, choices] = glob.match(/\{([^}]+)\}/);
|
||||
return choices.split(",").some((choice) => matches(path, glob.replace(part, choice)));
|
||||
}
|
||||
const pattern = glob
|
||||
.replace(/[.+^$()|[\]\\]/g, "\\$&")
|
||||
.replace(/\*\*\//g, "\0")
|
||||
.replace(/\*\*/g, "\x01")
|
||||
.replace(/\*/g, "[^/]*")
|
||||
.replace(/\?/g, "[^/]")
|
||||
.replaceAll("\0", "(?:.*/)?")
|
||||
.replaceAll("\x01", ".*");
|
||||
const regex = new RegExp(`^${pattern}$`);
|
||||
patterns.set(glob, regex);
|
||||
return regex.test(path);
|
||||
}
|
||||
|
||||
function readFilters(source) {
|
||||
return Object.fromEntries(
|
||||
[...source.matchAll(/^ ([\w-]+):\n((?: - .*\n)+)/gm)].map(
|
||||
([, name, body]) => {
|
||||
const globs = [...body.matchAll(/^ - (["'])(.*?)\1(?:\s+#.*)?$/gm)].map(
|
||||
(match) => match[2],
|
||||
);
|
||||
if (globs.some((glob) => glob.startsWith("!")))
|
||||
throw new Error("Negated path filters need an explicit reachability model");
|
||||
return [name, globs];
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function jobField(body, key) {
|
||||
return body.match(new RegExp(`^ ${key}: (.*)$`, "m"))?.[1] ?? "";
|
||||
}
|
||||
|
||||
function readJob(block) {
|
||||
const name = block.match(/^ ([\w-]+):/)[1];
|
||||
const body = block.slice(block.indexOf("\n") + 1);
|
||||
const condition = jobField(body, "if");
|
||||
const needs = jobField(body, "needs");
|
||||
return {
|
||||
name,
|
||||
body,
|
||||
condition,
|
||||
needs: needs.replace(/[[\]]/g, "").split(/,\s*/).filter(Boolean),
|
||||
};
|
||||
}
|
||||
|
||||
// Reject unsupported control syntax instead of treating it as an unconditional job.
|
||||
export function readWorkflow(source) {
|
||||
if (/^\s+(?:paths|paths-ignore|branches-ignore):/m.test(source.split("jobs:\n")[0]))
|
||||
throw new Error("Workflow trigger restrictions need an explicit reachability model");
|
||||
if (/^ (?:needs|if):\s*$/m.test(source))
|
||||
throw new Error("Block job conditions and dependencies need an explicit reachability model");
|
||||
if (/^\s+(?:- )?working-directory:/m.test(source))
|
||||
throw new Error("Working-directory overrides need an explicit reachability model");
|
||||
const body = source.split("jobs:\n")[1];
|
||||
const jobs = body
|
||||
.split(/(?=^ [\w-]+:$)/m)
|
||||
.filter((block) => /^ [\w-]+:/.test(block))
|
||||
.map(readJob);
|
||||
return { filters: readFilters(source), jobs };
|
||||
}
|
||||
|
||||
function conditionEnabled(condition, path, filters) {
|
||||
const resolved = condition
|
||||
.replace(/needs\.changes\.outputs\.([\w-]+) == 'true'/g, (_, name) => {
|
||||
if (!filters[name]) throw new Error(`Unknown path filter: ${name}`);
|
||||
return String(filters[name].some((glob) => matches(path, glob)));
|
||||
})
|
||||
.replace(/always\(\)/g, "true")
|
||||
.replace(/github.event_name == 'pull_request'/g, "true")
|
||||
.replace(/\$\{\{|\}\}/g, "")
|
||||
.trim();
|
||||
if (!/^(?:true|false|\s|\|\||&&)+$/.test(resolved)) return false;
|
||||
return resolved
|
||||
.split("||")
|
||||
.some((part) => part.split("&&").every((term) => term.trim() === "true"));
|
||||
}
|
||||
|
||||
function enabled(job, path, workflow, seen = []) {
|
||||
if (seen.includes(job.name)) throw new Error(`Cyclic job dependency: ${job.name}`);
|
||||
const active = conditionEnabled(job.condition || "true", path, workflow.filters);
|
||||
return (
|
||||
active &&
|
||||
job.needs.every((name) =>
|
||||
enabled(
|
||||
workflow.jobs.find((entry) => entry.name === name) ?? {
|
||||
name,
|
||||
condition: "false",
|
||||
needs: [],
|
||||
},
|
||||
path,
|
||||
workflow,
|
||||
[...seen, job.name],
|
||||
),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function blockCommand(step, start) {
|
||||
const rest = step.slice(start.index + start[0].length);
|
||||
const lines = (rest.match(/^(?: .*\n?)+/m)?.[0] ?? "")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => line.trim());
|
||||
return lines.join(start[1].startsWith(">") ? " " : "\n");
|
||||
}
|
||||
|
||||
function commands(body) {
|
||||
return body.split(/(?=^ - )/m).flatMap((step) => {
|
||||
if (/^ (?: |-[ ])if:/m.test(step)) return [];
|
||||
const start = step.match(/^\s+(?:- )?run: (.*)$/m);
|
||||
if (!start) return [];
|
||||
if (!/^[|>]-?$/.test(start[1])) return [start[1]];
|
||||
return [blockCommand(step, start)];
|
||||
});
|
||||
}
|
||||
|
||||
function flatOptions(tokens) {
|
||||
let depth = 1;
|
||||
let result = "";
|
||||
for (const token of tokens) {
|
||||
depth -= Number(token === "}");
|
||||
if (depth === 0) return result;
|
||||
if (depth === 1) result += token;
|
||||
depth += Number(token === "{");
|
||||
}
|
||||
throw new Error("Unsupported test configuration");
|
||||
}
|
||||
|
||||
function optionTokens(source) {
|
||||
return (
|
||||
source.match(
|
||||
/"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|\/\*[\s\S]*?\*\/|\/\/[^\n]*|[{}]|[^{}"'`/]+|./g,
|
||||
) ?? []
|
||||
);
|
||||
}
|
||||
|
||||
function rejectQuotedSelectors(config) {
|
||||
if (/["'](?:test|include|exclude|projects|workspace)["']\s*:/.test(config))
|
||||
throw new Error("Quoted test selection keys need an explicit reachability model");
|
||||
}
|
||||
|
||||
function testOptions(config) {
|
||||
rejectQuotedSelectors(config);
|
||||
const start = config.match(/\btest:\s*\{/);
|
||||
if (!start) {
|
||||
if (/\btest:/.test(config)) throw new Error("Nonliteral test configuration");
|
||||
return "";
|
||||
}
|
||||
const tokens = optionTokens(config.slice(start.index + start[0].length));
|
||||
const result = flatOptions(tokens.filter((token) => !/^\/[/*]/.test(token)));
|
||||
if (result.includes("...")) throw new Error("Unsupported test configuration");
|
||||
return result;
|
||||
}
|
||||
|
||||
function selectionGlobs(testConfig, key, fallback) {
|
||||
const selection = testConfig.match(new RegExp(`\\b${key}:\\s*\\[([^\\]]*)\\]`));
|
||||
if (!selection) {
|
||||
if (new RegExp(`\\b${key}:`).test(testConfig)) throw new Error(`Unsupported ${key}`);
|
||||
return fallback;
|
||||
}
|
||||
if (selection[1].replace(/["'][^"']*["']/g, "").replace(/[,\s]/g, ""))
|
||||
throw new Error(`Nonliteral ${key}`);
|
||||
return strings(selection[1]);
|
||||
}
|
||||
|
||||
function runnerSelection(cwd, files, read) {
|
||||
const configurations = files.filter(
|
||||
(path) => posix.dirname(path) === cwd && /(?:vitest|vite)\.config\./.test(path),
|
||||
);
|
||||
if (configurations.some((path) => !path.endsWith(".ts")))
|
||||
throw new Error(`Unsupported runner config: ${configurations.join(", ")}`);
|
||||
const config =
|
||||
["vitest", "vite"]
|
||||
.map((name) => read(posix.join(cwd, `${name}.config.ts`)))
|
||||
.find((value) => value !== undefined) ?? "";
|
||||
const testConfig = testOptions(config);
|
||||
if (/\bprojects:|\bworkspace:/.test(testConfig))
|
||||
throw new Error(`Unsupported test projects in ${cwd}`);
|
||||
return {
|
||||
includes: selectionGlobs(testConfig, "include", ["**/*.test.{mjs,ts,tsx}"]),
|
||||
excludes: selectionGlobs(testConfig, "exclude", ["**/node_modules/**", "**/.git/**"]),
|
||||
};
|
||||
}
|
||||
|
||||
function selectedByRunner(relative, args, { includes, excludes }) {
|
||||
if (excludes.some((glob) => matches(relative, glob))) return false;
|
||||
if (!includes.some((glob) => matches(relative, glob))) return false;
|
||||
return args.length === 0 || args.some((arg) => relative.includes(arg));
|
||||
}
|
||||
|
||||
function runnerArguments(tokens, node, command) {
|
||||
const tail = tokens.slice(node ? tokens.indexOf("--test") + 1 : 2);
|
||||
if (
|
||||
tail.some(
|
||||
(token) => token.startsWith("-") && !["--coverage", "--passWithNoTests"].includes(token),
|
||||
)
|
||||
)
|
||||
throw new Error(`Unsupported test option: ${command}`);
|
||||
return tail.filter((token) => !token.startsWith("-"));
|
||||
}
|
||||
|
||||
function runnerKind(command) {
|
||||
if (/^node\s.*--test(?:\s|$)/.test(command)) return "node";
|
||||
if (/^vitest run(?:\s|$)/.test(command)) return "vitest";
|
||||
if (/^bun test(?:\s|$)/.test(command)) return "bun";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function runnerCommand(command) {
|
||||
const kind = runnerKind(command);
|
||||
if (!kind) return undefined;
|
||||
const tokens = command.match(/"[^"]*"|'[^']*'|\S+/g).map((s) => s.replace(/^["']|["']$/g, ""));
|
||||
if (/[\n$`|;<>]/.test(command)) return undefined;
|
||||
return {
|
||||
node: kind === "node",
|
||||
vitest: kind === "vitest",
|
||||
args: runnerArguments(tokens, kind === "node", command),
|
||||
};
|
||||
}
|
||||
|
||||
function runnerTests(command, cwd, files, read) {
|
||||
const runner = runnerCommand(command);
|
||||
if (!runner) return [];
|
||||
const { node, vitest, args } = runner;
|
||||
const selection = vitest
|
||||
? runnerSelection(cwd, files, read)
|
||||
: { includes: ["**/*.test.{mjs,ts,tsx}"], excludes: [] };
|
||||
return files.filter((file) => {
|
||||
if (!TEST.test(file)) return false;
|
||||
const relative = posix.relative(cwd, file);
|
||||
if (relative.startsWith("../")) return false;
|
||||
if (node) return args.some((glob) => matches(relative, glob));
|
||||
return selectedByRunner(relative, args, selection);
|
||||
});
|
||||
}
|
||||
|
||||
function pinnedJob(text, name) {
|
||||
return readWorkflow(text).jobs.find((job) => job.name === name)?.body ?? "";
|
||||
}
|
||||
|
||||
function pinnedScript(text, name) {
|
||||
return JSON.parse(text).scripts[name] ?? "";
|
||||
}
|
||||
|
||||
export function pinnedSource(path, read) {
|
||||
const [file, kind, name] = path.split("#");
|
||||
const text = read(file);
|
||||
if (text === undefined) throw new Error(`Runner mapping needs review: ${path}`);
|
||||
if (kind === "job") return pinnedJob(text, name);
|
||||
if (kind === "script") return pinnedScript(text, name);
|
||||
return text;
|
||||
}
|
||||
|
||||
function adapterTests(adapter, files, read) {
|
||||
for (const [path, hash] of Object.entries(adapter.sources)) {
|
||||
if (digest(pinnedSource(path, read)) !== hash)
|
||||
throw new Error(`Runner mapping needs review: ${path}`);
|
||||
}
|
||||
return files.filter(
|
||||
(file) => TEST.test(file) && adapter.tests.some((glob) => matches(file, glob)),
|
||||
);
|
||||
}
|
||||
|
||||
function packageSelected(pkg, option, target, cwd) {
|
||||
if (option === "--cwd") return pkg.cwd === posix.join(cwd, target);
|
||||
if (option !== "--filter") return pkg.cwd === cwd;
|
||||
if (pkg.cwd === ".") return false;
|
||||
return filterPackageName(pkg.name, target);
|
||||
}
|
||||
function filterPackageName(name, target) {
|
||||
const negate = target.startsWith("!");
|
||||
const pattern = negate ? target.slice(1) : target;
|
||||
return matches(name.replaceAll("/", ":"), pattern.replaceAll("/", ":")) !== negate;
|
||||
}
|
||||
|
||||
function expandPackage(call, cwd, packages, files, read, adapters, seen) {
|
||||
const [, option, single, double, bare, script, args] = call;
|
||||
const target = single ?? double ?? bare;
|
||||
const selected = packages.filter((pkg) => packageSelected(pkg, option, target, cwd));
|
||||
return selected.flatMap((pkg) => {
|
||||
const body = pkg.scripts?.[script];
|
||||
if (!body) return [];
|
||||
const key = `${pkg.cwd}:${script}`;
|
||||
if (seen.includes(key)) throw new Error(`Cyclic package script: ${key}`);
|
||||
return expand(`${body}${args}`, pkg.cwd, packages, files, read, adapters, [...seen, key]);
|
||||
});
|
||||
}
|
||||
|
||||
const SHELL_CONTROL = new Set([
|
||||
"if",
|
||||
"then",
|
||||
"else",
|
||||
"elif",
|
||||
"fi",
|
||||
"for",
|
||||
"while",
|
||||
"until",
|
||||
"do",
|
||||
"done",
|
||||
"case",
|
||||
"esac",
|
||||
"test",
|
||||
"[",
|
||||
"[[",
|
||||
"cd",
|
||||
"false",
|
||||
"!",
|
||||
]);
|
||||
|
||||
function shellParts(command) {
|
||||
if (command.includes("\n") || command.includes("||")) return [];
|
||||
const parts = command.split(/\s*&&\s*/);
|
||||
if (parts.some((part) => SHELL_CONTROL.has(part.trim().split(/\s+/)[0]))) return [];
|
||||
return parts;
|
||||
}
|
||||
|
||||
function expand(command, cwd, packages, files, read, adapters, seen = []) {
|
||||
const adapter = adapters.find((entry) => entry.command === command && entry.cwd === cwd);
|
||||
if (adapter) return adapterTests(adapter, files, read);
|
||||
return shellParts(command).flatMap((part) =>
|
||||
expandSimple(part, cwd, packages, files, read, adapters, seen),
|
||||
);
|
||||
}
|
||||
function expandSimple(command, cwd, packages, files, read, adapters, seen) {
|
||||
const call = command.match(
|
||||
/^bun run (?:(--cwd|--filter) (?:'([^']+)'|"([^"]+)"|(\S+)) )?([\w:*-]+)(.*)$/,
|
||||
);
|
||||
if (!call) return runnerTests(command, cwd, files, read);
|
||||
return expandPackage(call, cwd, packages, files, read, adapters, seen);
|
||||
}
|
||||
|
||||
function testRoutes(workflow, packages, files, read, adapters) {
|
||||
const routes = new Map(files.filter((file) => TEST.test(file)).map((file) => [file, []]));
|
||||
for (const job of workflow.jobs) {
|
||||
for (const command of commands(job.body)) {
|
||||
for (const file of expand(command, ".", packages, files, read, adapters)) {
|
||||
routes.get(file).push(job);
|
||||
}
|
||||
}
|
||||
}
|
||||
return routes;
|
||||
}
|
||||
|
||||
function declaredGuards(file, source, declarations) {
|
||||
if (declarations[file]) return declarations[file];
|
||||
const comment = source.split("\n")[0].match(/^\/\/ guards: (.+)$/)?.[1];
|
||||
return comment ? comment.split(/,\s*/) : [`${posix.dirname(file)}/**`];
|
||||
}
|
||||
function expandGuard(guard, file, files, cache) {
|
||||
if (!cache.has(guard))
|
||||
cache.set(guard, guard === file ? [file] : files.filter((p) => matches(p, guard)));
|
||||
const guarded = cache.get(guard);
|
||||
if (!guarded.length) throw new Error(`${file}: guard matches no tracked files: ${guard}`);
|
||||
return guarded;
|
||||
}
|
||||
function guardIssues(file, guards, jobs, files, workflow, cache) {
|
||||
if (!jobs.length) return ["no CI runner selects this test"];
|
||||
return [file, ...guards].flatMap((guard) => {
|
||||
const guarded = expandGuard(guard, file, files, cache);
|
||||
const missed = guarded.find((p) => !jobs.some((job) => enabled(job, p, workflow)));
|
||||
return missed ? [`CI filters exclude ${guard} (for example ${missed})`] : [];
|
||||
});
|
||||
}
|
||||
|
||||
export function audit(files, read, manifest) {
|
||||
const workflow = readWorkflow(read(".github/workflows/ci.yml"));
|
||||
const packages = files
|
||||
.filter((p) => p === "package.json" || /^packages\/[^/]+\/package.json$/.test(p))
|
||||
.map((p) => ({ ...JSON.parse(read(p)), cwd: posix.dirname(p) }));
|
||||
const routes = testRoutes(workflow, packages, files, read, manifest.runners);
|
||||
const issues = {};
|
||||
const guardFiles = new Map();
|
||||
for (const file of files.filter((p) => TEST.test(p))) {
|
||||
const jobs = routes.get(file) ?? [];
|
||||
const guards = declaredGuards(file, read(file), manifest.guards);
|
||||
const failures = guardIssues(file, guards, jobs, files, workflow, guardFiles);
|
||||
if (failures.length) issues[file] = failures;
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
function baselineIssues(file, count, current, previous) {
|
||||
const errors = [];
|
||||
if (![Number.isInteger(count), count > 0, count <= previous].every(Boolean))
|
||||
errors.push(`${file}: baseline may only shrink`);
|
||||
if (current < count) errors.push(`${file}: lower baseline to ${current}`);
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function ratchet(issues, baseline, previous = baseline) {
|
||||
const errors = Object.entries(issues).flatMap(([file, failures]) => {
|
||||
return failures.length > (baseline.files[file] ?? 0) ? [`${file}: ${failures.join("; ")}`] : [];
|
||||
});
|
||||
const budgetErrors = Object.entries(baseline.files).flatMap(([file, count]) =>
|
||||
baselineIssues(file, count, issues[file]?.length ?? 0, previous.files[file] ?? 0),
|
||||
);
|
||||
if (baseline.total !== Object.values(baseline.files).reduce((sum, n) => sum + n, 0))
|
||||
errors.push("Incorrect baseline total");
|
||||
return [...errors, ...budgetErrors];
|
||||
}
|
||||
|
||||
function baseArgument(index) {
|
||||
const base = process.argv[index + 1];
|
||||
if (!base || base.startsWith("-")) throw new Error("--base needs a Git ref");
|
||||
return base;
|
||||
}
|
||||
|
||||
function previousBaseline(baseline) {
|
||||
const index = process.argv.indexOf("--base");
|
||||
let previous = baseline;
|
||||
if (index !== -1) {
|
||||
const base = baseArgument(index);
|
||||
const paths = execFileSync("git", ["ls-tree", "--name-only", base, "--", BASELINE], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (paths.trim())
|
||||
previous = JSON.parse(
|
||||
execFileSync("git", ["show", `${base}:${BASELINE}`], { encoding: "utf8" }),
|
||||
);
|
||||
}
|
||||
return previous;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const root = process.cwd();
|
||||
const files = readTrackedPaths(root);
|
||||
const read = (path) =>
|
||||
existsSync(resolve(root, path)) ? readFileSync(resolve(root, path), "utf8") : undefined;
|
||||
const issues = audit(files, read, JSON.parse(read(MANIFEST)));
|
||||
if (process.argv.includes("--report")) {
|
||||
console.log(JSON.stringify(issues, null, 2));
|
||||
return;
|
||||
}
|
||||
const baseline = JSON.parse(read(BASELINE));
|
||||
const previous = previousBaseline(baseline);
|
||||
const errors = ratchet(issues, baseline, previous);
|
||||
if (errors.length) {
|
||||
console.error(errors.join("\n"));
|
||||
process.exitCode = 1;
|
||||
} else
|
||||
console.log(`Test reachability verified: ${Object.keys(issues).length} baselined test files.`);
|
||||
}
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main();
|
||||
@@ -0,0 +1,228 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { audit, digest, pinnedSource, ratchet } from "./check-test-reachability.mjs";
|
||||
|
||||
function fixture(command = "bun run test:scripts", filter = '"scripts/**"') {
|
||||
return {
|
||||
".github/workflows/ci.yml": `name: CI
|
||||
jobs:
|
||||
changes:
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v4
|
||||
with:
|
||||
filters: |
|
||||
code:
|
||||
- ${filter}
|
||||
test:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
steps:
|
||||
- run: ${command}
|
||||
`,
|
||||
"package.json": JSON.stringify({
|
||||
scripts: { "test:scripts": "node --test scripts/parity.test.mjs" },
|
||||
}),
|
||||
"scripts/parity.test.mjs": "// guards: skills/lib/**\n",
|
||||
"skills/lib/a.mjs": "",
|
||||
};
|
||||
}
|
||||
const check = (tree, manifest = { guards: {}, runners: [] }) =>
|
||||
audit(Object.keys(tree), (path) => tree[path], manifest);
|
||||
const empty = { total: 0, files: {} };
|
||||
|
||||
test("planted guard-filter hole fails and adding its directory passes", () => {
|
||||
const tree = fixture();
|
||||
assert.match(ratchet(check(tree), empty)[0], /CI filters exclude skills\/lib/);
|
||||
tree[".github/workflows/ci.yml"] = tree[".github/workflows/ci.yml"].replace(
|
||||
'"scripts/**"',
|
||||
'"scripts/**"\n - "skills/**"',
|
||||
);
|
||||
assert.deepEqual(check(tree), {});
|
||||
});
|
||||
|
||||
test("removing a test from a package script makes it an orphan", () => {
|
||||
const tree = fixture("bun run test:scripts", '"**"');
|
||||
assert.deepEqual(check(tree), {});
|
||||
tree["package.json"] = JSON.stringify({
|
||||
scripts: { "test:scripts": "node --test scripts/other.test.mjs" },
|
||||
});
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("workspace scripts honor Vite include and exclude instead of package membership", () => {
|
||||
const tree = fixture("bun run --filter '*' test", '"**"');
|
||||
delete tree["scripts/parity.test.mjs"];
|
||||
tree["packages/a/package.json"] = JSON.stringify({
|
||||
name: "@scope/a",
|
||||
scripts: { test: "vitest run" },
|
||||
});
|
||||
tree["packages/a/src/a.test.ts"] = "";
|
||||
tree["packages/a/vite.config.ts"] =
|
||||
'export default { test: { include: ["src/**/*.test.ts"], exclude: ["src/a.test.ts"] } }';
|
||||
assert.match(check(tree)["packages/a/src/a.test.ts"][0], /no CI runner/);
|
||||
tree["packages/a/vite.config.ts"] =
|
||||
'export default { test: { include: ["src/**/*.test.ts"], coverage: { include: ["missing/**"] } } }';
|
||||
assert.deepEqual(check(tree), {});
|
||||
});
|
||||
|
||||
test("job dependencies can prevent an otherwise unfiltered job from running", () => {
|
||||
const tree = fixture("node --test scripts/parity.test.mjs", '"**"');
|
||||
tree[".github/workflows/ci.yml"] =
|
||||
tree[".github/workflows/ci.yml"].replace(" needs: changes", " needs: disabled") +
|
||||
" disabled:\n if: false\n steps: []\n";
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"].join("\n"), /CI filters exclude/);
|
||||
});
|
||||
|
||||
test("folded commands select tests but conditional steps are not assumed to run", () => {
|
||||
const tree = fixture(">-\n node --test\n scripts/parity.test.mjs", '"**"');
|
||||
assert.deepEqual(check(tree), {});
|
||||
tree[".github/workflows/ci.yml"] += " if: false\n";
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("workflow path restrictions cannot silently manufacture reachability", () => {
|
||||
const tree = fixture();
|
||||
tree[".github/workflows/ci.yml"] =
|
||||
"on:\n pull_request:\n paths: [docs/**]\n" + tree[".github/workflows/ci.yml"];
|
||||
assert.throws(() => check(tree), /trigger restrictions/);
|
||||
});
|
||||
|
||||
test("unknown runner options and config formats fail closed", () => {
|
||||
const tree = fixture("vitest run --exclude scripts/parity.test.mjs", '"**"');
|
||||
assert.throws(() => check(tree), /Unsupported test option/);
|
||||
tree[".github/workflows/ci.yml"] = fixture("vitest run", '"**"')[".github/workflows/ci.yml"];
|
||||
tree["vitest.config.js"] = 'export default { test: { exclude: ["**"] } }';
|
||||
assert.throws(() => check(tree), /Unsupported runner config/);
|
||||
});
|
||||
|
||||
test("custom runner mappings require matching commands and unchanged producer sources", () => {
|
||||
const tree = fixture("node custom.mjs", '"**"');
|
||||
tree["custom.mjs"] = "original runner";
|
||||
const manifest = {
|
||||
guards: {},
|
||||
runners: [
|
||||
{
|
||||
cwd: ".",
|
||||
command: "node custom.mjs",
|
||||
tests: ["scripts/*.test.mjs"],
|
||||
sources: { "custom.mjs": digest(tree["custom.mjs"]) },
|
||||
reason: "fixture",
|
||||
},
|
||||
],
|
||||
};
|
||||
assert.deepEqual(check(tree, manifest), {});
|
||||
tree["custom.mjs"] = "changed selection";
|
||||
assert.throws(() => check(tree, manifest), /mapping needs review/);
|
||||
});
|
||||
|
||||
test("baselines accept existing debt, reject new debt and cannot be raised", () => {
|
||||
const issues = { "a.test.ts": ["orphan"] };
|
||||
const baseline = { total: 1, files: { "a.test.ts": 1 } };
|
||||
assert.deepEqual(ratchet(issues, baseline), []);
|
||||
assert.match(ratchet({ ...issues, "new.test.ts": ["orphan"] }, baseline)[0], /new.test.ts/);
|
||||
assert.match(ratchet(issues, baseline, empty)[0], /only shrink/);
|
||||
assert.match(ratchet({}, baseline)[0], /lower baseline/);
|
||||
});
|
||||
|
||||
test("conditions in the first step key are never credited", () => {
|
||||
const tree = fixture("node --test scripts/parity.test.mjs", '"**"');
|
||||
tree[".github/workflows/ci.yml"] = tree[".github/workflows/ci.yml"].replace(
|
||||
" - run:",
|
||||
" - if: false\n run:",
|
||||
);
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("block dependency syntax fails closed instead of losing the dependency", () => {
|
||||
const tree = fixture();
|
||||
tree[".github/workflows/ci.yml"] = tree[".github/workflows/ci.yml"].replace(
|
||||
"needs: changes",
|
||||
"needs:\n - disabled",
|
||||
);
|
||||
assert.throws(() => check(tree), /Block job/);
|
||||
});
|
||||
|
||||
test("runner excludes after nested coverage options still apply", () => {
|
||||
const tree = fixture("vitest run", '"**"');
|
||||
tree["vitest.config.ts"] =
|
||||
'export default { test: { coverage: { include: ["**"] }, exclude: ["**"] } }';
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("a disabled change detector prevents dependent tests from running", () => {
|
||||
const tree = fixture("node --test scripts/parity.test.mjs", '"**"');
|
||||
tree[".github/workflows/ci.yml"] = tree[".github/workflows/ci.yml"].replace(
|
||||
" changes:\n",
|
||||
" changes:\n if: false\n",
|
||||
);
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"].join("\n"), /CI filters exclude/);
|
||||
});
|
||||
|
||||
test("quoted runner selection keys cannot silently change collection", () => {
|
||||
const tree = fixture("vitest run", '"**"');
|
||||
tree["vitest.config.ts"] = 'export default { test: { "exclude": ["**"] } }';
|
||||
assert.throws(() => check(tree), /Quoted test selection/);
|
||||
});
|
||||
|
||||
test("quoted comments do not widen path filters", () => {
|
||||
const tree = fixture("node --test scripts/parity.test.mjs", '"scripts/**" # "skills/**"');
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"].join("\n"), /CI filters exclude skills/);
|
||||
});
|
||||
|
||||
test("runner verbs must match exactly", () => {
|
||||
const tree = fixture("vitest run-anything", '"**"');
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("conditional shell blocks are not split into unconditional runners", () => {
|
||||
const tree = fixture(
|
||||
"|\n if false; then\n true && node --test scripts/parity.test.mjs && true\n fi",
|
||||
'"**"',
|
||||
);
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("single-line shell conditions cannot credit their guarded runner", () => {
|
||||
const tree = fixture("test -f dist/bundle.js && node --test scripts/parity.test.mjs", '"**"');
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("folded shell conditions cannot credit their guarded runner", () => {
|
||||
const tree = fixture(
|
||||
">-\n test -f dist/bundle.js &&\n node --test scripts/parity.test.mjs",
|
||||
'"**"',
|
||||
);
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("directory changes cannot credit a runner at the original working directory", () => {
|
||||
const tree = fixture("cd elsewhere && node --test scripts/parity.test.mjs", '"**"');
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("fallback shell branches cannot credit conditional runners", () => {
|
||||
const tree = fixture("false || true && node --test scripts/parity.test.mjs", '"**"');
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("ordinary sequential runners remain reachable", () => {
|
||||
const tree = fixture("node --test scripts/parity.test.mjs && vitest run", '"**"');
|
||||
assert.deepEqual(check(tree), {});
|
||||
});
|
||||
|
||||
test("known false commands cannot credit later runners", () => {
|
||||
const tree = fixture("false && node --test scripts/parity.test.mjs", '"**"');
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("negated shell commands cannot credit later runners", () => {
|
||||
const tree = fixture("! true && node --test scripts/parity.test.mjs", '"**"');
|
||||
assert.match(check(tree)["scripts/parity.test.mjs"][0], /no CI runner/);
|
||||
});
|
||||
|
||||
test("a missing pinned package gives the runner mapping diagnostic", () => {
|
||||
assert.throws(
|
||||
() => pinnedSource("missing/package.json#script#test", () => undefined),
|
||||
/Runner mapping needs review: missing\/package.json#script#test/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"total": 83,
|
||||
"files": {
|
||||
"packages/cli/src/media-use/lib/adopt.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/candidates.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/codex-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/coverage.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/cube-build.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/cube-validate.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/cutlist.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/duck.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/error-diffusion.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/freeze.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/grade-analyzer.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/heygen-cli.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/heygen-search.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/heygen-video-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/local-models.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/local-run.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/logo-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/ltx-video-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/lut-preset-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/manifest.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/media-fetch.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/mflux-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/misses.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/npx-sync.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/parakeet-words.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/prefs-store.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/probe.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/recipe-store.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/registry.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/search.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/specs.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/stats.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/telemetry.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/tts-local-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/usage.test.mjs": 2,
|
||||
"packages/cli/src/media-use/lib/voice-provider.test.mjs": 2,
|
||||
"packages/cli/src/media-use/resolve.test.mjs": 2,
|
||||
"packages/producer/tests/distributed/_smoke/webm-concat-copy.test.ts": 1,
|
||||
"packages/producer/tests/playback-rate-av-parity/playback-rate-av-parity.test.ts": 1,
|
||||
"registry/components/pan-stations/pan-stations.test.mjs": 1,
|
||||
"scripts/creator-editing-capabilities.test.mjs": 1,
|
||||
"scripts/generate-catalog-payloads.test.ts": 1,
|
||||
"scripts/generate-catalog-previews.test.ts": 1,
|
||||
"scripts/registry-primitive-payloads.test.ts": 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"guards": {
|
||||
"scripts/check-media-use-copy-parity.test.mjs": [
|
||||
"skills/media-use/scripts/lib/**",
|
||||
"packages/cli/src/media-use/lib/**"
|
||||
],
|
||||
"scripts/check-test-reachability.test.mjs": [
|
||||
".github/workflows/ci.yml",
|
||||
"**/package.json",
|
||||
"**/*.test.{mjs,ts,tsx}"
|
||||
]
|
||||
},
|
||||
"runners": [
|
||||
{
|
||||
"cwd": ".",
|
||||
"command": "set -euo pipefail\nmapfile -t SKILLS_TESTS < <(find skills packages/cli/src/media-use -type f -name \"*.test.mjs\" | sort)\nif [ \"${#SKILLS_TESTS[@]}\" -eq 0 ]; then\necho \"::error::No skills/**/*.test.mjs files found. Did the layout change?\"\nexit 1\nfi\nSKILLS_TESTS+=(scripts/check-media-use-copy-parity.test.mjs)\nprintf 'Running %d skills test file(s):\\n' \"${#SKILLS_TESTS[@]}\"\nprintf ' * %s\\n' \"${SKILLS_TESTS[@]}\"\nnode --test \"${SKILLS_TESTS[@]}\"",
|
||||
"tests": [
|
||||
"skills/**/*.test.mjs",
|
||||
"packages/cli/src/media-use/**/*.test.mjs",
|
||||
"scripts/check-media-use-copy-parity.test.mjs"
|
||||
],
|
||||
"sources": {},
|
||||
"reason": "Shell find plus an explicit parity file; exact command match invalidates this mapping on edits."
|
||||
},
|
||||
{
|
||||
"cwd": ".",
|
||||
"command": "bun run producer:test:${{ matrix.lane }}",
|
||||
"tests": [
|
||||
"packages/producer/src/**/*.test.ts",
|
||||
"packages/producer/scripts/test-classification.test.mjs"
|
||||
],
|
||||
"sources": {
|
||||
"packages/producer/scripts/run-test-lane.mjs": "bff02680f19eebcce0df72cdeedbbc05edfdcede3f3f60658fc0cb18ed8c59f6",
|
||||
"packages/producer/scripts/test-classification.mjs": "e3086467feb160d0b2debcb0deb70c4f07c8e2aa9d9063b0da8e97e5728e89cc",
|
||||
".github/workflows/ci.yml#job#producer-source-tests": "3709c6ac6c9f5c7c66f57fc6c1c106aec4ffe906f919eb7cdf2a6ce1d4e0974c",
|
||||
"packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1",
|
||||
"packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",
|
||||
"packages/producer/package.json#script#test:classification": "4ad50d88c81de374dd6ef8a53d702a4b1901483837ecd3d833b1cb0c9a29a058",
|
||||
"package.json#script#producer:test:unit": "ded6d4183d455d1f414e41d15794c1aef5323e53b5ea5e2ff349e3bb049c18db",
|
||||
"package.json#script#producer:test:integration": "8bb545947f4f8fef3bd5c6c9b1bcb394c7c942d7a3ab90cf4540c662572a278e"
|
||||
},
|
||||
"reason": "Both unit and integration matrix lanes run source discovery and the classification test; workflow and scripts are pinned to invalidate changed selection."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
# Test reachability manifest
|
||||
|
||||
`test-reachability.json` supplies guards and reviewed runner mappings for
|
||||
`check-test-reachability.mjs`. Only `.github/workflows/ci.yml` is modelled.
|
||||
A test selected only by another workflow is reported as unreachable by this check.
|
||||
For example, `windows-render.yml` selects four tests that are also routed by `ci.yml`.
|
||||
|
||||
`guards` maps test paths to the source paths that must trigger them. Custom runner
|
||||
entries name the exact command, selected tests, and hashes of the source that owns
|
||||
selection. A changed or missing pinned source requires the mapping to be reviewed.
|
||||
Reference in New Issue
Block a user