mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
ci(studio): re-run an edit accuracy shard on a fresh runner when its runner is too slow (#5659)
This commit is contained in:
Miguel Ángel
+74
-10
@@ -264,6 +264,7 @@ jobs:
|
||||
- "scripts/package-subpaths.mjs"
|
||||
- "scripts/rewrite-relative-import-extensions.mjs"
|
||||
- "scripts/ci/base-file.sh"
|
||||
- "scripts/ci/edit-accuracy-shard.sh"
|
||||
- "scripts/ci/install-workspace-dependencies.sh"
|
||||
- ".github/actions/prepare-ffmpeg-bin/**"
|
||||
- "package.json"
|
||||
@@ -1154,7 +1155,8 @@ jobs:
|
||||
needs: [changes, build]
|
||||
if: needs.changes.outputs.edit_accuracy == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
# Healthy shards take 6-9 min; 25 leaves a cancelled shard's fresh re-run inside the queue's 60 min check wait.
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -1182,15 +1184,14 @@ jobs:
|
||||
bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
|
||||
[ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
|
||||
- name: Run this shard, then re-run twice each case whose verdict differs from the base branch
|
||||
run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${{ strategy.job-total }}"
|
||||
# A slow runner stops the bench (slow-runner.json) or runs into the timeout; either gets one fresh re-run.
|
||||
- name: Mark a slow or cancelled shard for a re-run
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
|
||||
bench --shard "${{ matrix.shard }}/${{ strategy.job-total }}" --out /tmp/edit-accuracy/run1
|
||||
mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
|
||||
/tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
|
||||
if (( ${#FLIPPED[@]} > 0 )); then
|
||||
FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"
|
||||
for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
|
||||
mkdir -p /tmp/edit-accuracy
|
||||
if [ "${{ job.status }}" = cancelled ] || [ -n "$(find /tmp/edit-accuracy -name slow-runner.json)" ]; then
|
||||
touch /tmp/edit-accuracy/retry
|
||||
fi
|
||||
- name: Upload shard evidence
|
||||
if: always()
|
||||
@@ -1200,9 +1201,72 @@ jobs:
|
||||
path: /tmp/edit-accuracy
|
||||
if-no-files-found: error
|
||||
|
||||
studio-edit-accuracy-retry-plan:
|
||||
name: "Studio: edit accuracy retry plan"
|
||||
needs: [changes, studio-edit-accuracy]
|
||||
if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' && needs.studio-edit-accuracy.result != 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
shards: ${{ steps.plan.outputs.shards }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
pattern: edit-accuracy-shard-*
|
||||
path: /tmp/edit-accuracy-shards
|
||||
- name: List the shards marked for a re-run
|
||||
id: plan
|
||||
run: |
|
||||
shards=$(find /tmp/edit-accuracy-shards -path '*/edit-accuracy-shard-*/retry' |
|
||||
sed -E 's#.*/edit-accuracy-shard-([0-9]+)/retry$#\1#' | sort -n | paste -sd, -)
|
||||
echo "shards=[${shards}]" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Each marked shard runs once more on a fresh runner and replaces its evidence before the gate reads it.
|
||||
studio-edit-accuracy-retry:
|
||||
name: "Studio: edit accuracy retry (${{ matrix.shard }})"
|
||||
needs: [changes, build, studio-edit-accuracy-retry-plan]
|
||||
if: ${{ !cancelled() && needs.studio-edit-accuracy-retry-plan.outputs.shards != '[]' && needs.studio-edit-accuracy-retry-plan.outputs.shards != '' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: ${{ fromJSON(needs.studio-edit-accuracy-retry-plan.outputs.shards) }}
|
||||
env:
|
||||
EDIT_ACCURACY_GRID: ${{ github.event_name == 'merge_group' && 'full' || 'pr' }}
|
||||
SHARDS: ${{ github.event_name == 'merge_group' && 20 || 5 }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
- uses: ./.github/actions/prepare-ffmpeg-bin
|
||||
- run: bash scripts/ci/install-workspace-dependencies.sh
|
||||
- run: bun run --cwd packages/core build
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: edit-accuracy-cli
|
||||
path: packages/cli/dist
|
||||
- run: node packages/cli/dist/cli.js browser ensure
|
||||
- name: Read the base branch's edit accuracy baseline
|
||||
run: |
|
||||
bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
|
||||
[ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
|
||||
- name: Run the shard again
|
||||
run: bash scripts/ci/edit-accuracy-shard.sh "${{ matrix.shard }}/${SHARDS}"
|
||||
- name: Replace the shard's evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: edit-accuracy-shard-${{ matrix.shard }}
|
||||
path: /tmp/edit-accuracy
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
|
||||
studio-edit-accuracy-gate:
|
||||
name: "Studio: edit accuracy gate"
|
||||
needs: [changes, studio-edit-accuracy]
|
||||
needs: [changes, studio-edit-accuracy, studio-edit-accuracy-retry]
|
||||
if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@
|
||||
"player:perf": "bun run --filter @hyperframes/player perf",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/edit-accuracy-retry.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/rewrite-relative-import-extensions.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
|
||||
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
|
||||
"test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
|
||||
@@ -11,6 +11,14 @@ const lateUndo = (r) => Math.max(r.undo.ms ?? 0, r.undo.redoMs ?? 0) > UNDO_WRIT
|
||||
const DROPPED_FRAME_MS = 25;
|
||||
/** Smooth means no main-thread frame over this, and no more dropped frames than the blank-page control. */
|
||||
const WORK_MS = 8;
|
||||
/** A runner whose first cases take a median past this (3x a healthy runner's) is too slow to measure on. */
|
||||
export const SLOW_RUNNER = { cases: 4, medianSeconds: 60 };
|
||||
/** The first cases' median seconds when it marks a slow runner, else null; the blank-page control misses it. */
|
||||
export function slowRunner(seconds) {
|
||||
if (seconds.length < SLOW_RUNNER.cases) return null;
|
||||
const median = percentile(seconds.slice(0, SLOW_RUNNER.cases), 50);
|
||||
return median > SLOW_RUNNER.medianSeconds ? median : null;
|
||||
}
|
||||
export const METRICS = [
|
||||
"tracking",
|
||||
"press",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { accurate } from "./ratchet.mjs";
|
||||
import { entry, score, UNDO_WRITE_MAX_MS } from "./report.mjs";
|
||||
import { entry, score, slowRunner, UNDO_WRITE_MAX_MS } from "./report.mjs";
|
||||
|
||||
const frames = { intervals: [16, 17, 16], work: [2, 3, 2] };
|
||||
const result = (undo) => ({
|
||||
@@ -60,3 +60,20 @@ describe("a case's verdict", () => {
|
||||
expect(score({ id: "case" }, { error: "boom" }).pass).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the slow runner check", () => {
|
||||
it("stops a shard whose first four cases ran about 6x slower than a healthy runner", () => {
|
||||
// A merge queue shard that ran into its timeout: its first cases, in seconds.
|
||||
expect(slowRunner([95.5, 133.7, 57.9, 107.5, 91.6])).toBe(95.5);
|
||||
});
|
||||
|
||||
it("lets a healthy runner through, including one slow case among its first four", () => {
|
||||
// The slowest first four of 20 healthy shards had a 22 s median; one healthy case took 92.7 s.
|
||||
expect(slowRunner([22, 21.4, 19.8, 23.1])).toBeNull();
|
||||
expect(slowRunner([14, 92.7, 16.3, 15.2])).toBeNull();
|
||||
});
|
||||
|
||||
it("waits for four cases before judging", () => {
|
||||
expect(slowRunner([95.5, 133.7, 107.5])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,7 +14,7 @@ import { resolveHeadlessShellPath } from "../../../../engine/src/index.ts";
|
||||
import { buildGrid, writeFixture } from "./grid.mjs";
|
||||
import { killServers, runCase, startServer, stopServer } from "./case.mjs";
|
||||
import { runSequence } from "./sequences.mjs";
|
||||
import { METRICS, score, writeReport } from "./report.mjs";
|
||||
import { METRICS, SLOW_RUNNER, score, slowRunner, writeReport } from "./report.mjs";
|
||||
import { holdProducerBrowser, renderBox } from "./render.mjs";
|
||||
import { aabb, boxDistance } from "./geometry.mjs";
|
||||
|
||||
@@ -152,19 +152,28 @@ function acquireLock(path) {
|
||||
});
|
||||
}
|
||||
|
||||
// Set once the first cases show a slow runner: no new case starts, and the run exits 75 (see the end).
|
||||
let slow = null;
|
||||
|
||||
// The suite lock is taken per chunk, so a full grid never holds it for its whole run.
|
||||
const LOCK_CHUNK = 8;
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
async function runChunks(queue, browsers, decoders, results) {
|
||||
for (let start = 0; start < queue.length; start += opt.lock ? LOCK_CHUNK : queue.length) {
|
||||
for (
|
||||
let start = 0;
|
||||
start < queue.length && slow === null;
|
||||
start += opt.lock ? LOCK_CHUNK : queue.length
|
||||
) {
|
||||
const chunk = queue.slice(start, opt.lock ? start + LOCK_CHUNK : queue.length);
|
||||
const release = opt.lock ? await acquireLock(opt.lock) : () => undefined;
|
||||
try {
|
||||
await Promise.all(
|
||||
browsers.map(async (browser, i) => {
|
||||
for (let spec = chunk.shift(); spec; spec = chunk.shift())
|
||||
for (let spec = chunk.shift(); spec && slow === null; spec = chunk.shift()) {
|
||||
results.push(await runOne(spec, browser, decoders[i], Number(opt.port) + i));
|
||||
slow = slowRunner(results.map((r) => r.seconds));
|
||||
}
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
@@ -228,6 +237,18 @@ try {
|
||||
} finally {
|
||||
await Promise.all([releaseProducer(), ...browsers.map((b) => b.close())]);
|
||||
}
|
||||
// CI reads slow-runner.json to re-run the shard once on a fresh runner.
|
||||
if (slow !== null) {
|
||||
const seconds = results.slice(0, SLOW_RUNNER.cases).map((r) => r.seconds);
|
||||
writeFileSync(
|
||||
join(out, "slow-runner.json"),
|
||||
JSON.stringify({ seconds, median: slow, ...SLOW_RUNNER }),
|
||||
);
|
||||
console.error(
|
||||
`runner too slow: the first ${SLOW_RUNNER.cases} cases took a median ${slow.toFixed(1)}s, over ${SLOW_RUNNER.medianSeconds}s; stopping`,
|
||||
);
|
||||
process.exit(75);
|
||||
}
|
||||
results.sort((a, b) => a.id.localeCompare(b.id));
|
||||
const meta = {
|
||||
studio,
|
||||
|
||||
@@ -180,11 +180,18 @@ class Closure {
|
||||
}
|
||||
}
|
||||
|
||||
// A shell script a step runs is read like a step: the scripts and commands it runs count too.
|
||||
function seedRun(closure, path) {
|
||||
const fresh = closure.isNew(path);
|
||||
closure.add(path, true);
|
||||
if (fresh && path.endsWith(".sh")) seed(closure, read(path));
|
||||
}
|
||||
|
||||
// Repo files and package scripts the jobs' steps name directly.
|
||||
function seed(closure, text) {
|
||||
for (const m of text.matchAll(/uses: \.\/(\S+)/g)) closure.add(`${m[1]}/action.yml`);
|
||||
for (const m of text.matchAll(/\b(?:bash|sh|node|bun) ((?:scripts|packages)\/\S+)/g))
|
||||
closure.add(m[1], true);
|
||||
seedRun(closure, m[1]);
|
||||
for (const m of text.matchAll(/\bbun run --cwd (\S+) ([\w:-]+)/g)) closure.script(m[1], m[2]);
|
||||
}
|
||||
|
||||
@@ -196,7 +203,12 @@ const cliCommands = (texts) =>
|
||||
|
||||
function benchInputs() {
|
||||
const closure = new Closure();
|
||||
const steps = ["build", "studio-edit-accuracy", "studio-edit-accuracy-gate"].map(job);
|
||||
const steps = [
|
||||
"build",
|
||||
"studio-edit-accuracy",
|
||||
"studio-edit-accuracy-retry",
|
||||
"studio-edit-accuracy-gate",
|
||||
].map(job);
|
||||
steps.forEach((text) => seed(closure, text));
|
||||
for (const name of cliCommands([...steps, read(`${BENCH}/case.mjs`)]))
|
||||
closure.add(`packages/cli/src/commands/${name}.ts`);
|
||||
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# edit-accuracy-shard.sh <i/n>: runs one edit accuracy shard into /tmp/edit-accuracy, then re-runs twice each case
|
||||
# whose verdict differs from the base branch (/tmp/base-edit-accuracy.json). A slow runner makes the bench exit 75.
|
||||
set -euo pipefail
|
||||
bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
|
||||
bench --shard "$1" --out /tmp/edit-accuracy/run1
|
||||
mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
|
||||
/tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
|
||||
if (( ${#FLIPPED[@]} > 0 )); then
|
||||
FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"
|
||||
for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
|
||||
fi
|
||||
@@ -0,0 +1,103 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { parse } from "yaml";
|
||||
|
||||
// A slow runner must not drop a PR from the merge queue: its shard is marked, re-run once on a fresh runner,
|
||||
// and the re-run's evidence is what the gate reads.
|
||||
const ci = parse(
|
||||
readFileSync(join(import.meta.dirname, "..", ".github", "workflows", "ci.yml"), "utf8"),
|
||||
);
|
||||
const jobs = ci.jobs;
|
||||
const step = (job, name) => jobs[job].steps.find((s) => s.name === name);
|
||||
|
||||
// Runs a step's shell with its fixed paths moved under root; returns what it wrote to GITHUB_OUTPUT.
|
||||
function runStep(script, root, env = {}) {
|
||||
const output = join(root, "github-output");
|
||||
writeFileSync(output, "");
|
||||
execFileSync("bash", ["-euo", "pipefail", "-c", script.replaceAll("/tmp/", `${root}/`)], {
|
||||
env: { ...process.env, GITHUB_OUTPUT: output, ...env },
|
||||
});
|
||||
return readFileSync(output, "utf8");
|
||||
}
|
||||
|
||||
function withTree(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), "edit-accuracy-retry-"));
|
||||
try {
|
||||
return fn(root);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const markScript = (status) =>
|
||||
step("studio-edit-accuracy", "Mark a slow or cancelled shard for a re-run").run.replaceAll(
|
||||
"${{ job.status }}",
|
||||
status,
|
||||
);
|
||||
|
||||
test("a shard is marked for a re-run when it timed out or its bench stopped on a slow runner", () => {
|
||||
withTree((root) => {
|
||||
runStep(markScript("cancelled"), root);
|
||||
assert.ok(existsSync(join(root, "edit-accuracy/retry")));
|
||||
});
|
||||
withTree((root) => {
|
||||
mkdirSync(join(root, "edit-accuracy/run1"), { recursive: true });
|
||||
writeFileSync(join(root, "edit-accuracy/run1/slow-runner.json"), "{}");
|
||||
runStep(markScript("failure"), root);
|
||||
assert.ok(existsSync(join(root, "edit-accuracy/retry")));
|
||||
});
|
||||
withTree((root) => {
|
||||
runStep(markScript("success"), root);
|
||||
assert.ok(!existsSync(join(root, "edit-accuracy/retry")));
|
||||
});
|
||||
});
|
||||
|
||||
test("the retry plan lists exactly the marked shards", () => {
|
||||
const plan = step("studio-edit-accuracy-retry-plan", "List the shards marked for a re-run").run;
|
||||
withTree((root) => {
|
||||
for (const shard of [7, 12, 3])
|
||||
mkdirSync(join(root, `edit-accuracy-shards/edit-accuracy-shard-${shard}`), {
|
||||
recursive: true,
|
||||
});
|
||||
writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-7/retry"), "");
|
||||
writeFileSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-12/retry"), "");
|
||||
assert.equal(runStep(plan, root), "shards=[7,12]\n");
|
||||
});
|
||||
withTree((root) => {
|
||||
mkdirSync(join(root, "edit-accuracy-shards/edit-accuracy-shard-1"), { recursive: true });
|
||||
assert.equal(runStep(plan, root), "shards=[]\n");
|
||||
});
|
||||
});
|
||||
|
||||
test("each marked shard re-runs once and replaces its evidence before the gate reads it", () => {
|
||||
const retry = jobs["studio-edit-accuracy-retry"];
|
||||
assert.match(retry.needs.join(" "), /studio-edit-accuracy-retry-plan/);
|
||||
assert.match(
|
||||
retry.strategy.matrix.shard,
|
||||
/needs\.studio-edit-accuracy-retry-plan\.outputs\.shards/,
|
||||
);
|
||||
const shardRun = retry.steps.find((s) => s.run?.includes("edit-accuracy-shard.sh"));
|
||||
assert.ok(shardRun, "the retry runs the same shard script as the shard job");
|
||||
assert.ok(
|
||||
jobs["studio-edit-accuracy"].steps.some((s) =>
|
||||
s.run?.includes("scripts/ci/edit-accuracy-shard.sh"),
|
||||
),
|
||||
);
|
||||
const upload = retry.steps.find((s) => s.uses?.startsWith("actions/upload-artifact@"));
|
||||
assert.equal(upload.with.name, "edit-accuracy-shard-${{ matrix.shard }}");
|
||||
assert.equal(upload.with.overwrite, true);
|
||||
assert.ok(jobs["studio-edit-accuracy-gate"].needs.includes("studio-edit-accuracy-retry"));
|
||||
assert.match(jobs["studio-edit-accuracy-gate"].if, /!cancelled\(\)/);
|
||||
});
|
||||
|
||||
test("a cancelled shard and its re-run both fit in the merge queue's 60 minute check wait", () => {
|
||||
const shard = jobs["studio-edit-accuracy"]["timeout-minutes"];
|
||||
const plan = jobs["studio-edit-accuracy-retry-plan"]["timeout-minutes"];
|
||||
const retry = jobs["studio-edit-accuracy-retry"]["timeout-minutes"];
|
||||
const gate = jobs["studio-edit-accuracy-gate"]["timeout-minutes"];
|
||||
assert.ok(shard + plan + retry + gate <= 60, `${shard}+${plan}+${retry}+${gate} min`);
|
||||
});
|
||||
Reference in New Issue
Block a user