Merge pull request #2146 from heygen-com/07-10-test_producer_gate_source_tests_by_execution_lane

test(producer): gate source tests by execution lane
This commit is contained in:
James Russo
2026-07-11 11:03:17 -07:00
committed by GitHub
16 changed files with 285 additions and 37 deletions
+30
View File
@@ -235,6 +235,36 @@ jobs:
- run: bun run --cwd packages/core build:hyperframes-runtime
- run: bun run --filter '!@hyperframes/producer' test
producer-source-tests:
name: "Producer: ${{ matrix.lane }} tests"
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
lane: [unit, integration]
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
lfs: true
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
- name: Install FFmpeg for integration tests
if: matrix.lane == 'integration'
run: |
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends ffmpeg
- uses: ./.github/actions/prepare-ffmpeg-bin
- run: bun install --frozen-lockfile
- run: bun run --filter '@hyperframes/{parsers,lint,studio-server}' build
- run: bun run --cwd packages/core build
- run: bun run --filter @hyperframes/engine build
- run: bun run producer:test:${{ matrix.lane }}
# Tests under skills/**/*.test.mjs are bare `node --test` files with only
# `node:` built-in imports. They aren't part of any workspace package, and
# the main `Test` job's `code` path filter excludes `skills/**`, so without
+14 -13
View File
@@ -22,7 +22,7 @@
},
"packages/aws-lambda": {
"name": "@hyperframes/aws-lambda",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@aws-sdk/client-s3": "^3.700.0",
"@aws-sdk/client-sfn": "^3.700.0",
@@ -54,7 +54,7 @@
},
"packages/cli": {
"name": "@hyperframes/cli",
"version": "0.7.45",
"version": "0.7.48",
"bin": {
"hyperframes": "./dist/cli.js",
},
@@ -103,7 +103,7 @@
},
"packages/core": {
"name": "@hyperframes/core",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@chenglou/pretext": "^0.0.5",
"@hyperframes/lint": "workspace:*",
@@ -128,7 +128,7 @@
},
"packages/engine": {
"name": "@hyperframes/engine",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@hono/node-server": "^1.13.0",
"@hyperframes/core": "workspace:^",
@@ -146,7 +146,7 @@
},
"packages/gcp-cloud-run": {
"name": "@hyperframes/gcp-cloud-run",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@google-cloud/storage": "^7.14.0",
"@google-cloud/workflows": "^4.2.0",
@@ -166,7 +166,7 @@
},
"packages/lint": {
"name": "@hyperframes/lint",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@hyperframes/parsers": "workspace:*",
"htmlparser2": "^10.1.0",
@@ -183,7 +183,7 @@
},
"packages/parsers": {
"name": "@hyperframes/parsers",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@babel/parser": "^7.27.0",
"acorn": "^8.17.0",
@@ -203,7 +203,7 @@
},
"packages/player": {
"name": "@hyperframes/player",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@hyperframes/core": "workspace:*",
},
@@ -218,7 +218,7 @@
},
"packages/producer": {
"name": "@hyperframes/producer",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@fontsource/archivo-black": "^5.2.8",
"@fontsource/eb-garamond": "^5.2.7",
@@ -258,11 +258,12 @@
"esbuild": "^0.25.12",
"tsx": "^4.21.0",
"typescript": "^5.7.2",
"vitest": "^3.2.4",
},
},
"packages/sdk": {
"name": "@hyperframes/sdk",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@hyperframes/core": "workspace:*",
"@hyperframes/parsers": "workspace:*",
@@ -289,7 +290,7 @@
},
"packages/shader-transitions": {
"name": "@hyperframes/shader-transitions",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"html2canvas": "^1.4.1",
},
@@ -301,7 +302,7 @@
},
"packages/studio": {
"name": "@hyperframes/studio",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@codemirror/autocomplete": "^6.20.1",
"@codemirror/commands": "^6.10.3",
@@ -349,7 +350,7 @@
},
"packages/studio-server": {
"name": "@hyperframes/studio-server",
"version": "0.7.45",
"version": "0.7.48",
"dependencies": {
"@hyperframes/core": "workspace:*",
"@hyperframes/parsers": "workspace:*",
+8 -1
View File
@@ -29,7 +29,14 @@
"lint:fix": "oxlint --fix .",
"check:tracked-artifacts": "node scripts/check-tracked-artifacts.mjs",
"format": "oxfmt .",
"test": "bun run --filter '*' test",
"test": "bun run test:unit",
"test:unit": "bun run --filter '*' test",
"producer:test:classification": "bun run --cwd packages/producer test:classification",
"producer:test:unit": "bun run --cwd packages/producer test:unit",
"producer:test:unit:bun": "bun run --cwd packages/producer test:unit:bun",
"producer:test:unit:vitest": "bun run --cwd packages/producer test:unit:vitest",
"producer:test:integration": "bun run --cwd packages/producer test:integration",
"test:regression": "bun run --cwd packages/producer test:regression",
"player:perf": "bun run --filter @hyperframes/player perf",
"format:check": "oxfmt --check .",
"knip": "knip",
+12 -3
View File
@@ -43,8 +43,16 @@
"check:runtime-conformance": "tsx src/runtime-conformance.ts",
"benchmark": "tsx src/benchmark.ts",
"bench:hdr": "tsx src/benchmark.ts --tags hdr",
"test": "tsx src/regression-harness.ts --exclude-tags transparency",
"test:update": "tsx src/regression-harness.ts --update --exclude-tags transparency",
"test": "bun run test:unit",
"test:classification": "node --test scripts/test-classification.test.mjs && node scripts/check-test-classification.mjs",
"test:unit": "bun run test:classification && node scripts/run-test-lane.mjs unit",
"test:unit:bun": "node scripts/run-test-lane.mjs unit bun",
"test:unit:vitest": "node scripts/run-test-lane.mjs unit vitest",
"test:integration": "bun run test:classification && node scripts/run-test-lane.mjs integration",
"test:integration:bun": "node scripts/run-test-lane.mjs integration bun",
"test:integration:vitest": "node scripts/run-test-lane.mjs integration vitest",
"test:regression": "tsx src/regression-harness.ts --exclude-tags transparency",
"test:regression:update": "tsx src/regression-harness.ts --update --exclude-tags transparency",
"test:distributed": "tsx src/regression-harness.ts --exclude-tags transparency --mode=distributed-simulated",
"test:transparency": "tsx src/transparency-test.ts",
"docker:build:test": "docker build -f ../../Dockerfile.test -t hyperframes-producer:test ../..",
@@ -93,7 +101,8 @@
"@webgpu/types": "^0.1.69",
"esbuild": "^0.25.12",
"tsx": "^4.21.0",
"typescript": "^5.7.2"
"typescript": "^5.7.2",
"vitest": "^3.2.4"
},
"engines": {
"node": ">=22"
@@ -0,0 +1,9 @@
import { discoverProducerTests, summarizeTests } from "./test-classification.mjs";
try {
const tests = discoverProducerTests();
console.log(JSON.stringify({ event: "producer_tests_classified", ...summarizeTests(tests) }));
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
}
@@ -0,0 +1,34 @@
import { spawnSync } from "node:child_process";
import { discoverProducerTests, PRODUCER_ROOT } from "./test-classification.mjs";
const lane = process.argv[2];
const requestedRunner = process.argv[3];
if (lane !== "unit" && lane !== "integration") {
throw new Error("Usage: node scripts/run-test-lane.mjs <unit|integration> [bun|vitest]");
}
if (requestedRunner && requestedRunner !== "bun" && requestedRunner !== "vitest") {
throw new Error(`Unknown test runner: ${requestedRunner}`);
}
const tests = discoverProducerTests().filter(
(test) => test.lane === lane && (!requestedRunner || test.runner === requestedRunner),
);
function run(args) {
const result = spawnSync("bun", args, {
cwd: PRODUCER_ROOT,
env: { ...process.env, HYPERFRAMES_TEST_LANE: lane },
stdio: "inherit",
});
if (result.error) throw result.error;
if (result.status !== 0) process.exit(result.status ?? 1);
}
const vitestFiles = tests.filter((test) => test.runner === "vitest").map((test) => test.file);
if (vitestFiles.length > 0) run(["x", "vitest", "run", ...vitestFiles]);
// Bun's mock.module registry is process-global. Run each file in a fresh
// process so mocks from one source test cannot mutate another test's imports.
for (const test of tests.filter((entry) => entry.runner === "bun")) {
run(["test", test.file]);
}
@@ -0,0 +1,74 @@
// fallow-ignore-file complexity
import { readdirSync, readFileSync } from "node:fs";
import { dirname, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
export const PRODUCER_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
// These tests need host capabilities such as Chrome, ffmpeg, worker threads,
// or local sockets. Keep the list explicit so a filename-only rename does not
// make Git/fallow re-audit thousands of unchanged test lines as new code.
const INTEGRATION_TEST_FILES = new Set([
"src/services/deterministicFonts-systemCapture.test.ts",
"src/services/distributed/assemble.test.ts",
"src/services/distributed/chunkBoundary.test.ts",
"src/services/distributed/crossWorkerIdempotency.test.ts",
"src/services/distributed/plan.test.ts",
"src/services/distributed/planSizeCap.test.ts",
"src/services/distributed/renderChunk.test.ts",
"src/services/fileServer.test.ts",
"src/services/healthWorker.test.ts",
"src/utils/audioRegression.test.ts",
"src/utils/streamDurationParity.test.ts",
]);
function collectTestFiles(directory, files = []) {
for (const entry of readdirSync(directory, { withFileTypes: true })) {
const entryPath = resolve(directory, entry.name);
if (entry.isDirectory()) collectTestFiles(entryPath, files);
else if (entry.isFile() && entry.name.endsWith(".test.ts")) files.push(entryPath);
}
return files;
}
export function classifyTestSource(filePath, source, integrationFiles = INTEGRATION_TEST_FILES) {
const importsBun = /\bfrom\s+["']bun:test["']/.test(source);
const importsVitest = /\bfrom\s+["']vitest["']/.test(source);
if (importsBun === importsVitest) {
const detail = importsBun ? "imports both bun:test and vitest" : "imports neither runner";
throw new Error(`${filePath}: ${detail}`);
}
return {
file: filePath,
runner: importsBun ? "bun" : "vitest",
lane: integrationFiles.has(filePath) ? "integration" : "unit",
};
}
export function discoverProducerTests(producerRoot = PRODUCER_ROOT) {
const srcDir = resolve(producerRoot, "src");
const files = collectTestFiles(srcDir).map((absolutePath) => ({
absolutePath,
filePath: relative(producerRoot, absolutePath).replaceAll("\\", "/"),
}));
const discoveredFiles = new Set(files.map((test) => test.filePath));
const staleEntries = [...INTEGRATION_TEST_FILES].filter((file) => !discoveredFiles.has(file));
if (staleEntries.length > 0) {
throw new Error(`Integration test manifest contains missing files: ${staleEntries.join(", ")}`);
}
return files
.map(({ absolutePath, filePath }) => {
return classifyTestSource(filePath, readFileSync(absolutePath, "utf8"));
})
.sort((left, right) => left.file.localeCompare(right.file));
}
export function summarizeTests(tests) {
const summary = {
total: tests.length,
unit: { bun: 0, vitest: 0 },
integration: { bun: 0, vitest: 0 },
};
for (const test of tests) summary[test.lane][test.runner] += 1;
return summary;
}
@@ -0,0 +1,48 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
classifyTestSource,
discoverProducerTests,
summarizeTests,
} from "./test-classification.mjs";
describe("producer test classification", () => {
it("classifies each supported runner into one lane", () => {
assert.deepEqual(classifyTestSource("src/a.test.ts", 'import { it } from "bun:test";'), {
file: "src/a.test.ts",
runner: "bun",
lane: "unit",
});
assert.deepEqual(
classifyTestSource(
"src/a.test.ts",
'import { it } from "vitest";',
new Set(["src/a.test.ts"]),
),
{ file: "src/a.test.ts", runner: "vitest", lane: "integration" },
);
});
it("rejects missing and ambiguous runner imports", () => {
assert.throws(() => classifyTestSource("src/a.test.ts", "export {};"), /neither runner/);
assert.throws(
() =>
classifyTestSource(
"src/a.test.ts",
'import { it } from "bun:test"; import { expect } from "vitest";',
),
/both bun:test and vitest/,
);
});
it("classifies every current source test exactly once", () => {
const tests = discoverProducerTests();
assert.equal(new Set(tests.map((test) => test.file)).size, tests.length);
const summary = summarizeTests(tests);
assert.ok(summary.total > 0);
assert.equal(
summary.unit.bun + summary.unit.vitest + summary.integration.bun + summary.integration.vitest,
summary.total,
);
});
});
+12 -9
View File
@@ -1,3 +1,4 @@
// fallow-ignore-file complexity
import {
existsSync,
mkdirSync,
@@ -1510,12 +1511,14 @@ async function run(): Promise<void> {
}
}
void run().catch((error) => {
console.error(
JSON.stringify({
event: "test_suite_fatal",
message: error instanceof Error ? error.message : String(error),
}),
);
process.exitCode = 1;
});
if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
void run().catch((error) => {
console.error(
JSON.stringify({
event: "test_suite_fatal",
message: error instanceof Error ? error.message : String(error),
}),
);
process.exitCode = 1;
});
}
@@ -59,6 +59,7 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)"
it("swallows a network failure and returns the original HTML (no throw)", async () => {
const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
failClosedFontFetch: false,
allowSystemFontCapture: false,
fetchImpl: makeFailingFetch(),
});
// No @font-face was injected because the fetch failed — but the call
@@ -69,6 +70,7 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)"
it("swallows a 404 response and returns the original HTML (no throw)", async () => {
const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
failClosedFontFetch: false,
allowSystemFontCapture: false,
fetchImpl: makeHttp404Fetch(),
});
expect(result.includes("data-hyperframes-deterministic-fonts")).toBe(false);
@@ -77,6 +79,7 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)"
it("swallows a 5xx response and returns the original HTML (no throw)", async () => {
const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
failClosedFontFetch: false,
allowSystemFontCapture: false,
fetchImpl: makeHttp503Fetch(),
});
expect(result.includes("data-hyperframes-deterministic-fonts")).toBe(false);
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication complexity
/**
* Cross-worker idempotency: rendering the same `(planDir, chunkIndex)` on two
* different workers MUST produce byte-identical output. This is what makes a
@@ -41,7 +42,7 @@ const FIXTURE_HTML = `<!doctype html>
<html>
<head><meta charset="utf-8"><title>cross-worker idempotency fixture</title></head>
<body style="margin:0;background:#000;color:#fff;font:32px sans-serif">
<div data-composition-id="root" data-width="160" data-height="120" data-duration="0.16667">
<div data-composition-id="root" data-no-timeline data-width="160" data-height="120" data-duration="0.16667">
<p style="padding:1rem">chunk fixture</p>
</div>
</body>
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication complexity
/**
* Unit tests for `services/distributed/renderChunk.ts`.
*
@@ -41,7 +42,7 @@ const FIXTURE_HTML = `<!doctype html>
<html>
<head><meta charset="utf-8"><title>renderChunk fixture</title></head>
<body style="margin:0;background:#000;color:#fff;font:32px sans-serif">
<div data-composition-id="root" data-width="160" data-height="120" data-duration="0.16667">
<div data-composition-id="root" data-no-timeline data-width="160" data-height="120" data-duration="0.16667">
<p style="padding:1rem">chunk fixture</p>
</div>
</body>
@@ -134,7 +135,7 @@ beforeAll(async () => {
planDir = join(runRoot, "plan");
mkdirSync(planDir, { recursive: true });
await plan(projectDir, { fps: 30, width: 160, height: 120, format: "png-sequence" }, planDir);
});
}, 30_000);
afterAll(() => {
rmSync(runRoot, { recursive: true, force: true });
@@ -331,7 +332,7 @@ describe("renderChunk() — variables threading", () => {
<html data-composition-variables='{"color":"string"}'>
<head><meta charset="utf-8"><title>renderChunk variables fixture</title></head>
<body style="margin:0">
<div data-composition-id="root" data-width="160" data-height="120" data-duration="0.16667">
<div data-composition-id="root" data-no-timeline data-width="160" data-height="120" data-duration="0.16667">
<div id="paint" style="width:160px;height:120px;background:#000"></div>
</div>
<script>
@@ -19,11 +19,14 @@ let failInitializeSession = false;
let initializeSessionErrorMessage = "initialize failed";
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
const closeCaptureSession = mock(async () => {});
class DrawElementVerificationError extends Error {}
mock.module("@hyperframes/engine", () => ({
calculateOptimalWorkers: () => 1,
convertTransfer: () => {},
captureFrame: async () => {},
captureFrameToBufferPipelined: async () => ({ encodeResult: { buffer: Buffer.from("frame") } }),
captureFramesBatchPipelined: async () => [],
captureFrameToBuffer: async () => {
if (failCaptureFrameToBuffer) {
throw new Error("captureFrameToBuffer failed");
@@ -31,13 +34,23 @@ mock.module("@hyperframes/engine", () => ({
return { buffer: Buffer.from("frame"), captureTimeMs: 1 };
},
closeCaptureSession,
createCaptureSession: async () => ({ isInitialized: false, browserConsoleBuffer }),
completeDeferredDrawElementInit: async () => {},
createCaptureSession: async () => ({
isInitialized: false,
browserConsoleBuffer,
options: { captureBeyondViewport: false },
workerEncodeEnabled: false,
}),
createFrameReorderBuffer: () => ({
waitForFrame: async () => {},
advanceTo: () => {},
}),
distributeFrames: () => [],
distributeFramesInterleaved: () => [],
DrawElementVerificationError,
executeParallelCapture: async () => {},
getCapturePerfSummary: () => ({}),
getFfmpegBinary: () => "ffmpeg",
initializeSession: async (session: { isInitialized: boolean }) => {
if (failInitializeSession) {
throw new Error(initializeSessionErrorMessage);
@@ -52,12 +65,16 @@ mock.module("@hyperframes/engine", () => ({
}),
initTransparentBackground: async () => {},
prepareCaptureSessionForReuse: () => {},
recaptureDrawElementFrameForVerify: async () => Buffer.from("frame"),
spawnStreamingEncoder,
writeCapturedFrame: async () => {},
}));
mock.module("@hyperframes/core", () => ({
CANVAS_DIMENSIONS: {},
checkOutputResolutionCompatibility: () => ({ ok: true }),
fpsToNumber: () => 30,
redactTelemetryString: (value: string) => value,
}));
mock.module("../../renderOrchestrator.js", () => ({
@@ -67,6 +84,11 @@ mock.module("../../renderOrchestrator.js", () => ({
resolveCompositeTransfer: () => "srgb",
}));
mock.module("../../hdrCompositor.js", () => ({
closeHdrVideoFrameSource: () => {},
resolveCompositeTransfer: () => "srgb",
}));
mock.module("./captureHdrResources.js", () => ({
decodeHdrImageBuffers: () => new Map(),
extractHdrVideoFrames: async () => new Map(),
@@ -79,6 +101,7 @@ mock.module("./captureHdrResources.js", () => ({
}));
mock.module("./captureHdrFrameShared.js", () => ({
ensureFrameWritten: () => {},
partitionTransitionFrames: () => new Set(),
shouldUseHybridLayeredPath: () => false,
}));
@@ -1165,10 +1165,15 @@ describe("resolveRenderWorkerCount", () => {
debug: vi.fn(),
};
const stableCfg = {
...cfg,
concurrency: 2 as const,
largeRenderThreshold: 1_000,
};
const workers = resolveRenderWorkerCount(
180,
undefined,
{ ...cfg, forceScreenshot: true },
{ ...stableCfg, forceScreenshot: true },
{
hasShaderTransitions: false,
renderModeHints: { recommendScreenshot: false, reasons: [] },
@@ -1177,7 +1182,7 @@ describe("resolveRenderWorkerCount", () => {
{ multiplier: 1, reasons: [], p95Ms: 180 },
);
expect(workers).toBe(6);
expect(workers).toBe(2);
expect(log.warn).not.toHaveBeenCalled();
});
});
@@ -36,9 +36,9 @@ remuxed (no re-encode) so the HLG color tags survive verbatim.
```bash
cd packages/producer
bun run test hdr-hlg-regression
bun run test:regression hdr-hlg-regression
bun run test:update hdr-hlg-regression
bun run test:regression:update hdr-hlg-regression
```
In CI it runs in the `hdr` shard alongside `hdr-regression`
@@ -35,8 +35,8 @@ python3 packages/producer/tests/hdr-regression/scripts/generate-hdr-photo-pq.py
```bash
cd packages/producer
bun run test hdr-regression
bun run test:update hdr-regression
bun run test:regression hdr-regression
bun run test:regression:update hdr-regression
```
In CI it runs in the `hdr` shard alongside `hdr-hlg-regression`