From 04c48d5bc59a4d0008d45c1a1db6e6351380fac1 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 23 Mar 2026 19:08:25 +0000 Subject: [PATCH] ci(regression): add Docker-based regression test pipeline Port the regression test infrastructure from the internal repo to OSS. Runs golden-baseline visual/audio comparisons inside Docker for deterministic results. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/regression.yml | 151 +++++++++++++++++++++++++++++++ .gitignore | 3 + Dockerfile.test | 78 ++++++++++++++++ packages/producer/package.json | 3 + 4 files changed, 235 insertions(+) create mode 100644 .github/workflows/regression.yml create mode 100644 Dockerfile.test diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml new file mode 100644 index 000000000..5e36de820 --- /dev/null +++ b/.github/workflows/regression.yml @@ -0,0 +1,151 @@ +name: regression + +on: + pull_request: + push: + branches: + - main + +jobs: + regression-shards: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - shard: fast + args: "--sequential --exclude-tags slow" + - shard: required-styles + tests: "style-5-prod style-6-prod style-10-prod" + steps: + - name: Checkout (with LFS) + uses: actions/checkout@v4 + with: + lfs: true + + - name: Validate LFS files + run: | + echo "Checking golden baseline MP4s are real files (not LFS pointers)..." + for mp4 in packages/producer/tests/*/output/output.mp4; do + if [ -f "$mp4" ]; then + size=$(stat --format=%s "$mp4") + if [ "$size" -lt 1000 ]; then + echo "ERROR: $mp4 appears to be an LFS pointer ($size bytes)" + exit 1 + fi + echo "OK: $mp4 ($size bytes)" + fi + done + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build test Docker image (cached) + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.test + load: true + tags: hyperframes-producer:test + cache-from: type=gha,scope=regression-test-image + cache-to: type=gha,mode=max,scope=regression-test-image + + - name: "Run regression shard: ${{ matrix.shard }}" + run: | + echo "Shard: ${{ matrix.shard }}" + echo "Args: ${{ matrix.args || matrix.tests }}" + docker run --rm \ + --security-opt seccomp=unconfined \ + --shm-size=4g \ + -v ${{ github.workspace }}/packages/producer/tests:/app/packages/producer/tests \ + hyperframes-producer:test \ + ${{ matrix.args || matrix.tests }} + + - name: Upload failure artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: regression-failures-${{ matrix.shard }} + path: packages/producer/tests/*/failures/ + if-no-files-found: ignore + + # Summary job — matches the required check name in branch protection + regression: + runs-on: ubuntu-latest + needs: regression-shards + if: always() + steps: + - name: Check shard results + run: | + if [ "${{ needs.regression-shards.result }}" != "success" ]; then + echo "One or more regression shards failed" + exit 1 + fi + + regression-optional-styles: + runs-on: ubuntu-latest + continue-on-error: true + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - shard: styles-a + tests: "style-1-prod style-2-prod style-3-prod" + - shard: styles-b + tests: "style-4-prod style-11-prod style-12-prod" + - shard: styles-c + tests: "style-15-prod style-17-prod style-18-prod" + - shard: styles-d + tests: "style-7-prod style-13-prod style-16-prod" + steps: + - name: Checkout (with LFS) + uses: actions/checkout@v4 + with: + lfs: true + + - name: Validate LFS files + run: | + echo "Checking golden baseline MP4s (shard ${{ matrix.shard }})..." + for mp4 in packages/producer/tests/*/output/output.mp4; do + if [ -f "$mp4" ]; then + size=$(stat --format=%s "$mp4") + if [ "$size" -lt 1000 ]; then + echo "ERROR: $mp4 appears to be an LFS pointer ($size bytes)" + exit 1 + fi + fi + done + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build test Docker image (cached) + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.test + load: true + tags: hyperframes-producer:test + cache-from: type=gha,scope=regression-test-image + cache-to: type=gha,mode=max,scope=regression-test-image + + - name: "Run optional styles shard: ${{ matrix.shard }}" + run: | + echo "Shard: ${{ matrix.shard }}" + echo "Tests: ${{ matrix.tests }}" + docker run --rm \ + --security-opt seccomp=unconfined \ + --shm-size=4g \ + -v ${{ github.workspace }}/packages/producer/tests:/app/packages/producer/tests \ + hyperframes-producer:test \ + ${{ matrix.tests }} + + - name: Upload failure artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: regression-failures-${{ matrix.shard }} + path: packages/producer/tests/*/failures/ + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index a0abccb98..e997ddfc7 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ Thumbs.db coverage/ .debug/ +# Producer regression test failures (generated debugging artifacts) +packages/producer/tests/*/failures/ + # Rendered output (not test fixtures — those use git LFS) output/ !packages/producer/tests/*/output/ diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 000000000..8977ba742 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,78 @@ +# HyperFrames Producer - Regression Test Image +# +# Matches the production rendering environment (same Chromium, fonts, FFmpeg) +# but includes the full source + devDependencies for running the test harness. +# +# This ensures golden baselines match what production actually renders. +# +# Usage: +# docker build -f Dockerfile.test -t hyperframes-producer:test . +# docker run --rm -v ./packages/producer/tests:/app/packages/producer/tests hyperframes-producer:test +# docker run --rm -v ./packages/producer/tests:/app/packages/producer/tests hyperframes-producer:test --update + +FROM node:22-bookworm-slim + +# ── System dependencies (identical to production) ──────────────────────────── +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + chromium \ + libgbm1 \ + libnss3 \ + libatk-bridge2.0-0 \ + libdrm2 \ + libxcomposite1 \ + libxdamage1 \ + libxrandr2 \ + libcups2 \ + libasound2 \ + libpangocairo-1.0-0 \ + libxshmfence1 \ + libgtk-3-0 \ + # Font support — matches production + fonts-liberation \ + fonts-noto-color-emoji \ + fonts-noto-cjk \ + fonts-noto-core \ + fonts-noto-extra \ + fonts-noto-ui-core \ + fonts-freefont-ttf \ + fonts-dejavu-core \ + fontconfig \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean \ + && fc-cache -fv + +# Use system Chromium (same as production) +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium +ENV CONTAINER=true + +# Install chrome-headless-shell for deterministic BeginFrame rendering. +# This lightweight Chrome binary supports HeadlessExperimental.beginFrame. +# Install to ~/.cache/puppeteer/ where resolveHeadlessShellPath() looks. +RUN npx --yes @puppeteer/browsers install chrome-headless-shell@stable \ + --path /root/.cache/puppeteer \ + && find /root/.cache/puppeteer/chrome-headless-shell -name "chrome-headless-shell" -type f \ + && echo "chrome-headless-shell installed" + +WORKDIR /app + +# Install dependencies (full, including devDependencies for tsx + test harness) +COPY pnpm-workspace.yaml package.json pnpm-lock.yaml ./ +COPY packages/core/package.json packages/core/package.json +COPY packages/engine/package.json packages/engine/package.json +COPY packages/producer/package.json packages/producer/package.json +RUN corepack enable && corepack prepare pnpm@10 --activate \ + && pnpm install --frozen-lockfile + +# Copy source +COPY packages/core/ packages/core/ +COPY packages/engine/ packages/engine/ +COPY packages/producer/ packages/producer/ + +# Build core runtime artifacts (needed by renderer) +RUN pnpm --filter @hyperframes/core build:hyperframes-runtime:modular + +WORKDIR /app/packages/producer + +ENTRYPOINT ["pnpm", "exec", "tsx", "src/regression-harness.ts", "--", "--sequential"] diff --git a/packages/producer/package.json b/packages/producer/package.json index f3c9f59ab..da36546cc 100644 --- a/packages/producer/package.json +++ b/packages/producer/package.json @@ -36,6 +36,9 @@ "benchmark": "tsx src/benchmark.ts", "test": "tsx src/regression-harness.ts", "test:update": "tsx src/regression-harness.ts --update", + "docker:build:test": "docker build -f ../../Dockerfile.test -t hyperframes-producer:test ../..", + "docker:test": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test", + "docker:test:update": "docker run --rm --security-opt seccomp=unconfined --shm-size=2g -v ./tests:/app/packages/producer/tests hyperframes-producer:test --update", "prepublishOnly": "pnpm build" }, "dependencies": {