diff --git a/.github/workflows/preview-regression.yml b/.github/workflows/preview-regression.yml new file mode 100644 index 000000000..161d7a4e1 --- /dev/null +++ b/.github/workflows/preview-regression.yml @@ -0,0 +1,135 @@ +name: preview-regression + +on: + pull_request: + push: + branches: + - main + +concurrency: + group: preview-regression-${{ github.ref }} + cancel-in-progress: true + +jobs: + changes: + name: Detect changes + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + preview: ${{ steps.filter.outputs.preview }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: dorny/paths-filter@v4 + id: filter + with: + token: "" + filters: | + preview: + - "packages/core/**" + - "packages/player/**" + - "packages/studio/**" + - "packages/cli/**" + - "packages/producer/src/parity-harness.ts" + - "packages/producer/src/parity-fixtures.ts" + - "packages/producer/tests/parity/**" + - "package.json" + - "bun.lock" + - ".github/workflows/preview-regression.yml" + + preview-parity: + name: Preview parity + needs: changes + if: needs.changes.outputs.preview == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - run: bun install --frozen-lockfile + + - name: Build preview runtime + run: bun run --cwd packages/core build:hyperframes-runtime + + - name: Prepare parity fixtures + run: bun run --cwd packages/producer parity:fixtures:ci + + - name: Install ffmpeg + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg + ffmpeg -version | head -n 1 + + - name: Set up Chrome + id: setup-chrome + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Start parity fixture server + run: | + cd packages/producer/tests/parity/fixtures + python3 -m http.server 4173 --bind 127.0.0.1 > /tmp/preview-parity-http.log 2>&1 & + echo "$!" > /tmp/preview-parity-http.pid + for _ in $(seq 1 30); do + if curl -fsS http://127.0.0.1:4173/minimal-wysiwyg.html >/dev/null; then + exit 0 + fi + sleep 1 + done + cat /tmp/preview-parity-http.log + exit 1 + + - name: Run preview parity check + working-directory: packages/producer + env: + PUPPETEER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} + run: bun run parity:check:ci + + - name: Upload parity artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: preview-parity-artifacts + path: packages/producer/.debug/parity-harness-ci/ + if-no-files-found: ignore + retention-days: 30 + + preview-regression: + runs-on: ubuntu-latest + needs: [changes, preview-parity] + if: always() + steps: + - name: Check results + env: + PREVIEW_FILTER_RESULT: ${{ needs.changes.outputs.preview }} + PREVIEW_PARITY_RESULT: ${{ needs.preview-parity.result }} + run: | + { + echo "## Preview regression gate" + echo "" + echo "- paths-filter \`preview\` matched: \`${PREVIEW_FILTER_RESULT}\`" + echo "- preview-parity result: \`${PREVIEW_PARITY_RESULT}\`" + echo "" + } >> "$GITHUB_STEP_SUMMARY" + + if [ "${PREVIEW_FILTER_RESULT}" != "true" ]; then + echo "::notice title=Preview regression::SKIPPED — no preview/runtime changes. Auto-pass." + echo "**Status:** SKIPPED (no preview/runtime changes — auto-pass)" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + if [ "${PREVIEW_PARITY_RESULT}" != "success" ]; then + echo "**Status:** FAILED" >> "$GITHUB_STEP_SUMMARY" + echo "Preview parity check failed" + exit 1 + fi + + echo "**Status:** PASSED" >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 55b6babf0..24a7d2c79 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ coverage/ # Producer regression test failures (generated debugging artifacts) packages/producer/tests/*/failures/ +packages/producer/tests/parity/fixtures/hyperframe.runtime.iife.js # Player perf test results (generated each run, attached as CI artifact) packages/player/tests/perf/results/ diff --git a/packages/producer/src/parity-fixtures.ts b/packages/producer/src/parity-fixtures.ts new file mode 100644 index 000000000..628e6628e --- /dev/null +++ b/packages/producer/src/parity-fixtures.ts @@ -0,0 +1,21 @@ +import { copyFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const sourceDir = dirname(fileURLToPath(import.meta.url)); +const producerRoot = resolve(sourceDir, ".."); +const repoRoot = resolve(producerRoot, "../.."); +const runtimePath = resolve(repoRoot, "packages/core/dist/hyperframe.runtime.iife.js"); +const fixturesDir = resolve(producerRoot, "tests/parity/fixtures"); +const fixtureRuntimePath = resolve(fixturesDir, "hyperframe.runtime.iife.js"); + +if (!existsSync(runtimePath)) { + throw new Error( + `Missing preview runtime at ${runtimePath}. Run "bun run --cwd packages/core build:hyperframes-runtime" first.`, + ); +} + +mkdirSync(fixturesDir, { recursive: true }); +copyFileSync(runtimePath, fixtureRuntimePath); + +console.log(`[ParityFixtures] copied ${runtimePath} -> ${fixtureRuntimePath}`); diff --git a/packages/producer/src/parity-harness.ts b/packages/producer/src/parity-harness.ts index 5a93f3127..e7d324f87 100644 --- a/packages/producer/src/parity-harness.ts +++ b/packages/producer/src/parity-harness.ts @@ -3,7 +3,7 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import process from "node:process"; import { spawnSync } from "node:child_process"; -import puppeteer, { type Page } from "puppeteer-core"; +import puppeteer, { type Browser, type Page } from "puppeteer-core"; import { MEDIA_VISUAL_STYLE_PROPERTIES, quantizeTimeToFrame } from "./utils/parityContract.js"; type ParityHarnessOptions = { @@ -262,12 +262,41 @@ async function captureCheckpoint( if (emulateProducerSwap) { await emulateProducerVideoSwap(page); } - await page.evaluate( - () => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))), - ); + await page.evaluate(`new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + resolve(); + }; + window.setTimeout(finish, 100); + requestAnimationFrame(() => requestAnimationFrame(finish)); + })`); return (await page.screenshot({ type: "png" })) as Buffer; } +async function captureParitySide( + browser: Browser, + url: string, + checkpointSec: number, + fps: number, + emulateProducerSwap: boolean, +): Promise<{ buffer: Buffer; styles: Record }> { + const page = await browser.newPage(); + try { + await page.goto(url, { + waitUntil: "load", + timeout: 60_000, + }); + await waitForParityReady(page); + const buffer = await captureCheckpoint(page, checkpointSec, fps, emulateProducerSwap); + const styles = await captureStyleSnapshot(page); + return { buffer, styles }; + } finally { + await page.close().catch(() => {}); + } +} + function sha256(data: Buffer): string { return createHash("sha256").update(data).digest("hex"); } @@ -277,8 +306,11 @@ async function run(): Promise { console.log(`[ParityHarness] options=${JSON.stringify(options)}`); ensureDir(options.artifactsDir); + const browserTarget = process.env.PUPPETEER_EXECUTABLE_PATH + ? { executablePath: process.env.PUPPETEER_EXECUTABLE_PATH } + : { channel: "chrome" as const }; const browser = await puppeteer.launch({ - channel: "chrome", + ...browserTarget, headless: true, defaultViewport: { width: options.width, @@ -301,21 +333,6 @@ async function run(): Promise { }); try { - const previewPage = await browser.newPage(); - const producerPage = await browser.newPage(); - - await Promise.all([ - previewPage.goto(options.previewUrl, { - waitUntil: ["load", "networkidle2"], - timeout: 60_000, - }), - producerPage.goto(options.producerUrl, { - waitUntil: ["load", "networkidle2"], - timeout: 60_000, - }), - ]); - await Promise.all([waitForParityReady(previewPage), waitForParityReady(producerPage)]); - let mismatches = 0; const results: Array<{ checkpointSec: number; @@ -333,10 +350,22 @@ async function run(): Promise { const checkpointKey = checkpointSec.toFixed(3).replace(/\./g, "_"); const artifactDir = join(options.artifactsDir, `checkpoint_${checkpointKey}s`); ensureDir(artifactDir); - const [previewBuffer, producerBuffer] = await Promise.all([ - captureCheckpoint(previewPage, checkpointSec, options.fps, false), - captureCheckpoint(producerPage, checkpointSec, options.fps, options.emulateProducerSwap), - ]); + const previewCapture = await captureParitySide( + browser, + options.previewUrl, + checkpointSec, + options.fps, + false, + ); + const producerCapture = await captureParitySide( + browser, + options.producerUrl, + checkpointSec, + options.fps, + options.emulateProducerSwap, + ); + const previewBuffer = previewCapture.buffer; + const producerBuffer = producerCapture.buffer; const previewHash = sha256(previewBuffer); const producerHash = sha256(producerBuffer); const match = previewHash === producerHash; @@ -349,10 +378,8 @@ async function run(): Promise { if (diffImagePath) { writeImageDiff(previewImagePath, producerImagePath, diffImagePath); } - const [previewStyles, producerStyles] = await Promise.all([ - captureStyleSnapshot(previewPage), - captureStyleSnapshot(producerPage), - ]); + const previewStyles = previewCapture.styles; + const producerStyles = producerCapture.styles; const previewStylesPath = join(artifactDir, "preview-styles.json"); const producerStylesPath = join(artifactDir, "producer-styles.json"); writeJson(previewStylesPath, previewStyles); diff --git a/packages/producer/tests/parity/fixtures/minimal-wysiwyg.html b/packages/producer/tests/parity/fixtures/minimal-wysiwyg.html index 8f81026d2..0a5f170d7 100644 --- a/packages/producer/tests/parity/fixtures/minimal-wysiwyg.html +++ b/packages/producer/tests/parity/fixtures/minimal-wysiwyg.html @@ -4,7 +4,6 @@ Parity Fixture - @@ -45,24 +62,5 @@ >
-