mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-15 13:58:20 +00:00
test(studio): gate playback cost with the Catalog open
A new step in the Studio timeline viewport job plays a small GSAP film with the Catalog showing. It holds three work counts exactly through perf-ceilings.json (Catalog elements, composited layers, eventful timeline seeks per frame), and times main-thread CPU per frame against the base branch's Studio built on the same runner, in alternating runs, failing above 1.15x. Chrome runs without vsync or a frame cap, so a 60 Hz runner measures what a frame costs, not the display's pace. The catalog, its posters and GSAP are served by the script, so the counts never follow the live registry.
This commit is contained in:
Miguel Ángel
@@ -144,6 +144,7 @@ jobs:
|
||||
- "packages/studio/**"
|
||||
- "packages/core/**"
|
||||
- "packages/studio-server/**"
|
||||
- "packages/player/**"
|
||||
- "bun.lock"
|
||||
- ".github/workflows/ci.yml"
|
||||
gcp_beginframe:
|
||||
@@ -763,7 +764,7 @@ jobs:
|
||||
needs: [changes]
|
||||
if: needs.changes.outputs.studio == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 18
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
@@ -786,6 +787,8 @@ jobs:
|
||||
packages/studio/data/projects/timeline-virtualization
|
||||
cp -R packages/studio/tests/e2e/fixtures/studio-open \
|
||||
packages/studio/data/projects/studio-open
|
||||
cp -R packages/studio/tests/e2e/fixtures/studio-playback \
|
||||
packages/studio/data/projects/studio-playback
|
||||
- name: Read the base branch's work-count ceilings
|
||||
# A ceiling may not disappear, or rise without a new reason, against the branch this PR merges into.
|
||||
if: github.event_name == 'pull_request'
|
||||
@@ -797,6 +800,21 @@ jobs:
|
||||
rm -f /tmp/base-perf-ceilings.json
|
||||
echo "The base branch has no perf-ceilings.json yet"
|
||||
fi
|
||||
- name: Build the base branch's Studio for the playback timing
|
||||
# The playback gate times this Studio against its base on the same runner, in
|
||||
# alternating runs, so the runner's speed cancels out of the ratio.
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git worktree add --detach /tmp/hf-base FETCH_HEAD
|
||||
cd /tmp/hf-base
|
||||
bash scripts/ci/install-workspace-dependencies.sh
|
||||
bun run --filter '@hyperframes/{parsers,lint,studio-server}' build
|
||||
bun run --cwd packages/core build
|
||||
bun run --cwd packages/core build:hyperframes-runtime
|
||||
mkdir -p packages/studio/data/projects
|
||||
cp -R "$GITHUB_WORKSPACE/packages/studio/tests/e2e/fixtures/studio-playback" \
|
||||
packages/studio/data/projects/studio-playback
|
||||
- name: Run both arms of the timeline viewport gate
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -893,6 +911,48 @@ jobs:
|
||||
node packages/studio/tests/e2e/perf-ratchet.mjs check \
|
||||
packages/studio/tests/e2e/perf-ceilings.json studio-open /tmp/studio-open-counts.json \
|
||||
/tmp/base-perf-ceilings.json
|
||||
- name: Count and time playback with the Catalog open
|
||||
# Counts go through perf-ceilings.json like studio-open. Time is only ever a ratio
|
||||
# against the base branch's Studio on this runner, so a slow runner cannot fail it.
|
||||
# Both servers stay up; they sit idle while the other one is measured.
|
||||
if: success() || failure()
|
||||
run: |
|
||||
set -uo pipefail
|
||||
PIDS=()
|
||||
trap 'kill "${PIDS[@]}" 2>/dev/null || true' EXIT
|
||||
NODE_ENV=production bun run --cwd packages/studio dev -- --port 5316 --strictPort &
|
||||
PIDS+=($!)
|
||||
PORTS=(5316)
|
||||
BASE_URL=""
|
||||
if [[ -d /tmp/hf-base ]]; then
|
||||
(cd /tmp/hf-base && NODE_ENV=production bun run --cwd packages/studio dev -- --port 5317 --strictPort) &
|
||||
PIDS+=($!)
|
||||
PORTS+=(5317)
|
||||
BASE_URL="http://localhost:5317/#project/studio-playback"
|
||||
fi
|
||||
for port in "${PORTS[@]}"; do
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf "http://localhost:${port}/" >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
if ! curl -sf "http://localhost:${port}/" >/dev/null 2>&1; then
|
||||
echo "FAIL: studio dev server did not start on port ${port}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
STUDIO_URL="http://localhost:5316/#project/studio-playback" \
|
||||
STUDIO_BASE_URL="$BASE_URL" \
|
||||
node packages/studio/tests/e2e/studio-playback.mjs \
|
||||
| tee /tmp/studio-playback.json
|
||||
TIMING_STATUS=$?
|
||||
node packages/studio/tests/e2e/perf-ratchet.mjs check \
|
||||
packages/studio/tests/e2e/perf-ceilings.json studio-playback /tmp/studio-playback.json \
|
||||
/tmp/base-perf-ceilings.json
|
||||
COUNTS_STATUS=$?
|
||||
if (( TIMING_STATUS != 0 || COUNTS_STATUS != 0 )); then
|
||||
echo "FAIL: playback timing=${TIMING_STATUS}, counts=${COUNTS_STATUS}"
|
||||
exit 1
|
||||
fi
|
||||
- name: Upload gate evidence
|
||||
# The gate's whole output is machine-readable evidence, and a red run is
|
||||
# exactly when someone needs to read it. Keep it on failure too.
|
||||
@@ -903,6 +963,7 @@ jobs:
|
||||
path: |
|
||||
/tmp/timeline-gate-*.json
|
||||
/tmp/studio-open-counts.json
|
||||
/tmp/studio-playback.json
|
||||
if-no-files-found: error
|
||||
|
||||
smoke-global-install:
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# Studio playback fixture
|
||||
|
||||
A 12 s film of 24 GSAP-animated tiles and a title, with no media. `studio-playback.mjs` opens it,
|
||||
closes the Compositions tab so the Catalog shows, plays it and records what each frame costs.
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "https://hyperframes.heygen.com/schema/hyperframes.json",
|
||||
"paths": {
|
||||
"blocks": "compositions",
|
||||
"components": "compositions/components",
|
||||
"assets": "assets"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Studio playback fixture</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/gsap@3.14.2/dist/gsap.min.js"></script>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
width: 1920px;
|
||||
height: 1080px;
|
||||
margin: 0;
|
||||
overflow: hidden;
|
||||
background: #0f172a;
|
||||
font-family: Arial, sans-serif;
|
||||
}
|
||||
.tile {
|
||||
position: absolute;
|
||||
width: 160px;
|
||||
height: 160px;
|
||||
border-radius: 24px;
|
||||
background: #38bdf8;
|
||||
}
|
||||
.title {
|
||||
position: absolute;
|
||||
left: 120px;
|
||||
top: 820px;
|
||||
margin: 0;
|
||||
font-size: 120px;
|
||||
color: #fff;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main
|
||||
id="studio-playback"
|
||||
data-hf-id="studio-playback"
|
||||
data-composition-id="studio-playback"
|
||||
data-width="1920"
|
||||
data-height="1080"
|
||||
data-start="0"
|
||||
data-duration="12"
|
||||
data-fps="30"
|
||||
>
|
||||
<h1 class="title" data-hf-id="title">Playback</h1>
|
||||
</main>
|
||||
<script>
|
||||
const root = document.getElementById("studio-playback");
|
||||
const tl = gsap.timeline({ paused: true });
|
||||
for (let i = 0; i < 24; i += 1) {
|
||||
const tile = document.createElement("div");
|
||||
tile.className = "tile";
|
||||
tile.dataset.hfId = `tile-${i}`;
|
||||
tile.style.left = `${120 + (i % 8) * 210}px`;
|
||||
tile.style.top = `${120 + Math.floor(i / 8) * 210}px`;
|
||||
root.appendChild(tile);
|
||||
tl.to(
|
||||
tile,
|
||||
{
|
||||
x: 60,
|
||||
rotation: 180,
|
||||
opacity: 0.4,
|
||||
duration: 2,
|
||||
repeat: 5,
|
||||
yoyo: true,
|
||||
ease: "sine.inOut",
|
||||
},
|
||||
i * 0.05,
|
||||
);
|
||||
}
|
||||
tl.from(".title", { y: 80, opacity: 0, duration: 1 }, 0);
|
||||
window.__timelines = { "studio-playback": tl };
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -10,5 +10,13 @@
|
||||
"raised": {
|
||||
"total.requests.thumbnail": "#4401 shows a cached frame-0 poster while the preview boots: a cache-only ask that misses on a fresh open, then a render of that poster for the next open (+2 requests)"
|
||||
}
|
||||
},
|
||||
"studio-playback": {
|
||||
"browser": "154",
|
||||
"counts": {
|
||||
"playing.catalogElements": 6878,
|
||||
"playing.compositedLayers": 172,
|
||||
"playing.maxSeeksPerFrame": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env node
|
||||
// Plays the studio-playback fixture with the Catalog open and prints evidence: exact work counts for
|
||||
// perf-ratchet.mjs and main-thread CPU per frame. With STUDIO_BASE_URL (the base branch's Studio on this runner)
|
||||
// it alternates runs of both and exits 1 when this build's median CPU per frame exceeds MAX_CPU_RATIO of the base's.
|
||||
// Chrome runs without vsync or a frame cap, so a frame costs what the page makes it cost, even on a 60 Hz runner.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import puppeteer from "puppeteer-core";
|
||||
import { resolveChromeExecutable } from "./chrome-executable.mjs";
|
||||
|
||||
const HEAD_URL = process.env.STUDIO_URL;
|
||||
const BASE_URL = process.env.STUDIO_BASE_URL;
|
||||
const ROUNDS = Number(process.env.STUDIO_PLAYBACK_ROUNDS || 3);
|
||||
const PLAY_MS = 4_000;
|
||||
const MAX_CPU_RATIO = 1.15;
|
||||
// A catalog the size of the real registry, served by this script so the counts never follow it.
|
||||
const CATALOG_SIZE = 400;
|
||||
const CATEGORIES = ["transitions", "vfx", "social", "data", "scenes", "captions", "effects"];
|
||||
|
||||
if (!HEAD_URL) {
|
||||
console.error("STUDIO_URL is required and must point at the studio-playback fixture");
|
||||
process.exit(2);
|
||||
}
|
||||
const executablePath = resolveChromeExecutable();
|
||||
if (!executablePath) {
|
||||
console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");
|
||||
process.exit(2);
|
||||
}
|
||||
const gsapSource = readFileSync(
|
||||
createRequire(import.meta.url).resolve("gsap/dist/gsap.min.js"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const catalog = Array.from({ length: CATALOG_SIZE }, (_, i) => ({
|
||||
name: `perf-block-${i}`,
|
||||
type: "hyperframes:block",
|
||||
title: `Block ${i}`,
|
||||
description: `Catalog entry ${i} for the playback gate`,
|
||||
tags: [CATEGORIES[i % CATEGORIES.length]],
|
||||
duration: 5,
|
||||
preview: { poster: `/__studio-playback/poster/${i}.svg` },
|
||||
}));
|
||||
const poster = (i) =>
|
||||
`<svg xmlns="http://www.w3.org/2000/svg" width="640" height="360"><rect width="640" height="360" fill="hsl(${(i * 37) % 360} 60% 40%)"/><text x="40" y="200" font-size="64" fill="#fff">${i}</text></svg>`;
|
||||
|
||||
/** A response for each fixture this script serves in place of the network, by URL. */
|
||||
// fallow-ignore-next-line complexity
|
||||
function fixtureResponse(url) {
|
||||
if (url.pathname.endsWith("/api/registry/blocks")) {
|
||||
return { contentType: "application/json", body: JSON.stringify(catalog) };
|
||||
}
|
||||
const posterMatch = /^\/__studio-playback\/poster\/(\d+)\.svg$/.exec(url.pathname);
|
||||
if (posterMatch) return { contentType: "image/svg+xml", body: poster(Number(posterMatch[1])) };
|
||||
const isGsap = url.hostname === "cdn.jsdelivr.net" && url.pathname.endsWith("/gsap.min.js");
|
||||
return isGsap ? { contentType: "application/javascript", body: gsapSource } : null;
|
||||
}
|
||||
|
||||
/** Serves the catalog, its posters and GSAP locally, and refuses every other off-origin request. */
|
||||
async function serveFixtures(page, origin) {
|
||||
await page.setRequestInterception(true);
|
||||
page.on("request", (request) => {
|
||||
const url = new URL(request.url());
|
||||
const fixture = fixtureResponse(url);
|
||||
if (fixture) return request.respond(fixture);
|
||||
return url.origin === origin ? request.continue() : request.abort();
|
||||
});
|
||||
}
|
||||
|
||||
/** Counts, per composition animation frame, the eventful seeks of GSAP timelines. */
|
||||
function instrumentPlayback() {
|
||||
const player = document.querySelector("hyperframes-player");
|
||||
const win = player.shadowRoot.querySelector("iframe").contentWindow;
|
||||
const timelines = Object.values(win.__timelines).filter(
|
||||
(tl) => typeof tl?.totalTime === "function",
|
||||
);
|
||||
if (timelines.length === 0) throw new Error("the composition registered no timeline to watch");
|
||||
const state = (window.__studioPlayback = { on: false, frames: [], current: 0 });
|
||||
// Studio wraps each timeline's own totalTime, so count on the instance, outside those wrappers.
|
||||
for (const tl of timelines) {
|
||||
const totalTime = tl.totalTime;
|
||||
tl.totalTime = function (time, suppressEvents) {
|
||||
const eventfulSeek = arguments.length > 0 && !suppressEvents;
|
||||
if (eventfulSeek && state.on) state.current += 1;
|
||||
return totalTime.apply(this, arguments);
|
||||
};
|
||||
}
|
||||
const frame = () => {
|
||||
if (state.on) state.frames.push(state.current);
|
||||
state.current = 0;
|
||||
win.requestAnimationFrame(frame);
|
||||
};
|
||||
win.requestAnimationFrame(frame);
|
||||
}
|
||||
|
||||
/** The renderer main thread's complete events, in time order. */
|
||||
function mainThreadEvents(traceEvents) {
|
||||
const main = traceEvents.find(
|
||||
(e) => e.ph === "M" && e.name === "thread_name" && e.args.name === "CrRendererMain",
|
||||
);
|
||||
return traceEvents
|
||||
.filter((e) => e.pid === main.pid && e.tid === main.tid && e.ph === "X")
|
||||
.sort((a, b) => a.ts - b.ts);
|
||||
}
|
||||
|
||||
/** Main-thread CPU per produced frame: top-level task CPU between consecutive BeginMainFrames. */
|
||||
// fallow-ignore-next-line complexity
|
||||
function cpuPerFrameMs(traceEvents) {
|
||||
const events = mainThreadEvents(traceEvents);
|
||||
const frameStarts = events.filter((e) => e.name === "ProxyMain::BeginMainFrame").map((e) => e.ts);
|
||||
const cpu = new Array(Math.max(0, frameStarts.length - 1)).fill(0);
|
||||
let taskEnd = -1;
|
||||
for (const event of events) {
|
||||
if (event.ts < taskEnd) continue; // nested inside the previous top-level task
|
||||
taskEnd = event.ts + event.dur;
|
||||
const frame = frameStarts.findLastIndex((start) => start <= event.ts);
|
||||
if (frame >= 0 && frame < cpu.length) cpu[frame] += (event.tdur ?? event.dur) / 1000;
|
||||
}
|
||||
return cpu;
|
||||
}
|
||||
|
||||
const median = (values) => {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted.length ? sorted[Math.floor(sorted.length / 2)] : Number.NaN;
|
||||
};
|
||||
const round = (value) => Math.round(value * 100) / 100;
|
||||
|
||||
async function measure(url) {
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath,
|
||||
headless: true,
|
||||
args: [
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu-vsync",
|
||||
"--disable-frame-rate-limit",
|
||||
"--window-size=1600,900",
|
||||
],
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1600, height: 900 });
|
||||
await serveFixtures(page, new URL(url).origin);
|
||||
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const play = document.querySelector('button[aria-label="Play"]');
|
||||
return play instanceof HTMLButtonElement && !play.disabled;
|
||||
},
|
||||
{ timeout: 120_000, polling: 50 },
|
||||
);
|
||||
// #4529's setup: closing the Compositions tab leaves the Catalog showing.
|
||||
await page.click('[aria-label="Close Compositions"]');
|
||||
await page.waitForFunction(
|
||||
() => document.querySelectorAll('img[src*="/__studio-playback/poster/"]').length > 0,
|
||||
{ timeout: 30_000, polling: 100 },
|
||||
);
|
||||
// Posters below the panel's fold are lazy and never load, so wait for the first rows only.
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
[...document.querySelectorAll('img[src*="/__studio-playback/poster/"]')]
|
||||
.slice(0, 4)
|
||||
.every((img) => img.complete),
|
||||
{ timeout: 30_000, polling: 100 },
|
||||
);
|
||||
await page.evaluate(instrumentPlayback);
|
||||
await page.mouse.move(0, 0);
|
||||
await page.click('button[aria-label="Play"]');
|
||||
await page.mouse.move(0, 0);
|
||||
await page.waitForFunction(() => document.querySelector('button[aria-label="Pause"]'), {
|
||||
timeout: 10_000,
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
|
||||
await page.tracing.start({ categories: ["devtools.timeline", "toplevel", "cc"] });
|
||||
await page.evaluate(() => {
|
||||
window.__studioPlayback.on = true;
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, PLAY_MS));
|
||||
const counts = await page.evaluate(() => {
|
||||
window.__studioPlayback.on = false;
|
||||
// The Catalog's own subtree: the nearest ancestor of its search box that holds its posters.
|
||||
let panel = document.querySelector('input[aria-label="Search blocks"]');
|
||||
while (panel && !panel.querySelector('img[src*="/__studio-playback/poster/"]')) {
|
||||
panel = panel.parentElement;
|
||||
}
|
||||
return {
|
||||
catalogElements: panel ? panel.querySelectorAll("*").length : -1,
|
||||
seekFrames: window.__studioPlayback.frames,
|
||||
};
|
||||
});
|
||||
const trace = JSON.parse(Buffer.from(await page.tracing.stop()).toString("utf8"));
|
||||
// Layers are read paused: while playing, the film's own layers come and go with its tweens.
|
||||
await page.click('button[aria-label="Pause"]');
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
const cdp = await page.createCDPSession();
|
||||
const layers = new Promise((resolve) =>
|
||||
cdp.on("LayerTree.layerTreeDidChange", (event) => event.layers && resolve(event.layers)),
|
||||
);
|
||||
await cdp.send("LayerTree.enable");
|
||||
const compositedLayers = (await layers).length;
|
||||
const cpu = cpuPerFrameMs(trace.traceEvents ?? trace);
|
||||
const seekFrames = counts.seekFrames.slice(1);
|
||||
return {
|
||||
browser: await browser.version(),
|
||||
frames: cpu.length,
|
||||
cpuMedianMs: round(median(cpu)),
|
||||
workCounts: {
|
||||
"playing.catalogElements": counts.catalogElements,
|
||||
"playing.compositedLayers": compositedLayers,
|
||||
"playing.maxSeeksPerFrame": seekFrames.length ? Math.max(...seekFrames) : -1,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
const head = [];
|
||||
const base = [];
|
||||
for (let round_ = 0; round_ < ROUNDS; round_ += 1) {
|
||||
head.push(await measure(HEAD_URL));
|
||||
if (BASE_URL) base.push(await measure(BASE_URL));
|
||||
}
|
||||
// Counts must repeat exactly across runs of one build, or the ratchet would fail at random.
|
||||
const countSets = new Set(head.map((run) => JSON.stringify(run.workCounts)));
|
||||
if (countSets.size !== 1) {
|
||||
console.error(`work counts differ between runs of one build: ${[...countSets].join(" vs ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const evidence = {
|
||||
journey: "studio-playback",
|
||||
browser: head[0].browser,
|
||||
workCounts: head[0].workCounts,
|
||||
cpuMedianMsPerRun: head.map((run) => run.cpuMedianMs),
|
||||
framesPerRun: head.map((run) => run.frames),
|
||||
};
|
||||
let failed = false;
|
||||
if (BASE_URL) {
|
||||
const headMs = median(head.map((run) => run.cpuMedianMs));
|
||||
const baseMs = median(base.map((run) => run.cpuMedianMs));
|
||||
evidence.ab = {
|
||||
headCpuMedianMs: headMs,
|
||||
baseCpuMedianMs: baseMs,
|
||||
baseCpuMedianMsPerRun: base.map((run) => run.cpuMedianMs),
|
||||
baseWorkCounts: base[0].workCounts,
|
||||
ratio: round(headMs / baseMs),
|
||||
maxRatio: MAX_CPU_RATIO,
|
||||
};
|
||||
failed = headMs > baseMs * MAX_CPU_RATIO;
|
||||
}
|
||||
console.log(JSON.stringify(evidence, null, 2));
|
||||
if (failed) {
|
||||
console.error(
|
||||
`FAIL main-thread CPU per frame ${evidence.ab.headCpuMedianMs} ms against the base's ${evidence.ab.baseCpuMedianMs} ms (x${evidence.ab.ratio}, limit x${MAX_CPU_RATIO})`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user