hyperframes/packages/player/tests/perf/perf-gate.ts

import { readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";

/**
 * Compares measured perf metrics against baseline.json with an allowed regression ratio.
 *
 * Mirrors packages/producer/src/perf-gate.ts: each metric has a baseline value, the
 * gate computes `max = baseline * (1 + allowedRegressionRatio)`, and any measured
 * value above max counts as a regression. In "measure" mode the script logs but
 * never exits non-zero — useful for the first runs while we collect realistic
 * baselines on the CI runner. Flip to "enforce" once baselines are committed.
 */

const HERE = dirname(fileURLToPath(import.meta.url));
const DEFAULT_BASELINE_PATH = resolve(HERE, "baseline.json");

export type Direction = "lower-is-better" | "higher-is-better";

export type Metric = {
  /** Display name, e.g. "comp_load_cold_p95_ms" */
  name: string;
  /** Key into baseline.json, e.g. "compLoadColdP95Ms" */
  baselineKey: keyof PerfBaseline;
  value: number;
  unit: string;
  direction: Direction;
  samples?: number[];
};

export type PerfBaseline = {
  compLoadColdP95Ms: number;
  compLoadWarmP95Ms: number;
  /**
   * Floor on `(compositionTime advanced) / (wallClock elapsed)` over a sustained
   * playback window — see packages/player/tests/perf/scenarios/02-fps.ts. A
   * healthy player keeps up with its intended speed and reads ~1.0; values
   * below 1.0 mean the composition clock fell behind real time, which is the
   * actual user-visible jank we want to gate against. Refresh-rate independent
   * by construction, so it does not saturate to display refresh on high-Hz
   * runners the way the previous `fpsMin` did. Direction: higher-is-better.
   */
  compositionTimeAdvancementRatioMin: number;
  scrubLatencyP95IsolatedMs: number;
  scrubLatencyP95InlineMs: number;
  driftMaxMs: number;
  driftP95Ms: number;
  paritySsimMin: number;
  allowedRegressionRatio: number;
};

export type GateMode = "measure" | "enforce";

export type GateResult = {
  metric: Metric;
  baseline: number;
  threshold: number;
  passed: boolean;
  ratio: number;
};

export function loadBaseline(path?: string): PerfBaseline {
  const baselinePath = path ?? process.env.PLAYER_PERF_BASELINE_PATH ?? DEFAULT_BASELINE_PATH;
  const raw = readFileSync(baselinePath, "utf-8");
  return JSON.parse(raw) as PerfBaseline;
}

export function evaluateMetric(metric: Metric, baseline: PerfBaseline): GateResult {
  const baselineValue = baseline[metric.baselineKey];
  if (typeof baselineValue !== "number") {
    throw new Error(`[player-perf] baseline missing numeric key: ${String(metric.baselineKey)}`);
  }
  const allowed = baseline.allowedRegressionRatio;
  const threshold =
    metric.direction === "lower-is-better"
      ? baselineValue * (1 + allowed)
      : baselineValue * (1 - allowed);
  const passed =
    metric.direction === "lower-is-better" ? metric.value <= threshold : metric.value >= threshold;
  const ratio = baselineValue === 0 ? 0 : metric.value / baselineValue;
  return { metric, baseline: baselineValue, threshold, passed, ratio };
}

export type GateReport = {
  passed: boolean;
  rows: GateResult[];
};

export function reportAndGate(
  metrics: Metric[],
  // `mode` is resolved upstream in packages/player/tests/perf/index.ts
  // (`parseArgs`): the default comes from PLAYER_PERF_MODE env or "measure", and
  // the CLI flag `--mode=measure|enforce` overrides it. The "flip to enforce"
  // TODO lives at that call site so it is a one-line change.
  mode: GateMode,
  baselinePath?: string,
): GateReport {
  const baseline = loadBaseline(baselinePath);
  const rows = metrics.map((m) => evaluateMetric(m, baseline));
  console.log("[PerfGate] mode=" + mode);
  for (const row of rows) {
    const status = row.passed ? "PASS" : "FAIL";
    const dir = row.metric.direction === "lower-is-better" ? "≤" : "≥";
    console.log(
      `[PerfGate] ${status} ${row.metric.name} = ${row.metric.value.toFixed(2)}${row.metric.unit} (baseline=${row.baseline}${row.metric.unit}, threshold ${dir} ${row.threshold.toFixed(2)}${row.metric.unit}, ratio=${row.ratio.toFixed(3)})`,
    );
  }
  const failed = rows.filter((r) => !r.passed);
  if (failed.length === 0) return { passed: true, rows };
  if (mode === "measure") {
    console.log(`[PerfGate] ${failed.length} regression(s) detected — measure mode, not failing`);
    return { passed: true, rows };
  }
  console.error(`[PerfGate] ${failed.length} regression(s) detected — enforce mode, failing`);
  return { passed: false, rows };
}