mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
* fix(cli): benchmark applies the received-files rule for files outside the project hyperframes benchmark now asks outsideAssetsAllowed() before it creates its render jobs, the same check hyperframes render and the Studio render route use. A project whose received-files record lists code files, or whose record cannot be read, renders without files from outside the project. * fix(cli): a received .env key never stands in for an expired login A HeyGen key that a received project's .env set is now used only when no login or API key is saved. When the saved login has expired, cannot refresh, belongs to another OAuth client, or HeyGen refused its refresh, the CLI resolver, afterRefreshFailure and the media-use skill's resolver raise the login error instead of switching to that key. A person's own .env key and a shell-exported key keep their current priority. * fix(cli): a broken credentials file never gives way to a received .env key A malformed or unreadable ~/.hyperframes/credentials now counts as a saved credential when deciding whether a received project's .env key may be used. The CLI resolver raises INVALID_STORE and the media-use skill says the file can't be read. A person's own .env key keeps its current behaviour. The skill now tells a person whose saved login belongs to another OAuth client to run auth login rather than auth refresh.
259 lines
9.7 KiB
TypeScript
259 lines
9.7 KiB
TypeScript
import { failCommand, failUsage, setCommandExitCode } from "../utils/commandResult.js";
|
|
import { defineCommand } from "../utils/defineCommand.js";
|
|
import type { Example } from "./_examples.js";
|
|
import { existsSync, mkdtempSync, rmSync, statSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
|
|
export const examples: Example[] = [
|
|
["Run benchmarks with default settings (3 runs)", "hyperframes benchmark"],
|
|
["Run 5 iterations per config", "hyperframes benchmark --runs 5"],
|
|
["Output results as JSON", "hyperframes benchmark --json"],
|
|
];
|
|
import { join } from "node:path";
|
|
import { resolveProject } from "../utils/project.js";
|
|
import { loadProducer } from "../utils/producer.js";
|
|
import { outsideAssetsAllowed } from "../utils/receivedRender.js";
|
|
import { c } from "../ui/colors.js";
|
|
import { formatBytes, formatDuration, errorBox } from "../ui/format.js";
|
|
import * as clack from "../ui/prompts.js";
|
|
import { withMeta } from "../utils/updateCheck.js";
|
|
import { fpsToFfmpegArg } from "@hyperframes/core";
|
|
|
|
interface BenchmarkConfig {
|
|
label: string;
|
|
fps: import("@hyperframes/core").Fps;
|
|
quality: "draft" | "standard" | "high";
|
|
workers: number;
|
|
}
|
|
|
|
interface RunResult {
|
|
elapsedMs: number;
|
|
fileSize: number | null;
|
|
}
|
|
|
|
interface ConfigResult {
|
|
config: BenchmarkConfig;
|
|
runs: RunResult[];
|
|
failures: number;
|
|
avgTime: number | null;
|
|
avgSize: number | null;
|
|
}
|
|
|
|
const FPS_30: import("@hyperframes/core").Fps = { num: 30, den: 1 };
|
|
const FPS_60: import("@hyperframes/core").Fps = { num: 60, den: 1 };
|
|
const DEFAULT_CONFIGS: BenchmarkConfig[] = [
|
|
{ label: "30fps \u00B7 draft \u00B7 2w", fps: FPS_30, quality: "draft", workers: 2 },
|
|
{ label: "30fps \u00B7 standard \u00B7 2w", fps: FPS_30, quality: "standard", workers: 2 },
|
|
{ label: "30fps \u00B7 high \u00B7 2w", fps: FPS_30, quality: "high", workers: 2 },
|
|
{ label: "30fps \u00B7 standard \u00B7 4w", fps: FPS_30, quality: "standard", workers: 4 },
|
|
{ label: "60fps \u00B7 standard \u00B7 4w", fps: FPS_60, quality: "standard", workers: 4 },
|
|
];
|
|
|
|
export default defineCommand({
|
|
meta: {
|
|
name: "benchmark",
|
|
description: "Render with preset fps/quality/worker configs and compare speed and file size",
|
|
},
|
|
args: {
|
|
dir: { type: "positional", description: "Project directory", required: false },
|
|
runs: { type: "string", description: "Number of runs per config", default: "3" },
|
|
json: { type: "boolean", description: "Output results as JSON", default: false },
|
|
},
|
|
async run({ args }) {
|
|
// ── Parse runs ───────────────────────────────────────────────────────
|
|
const runsPerConfig = parseInt(args.runs ?? "3", 10);
|
|
if (isNaN(runsPerConfig) || runsPerConfig < 1 || runsPerConfig > 20) {
|
|
errorBox("Invalid runs", `Got "${args.runs ?? "3"}". Must be between 1 and 20.`);
|
|
failUsage();
|
|
}
|
|
|
|
// ── Resolve project ──────────────────────────────────────────────────
|
|
const project = resolveProject(args.dir);
|
|
|
|
const jsonOutput = args.json ?? false;
|
|
|
|
// ── Temp output for benchmark renders ────────────────────────────────
|
|
const benchDir = mkdtempSync(join(tmpdir(), "hyperframes-benchmark-"));
|
|
|
|
// ── Load producer ────────────────────────────────────────────────────
|
|
let producer: Awaited<ReturnType<typeof loadProducer>> | null = null;
|
|
try {
|
|
producer = await loadProducer();
|
|
} catch {
|
|
if (jsonOutput) {
|
|
console.log(
|
|
JSON.stringify({ error: "Producer module not available. Is the project built?" }),
|
|
);
|
|
} else {
|
|
errorBox(
|
|
"Producer module not available",
|
|
"The rendering pipeline could not be loaded.",
|
|
"Ensure @hyperframes/producer is built and linked.",
|
|
);
|
|
}
|
|
failCommand();
|
|
}
|
|
|
|
// ── Print header ─────────────────────────────────────────────────────
|
|
if (!jsonOutput) {
|
|
console.log("");
|
|
console.log(
|
|
c.accent("\u25C6") +
|
|
" Benchmarking " +
|
|
c.accent(project.name) +
|
|
c.dim(` (${runsPerConfig} runs each)`),
|
|
);
|
|
console.log("");
|
|
}
|
|
|
|
// ── Run benchmarks ───────────────────────────────────────────────────
|
|
const allowOutsideProjectAssets = outsideAssetsAllowed(project.dir, undefined, (line) =>
|
|
console.warn(c.warn(` ${line}`)),
|
|
);
|
|
const results: ConfigResult[] = [];
|
|
|
|
for (const config of DEFAULT_CONFIGS) {
|
|
const runs: RunResult[] = [];
|
|
let failures = 0;
|
|
|
|
const s = !jsonOutput ? clack.spinner() : undefined;
|
|
s?.start(`Benchmarking ${config.label}...`);
|
|
|
|
for (let i = 0; i < runsPerConfig; i++) {
|
|
s?.message(`${config.label} — run ${i + 1}/${runsPerConfig}`);
|
|
const outputPath = join(
|
|
benchDir,
|
|
`${config.label.replace(/[^a-zA-Z0-9]/g, "_")}_run${i}.mp4`,
|
|
);
|
|
|
|
try {
|
|
const startTime = Date.now();
|
|
const job = producer.createRenderJob({
|
|
fps: config.fps,
|
|
quality: config.quality,
|
|
workers: config.workers,
|
|
allowOutsideProjectAssets,
|
|
});
|
|
await producer.executeRenderJob(job, project.dir, outputPath);
|
|
const elapsedMs = Date.now() - startTime;
|
|
|
|
let fileSize: number | null = null;
|
|
if (existsSync(outputPath)) {
|
|
const stat = statSync(outputPath);
|
|
fileSize = stat.size;
|
|
}
|
|
|
|
runs.push({ elapsedMs, fileSize });
|
|
} catch {
|
|
failures++;
|
|
}
|
|
}
|
|
|
|
s?.stop(`${config.label} — ${runs.length} runs${failures > 0 ? `, ${failures} failed` : ""}`);
|
|
|
|
const successfulRuns = runs.filter((r) => r.elapsedMs > 0);
|
|
const avgTime =
|
|
successfulRuns.length > 0
|
|
? successfulRuns.reduce((sum, r) => sum + r.elapsedMs, 0) / successfulRuns.length
|
|
: null;
|
|
const sizesWithValues = runs.map((r) => r.fileSize).filter((s): s is number => s != null);
|
|
const avgSize =
|
|
sizesWithValues.length > 0
|
|
? sizesWithValues.reduce((sum, s) => sum + s, 0) / sizesWithValues.length
|
|
: null;
|
|
|
|
results.push({ config, runs, failures, avgTime, avgSize });
|
|
}
|
|
|
|
rmSync(benchDir, { recursive: true, force: true });
|
|
|
|
if (results.every((result) => result.runs.length === 0)) {
|
|
setCommandExitCode(1);
|
|
}
|
|
|
|
// ── Output results ───────────────────────────────────────────────────
|
|
if (jsonOutput) {
|
|
console.log(
|
|
JSON.stringify(
|
|
withMeta({
|
|
results: results.map((r) => ({
|
|
config: r.config.label,
|
|
// Emit fps in the on-the-wire form so the JSON payload is
|
|
// round-trippable through `parseFps` (e.g. "30000/1001" stays
|
|
// exact; integer fps stays integer).
|
|
fps: fpsToFfmpegArg(r.config.fps),
|
|
quality: r.config.quality,
|
|
workers: r.config.workers,
|
|
avgTimeMs: r.avgTime,
|
|
avgSizeBytes: r.avgSize,
|
|
failures: r.failures,
|
|
runs: r.runs,
|
|
})),
|
|
}),
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
return;
|
|
}
|
|
|
|
// ── Table output ─────────────────────────────────────────────────────
|
|
const configColWidth = 26;
|
|
const timeColWidth = 10;
|
|
const sizeColWidth = 10;
|
|
|
|
const header =
|
|
" " +
|
|
c.bold("Config".padEnd(configColWidth)) +
|
|
c.bold("Time".padEnd(timeColWidth)) +
|
|
c.bold("Size".padEnd(sizeColWidth));
|
|
const separator = " " + c.dim("\u2500".repeat(configColWidth + timeColWidth + sizeColWidth));
|
|
|
|
console.log(header);
|
|
console.log(separator);
|
|
|
|
for (const result of results) {
|
|
const timeStr = result.avgTime != null ? formatDuration(result.avgTime) : c.dim("failed");
|
|
const sizeStr = result.avgSize != null ? formatBytes(result.avgSize) : c.dim("n/a");
|
|
const failStr = result.failures > 0 ? c.warn(` (${result.failures} failed)`) : "";
|
|
|
|
console.log(
|
|
" " +
|
|
result.config.label.padEnd(configColWidth) +
|
|
timeStr.padEnd(timeColWidth) +
|
|
sizeStr.padEnd(sizeColWidth) +
|
|
failStr,
|
|
);
|
|
}
|
|
|
|
// ── Summary ──────────────────────────────────────────────────────────
|
|
const successfulResults = results.filter((r) => r.avgTime != null);
|
|
if (successfulResults.length > 0) {
|
|
let fastest = successfulResults[0];
|
|
for (const r of successfulResults) {
|
|
if (fastest == null || r.avgTime == null) continue;
|
|
if (fastest.avgTime == null || r.avgTime < fastest.avgTime) {
|
|
fastest = r;
|
|
}
|
|
}
|
|
|
|
if (fastest?.avgTime != null) {
|
|
console.log("");
|
|
console.log(
|
|
c.success("\u25C7") +
|
|
" Fastest: " +
|
|
c.accent(fastest.config.label) +
|
|
c.dim(` (${formatDuration(fastest.avgTime)})`),
|
|
);
|
|
}
|
|
} else {
|
|
console.log("");
|
|
console.log(
|
|
c.error("\u2717") + " All configurations failed. Ensure the rendering pipeline is set up.",
|
|
);
|
|
}
|
|
|
|
console.log("");
|
|
},
|
|
});
|