mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
feat(cli): batch rendering — one output per variables row with manifest (#1336)
Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
This commit is contained in:
Matt Van Horn
co-authored by
Matt Van Horn
Matt Van Horn
parent
d580f2a1d8
commit
e5a78ef6a2
@@ -0,0 +1,280 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import {
|
||||
BatchRenderInputError,
|
||||
parseBatchRows,
|
||||
prepareBatchRender,
|
||||
resolveOutputTemplate,
|
||||
runBatchRender,
|
||||
} from "./batchRender.js";
|
||||
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "hf-local-batch-render-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function writeJson(name: string, content: string): string {
|
||||
const path = join(tmpDir, name);
|
||||
writeFileSync(path, content, "utf8");
|
||||
return path;
|
||||
}
|
||||
|
||||
function writeIndex(schema = "[]"): string {
|
||||
return writeJson(
|
||||
"index.html",
|
||||
`<html data-composition-variables='${schema}'><body><div data-composition-id="root"></div></body></html>`,
|
||||
);
|
||||
}
|
||||
|
||||
function expectBatchError(fn: () => unknown, title: string): BatchRenderInputError {
|
||||
try {
|
||||
fn();
|
||||
} catch (error: unknown) {
|
||||
expect(error).toBeInstanceOf(BatchRenderInputError);
|
||||
if (error instanceof BatchRenderInputError) {
|
||||
expect(error.title).toBe(title);
|
||||
return error;
|
||||
}
|
||||
}
|
||||
throw new Error("Expected BatchRenderInputError");
|
||||
}
|
||||
|
||||
function eventType(value: unknown): string | undefined {
|
||||
if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||
return "type" in value && typeof value.type === "string" ? value.type : undefined;
|
||||
}
|
||||
|
||||
describe("parseBatchRows", () => {
|
||||
it("parses a JSON array of variable rows", () => {
|
||||
expect(parseBatchRows('[{"name":"Alice"},{"name":"Bob"}]', "rows.json")).toEqual([
|
||||
{ name: "Alice" },
|
||||
{ name: "Bob" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("parses an object with a rows array", () => {
|
||||
expect(parseBatchRows('{"rows":[{"name":"Alice"}]}', "rows.json")).toEqual([{ name: "Alice" }]);
|
||||
});
|
||||
|
||||
it("rejects non-object rows", () => {
|
||||
const error = expectBatchError(
|
||||
() => parseBatchRows('[{"name":"Alice"},null]', "rows.json"),
|
||||
"Invalid batch row",
|
||||
);
|
||||
expect(error.message).toMatch(/Row 1/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveOutputTemplate", () => {
|
||||
it("replaces row placeholders and index", () => {
|
||||
expect(resolveOutputTemplate("renders/{name}-{index}.mp4", { name: "Alice" }, 3)).toBe(
|
||||
"renders/Alice-3.mp4",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects missing placeholder keys", () => {
|
||||
const error = expectBatchError(
|
||||
() => resolveOutputTemplate("renders/{slug}.mp4", { name: "Alice" }, 0),
|
||||
"Invalid output template",
|
||||
);
|
||||
expect(error.message).toMatch(/Missing value/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prepareBatchRender", () => {
|
||||
it("resolves output paths and the manifest path", () => {
|
||||
const batchPath = writeJson("rows.json", '[{"name":"Alice"},{"name":"Bob"}]');
|
||||
const outDir = join(tmpDir, "renders");
|
||||
const prepared = prepareBatchRender({
|
||||
batchPath,
|
||||
outputTemplate: join(outDir, "{name}.mp4"),
|
||||
indexPath: writeIndex(),
|
||||
strictVariables: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
});
|
||||
|
||||
expect(prepared.rows.map((row) => row.outputPath)).toEqual([
|
||||
resolve(outDir, "Alice.mp4"),
|
||||
resolve(outDir, "Bob.mp4"),
|
||||
]);
|
||||
expect(prepared.manifestPath).toBe(resolve(outDir, "manifest.json"));
|
||||
});
|
||||
|
||||
it("rejects output collisions before rendering", () => {
|
||||
const batchPath = writeJson("rows.json", '[{"name":"Alice"},{"name":"Bob"}]');
|
||||
const error = expectBatchError(
|
||||
() =>
|
||||
prepareBatchRender({
|
||||
batchPath,
|
||||
outputTemplate: join(tmpDir, "same.mp4"),
|
||||
indexPath: writeIndex(),
|
||||
strictVariables: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
}),
|
||||
"Batch output collision",
|
||||
);
|
||||
expect(error.message).toMatch(/Rows 0 and 1/);
|
||||
});
|
||||
|
||||
it("fails strict variable validation per row", () => {
|
||||
const batchPath = writeJson("rows.json", '[{"title":"Hello"},{"title":3}]');
|
||||
const schema = '[{"id":"title","type":"string","label":"Title","default":"Untitled"}]';
|
||||
const error = expectBatchError(
|
||||
() =>
|
||||
prepareBatchRender({
|
||||
batchPath,
|
||||
outputTemplate: join(tmpDir, "{index}.mp4"),
|
||||
indexPath: writeIndex(schema),
|
||||
strictVariables: true,
|
||||
quiet: true,
|
||||
json: true,
|
||||
}),
|
||||
"Variable validation failed",
|
||||
);
|
||||
expect(error.message).toMatch(/row 1/);
|
||||
});
|
||||
|
||||
it("counts non-strict variable validation issues without failing", () => {
|
||||
const batchPath = writeJson("rows.json", '[{"title":3}]');
|
||||
const schema = '[{"id":"title","type":"string","label":"Title","default":"Untitled"}]';
|
||||
const prepared = prepareBatchRender({
|
||||
batchPath,
|
||||
outputTemplate: join(tmpDir, "{index}.mp4"),
|
||||
indexPath: writeIndex(schema),
|
||||
strictVariables: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
});
|
||||
|
||||
expect(prepared.variableIssueCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runBatchRender", () => {
|
||||
it("writes a manifest with completed rows", async () => {
|
||||
const prepared = prepareBatchRender({
|
||||
batchPath: writeJson("rows.json", '[{"name":"Alice"}]'),
|
||||
outputTemplate: join(tmpDir, "renders/{name}.mp4"),
|
||||
indexPath: writeIndex(),
|
||||
strictVariables: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
});
|
||||
|
||||
const manifest = await runBatchRender({
|
||||
prepared,
|
||||
concurrency: 1,
|
||||
failFast: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
renderOne: async () => ({ durationMs: 3000, renderTimeMs: 42 }),
|
||||
});
|
||||
|
||||
expect(manifest.completed).toBe(1);
|
||||
expect(manifest.failed).toBe(0);
|
||||
expect(manifest.rows[0]).toMatchObject({
|
||||
index: 0,
|
||||
status: "completed",
|
||||
durationMs: 3000,
|
||||
renderTimeMs: 42,
|
||||
error: null,
|
||||
});
|
||||
expect(readFileSync(prepared.manifestPath, "utf8")).toContain('"status": "completed"');
|
||||
});
|
||||
|
||||
it("emits JSON progress events when json mode is enabled", async () => {
|
||||
const prepared = prepareBatchRender({
|
||||
batchPath: writeJson("rows.json", '[{"name":"Alice"}]'),
|
||||
outputTemplate: join(tmpDir, "renders/{name}.mp4"),
|
||||
indexPath: writeIndex(),
|
||||
strictVariables: false,
|
||||
quiet: true,
|
||||
json: true,
|
||||
});
|
||||
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
|
||||
|
||||
await runBatchRender({
|
||||
prepared,
|
||||
concurrency: 1,
|
||||
failFast: false,
|
||||
quiet: true,
|
||||
json: true,
|
||||
renderOne: async () => ({ renderTimeMs: 10 }),
|
||||
});
|
||||
|
||||
const events = log.mock.calls.map((call): unknown => JSON.parse(String(call[0])));
|
||||
expect(events.map(eventType)).toEqual([
|
||||
"batch-row-start",
|
||||
"batch-row-complete",
|
||||
"batch-complete",
|
||||
]);
|
||||
});
|
||||
|
||||
it("continues after row failure by default", async () => {
|
||||
const prepared = prepareBatchRender({
|
||||
batchPath: writeJson("rows.json", '[{"name":"Alice"},{"name":"Bob"}]'),
|
||||
outputTemplate: join(tmpDir, "renders/{name}.mp4"),
|
||||
indexPath: writeIndex(),
|
||||
strictVariables: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
});
|
||||
|
||||
const seen: number[] = [];
|
||||
const manifest = await runBatchRender({
|
||||
prepared,
|
||||
concurrency: 1,
|
||||
failFast: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
renderOne: async (row) => {
|
||||
seen.push(row.index);
|
||||
if (row.index === 0) throw new Error("boom");
|
||||
return { renderTimeMs: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(seen).toEqual([0, 1]);
|
||||
expect(manifest.failed).toBe(1);
|
||||
expect(manifest.completed).toBe(1);
|
||||
});
|
||||
|
||||
it("marks unstarted rows skipped when fail-fast is enabled", async () => {
|
||||
const prepared = prepareBatchRender({
|
||||
batchPath: writeJson("rows.json", '[{"name":"Alice"},{"name":"Bob"},{"name":"Cleo"}]'),
|
||||
outputTemplate: join(tmpDir, "renders/{name}.mp4"),
|
||||
indexPath: writeIndex(),
|
||||
strictVariables: false,
|
||||
quiet: true,
|
||||
json: false,
|
||||
});
|
||||
|
||||
const seen: number[] = [];
|
||||
const manifest = await runBatchRender({
|
||||
prepared,
|
||||
concurrency: 1,
|
||||
failFast: true,
|
||||
quiet: true,
|
||||
json: false,
|
||||
renderOne: async (row) => {
|
||||
seen.push(row.index);
|
||||
if (row.index === 1) throw new Error("boom");
|
||||
return { renderTimeMs: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(seen).toEqual([0, 1]);
|
||||
expect(manifest.rows.map((row) => row.status)).toEqual(["completed", "failed", "skipped"]);
|
||||
expect(manifest.skipped).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,464 @@
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve, sep } from "node:path";
|
||||
import { c } from "../ui/colors.js";
|
||||
import { errorBox } from "../ui/format.js";
|
||||
import {
|
||||
loadProjectVariableSchema,
|
||||
reportVariableIssues,
|
||||
validateVariablesAgainstSchema,
|
||||
} from "../utils/variables.js";
|
||||
|
||||
export class BatchRenderInputError extends Error {
|
||||
readonly title: string;
|
||||
readonly hint: string | undefined;
|
||||
|
||||
constructor(title: string, message: string, hint?: string) {
|
||||
super(message);
|
||||
this.name = "BatchRenderInputError";
|
||||
this.title = title;
|
||||
this.hint = hint;
|
||||
}
|
||||
}
|
||||
|
||||
export interface PreparedBatchRow {
|
||||
index: number;
|
||||
variables: Record<string, unknown>;
|
||||
outputPath: string;
|
||||
}
|
||||
|
||||
export interface PreparedBatchRender {
|
||||
batchPath: string;
|
||||
manifestPath: string;
|
||||
variableIssueCount: number;
|
||||
rows: PreparedBatchRow[];
|
||||
}
|
||||
|
||||
export interface BatchRenderResult {
|
||||
durationMs?: number;
|
||||
renderTimeMs: number;
|
||||
}
|
||||
|
||||
export interface BatchManifestRow {
|
||||
index: number;
|
||||
outputPath: string;
|
||||
status: "pending" | "running" | "completed" | "failed" | "skipped";
|
||||
durationMs: number | null;
|
||||
renderTimeMs: number | null;
|
||||
error: string | null;
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
variables: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface BatchManifest {
|
||||
version: 1;
|
||||
batchPath: string;
|
||||
manifestPath: string;
|
||||
total: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
skipped: number;
|
||||
rows: BatchManifestRow[];
|
||||
}
|
||||
|
||||
interface PrepareBatchRenderOptions {
|
||||
batchPath: string;
|
||||
outputTemplate: string;
|
||||
indexPath: string;
|
||||
strictVariables: boolean;
|
||||
quiet: boolean;
|
||||
json: boolean;
|
||||
readFile?: (path: string) => string;
|
||||
}
|
||||
|
||||
interface RunBatchRenderOptions {
|
||||
prepared: PreparedBatchRender;
|
||||
concurrency: number;
|
||||
failFast: boolean;
|
||||
quiet: boolean;
|
||||
json: boolean;
|
||||
renderOne: (row: PreparedBatchRow) => Promise<BatchRenderResult>;
|
||||
}
|
||||
|
||||
const PLACEHOLDER_RE = /\{([A-Za-z0-9_.-]+)\}/g;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function parseJson(raw: string, source: string): unknown {
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch (error: unknown) {
|
||||
throw new BatchRenderInputError(
|
||||
"Invalid JSON in --batch",
|
||||
`${source}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function parseBatchRows(raw: string, source: string): Record<string, unknown>[] {
|
||||
const parsed = parseJson(raw, source);
|
||||
const rows = Array.isArray(parsed) ? parsed : isRecord(parsed) ? parsed.rows : undefined;
|
||||
|
||||
if (!Array.isArray(rows)) {
|
||||
throw new BatchRenderInputError(
|
||||
"Invalid batch payload",
|
||||
'--batch must be a JSON array of objects, or an object with a "rows" array.',
|
||||
);
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
throw new BatchRenderInputError("Empty batch", `${source} contains zero rows.`);
|
||||
}
|
||||
|
||||
return rows.map((row, index) => {
|
||||
if (!isRecord(row)) {
|
||||
throw new BatchRenderInputError(
|
||||
"Invalid batch row",
|
||||
`Row ${index} must be a JSON object of variable values.`,
|
||||
);
|
||||
}
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
function placeholderValue(row: Record<string, unknown>, key: string, index: number): string {
|
||||
if (key === "index") return String(index);
|
||||
if (!Object.hasOwn(row, key)) {
|
||||
throw new BatchRenderInputError(
|
||||
"Invalid output template",
|
||||
`Missing value for placeholder {${key}} in row ${index}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const value = row[key];
|
||||
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
|
||||
throw new BatchRenderInputError(
|
||||
"Invalid output template",
|
||||
`Placeholder {${key}} in row ${index} must resolve to a string, number, or boolean.`,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveOutputTemplate(
|
||||
template: string,
|
||||
row: Record<string, unknown>,
|
||||
index: number,
|
||||
): string {
|
||||
return template.replace(PLACEHOLDER_RE, (_match, key: string) =>
|
||||
placeholderValue(row, key, index),
|
||||
);
|
||||
}
|
||||
|
||||
function isSameOrChildPath(path: string, parent: string): boolean {
|
||||
return path === parent || path.startsWith(parent.endsWith(sep) ? parent : parent + sep);
|
||||
}
|
||||
|
||||
export function commonOutputDirectory(outputPaths: readonly string[]): string {
|
||||
const firstPath = outputPaths[0];
|
||||
if (!firstPath) return resolve("renders");
|
||||
|
||||
let common = dirname(firstPath);
|
||||
for (const outputPath of outputPaths.slice(1)) {
|
||||
const dir = dirname(outputPath);
|
||||
while (!isSameOrChildPath(dir, common)) {
|
||||
const parent = dirname(common);
|
||||
if (parent === common) return common;
|
||||
common = parent;
|
||||
}
|
||||
}
|
||||
return common;
|
||||
}
|
||||
|
||||
function checkOutputCollisions(rows: readonly PreparedBatchRow[], manifestPath: string): void {
|
||||
const seen = new Map<string, number>();
|
||||
for (const row of rows) {
|
||||
const previous = seen.get(row.outputPath);
|
||||
if (previous !== undefined) {
|
||||
throw new BatchRenderInputError(
|
||||
"Batch output collision",
|
||||
`Rows ${previous} and ${row.index} both resolve to ${row.outputPath}.`,
|
||||
"Use placeholders such as {index} or a unique row key in --output.",
|
||||
);
|
||||
}
|
||||
if (row.outputPath === manifestPath) {
|
||||
throw new BatchRenderInputError(
|
||||
"Batch output collision",
|
||||
`Row ${row.index} resolves to the manifest path: ${manifestPath}.`,
|
||||
);
|
||||
}
|
||||
seen.set(row.outputPath, row.index);
|
||||
}
|
||||
}
|
||||
|
||||
function validateBatchVariables(
|
||||
rows: readonly Record<string, unknown>[],
|
||||
indexPath: string,
|
||||
strictVariables: boolean,
|
||||
quiet: boolean,
|
||||
json: boolean,
|
||||
): number {
|
||||
const schema = loadProjectVariableSchema(indexPath);
|
||||
let issueCount = 0;
|
||||
const strictRows: number[] = [];
|
||||
|
||||
for (let index = 0; index < rows.length; index++) {
|
||||
const row = rows[index];
|
||||
if (!row || Object.keys(row).length === 0) continue;
|
||||
|
||||
const issues = validateVariablesAgainstSchema(row, schema);
|
||||
if (issues.length === 0) continue;
|
||||
|
||||
issueCount += issues.length;
|
||||
if (!quiet && !json) {
|
||||
console.log("");
|
||||
console.log(c.dim(`Batch row ${index}:`));
|
||||
}
|
||||
reportVariableIssues(issues, { strict: false, quiet: quiet || json });
|
||||
if (strictVariables) strictRows.push(index);
|
||||
}
|
||||
|
||||
if (strictRows.length > 0) {
|
||||
throw new BatchRenderInputError(
|
||||
"Variable validation failed",
|
||||
`Aborting batch due to variable issues in row ${strictRows.join(", ")} (--strict-variables mode).`,
|
||||
);
|
||||
}
|
||||
|
||||
return issueCount;
|
||||
}
|
||||
|
||||
export function prepareBatchRender(options: PrepareBatchRenderOptions): PreparedBatchRender {
|
||||
const batchPath = resolve(options.batchPath);
|
||||
const read = options.readFile ?? ((path: string) => readFileSync(path, "utf8"));
|
||||
let raw: string;
|
||||
try {
|
||||
raw = read(batchPath);
|
||||
} catch (error: unknown) {
|
||||
throw new BatchRenderInputError(
|
||||
"Could not read --batch",
|
||||
`${batchPath}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const variableRows = parseBatchRows(raw, batchPath);
|
||||
const rows = variableRows.map((variables, index) => ({
|
||||
index,
|
||||
variables,
|
||||
outputPath: resolve(resolveOutputTemplate(options.outputTemplate, variables, index)),
|
||||
}));
|
||||
const manifestPath = join(
|
||||
commonOutputDirectory(rows.map((row) => row.outputPath)),
|
||||
"manifest.json",
|
||||
);
|
||||
checkOutputCollisions(rows, manifestPath);
|
||||
|
||||
const variableIssueCount = validateBatchVariables(
|
||||
variableRows,
|
||||
options.indexPath,
|
||||
options.strictVariables,
|
||||
options.quiet,
|
||||
options.json,
|
||||
);
|
||||
|
||||
return {
|
||||
batchPath,
|
||||
manifestPath,
|
||||
variableIssueCount,
|
||||
rows,
|
||||
};
|
||||
}
|
||||
|
||||
function makeInitialManifest(prepared: PreparedBatchRender): BatchManifest {
|
||||
return {
|
||||
version: 1,
|
||||
batchPath: prepared.batchPath,
|
||||
manifestPath: prepared.manifestPath,
|
||||
total: prepared.rows.length,
|
||||
completed: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
rows: prepared.rows.map((row) => ({
|
||||
index: row.index,
|
||||
outputPath: row.outputPath,
|
||||
status: "pending",
|
||||
durationMs: null,
|
||||
renderTimeMs: null,
|
||||
error: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
variables: row.variables,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeManifest(manifest: BatchManifest): void {
|
||||
manifest.completed = manifest.rows.filter((row) => row.status === "completed").length;
|
||||
manifest.failed = manifest.rows.filter((row) => row.status === "failed").length;
|
||||
manifest.skipped = manifest.rows.filter((row) => row.status === "skipped").length;
|
||||
}
|
||||
|
||||
function writeManifest(manifest: BatchManifest): void {
|
||||
summarizeManifest(manifest);
|
||||
mkdirSync(dirname(manifest.manifestPath), { recursive: true });
|
||||
writeFileSync(manifest.manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf8");
|
||||
}
|
||||
|
||||
function emitJsonEvent(event: Record<string, unknown>, json: boolean): void {
|
||||
if (json) console.log(JSON.stringify(event));
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
async function renderBatchRow(
|
||||
row: PreparedBatchRow,
|
||||
manifest: BatchManifest,
|
||||
options: RunBatchRenderOptions,
|
||||
): Promise<boolean> {
|
||||
const manifestRow = manifest.rows[row.index];
|
||||
if (!manifestRow) {
|
||||
throw new Error(`Batch manifest is missing row ${row.index}`);
|
||||
}
|
||||
|
||||
manifestRow.status = "running";
|
||||
manifestRow.startedAt = new Date().toISOString();
|
||||
writeManifest(manifest);
|
||||
emitJsonEvent(
|
||||
{ type: "batch-row-start", index: row.index, outputPath: row.outputPath },
|
||||
options.json,
|
||||
);
|
||||
|
||||
if (!options.quiet && !options.json) {
|
||||
console.log(c.dim(`Batch row ${row.index}: ${row.outputPath}`));
|
||||
}
|
||||
|
||||
try {
|
||||
mkdirSync(dirname(row.outputPath), { recursive: true });
|
||||
const result = await options.renderOne(row);
|
||||
manifestRow.status = "completed";
|
||||
manifestRow.durationMs = result.durationMs ?? null;
|
||||
manifestRow.renderTimeMs = result.renderTimeMs;
|
||||
manifestRow.completedAt = new Date().toISOString();
|
||||
writeManifest(manifest);
|
||||
emitJsonEvent(
|
||||
{
|
||||
type: "batch-row-complete",
|
||||
index: row.index,
|
||||
outputPath: row.outputPath,
|
||||
durationMs: manifestRow.durationMs,
|
||||
renderTimeMs: manifestRow.renderTimeMs,
|
||||
},
|
||||
options.json,
|
||||
);
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
manifestRow.status = "failed";
|
||||
manifestRow.error = errorMessage(error);
|
||||
manifestRow.completedAt = new Date().toISOString();
|
||||
writeManifest(manifest);
|
||||
emitJsonEvent(
|
||||
{
|
||||
type: "batch-row-error",
|
||||
index: row.index,
|
||||
outputPath: row.outputPath,
|
||||
error: manifestRow.error,
|
||||
},
|
||||
options.json,
|
||||
);
|
||||
if (!options.quiet && !options.json) {
|
||||
console.log(c.error(` Row ${row.index} failed: ${manifestRow.error}`));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function markUnstartedRowsSkipped(manifest: BatchManifest): void {
|
||||
for (const row of manifest.rows) {
|
||||
if (row.status !== "pending") continue;
|
||||
row.status = "skipped";
|
||||
row.error = "Skipped after --batch-fail-fast.";
|
||||
row.completedAt = new Date().toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
export async function runBatchRender(options: RunBatchRenderOptions): Promise<BatchManifest> {
|
||||
if (options.concurrency < 1) {
|
||||
throw new BatchRenderInputError(
|
||||
"Invalid batch-concurrency",
|
||||
`Got "${options.concurrency}". Must be a positive integer.`,
|
||||
);
|
||||
}
|
||||
|
||||
const manifest = makeInitialManifest(options.prepared);
|
||||
writeManifest(manifest);
|
||||
|
||||
if (!options.quiet && !options.json) {
|
||||
console.log("");
|
||||
console.log(
|
||||
c.accent("◆") +
|
||||
` Batch rendering ${options.prepared.rows.length} rows` +
|
||||
c.dim(` → ${options.prepared.manifestPath}`),
|
||||
);
|
||||
console.log(c.dim(` batch concurrency: ${options.concurrency}`));
|
||||
console.log("");
|
||||
}
|
||||
|
||||
let cursor = 0;
|
||||
let stopLaunching = false;
|
||||
const workerCount = Math.min(options.concurrency, options.prepared.rows.length);
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: workerCount }, async () => {
|
||||
while (!stopLaunching) {
|
||||
const row = options.prepared.rows[cursor];
|
||||
if (!row) return;
|
||||
cursor++;
|
||||
|
||||
const ok = await renderBatchRow(row, manifest, options);
|
||||
if (!ok && options.failFast) {
|
||||
stopLaunching = true;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
if (stopLaunching) markUnstartedRowsSkipped(manifest);
|
||||
writeManifest(manifest);
|
||||
emitJsonEvent(
|
||||
{
|
||||
type: "batch-complete",
|
||||
manifestPath: manifest.manifestPath,
|
||||
total: manifest.total,
|
||||
completed: manifest.completed,
|
||||
failed: manifest.failed,
|
||||
skipped: manifest.skipped,
|
||||
},
|
||||
options.json,
|
||||
);
|
||||
|
||||
if (!options.quiet && !options.json) {
|
||||
console.log("");
|
||||
console.log(
|
||||
manifest.failed > 0
|
||||
? c.warn(
|
||||
`Batch complete: ${manifest.completed} completed, ${manifest.failed} failed, ${manifest.skipped} skipped.`,
|
||||
)
|
||||
: c.success(`Batch complete: ${manifest.completed} completed.`),
|
||||
);
|
||||
console.log(c.dim(`Manifest: ${manifest.manifestPath}`));
|
||||
}
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function exitBatchRenderInputError(error: unknown): never {
|
||||
if (error instanceof BatchRenderInputError) {
|
||||
errorBox(error.title, error.message, error.hint);
|
||||
process.exit(1);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -42,6 +42,10 @@ export const examples: Example[] = [
|
||||
"Variables from a JSON file",
|
||||
"hyperframes render --variables-file ./vars.json --output out.mp4",
|
||||
],
|
||||
[
|
||||
"Batch render one output per variables row",
|
||||
'hyperframes render --batch rows.json --output "renders/{name}.mp4"',
|
||||
],
|
||||
];
|
||||
import { cpus, freemem, tmpdir } from "node:os";
|
||||
import { resolve, dirname, join, basename } from "node:path";
|
||||
@@ -248,6 +252,26 @@ export default defineCommand({
|
||||
"Fail render if any --variables key is undeclared or has a wrong type vs the composition's data-composition-variables. Without this flag, mismatches are warnings.",
|
||||
default: false,
|
||||
},
|
||||
batch: {
|
||||
type: "string",
|
||||
description:
|
||||
'Path to a JSON array of variable rows (or {"rows":[...]}). Renders one output per row.',
|
||||
},
|
||||
"batch-concurrency": {
|
||||
type: "string",
|
||||
description:
|
||||
"Maximum number of batch rows to render at once. Default: 1, because each render already parallelizes across workers.",
|
||||
},
|
||||
"batch-fail-fast": {
|
||||
type: "boolean",
|
||||
description: "Stop launching new batch rows after the first row failure.",
|
||||
default: false,
|
||||
},
|
||||
json: {
|
||||
type: "boolean",
|
||||
description: "With --batch, emit JSON progress events.",
|
||||
default: false,
|
||||
},
|
||||
resolution: {
|
||||
type: "string",
|
||||
description:
|
||||
@@ -440,6 +464,39 @@ export default defineCommand({
|
||||
process.env.PRODUCER_MAX_CONCURRENT_RENDERS = String(parsed);
|
||||
}
|
||||
|
||||
// ── Validate batch mode ───────────────────────────────────────────────
|
||||
const batchPath =
|
||||
typeof args.batch === "string" && args.batch.trim() !== "" ? args.batch.trim() : undefined;
|
||||
if (batchPath && (args.variables != null || args["variables-file"] != null)) {
|
||||
errorBox(
|
||||
"Conflicting variables flags",
|
||||
"Use either --batch or --variables/--variables-file, not both.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!batchPath && args["batch-concurrency"] != null) {
|
||||
errorBox("Invalid batch-concurrency", "--batch-concurrency requires --batch.");
|
||||
process.exit(1);
|
||||
}
|
||||
if (!batchPath && args["batch-fail-fast"]) {
|
||||
errorBox("Invalid batch-fail-fast", "--batch-fail-fast requires --batch.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let batchConcurrency = 1;
|
||||
if (args["batch-concurrency"] != null) {
|
||||
const parsed = parseInt(args["batch-concurrency"], 10);
|
||||
if (isNaN(parsed) || parsed < 1) {
|
||||
errorBox(
|
||||
"Invalid batch-concurrency",
|
||||
`Got "${args["batch-concurrency"]}". Must be a positive integer.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
batchConcurrency = parsed;
|
||||
}
|
||||
|
||||
// ── Resolve output path ───────────────────────────────────────────────
|
||||
const rendersDir = resolve("renders");
|
||||
const ext = FORMAT_EXT[format] ?? ".mp4";
|
||||
@@ -447,18 +504,23 @@ export default defineCommand({
|
||||
const now = new Date();
|
||||
const datePart = now.toISOString().slice(0, 10);
|
||||
const timePart = now.toTimeString().slice(0, 8).replace(/:/g, "-");
|
||||
const batchOutputTemplate = args.output
|
||||
? args.output
|
||||
: join(rendersDir, `${project.name}_${datePart}_${timePart}_{index}${ext}`);
|
||||
const outputPath = args.output
|
||||
? resolve(args.output)
|
||||
: join(rendersDir, `${project.name}_${datePart}_${timePart}${ext}`);
|
||||
|
||||
// Ensure output directory exists
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
if (!batchPath) mkdirSync(dirname(outputPath), { recursive: true });
|
||||
|
||||
const useDocker = args.docker ?? false;
|
||||
const useGpu = args.gpu ?? false;
|
||||
const browserGpuArg = args["browser-gpu"];
|
||||
const browserGpuMode = resolveBrowserGpuForCli(useDocker, browserGpuArg);
|
||||
const quiet = args.quiet ?? false;
|
||||
const batchJson = args.json ?? false;
|
||||
const effectiveQuiet = quiet || (batchPath != null && batchJson);
|
||||
const strictAll = args["strict-all"] ?? false;
|
||||
const strictErrors = (args.strict ?? false) || strictAll;
|
||||
const crfRaw = args.crf;
|
||||
@@ -507,8 +569,27 @@ export default defineCommand({
|
||||
const pageNavigationTimeoutMs = resolveBrowserTimeoutMsArg(args["browser-timeout"]);
|
||||
const entryFile = resolveCompositionEntryArg(args.composition, project.dir, statSync);
|
||||
|
||||
// ── Preflight batch rows before browser/lint work ────────────────────
|
||||
let batchModule: typeof import("./batchRender.js") | undefined;
|
||||
let preparedBatch: import("./batchRender.js").PreparedBatchRender | undefined;
|
||||
if (batchPath) {
|
||||
batchModule = await import("./batchRender.js");
|
||||
try {
|
||||
preparedBatch = batchModule.prepareBatchRender({
|
||||
batchPath,
|
||||
outputTemplate: batchOutputTemplate,
|
||||
indexPath: project.indexPath,
|
||||
strictVariables: args["strict-variables"] ?? false,
|
||||
quiet: quiet || batchJson,
|
||||
json: batchJson,
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
batchModule.exitBatchRenderInputError(error);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Print render plan ─────────────────────────────────────────────────
|
||||
if (!quiet) {
|
||||
if (!quiet && !batchPath) {
|
||||
const workerLabel =
|
||||
workers != null ? `${workers} workers` : `auto workers (${CPU_CORE_COUNT} cores detected)`;
|
||||
console.log("");
|
||||
@@ -544,23 +625,35 @@ export default defineCommand({
|
||||
let browserPath: string | undefined;
|
||||
if (!useDocker) {
|
||||
const { ensureBrowser } = await import("../browser/manager.js");
|
||||
const clack = await import("@clack/prompts");
|
||||
const s = clack.spinner();
|
||||
s.start("Checking browser...");
|
||||
let browserSpinner:
|
||||
| {
|
||||
start: (message?: string) => void;
|
||||
message: (message: string) => void;
|
||||
stop: (message?: string) => void;
|
||||
}
|
||||
| undefined;
|
||||
try {
|
||||
const info = await ensureBrowser({
|
||||
onProgress: (downloaded, total) => {
|
||||
if (total <= 0) return;
|
||||
const pct = Math.floor((downloaded / total) * 100);
|
||||
s.message(
|
||||
`Downloading Chrome... ${c.progress(pct + "%")} ${c.dim("(" + formatBytes(downloaded) + " / " + formatBytes(total) + ")")}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
browserPath = info.executablePath;
|
||||
s.stop(c.dim(`Browser: ${info.source}`));
|
||||
if (effectiveQuiet) {
|
||||
const info = await ensureBrowser();
|
||||
browserPath = info.executablePath;
|
||||
} else {
|
||||
const clack = await import("@clack/prompts");
|
||||
browserSpinner = clack.spinner();
|
||||
browserSpinner.start("Checking browser...");
|
||||
const info = await ensureBrowser({
|
||||
onProgress: (downloaded, total) => {
|
||||
if (total <= 0) return;
|
||||
const pct = Math.floor((downloaded / total) * 100);
|
||||
browserSpinner?.message(
|
||||
`Downloading Chrome... ${c.progress(pct + "%")} ${c.dim("(" + formatBytes(downloaded) + " / " + formatBytes(total) + ")")}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
browserPath = info.executablePath;
|
||||
browserSpinner.stop(c.dim(`Browser: ${info.source}`));
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
s.stop(c.error("Browser not available"));
|
||||
browserSpinner?.stop(c.error("Browser not available"));
|
||||
errorBox(
|
||||
"Chrome not found",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
@@ -601,6 +694,57 @@ export default defineCommand({
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── Batch render ──────────────────────────────────────────────────────
|
||||
if (batchPath && batchModule && preparedBatch) {
|
||||
const batchQuiet = quiet || batchJson;
|
||||
const hdrMode: RenderOptions["hdrMode"] = args.sdr
|
||||
? "force-sdr"
|
||||
: args.hdr
|
||||
? "force-hdr"
|
||||
: "auto";
|
||||
const renderOptionsBase: RenderOptions = {
|
||||
fps,
|
||||
quality,
|
||||
format,
|
||||
workers,
|
||||
gpu: useGpu,
|
||||
browserGpuMode,
|
||||
hdrMode,
|
||||
crf,
|
||||
videoBitrate,
|
||||
quiet: batchQuiet,
|
||||
browserPath,
|
||||
entryFile,
|
||||
outputResolution,
|
||||
pageNavigationTimeoutMs,
|
||||
protocolTimeout,
|
||||
playerReadyTimeout,
|
||||
exitAfterComplete: false,
|
||||
throwOnError: true,
|
||||
skipFeedback: true,
|
||||
};
|
||||
const manifest = await batchModule.runBatchRender({
|
||||
prepared: preparedBatch,
|
||||
concurrency: batchConcurrency,
|
||||
failFast: args["batch-fail-fast"] ?? false,
|
||||
quiet: batchQuiet,
|
||||
json: batchJson,
|
||||
renderOne: (row) =>
|
||||
useDocker
|
||||
? renderDocker(project.dir, row.outputPath, {
|
||||
...renderOptionsBase,
|
||||
variables: row.variables,
|
||||
pageSideCompositing: args["page-side-compositing"] !== false,
|
||||
})
|
||||
: renderLocal(project.dir, row.outputPath, {
|
||||
...renderOptionsBase,
|
||||
variables: row.variables,
|
||||
}),
|
||||
});
|
||||
if (manifest.failed > 0) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Resolve --variables / --variables-file ──────────────────────────
|
||||
const variables = resolveVariablesArg(args.variables, args["variables-file"]);
|
||||
|
||||
@@ -660,6 +804,11 @@ export default defineCommand({
|
||||
},
|
||||
});
|
||||
|
||||
export interface SingleRenderResult {
|
||||
durationMs?: number;
|
||||
renderTimeMs: number;
|
||||
}
|
||||
|
||||
interface RenderOptions {
|
||||
fps: Fps;
|
||||
quality: "draft" | "standard" | "high";
|
||||
@@ -695,6 +844,10 @@ interface RenderOptions {
|
||||
protocolTimeout?: number;
|
||||
/** Player-ready timeout override (ms). */
|
||||
playerReadyTimeout?: number;
|
||||
/** Throw render failures to the caller instead of printing and exiting. */
|
||||
throwOnError?: boolean;
|
||||
/** Skip the interactive feedback prompt after a successful render. */
|
||||
skipFeedback?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -868,7 +1021,7 @@ async function renderDocker(
|
||||
projectDir: string,
|
||||
outputPath: string,
|
||||
options: RenderOptions,
|
||||
): Promise<void> {
|
||||
): Promise<SingleRenderResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
// Dev mode (tsx/ts-node) uses "latest" since the local version isn't on npm
|
||||
@@ -959,6 +1112,7 @@ async function renderDocker(
|
||||
|
||||
printRenderComplete(outputPath, elapsed, options.quiet);
|
||||
if (options.exitAfterComplete) scheduleRenderProcessExit();
|
||||
return { renderTimeMs: elapsed };
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
@@ -966,7 +1120,7 @@ export async function renderLocal(
|
||||
projectDir: string,
|
||||
outputPath: string,
|
||||
options: RenderOptions,
|
||||
): Promise<void> {
|
||||
): Promise<SingleRenderResult> {
|
||||
const preflight = await runEnvironmentChecks({
|
||||
projectDir,
|
||||
browserPath: options.browserPath,
|
||||
@@ -1050,11 +1204,17 @@ export async function renderLocal(
|
||||
const elapsed = Date.now() - startTime;
|
||||
trackRenderMetrics(job, elapsed, options, false);
|
||||
printRenderComplete(outputPath, elapsed, options.quiet);
|
||||
await maybePromptRenderFeedback({
|
||||
renderDurationMs: elapsed,
|
||||
quiet: options.quiet,
|
||||
});
|
||||
if (!options.skipFeedback) {
|
||||
await maybePromptRenderFeedback({
|
||||
renderDurationMs: elapsed,
|
||||
quiet: options.quiet,
|
||||
});
|
||||
}
|
||||
if (options.exitAfterComplete) scheduleRenderProcessExit();
|
||||
const durationMs = job.perfSummary
|
||||
? Math.round(job.perfSummary.compositionDurationSeconds * 1000)
|
||||
: undefined;
|
||||
return { renderTimeMs: elapsed, durationMs };
|
||||
}
|
||||
|
||||
type UnrefableTimer = {
|
||||
@@ -1189,6 +1349,9 @@ function handleRenderError(
|
||||
...renderJobObservabilityTelemetryPayload(job),
|
||||
...getMemorySnapshot(),
|
||||
});
|
||||
if (options.throwOnError) {
|
||||
throw new Error(message);
|
||||
}
|
||||
errorBox("Render failed", message, hint);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user