mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-14 13:27:13 +00:00
New subcommand for automated template-rendering pipelines. Given a
project dir + a JSONL batch file, fans out N personalised renders by
calling renderToLambda once per batch row with per-entry variables and
outputKey:
hyperframes lambda render-batch ./my-template \
--batch ./users.jsonl \
--width 1920 --height 1080 \
--max-concurrent 10
JSONL format (one JSON object per line):
{"outputKey": "renders/alice.mp4", "variables": {"name": "Alice"}}
{"outputKey": "renders/bob.mp4", "variables": {"name": "Bob"}}
The verb deploys the site once and reuses it across renders (--site-id
skips the deploy when the project was pre-uploaded). Concurrent Step
Functions starts are capped at --max-concurrent (default 50) via a
semaphore so a 10 000-entry batch doesn't try to spawn 10 000
executions simultaneously and trip the AWS account's concurrent-
execution quota.
Per-entry results land in a manifest (one row per input line) with
executionArn + status. --json emits the manifest as machine-readable
JSON. --dry-run prints the manifest with status: "would-invoke" for
every entry without calling AWS, so callers can lint their batch file
before paying for N executions.
Variables in each batch entry pre-validate against the composition's
data-composition-variables declaration (mirroring the local
hyperframes render UX). --strict-variables aborts the run on the first
failing entry before any AWS call. The reportVariableIssues helper from
PR 9.3 is reused so the warning format matches the single-render path
exactly.
Distinction from --max-parallel-chunks: --max-concurrent caps
ORCHESTRATOR-side fan-out (how many StartExecution calls run at once);
--max-parallel-chunks caps chunks PER render. AWS account-level Lambda
concurrent-execution limits live one level up and render-batch can't
enforce those; pick --max-concurrent based on your account quota +
the reserved concurrency you provisioned via lambda deploy.
Tests cover the concurrency-cap semaphore (preserve-order,
peak-in-flight, empty-input, limit > inputs.length, propagate
rejection) and the JSONL parser (blank-line handling, malformed JSON,
missing outputKey, non-object variables).
Phase 9 PR 9.4 of the distributed rendering plan.
140 lines
4.6 KiB
TypeScript
140 lines
4.6 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { parseBatchFile, runWithConcurrencyLimit } from "./render-batch.js";
|
|
|
|
let tmpDir: string;
|
|
|
|
beforeEach(() => {
|
|
tmpDir = mkdtempSync(join(tmpdir(), "hf-render-batch-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(tmpDir, { recursive: true, force: true });
|
|
});
|
|
|
|
function writeBatch(content: string): string {
|
|
const p = join(tmpDir, "batch.jsonl");
|
|
writeFileSync(p, content, "utf8");
|
|
return p;
|
|
}
|
|
|
|
describe("runWithConcurrencyLimit", () => {
|
|
it("preserves input order in the output array regardless of completion order", async () => {
|
|
// First input takes longest to resolve; output array still positional.
|
|
const delays = [40, 10, 20];
|
|
const out = await runWithConcurrencyLimit(delays, 3, async (ms, i) => {
|
|
await new Promise((r) => setTimeout(r, ms));
|
|
return `done-${i}`;
|
|
});
|
|
expect(out).toEqual(["done-0", "done-1", "done-2"]);
|
|
});
|
|
|
|
it("caps simultaneous in-flight work to the limit", async () => {
|
|
let inFlight = 0;
|
|
let peak = 0;
|
|
const inputs = Array.from({ length: 12 }, (_, i) => i);
|
|
const worker = async (i: number): Promise<number> => {
|
|
inFlight++;
|
|
peak = Math.max(peak, inFlight);
|
|
await new Promise((r) => setTimeout(r, 5));
|
|
inFlight--;
|
|
return i;
|
|
};
|
|
await runWithConcurrencyLimit(inputs, 3, worker);
|
|
expect(peak).toBe(3);
|
|
});
|
|
|
|
it("does not exceed the input length even when limit > inputs.length", async () => {
|
|
let inFlight = 0;
|
|
let peak = 0;
|
|
const inputs = [1, 2];
|
|
await runWithConcurrencyLimit(inputs, 50, async (n) => {
|
|
inFlight++;
|
|
peak = Math.max(peak, inFlight);
|
|
await new Promise((r) => setTimeout(r, 2));
|
|
inFlight--;
|
|
return n;
|
|
});
|
|
// Only 2 inputs → only 2 concurrent workers, even with limit=50.
|
|
expect(peak).toBe(2);
|
|
});
|
|
|
|
it("rejects a limit < 1", async () => {
|
|
await expect(runWithConcurrencyLimit([1, 2], 0, async (n) => n)).rejects.toThrow(
|
|
/limit must be/,
|
|
);
|
|
});
|
|
|
|
it("returns immediately for an empty input array", async () => {
|
|
const out = await runWithConcurrencyLimit([], 10, async (n: number) => n * 2);
|
|
expect(out).toEqual([]);
|
|
});
|
|
|
|
it("propagates the first worker rejection", async () => {
|
|
await expect(
|
|
runWithConcurrencyLimit([1, 2, 3], 2, async (n) => {
|
|
if (n === 2) throw new Error("boom");
|
|
return n;
|
|
}),
|
|
).rejects.toThrow(/boom/);
|
|
});
|
|
});
|
|
|
|
describe("parseBatchFile", () => {
|
|
it("parses a JSONL file into ordered entries (line numbers preserve source order)", () => {
|
|
const path = writeBatch(
|
|
[
|
|
'{"outputKey":"renders/alice.mp4","variables":{"name":"Alice"}}',
|
|
'{"outputKey":"renders/bob.mp4","variables":{"name":"Bob"},"executionName":"hf-bob-001"}',
|
|
].join("\n") + "\n",
|
|
);
|
|
const out = parseBatchFile(path);
|
|
expect(out).toHaveLength(2);
|
|
expect(out[0]?.entry.outputKey).toBe("renders/alice.mp4");
|
|
expect(out[0]?.entry.variables).toEqual({ name: "Alice" });
|
|
expect(out[0]?.lineNumber).toBe(1);
|
|
expect(out[1]?.entry.executionName).toBe("hf-bob-001");
|
|
expect(out[1]?.lineNumber).toBe(2);
|
|
});
|
|
|
|
it("skips blank lines and preserves line numbers", () => {
|
|
const path = writeBatch(
|
|
["", '{"outputKey":"renders/a.mp4"}', "", "", '{"outputKey":"renders/b.mp4"}'].join("\n") +
|
|
"\n",
|
|
);
|
|
const out = parseBatchFile(path);
|
|
expect(out).toHaveLength(2);
|
|
expect(out[0]?.lineNumber).toBe(2);
|
|
expect(out[1]?.lineNumber).toBe(5);
|
|
});
|
|
|
|
// Helper: stub `process.exit` to throw a sentinel, run the parser, and
|
|
// verify it called exit(1). Dedupes the 3 error-path tests so each one
|
|
// is a single readable assertion.
|
|
function expectExitOne(content: string): void {
|
|
const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => {
|
|
throw new Error("EXIT_CALLED");
|
|
});
|
|
try {
|
|
expect(() => parseBatchFile(writeBatch(content))).toThrow(/EXIT_CALLED/);
|
|
expect(exitSpy).toHaveBeenCalledWith(1);
|
|
} finally {
|
|
exitSpy.mockRestore();
|
|
}
|
|
}
|
|
|
|
it("exits with a clear message on malformed JSON, naming the offending line", () => {
|
|
expectExitOne(['{"outputKey":"renders/a.mp4"}', "{not json"].join("\n"));
|
|
});
|
|
|
|
it("rejects entries missing outputKey", () => {
|
|
expectExitOne('{"variables":{"name":"Alice"}}\n');
|
|
});
|
|
|
|
it("rejects variables that's not a plain object", () => {
|
|
expectExitOne('{"outputKey":"renders/a.mp4","variables":[1,2,3]}\n');
|
|
});
|
|
});
|