feat(cli): hyperframes lambda render-batch verb

New subcommand for automated template-rendering pipelines. Given a
project dir + a JSONL batch file, fans out N personalised renders by
calling renderToLambda once per batch row with per-entry variables and
outputKey:

  hyperframes lambda render-batch ./my-template \
    --batch ./users.jsonl \
    --width 1920 --height 1080 \
    --max-concurrent 10

JSONL format (one JSON object per line):

  {"outputKey": "renders/alice.mp4", "variables": {"name": "Alice"}}
  {"outputKey": "renders/bob.mp4",   "variables": {"name": "Bob"}}

The verb deploys the site once and reuses it across renders (--site-id
skips the deploy when the project was pre-uploaded). Concurrent Step
Functions starts are capped at --max-concurrent (default 50) via a
semaphore so a 10 000-entry batch doesn't try to spawn 10 000
executions simultaneously and trip the AWS account's concurrent-
execution quota.

Per-entry results land in a manifest (one row per input line) with
executionArn + status. --json emits the manifest as machine-readable
JSON. --dry-run prints the manifest with status: "would-invoke" for
every entry without calling AWS, so callers can lint their batch file
before paying for N executions.

Variables in each batch entry pre-validate against the composition's
data-composition-variables declaration (mirroring the local
hyperframes render UX). --strict-variables aborts the run on the first
failing entry before any AWS call. The reportVariableIssues helper from
PR 9.3 is reused so the warning format matches the single-render path
exactly.

Distinction from --max-parallel-chunks: --max-concurrent caps
ORCHESTRATOR-side fan-out (how many StartExecution calls run at once);
--max-parallel-chunks caps chunks PER render. AWS account-level Lambda
concurrent-execution limits live one level up and render-batch can't
enforce those; pick --max-concurrent based on your account quota +
the reserved concurrency you provisioned via lambda deploy.

Tests cover the concurrency-cap semaphore (preserve-order,
peak-in-flight, empty-input, limit > inputs.length, propagate
rejection) and the JSONL parser (blank-line handling, malformed JSON,
missing outputKey, non-object variables).

Phase 9 PR 9.4 of the distributed rendering plan.
This commit is contained in:
James
2026-05-19 19:54:30 -04:00
committed by James Russo
parent cb948d5fcf
commit f0a2740f6e
4 changed files with 702 additions and 1 deletions
+77 -1
View File
@@ -31,6 +31,10 @@ export const examples: Example[] = [
"Render with variables from a JSON file",
"hyperframes lambda render ./my-template --site-id abc1234deadbeef0 --width 1920 --height 1080 --variables-file ./alice.json",
],
[
"Batch-render N personalised videos from a JSONL file (deploys the site once)",
"hyperframes lambda render-batch ./my-template --batch ./users.jsonl --width 1920 --height 1080 --max-concurrent 10",
],
["Check progress for a started render", "hyperframes lambda progress hf-render-abcd1234"],
[
"Pre-upload a project so multiple renders share the upload",
@@ -53,6 +57,7 @@ ${c.bold("SUBCOMMANDS:")}
${c.accent("deploy")} ${c.dim("Provision the Lambda + Step Functions + S3 stack via SAM")}
${c.accent("sites create")} ${c.dim("Tar + upload a project to S3 (reusable across renders)")}
${c.accent("render")} ${c.dim("Start a distributed render (returns a renderId)")}
${c.accent("render-batch")} ${c.dim("Fan out N personalised renders from a JSONL batch file")}
${c.accent("progress")} ${c.dim("Print progress + cost for an in-flight or finished render")}
${c.accent("destroy")} ${c.dim("Tear the stack down (S3 bucket is retained)")}
${c.accent("policies")} ${c.dim("Print or validate the IAM permissions the CLI needs")}
@@ -141,6 +146,23 @@ export default defineCommand({
"Fail the render command if any --variables key is undeclared or has a wrong type vs the composition's data-composition-variables. Without this flag, mismatches are warnings.",
default: false,
},
// render-batch
batch: {
type: "string",
description:
'Path to a JSONL batch file for `render-batch`. Each line: {"outputKey":"...","variables":{...}}',
},
"max-concurrent": {
type: "string",
description:
"Max in-flight Step Functions executions for `render-batch` (default: 50). Distinct from --max-parallel-chunks (which caps chunks per render).",
},
"dry-run": {
type: "boolean",
description:
"For `render-batch`: parse the batch file and print the manifest without invoking AWS. Every entry's status becomes `would-invoke`.",
default: false,
},
wait: { type: "boolean", description: "Block until the render finishes" },
"wait-interval-ms": {
type: "string",
@@ -179,7 +201,14 @@ export default defineCommand({
// dep) so the published CLI install stays small for users who don't
// deploy to Lambda. Subverbs other than `policies` need aws-lambda;
// catch the missing-module error here and turn it into a friendly hint.
const verbsNeedingSDK = new Set(["deploy", "sites", "render", "progress", "destroy"]);
const verbsNeedingSDK = new Set([
"deploy",
"sites",
"render",
"render-batch",
"progress",
"destroy",
]);
if (verbsNeedingSDK.has(subcommand)) {
try {
await import("@hyperframes/aws-lambda/sdk");
@@ -278,6 +307,53 @@ export default defineCommand({
});
return;
}
case "render-batch": {
const projectDir = args.target as string | undefined;
if (!projectDir) {
console.error(
"[lambda render-batch] usage: hyperframes lambda render-batch <projectDir> --batch <path.jsonl> --width <px> --height <px>",
);
process.exit(1);
}
const batch = args.batch as string | undefined;
if (!batch) {
console.error(
"[lambda render-batch] --batch <path.jsonl> is required. Each line is a JSON object with at least { outputKey: '...' }.",
);
process.exit(1);
}
const width = parsePositiveInt(args.width, "--width");
const height = parsePositiveInt(args.height, "--height");
if (width === undefined || height === undefined) {
console.error("[lambda render-batch] --width and --height are required.");
process.exit(1);
}
const fpsRaw = parseIntFlag(args.fps) ?? 30;
if (fpsRaw !== 24 && fpsRaw !== 30 && fpsRaw !== 60) {
console.error(`[lambda render-batch] --fps must be 24, 30, or 60; got ${fpsRaw}.`);
process.exit(1);
}
const { runRenderBatch } = await import("./lambda/render-batch.js");
await runRenderBatch({
projectDir,
stackName,
batch,
siteId: args["site-id"] as string | undefined,
fps: fpsRaw,
width,
height,
format: parseFormat(args.format),
codec: parseCodec(args.codec),
quality: parseQuality(args.quality),
chunkSize: parsePositiveInt(args["chunk-size"], "--chunk-size"),
maxParallelChunks: parsePositiveInt(args["max-parallel-chunks"], "--max-parallel-chunks"),
maxConcurrent: parsePositiveInt(args["max-concurrent"], "--max-concurrent"),
strictVariables: Boolean(args["strict-variables"]),
dryRun: Boolean(args["dry-run"]),
json: Boolean(args.json),
});
return;
}
case "progress": {
const target = args.target as string | undefined;
if (!target) {
@@ -0,0 +1,139 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { parseBatchFile, runWithConcurrencyLimit } from "./render-batch.js";
let tmpDir: string;
beforeEach(() => {
tmpDir = mkdtempSync(join(tmpdir(), "hf-render-batch-"));
});
afterEach(() => {
rmSync(tmpDir, { recursive: true, force: true });
});
function writeBatch(content: string): string {
const p = join(tmpDir, "batch.jsonl");
writeFileSync(p, content, "utf8");
return p;
}
describe("runWithConcurrencyLimit", () => {
it("preserves input order in the output array regardless of completion order", async () => {
// First input takes longest to resolve; output array still positional.
const delays = [40, 10, 20];
const out = await runWithConcurrencyLimit(delays, 3, async (ms, i) => {
await new Promise((r) => setTimeout(r, ms));
return `done-${i}`;
});
expect(out).toEqual(["done-0", "done-1", "done-2"]);
});
it("caps simultaneous in-flight work to the limit", async () => {
let inFlight = 0;
let peak = 0;
const inputs = Array.from({ length: 12 }, (_, i) => i);
const worker = async (i: number): Promise<number> => {
inFlight++;
peak = Math.max(peak, inFlight);
await new Promise((r) => setTimeout(r, 5));
inFlight--;
return i;
};
await runWithConcurrencyLimit(inputs, 3, worker);
expect(peak).toBe(3);
});
it("does not exceed the input length even when limit > inputs.length", async () => {
let inFlight = 0;
let peak = 0;
const inputs = [1, 2];
await runWithConcurrencyLimit(inputs, 50, async (n) => {
inFlight++;
peak = Math.max(peak, inFlight);
await new Promise((r) => setTimeout(r, 2));
inFlight--;
return n;
});
// Only 2 inputs → only 2 concurrent workers, even with limit=50.
expect(peak).toBe(2);
});
it("rejects a limit < 1", async () => {
await expect(runWithConcurrencyLimit([1, 2], 0, async (n) => n)).rejects.toThrow(
/limit must be/,
);
});
it("returns immediately for an empty input array", async () => {
const out = await runWithConcurrencyLimit([], 10, async (n: number) => n * 2);
expect(out).toEqual([]);
});
it("propagates the first worker rejection", async () => {
await expect(
runWithConcurrencyLimit([1, 2, 3], 2, async (n) => {
if (n === 2) throw new Error("boom");
return n;
}),
).rejects.toThrow(/boom/);
});
});
describe("parseBatchFile", () => {
it("parses a JSONL file into ordered entries (line numbers preserve source order)", () => {
const path = writeBatch(
[
'{"outputKey":"renders/alice.mp4","variables":{"name":"Alice"}}',
'{"outputKey":"renders/bob.mp4","variables":{"name":"Bob"},"executionName":"hf-bob-001"}',
].join("\n") + "\n",
);
const out = parseBatchFile(path);
expect(out).toHaveLength(2);
expect(out[0]?.entry.outputKey).toBe("renders/alice.mp4");
expect(out[0]?.entry.variables).toEqual({ name: "Alice" });
expect(out[0]?.lineNumber).toBe(1);
expect(out[1]?.entry.executionName).toBe("hf-bob-001");
expect(out[1]?.lineNumber).toBe(2);
});
it("skips blank lines and preserves line numbers", () => {
const path = writeBatch(
["", '{"outputKey":"renders/a.mp4"}', "", "", '{"outputKey":"renders/b.mp4"}'].join("\n") +
"\n",
);
const out = parseBatchFile(path);
expect(out).toHaveLength(2);
expect(out[0]?.lineNumber).toBe(2);
expect(out[1]?.lineNumber).toBe(5);
});
// Helper: stub `process.exit` to throw a sentinel, run the parser, and
// verify it called exit(1). Dedupes the 3 error-path tests so each one
// is a single readable assertion.
function expectExitOne(content: string): void {
const exitSpy = vi.spyOn(process, "exit").mockImplementation(() => {
throw new Error("EXIT_CALLED");
});
try {
expect(() => parseBatchFile(writeBatch(content))).toThrow(/EXIT_CALLED/);
expect(exitSpy).toHaveBeenCalledWith(1);
} finally {
exitSpy.mockRestore();
}
}
it("exits with a clear message on malformed JSON, naming the offending line", () => {
expectExitOne(['{"outputKey":"renders/a.mp4"}', "{not json"].join("\n"));
});
it("rejects entries missing outputKey", () => {
expectExitOne('{"variables":{"name":"Alice"}}\n');
});
it("rejects variables that's not a plain object", () => {
expectExitOne('{"outputKey":"renders/a.mp4","variables":[1,2,3]}\n');
});
});
@@ -0,0 +1,446 @@
/**
* `hyperframes lambda render-batch <projectDir> --batch <path.jsonl>` —
* fan out N personalised renders of the same project, one per JSONL line.
*
* The headline ergonomic for automated template-rendering pipelines on
* Lambda: deploy the site once (or accept a `--site-id` to skip), then
* call `renderToLambda` for each batch entry with per-entry `variables`
* and `outputKey`. Concurrent Step Functions executions are capped at
* `--max-concurrent` (default 50) via a semaphore so a 10 000-entry batch
* file doesn't try to start 10 000 executions simultaneously and trip the
* AWS account's concurrent-execution limit.
*
* Per-entry results land in a manifest: one row per input line with the
* `executionArn` + status. `--dry-run` skips the AWS calls and prints the
* manifest with `status: "would-invoke"` for each entry so callers can
* lint their batch file without paying for any executions.
*
* JSONL format (one JSON object per line):
*
* {"outputKey": "renders/alice.mp4", "variables": {"name": "Alice"}}
* {"outputKey": "renders/bob.mp4", "variables": {"name": "Bob"}}
*/
import { existsSync, readFileSync } from "node:fs";
import { join, resolve as resolvePath } from "node:path";
import type {
DistributedFormat,
SerializableDistributedRenderConfig,
SiteHandle,
} from "@hyperframes/aws-lambda/sdk";
import { c } from "../../ui/colors.js";
import { errorBox } from "../../ui/format.js";
import {
loadProjectVariableSchema,
reportVariableIssues,
validateVariablesAgainstSchema,
} from "../../utils/variables.js";
import { requireStack } from "./state.js";
// Dynamic-import the SDK so tsup keeps it out of the static-import head of
// the CLI bundle. See sites.ts loadSDK() for the full rationale.
async function loadSDK(): Promise<typeof import("@hyperframes/aws-lambda/sdk")> {
return import("@hyperframes/aws-lambda/sdk");
}
/** Arguments accepted by `hyperframes lambda render-batch`. */
export interface RenderBatchArgs {
projectDir: string;
stackName: string;
/** Path to the JSONL batch file. Each line is a {@link BatchEntry}. */
batch: string;
/**
* Skip the project upload and re-use an existing pre-deployed site. The
* batch verb deploys the site once and reuses it across renders by
* default — this flag is for cases where the site was uploaded by a
* separate `sites create` step (CI / cross-machine).
*/
siteId?: string;
/** Composition config — fps/width/height/format required, rest optional. */
fps: 24 | 30 | 60;
width: number;
height: number;
format: DistributedFormat;
codec?: "h264" | "h265";
quality?: "draft" | "standard" | "high";
chunkSize?: number;
maxParallelChunks?: number;
/**
* Maximum in-flight Step Functions starts at any moment. Caps fan-out
* so a 10 000-entry batch doesn't try to spawn 10 000 executions
* simultaneously. Defaults to 50.
*
* Distinct from `maxParallelChunks` (which caps chunks PER render).
* Lambda concurrent-execution limits live one level up at the AWS
* account level and this CLI cannot enforce those; the cap here is
* purely orchestrator-side.
*/
maxConcurrent?: number;
/**
* `--strict-variables` applies to every batch entry's pre-validation.
* Mismatches print as warnings; in strict mode the first failing entry
* aborts the run before any AWS call.
*/
strictVariables?: boolean;
/**
* Don't actually invoke `renderToLambda`. Print the manifest with
* `status: "would-invoke"` for every entry. Used to lint the batch
* file before committing to N billable executions.
*/
dryRun?: boolean;
/** Print machine-readable JSON instead of the human-friendly summary. */
json: boolean;
}
/**
* A single line in the JSONL batch file. Each entry produces one Step
* Functions execution with the supplied `variables` injected into the
* composition's `window.__hfVariables`.
*/
export interface BatchEntry {
/**
* Final output S3 key for this entry's render. Per-entry so the caller
* controls the output layout (e.g. `renders/users/alice.mp4`). Without
* an explicit value the SDK falls back to its
* `renders/<executionName>/output.<ext>` default, which makes a 100-row
* batch unreadable.
*/
outputKey: string;
/**
* Variable overrides for this entry. Merged over the composition's
* declared defaults inside the chunk worker (via
* `window.__hfVariables`). Optional — pass `{}` if a row needs the
* composition's defaults verbatim.
*/
variables?: Record<string, unknown>;
/**
* Optional explicit Step Functions execution name. Defaults to
* `hf-render-<uuid>` (generated by the SDK). Useful when the caller
* wants to correlate batch rows with downstream systems.
*/
executionName?: string;
}
/** Single row of the manifest emitted by `render-batch`. */
interface BatchManifestEntry {
/** 1-based index of the source JSONL line (after blank-line stripping). */
inputLine: number;
/** Output S3 key the SDK was asked to produce. */
outputKey: string;
/**
* SFN execution ARN, or `null` for entries that failed-to-start or were
* `--dry-run`-skipped. The latter case has `status: "would-invoke"`.
*/
executionArn: string | null;
/** Stable status discriminator the caller's manifest consumer can switch on. */
status: "started" | "would-invoke" | "failed-to-start";
/** Error message when `status === "failed-to-start"`. */
error?: string;
}
const DEFAULT_MAX_CONCURRENT = 50;
/**
* Run the batch render. Throws on usage errors (bad CLI flags, missing
* project dir, malformed batch file); per-entry failures are captured in
* the manifest with `status: "failed-to-start"` rather than aborting the
* whole batch.
*/
// fallow-ignore-next-line complexity
export async function runRenderBatch(args: RenderBatchArgs): Promise<void> {
const projectDir = resolvePath(args.projectDir);
const stack = requireStack(args.stackName);
const batchPath = resolvePath(args.batch);
if (!existsSync(batchPath)) {
errorBox("Batch file not found", `No such file: ${batchPath}`);
process.exit(1);
}
const entries = parseBatchFile(batchPath);
if (entries.length === 0) {
errorBox("Empty batch", `${batchPath} contains zero entries (every line was blank).`);
process.exit(1);
}
// Pre-validate every entry's variables against the composition's
// schema. Mismatches print as warnings; strict mode aborts before any
// AWS call. Schema is loaded once and reused across entries — a 10k-row
// batch with the per-entry parser would do 10k readFile + DOM parses.
//
// In strict mode we accumulate every failing entry first, then exit
// once with the full list. The naive "exit on first failure" pattern
// would force the caller to fix-one → re-run × N times on a batch with
// N broken rows.
const schema = loadProjectVariableSchema(join(projectDir, "index.html"));
const strict = args.strictVariables ?? false;
let hadStrictIssue = false;
for (const { entry, lineNumber } of entries) {
if (!entry.variables || Object.keys(entry.variables).length === 0) continue;
const issues = validateVariablesAgainstSchema(entry.variables, schema);
if (issues.length === 0) continue;
if (!args.json) {
console.log("");
console.log(c.dim(`Batch entry on line ${lineNumber}:`));
}
// Pass strict: false here so the helper just prints; we own the
// single exit-at-end below.
reportVariableIssues(issues, { strict: false, quiet: args.json });
if (strict) hadStrictIssue = true;
}
if (hadStrictIssue) {
errorBox(
"Variable validation failed",
"Aborting batch due to variable issues in one or more entries (--strict-variables mode).",
);
process.exit(1);
}
const config: SerializableDistributedRenderConfig = {
fps: args.fps,
width: args.width,
height: args.height,
format: args.format,
codec: args.codec,
quality: args.quality,
chunkSize: args.chunkSize,
maxParallelChunks: args.maxParallelChunks,
runtimeCap: "lambda",
};
// Deploy the site once and reuse it across every entry. --site-id and
// --dry-run both skip the deploy via a synthesised handle.
let siteHandle: SiteHandle;
if (args.siteId) {
siteHandle = makePlaceholderSiteHandle(args.siteId, stack.bucketName);
} else if (args.dryRun) {
siteHandle = makePlaceholderSiteHandle("dry-run-site", stack.bucketName);
} else {
const { deploySite } = await loadSDK();
siteHandle = await deploySite({
projectDir,
bucketName: stack.bucketName,
region: stack.region,
});
if (!args.json) {
console.log(
c.success(
siteHandle.uploaded
? `Site uploaded once for the batch: ${siteHandle.siteId}`
: `Site already up to date (skipped upload): ${siteHandle.siteId}`,
),
);
console.log();
}
}
const maxConcurrent = args.maxConcurrent ?? DEFAULT_MAX_CONCURRENT;
// Skip the SDK import entirely on --dry-run; the startEntry closure
// short-circuits before touching `renderToLambda` so it can stay
// undefined.
const renderToLambda = args.dryRun ? undefined : (await loadSDK()).renderToLambda;
const startEntry = async (item: {
entry: BatchEntry;
lineNumber: number;
}): Promise<BatchManifestEntry> => {
const { entry, lineNumber } = item;
if (args.dryRun) {
return {
inputLine: lineNumber,
outputKey: entry.outputKey,
executionArn: null,
status: "would-invoke",
};
}
if (!renderToLambda) {
// Unreachable: dryRun returns above; this branch is for the TS
// narrower since `renderToLambda` is undefined under dryRun.
throw new Error("[render-batch] renderToLambda is undefined outside --dry-run");
}
try {
const handle = await renderToLambda({
siteHandle,
bucketName: stack.bucketName,
stateMachineArn: stack.stateMachineArn,
region: stack.region,
config: { ...config, variables: entry.variables },
executionName: entry.executionName,
outputKey: entry.outputKey,
});
return {
inputLine: lineNumber,
outputKey: entry.outputKey,
executionArn: handle.executionArn,
status: "started",
};
} catch (err) {
return {
inputLine: lineNumber,
outputKey: entry.outputKey,
executionArn: null,
status: "failed-to-start",
error: err instanceof Error ? err.message : String(err),
};
}
};
const manifest = await runWithConcurrencyLimit(entries, maxConcurrent, startEntry);
if (args.json) {
console.log(JSON.stringify(manifest, null, 2));
return;
}
const started = manifest.filter((m) => m.status === "started").length;
const failed = manifest.filter((m) => m.status === "failed-to-start").length;
const wouldInvoke = manifest.filter((m) => m.status === "would-invoke").length;
if (args.dryRun) {
console.log(c.success(`Dry-run complete: ${wouldInvoke} entries would invoke.`));
} else {
console.log(c.success(`Batch dispatched: ${started} started, ${failed} failed-to-start.`));
}
console.log();
for (const row of manifest) {
const tag =
row.status === "started"
? c.success("✓")
: row.status === "would-invoke"
? c.dim("·")
: c.error("✗");
const detail =
row.status === "failed-to-start"
? c.error(row.error ?? "unknown error")
: (row.executionArn ?? c.dim("(no execution)"));
console.log(` ${tag} line ${row.inputLine} ${c.dim(row.outputKey)} ${detail}`);
}
if (failed > 0) process.exitCode = 1;
}
/**
* Synthesise a SiteHandle from a known siteId for paths that skip the
* `deploySite` upload (`--site-id` and `--dry-run`). The SDK reads only
* `siteId` + `projectS3Uri` when `uploaded: false`, so the byte / time
* fields are intentional placeholders.
*/
function makePlaceholderSiteHandle(siteId: string, bucketName: string): SiteHandle {
return {
siteId,
bucketName,
projectS3Uri: `s3://${bucketName}/sites/${siteId}/project.tar.gz`,
bytes: 0,
uploadedAt: "",
uploaded: false,
};
}
/**
* Read the JSONL batch file and return one parsed entry per non-blank
* line. Reads the whole file into memory — fine for typical batch sizes,
* an in-memory bound the caller can size around. Calls `errorBox` and
* `process.exit(1)` on the first malformed line so the caller doesn't
* have to sift through thousands of rows looking for the typo.
*
* Exported for unit-test coverage; production callers go through
* {@link runRenderBatch} which handles the file-not-found case.
*/
// fallow-ignore-next-line complexity
export function parseBatchFile(path: string): Array<{ entry: BatchEntry; lineNumber: number }> {
const raw = readFileSync(path, "utf8");
const lines = raw.split(/\r?\n/);
const out: Array<{ entry: BatchEntry; lineNumber: number }> = [];
for (let i = 0; i < lines.length; i++) {
const line = lines[i]!.trim();
if (line === "") continue;
let parsed: unknown;
try {
parsed = JSON.parse(line);
} catch (err) {
errorBox(
`Invalid JSON in batch file on line ${i + 1}`,
err instanceof Error ? err.message : String(err),
);
process.exit(1);
}
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
errorBox(
`Invalid batch entry on line ${i + 1}`,
'Each line must be a JSON object with at least { "outputKey": "..." }.',
);
process.exit(1);
}
const obj = parsed as Record<string, unknown>;
const outputKey = obj.outputKey;
if (typeof outputKey !== "string" || outputKey.length === 0) {
errorBox(
`Missing outputKey on line ${i + 1}`,
'Each batch entry needs a non-empty "outputKey" string (e.g. "renders/alice.mp4").',
);
process.exit(1);
}
if (obj.variables !== undefined) {
if (
obj.variables === null ||
typeof obj.variables !== "object" ||
Array.isArray(obj.variables)
) {
errorBox(
`Invalid variables on line ${i + 1}`,
'"variables" must be a JSON object (or omitted).',
);
process.exit(1);
}
}
if (obj.executionName !== undefined && typeof obj.executionName !== "string") {
errorBox(
`Invalid executionName on line ${i + 1}`,
'"executionName" must be a string (or omitted).',
);
process.exit(1);
}
out.push({
entry: {
outputKey,
variables: obj.variables as Record<string, unknown> | undefined,
executionName: obj.executionName as string | undefined,
},
lineNumber: i + 1,
});
}
return out;
}
/**
* Run `worker` against every input with at most `limit` concurrent
* invocations. Preserves input order in the returned array; each output
* is positional with its input.
*
* Implementation: index-cursor + N concurrent producers each picking the
* next index until the input is drained. Uses `Promise.all` and
* propagates the first worker rejection — partial-failure isolation is
* the caller's responsibility (in this file, `startEntry` wraps the
* per-entry render in try/catch so a single failure surfaces in the
* manifest rather than aborting the batch).
*
* Exported so unit tests can pin the concurrency cap independently of
* the AWS-dependent fan-out path.
*/
export async function runWithConcurrencyLimit<T, R>(
inputs: readonly T[],
limit: number,
worker: (input: T, index: number) => Promise<R>,
): Promise<R[]> {
if (limit < 1) throw new Error(`runWithConcurrencyLimit: limit must be ≥ 1, got ${limit}`);
const results = new Array<R>(inputs.length);
let cursor = 0;
const workerCount = Math.min(limit, inputs.length);
await Promise.all(
Array.from({ length: workerCount }, async () => {
while (cursor < inputs.length) {
const idx = cursor++;
results[idx] = await worker(inputs[idx]!, idx);
}
}),
);
return results;
}