ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-03 04:38:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): make cloud archives size-aware

Browse Source
This commit is contained in:
James
2026-07-17 18:44:58 -04:00
parent 43b52f7d6e
commit e73304fb0e
16 changed files with 426 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/package.json
+1
View File
@@ -39,6 +39,7 @@
"fontkit": "^2.0.4",
"giget": "^3.2.0",
"hono": "^4.0.0",
"ignore": "^5.3.2",
"onnxruntime-node": "1.23.2",
"open": "^10.0.0",
"postcss": "^8.5.8",
packages/cli/src/cloud/errors.test.ts
+14
View File
@@ -50,6 +50,20 @@ describe("cloud/errors reportApiError", () => {
);
});
it("points oversized projects to dry-run and .hyperframesignore", () => {
const err = new HyperframesApiError({
status: 413,
message: "project too large",
code: "hyperframes_project_too_large",
});
expect(() => reportApiError("Upload failed", err)).toThrow("process.exit called");
expect(errorBox).toHaveBeenCalledWith(
"Upload failed (HTTP 413)",
"project too large",
"The zip exceeded the 200 MB limit. Run `hyperframes cloud render --dry-run`, then add only verified-unneeded paths to `.hyperframesignore` or pre-host required large media.",
);
});
it("prefers the code-specific hint over a caller suggestion", () => {
const err = new HyperframesApiError({
status: 400,
packages/cli/src/cloud/errors.ts
+1 -1
View File
@@ -19,7 +19,7 @@ import { HyperframesApiError } from "./_gen/client.js";
*/
const ERROR_CODE_HINTS: Record<string, string> = {
hyperframes_project_too_large:
"The zip exceeded the 200 MB limit. Trim large media (or pre-host them and reference by URL), then try again.",
"The zip exceeded the 200 MB limit. Run `hyperframes cloud render --dry-run`, then add only verified-unneeded paths to `.hyperframesignore` or pre-host required large media.",
hyperframes_render_not_found:
"The render_id no longer exists — either soft-deleted or never created.",
invalid_parameter:
packages/cli/src/commands/cloud.ts
+1
View File
@@ -18,6 +18,7 @@ import { c } from "../ui/colors.js";
export const examples: Example[] = [
["Render the current directory in the cloud", "hyperframes cloud render"],
["Render a specific project", "hyperframes cloud render ./my-video"],
["Inspect the project zip without uploading", "hyperframes cloud render --dry-run"],
[
"Render at 60fps + high quality, save to a path",
"hyperframes cloud render ./my-video --fps 60 --quality high -o ./out.mp4",
packages/cli/src/commands/cloud/render.test.ts
+77 -2
View File
@@ -1,14 +1,19 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { spawnSync } from "node:child_process";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
import {
resolveAspectRatioForSubmit,
validateDryRunSource,
validateResolutionFormatCombo,
type ProjectInputSource,
} from "./render.js";
const cliEntry = resolve(fileURLToPath(import.meta.url), "..", "..", "..", "cli.ts");
// errorBox writes to console; silence it so test output stays clean.
beforeEach(() => {
vi.spyOn(console, "log").mockImplementation(() => undefined);
@@ -51,6 +56,76 @@ describe("validateResolutionFormatCombo", () => {
});
});
describe("validateDryRunSource", () => {
it("accepts a local directory and rejects already-uploaded sources", () => {
const exit = trapExit();
expect(() => validateDryRunSource({ kind: "dir", dir: "." }, true)).not.toThrow();
expect(() => validateDryRunSource({ kind: "asset_id", assetId: "asst_123" }, true)).toThrow(
"process.exit:1",
);
expect(() =>
validateDryRunSource({ kind: "url", url: "https://example.com/project.zip" }, true),
).toThrow("process.exit:1");
expect(exit).toHaveBeenCalledTimes(2);
});
});
describe("cloud render --dry-run", () => {
it("reports the real archive without contacting the cloud", () => {
const dir = writeComposition(1920, 1080);
try {
writeFileSync(
join(dir, "index.html"),
`<!doctype html><html data-composition-variables='[{"id":"title","type":"string","label":"Title","default":"x"}]'><body><div data-composition-id="main" data-width="1920" data-height="1080"></div></body></html>`,
"utf-8",
);
writeFileSync(join(dir, "asset.bin"), "archive-input", "utf-8");
const result = spawnSync(
"bun",
[
"run",
cliEntry,
"cloud",
"render",
dir,
"--dry-run",
"--json",
"--variables",
'{"extra":1}',
],
{
encoding: "utf-8",
timeout: 30_000,
env: {
...process.env,
CI: "1",
HEYGEN_API_URL: "http://127.0.0.1:1",
HYPERFRAMES_NO_UPDATE_CHECK: "1",
},
},
);
expect(result.status).toBe(0);
const payload = JSON.parse(result.stdout) as {
archive: {
file_count: number;
size_bytes: number;
upload_limit_bytes: number;
exceeds_upload_limit: boolean;
largest_files: Array<{ path: string }>;
};
};
expect(payload.archive.file_count).toBe(2);
expect(payload.archive.size_bytes).toBeGreaterThan(0);
expect(payload.archive.upload_limit_bytes).toBe(200 * 1024 * 1024);
expect(payload.archive.exceeds_upload_limit).toBe(false);
expect(payload.archive.largest_files.map((file) => file.path)).toContain("asset.bin");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
describe("resolveAspectRatioForSubmit — non-local sources", () => {
it("trusts an explicit flag for asset_id / url", () => {
trapExit();
packages/cli/src/commands/cloud/render.ts
+141 -20
View File
@@ -35,9 +35,13 @@ import {
} from "../../cloud/detectAspectRatio.js";
import { c } from "../../ui/colors.js";
import { errorBox, formatBytes, formatDuration } from "../../ui/format.js";
import { resolveProject } from "../../utils/project.js";
import { resolveProject, type ProjectDir } from "../../utils/project.js";
import { normalizeErrorMessage } from "../../utils/errorMessage.js";
import { createPublishArchive } from "../../utils/publishProject.js";
import {
buildPublishFileMap,
type PublishArchiveResult,
zipPublishFileMap,
} from "../../utils/publishProject.js";
import {
reportVariableIssues,
resolveVariablesArg,
@@ -63,18 +67,23 @@ import type {
HyperframesCloudClient,
HyperframesRenderDetail,
} from "../../cloud/index.js";
import { isAbsolute, resolve as resolvePath } from "node:path";
import { isAbsolute, relative, resolve as resolvePath } from "node:path";
import { existsSync } from "node:fs";
const VALID_QUALITY = ["draft", "standard", "high"] as const;
const VALID_FORMAT = ["mp4", "webm", "mov"] as const;
const VALID_RESOLUTION = ["1080p", "4k"] as const;
const VALID_ASPECT_RATIO = ["16:9", "9:16", "1:1"] as const;
// Mirrors the binary-byte max_bytes contract used by direct asset uploads.
const DIRECT_UPLOAD_LIMIT_BYTES = 200 * 1024 * 1024;
const ARCHIVE_DIAGNOSTIC_THRESHOLD_BYTES = 150 * 1024 * 1024;
const LARGEST_FILE_COUNT = 10;
const FORMAT_EXT: Record<string, string> = { mp4: ".mp4", webm: ".webm", mov: ".mov" };
export const examples: Example[] = [
["Render the current directory in the cloud", "hyperframes cloud render"],
["Inspect archive size without uploading", "hyperframes cloud render --dry-run"],
[
"Pick a specific composition + output path",
"hyperframes cloud render . --composition compositions/intro.html -o ./renders/intro.mp4",
@@ -178,6 +187,11 @@ export default defineCommand({
description: "Emit machine-readable JSON instead of human-friendly progress",
default: false,
},
"dry-run": {
type: "boolean",
description: "Build and inspect the project zip without authenticating or uploading",
default: false,
},
"idempotency-key": {
type: "string",
description: "Optional Idempotency-Key for safe retries (1-255 chars from [A-Za-z0-9_:.-])",
@@ -207,6 +221,7 @@ export default defineCommand({
assetId: args["asset-id"],
url: args.url,
});
validateDryRunSource(project, args["dry-run"] ?? false);
// 4k supersampling runs through the alpha-incompatible screenshot path;
// reject the combination client-side instead of failing mid-render.
@@ -226,6 +241,12 @@ export default defineCommand({
asJson,
);
if (args["dry-run"]) {
if (project.kind !== "dir") throw new Error("Dry-run project must be a local directory");
reportDryRun(prepareLocalArchive(project, args.composition, asJson), asJson);
return;
}
const variables = resolveVariablesAndValidateIfLocal(
args.variables,
args["variables-file"],
@@ -234,8 +255,16 @@ export default defineCommand({
);
const client = await createCloudClient();
const preparedArchive =
project.kind === "dir" ? prepareLocalArchive(project, args.composition, asJson) : undefined;
const upload = await maybeUploadProject(client, project, asJson, args["idempotency-key"]);
const upload = await maybeUploadProject(
client,
project,
preparedArchive,
asJson,
args["idempotency-key"],
);
const submitted = await submitRender(client, {
projectInput: upload.projectInput,
fps,
@@ -334,6 +363,15 @@ function validateIdempotencyKey(key: string | undefined): void {
}
}
export function validateDryRunSource(source: ProjectInputSource, dryRun: boolean): void {
if (!dryRun || source.kind === "dir") return;
errorBox(
"Invalid --dry-run input",
"--dry-run inspects a local project directory and cannot be combined with --asset-id or --url.",
);
process.exit(1);
}
// ---------------------------------------------------------------------------
// Project resolution (dir | asset-id | url) — exactly one source
// ---------------------------------------------------------------------------
@@ -523,10 +561,107 @@ interface UploadResult {
projectInput: CreateHyperframesRenderRequest["project"];
}
interface ArchiveFileSummary {
path: string;
size_bytes: number;
}
interface PreparedLocalArchive {
project: ProjectDir;
archive: PublishArchiveResult;
largestFiles: ArchiveFileSummary[];
}
function summarizeLargestFiles(fileMap: Map<string, Buffer>): ArchiveFileSummary[] {
return [...fileMap.entries()]
.map(([path, content]) => ({ path, size_bytes: content.byteLength }))
.sort((a, b) => b.size_bytes - a.size_bytes || a.path.localeCompare(b.path))
.slice(0, LARGEST_FILE_COUNT);
}
function prepareLocalArchive(
source: ProjectInputSource,
composition: string | undefined,
asJson: boolean,
): PreparedLocalArchive {
const project = resolveProject(source.dir);
if (!asJson) {
console.log("");
console.log(`${c.accent("◆")} Zipping ${c.accent(project.name)}`);
}
try {
const fileMap = buildPublishFileMap(project.dir);
const entryPath = relative(
project.dir,
resolvePath(project.dir, composition ?? "index.html"),
).replaceAll("\\", "/");
if (!fileMap.has(entryPath)) {
throw new Error(
`Composition "${composition ?? "index.html"}" is excluded from the archive. Check .hyperframesignore.`,
);
}
const archive = zipPublishFileMap(fileMap);
if (!asJson) {
console.log(
c.dim(` ${archive.fileCount} files · ${formatBytes(archive.buffer.byteLength)}`),
);
}
const prepared = { project, archive, largestFiles: summarizeLargestFiles(fileMap) };
if (!asJson && archive.buffer.byteLength >= ARCHIVE_DIAGNOSTIC_THRESHOLD_BYTES) {
reportLargestFiles(prepared);
}
return prepared;
} catch (err) {
const msg = normalizeErrorMessage(err);
errorBox("Zip failed", msg, "Check the project and .hyperframesignore for missing files.");
process.exit(1);
}
}
function reportLargestFiles(prepared: PreparedLocalArchive): void {
console.log(c.dim(" Largest included files:"));
for (const file of prepared.largestFiles) {
console.log(c.dim(` ${formatBytes(file.size_bytes).padStart(9)} ${file.path}`));
}
console.log(c.dim(" Exclude verified-unneeded files with .hyperframesignore."));
}
function reportDryRun(prepared: PreparedLocalArchive, asJson: boolean): void {
const sizeBytes = prepared.archive.buffer.byteLength;
if (asJson) {
console.log(
JSON.stringify(
withMeta({
archive: {
project: prepared.project.name,
file_count: prepared.archive.fileCount,
size_bytes: sizeBytes,
upload_limit_bytes: DIRECT_UPLOAD_LIMIT_BYTES,
exceeds_upload_limit: sizeBytes > DIRECT_UPLOAD_LIMIT_BYTES,
largest_files: prepared.largestFiles,
},
}),
null,
2,
),
);
return;
}
if (sizeBytes < ARCHIVE_DIAGNOSTIC_THRESHOLD_BYTES) reportLargestFiles(prepared);
console.log("");
console.log(`${c.success("✓")} Dry run complete — nothing uploaded`);
if (sizeBytes > DIRECT_UPLOAD_LIMIT_BYTES) {
console.log(c.warn(` Archive exceeds the 200 MB direct-upload limit.`));
}
}
// fallow-ignore-next-line complexity
async function maybeUploadProject(
client: HyperframesCloudClient,
source: ProjectInputSource,
preparedArchive: PreparedLocalArchive | undefined,
asJson: boolean,
idempotencyKey: string | undefined,
): Promise<UploadResult> {
@@ -537,22 +672,8 @@ async function maybeUploadProject(
return { projectInput: { type: "url", url: source.url! } };
}
const project = resolveProject(source.dir);
if (!asJson) {
console.log("");
console.log(`${c.accent("◆")} Zipping ${c.accent(project.name)}`);
}
let archive;
try {
archive = createPublishArchive(project.dir);
} catch (err) {
const msg = normalizeErrorMessage(err);
errorBox("Zip failed", msg, "Check the project for missing files or unreadable permissions.");
process.exit(1);
}
if (!asJson) {
console.log(c.dim(` ${archive.fileCount} files · ${formatBytes(archive.buffer.byteLength)}`));
}
if (!preparedArchive) throw new Error("Local project archive was not prepared");
const { project, archive } = preparedArchive;
if (!asJson) {
console.log("");
packages/cli/src/commands/coreSkillContent.test.ts
+11
View File
@@ -49,6 +49,17 @@ describe("hyperframes-core contract docs", () => {
expect(renderReference).toContain("timeline:");
expect(renderReference).toContain("buildCompositionCensus");
});
it("teaches safe cloud archive size remediation", () => {
const skill = read("skills", "hyperframes-cli", "SKILL.md");
const cloudReference = read("skills", "hyperframes-cli", "references", "cloud.md");
expect(skill).toContain("cloud render --dry-run --json");
expect(skill).toContain("Never ignore an asset merely because it is large");
expect(cloudReference).toContain(".hyperframesignore");
expect(cloudReference).toContain("Never ignore all of `assets/`");
expect(cloudReference).toContain("dynamically computed asset path");
});
});
describe("media-use TTS documentation", () => {
packages/cli/src/utils/publishProject.test.ts
+78
View File
@@ -208,6 +208,84 @@ describe("createPublishArchive", () => {
rmSync(dir, { recursive: true, force: true });
}
});
it("skips root render outputs but keeps same-named nested source directories", () => {
const dir = makeProjectDir();
try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
mkdirSync(join(dir, "renders"));
writeFileSync(join(dir, "renders", "old.mp4"), "render-output", "utf-8");
mkdirSync(join(dir, "snapshots"));
writeFileSync(join(dir, "snapshots", "frame.png"), "snapshot-output", "utf-8");
mkdirSync(join(dir, "assets", "renders"), { recursive: true });
writeFileSync(join(dir, "assets", "renders", "source.mp4"), "source", "utf-8");
const zip = new AdmZip(createPublishArchive(dir).buffer);
const entries = zip.getEntries().map((entry) => entry.entryName);
expect(entries).toContain("assets/renders/source.mp4");
expect(entries).not.toContain("renders/old.mp4");
expect(entries).not.toContain("snapshots/frame.png");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("applies gitignore-style rules from .hyperframesignore", () => {
const dir = makeProjectDir();
try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
mkdirSync(join(dir, "exports"));
writeFileSync(join(dir, "exports", "draft.mp4"), "draft", "utf-8");
mkdirSync(join(dir, "assets"));
writeFileSync(join(dir, "assets", "discard.psd"), "discard", "utf-8");
writeFileSync(join(dir, "assets", "keep.psd"), "keep", "utf-8");
writeFileSync(
join(dir, ".hyperframesignore"),
["# Generated source files", "/exports/", "*.psd", "!assets/keep.psd", ""].join("\n"),
"utf-8",
);
const zip = new AdmZip(createPublishArchive(dir).buffer);
const entries = zip.getEntries().map((entry) => entry.entryName);
expect(entries).toContain("assets/keep.psd");
expect(entries).not.toContain("exports/draft.mp4");
expect(entries).not.toContain("assets/discard.psd");
expect(entries).not.toContain(".hyperframesignore");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("allows .hyperframesignore to re-include a default output directory", () => {
const dir = makeProjectDir();
try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
mkdirSync(join(dir, "snapshots"));
writeFileSync(join(dir, "snapshots", "reference.png"), "reference", "utf-8");
writeFileSync(join(dir, ".hyperframesignore"), "!/snapshots/\n", "utf-8");
const zip = new AdmZip(createPublishArchive(dir).buffer);
expect(zip.getEntries().map((entry) => entry.entryName)).toContain("snapshots/reference.png");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("fails clearly when .hyperframesignore excludes index.html", () => {
const dir = makeProjectDir();
try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");
writeFileSync(join(dir, ".hyperframesignore"), "/index.html\n", "utf-8");
expect(() => createPublishArchive(dir)).toThrow(
"Project archive must include index.html at the root. Check that .hyperframesignore does not exclude it.",
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
describe("createPublishArchive (U6 cloud-render regression guard)", () => {
packages/cli/src/utils/publishProject.ts
+29 -8
View File
@@ -2,6 +2,7 @@ import { basename, dirname, join, posix, relative, resolve } from "node:path";
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
import { parseHTML } from "linkedom";
import AdmZip from "adm-zip";
import ignore, { type Ignore } from "ignore";
import { CSS_URL_RE, isNonRelativeUrl, isPathInside } from "@hyperframes/core";
import { buildAuthHeaders } from "../auth/client.js";
import { tryResolveCredential } from "../auth/index.js";
@@ -9,6 +10,8 @@ import { writeProjectLink } from "./projectLink.js";
const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]);
const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]);
const HYPERFRAMES_IGNORE_FILE = ".hyperframesignore";
const DEFAULT_PROJECT_IGNORE = ["/renders/", "/snapshots/"];
const PUBLISH_CONTENT_TYPE = "application/zip";
const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
@@ -174,7 +177,21 @@ function shouldIgnoreSegment(segment: string): boolean {
return segment.startsWith(".") || IGNORED_DIRS.has(segment) || IGNORED_FILES.has(segment);
}
function collectProjectFiles(rootDir: string, currentDir: string, paths: string[]): void {
function createProjectIgnore(rootDir: string): Ignore {
const matcher = ignore().add(DEFAULT_PROJECT_IGNORE);
const ignorePath = join(rootDir, HYPERFRAMES_IGNORE_FILE);
if (existsSync(ignorePath)) {
matcher.add(readFileSync(ignorePath, "utf-8"));
}
return matcher;
}
function collectProjectFiles(
rootDir: string,
currentDir: string,
paths: string[],
matcher: Ignore,
): void {
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
if (shouldIgnoreSegment(entry.name)) continue;
const absolutePath = join(currentDir, entry.name);
@@ -182,11 +199,13 @@ function collectProjectFiles(rootDir: string, currentDir: string, paths: string[
if (!relativePath) continue;
if (entry.isDirectory()) {
collectProjectFiles(rootDir, absolutePath, paths);
if (matcher.ignores(`${relativePath}/`)) continue;
collectProjectFiles(rootDir, absolutePath, paths, matcher);
continue;
}
if (!statSync(absolutePath).isFile()) continue;
if (matcher.ignores(relativePath)) continue;
paths.push(relativePath);
}
}
@@ -388,9 +407,11 @@ export function localizeExternalAssets(
export function buildPublishFileMap(projectDir: string): Map<string, Buffer> {
const absProjectDir = resolve(projectDir);
const filePaths: string[] = [];
collectProjectFiles(absProjectDir, absProjectDir, filePaths);
collectProjectFiles(absProjectDir, absProjectDir, filePaths, createProjectIgnore(absProjectDir));
if (!filePaths.includes("index.html")) {
throw new Error("Project must include an index.html file at the root before publish.");
throw new Error(
"Project archive must include index.html at the root. Check that .hyperframesignore does not exclude it.",
);
}
const fileContents = new Map<string, Buffer>();
@@ -418,10 +439,10 @@ export function zipPublishFileMap(fileContents: Map<string, Buffer>): PublishArc
/**
* Thin composition of `buildPublishFileMap` + `zipPublishFileMap` — signature
* and behavior UNCHANGED from before the U6 split. `cloud render`
* (`commands/cloud/render.ts`, `maybeUploadProject`) calls this directly and
* must stay byte-identical (never see baked proxies); only `publish.ts` calls
* the two halves separately with a baking transform in between.
* and behavior UNCHANGED from before the U6 split. `cloud render` composes the
* same two functions without an intermediate transform and must stay
* byte-identical (never see baked proxies); only `publish.ts` inserts a baking
* transform between them.
*/
export function createPublishArchive(projectDir: string): PublishArchiveResult {
return zipPublishFileMap(buildPublishFileMap(projectDir));
packages/cli/src/utils/publishProxyBake.ts
+2 -2
View File
@@ -17,8 +17,8 @@
* original untouched. On-disk project files are never modified — only the
* in-memory archive file map passed in by `publish.ts` (built via
* `buildPublishFileMap`, baked here, then zipped via `zipPublishFileMap`).
* `cloud render` never calls this: it uses `createPublishArchive` directly,
* which has no baking hook (R2 in the plan).
* `cloud render` never calls this: it builds and zips the file map without an
* intermediate baking transform (R2 in the plan).
*
* Alpha-bearing sources bake as VP9/WebM so transparency survives. A failed
* hostile transcode aborts publish with a