diff --git a/packages/cli/src/cloud/_gen/client.ts b/packages/cli/src/cloud/_gen/client.ts index 42a0f5764..244f6d1ea 100644 --- a/packages/cli/src/cloud/_gen/client.ts +++ b/packages/cli/src/cloud/_gen/client.ts @@ -5,6 +5,10 @@ * experiment-framework to regenerate. */ import type { + CompleteAssetUploadRequest, + CompleteAssetUploadResponse, + CreateAssetUploadRequest, + CreateAssetUploadResponse, CreateHyperframesRenderRequest, CreateHyperframesRenderResponse, DeleteHyperframesRenderResponse, @@ -225,6 +229,43 @@ export class HyperframesCloudClient { }); } + /** + * Create Asset Upload + * + * Begin a direct-to-S3 upload. Returns an asset_id and a presigned upload_url; PUT the file bytes to upload_url, then call POST /v3/assets/{asset_id}/complete. Unlike POST /v3/assets (which proxies the bytes), this never sends the file through the API. + */ + async createAssetUpload(args: { + body: CreateAssetUploadRequest; + idempotencyKey?: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/assets/direct-uploads", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + /** + * Complete Asset Upload + * + * Finalize a direct-to-S3 upload into a reusable asset. Call after the upload PUT returns 200. Idempotent: repeated calls return the same finalized asset. + */ + async completeAssetUpload(args: { + asset_id: string; + body: CompleteAssetUploadRequest; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: `/v3/assets/${encodeURIComponent(args.asset_id)}/complete`, + body: args.body, + signal: args.signal, + }); + } + /** * Create HyperFrames Render * diff --git a/packages/cli/src/cloud/_gen/types.ts b/packages/cli/src/cloud/_gen/types.ts index a56c51120..c73b6d1cd 100644 --- a/packages/cli/src/cloud/_gen/types.ts +++ b/packages/cli/src/cloud/_gen/types.ts @@ -55,6 +55,100 @@ export interface AssetUrl { url: string; } +/** + * Finalize a presigned upload (POST /v3/assets/{asset_id}/complete). + */ +export interface CompleteAssetUploadRequest { + /** + * Optional SHA256 (hex) cross-check. + */ + checksum_sha256?: string | null; +} + +/** + * Result of finalizing an upload. + */ +export interface CompleteAssetUploadResponse { + /** + * The reusable asset identifier. + */ + asset_id: string; + /** + * Public URL of the finalized asset. + */ + url: string; + /** + * MIME type detected from the stored bytes. + */ + mime_type: string; + /** + * Size of the stored object in bytes. + */ + size_bytes: number; + /** + * Asset status, e.g. 'processing'. + */ + status: "processing"; +} + +/** + * Request to begin a presigned direct-to-S3 upload (POST + * /v3/assets/direct-uploads). + */ +export interface CreateAssetUploadRequest { + /** + * Original filename for reference/metadata. The stored object's extension is + * derived from content_type. + */ + filename: string; + /** + * Declared MIME type (e.g. 'video/mp4', 'image/png', 'audio/mpeg', + * 'application/pdf', 'application/zip'). Verified against the stored bytes at + * completion. + */ + content_type: string; + /** + * Exact byte size of the file. Signed into the upload URL so it cannot be + * exceeded. + */ + size_bytes: number; + /** + * Optional SHA256 of the file as hex. When provided, S3 enforces it on upload. + */ + checksum_sha256?: string | null; +} + +/** + * Presigned upload instructions. + */ +export interface CreateAssetUploadResponse { + /** + * Reusable asset identifier. Becomes usable after POST + * /v3/assets/{asset_id}/complete. + */ + asset_id: string; + /** + * Presigned S3 URL. PUT the raw file bytes here. + */ + upload_url: string; + /** + * Headers that must be sent verbatim on the PUT request. + */ + upload_headers: Record; + /** + * Seconds until the upload URL expires. + */ + expires_in_seconds: number; + /** + * Maximum allowed upload size in bytes. + */ + max_bytes: number; + /** + * Upload lifecycle status. Always 'pending_upload' here. + */ + status: "pending_upload"; +} + /** * Request body for POST /v3/hyperframes/renders. */ diff --git a/packages/cli/src/cloud/errors.ts b/packages/cli/src/cloud/errors.ts index 4c823c90c..7a6c248e4 100644 --- a/packages/cli/src/cloud/errors.ts +++ b/packages/cli/src/cloud/errors.ts @@ -19,7 +19,7 @@ import { HyperframesApiError } from "./_gen/client.js"; */ const ERROR_CODE_HINTS: Record = { hyperframes_project_too_large: - "The zip exceeded the 32 MB limit. Trim large media (or pre-host them and reference by URL), then try again.", + "The zip exceeded the 200 MB limit. Trim large media (or pre-host them and reference by URL), then try again.", hyperframes_render_not_found: "The render_id no longer exists — either soft-deleted or never created.", invalid_parameter: diff --git a/packages/cli/src/cloud/upload.test.ts b/packages/cli/src/cloud/upload.test.ts new file mode 100644 index 000000000..89713f83a --- /dev/null +++ b/packages/cli/src/cloud/upload.test.ts @@ -0,0 +1,234 @@ +import { describe, it, expect, vi } from "vitest"; +import { createHash } from "node:crypto"; +import { HyperframesApiError } from "./_gen/client.js"; +import type { HyperframesCloudClient } from "./_gen/client.js"; +import { uploadZipViaDirectUpload } from "./upload.js"; + +function sha256Hex(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function makeClient(overrides: Partial = {}): HyperframesCloudClient { + return { + createAssetUpload: vi.fn(async () => ({ + asset_id: "asset_xyz", + upload_url: "https://s3.example/asset_xyz?sig=abc", + upload_headers: { "x-amz-checksum-sha256-b64": "..." }, + expires_in_seconds: 3600, + max_bytes: 200 * 1024 * 1024, + status: "pending_upload" as const, + })), + completeAssetUpload: vi.fn(async () => ({ + asset_id: "asset_xyz", + url: "https://files.heygen.com/document/asset_xyz/original.zip", + mime_type: "application/zip", + size_bytes: 42, + status: "processing" as const, + })), + ...overrides, + } as unknown as HyperframesCloudClient; +} + +function makeFetchOk(): typeof fetch { + return vi.fn(async () => new Response("", { status: 200 })) as unknown as typeof fetch; +} + +describe("uploadZipViaDirectUpload", () => { + it("sends the correct filename, content_type, size_bytes, and SHA256 to createAssetUpload", async () => { + const bytes = new TextEncoder().encode("hello-hyperframes-zip"); + const expectedSha = sha256Hex(bytes); + const client = makeClient(); + const fetchImpl = makeFetchOk(); + + await uploadZipViaDirectUpload({ + client, + bytes, + filename: "my-comp.zip", + fetchImpl, + }); + + expect(client.createAssetUpload).toHaveBeenCalledOnce(); + const arg = (client.createAssetUpload as ReturnType).mock.calls[0]![0]; + expect(arg.body).toEqual({ + filename: "my-comp.zip", + content_type: "application/zip", + size_bytes: bytes.byteLength, + checksum_sha256: expectedSha, + }); + }); + + it("PUTs to the returned upload_url with the returned upload_headers verbatim + content-type", async () => { + const bytes = new Uint8Array([1, 2, 3, 4]); + const client = makeClient({ + createAssetUpload: vi.fn(async () => ({ + asset_id: "asset_xyz", + upload_url: "https://s3.example/target?sig=xxx", + upload_headers: { "x-signed-header": "signed-value", "x-other": "other-value" }, + expires_in_seconds: 3600, + max_bytes: 200 * 1024 * 1024, + status: "pending_upload" as const, + })) as HyperframesCloudClient["createAssetUpload"], + }); + const fetchImpl = makeFetchOk(); + + await uploadZipViaDirectUpload({ client, bytes, filename: "x.zip", fetchImpl }); + + expect(fetchImpl).toHaveBeenCalledOnce(); + const [url, init] = (fetchImpl as ReturnType).mock.calls[0]!; + expect(url).toBe("https://s3.example/target?sig=xxx"); + expect(init.method).toBe("PUT"); + expect(init.body).toBe(bytes); + expect(init.headers).toEqual({ + "content-type": "application/zip", + "x-signed-header": "signed-value", + "x-other": "other-value", + }); + }); + + it("does NOT attach CLI auth headers to the S3 PUT — presigned URL carries auth", async () => { + const client = makeClient(); + const fetchImpl = makeFetchOk(); + + await uploadZipViaDirectUpload({ + client, + bytes: new Uint8Array([0]), + filename: "x.zip", + fetchImpl, + }); + + const init = (fetchImpl as ReturnType).mock.calls[0]![1]; + // No Authorization / x-api-key / Bearer header should be present. + const headerKeys = Object.keys(init.headers as Record).map((k) => + k.toLowerCase(), + ); + expect(headerKeys).not.toContain("authorization"); + expect(headerKeys).not.toContain("x-api-key"); + }); + + it("calls completeAssetUpload with the initialize's asset_id + same checksum", async () => { + const bytes = new TextEncoder().encode("determinism"); + const expectedSha = sha256Hex(bytes); + const client = makeClient(); + const fetchImpl = makeFetchOk(); + + const result = await uploadZipViaDirectUpload({ + client, + bytes, + filename: "x.zip", + fetchImpl, + }); + + expect(client.completeAssetUpload).toHaveBeenCalledOnce(); + const completeArg = (client.completeAssetUpload as ReturnType).mock.calls[0]![0]; + expect(completeArg.asset_id).toBe("asset_xyz"); + expect(completeArg.body).toEqual({ checksum_sha256: expectedSha }); + expect(result.asset_id).toBe("asset_xyz"); + expect(result.size_bytes).toBe(bytes.byteLength); + }); + + it("retries completeAssetUpload on 409 and succeeds on a later attempt", async () => { + let completeCalls = 0; + const completeAssetUpload = vi.fn(async () => { + completeCalls++; + if (completeCalls < 3) { + throw new HyperframesApiError({ + status: 409, + message: "Uploaded object not found yet. Retry after upload PUT returns 200.", + code: "conflict", + }); + } + return { + asset_id: "asset_xyz", + url: "u", + mime_type: "application/zip", + size_bytes: 1, + status: "processing" as const, + }; + }); + const client = makeClient({ + completeAssetUpload: + completeAssetUpload as unknown as HyperframesCloudClient["completeAssetUpload"], + }); + + const result = await uploadZipViaDirectUpload({ + client, + bytes: new Uint8Array([0]), + filename: "x.zip", + fetchImpl: makeFetchOk(), + }); + + expect(completeCalls).toBe(3); + expect(result.asset_id).toBe("asset_xyz"); + }); + + it("surfaces non-409 errors from complete without retrying", async () => { + let completeCalls = 0; + const completeAssetUpload = vi.fn(async () => { + completeCalls++; + throw new HyperframesApiError({ + status: 400, + message: "invalid checksum", + code: "invalid_parameter", + }); + }); + const client = makeClient({ + completeAssetUpload: + completeAssetUpload as unknown as HyperframesCloudClient["completeAssetUpload"], + }); + + await expect( + uploadZipViaDirectUpload({ + client, + bytes: new Uint8Array([0]), + filename: "x.zip", + fetchImpl: makeFetchOk(), + }), + ).rejects.toThrow(/invalid checksum/); + expect(completeCalls).toBe(1); + }); + + it("surfaces PUT failures with response body detail", async () => { + const client = makeClient(); + const fetchImpl = vi.fn( + async () => + new Response("SignatureDoesNotMatch: request signature we calculated does not match", { + status: 403, + }), + ) as unknown as typeof fetch; + + await expect( + uploadZipViaDirectUpload({ + client, + bytes: new Uint8Array([0]), + filename: "x.zip", + fetchImpl, + }), + ).rejects.toThrow(/Direct upload PUT failed: 403.*SignatureDoesNotMatch/); + }); + + it("passes idempotencyKey through to createAssetUpload", async () => { + const client = makeClient(); + await uploadZipViaDirectUpload({ + client, + bytes: new Uint8Array([0]), + filename: "x.zip", + idempotencyKey: "test-key-123", + fetchImpl: makeFetchOk(), + }); + const arg = (client.createAssetUpload as ReturnType).mock.calls[0]![0]; + expect(arg.idempotencyKey).toBe("test-key-123"); + }); + + it("emits progress events in order", async () => { + const client = makeClient(); + const events: Array<{ phase: string }> = []; + await uploadZipViaDirectUpload({ + client, + bytes: new Uint8Array([0]), + filename: "x.zip", + fetchImpl: makeFetchOk(), + onProgress: (e) => events.push({ phase: e.phase }), + }); + expect(events.map((e) => e.phase)).toEqual(["initialize", "upload", "upload", "complete"]); + }); +}); diff --git a/packages/cli/src/cloud/upload.ts b/packages/cli/src/cloud/upload.ts new file mode 100644 index 000000000..51c4466f0 --- /dev/null +++ b/packages/cli/src/cloud/upload.ts @@ -0,0 +1,165 @@ +/** + * Direct-upload flow for `hyperframes cloud render` project asset uploads. + * + * The legacy `POST /v3/assets` path proxies bytes through the API and is + * capped at 32 MB in-memory. This module implements the three-step + * direct-to-S3 flow the CLI now uses instead, lifting the practical + * per-project ceiling to 200 MB (the direct-upload cap enforced by the + * signed presigned URL): + * + * 1. `client.createAssetUpload({filename, content_type, size_bytes, + * checksum_sha256})` → returns `{asset_id, upload_url, + * upload_headers, expires_in_seconds, max_bytes}`. + * 2. Raw `PUT` to `upload_url` with the zip bytes + `upload_headers` + * verbatim. No CLI auth headers on this call — the presigned URL + * signature carries authorization. + * 3. `client.completeAssetUpload({asset_id, body: {checksum_sha256}})` + * to finalize. Docs explicitly note a 409 "Uploaded object not + * found yet" is possible if the PUT hasn't been fully committed + * server-side; a small retry loop absorbs that. + * + * The returned `asset_id` is the same id namespace the legacy path + * produced, so `createRender({project: {type: "asset_id", asset_id}})` + * on the render side is a drop-in swap. + */ + +import { createHash } from "node:crypto"; +import type { HyperframesCloudClient } from "./_gen/client.js"; +import { HyperframesApiError } from "./_gen/client.js"; + +export interface UploadZipViaDirectResult { + asset_id: string; + size_bytes: number; + duration_ms: number; +} + +export type UploadProgressEvent = + | { phase: "initialize" } + | { phase: "upload"; percent: number } + | { phase: "complete" }; + +export interface UploadZipViaDirectOptions { + client: HyperframesCloudClient; + bytes: Uint8Array; + filename: string; + idempotencyKey?: string; + fetchImpl?: typeof fetch; + onProgress?: (event: UploadProgressEvent) => void; +} + +// Per api-docs: completeAssetUpload can return 409 "Uploaded object not found +// yet" if the S3 PUT's write hasn't propagated to the read plane by the time +// complete runs. Small backoff loop absorbs it. +const COMPLETE_MAX_RETRIES = 5; +const COMPLETE_RETRY_BASE_MS = 500; +const CONTENT_TYPE_ZIP = "application/zip"; + +function sha256Hex(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +// upload_headers is generated as `Record` from the +// OpenAPI spec, but the values are always strings at runtime (HTTP header +// values). Coerce defensively so a spec quirk can't slip a non-string in. +function normalizeUploadHeaders(raw: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(raw ?? {})) { + if (typeof v === "string") out[k] = v; + else if (typeof v === "number" || typeof v === "boolean") out[k] = String(v); + // Skip anything else — better to omit than send `[object Object]`. + } + return out; +} + +// PUT bytes to the presigned URL. Do NOT attach CLI auth headers; the +// presigned URL signature carries authorization and the signature is +// bound to the *headers signed at presign time* — extra headers invalidate +// the signature (S3 returns 403). Content-Type must match the declared +// content_type from step 1 (also signed). +async function putBytesToPresignedUrl( + fetchImpl: typeof fetch, + uploadUrl: string, + uploadHeaders: Record, + bytes: Uint8Array, +): Promise { + const headers: Record = { + "content-type": CONTENT_TYPE_ZIP, + ...normalizeUploadHeaders(uploadHeaders), + }; + // `Uint8Array` is a valid `BodyInit` at runtime but + // not strictly assignable per lib.dom.d.ts — cast rather than copy, + // since a 200MB buffer copy would be wasteful. + const res = await fetchImpl(uploadUrl, { + method: "PUT", + headers, + body: bytes as unknown as BodyInit, + }); + if (!res.ok) { + const detail = await res.text().catch(() => ""); + throw new Error( + `Direct upload PUT failed: ${res.status} ${res.statusText}${ + detail ? ` — ${detail.slice(0, 300)}` : "" + }`, + ); + } +} + +// Complete with retry-on-409. Retry ONLY on the documented "PUT not +// visible yet" race between S3 write consistency and finalize; any other +// error surfaces immediately. `completeAssetUpload` itself is idempotent, +// so retrying an already-succeeded call is safe. +async function completeWithRetry( + client: HyperframesCloudClient, + asset_id: string, + checksum_sha256: string, +): Promise<{ asset_id: string }> { + let lastErr: unknown; + for (let attempt = 0; attempt < COMPLETE_MAX_RETRIES; attempt++) { + try { + return await client.completeAssetUpload({ + asset_id, + body: { checksum_sha256 }, + }); + } catch (err) { + const retryable = err instanceof HyperframesApiError && err.status === 409; + if (!retryable || attempt === COMPLETE_MAX_RETRIES - 1) { + throw err; + } + lastErr = err; + await sleep(COMPLETE_RETRY_BASE_MS * (attempt + 1)); + } + } + throw lastErr ?? new Error("completeAssetUpload retries exhausted"); +} + +export async function uploadZipViaDirectUpload( + opts: UploadZipViaDirectOptions, +): Promise { + const start = Date.now(); + const { bytes, filename, idempotencyKey } = opts; + const size_bytes = bytes.byteLength; + const checksum_sha256 = sha256Hex(bytes); + const fetchImpl = opts.fetchImpl ?? fetch; + + opts.onProgress?.({ phase: "initialize" }); + const initialize = await opts.client.createAssetUpload({ + body: { filename, content_type: CONTENT_TYPE_ZIP, size_bytes, checksum_sha256 }, + idempotencyKey, + }); + + opts.onProgress?.({ phase: "upload", percent: 0 }); + await putBytesToPresignedUrl(fetchImpl, initialize.upload_url, initialize.upload_headers, bytes); + opts.onProgress?.({ phase: "upload", percent: 100 }); + + opts.onProgress?.({ phase: "complete" }); + const completed = await completeWithRetry(opts.client, initialize.asset_id, checksum_sha256); + return { + asset_id: completed.asset_id, + size_bytes, + duration_ms: Date.now() - start, + }; +} diff --git a/packages/cli/src/commands/cloud/render.ts b/packages/cli/src/commands/cloud/render.ts index c483eef9e..e6dd5532c 100644 --- a/packages/cli/src/commands/cloud/render.ts +++ b/packages/cli/src/commands/cloud/render.ts @@ -6,8 +6,11 @@ * `--url`). * 2. Zip the project (reuses `createPublishArchive` so the * file-ignore set matches the existing `publish` command exactly). - * 3. Upload the zip via `POST /v3/assets` (multipart) — the server - * branches on the detected `application/zip` MIME. + * 3. Upload the zip via the direct-to-S3 flow: `POST /v3/assets/ + * direct-uploads` returns a presigned URL, we PUT the zip bytes + * to it, then `POST /v3/assets/{asset_id}/complete` finalizes. + * Cap: 200 MB. See `../../cloud/upload.ts` for the three-step + * contract. (The legacy `POST /v3/assets` proxy path was 32 MB.) * 4. Submit the render via `POST /v3/hyperframes/renders` with a * `project: {type:"asset_id", asset_id}` shape. * 5. If `--no-wait`: print the `render_id` and exit immediately. @@ -53,6 +56,7 @@ import { } from "../../cloud/index.js"; import { reportApiError } from "../../cloud/errors.js"; import { parseEnumFlag, parseIntFlag, parseNumericFlag } from "../../cloud/parsing.js"; +import { uploadZipViaDirectUpload } from "../../cloud/upload.js"; import { colorStatus } from "../../cloud/statusColor.js"; import type { CreateHyperframesRenderRequest, @@ -552,22 +556,27 @@ async function maybeUploadProject( if (!asJson) { console.log(""); - console.log(`${c.accent("◆")} Uploading to /v3/assets`); + console.log(`${c.accent("◆")} Uploading (direct-to-S3)`); } const uploadStart = Date.now(); let uploaded; try { - uploaded = await client.uploadAsset({ - file: archive.buffer, + uploaded = await uploadZipViaDirectUpload({ + client, + bytes: archive.buffer, filename: `${project.name}.zip`, - // Tag the multipart part with application/zip so downstream - // proxies / WAFs / any server-side path that keys off the - // part Content-Type see the intended type. The asset - // controller currently sniffs magic bytes from the file - // bytes, so this is belt-and-suspenders today; without it, - // FormData defaults to application/octet-stream. - mimeType: "application/zip", idempotencyKey, + onProgress: !asJson + ? (ev) => { + if (ev.phase === "initialize") { + console.log(c.dim(` initializing…`)); + } else if (ev.phase === "upload" && ev.percent === 0) { + console.log(c.dim(` uploading to S3…`)); + } else if (ev.phase === "complete") { + console.log(c.dim(` finalizing…`)); + } + } + : undefined, }); } catch (err) { reportApiError("Upload failed", err);