diff --git a/packages/cli/src/cloud/_gen/client.ts b/packages/cli/src/cloud/_gen/client.ts index 850fedefb1..fa7ba00992 100644 --- a/packages/cli/src/cloud/_gen/client.ts +++ b/packages/cli/src/cloud/_gen/client.ts @@ -5,6 +5,7 @@ * experiment-framework to regenerate. */ import type { + AbortAssetUploadResponse, BackgroundRemovalJob, CompleteAssetUploadRequest, CompleteAssetUploadResponse, @@ -25,6 +26,9 @@ import type { ImageCreateRequest, LipsyncCreateResponse, LipsyncDetail, + ListUploadedPartsResponse, + MintPartUrlsRequest, + MintPartUrlsResponse, ModelVideoResponse, MusicCreateRequest, MusicResponse, @@ -271,6 +275,51 @@ export class HyperframesCloudClient { }); } + async getAssetUploadPartUrls(args: { + asset_id: string; + body: MintPartUrlsRequest; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`, + body: args.body, + signal: args.signal, + }); + } + + async listAssetUploadParts(args: { + asset_id: string; + multipart_upload_id: string; + signal?: AbortSignal; + }): Promise { + const query: Record = { + multipart_upload_id: args.multipart_upload_id, + }; + return await this.request({ + method: "GET", + path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`, + query, + signal: args.signal, + }); + } + + async abortAssetUploadParts(args: { + asset_id: string; + multipart_upload_id: string; + signal?: AbortSignal; + }): Promise { + const query: Record = { + multipart_upload_id: args.multipart_upload_id, + }; + return await this.request({ + method: "DELETE", + path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`, + query, + signal: args.signal, + }); + } + async createRender(args: { body: CreateHyperframesRenderRequest; idempotencyKey?: string; diff --git a/packages/cli/src/cloud/_gen/types.ts b/packages/cli/src/cloud/_gen/types.ts index 6fd2f63401..62759e49c9 100644 --- a/packages/cli/src/cloud/_gen/types.ts +++ b/packages/cli/src/cloud/_gen/types.ts @@ -9,6 +9,11 @@ // Add a new path/method to TARGET_ENDPOINTS in // scripts/generate_hyperframes_cli_client.py to extend this list. +export interface AbortAssetUploadResponse { + asset_id: string; + status: "aborted"; +} + export interface AssetBase64 { type: "base64"; media_type: string; @@ -78,6 +83,8 @@ export type CinematicAvatarModel = "seedance_2" | "heygen_video_1"; export interface CompleteAssetUploadRequest { checksum_sha256?: string | null; + multipart_upload_id?: string | null; + parts?: Array | null; } export interface CompleteAssetUploadResponse { @@ -88,11 +95,17 @@ export interface CompleteAssetUploadResponse { status: "processing"; } +export interface CompletedPart { + part_number: number; + etag: string; +} + export interface CreateAssetUploadRequest { filename: string; content_type: string; size_bytes: number; checksum_sha256?: string | null; + multipart?: boolean; } export interface CreateAssetUploadResponse { @@ -102,6 +115,7 @@ export interface CreateAssetUploadResponse { expires_in_seconds: number; max_bytes: number; status: "pending_upload"; + multipart?: MultipartUploadPlan | null; } export interface CreateBackgroundRemovalRequest { @@ -358,6 +372,22 @@ export interface GptImage25Request { reference_images?: Array; } +export interface GptImage26FlareRequest { + model: "gpt-image-2.6-flare"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null; + reference_images?: Array; +} + +export interface GptImage26Request { + model: "gpt-image-2.6"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null; + reference_images?: Array; +} + export interface HeyGenVoiceEngineSettings { engine_type: "heygen_voice" | "orca"; } @@ -394,7 +424,12 @@ export type HyperframesRenderStatus = "queued" | "rendering" | "completed" | "fa export type HyperframesResolution = "1080p" | "4k"; -export type ImageCreateRequest = GptImage25Request | NanoBanana2Request | NanoBanana21Request; +export type ImageCreateRequest = + | GptImage25Request + | GptImage26Request + | GptImage26FlareRequest + | NanoBanana2Request + | NanoBanana21Request; export interface ImageScene { type: "image"; @@ -440,6 +475,10 @@ export interface LipsyncDetail { failure_code?: string | null; } +export interface ListUploadedPartsResponse { + parts: Array; +} + export interface LyriaMusicRequest { model: "lyria-3-pro"; prompt: string; @@ -450,6 +489,16 @@ export interface LyriaMusicRequest { lyrics?: string | null; } +export interface MintPartUrlsRequest { + multipart_upload_id: string; + part_numbers: Array; +} + +export interface MintPartUrlsResponse { + parts: Array; + expires_in_seconds: number; +} + export interface ModelVideoResponse { video_id: string; status: "pending" | "processing" | "completed" | "failed" | "cancelled"; @@ -471,6 +520,12 @@ export interface ModelVideoTimings { inference: number; } +export interface MultipartUploadPlan { + multipart_upload_id: string; + part_size_bytes: number; + part_count: number; +} + export type MusicCreateRequest = LyriaMusicRequest; export type MusicFailureCode = @@ -520,6 +575,11 @@ export interface OutputUploadRequest { thumbnail?: UploadTargetRequest | null; } +export interface PartUrl { + part_number: number; + upload_url: string; +} + export interface PlaybackInput { mode?: "freeze" | "loop" | "fit_to_scene" | null; volume?: number; @@ -702,6 +762,12 @@ export interface UploadTargetRequest { headers?: Record; } +export interface UploadedPart { + part_number: number; + size_bytes: number; + etag: string; +} + export type VideoAspectRatio = "16:9" | "9:16" | "4:5" | "5:4" | "1:1" | "auto"; export interface VideoDetail { diff --git a/packages/cli/src/cloud/upload.ts b/packages/cli/src/cloud/upload.ts index 55469c7ebd..3f104e7871 100644 --- a/packages/cli/src/cloud/upload.ts +++ b/packages/cli/src/cloud/upload.ts @@ -26,6 +26,11 @@ import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import type { HyperframesCloudClient } from "./_gen/client.js"; +import type { + CompleteAssetUploadRequest, + CreateAssetUploadResponse, + MultipartUploadPlan, +} from "./_gen/types.js"; import { HyperframesApiError } from "./_gen/client.js"; import { PutRefusedError, @@ -36,6 +41,14 @@ import { type PutTry, type UploadFileSource, } from "./uploadPut.js"; +import { + forgetSaved, + PARTS_ABOVE_BYTES, + resumable, + saveUpload, + savedUploadFile, + uploadParts, +} from "./uploadParts.js"; export { putTimings, type UploadFileSource }; @@ -137,7 +150,7 @@ async function putBytesToPresignedUrl( ...normalizeUploadHeaders(uploadHeaders), }; return putWithRetry( - (tried) => { + async (tried) => { if (tried.attempt > 1) onProgress?.({ phase: "upload", @@ -146,12 +159,12 @@ async function putBytesToPresignedUrl( totalBytes: total, attempt: tried.attempt, }); - return fetchImpl && body instanceof Uint8Array + await (fetchImpl && body instanceof Uint8Array ? putBytesOnce(fetchImpl, uploadUrl, headers, body, signal) : putStreamToPresignedUrl(uploadUrl, headers, body, signal, (written) => { tried.sent = written; report(tried); - }); + })); }, total, signal, @@ -208,13 +221,14 @@ export async function completeWithRetry( checksum_sha256: string, sizeBytes: number, signal?: AbortSignal, + inParts: CompleteParts = {}, ): Promise<{ asset_id: string; url: string }> { const budget = completeBudgetMs(sizeBytes); const startedAt = Date.now(); let poll = completeWait.firstPollMs; for (;;) { try { - return await client.completeAssetUpload({ asset_id, body: { checksum_sha256 } }); + return await client.completeAssetUpload({ asset_id, body: { checksum_sha256, ...inParts } }); } catch (err) { if (!(err instanceof HyperframesApiError && err.status === 409)) throw err; const left = startedAt + budget - Date.now(); @@ -237,6 +251,17 @@ export async function completeWithRetry( // complete answers invalid_parameter for type, size, checksum and expiry; only type refusals blame the type. const TYPE_REFUSAL = /^(Stored file type|content_type not allowed|Stored file does not have a ZIP)/; +const STATUSES_NOT_ABOUT_THE_FILE = new Set([401, 403, 408, 409, 429]); + +function refusesTheFile(err: unknown): boolean { + return ( + err instanceof HyperframesApiError && + err.status >= 400 && + err.status < 500 && + !STATUSES_NOT_ABOUT_THE_FILE.has(err.status) + ); +} + function nameRefusedFile(err: unknown, filename: string, contentType: string): unknown { if (!(err instanceof HyperframesApiError) || err.code !== "invalid_parameter") return err; const kind = contentType.split("/")[0]; @@ -247,48 +272,131 @@ function nameRefusedFile(err: unknown, filename: string, contentType: string): u return new HyperframesApiError({ ...err, message: `${lead} ${err.message}` }); } -export async function uploadAssetViaDirectUpload( - opts: Omit & { contentType: string } & ( - | { bytes: Uint8Array } - | UploadFileSource - ), -): Promise { - const start = Date.now(); - const { filename, idempotencyKey } = opts; - const body = "bytes" in opts ? opts.bytes : { path: opts.path, size: opts.size }; - const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size; - const checksum_sha256 = - body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path); - const fetchImpl = opts.fetchImpl; +type CompleteParts = Pick; - opts.onProgress?.({ phase: "initialize" }); - const initialize = await opts.client.createAssetUpload({ - body: { filename, content_type: opts.contentType, size_bytes, checksum_sha256 }, - idempotencyKey, +/** Asks for parts on a big file; a server that does not know them answers with the single PUT. */ +async function createUpload( + client: HyperframesCloudClient, + o: { + filename: string; + contentType: string; + size_bytes: number; + checksum_sha256: string; + inParts: boolean; + idempotencyKey?: string; + }, +): Promise { + const body = { + filename: o.filename, + content_type: o.contentType, + size_bytes: o.size_bytes, + checksum_sha256: o.checksum_sha256, + }; + try { + return await client.createAssetUpload({ + body: o.inParts ? { ...body, multipart: true } : body, + idempotencyKey: o.idempotencyKey, + }); + } catch (err) { + const unknownField = + err instanceof HyperframesApiError && (err.status === 400 || err.status === 422); + if (!o.inParts || !unknownField) throw err; + return await client.createAssetUpload({ body, idempotencyKey: o.idempotencyKey }); + } +} + +type DirectUploadOptions = Omit & { contentType: string }; + +/** Sends the file (in parts above PARTS_ABOVE_BYTES, resuming a saved upload) and says what `complete` must be told. */ +async function sendInParts( + opts: DirectUploadOptions, + body: Uint8Array | UploadFileSource, + size_bytes: number, + asset_id: string, + plan: MultipartUploadPlan, + saved: string | undefined, +): Promise<{ asset_id: string; inParts: CompleteParts; forget?: () => void }> { + if (saved) saveUpload(saved, { asset_id, plan }); + const parts = await uploadParts({ + client: opts.client, + asset_id, + plan, + body, + total: size_bytes, + onProgress: opts.onProgress, + signal: opts.signal, + }); + return { + asset_id, + inParts: { multipart_upload_id: plan.multipart_upload_id, parts }, + forget: saved ? () => forgetSaved(saved) : undefined, + }; +} + +async function sendFile( + opts: DirectUploadOptions, + body: Uint8Array | UploadFileSource, + size_bytes: number, + checksum_sha256: string, +): Promise<{ asset_id: string; inParts: CompleteParts; forget?: () => void }> { + const wantParts = size_bytes > PARTS_ABOVE_BYTES; + const saved = wantParts ? savedUploadFile(checksum_sha256, size_bytes) : undefined; + const resumed = saved ? await resumable(opts.client, saved) : undefined; + if (resumed) { + opts.onProgress?.({ phase: "upload", percent: 0 }); + return sendInParts(opts, body, size_bytes, resumed.asset_id, resumed.plan, saved); + } + const created = await createUpload(opts.client, { + filename: opts.filename, + contentType: opts.contentType, + size_bytes, + checksum_sha256, + inParts: wantParts, + idempotencyKey: opts.idempotencyKey, }); opts.onProgress?.({ phase: "upload", percent: 0 }); + if (created.multipart) { + return sendInParts(opts, body, size_bytes, created.asset_id, created.multipart, saved); + } await putBytesToPresignedUrl( - fetchImpl, - initialize.upload_url, - initialize.upload_headers, + opts.fetchImpl, + created.upload_url, + created.upload_headers, body, opts.contentType, opts.signal, opts.onProgress, ); + return { asset_id: created.asset_id, inParts: {} }; +} + +export async function uploadAssetViaDirectUpload( + opts: DirectUploadOptions & ({ bytes: Uint8Array } | UploadFileSource), +): Promise { + const start = Date.now(); + const body = "bytes" in opts ? opts.bytes : { path: opts.path, size: opts.size }; + const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size; + const checksum_sha256 = + body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path); + + opts.onProgress?.({ phase: "initialize" }); + const sent = await sendFile(opts, body, size_bytes, checksum_sha256); opts.onProgress?.({ phase: "upload", percent: 100 }); opts.onProgress?.({ phase: "complete" }); const completed = await completeWithRetry( opts.client, - initialize.asset_id, + sent.asset_id, checksum_sha256, size_bytes, opts.signal, + sent.inParts, ).catch((err: unknown) => { - throw nameRefusedFile(err, filename, opts.contentType); + if (refusesTheFile(err)) sent.forget?.(); + throw nameRefusedFile(err, opts.filename, opts.contentType); }); + sent.forget?.(); return { asset_id: completed.asset_id, url: completed.url, diff --git a/packages/cli/src/cloud/uploadParts.test.ts b/packages/cli/src/cloud/uploadParts.test.ts new file mode 100644 index 0000000000..e5513d3139 --- /dev/null +++ b/packages/cli/src/cloud/uploadParts.test.ts @@ -0,0 +1,282 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { HyperframesApiError } from "./_gen/client.js"; +import type { HyperframesCloudClient } from "./_gen/client.js"; +import { putTimings, uploadAssetViaDirectUpload, type UploadProgressEvent } from "./upload.js"; + +const MIB = 1024 * 1024; +const PART = 4 * MIB; +const RATE_BYTES_PER_MS = 8 * 1024; + +let dir: string; +let server: Server; +let base: string; +let received: Map; +let puts: number[]; +let cutAfterMs: number | undefined; +let cutOnce: Set; +const saved = { ...putTimings }; + +beforeEach(async () => { + dir = mkdtempSync(join(tmpdir(), "hf-parts-")); + process.env["HYPERFRAMES_CONFIG_DIR"] = join(dir, "config"); + Object.assign(putTimings, { stallMs: 5000, backoffMs: 5, progressMs: 50, sampleMs: 50 }); + received = new Map(); + puts = []; + cutAfterMs = undefined; + cutOnce = new Set(); + server = createServer((req, res) => { + const part = Number(req.url?.split("/").pop()) || 0; + puts.push(part); + const cutAt = cutOnce.delete(part) ? 100 : cutAfterMs; + let size = 0; + const startedAt = Date.now(); + req.on("error", () => undefined); + req.on("data", (chunk: Buffer) => { + size += chunk.length; + req.pause(); + setTimeout( + () => { + if (cutAt !== undefined && Date.now() - startedAt >= cutAt) req.socket.destroy(); + else req.resume(); + }, + Math.ceil(chunk.length / RATE_BYTES_PER_MS), + ); + }); + req.on("end", () => { + received.set(part, size); + res.writeHead(200, { etag: `"e${part}"` }).end(); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +afterEach(() => { + Object.assign(putTimings, saved); + delete process.env["HYPERFRAMES_CONFIG_DIR"]; + server.closeAllConnections(); + server.close(); + rmSync(dir, { recursive: true, force: true }); +}); + +function bigFile(bytes: number): { path: string; size: number } { + const path = join(dir, "big.mp4"); + writeFileSync(path, Buffer.alloc(bytes, 7)); + return { path, size: bytes }; +} + +function cloud( + over: { multipart?: boolean; held?: { part_number: number; size_bytes: number }[] } = {}, +) { + const complete = vi.fn(async (_args: unknown) => ({ + asset_id: "a1", + url: "u", + mime_type: "video/mp4", + size_bytes: 1, + status: "processing" as const, + })); + const client = { + createAssetUpload: vi.fn(async (_args?: unknown) => ({ + asset_id: "a1", + upload_url: `${base}/single`, + upload_headers: {}, + expires_in_seconds: 60, + max_bytes: 1, + status: "pending_upload" as const, + ...(over.multipart === false + ? {} + : { multipart: { multipart_upload_id: "mp1", part_size_bytes: PART, part_count: 5 } }), + })), + getAssetUploadPartUrls: vi.fn(async (args: { body: { part_numbers: number[] } }) => ({ + expires_in_seconds: 60, + parts: args.body.part_numbers.map((n) => ({ + part_number: n, + upload_url: `${base}/part/${n}`, + })), + })), + listAssetUploadParts: vi.fn(async () => ({ + parts: (over.held ?? []).map((p) => ({ ...p, etag: `"e${p.part_number}"` })), + })), + completeAssetUpload: complete, + }; + return { client: client as unknown as HyperframesCloudClient, mocks: client, complete }; +} + +const FILE_BYTES = 17 * MIB; + +describe("a big file goes up in parts", () => { + it("delivers a file a link cannot finish before the cut, because each part finishes alone", async () => { + cutAfterMs = 1200; + const { client, complete } = cloud(); + + await uploadAssetViaDirectUpload({ + client, + ...bigFile(FILE_BYTES), + filename: "big.mp4", + contentType: "video/mp4", + }); + + expect([...received.keys()].sort()).toEqual([1, 2, 3, 4, 5]); + const body = ( + complete.mock.calls[0]![0] as { body: { multipart_upload_id: string; parts: unknown[] } } + ).body; + expect(body.multipart_upload_id).toBe("mp1"); + expect(body.parts).toEqual([1, 2, 3, 4, 5].map((n) => ({ part_number: n, etag: `"e${n}"` }))); + }, 30_000); + + it("the same link cannot deliver the file in one PUT", async () => { + cutAfterMs = 1200; + const { client } = cloud({ multipart: false }); + + await expect( + uploadAssetViaDirectUpload({ + client, + ...bigFile(FILE_BYTES), + filename: "big.mp4", + contentType: "video/mp4", + }), + ).rejects.toThrow(/cut at .* on two tries in a row/); + }, 30_000); + + it("sends again only the part that was cut, from its own start", async () => { + cutOnce.add(3); + const { client } = cloud(); + + await uploadAssetViaDirectUpload({ + client, + ...bigFile(FILE_BYTES), + filename: "big.mp4", + contentType: "video/mp4", + }); + + expect(puts.filter((n) => n === 3)).toHaveLength(2); + expect(puts.filter((n) => n !== 3)).toHaveLength(4); + expect(received.get(3)).toBe(PART); + }, 30_000); + + it("resumes after a restart: parts the cloud holds are not sent again", async () => { + const held = [ + { part_number: 1, size_bytes: PART }, + { part_number: 2, size_bytes: PART }, + ]; + const first = cloud({ held }); + first.complete.mockRejectedValue(new Error("the CLI was closed before complete")); + const file = bigFile(FILE_BYTES); + await expect( + uploadAssetViaDirectUpload({ + client: first.client, + ...file, + filename: "big.mp4", + contentType: "video/mp4", + }), + ).rejects.toThrow(); + puts.length = 0; + received.clear(); + const second = cloud({ held }); + + await uploadAssetViaDirectUpload({ + client: second.client, + ...file, + filename: "big.mp4", + contentType: "video/mp4", + }); + + expect([...puts].sort()).toEqual([3, 4, 5]); + expect(second.mocks.createAssetUpload).not.toHaveBeenCalled(); + }, 30_000); + + it("forgets the saved plan when the cloud refuses the file, and keeps it when complete only failed", async () => { + const file = bigFile(FILE_BYTES); + const upload = (client: HyperframesCloudClient) => + uploadAssetViaDirectUpload({ + client, + ...file, + filename: "big.mp4", + contentType: "video/mp4", + }); + + const refused = cloud(); + refused.complete.mockRejectedValue( + new HyperframesApiError({ status: 400, code: "invalid_parameter", message: "bad type" }), + ); + await expect(upload(refused.client)).rejects.toThrow(/bad type/); + const afterRefusal = cloud(); + await upload(afterRefusal.client); + expect(afterRefusal.mocks.createAssetUpload).toHaveBeenCalledTimes(1); + + const failed = cloud(); + failed.complete.mockRejectedValue( + new HyperframesApiError({ status: 503, message: "try again later" }), + ); + await expect(upload(failed.client)).rejects.toThrow(/try again later/); + const afterFailure = cloud(); + await upload(afterFailure.client); + expect(afterFailure.mocks.createAssetUpload).not.toHaveBeenCalled(); + }, 60_000); + + it("reports bytes from the parts done plus the live ones", async () => { + const events: UploadProgressEvent[] = []; + const { client } = cloud(); + + await uploadAssetViaDirectUpload({ + client, + ...bigFile(FILE_BYTES), + filename: "big.mp4", + contentType: "video/mp4", + onProgress: (e) => events.push(e), + }); + + const sent = events.flatMap((e) => + e.phase === "upload" && e.sentBytes !== undefined ? [e.sentBytes] : [], + ); + expect(sent.length).toBeGreaterThan(2); + expect(Math.max(...sent)).toBeGreaterThan(PART); + }, 30_000); +}); + +describe("files and servers that keep the single PUT", () => { + it("a small file never asks for parts", async () => { + const { client, mocks } = cloud({ multipart: false }); + await uploadAssetViaDirectUpload({ + client, + ...bigFile(2 * MIB), + filename: "s.mp4", + contentType: "video/mp4", + }); + expect((mocks.createAssetUpload.mock.calls[0]![0] as { body: object }).body).not.toHaveProperty( + "multipart", + ); + expect(puts).toEqual([0]); + }); + + it("a server that answers without a plan gets the single PUT", async () => { + const { client, mocks } = cloud({ multipart: false }); + await uploadAssetViaDirectUpload({ + client, + ...bigFile(FILE_BYTES), + filename: "b.mp4", + contentType: "video/mp4", + }); + expect(mocks.getAssetUploadPartUrls).not.toHaveBeenCalled(); + expect(received.get(0)).toBe(FILE_BYTES); + }, 30_000); + + it("a server that refuses the unknown field is asked again without it", async () => { + const { client, mocks } = cloud({ multipart: false }); + mocks.createAssetUpload.mockRejectedValueOnce( + new HyperframesApiError({ status: 422, code: "invalid_parameter", message: "extra field" }), + ); + await uploadAssetViaDirectUpload({ + client, + ...bigFile(FILE_BYTES), + filename: "b.mp4", + contentType: "video/mp4", + }); + expect(mocks.createAssetUpload).toHaveBeenCalledTimes(2); + }, 30_000); +}); diff --git a/packages/cli/src/cloud/uploadParts.ts b/packages/cli/src/cloud/uploadParts.ts new file mode 100644 index 0000000000..641d1a2e88 --- /dev/null +++ b/packages/cli/src/cloud/uploadParts.ts @@ -0,0 +1,183 @@ +/** Sends a file to the cloud in parts, so a cut connection costs one part and a restart sends only the rest. */ + +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { hyperframesConfigDir } from "../utils/configDir.js"; +import type { HyperframesCloudClient } from "./_gen/client.js"; +import type { CompletedPart, MultipartUploadPlan } from "./_gen/types.js"; +import { + putStreamToPresignedUrl, + putTimings, + putWithRetry, + type PutTry, + type UploadFileSource, +} from "./uploadPut.js"; +import type { UploadProgressEvent } from "./upload.js"; + +/** Files above this go in parts; the server fixes the part size. */ +export const PARTS_ABOVE_BYTES = 16 * 1024 * 1024; +const PARTS_IN_FLIGHT = 3; + +/** The upload a restart picks up again: it is found by the file's content, so a changed file starts over. */ +interface SavedUpload { + asset_id: string; + plan: MultipartUploadPlan; +} + +export function savedUploadFile(sha256: string, size: number): string { + return hyperframesConfigDir("uploads", `${sha256.slice(0, 32)}-${size}.json`); +} + +function readSaved(file: string): SavedUpload | undefined { + try { + return JSON.parse(readFileSync(file, "utf8")) as SavedUpload; + } catch { + return undefined; + } +} + +export function forgetSaved(file: string): void { + rmSync(file, { force: true }); +} + +export function saveUpload(file: string, saved: SavedUpload): void { + try { + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + writeFileSync(file, JSON.stringify(saved), { mode: 0o600 }); + } catch { + // expected: a read-only config folder only costs the resume after a restart + } +} + +/** The upload saved for this content that the cloud still holds, or nothing. */ +export async function resumable( + client: HyperframesCloudClient, + file: string, +): Promise { + const saved = readSaved(file); + if (!saved) return undefined; + try { + await client.listAssetUploadParts({ + asset_id: saved.asset_id, + multipart_upload_id: saved.plan.multipart_upload_id, + }); + return saved; + } catch { + forgetSaved(file); + return undefined; + } +} + +function partRange(plan: MultipartUploadPlan, total: number, number: number) { + const start = (number - 1) * plan.part_size_bytes; + return { start, length: Math.min(plan.part_size_bytes, total - start) }; +} + +export interface PartsUpload { + client: HyperframesCloudClient; + asset_id: string; + plan: MultipartUploadPlan; + body: Uint8Array | UploadFileSource; + total: number; + onProgress?: (event: UploadProgressEvent) => void; + signal?: AbortSignal; +} + +/** Sends every part the cloud does not hold yet, a few at a time, each part retried alone from its own start. */ +export async function uploadParts(job: PartsUpload): Promise { + const { client, asset_id, plan, total } = job; + const id = plan.multipart_upload_id; + const held = await client.listAssetUploadParts({ asset_id, multipart_upload_id: id }); + const done = new Map(); + for (const part of held.parts) { + const range = partRange(plan, total, part.part_number); + if (part.size_bytes === range.length) done.set(part.part_number, part.etag); + } + const progress = partsProgress( + job, + [...done.keys()].reduce((sum, n) => sum + partRange(plan, total, n).length, 0), + ); + progress.report(1); + + const todo = Array.from({ length: plan.part_count }, (_, i) => i + 1).filter((n) => !done.has(n)); + const failure: { error?: unknown } = {}; + const worker = async () => { + for (let number = todo.shift(); number !== undefined; number = todo.shift()) { + if (failure.error) return; + try { + done.set(number, await sendPart(job, number, progress)); + progress.finish(number, partRange(plan, total, number).length); + } catch (err) { + failure.error ??= err; + return; + } + } + }; + await Promise.all(Array.from({ length: Math.min(PARTS_IN_FLIGHT, todo.length) }, worker)); + if (failure.error) throw failure.error; + return [...done.entries()] + .sort((a, b) => a[0] - b[0]) + .map(([part_number, etag]) => ({ part_number, etag })); +} + +/** Bytes sent so far: the parts done plus what each part in flight has written, reported at most every progressMs + * or 5 points. */ +function partsProgress(job: PartsUpload, finishedBytes: number) { + const live = new Map(); + let finished = finishedBytes; + let lastAt = Date.now(); + let lastPercent = -5; + const report = (attempt: number) => { + const sent = Math.min(job.total, finished + [...live.values()].reduce((a, b) => a + b, 0)); + const percent = job.total > 0 ? Math.min(99, Math.floor((100 * sent) / job.total)) : 0; + const now = Date.now(); + if (now - lastAt < putTimings.progressMs && percent < lastPercent + 5) return; + [lastAt, lastPercent] = [now, percent]; + job.onProgress?.({ phase: "upload", percent, sentBytes: sent, totalBytes: job.total, attempt }); + }; + return { + report, + written: (number: number, bytes: number, attempt: number) => ( + live.set(number, bytes), report(attempt) + ), + finish: (number: number, bytes: number) => ( + (finished += bytes), live.delete(number), report(1) + ), + }; +} + +async function sendPart( + job: PartsUpload, + number: number, + progress: ReturnType, +): Promise { + const { client, asset_id, plan, body, total, signal } = job; + const range = partRange(plan, total, number); + return putWithRetry( + async (tried: PutTry) => { + const minted = await client.getAssetUploadPartUrls({ + asset_id, + body: { multipart_upload_id: plan.multipart_upload_id, part_numbers: [number] }, + signal, + }); + const url = minted.parts.find((part) => part.part_number === number)?.upload_url; + if (!url) throw new Error(`The cloud gave no address for part ${number}`); + progress.written(number, 0, tried.attempt); + const etag = await putStreamToPresignedUrl( + url, + {}, + body, + signal, + (written) => { + tried.sent = written; + progress.written(number, Math.min(written, range.length), tried.attempt); + }, + range, + ); + if (!etag) throw new Error(`The cloud gave no ETag for part ${number}`); + return etag; + }, + range.length, + signal, + ); +} diff --git a/packages/cli/src/cloud/uploadPut.ts b/packages/cli/src/cloud/uploadPut.ts index d3566fb284..9ee67bd2d8 100644 --- a/packages/cli/src/cloud/uploadPut.ts +++ b/packages/cli/src/cloud/uploadPut.ts @@ -125,12 +125,23 @@ export function sleep(ms: number, signal?: AbortSignal): Promise { // same way, in slices, so a stalled link ends its try the same way (a plain fetch has no progress to watch). const SLICE_BYTES = 64 * 1024; -function bodyStream(body: Uint8Array | UploadFileSource): Readable { - if (!(body instanceof Uint8Array)) return createReadStream(body.path); +export interface ByteRange { + start: number; + length: number; +} + +function bodyStream(body: Uint8Array | UploadFileSource, range?: ByteRange): Readable { + if (!(body instanceof Uint8Array)) + return createReadStream( + body.path, + range && { start: range.start, end: range.start + range.length - 1 }, + ); + const from = range?.start ?? 0; + const to = from + (range?.length ?? body.byteLength); return Readable.from( (function* () { - for (let at = 0; at < body.byteLength; at += SLICE_BYTES) - yield body.subarray(at, at + SLICE_BYTES); + for (let at = from; at < to; at += SLICE_BYTES) + yield body.subarray(at, Math.min(at + SLICE_BYTES, to)); })(), ); } @@ -159,8 +170,9 @@ export async function putStreamToPresignedUrl( body: Uint8Array | UploadFileSource, signal: AbortSignal | undefined, onSample: (written: number) => void, -): Promise { - const size = body instanceof Uint8Array ? body.byteLength : body.size; + range?: ByteRange, +): Promise { + const size = range?.length ?? (body instanceof Uint8Array ? body.byteLength : body.size); const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest; const req = send(uploadUrl, { method: "PUT", @@ -171,7 +183,7 @@ export async function putStreamToPresignedUrl( req.once("response", resolve).once("error", reject); }); const stopWatching = watchStall(req, onSample); - const source = bodyStream(body); + const source = bodyStream(body, range); const sent = pipeline(source, req); let res: IncomingMessage; try { @@ -183,6 +195,7 @@ export async function putStreamToPresignedUrl( stopWatching(); } await settle(res, sent); + return String(res.headers.etag ?? "") || undefined; } /** Reads S3's answer: a refusal is thrown with its words; a refused upload can close the connection mid-body, so the