feat(cli): cloud uploads send a big file in parts, so a cut connection resends one part (#7375)

* refactor(cli): the cloud upload's PUT and retry code moves to its own file

* feat(cli): cloud uploads send a big file in parts, so a cut connection resends one part

* test(cli): the cutting link fails a single PUT of the same file

* fix(cli): a file the cloud refuses forgets its saved parts plan

* fix(cli): a resumed upload goes straight to its parts, so a saved plan never feeds the single PUT
This commit is contained in:
Miguel Ángel
2026-11-07 21:14:19 +00:00
committed by GitHub
parent f5bde9533f
commit 3d7f12acd7
6 changed files with 735 additions and 34 deletions
+49
View File
@@ -5,6 +5,7 @@
* experiment-framework to regenerate.
*/
import type {
AbortAssetUploadResponse,
BackgroundRemovalJob,
CompleteAssetUploadRequest,
CompleteAssetUploadResponse,
@@ -25,6 +26,9 @@ import type {
ImageCreateRequest,
LipsyncCreateResponse,
LipsyncDetail,
ListUploadedPartsResponse,
MintPartUrlsRequest,
MintPartUrlsResponse,
ModelVideoResponse,
MusicCreateRequest,
MusicResponse,
@@ -271,6 +275,51 @@ export class HyperframesCloudClient {
});
}
async getAssetUploadPartUrls(args: {
asset_id: string;
body: MintPartUrlsRequest;
signal?: AbortSignal;
}): Promise<MintPartUrlsResponse> {
return await this.request<MintPartUrlsResponse>({
method: "POST",
path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`,
body: args.body,
signal: args.signal,
});
}
async listAssetUploadParts(args: {
asset_id: string;
multipart_upload_id: string;
signal?: AbortSignal;
}): Promise<ListUploadedPartsResponse> {
const query: Record<string, string | number | undefined> = {
multipart_upload_id: args.multipart_upload_id,
};
return await this.request<ListUploadedPartsResponse>({
method: "GET",
path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`,
query,
signal: args.signal,
});
}
async abortAssetUploadParts(args: {
asset_id: string;
multipart_upload_id: string;
signal?: AbortSignal;
}): Promise<AbortAssetUploadResponse> {
const query: Record<string, string | number | undefined> = {
multipart_upload_id: args.multipart_upload_id,
};
return await this.request<AbortAssetUploadResponse>({
method: "DELETE",
path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`,
query,
signal: args.signal,
});
}
async createRender(args: {
body: CreateHyperframesRenderRequest;
idempotencyKey?: string;
+67 -1
View File
@@ -9,6 +9,11 @@
// Add a new path/method to TARGET_ENDPOINTS in
// scripts/generate_hyperframes_cli_client.py to extend this list.
export interface AbortAssetUploadResponse {
asset_id: string;
status: "aborted";
}
export interface AssetBase64 {
type: "base64";
media_type: string;
@@ -78,6 +83,8 @@ export type CinematicAvatarModel = "seedance_2" | "heygen_video_1";
export interface CompleteAssetUploadRequest {
checksum_sha256?: string | null;
multipart_upload_id?: string | null;
parts?: Array<CompletedPart> | null;
}
export interface CompleteAssetUploadResponse {
@@ -88,11 +95,17 @@ export interface CompleteAssetUploadResponse {
status: "processing";
}
export interface CompletedPart {
part_number: number;
etag: string;
}
export interface CreateAssetUploadRequest {
filename: string;
content_type: string;
size_bytes: number;
checksum_sha256?: string | null;
multipart?: boolean;
}
export interface CreateAssetUploadResponse {
@@ -102,6 +115,7 @@ export interface CreateAssetUploadResponse {
expires_in_seconds: number;
max_bytes: number;
status: "pending_upload";
multipart?: MultipartUploadPlan | null;
}
export interface CreateBackgroundRemovalRequest {
@@ -358,6 +372,22 @@ export interface GptImage25Request {
reference_images?: Array<AssetUrl | AssetId | AssetBase64>;
}
export interface GptImage26FlareRequest {
model: "gpt-image-2.6-flare";
prompt: string;
num_images?: number;
aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null;
reference_images?: Array<AssetUrl | AssetId | AssetBase64>;
}
export interface GptImage26Request {
model: "gpt-image-2.6";
prompt: string;
num_images?: number;
aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null;
reference_images?: Array<AssetUrl | AssetId | AssetBase64>;
}
export interface HeyGenVoiceEngineSettings {
engine_type: "heygen_voice" | "orca";
}
@@ -394,7 +424,12 @@ export type HyperframesRenderStatus = "queued" | "rendering" | "completed" | "fa
export type HyperframesResolution = "1080p" | "4k";
export type ImageCreateRequest = GptImage25Request | NanoBanana2Request | NanoBanana21Request;
export type ImageCreateRequest =
| GptImage25Request
| GptImage26Request
| GptImage26FlareRequest
| NanoBanana2Request
| NanoBanana21Request;
export interface ImageScene {
type: "image";
@@ -440,6 +475,10 @@ export interface LipsyncDetail {
failure_code?: string | null;
}
export interface ListUploadedPartsResponse {
parts: Array<UploadedPart>;
}
export interface LyriaMusicRequest {
model: "lyria-3-pro";
prompt: string;
@@ -450,6 +489,16 @@ export interface LyriaMusicRequest {
lyrics?: string | null;
}
export interface MintPartUrlsRequest {
multipart_upload_id: string;
part_numbers: Array<number>;
}
export interface MintPartUrlsResponse {
parts: Array<PartUrl>;
expires_in_seconds: number;
}
export interface ModelVideoResponse {
video_id: string;
status: "pending" | "processing" | "completed" | "failed" | "cancelled";
@@ -471,6 +520,12 @@ export interface ModelVideoTimings {
inference: number;
}
export interface MultipartUploadPlan {
multipart_upload_id: string;
part_size_bytes: number;
part_count: number;
}
export type MusicCreateRequest = LyriaMusicRequest;
export type MusicFailureCode =
@@ -520,6 +575,11 @@ export interface OutputUploadRequest {
thumbnail?: UploadTargetRequest | null;
}
export interface PartUrl {
part_number: number;
upload_url: string;
}
export interface PlaybackInput {
mode?: "freeze" | "loop" | "fit_to_scene" | null;
volume?: number;
@@ -702,6 +762,12 @@ export interface UploadTargetRequest {
headers?: Record<string, unknown>;
}
export interface UploadedPart {
part_number: number;
size_bytes: number;
etag: string;
}
export type VideoAspectRatio = "16:9" | "9:16" | "4:5" | "5:4" | "1:1" | "auto";
export interface VideoDetail {
+134 -26
View File
@@ -26,6 +26,11 @@
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import type { HyperframesCloudClient } from "./_gen/client.js";
import type {
CompleteAssetUploadRequest,
CreateAssetUploadResponse,
MultipartUploadPlan,
} from "./_gen/types.js";
import { HyperframesApiError } from "./_gen/client.js";
import {
PutRefusedError,
@@ -36,6 +41,14 @@ import {
type PutTry,
type UploadFileSource,
} from "./uploadPut.js";
import {
forgetSaved,
PARTS_ABOVE_BYTES,
resumable,
saveUpload,
savedUploadFile,
uploadParts,
} from "./uploadParts.js";
export { putTimings, type UploadFileSource };
@@ -137,7 +150,7 @@ async function putBytesToPresignedUrl(
...normalizeUploadHeaders(uploadHeaders),
};
return putWithRetry(
(tried) => {
async (tried) => {
if (tried.attempt > 1)
onProgress?.({
phase: "upload",
@@ -146,12 +159,12 @@ async function putBytesToPresignedUrl(
totalBytes: total,
attempt: tried.attempt,
});
return fetchImpl && body instanceof Uint8Array
await (fetchImpl && body instanceof Uint8Array
? putBytesOnce(fetchImpl, uploadUrl, headers, body, signal)
: putStreamToPresignedUrl(uploadUrl, headers, body, signal, (written) => {
tried.sent = written;
report(tried);
});
}));
},
total,
signal,
@@ -208,13 +221,14 @@ export async function completeWithRetry(
checksum_sha256: string,
sizeBytes: number,
signal?: AbortSignal,
inParts: CompleteParts = {},
): Promise<{ asset_id: string; url: string }> {
const budget = completeBudgetMs(sizeBytes);
const startedAt = Date.now();
let poll = completeWait.firstPollMs;
for (;;) {
try {
return await client.completeAssetUpload({ asset_id, body: { checksum_sha256 } });
return await client.completeAssetUpload({ asset_id, body: { checksum_sha256, ...inParts } });
} catch (err) {
if (!(err instanceof HyperframesApiError && err.status === 409)) throw err;
const left = startedAt + budget - Date.now();
@@ -237,6 +251,17 @@ export async function completeWithRetry(
// complete answers invalid_parameter for type, size, checksum and expiry; only type refusals blame the type.
const TYPE_REFUSAL = /^(Stored file type|content_type not allowed|Stored file does not have a ZIP)/;
const STATUSES_NOT_ABOUT_THE_FILE = new Set([401, 403, 408, 409, 429]);
function refusesTheFile(err: unknown): boolean {
return (
err instanceof HyperframesApiError &&
err.status >= 400 &&
err.status < 500 &&
!STATUSES_NOT_ABOUT_THE_FILE.has(err.status)
);
}
function nameRefusedFile(err: unknown, filename: string, contentType: string): unknown {
if (!(err instanceof HyperframesApiError) || err.code !== "invalid_parameter") return err;
const kind = contentType.split("/")[0];
@@ -247,48 +272,131 @@ function nameRefusedFile(err: unknown, filename: string, contentType: string): u
return new HyperframesApiError({ ...err, message: `${lead} ${err.message}` });
}
export async function uploadAssetViaDirectUpload(
opts: Omit<UploadZipViaDirectOptions, "bytes"> & { contentType: string } & (
| { bytes: Uint8Array }
| UploadFileSource
),
): Promise<UploadZipViaDirectResult> {
const start = Date.now();
const { filename, idempotencyKey } = opts;
const body = "bytes" in opts ? opts.bytes : { path: opts.path, size: opts.size };
const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size;
const checksum_sha256 =
body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path);
const fetchImpl = opts.fetchImpl;
type CompleteParts = Pick<CompleteAssetUploadRequest, "multipart_upload_id" | "parts">;
opts.onProgress?.({ phase: "initialize" });
const initialize = await opts.client.createAssetUpload({
body: { filename, content_type: opts.contentType, size_bytes, checksum_sha256 },
idempotencyKey,
/** Asks for parts on a big file; a server that does not know them answers with the single PUT. */
async function createUpload(
client: HyperframesCloudClient,
o: {
filename: string;
contentType: string;
size_bytes: number;
checksum_sha256: string;
inParts: boolean;
idempotencyKey?: string;
},
): Promise<CreateAssetUploadResponse> {
const body = {
filename: o.filename,
content_type: o.contentType,
size_bytes: o.size_bytes,
checksum_sha256: o.checksum_sha256,
};
try {
return await client.createAssetUpload({
body: o.inParts ? { ...body, multipart: true } : body,
idempotencyKey: o.idempotencyKey,
});
} catch (err) {
const unknownField =
err instanceof HyperframesApiError && (err.status === 400 || err.status === 422);
if (!o.inParts || !unknownField) throw err;
return await client.createAssetUpload({ body, idempotencyKey: o.idempotencyKey });
}
}
type DirectUploadOptions = Omit<UploadZipViaDirectOptions, "bytes"> & { contentType: string };
/** Sends the file (in parts above PARTS_ABOVE_BYTES, resuming a saved upload) and says what `complete` must be told. */
async function sendInParts(
opts: DirectUploadOptions,
body: Uint8Array | UploadFileSource,
size_bytes: number,
asset_id: string,
plan: MultipartUploadPlan,
saved: string | undefined,
): Promise<{ asset_id: string; inParts: CompleteParts; forget?: () => void }> {
if (saved) saveUpload(saved, { asset_id, plan });
const parts = await uploadParts({
client: opts.client,
asset_id,
plan,
body,
total: size_bytes,
onProgress: opts.onProgress,
signal: opts.signal,
});
return {
asset_id,
inParts: { multipart_upload_id: plan.multipart_upload_id, parts },
forget: saved ? () => forgetSaved(saved) : undefined,
};
}
async function sendFile(
opts: DirectUploadOptions,
body: Uint8Array | UploadFileSource,
size_bytes: number,
checksum_sha256: string,
): Promise<{ asset_id: string; inParts: CompleteParts; forget?: () => void }> {
const wantParts = size_bytes > PARTS_ABOVE_BYTES;
const saved = wantParts ? savedUploadFile(checksum_sha256, size_bytes) : undefined;
const resumed = saved ? await resumable(opts.client, saved) : undefined;
if (resumed) {
opts.onProgress?.({ phase: "upload", percent: 0 });
return sendInParts(opts, body, size_bytes, resumed.asset_id, resumed.plan, saved);
}
const created = await createUpload(opts.client, {
filename: opts.filename,
contentType: opts.contentType,
size_bytes,
checksum_sha256,
inParts: wantParts,
idempotencyKey: opts.idempotencyKey,
});
opts.onProgress?.({ phase: "upload", percent: 0 });
if (created.multipart) {
return sendInParts(opts, body, size_bytes, created.asset_id, created.multipart, saved);
}
await putBytesToPresignedUrl(
fetchImpl,
initialize.upload_url,
initialize.upload_headers,
opts.fetchImpl,
created.upload_url,
created.upload_headers,
body,
opts.contentType,
opts.signal,
opts.onProgress,
);
return { asset_id: created.asset_id, inParts: {} };
}
export async function uploadAssetViaDirectUpload(
opts: DirectUploadOptions & ({ bytes: Uint8Array } | UploadFileSource),
): Promise<UploadZipViaDirectResult> {
const start = Date.now();
const body = "bytes" in opts ? opts.bytes : { path: opts.path, size: opts.size };
const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size;
const checksum_sha256 =
body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path);
opts.onProgress?.({ phase: "initialize" });
const sent = await sendFile(opts, body, size_bytes, checksum_sha256);
opts.onProgress?.({ phase: "upload", percent: 100 });
opts.onProgress?.({ phase: "complete" });
const completed = await completeWithRetry(
opts.client,
initialize.asset_id,
sent.asset_id,
checksum_sha256,
size_bytes,
opts.signal,
sent.inParts,
).catch((err: unknown) => {
throw nameRefusedFile(err, filename, opts.contentType);
if (refusesTheFile(err)) sent.forget?.();
throw nameRefusedFile(err, opts.filename, opts.contentType);
});
sent.forget?.();
return {
asset_id: completed.asset_id,
url: completed.url,
+282
View File
@@ -0,0 +1,282 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { HyperframesApiError } from "./_gen/client.js";
import type { HyperframesCloudClient } from "./_gen/client.js";
import { putTimings, uploadAssetViaDirectUpload, type UploadProgressEvent } from "./upload.js";
const MIB = 1024 * 1024;
const PART = 4 * MIB;
const RATE_BYTES_PER_MS = 8 * 1024;
let dir: string;
let server: Server;
let base: string;
let received: Map<number, number>;
let puts: number[];
let cutAfterMs: number | undefined;
let cutOnce: Set<number>;
const saved = { ...putTimings };
beforeEach(async () => {
dir = mkdtempSync(join(tmpdir(), "hf-parts-"));
process.env["HYPERFRAMES_CONFIG_DIR"] = join(dir, "config");
Object.assign(putTimings, { stallMs: 5000, backoffMs: 5, progressMs: 50, sampleMs: 50 });
received = new Map();
puts = [];
cutAfterMs = undefined;
cutOnce = new Set();
server = createServer((req, res) => {
const part = Number(req.url?.split("/").pop()) || 0;
puts.push(part);
const cutAt = cutOnce.delete(part) ? 100 : cutAfterMs;
let size = 0;
const startedAt = Date.now();
req.on("error", () => undefined);
req.on("data", (chunk: Buffer) => {
size += chunk.length;
req.pause();
setTimeout(
() => {
if (cutAt !== undefined && Date.now() - startedAt >= cutAt) req.socket.destroy();
else req.resume();
},
Math.ceil(chunk.length / RATE_BYTES_PER_MS),
);
});
req.on("end", () => {
received.set(part, size);
res.writeHead(200, { etag: `"e${part}"` }).end();
});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
afterEach(() => {
Object.assign(putTimings, saved);
delete process.env["HYPERFRAMES_CONFIG_DIR"];
server.closeAllConnections();
server.close();
rmSync(dir, { recursive: true, force: true });
});
function bigFile(bytes: number): { path: string; size: number } {
const path = join(dir, "big.mp4");
writeFileSync(path, Buffer.alloc(bytes, 7));
return { path, size: bytes };
}
function cloud(
over: { multipart?: boolean; held?: { part_number: number; size_bytes: number }[] } = {},
) {
const complete = vi.fn(async (_args: unknown) => ({
asset_id: "a1",
url: "u",
mime_type: "video/mp4",
size_bytes: 1,
status: "processing" as const,
}));
const client = {
createAssetUpload: vi.fn(async (_args?: unknown) => ({
asset_id: "a1",
upload_url: `${base}/single`,
upload_headers: {},
expires_in_seconds: 60,
max_bytes: 1,
status: "pending_upload" as const,
...(over.multipart === false
? {}
: { multipart: { multipart_upload_id: "mp1", part_size_bytes: PART, part_count: 5 } }),
})),
getAssetUploadPartUrls: vi.fn(async (args: { body: { part_numbers: number[] } }) => ({
expires_in_seconds: 60,
parts: args.body.part_numbers.map((n) => ({
part_number: n,
upload_url: `${base}/part/${n}`,
})),
})),
listAssetUploadParts: vi.fn(async () => ({
parts: (over.held ?? []).map((p) => ({ ...p, etag: `"e${p.part_number}"` })),
})),
completeAssetUpload: complete,
};
return { client: client as unknown as HyperframesCloudClient, mocks: client, complete };
}
const FILE_BYTES = 17 * MIB;
describe("a big file goes up in parts", () => {
it("delivers a file a link cannot finish before the cut, because each part finishes alone", async () => {
cutAfterMs = 1200;
const { client, complete } = cloud();
await uploadAssetViaDirectUpload({
client,
...bigFile(FILE_BYTES),
filename: "big.mp4",
contentType: "video/mp4",
});
expect([...received.keys()].sort()).toEqual([1, 2, 3, 4, 5]);
const body = (
complete.mock.calls[0]![0] as { body: { multipart_upload_id: string; parts: unknown[] } }
).body;
expect(body.multipart_upload_id).toBe("mp1");
expect(body.parts).toEqual([1, 2, 3, 4, 5].map((n) => ({ part_number: n, etag: `"e${n}"` })));
}, 30_000);
it("the same link cannot deliver the file in one PUT", async () => {
cutAfterMs = 1200;
const { client } = cloud({ multipart: false });
await expect(
uploadAssetViaDirectUpload({
client,
...bigFile(FILE_BYTES),
filename: "big.mp4",
contentType: "video/mp4",
}),
).rejects.toThrow(/cut at .* on two tries in a row/);
}, 30_000);
it("sends again only the part that was cut, from its own start", async () => {
cutOnce.add(3);
const { client } = cloud();
await uploadAssetViaDirectUpload({
client,
...bigFile(FILE_BYTES),
filename: "big.mp4",
contentType: "video/mp4",
});
expect(puts.filter((n) => n === 3)).toHaveLength(2);
expect(puts.filter((n) => n !== 3)).toHaveLength(4);
expect(received.get(3)).toBe(PART);
}, 30_000);
it("resumes after a restart: parts the cloud holds are not sent again", async () => {
const held = [
{ part_number: 1, size_bytes: PART },
{ part_number: 2, size_bytes: PART },
];
const first = cloud({ held });
first.complete.mockRejectedValue(new Error("the CLI was closed before complete"));
const file = bigFile(FILE_BYTES);
await expect(
uploadAssetViaDirectUpload({
client: first.client,
...file,
filename: "big.mp4",
contentType: "video/mp4",
}),
).rejects.toThrow();
puts.length = 0;
received.clear();
const second = cloud({ held });
await uploadAssetViaDirectUpload({
client: second.client,
...file,
filename: "big.mp4",
contentType: "video/mp4",
});
expect([...puts].sort()).toEqual([3, 4, 5]);
expect(second.mocks.createAssetUpload).not.toHaveBeenCalled();
}, 30_000);
it("forgets the saved plan when the cloud refuses the file, and keeps it when complete only failed", async () => {
const file = bigFile(FILE_BYTES);
const upload = (client: HyperframesCloudClient) =>
uploadAssetViaDirectUpload({
client,
...file,
filename: "big.mp4",
contentType: "video/mp4",
});
const refused = cloud();
refused.complete.mockRejectedValue(
new HyperframesApiError({ status: 400, code: "invalid_parameter", message: "bad type" }),
);
await expect(upload(refused.client)).rejects.toThrow(/bad type/);
const afterRefusal = cloud();
await upload(afterRefusal.client);
expect(afterRefusal.mocks.createAssetUpload).toHaveBeenCalledTimes(1);
const failed = cloud();
failed.complete.mockRejectedValue(
new HyperframesApiError({ status: 503, message: "try again later" }),
);
await expect(upload(failed.client)).rejects.toThrow(/try again later/);
const afterFailure = cloud();
await upload(afterFailure.client);
expect(afterFailure.mocks.createAssetUpload).not.toHaveBeenCalled();
}, 60_000);
it("reports bytes from the parts done plus the live ones", async () => {
const events: UploadProgressEvent[] = [];
const { client } = cloud();
await uploadAssetViaDirectUpload({
client,
...bigFile(FILE_BYTES),
filename: "big.mp4",
contentType: "video/mp4",
onProgress: (e) => events.push(e),
});
const sent = events.flatMap((e) =>
e.phase === "upload" && e.sentBytes !== undefined ? [e.sentBytes] : [],
);
expect(sent.length).toBeGreaterThan(2);
expect(Math.max(...sent)).toBeGreaterThan(PART);
}, 30_000);
});
describe("files and servers that keep the single PUT", () => {
it("a small file never asks for parts", async () => {
const { client, mocks } = cloud({ multipart: false });
await uploadAssetViaDirectUpload({
client,
...bigFile(2 * MIB),
filename: "s.mp4",
contentType: "video/mp4",
});
expect((mocks.createAssetUpload.mock.calls[0]![0] as { body: object }).body).not.toHaveProperty(
"multipart",
);
expect(puts).toEqual([0]);
});
it("a server that answers without a plan gets the single PUT", async () => {
const { client, mocks } = cloud({ multipart: false });
await uploadAssetViaDirectUpload({
client,
...bigFile(FILE_BYTES),
filename: "b.mp4",
contentType: "video/mp4",
});
expect(mocks.getAssetUploadPartUrls).not.toHaveBeenCalled();
expect(received.get(0)).toBe(FILE_BYTES);
}, 30_000);
it("a server that refuses the unknown field is asked again without it", async () => {
const { client, mocks } = cloud({ multipart: false });
mocks.createAssetUpload.mockRejectedValueOnce(
new HyperframesApiError({ status: 422, code: "invalid_parameter", message: "extra field" }),
);
await uploadAssetViaDirectUpload({
client,
...bigFile(FILE_BYTES),
filename: "b.mp4",
contentType: "video/mp4",
});
expect(mocks.createAssetUpload).toHaveBeenCalledTimes(2);
}, 30_000);
});
+183
View File
@@ -0,0 +1,183 @@
/** Sends a file to the cloud in parts, so a cut connection costs one part and a restart sends only the rest. */
import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import { hyperframesConfigDir } from "../utils/configDir.js";
import type { HyperframesCloudClient } from "./_gen/client.js";
import type { CompletedPart, MultipartUploadPlan } from "./_gen/types.js";
import {
putStreamToPresignedUrl,
putTimings,
putWithRetry,
type PutTry,
type UploadFileSource,
} from "./uploadPut.js";
import type { UploadProgressEvent } from "./upload.js";
/** Files above this go in parts; the server fixes the part size. */
export const PARTS_ABOVE_BYTES = 16 * 1024 * 1024;
const PARTS_IN_FLIGHT = 3;
/** The upload a restart picks up again: it is found by the file's content, so a changed file starts over. */
interface SavedUpload {
asset_id: string;
plan: MultipartUploadPlan;
}
export function savedUploadFile(sha256: string, size: number): string {
return hyperframesConfigDir("uploads", `${sha256.slice(0, 32)}-${size}.json`);
}
function readSaved(file: string): SavedUpload | undefined {
try {
return JSON.parse(readFileSync(file, "utf8")) as SavedUpload;
} catch {
return undefined;
}
}
export function forgetSaved(file: string): void {
rmSync(file, { force: true });
}
export function saveUpload(file: string, saved: SavedUpload): void {
try {
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
writeFileSync(file, JSON.stringify(saved), { mode: 0o600 });
} catch {
// expected: a read-only config folder only costs the resume after a restart
}
}
/** The upload saved for this content that the cloud still holds, or nothing. */
export async function resumable(
client: HyperframesCloudClient,
file: string,
): Promise<SavedUpload | undefined> {
const saved = readSaved(file);
if (!saved) return undefined;
try {
await client.listAssetUploadParts({
asset_id: saved.asset_id,
multipart_upload_id: saved.plan.multipart_upload_id,
});
return saved;
} catch {
forgetSaved(file);
return undefined;
}
}
function partRange(plan: MultipartUploadPlan, total: number, number: number) {
const start = (number - 1) * plan.part_size_bytes;
return { start, length: Math.min(plan.part_size_bytes, total - start) };
}
export interface PartsUpload {
client: HyperframesCloudClient;
asset_id: string;
plan: MultipartUploadPlan;
body: Uint8Array | UploadFileSource;
total: number;
onProgress?: (event: UploadProgressEvent) => void;
signal?: AbortSignal;
}
/** Sends every part the cloud does not hold yet, a few at a time, each part retried alone from its own start. */
export async function uploadParts(job: PartsUpload): Promise<CompletedPart[]> {
const { client, asset_id, plan, total } = job;
const id = plan.multipart_upload_id;
const held = await client.listAssetUploadParts({ asset_id, multipart_upload_id: id });
const done = new Map<number, string>();
for (const part of held.parts) {
const range = partRange(plan, total, part.part_number);
if (part.size_bytes === range.length) done.set(part.part_number, part.etag);
}
const progress = partsProgress(
job,
[...done.keys()].reduce((sum, n) => sum + partRange(plan, total, n).length, 0),
);
progress.report(1);
const todo = Array.from({ length: plan.part_count }, (_, i) => i + 1).filter((n) => !done.has(n));
const failure: { error?: unknown } = {};
const worker = async () => {
for (let number = todo.shift(); number !== undefined; number = todo.shift()) {
if (failure.error) return;
try {
done.set(number, await sendPart(job, number, progress));
progress.finish(number, partRange(plan, total, number).length);
} catch (err) {
failure.error ??= err;
return;
}
}
};
await Promise.all(Array.from({ length: Math.min(PARTS_IN_FLIGHT, todo.length) }, worker));
if (failure.error) throw failure.error;
return [...done.entries()]
.sort((a, b) => a[0] - b[0])
.map(([part_number, etag]) => ({ part_number, etag }));
}
/** Bytes sent so far: the parts done plus what each part in flight has written, reported at most every progressMs
* or 5 points. */
function partsProgress(job: PartsUpload, finishedBytes: number) {
const live = new Map<number, number>();
let finished = finishedBytes;
let lastAt = Date.now();
let lastPercent = -5;
const report = (attempt: number) => {
const sent = Math.min(job.total, finished + [...live.values()].reduce((a, b) => a + b, 0));
const percent = job.total > 0 ? Math.min(99, Math.floor((100 * sent) / job.total)) : 0;
const now = Date.now();
if (now - lastAt < putTimings.progressMs && percent < lastPercent + 5) return;
[lastAt, lastPercent] = [now, percent];
job.onProgress?.({ phase: "upload", percent, sentBytes: sent, totalBytes: job.total, attempt });
};
return {
report,
written: (number: number, bytes: number, attempt: number) => (
live.set(number, bytes), report(attempt)
),
finish: (number: number, bytes: number) => (
(finished += bytes), live.delete(number), report(1)
),
};
}
async function sendPart(
job: PartsUpload,
number: number,
progress: ReturnType<typeof partsProgress>,
): Promise<string> {
const { client, asset_id, plan, body, total, signal } = job;
const range = partRange(plan, total, number);
return putWithRetry(
async (tried: PutTry) => {
const minted = await client.getAssetUploadPartUrls({
asset_id,
body: { multipart_upload_id: plan.multipart_upload_id, part_numbers: [number] },
signal,
});
const url = minted.parts.find((part) => part.part_number === number)?.upload_url;
if (!url) throw new Error(`The cloud gave no address for part ${number}`);
progress.written(number, 0, tried.attempt);
const etag = await putStreamToPresignedUrl(
url,
{},
body,
signal,
(written) => {
tried.sent = written;
progress.written(number, Math.min(written, range.length), tried.attempt);
},
range,
);
if (!etag) throw new Error(`The cloud gave no ETag for part ${number}`);
return etag;
},
range.length,
signal,
);
}
+20 -7
View File
@@ -125,12 +125,23 @@ export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
// same way, in slices, so a stalled link ends its try the same way (a plain fetch has no progress to watch).
const SLICE_BYTES = 64 * 1024;
function bodyStream(body: Uint8Array | UploadFileSource): Readable {
if (!(body instanceof Uint8Array)) return createReadStream(body.path);
export interface ByteRange {
start: number;
length: number;
}
function bodyStream(body: Uint8Array | UploadFileSource, range?: ByteRange): Readable {
if (!(body instanceof Uint8Array))
return createReadStream(
body.path,
range && { start: range.start, end: range.start + range.length - 1 },
);
const from = range?.start ?? 0;
const to = from + (range?.length ?? body.byteLength);
return Readable.from(
(function* () {
for (let at = 0; at < body.byteLength; at += SLICE_BYTES)
yield body.subarray(at, at + SLICE_BYTES);
for (let at = from; at < to; at += SLICE_BYTES)
yield body.subarray(at, Math.min(at + SLICE_BYTES, to));
})(),
);
}
@@ -159,8 +170,9 @@ export async function putStreamToPresignedUrl(
body: Uint8Array | UploadFileSource,
signal: AbortSignal | undefined,
onSample: (written: number) => void,
): Promise<void> {
const size = body instanceof Uint8Array ? body.byteLength : body.size;
range?: ByteRange,
): Promise<string | undefined> {
const size = range?.length ?? (body instanceof Uint8Array ? body.byteLength : body.size);
const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest;
const req = send(uploadUrl, {
method: "PUT",
@@ -171,7 +183,7 @@ export async function putStreamToPresignedUrl(
req.once("response", resolve).once("error", reject);
});
const stopWatching = watchStall(req, onSample);
const source = bodyStream(body);
const source = bodyStream(body, range);
const sent = pipeline(source, req);
let res: IncomingMessage;
try {
@@ -183,6 +195,7 @@ export async function putStreamToPresignedUrl(
stopWatching();
}
await settle(res, sent);
return String(res.headers.etag ?? "") || undefined;
}
/** Reads S3's answer: a refusal is thrown with its words; a refused upload can close the connection mid-body, so the