mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
feat(cli): cloud uploads send a big file in parts, so a cut connection resends one part (#7375)
* refactor(cli): the cloud upload's PUT and retry code moves to its own file * feat(cli): cloud uploads send a big file in parts, so a cut connection resends one part * test(cli): the cutting link fails a single PUT of the same file * fix(cli): a file the cloud refuses forgets its saved parts plan * fix(cli): a resumed upload goes straight to its parts, so a saved plan never feeds the single PUT
This commit is contained in:
Generated
+49
@@ -5,6 +5,7 @@
|
||||
* experiment-framework to regenerate.
|
||||
*/
|
||||
import type {
|
||||
AbortAssetUploadResponse,
|
||||
BackgroundRemovalJob,
|
||||
CompleteAssetUploadRequest,
|
||||
CompleteAssetUploadResponse,
|
||||
@@ -25,6 +26,9 @@ import type {
|
||||
ImageCreateRequest,
|
||||
LipsyncCreateResponse,
|
||||
LipsyncDetail,
|
||||
ListUploadedPartsResponse,
|
||||
MintPartUrlsRequest,
|
||||
MintPartUrlsResponse,
|
||||
ModelVideoResponse,
|
||||
MusicCreateRequest,
|
||||
MusicResponse,
|
||||
@@ -271,6 +275,51 @@ export class HyperframesCloudClient {
|
||||
});
|
||||
}
|
||||
|
||||
async getAssetUploadPartUrls(args: {
|
||||
asset_id: string;
|
||||
body: MintPartUrlsRequest;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<MintPartUrlsResponse> {
|
||||
return await this.request<MintPartUrlsResponse>({
|
||||
method: "POST",
|
||||
path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`,
|
||||
body: args.body,
|
||||
signal: args.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async listAssetUploadParts(args: {
|
||||
asset_id: string;
|
||||
multipart_upload_id: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ListUploadedPartsResponse> {
|
||||
const query: Record<string, string | number | undefined> = {
|
||||
multipart_upload_id: args.multipart_upload_id,
|
||||
};
|
||||
return await this.request<ListUploadedPartsResponse>({
|
||||
method: "GET",
|
||||
path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`,
|
||||
query,
|
||||
signal: args.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async abortAssetUploadParts(args: {
|
||||
asset_id: string;
|
||||
multipart_upload_id: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AbortAssetUploadResponse> {
|
||||
const query: Record<string, string | number | undefined> = {
|
||||
multipart_upload_id: args.multipart_upload_id,
|
||||
};
|
||||
return await this.request<AbortAssetUploadResponse>({
|
||||
method: "DELETE",
|
||||
path: `/v3/assets/${encodeURIComponent(args.asset_id)}/parts`,
|
||||
query,
|
||||
signal: args.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async createRender(args: {
|
||||
body: CreateHyperframesRenderRequest;
|
||||
idempotencyKey?: string;
|
||||
|
||||
Generated
+67
-1
@@ -9,6 +9,11 @@
|
||||
// Add a new path/method to TARGET_ENDPOINTS in
|
||||
// scripts/generate_hyperframes_cli_client.py to extend this list.
|
||||
|
||||
export interface AbortAssetUploadResponse {
|
||||
asset_id: string;
|
||||
status: "aborted";
|
||||
}
|
||||
|
||||
export interface AssetBase64 {
|
||||
type: "base64";
|
||||
media_type: string;
|
||||
@@ -78,6 +83,8 @@ export type CinematicAvatarModel = "seedance_2" | "heygen_video_1";
|
||||
|
||||
export interface CompleteAssetUploadRequest {
|
||||
checksum_sha256?: string | null;
|
||||
multipart_upload_id?: string | null;
|
||||
parts?: Array<CompletedPart> | null;
|
||||
}
|
||||
|
||||
export interface CompleteAssetUploadResponse {
|
||||
@@ -88,11 +95,17 @@ export interface CompleteAssetUploadResponse {
|
||||
status: "processing";
|
||||
}
|
||||
|
||||
export interface CompletedPart {
|
||||
part_number: number;
|
||||
etag: string;
|
||||
}
|
||||
|
||||
export interface CreateAssetUploadRequest {
|
||||
filename: string;
|
||||
content_type: string;
|
||||
size_bytes: number;
|
||||
checksum_sha256?: string | null;
|
||||
multipart?: boolean;
|
||||
}
|
||||
|
||||
export interface CreateAssetUploadResponse {
|
||||
@@ -102,6 +115,7 @@ export interface CreateAssetUploadResponse {
|
||||
expires_in_seconds: number;
|
||||
max_bytes: number;
|
||||
status: "pending_upload";
|
||||
multipart?: MultipartUploadPlan | null;
|
||||
}
|
||||
|
||||
export interface CreateBackgroundRemovalRequest {
|
||||
@@ -358,6 +372,22 @@ export interface GptImage25Request {
|
||||
reference_images?: Array<AssetUrl | AssetId | AssetBase64>;
|
||||
}
|
||||
|
||||
export interface GptImage26FlareRequest {
|
||||
model: "gpt-image-2.6-flare";
|
||||
prompt: string;
|
||||
num_images?: number;
|
||||
aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null;
|
||||
reference_images?: Array<AssetUrl | AssetId | AssetBase64>;
|
||||
}
|
||||
|
||||
export interface GptImage26Request {
|
||||
model: "gpt-image-2.6";
|
||||
prompt: string;
|
||||
num_images?: number;
|
||||
aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null;
|
||||
reference_images?: Array<AssetUrl | AssetId | AssetBase64>;
|
||||
}
|
||||
|
||||
export interface HeyGenVoiceEngineSettings {
|
||||
engine_type: "heygen_voice" | "orca";
|
||||
}
|
||||
@@ -394,7 +424,12 @@ export type HyperframesRenderStatus = "queued" | "rendering" | "completed" | "fa
|
||||
|
||||
export type HyperframesResolution = "1080p" | "4k";
|
||||
|
||||
export type ImageCreateRequest = GptImage25Request | NanoBanana2Request | NanoBanana21Request;
|
||||
export type ImageCreateRequest =
|
||||
| GptImage25Request
|
||||
| GptImage26Request
|
||||
| GptImage26FlareRequest
|
||||
| NanoBanana2Request
|
||||
| NanoBanana21Request;
|
||||
|
||||
export interface ImageScene {
|
||||
type: "image";
|
||||
@@ -440,6 +475,10 @@ export interface LipsyncDetail {
|
||||
failure_code?: string | null;
|
||||
}
|
||||
|
||||
export interface ListUploadedPartsResponse {
|
||||
parts: Array<UploadedPart>;
|
||||
}
|
||||
|
||||
export interface LyriaMusicRequest {
|
||||
model: "lyria-3-pro";
|
||||
prompt: string;
|
||||
@@ -450,6 +489,16 @@ export interface LyriaMusicRequest {
|
||||
lyrics?: string | null;
|
||||
}
|
||||
|
||||
export interface MintPartUrlsRequest {
|
||||
multipart_upload_id: string;
|
||||
part_numbers: Array<number>;
|
||||
}
|
||||
|
||||
export interface MintPartUrlsResponse {
|
||||
parts: Array<PartUrl>;
|
||||
expires_in_seconds: number;
|
||||
}
|
||||
|
||||
export interface ModelVideoResponse {
|
||||
video_id: string;
|
||||
status: "pending" | "processing" | "completed" | "failed" | "cancelled";
|
||||
@@ -471,6 +520,12 @@ export interface ModelVideoTimings {
|
||||
inference: number;
|
||||
}
|
||||
|
||||
export interface MultipartUploadPlan {
|
||||
multipart_upload_id: string;
|
||||
part_size_bytes: number;
|
||||
part_count: number;
|
||||
}
|
||||
|
||||
export type MusicCreateRequest = LyriaMusicRequest;
|
||||
|
||||
export type MusicFailureCode =
|
||||
@@ -520,6 +575,11 @@ export interface OutputUploadRequest {
|
||||
thumbnail?: UploadTargetRequest | null;
|
||||
}
|
||||
|
||||
export interface PartUrl {
|
||||
part_number: number;
|
||||
upload_url: string;
|
||||
}
|
||||
|
||||
export interface PlaybackInput {
|
||||
mode?: "freeze" | "loop" | "fit_to_scene" | null;
|
||||
volume?: number;
|
||||
@@ -702,6 +762,12 @@ export interface UploadTargetRequest {
|
||||
headers?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface UploadedPart {
|
||||
part_number: number;
|
||||
size_bytes: number;
|
||||
etag: string;
|
||||
}
|
||||
|
||||
export type VideoAspectRatio = "16:9" | "9:16" | "4:5" | "5:4" | "1:1" | "auto";
|
||||
|
||||
export interface VideoDetail {
|
||||
|
||||
@@ -26,6 +26,11 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import type { HyperframesCloudClient } from "./_gen/client.js";
|
||||
import type {
|
||||
CompleteAssetUploadRequest,
|
||||
CreateAssetUploadResponse,
|
||||
MultipartUploadPlan,
|
||||
} from "./_gen/types.js";
|
||||
import { HyperframesApiError } from "./_gen/client.js";
|
||||
import {
|
||||
PutRefusedError,
|
||||
@@ -36,6 +41,14 @@ import {
|
||||
type PutTry,
|
||||
type UploadFileSource,
|
||||
} from "./uploadPut.js";
|
||||
import {
|
||||
forgetSaved,
|
||||
PARTS_ABOVE_BYTES,
|
||||
resumable,
|
||||
saveUpload,
|
||||
savedUploadFile,
|
||||
uploadParts,
|
||||
} from "./uploadParts.js";
|
||||
|
||||
export { putTimings, type UploadFileSource };
|
||||
|
||||
@@ -137,7 +150,7 @@ async function putBytesToPresignedUrl(
|
||||
...normalizeUploadHeaders(uploadHeaders),
|
||||
};
|
||||
return putWithRetry(
|
||||
(tried) => {
|
||||
async (tried) => {
|
||||
if (tried.attempt > 1)
|
||||
onProgress?.({
|
||||
phase: "upload",
|
||||
@@ -146,12 +159,12 @@ async function putBytesToPresignedUrl(
|
||||
totalBytes: total,
|
||||
attempt: tried.attempt,
|
||||
});
|
||||
return fetchImpl && body instanceof Uint8Array
|
||||
await (fetchImpl && body instanceof Uint8Array
|
||||
? putBytesOnce(fetchImpl, uploadUrl, headers, body, signal)
|
||||
: putStreamToPresignedUrl(uploadUrl, headers, body, signal, (written) => {
|
||||
tried.sent = written;
|
||||
report(tried);
|
||||
});
|
||||
}));
|
||||
},
|
||||
total,
|
||||
signal,
|
||||
@@ -208,13 +221,14 @@ export async function completeWithRetry(
|
||||
checksum_sha256: string,
|
||||
sizeBytes: number,
|
||||
signal?: AbortSignal,
|
||||
inParts: CompleteParts = {},
|
||||
): Promise<{ asset_id: string; url: string }> {
|
||||
const budget = completeBudgetMs(sizeBytes);
|
||||
const startedAt = Date.now();
|
||||
let poll = completeWait.firstPollMs;
|
||||
for (;;) {
|
||||
try {
|
||||
return await client.completeAssetUpload({ asset_id, body: { checksum_sha256 } });
|
||||
return await client.completeAssetUpload({ asset_id, body: { checksum_sha256, ...inParts } });
|
||||
} catch (err) {
|
||||
if (!(err instanceof HyperframesApiError && err.status === 409)) throw err;
|
||||
const left = startedAt + budget - Date.now();
|
||||
@@ -237,6 +251,17 @@ export async function completeWithRetry(
|
||||
// complete answers invalid_parameter for type, size, checksum and expiry; only type refusals blame the type.
|
||||
const TYPE_REFUSAL = /^(Stored file type|content_type not allowed|Stored file does not have a ZIP)/;
|
||||
|
||||
const STATUSES_NOT_ABOUT_THE_FILE = new Set([401, 403, 408, 409, 429]);
|
||||
|
||||
function refusesTheFile(err: unknown): boolean {
|
||||
return (
|
||||
err instanceof HyperframesApiError &&
|
||||
err.status >= 400 &&
|
||||
err.status < 500 &&
|
||||
!STATUSES_NOT_ABOUT_THE_FILE.has(err.status)
|
||||
);
|
||||
}
|
||||
|
||||
function nameRefusedFile(err: unknown, filename: string, contentType: string): unknown {
|
||||
if (!(err instanceof HyperframesApiError) || err.code !== "invalid_parameter") return err;
|
||||
const kind = contentType.split("/")[0];
|
||||
@@ -247,48 +272,131 @@ function nameRefusedFile(err: unknown, filename: string, contentType: string): u
|
||||
return new HyperframesApiError({ ...err, message: `${lead} ${err.message}` });
|
||||
}
|
||||
|
||||
export async function uploadAssetViaDirectUpload(
|
||||
opts: Omit<UploadZipViaDirectOptions, "bytes"> & { contentType: string } & (
|
||||
| { bytes: Uint8Array }
|
||||
| UploadFileSource
|
||||
),
|
||||
): Promise<UploadZipViaDirectResult> {
|
||||
const start = Date.now();
|
||||
const { filename, idempotencyKey } = opts;
|
||||
const body = "bytes" in opts ? opts.bytes : { path: opts.path, size: opts.size };
|
||||
const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size;
|
||||
const checksum_sha256 =
|
||||
body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path);
|
||||
const fetchImpl = opts.fetchImpl;
|
||||
type CompleteParts = Pick<CompleteAssetUploadRequest, "multipart_upload_id" | "parts">;
|
||||
|
||||
opts.onProgress?.({ phase: "initialize" });
|
||||
const initialize = await opts.client.createAssetUpload({
|
||||
body: { filename, content_type: opts.contentType, size_bytes, checksum_sha256 },
|
||||
idempotencyKey,
|
||||
/** Asks for parts on a big file; a server that does not know them answers with the single PUT. */
|
||||
async function createUpload(
|
||||
client: HyperframesCloudClient,
|
||||
o: {
|
||||
filename: string;
|
||||
contentType: string;
|
||||
size_bytes: number;
|
||||
checksum_sha256: string;
|
||||
inParts: boolean;
|
||||
idempotencyKey?: string;
|
||||
},
|
||||
): Promise<CreateAssetUploadResponse> {
|
||||
const body = {
|
||||
filename: o.filename,
|
||||
content_type: o.contentType,
|
||||
size_bytes: o.size_bytes,
|
||||
checksum_sha256: o.checksum_sha256,
|
||||
};
|
||||
try {
|
||||
return await client.createAssetUpload({
|
||||
body: o.inParts ? { ...body, multipart: true } : body,
|
||||
idempotencyKey: o.idempotencyKey,
|
||||
});
|
||||
} catch (err) {
|
||||
const unknownField =
|
||||
err instanceof HyperframesApiError && (err.status === 400 || err.status === 422);
|
||||
if (!o.inParts || !unknownField) throw err;
|
||||
return await client.createAssetUpload({ body, idempotencyKey: o.idempotencyKey });
|
||||
}
|
||||
}
|
||||
|
||||
type DirectUploadOptions = Omit<UploadZipViaDirectOptions, "bytes"> & { contentType: string };
|
||||
|
||||
/** Sends the file (in parts above PARTS_ABOVE_BYTES, resuming a saved upload) and says what `complete` must be told. */
|
||||
async function sendInParts(
|
||||
opts: DirectUploadOptions,
|
||||
body: Uint8Array | UploadFileSource,
|
||||
size_bytes: number,
|
||||
asset_id: string,
|
||||
plan: MultipartUploadPlan,
|
||||
saved: string | undefined,
|
||||
): Promise<{ asset_id: string; inParts: CompleteParts; forget?: () => void }> {
|
||||
if (saved) saveUpload(saved, { asset_id, plan });
|
||||
const parts = await uploadParts({
|
||||
client: opts.client,
|
||||
asset_id,
|
||||
plan,
|
||||
body,
|
||||
total: size_bytes,
|
||||
onProgress: opts.onProgress,
|
||||
signal: opts.signal,
|
||||
});
|
||||
return {
|
||||
asset_id,
|
||||
inParts: { multipart_upload_id: plan.multipart_upload_id, parts },
|
||||
forget: saved ? () => forgetSaved(saved) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function sendFile(
|
||||
opts: DirectUploadOptions,
|
||||
body: Uint8Array | UploadFileSource,
|
||||
size_bytes: number,
|
||||
checksum_sha256: string,
|
||||
): Promise<{ asset_id: string; inParts: CompleteParts; forget?: () => void }> {
|
||||
const wantParts = size_bytes > PARTS_ABOVE_BYTES;
|
||||
const saved = wantParts ? savedUploadFile(checksum_sha256, size_bytes) : undefined;
|
||||
const resumed = saved ? await resumable(opts.client, saved) : undefined;
|
||||
if (resumed) {
|
||||
opts.onProgress?.({ phase: "upload", percent: 0 });
|
||||
return sendInParts(opts, body, size_bytes, resumed.asset_id, resumed.plan, saved);
|
||||
}
|
||||
const created = await createUpload(opts.client, {
|
||||
filename: opts.filename,
|
||||
contentType: opts.contentType,
|
||||
size_bytes,
|
||||
checksum_sha256,
|
||||
inParts: wantParts,
|
||||
idempotencyKey: opts.idempotencyKey,
|
||||
});
|
||||
|
||||
opts.onProgress?.({ phase: "upload", percent: 0 });
|
||||
if (created.multipart) {
|
||||
return sendInParts(opts, body, size_bytes, created.asset_id, created.multipart, saved);
|
||||
}
|
||||
await putBytesToPresignedUrl(
|
||||
fetchImpl,
|
||||
initialize.upload_url,
|
||||
initialize.upload_headers,
|
||||
opts.fetchImpl,
|
||||
created.upload_url,
|
||||
created.upload_headers,
|
||||
body,
|
||||
opts.contentType,
|
||||
opts.signal,
|
||||
opts.onProgress,
|
||||
);
|
||||
return { asset_id: created.asset_id, inParts: {} };
|
||||
}
|
||||
|
||||
export async function uploadAssetViaDirectUpload(
|
||||
opts: DirectUploadOptions & ({ bytes: Uint8Array } | UploadFileSource),
|
||||
): Promise<UploadZipViaDirectResult> {
|
||||
const start = Date.now();
|
||||
const body = "bytes" in opts ? opts.bytes : { path: opts.path, size: opts.size };
|
||||
const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size;
|
||||
const checksum_sha256 =
|
||||
body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path);
|
||||
|
||||
opts.onProgress?.({ phase: "initialize" });
|
||||
const sent = await sendFile(opts, body, size_bytes, checksum_sha256);
|
||||
opts.onProgress?.({ phase: "upload", percent: 100 });
|
||||
|
||||
opts.onProgress?.({ phase: "complete" });
|
||||
const completed = await completeWithRetry(
|
||||
opts.client,
|
||||
initialize.asset_id,
|
||||
sent.asset_id,
|
||||
checksum_sha256,
|
||||
size_bytes,
|
||||
opts.signal,
|
||||
sent.inParts,
|
||||
).catch((err: unknown) => {
|
||||
throw nameRefusedFile(err, filename, opts.contentType);
|
||||
if (refusesTheFile(err)) sent.forget?.();
|
||||
throw nameRefusedFile(err, opts.filename, opts.contentType);
|
||||
});
|
||||
sent.forget?.();
|
||||
return {
|
||||
asset_id: completed.asset_id,
|
||||
url: completed.url,
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { HyperframesApiError } from "./_gen/client.js";
|
||||
import type { HyperframesCloudClient } from "./_gen/client.js";
|
||||
import { putTimings, uploadAssetViaDirectUpload, type UploadProgressEvent } from "./upload.js";
|
||||
|
||||
const MIB = 1024 * 1024;
|
||||
const PART = 4 * MIB;
|
||||
const RATE_BYTES_PER_MS = 8 * 1024;
|
||||
|
||||
let dir: string;
|
||||
let server: Server;
|
||||
let base: string;
|
||||
let received: Map<number, number>;
|
||||
let puts: number[];
|
||||
let cutAfterMs: number | undefined;
|
||||
let cutOnce: Set<number>;
|
||||
const saved = { ...putTimings };
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-parts-"));
|
||||
process.env["HYPERFRAMES_CONFIG_DIR"] = join(dir, "config");
|
||||
Object.assign(putTimings, { stallMs: 5000, backoffMs: 5, progressMs: 50, sampleMs: 50 });
|
||||
received = new Map();
|
||||
puts = [];
|
||||
cutAfterMs = undefined;
|
||||
cutOnce = new Set();
|
||||
server = createServer((req, res) => {
|
||||
const part = Number(req.url?.split("/").pop()) || 0;
|
||||
puts.push(part);
|
||||
const cutAt = cutOnce.delete(part) ? 100 : cutAfterMs;
|
||||
let size = 0;
|
||||
const startedAt = Date.now();
|
||||
req.on("error", () => undefined);
|
||||
req.on("data", (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
req.pause();
|
||||
setTimeout(
|
||||
() => {
|
||||
if (cutAt !== undefined && Date.now() - startedAt >= cutAt) req.socket.destroy();
|
||||
else req.resume();
|
||||
},
|
||||
Math.ceil(chunk.length / RATE_BYTES_PER_MS),
|
||||
);
|
||||
});
|
||||
req.on("end", () => {
|
||||
received.set(part, size);
|
||||
res.writeHead(200, { etag: `"e${part}"` }).end();
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
Object.assign(putTimings, saved);
|
||||
delete process.env["HYPERFRAMES_CONFIG_DIR"];
|
||||
server.closeAllConnections();
|
||||
server.close();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function bigFile(bytes: number): { path: string; size: number } {
|
||||
const path = join(dir, "big.mp4");
|
||||
writeFileSync(path, Buffer.alloc(bytes, 7));
|
||||
return { path, size: bytes };
|
||||
}
|
||||
|
||||
function cloud(
|
||||
over: { multipart?: boolean; held?: { part_number: number; size_bytes: number }[] } = {},
|
||||
) {
|
||||
const complete = vi.fn(async (_args: unknown) => ({
|
||||
asset_id: "a1",
|
||||
url: "u",
|
||||
mime_type: "video/mp4",
|
||||
size_bytes: 1,
|
||||
status: "processing" as const,
|
||||
}));
|
||||
const client = {
|
||||
createAssetUpload: vi.fn(async (_args?: unknown) => ({
|
||||
asset_id: "a1",
|
||||
upload_url: `${base}/single`,
|
||||
upload_headers: {},
|
||||
expires_in_seconds: 60,
|
||||
max_bytes: 1,
|
||||
status: "pending_upload" as const,
|
||||
...(over.multipart === false
|
||||
? {}
|
||||
: { multipart: { multipart_upload_id: "mp1", part_size_bytes: PART, part_count: 5 } }),
|
||||
})),
|
||||
getAssetUploadPartUrls: vi.fn(async (args: { body: { part_numbers: number[] } }) => ({
|
||||
expires_in_seconds: 60,
|
||||
parts: args.body.part_numbers.map((n) => ({
|
||||
part_number: n,
|
||||
upload_url: `${base}/part/${n}`,
|
||||
})),
|
||||
})),
|
||||
listAssetUploadParts: vi.fn(async () => ({
|
||||
parts: (over.held ?? []).map((p) => ({ ...p, etag: `"e${p.part_number}"` })),
|
||||
})),
|
||||
completeAssetUpload: complete,
|
||||
};
|
||||
return { client: client as unknown as HyperframesCloudClient, mocks: client, complete };
|
||||
}
|
||||
|
||||
const FILE_BYTES = 17 * MIB;
|
||||
|
||||
describe("a big file goes up in parts", () => {
|
||||
it("delivers a file a link cannot finish before the cut, because each part finishes alone", async () => {
|
||||
cutAfterMs = 1200;
|
||||
const { client, complete } = cloud();
|
||||
|
||||
await uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...bigFile(FILE_BYTES),
|
||||
filename: "big.mp4",
|
||||
contentType: "video/mp4",
|
||||
});
|
||||
|
||||
expect([...received.keys()].sort()).toEqual([1, 2, 3, 4, 5]);
|
||||
const body = (
|
||||
complete.mock.calls[0]![0] as { body: { multipart_upload_id: string; parts: unknown[] } }
|
||||
).body;
|
||||
expect(body.multipart_upload_id).toBe("mp1");
|
||||
expect(body.parts).toEqual([1, 2, 3, 4, 5].map((n) => ({ part_number: n, etag: `"e${n}"` })));
|
||||
}, 30_000);
|
||||
|
||||
it("the same link cannot deliver the file in one PUT", async () => {
|
||||
cutAfterMs = 1200;
|
||||
const { client } = cloud({ multipart: false });
|
||||
|
||||
await expect(
|
||||
uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...bigFile(FILE_BYTES),
|
||||
filename: "big.mp4",
|
||||
contentType: "video/mp4",
|
||||
}),
|
||||
).rejects.toThrow(/cut at .* on two tries in a row/);
|
||||
}, 30_000);
|
||||
|
||||
it("sends again only the part that was cut, from its own start", async () => {
|
||||
cutOnce.add(3);
|
||||
const { client } = cloud();
|
||||
|
||||
await uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...bigFile(FILE_BYTES),
|
||||
filename: "big.mp4",
|
||||
contentType: "video/mp4",
|
||||
});
|
||||
|
||||
expect(puts.filter((n) => n === 3)).toHaveLength(2);
|
||||
expect(puts.filter((n) => n !== 3)).toHaveLength(4);
|
||||
expect(received.get(3)).toBe(PART);
|
||||
}, 30_000);
|
||||
|
||||
it("resumes after a restart: parts the cloud holds are not sent again", async () => {
|
||||
const held = [
|
||||
{ part_number: 1, size_bytes: PART },
|
||||
{ part_number: 2, size_bytes: PART },
|
||||
];
|
||||
const first = cloud({ held });
|
||||
first.complete.mockRejectedValue(new Error("the CLI was closed before complete"));
|
||||
const file = bigFile(FILE_BYTES);
|
||||
await expect(
|
||||
uploadAssetViaDirectUpload({
|
||||
client: first.client,
|
||||
...file,
|
||||
filename: "big.mp4",
|
||||
contentType: "video/mp4",
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
puts.length = 0;
|
||||
received.clear();
|
||||
const second = cloud({ held });
|
||||
|
||||
await uploadAssetViaDirectUpload({
|
||||
client: second.client,
|
||||
...file,
|
||||
filename: "big.mp4",
|
||||
contentType: "video/mp4",
|
||||
});
|
||||
|
||||
expect([...puts].sort()).toEqual([3, 4, 5]);
|
||||
expect(second.mocks.createAssetUpload).not.toHaveBeenCalled();
|
||||
}, 30_000);
|
||||
|
||||
it("forgets the saved plan when the cloud refuses the file, and keeps it when complete only failed", async () => {
|
||||
const file = bigFile(FILE_BYTES);
|
||||
const upload = (client: HyperframesCloudClient) =>
|
||||
uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...file,
|
||||
filename: "big.mp4",
|
||||
contentType: "video/mp4",
|
||||
});
|
||||
|
||||
const refused = cloud();
|
||||
refused.complete.mockRejectedValue(
|
||||
new HyperframesApiError({ status: 400, code: "invalid_parameter", message: "bad type" }),
|
||||
);
|
||||
await expect(upload(refused.client)).rejects.toThrow(/bad type/);
|
||||
const afterRefusal = cloud();
|
||||
await upload(afterRefusal.client);
|
||||
expect(afterRefusal.mocks.createAssetUpload).toHaveBeenCalledTimes(1);
|
||||
|
||||
const failed = cloud();
|
||||
failed.complete.mockRejectedValue(
|
||||
new HyperframesApiError({ status: 503, message: "try again later" }),
|
||||
);
|
||||
await expect(upload(failed.client)).rejects.toThrow(/try again later/);
|
||||
const afterFailure = cloud();
|
||||
await upload(afterFailure.client);
|
||||
expect(afterFailure.mocks.createAssetUpload).not.toHaveBeenCalled();
|
||||
}, 60_000);
|
||||
|
||||
it("reports bytes from the parts done plus the live ones", async () => {
|
||||
const events: UploadProgressEvent[] = [];
|
||||
const { client } = cloud();
|
||||
|
||||
await uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...bigFile(FILE_BYTES),
|
||||
filename: "big.mp4",
|
||||
contentType: "video/mp4",
|
||||
onProgress: (e) => events.push(e),
|
||||
});
|
||||
|
||||
const sent = events.flatMap((e) =>
|
||||
e.phase === "upload" && e.sentBytes !== undefined ? [e.sentBytes] : [],
|
||||
);
|
||||
expect(sent.length).toBeGreaterThan(2);
|
||||
expect(Math.max(...sent)).toBeGreaterThan(PART);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe("files and servers that keep the single PUT", () => {
|
||||
it("a small file never asks for parts", async () => {
|
||||
const { client, mocks } = cloud({ multipart: false });
|
||||
await uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...bigFile(2 * MIB),
|
||||
filename: "s.mp4",
|
||||
contentType: "video/mp4",
|
||||
});
|
||||
expect((mocks.createAssetUpload.mock.calls[0]![0] as { body: object }).body).not.toHaveProperty(
|
||||
"multipart",
|
||||
);
|
||||
expect(puts).toEqual([0]);
|
||||
});
|
||||
|
||||
it("a server that answers without a plan gets the single PUT", async () => {
|
||||
const { client, mocks } = cloud({ multipart: false });
|
||||
await uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...bigFile(FILE_BYTES),
|
||||
filename: "b.mp4",
|
||||
contentType: "video/mp4",
|
||||
});
|
||||
expect(mocks.getAssetUploadPartUrls).not.toHaveBeenCalled();
|
||||
expect(received.get(0)).toBe(FILE_BYTES);
|
||||
}, 30_000);
|
||||
|
||||
it("a server that refuses the unknown field is asked again without it", async () => {
|
||||
const { client, mocks } = cloud({ multipart: false });
|
||||
mocks.createAssetUpload.mockRejectedValueOnce(
|
||||
new HyperframesApiError({ status: 422, code: "invalid_parameter", message: "extra field" }),
|
||||
);
|
||||
await uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...bigFile(FILE_BYTES),
|
||||
filename: "b.mp4",
|
||||
contentType: "video/mp4",
|
||||
});
|
||||
expect(mocks.createAssetUpload).toHaveBeenCalledTimes(2);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,183 @@
|
||||
/** Sends a file to the cloud in parts, so a cut connection costs one part and a restart sends only the rest. */
|
||||
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { hyperframesConfigDir } from "../utils/configDir.js";
|
||||
import type { HyperframesCloudClient } from "./_gen/client.js";
|
||||
import type { CompletedPart, MultipartUploadPlan } from "./_gen/types.js";
|
||||
import {
|
||||
putStreamToPresignedUrl,
|
||||
putTimings,
|
||||
putWithRetry,
|
||||
type PutTry,
|
||||
type UploadFileSource,
|
||||
} from "./uploadPut.js";
|
||||
import type { UploadProgressEvent } from "./upload.js";
|
||||
|
||||
/** Files above this go in parts; the server fixes the part size. */
|
||||
export const PARTS_ABOVE_BYTES = 16 * 1024 * 1024;
|
||||
const PARTS_IN_FLIGHT = 3;
|
||||
|
||||
/** The upload a restart picks up again: it is found by the file's content, so a changed file starts over. */
|
||||
interface SavedUpload {
|
||||
asset_id: string;
|
||||
plan: MultipartUploadPlan;
|
||||
}
|
||||
|
||||
export function savedUploadFile(sha256: string, size: number): string {
|
||||
return hyperframesConfigDir("uploads", `${sha256.slice(0, 32)}-${size}.json`);
|
||||
}
|
||||
|
||||
function readSaved(file: string): SavedUpload | undefined {
|
||||
try {
|
||||
return JSON.parse(readFileSync(file, "utf8")) as SavedUpload;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function forgetSaved(file: string): void {
|
||||
rmSync(file, { force: true });
|
||||
}
|
||||
|
||||
export function saveUpload(file: string, saved: SavedUpload): void {
|
||||
try {
|
||||
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
||||
writeFileSync(file, JSON.stringify(saved), { mode: 0o600 });
|
||||
} catch {
|
||||
// expected: a read-only config folder only costs the resume after a restart
|
||||
}
|
||||
}
|
||||
|
||||
/** The upload saved for this content that the cloud still holds, or nothing. */
|
||||
export async function resumable(
|
||||
client: HyperframesCloudClient,
|
||||
file: string,
|
||||
): Promise<SavedUpload | undefined> {
|
||||
const saved = readSaved(file);
|
||||
if (!saved) return undefined;
|
||||
try {
|
||||
await client.listAssetUploadParts({
|
||||
asset_id: saved.asset_id,
|
||||
multipart_upload_id: saved.plan.multipart_upload_id,
|
||||
});
|
||||
return saved;
|
||||
} catch {
|
||||
forgetSaved(file);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function partRange(plan: MultipartUploadPlan, total: number, number: number) {
|
||||
const start = (number - 1) * plan.part_size_bytes;
|
||||
return { start, length: Math.min(plan.part_size_bytes, total - start) };
|
||||
}
|
||||
|
||||
export interface PartsUpload {
|
||||
client: HyperframesCloudClient;
|
||||
asset_id: string;
|
||||
plan: MultipartUploadPlan;
|
||||
body: Uint8Array | UploadFileSource;
|
||||
total: number;
|
||||
onProgress?: (event: UploadProgressEvent) => void;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/** Sends every part the cloud does not hold yet, a few at a time, each part retried alone from its own start. */
|
||||
export async function uploadParts(job: PartsUpload): Promise<CompletedPart[]> {
|
||||
const { client, asset_id, plan, total } = job;
|
||||
const id = plan.multipart_upload_id;
|
||||
const held = await client.listAssetUploadParts({ asset_id, multipart_upload_id: id });
|
||||
const done = new Map<number, string>();
|
||||
for (const part of held.parts) {
|
||||
const range = partRange(plan, total, part.part_number);
|
||||
if (part.size_bytes === range.length) done.set(part.part_number, part.etag);
|
||||
}
|
||||
const progress = partsProgress(
|
||||
job,
|
||||
[...done.keys()].reduce((sum, n) => sum + partRange(plan, total, n).length, 0),
|
||||
);
|
||||
progress.report(1);
|
||||
|
||||
const todo = Array.from({ length: plan.part_count }, (_, i) => i + 1).filter((n) => !done.has(n));
|
||||
const failure: { error?: unknown } = {};
|
||||
const worker = async () => {
|
||||
for (let number = todo.shift(); number !== undefined; number = todo.shift()) {
|
||||
if (failure.error) return;
|
||||
try {
|
||||
done.set(number, await sendPart(job, number, progress));
|
||||
progress.finish(number, partRange(plan, total, number).length);
|
||||
} catch (err) {
|
||||
failure.error ??= err;
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(PARTS_IN_FLIGHT, todo.length) }, worker));
|
||||
if (failure.error) throw failure.error;
|
||||
return [...done.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([part_number, etag]) => ({ part_number, etag }));
|
||||
}
|
||||
|
||||
/** Bytes sent so far: the parts done plus what each part in flight has written, reported at most every progressMs
|
||||
* or 5 points. */
|
||||
function partsProgress(job: PartsUpload, finishedBytes: number) {
|
||||
const live = new Map<number, number>();
|
||||
let finished = finishedBytes;
|
||||
let lastAt = Date.now();
|
||||
let lastPercent = -5;
|
||||
const report = (attempt: number) => {
|
||||
const sent = Math.min(job.total, finished + [...live.values()].reduce((a, b) => a + b, 0));
|
||||
const percent = job.total > 0 ? Math.min(99, Math.floor((100 * sent) / job.total)) : 0;
|
||||
const now = Date.now();
|
||||
if (now - lastAt < putTimings.progressMs && percent < lastPercent + 5) return;
|
||||
[lastAt, lastPercent] = [now, percent];
|
||||
job.onProgress?.({ phase: "upload", percent, sentBytes: sent, totalBytes: job.total, attempt });
|
||||
};
|
||||
return {
|
||||
report,
|
||||
written: (number: number, bytes: number, attempt: number) => (
|
||||
live.set(number, bytes), report(attempt)
|
||||
),
|
||||
finish: (number: number, bytes: number) => (
|
||||
(finished += bytes), live.delete(number), report(1)
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function sendPart(
|
||||
job: PartsUpload,
|
||||
number: number,
|
||||
progress: ReturnType<typeof partsProgress>,
|
||||
): Promise<string> {
|
||||
const { client, asset_id, plan, body, total, signal } = job;
|
||||
const range = partRange(plan, total, number);
|
||||
return putWithRetry(
|
||||
async (tried: PutTry) => {
|
||||
const minted = await client.getAssetUploadPartUrls({
|
||||
asset_id,
|
||||
body: { multipart_upload_id: plan.multipart_upload_id, part_numbers: [number] },
|
||||
signal,
|
||||
});
|
||||
const url = minted.parts.find((part) => part.part_number === number)?.upload_url;
|
||||
if (!url) throw new Error(`The cloud gave no address for part ${number}`);
|
||||
progress.written(number, 0, tried.attempt);
|
||||
const etag = await putStreamToPresignedUrl(
|
||||
url,
|
||||
{},
|
||||
body,
|
||||
signal,
|
||||
(written) => {
|
||||
tried.sent = written;
|
||||
progress.written(number, Math.min(written, range.length), tried.attempt);
|
||||
},
|
||||
range,
|
||||
);
|
||||
if (!etag) throw new Error(`The cloud gave no ETag for part ${number}`);
|
||||
return etag;
|
||||
},
|
||||
range.length,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
@@ -125,12 +125,23 @@ export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
// same way, in slices, so a stalled link ends its try the same way (a plain fetch has no progress to watch).
|
||||
const SLICE_BYTES = 64 * 1024;
|
||||
|
||||
function bodyStream(body: Uint8Array | UploadFileSource): Readable {
|
||||
if (!(body instanceof Uint8Array)) return createReadStream(body.path);
|
||||
export interface ByteRange {
|
||||
start: number;
|
||||
length: number;
|
||||
}
|
||||
|
||||
function bodyStream(body: Uint8Array | UploadFileSource, range?: ByteRange): Readable {
|
||||
if (!(body instanceof Uint8Array))
|
||||
return createReadStream(
|
||||
body.path,
|
||||
range && { start: range.start, end: range.start + range.length - 1 },
|
||||
);
|
||||
const from = range?.start ?? 0;
|
||||
const to = from + (range?.length ?? body.byteLength);
|
||||
return Readable.from(
|
||||
(function* () {
|
||||
for (let at = 0; at < body.byteLength; at += SLICE_BYTES)
|
||||
yield body.subarray(at, at + SLICE_BYTES);
|
||||
for (let at = from; at < to; at += SLICE_BYTES)
|
||||
yield body.subarray(at, Math.min(at + SLICE_BYTES, to));
|
||||
})(),
|
||||
);
|
||||
}
|
||||
@@ -159,8 +170,9 @@ export async function putStreamToPresignedUrl(
|
||||
body: Uint8Array | UploadFileSource,
|
||||
signal: AbortSignal | undefined,
|
||||
onSample: (written: number) => void,
|
||||
): Promise<void> {
|
||||
const size = body instanceof Uint8Array ? body.byteLength : body.size;
|
||||
range?: ByteRange,
|
||||
): Promise<string | undefined> {
|
||||
const size = range?.length ?? (body instanceof Uint8Array ? body.byteLength : body.size);
|
||||
const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest;
|
||||
const req = send(uploadUrl, {
|
||||
method: "PUT",
|
||||
@@ -171,7 +183,7 @@ export async function putStreamToPresignedUrl(
|
||||
req.once("response", resolve).once("error", reject);
|
||||
});
|
||||
const stopWatching = watchStall(req, onSample);
|
||||
const source = bodyStream(body);
|
||||
const source = bodyStream(body, range);
|
||||
const sent = pipeline(source, req);
|
||||
let res: IncomingMessage;
|
||||
try {
|
||||
@@ -183,6 +195,7 @@ export async function putStreamToPresignedUrl(
|
||||
stopWatching();
|
||||
}
|
||||
await settle(res, sent);
|
||||
return String(res.headers.etag ?? "") || undefined;
|
||||
}
|
||||
|
||||
/** Reads S3's answer: a refusal is thrown with its words; a refused upload can close the connection mid-body, so the
|
||||
|
||||
Reference in New Issue
Block a user