diff --git a/packages/cli/src/cloud/upload.test.ts b/packages/cli/src/cloud/upload.test.ts index f11162d201..66f9f8f9a9 100644 --- a/packages/cli/src/cloud/upload.test.ts +++ b/packages/cli/src/cloud/upload.test.ts @@ -7,7 +7,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { HyperframesApiError } from "./_gen/client.js"; import type { HyperframesCloudClient } from "./_gen/client.js"; -import { putTimings, uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "./upload.js"; +import { + putTimings, + uploadAssetViaDirectUpload, + uploadZipViaDirectUpload, + type UploadProgressEvent, +} from "./upload.js"; function sha256Hex(bytes: Uint8Array): string { return createHash("sha256").update(bytes).digest("hex"); @@ -372,19 +377,32 @@ describe("uploadAssetViaDirectUpload from a file on disk", () => { }); }); +const BIG_BYTES = 32 * 1024 * 1024; + describe("a PUT that stalls or fails on the network is sent again", () => { - /** A server that answers its Nth request as `plan[n]` ("hang" never answers; a number is the status) and keeps - * the last answer for any later one. */ + /** Answers its Nth request as `plan[n]`, the last one for any later: "hang" never answers, "cut" drops the + * connection at 400 ms, "slow" drains slowly and answers 200, a number is the status. */ async function withPlan( - plan: Array<"hang" | number>, + plan: Array<"hang" | "cut" | "slow" | number>, run: (url: string, requests: () => number) => Promise, ): Promise { let n = 0; const open: Array<() => void> = []; const server = createServer(async (req, res) => { const answer = plan[Math.min(n++, plan.length - 1)]!; - for await (const _ of req) void _; + const born = Date.now(); + try { + for await (const _ of req) { + void _; + if (answer !== "cut" && answer !== "slow") continue; + await new Promise((resolve) => setTimeout(resolve, 1)); + if (answer === "cut" && Date.now() - born >= 400) return void req.socket.destroy(); + } + } catch { + return; + } if (answer === "hang") return void open.push(() => res.destroy()); + if (answer === "cut" || answer === "slow") return void res.writeHead(200).end(); res.writeHead(answer).end(answer === 200 ? "" : "refused"); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); @@ -396,12 +414,17 @@ describe("a PUT that stalls or fails on the network is sent again", () => { } } - const upload = (url: string, asBytes = false) => { + const upload = ( + url: string, + asBytes = false, + big?: { onProgress: (event: UploadProgressEvent) => void }, + ) => { const dir = mkdtempSync(join(tmpdir(), "hf-upload-retry-")); onTestFinished(() => rmSync(dir, { recursive: true, force: true })); - writeFileSync(join(dir, "clip.mp4"), "bytes"); + const content = big ? Buffer.alloc(BIG_BYTES, 1) : Buffer.from("bytes"); + writeFileSync(join(dir, "clip.mp4"), content); const was = { ...putTimings }; - Object.assign(putTimings, { stallMs: 150, backoffMs: 5 }); + Object.assign(putTimings, { stallMs: 150, backoffMs: 5, progressMs: 50 }); onTestFinished(() => void Object.assign(putTimings, was)); return uploadAssetViaDirectUpload({ client: makeClient({ @@ -414,9 +437,12 @@ describe("a PUT that stalls or fails on the network is sent again", () => { status: "pending_upload" as const, })), } as Partial), - ...(asBytes ? { bytes: Buffer.from("bytes") } : { path: join(dir, "clip.mp4"), size: 5 }), + ...(asBytes + ? { bytes: Buffer.from("bytes") } + : { path: join(dir, "clip.mp4"), size: content.byteLength }), filename: "clip.mp4", contentType: "video/mp4", + onProgress: big?.onProgress, }); }; @@ -451,4 +477,35 @@ describe("a PUT that stalls or fails on the network is sent again", () => { expect(requests()).toBe(1); }); }); + it("stops after a second identical cut instead of repeating the doomed try", async () => { + await withPlan(["cut"], async (url, requests) => { + await expect(upload(url, false, { onProgress: () => undefined })).rejects.toThrow( + /cut at \d+ s on two tries in a row.*Every try restarts the file/, + ); + expect(requests()).toBe(2); + }); + }); + + it("reports the bytes sent while a slow PUT runs, and the restart of a second try", async () => { + const events: UploadProgressEvent[] = []; + await withPlan([503, "slow"], async (url) => { + await upload(url, false, { onProgress: (event) => events.push(event) }); + }); + const sent = events.flatMap((e) => (e.phase === "upload" && e.attempt === 2 ? [e] : [])); + const live = sent.filter((e) => (e.sentBytes ?? 0) > 0); + expect(live.length).toBeGreaterThan(1); + expect(live.map((e) => e.sentBytes)).toEqual( + [...live.map((e) => e.sentBytes)].sort((a, b) => a! - b!), + ); + expect(live.every((e) => e.totalBytes === BIG_BYTES)).toBe(true); + expect(events.some((e) => e.phase === "upload" && e.attempt === 2 && e.sentBytes === 0)).toBe( + true, + ); + }); + + it("says how many tries it made and how far the last got when the retries run out", async () => { + await withPlan([503], async (url) => { + await expect(upload(url)).rejects.toThrow(/503.*4 tries, each from the first byte/); + }); + }); }); diff --git a/packages/cli/src/cloud/upload.ts b/packages/cli/src/cloud/upload.ts index 646f3fb47f..cb2e643281 100644 --- a/packages/cli/src/cloud/upload.ts +++ b/packages/cli/src/cloud/upload.ts @@ -41,7 +41,15 @@ export interface UploadZipViaDirectResult { export type UploadProgressEvent = | { phase: "initialize" } - | { phase: "upload"; percent: number } + | { + phase: "upload"; + percent: number; + /** Bytes the connection has accepted so far in this try; the very last ones may still be in flight. */ + sentBytes?: number; + totalBytes?: number; + /** 1 for the first try; a higher number means the file restarted from its first byte. */ + attempt?: number; + } | { phase: "complete" }; /** A file on disk, hashed and uploaded as a stream so it is never held in memory. */ @@ -68,7 +76,43 @@ const COMPLETE_RETRY_BASE_MS = 500; const CONTENT_TYPE_ZIP = "application/zip"; /** A failed or stalled PUT is sent again to the same presigned URL (it stays valid and a PUT replaces the object). */ -export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000 }; +export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000, progressMs: 10_000 }; + +/** What one try of the PUT got through, read by the retry loop and the progress lines. */ +interface PutTry { + attempt: number; + startedAt: number; + sent: number; +} + +const CUT_CODES = new Set(["ECONNRESET", "EPIPE", "ECONNABORTED"]); + +function isConnectionCut(err: unknown): boolean { + const code = (err as { code?: unknown }).code; + return (typeof code === "string" && CUT_CODES.has(code)) || /socket hang up/.test(String(err)); +} + +/** The same cut twice: a connection that dies at the same age with the same bytes sent dies the same way again. */ +function isSameCut(before: PutTry & { ageMs: number }, now: PutTry & { ageMs: number }): boolean { + const near = (a: number, b: number) => Math.abs(a - b) <= 0.2 * Math.max(a, b); + return before.sent > 0 && near(before.ageMs, now.ageMs) && near(before.sent, now.sent); +} + +function seconds(ms: number): string { + return `${Math.round(ms / 1000)} s`; +} + +function exhausted(err: unknown, tries: number, last: PutTry & { ageMs: number }, total: number) { + const base = err instanceof Error ? err.message : String(err); + const percent = total > 0 ? Math.floor((100 * last.sent) / total) : 0; + return Object.assign( + new Error( + `${base} (${tries} tries, each from the first byte; the last got ${percent}% of the file through in ${seconds(last.ageMs)})`, + { cause: err }, + ), + { code: (err as { code?: unknown }).code }, + ); +} /** S3's answer to a PUT it refused; a refusal below 500 is final (an expired signature stays expired). */ class PutRefusedError extends Error { @@ -88,17 +132,59 @@ function isFinal(err: unknown): boolean { ); } -async function putWithRetry(attempt: () => Promise, signal?: AbortSignal): Promise { +async function putWithRetry( + attempt: (tried: PutTry) => Promise, + total: number, + signal?: AbortSignal, +): Promise { + let before: (PutTry & { ageMs: number }) | undefined; + let cutBefore = false; for (let tried = 0; ; tried++) { + const current: PutTry = { attempt: tried + 1, startedAt: Date.now(), sent: 0 }; try { - return await attempt(); + return await attempt(current); } catch (err) { - if (isFinal(err) || signal?.aborted || tried >= putTimings.retries) throw err; + if (isFinal(err) || signal?.aborted) throw err; + const now = { ...current, ageMs: Date.now() - current.startedAt }; + const cut = isConnectionCut(err); + if (cut && cutBefore && before && isSameCut(before, now)) { + throw Object.assign( + new Error( + `The connection was cut at ${seconds(now.ageMs)} on two tries in a row, after ${Math.floor((100 * now.sent) / total)}% of the file was sent (${(err as { code?: string }).code ?? "connection reset"}). Every try restarts the file, so this link cannot finish it before the cut: use a faster connection or send less media.`, + { cause: err }, + ), + { code: (err as { code?: unknown }).code }, + ); + } + if (tried >= putTimings.retries) throw exhausted(err, tried + 1, now, total); + [before, cutBefore] = [now, cut]; await sleep(putTimings.backoffMs * 2 ** tried, signal); } } } +/** Reports the bytes a try has sent at most every progressMs, or when it gains 5 points. */ +function progressReporter( + total: number, + emit: ((event: UploadProgressEvent) => void) | undefined, +): (tried: PutTry) => void { + let lastAt = Date.now(); + let lastPercent = 0; + return (tried) => { + const percent = total > 0 ? Math.min(99, Math.floor((100 * tried.sent) / total)) : 0; + const now = Date.now(); + if (now - lastAt < putTimings.progressMs && percent < lastPercent + 5) return; + [lastAt, lastPercent] = [now, percent]; + emit?.({ + phase: "upload", + percent, + sentBytes: Math.min(tried.sent, total), + totalBytes: total, + attempt: tried.attempt, + }); + }; +} + function sha256Hex(bytes: Uint8Array): string { return createHash("sha256").update(bytes).digest("hex"); } @@ -141,16 +227,32 @@ async function putBytesToPresignedUrl( body: Uint8Array | UploadFileSource, contentType: string, signal?: AbortSignal, + onProgress?: (event: UploadProgressEvent) => void, ): Promise { + const total = body instanceof Uint8Array ? body.byteLength : body.size; + const report = progressReporter(total, onProgress); const headers: Record = { "content-type": contentType, ...normalizeUploadHeaders(uploadHeaders), }; return putWithRetry( - () => - fetchImpl && body instanceof Uint8Array + (tried) => { + if (tried.attempt > 1) + onProgress?.({ + phase: "upload", + percent: 0, + sentBytes: 0, + totalBytes: total, + attempt: tried.attempt, + }); + return fetchImpl && body instanceof Uint8Array ? putBytesOnce(fetchImpl, uploadUrl, headers, body, signal) - : putStreamToPresignedUrl(uploadUrl, headers, body, signal), + : putStreamToPresignedUrl(uploadUrl, headers, body, signal, (written) => { + tried.sent = written; + report(tried); + }); + }, + total, signal, ); } @@ -198,12 +300,13 @@ function bodyStream(body: Uint8Array | UploadFileSource): Readable { /** Ends `req` when no byte has been accepted by its socket for stallMs (its buffer is full and the link drains * nothing). Returns the function that stops the watch. */ -function watchStall(req: ClientRequest): () => void { +function watchStall(req: ClientRequest, onSample: (written: number) => void): () => void { let written = -1; let movedAt = Date.now(); const watch = setInterval( () => { const now = req.socket?.bytesWritten ?? 0; + onSample(now); if (now !== written) [written, movedAt] = [now, Date.now()]; else if (Date.now() - movedAt >= putTimings.stallMs) req.destroy(new Error(`Upload stalled: no data sent for ${putTimings.stallMs / 1000} s`)); @@ -217,7 +320,8 @@ async function putStreamToPresignedUrl( uploadUrl: string, headers: Record, body: Uint8Array | UploadFileSource, - signal?: AbortSignal, + signal: AbortSignal | undefined, + onSample: (written: number) => void, ): Promise { const size = body instanceof Uint8Array ? body.byteLength : body.size; const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest; @@ -229,7 +333,7 @@ async function putStreamToPresignedUrl( const response = new Promise((resolve, reject) => { req.once("response", resolve).once("error", reject); }); - const stopWatching = watchStall(req); + const stopWatching = watchStall(req, onSample); const source = bodyStream(body); const sent = pipeline(source, req); let res: IncomingMessage; @@ -333,6 +437,7 @@ export async function uploadAssetViaDirectUpload( body, opts.contentType, opts.signal, + opts.onProgress, ); opts.onProgress?.({ phase: "upload", percent: 100 }); diff --git a/packages/cli/src/commands/cloud/render.ts b/packages/cli/src/commands/cloud/render.ts index f1657b9cd5..16a406fd4a 100644 --- a/packages/cli/src/commands/cloud/render.ts +++ b/packages/cli/src/commands/cloud/render.ts @@ -75,7 +75,11 @@ import { import { reportApiError } from "../../cloud/errors.js"; import { describeSwappedFonts, embedSystemFonts } from "../../cloud/fontPreflight.js"; import { parseEnumFlag, parseIntFlag, parseNumericFlag } from "../../cloud/parsing.js"; -import { uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "../../cloud/upload.js"; +import { + uploadAssetViaDirectUpload, + uploadZipViaDirectUpload, + type UploadProgressEvent, +} from "../../cloud/upload.js"; import { OFFLOAD_MIN_BYTES, offloadHeavyMedia, @@ -292,7 +296,12 @@ export default defineCommand({ project, args.composition, asJson, - (file) => uploadAssetViaDirectUpload({ client, ...file }), + (file) => + uploadAssetViaDirectUpload({ + client, + ...file, + onProgress: reportUploadProgress(file.filename, asJson), + }), variables && JSON.stringify(variables), ) : undefined; @@ -795,6 +804,33 @@ export function assertArchiveFits(sizeBytes: number, largestFiles: ArchiveFileSu failCommand(); } +/** A line per PUT progress event: JSON on stderr under --json, so a caller can tell a slow upload from a dead one. */ +function reportUploadProgress(name: string, asJson: boolean): (event: UploadProgressEvent) => void { + return (event) => { + if (event.phase !== "upload" || event.sentBytes === undefined) return; + const { percent, sentBytes, totalBytes, attempt = 1 } = event; + if (asJson) { + console.error( + JSON.stringify({ + event: "upload_progress", + file: name, + attempt, + percent, + sent_bytes: sentBytes, + total_bytes: totalBytes, + }), + ); + } else { + const retry = attempt > 1 ? ` (try ${attempt}, from the first byte)` : ""; + console.log( + c.dim( + ` ${name}: ${percent}% · ${formatBytes(sentBytes)} of ${formatBytes(totalBytes ?? 0)}${retry}`, + ), + ); + } + }; +} + function describeOffload(item: OffloadedMedia, dryRun: boolean): string { const verb = item.action === "cached" @@ -880,17 +916,17 @@ async function maybeUploadProject( bytes: archive.buffer, filename: `${project.name}.zip`, idempotencyKey, - onProgress: !asJson - ? (ev) => { - if (ev.phase === "initialize") { - console.log(c.dim(` initializing…`)); - } else if (ev.phase === "upload" && ev.percent === 0) { - console.log(c.dim(` uploading to S3…`)); - } else if (ev.phase === "complete") { - console.log(c.dim(` finalizing…`)); - } - } - : undefined, + onProgress: (ev) => { + reportUploadProgress(`${project.name}.zip`, asJson)(ev); + if (asJson) return; + if (ev.phase === "initialize") { + console.log(c.dim(` initializing…`)); + } else if (ev.phase === "upload" && ev.percent === 0 && ev.attempt === undefined) { + console.log(c.dim(` uploading to S3…`)); + } else if (ev.phase === "complete") { + console.log(c.dim(` finalizing…`)); + } + }, }); } catch (err) { reportApiError("Upload failed", err);