fix(cli): a cloud upload cut at a fixed age stops early and shows its progress (#7350)

* fix(cli): a cloud upload cut at a fixed age stops after two identical cuts and shows its progress

* test(cli): size the upload tests so the kernel cannot buffer the whole file

* test(cli): keep the upload test server quiet when the client drops the connection

* test(cli): shorten the upload test server comment
This commit is contained in:
Miguel Ángel
2026-11-07 04:45:44 +00:00
committed by GitHub
parent bc2773d047
commit 65c62f3145
3 changed files with 231 additions and 33 deletions
+66 -9
View File
@@ -7,7 +7,12 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { HyperframesApiError } from "./_gen/client.js";
import type { HyperframesCloudClient } from "./_gen/client.js";
import { putTimings, uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "./upload.js";
import {
putTimings,
uploadAssetViaDirectUpload,
uploadZipViaDirectUpload,
type UploadProgressEvent,
} from "./upload.js";
function sha256Hex(bytes: Uint8Array): string {
return createHash("sha256").update(bytes).digest("hex");
@@ -372,19 +377,32 @@ describe("uploadAssetViaDirectUpload from a file on disk", () => {
});
});
const BIG_BYTES = 32 * 1024 * 1024;
describe("a PUT that stalls or fails on the network is sent again", () => {
/** A server that answers its Nth request as `plan[n]` ("hang" never answers; a number is the status) and keeps
* the last answer for any later one. */
/** Answers its Nth request as `plan[n]`, the last one for any later: "hang" never answers, "cut" drops the
* connection at 400 ms, "slow" drains slowly and answers 200, a number is the status. */
async function withPlan(
plan: Array<"hang" | number>,
plan: Array<"hang" | "cut" | "slow" | number>,
run: (url: string, requests: () => number) => Promise<void>,
): Promise<void> {
let n = 0;
const open: Array<() => void> = [];
const server = createServer(async (req, res) => {
const answer = plan[Math.min(n++, plan.length - 1)]!;
for await (const _ of req) void _;
const born = Date.now();
try {
for await (const _ of req) {
void _;
if (answer !== "cut" && answer !== "slow") continue;
await new Promise((resolve) => setTimeout(resolve, 1));
if (answer === "cut" && Date.now() - born >= 400) return void req.socket.destroy();
}
} catch {
return;
}
if (answer === "hang") return void open.push(() => res.destroy());
if (answer === "cut" || answer === "slow") return void res.writeHead(200).end();
res.writeHead(answer).end(answer === 200 ? "" : "refused");
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
@@ -396,12 +414,17 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
}
}
const upload = (url: string, asBytes = false) => {
const upload = (
url: string,
asBytes = false,
big?: { onProgress: (event: UploadProgressEvent) => void },
) => {
const dir = mkdtempSync(join(tmpdir(), "hf-upload-retry-"));
onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
writeFileSync(join(dir, "clip.mp4"), "bytes");
const content = big ? Buffer.alloc(BIG_BYTES, 1) : Buffer.from("bytes");
writeFileSync(join(dir, "clip.mp4"), content);
const was = { ...putTimings };
Object.assign(putTimings, { stallMs: 150, backoffMs: 5 });
Object.assign(putTimings, { stallMs: 150, backoffMs: 5, progressMs: 50 });
onTestFinished(() => void Object.assign(putTimings, was));
return uploadAssetViaDirectUpload({
client: makeClient({
@@ -414,9 +437,12 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
status: "pending_upload" as const,
})),
} as Partial<HyperframesCloudClient>),
...(asBytes ? { bytes: Buffer.from("bytes") } : { path: join(dir, "clip.mp4"), size: 5 }),
...(asBytes
? { bytes: Buffer.from("bytes") }
: { path: join(dir, "clip.mp4"), size: content.byteLength }),
filename: "clip.mp4",
contentType: "video/mp4",
onProgress: big?.onProgress,
});
};
@@ -451,4 +477,35 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
expect(requests()).toBe(1);
});
});
it("stops after a second identical cut instead of repeating the doomed try", async () => {
await withPlan(["cut"], async (url, requests) => {
await expect(upload(url, false, { onProgress: () => undefined })).rejects.toThrow(
/cut at \d+ s on two tries in a row.*Every try restarts the file/,
);
expect(requests()).toBe(2);
});
});
it("reports the bytes sent while a slow PUT runs, and the restart of a second try", async () => {
const events: UploadProgressEvent[] = [];
await withPlan([503, "slow"], async (url) => {
await upload(url, false, { onProgress: (event) => events.push(event) });
});
const sent = events.flatMap((e) => (e.phase === "upload" && e.attempt === 2 ? [e] : []));
const live = sent.filter((e) => (e.sentBytes ?? 0) > 0);
expect(live.length).toBeGreaterThan(1);
expect(live.map((e) => e.sentBytes)).toEqual(
[...live.map((e) => e.sentBytes)].sort((a, b) => a! - b!),
);
expect(live.every((e) => e.totalBytes === BIG_BYTES)).toBe(true);
expect(events.some((e) => e.phase === "upload" && e.attempt === 2 && e.sentBytes === 0)).toBe(
true,
);
});
it("says how many tries it made and how far the last got when the retries run out", async () => {
await withPlan([503], async (url) => {
await expect(upload(url)).rejects.toThrow(/503.*4 tries, each from the first byte/);
});
});
});
+116 -11
View File
@@ -41,7 +41,15 @@ export interface UploadZipViaDirectResult {
export type UploadProgressEvent =
| { phase: "initialize" }
| { phase: "upload"; percent: number }
| {
phase: "upload";
percent: number;
/** Bytes the connection has accepted so far in this try; the very last ones may still be in flight. */
sentBytes?: number;
totalBytes?: number;
/** 1 for the first try; a higher number means the file restarted from its first byte. */
attempt?: number;
}
| { phase: "complete" };
/** A file on disk, hashed and uploaded as a stream so it is never held in memory. */
@@ -68,7 +76,43 @@ const COMPLETE_RETRY_BASE_MS = 500;
const CONTENT_TYPE_ZIP = "application/zip";
/** A failed or stalled PUT is sent again to the same presigned URL (it stays valid and a PUT replaces the object). */
export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000 };
export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000, progressMs: 10_000 };
/** What one try of the PUT got through, read by the retry loop and the progress lines. */
interface PutTry {
attempt: number;
startedAt: number;
sent: number;
}
const CUT_CODES = new Set(["ECONNRESET", "EPIPE", "ECONNABORTED"]);
function isConnectionCut(err: unknown): boolean {
const code = (err as { code?: unknown }).code;
return (typeof code === "string" && CUT_CODES.has(code)) || /socket hang up/.test(String(err));
}
/** The same cut twice: a connection that dies at the same age with the same bytes sent dies the same way again. */
function isSameCut(before: PutTry & { ageMs: number }, now: PutTry & { ageMs: number }): boolean {
const near = (a: number, b: number) => Math.abs(a - b) <= 0.2 * Math.max(a, b);
return before.sent > 0 && near(before.ageMs, now.ageMs) && near(before.sent, now.sent);
}
function seconds(ms: number): string {
return `${Math.round(ms / 1000)} s`;
}
function exhausted(err: unknown, tries: number, last: PutTry & { ageMs: number }, total: number) {
const base = err instanceof Error ? err.message : String(err);
const percent = total > 0 ? Math.floor((100 * last.sent) / total) : 0;
return Object.assign(
new Error(
`${base} (${tries} tries, each from the first byte; the last got ${percent}% of the file through in ${seconds(last.ageMs)})`,
{ cause: err },
),
{ code: (err as { code?: unknown }).code },
);
}
/** S3's answer to a PUT it refused; a refusal below 500 is final (an expired signature stays expired). */
class PutRefusedError extends Error {
@@ -88,17 +132,59 @@ function isFinal(err: unknown): boolean {
);
}
async function putWithRetry(attempt: () => Promise<void>, signal?: AbortSignal): Promise<void> {
async function putWithRetry(
attempt: (tried: PutTry) => Promise<void>,
total: number,
signal?: AbortSignal,
): Promise<void> {
let before: (PutTry & { ageMs: number }) | undefined;
let cutBefore = false;
for (let tried = 0; ; tried++) {
const current: PutTry = { attempt: tried + 1, startedAt: Date.now(), sent: 0 };
try {
return await attempt();
return await attempt(current);
} catch (err) {
if (isFinal(err) || signal?.aborted || tried >= putTimings.retries) throw err;
if (isFinal(err) || signal?.aborted) throw err;
const now = { ...current, ageMs: Date.now() - current.startedAt };
const cut = isConnectionCut(err);
if (cut && cutBefore && before && isSameCut(before, now)) {
throw Object.assign(
new Error(
`The connection was cut at ${seconds(now.ageMs)} on two tries in a row, after ${Math.floor((100 * now.sent) / total)}% of the file was sent (${(err as { code?: string }).code ?? "connection reset"}). Every try restarts the file, so this link cannot finish it before the cut: use a faster connection or send less media.`,
{ cause: err },
),
{ code: (err as { code?: unknown }).code },
);
}
if (tried >= putTimings.retries) throw exhausted(err, tried + 1, now, total);
[before, cutBefore] = [now, cut];
await sleep(putTimings.backoffMs * 2 ** tried, signal);
}
}
}
/** Reports the bytes a try has sent at most every progressMs, or when it gains 5 points. */
function progressReporter(
total: number,
emit: ((event: UploadProgressEvent) => void) | undefined,
): (tried: PutTry) => void {
let lastAt = Date.now();
let lastPercent = 0;
return (tried) => {
const percent = total > 0 ? Math.min(99, Math.floor((100 * tried.sent) / total)) : 0;
const now = Date.now();
if (now - lastAt < putTimings.progressMs && percent < lastPercent + 5) return;
[lastAt, lastPercent] = [now, percent];
emit?.({
phase: "upload",
percent,
sentBytes: Math.min(tried.sent, total),
totalBytes: total,
attempt: tried.attempt,
});
};
}
function sha256Hex(bytes: Uint8Array): string {
return createHash("sha256").update(bytes).digest("hex");
}
@@ -141,16 +227,32 @@ async function putBytesToPresignedUrl(
body: Uint8Array | UploadFileSource,
contentType: string,
signal?: AbortSignal,
onProgress?: (event: UploadProgressEvent) => void,
): Promise<void> {
const total = body instanceof Uint8Array ? body.byteLength : body.size;
const report = progressReporter(total, onProgress);
const headers: Record<string, string> = {
"content-type": contentType,
...normalizeUploadHeaders(uploadHeaders),
};
return putWithRetry(
() =>
fetchImpl && body instanceof Uint8Array
(tried) => {
if (tried.attempt > 1)
onProgress?.({
phase: "upload",
percent: 0,
sentBytes: 0,
totalBytes: total,
attempt: tried.attempt,
});
return fetchImpl && body instanceof Uint8Array
? putBytesOnce(fetchImpl, uploadUrl, headers, body, signal)
: putStreamToPresignedUrl(uploadUrl, headers, body, signal),
: putStreamToPresignedUrl(uploadUrl, headers, body, signal, (written) => {
tried.sent = written;
report(tried);
});
},
total,
signal,
);
}
@@ -198,12 +300,13 @@ function bodyStream(body: Uint8Array | UploadFileSource): Readable {
/** Ends `req` when no byte has been accepted by its socket for stallMs (its buffer is full and the link drains
* nothing). Returns the function that stops the watch. */
function watchStall(req: ClientRequest): () => void {
function watchStall(req: ClientRequest, onSample: (written: number) => void): () => void {
let written = -1;
let movedAt = Date.now();
const watch = setInterval(
() => {
const now = req.socket?.bytesWritten ?? 0;
onSample(now);
if (now !== written) [written, movedAt] = [now, Date.now()];
else if (Date.now() - movedAt >= putTimings.stallMs)
req.destroy(new Error(`Upload stalled: no data sent for ${putTimings.stallMs / 1000} s`));
@@ -217,7 +320,8 @@ async function putStreamToPresignedUrl(
uploadUrl: string,
headers: Record<string, string>,
body: Uint8Array | UploadFileSource,
signal?: AbortSignal,
signal: AbortSignal | undefined,
onSample: (written: number) => void,
): Promise<void> {
const size = body instanceof Uint8Array ? body.byteLength : body.size;
const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest;
@@ -229,7 +333,7 @@ async function putStreamToPresignedUrl(
const response = new Promise<IncomingMessage>((resolve, reject) => {
req.once("response", resolve).once("error", reject);
});
const stopWatching = watchStall(req);
const stopWatching = watchStall(req, onSample);
const source = bodyStream(body);
const sent = pipeline(source, req);
let res: IncomingMessage;
@@ -333,6 +437,7 @@ export async function uploadAssetViaDirectUpload(
body,
opts.contentType,
opts.signal,
opts.onProgress,
);
opts.onProgress?.({ phase: "upload", percent: 100 });
+49 -13
View File
@@ -75,7 +75,11 @@ import {
import { reportApiError } from "../../cloud/errors.js";
import { describeSwappedFonts, embedSystemFonts } from "../../cloud/fontPreflight.js";
import { parseEnumFlag, parseIntFlag, parseNumericFlag } from "../../cloud/parsing.js";
import { uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "../../cloud/upload.js";
import {
uploadAssetViaDirectUpload,
uploadZipViaDirectUpload,
type UploadProgressEvent,
} from "../../cloud/upload.js";
import {
OFFLOAD_MIN_BYTES,
offloadHeavyMedia,
@@ -292,7 +296,12 @@ export default defineCommand({
project,
args.composition,
asJson,
(file) => uploadAssetViaDirectUpload({ client, ...file }),
(file) =>
uploadAssetViaDirectUpload({
client,
...file,
onProgress: reportUploadProgress(file.filename, asJson),
}),
variables && JSON.stringify(variables),
)
: undefined;
@@ -795,6 +804,33 @@ export function assertArchiveFits(sizeBytes: number, largestFiles: ArchiveFileSu
failCommand();
}
/** A line per PUT progress event: JSON on stderr under --json, so a caller can tell a slow upload from a dead one. */
function reportUploadProgress(name: string, asJson: boolean): (event: UploadProgressEvent) => void {
return (event) => {
if (event.phase !== "upload" || event.sentBytes === undefined) return;
const { percent, sentBytes, totalBytes, attempt = 1 } = event;
if (asJson) {
console.error(
JSON.stringify({
event: "upload_progress",
file: name,
attempt,
percent,
sent_bytes: sentBytes,
total_bytes: totalBytes,
}),
);
} else {
const retry = attempt > 1 ? ` (try ${attempt}, from the first byte)` : "";
console.log(
c.dim(
` ${name}: ${percent}% · ${formatBytes(sentBytes)} of ${formatBytes(totalBytes ?? 0)}${retry}`,
),
);
}
};
}
function describeOffload(item: OffloadedMedia, dryRun: boolean): string {
const verb =
item.action === "cached"
@@ -880,17 +916,17 @@ async function maybeUploadProject(
bytes: archive.buffer,
filename: `${project.name}.zip`,
idempotencyKey,
onProgress: !asJson
? (ev) => {
if (ev.phase === "initialize") {
console.log(c.dim(` initializing…`));
} else if (ev.phase === "upload" && ev.percent === 0) {
console.log(c.dim(` uploading to S3…`));
} else if (ev.phase === "complete") {
console.log(c.dim(` finalizing…`));
}
}
: undefined,
onProgress: (ev) => {
reportUploadProgress(`${project.name}.zip`, asJson)(ev);
if (asJson) return;
if (ev.phase === "initialize") {
console.log(c.dim(` initializing…`));
} else if (ev.phase === "upload" && ev.percent === 0 && ev.attempt === undefined) {
console.log(c.dim(` uploading to S3…`));
} else if (ev.phase === "complete") {
console.log(c.dim(` finalizing…`));
}
},
});
} catch (err) {
reportApiError("Upload failed", err);