mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): a cloud upload cut at a fixed age stops early and shows its progress (#7350)
* fix(cli): a cloud upload cut at a fixed age stops after two identical cuts and shows its progress * test(cli): size the upload tests so the kernel cannot buffer the whole file * test(cli): keep the upload test server quiet when the client drops the connection * test(cli): shorten the upload test server comment
This commit is contained in:
@@ -7,7 +7,12 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { HyperframesApiError } from "./_gen/client.js";
|
||||
import type { HyperframesCloudClient } from "./_gen/client.js";
|
||||
import { putTimings, uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "./upload.js";
|
||||
import {
|
||||
putTimings,
|
||||
uploadAssetViaDirectUpload,
|
||||
uploadZipViaDirectUpload,
|
||||
type UploadProgressEvent,
|
||||
} from "./upload.js";
|
||||
|
||||
function sha256Hex(bytes: Uint8Array): string {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
@@ -372,19 +377,32 @@ describe("uploadAssetViaDirectUpload from a file on disk", () => {
|
||||
});
|
||||
});
|
||||
|
||||
const BIG_BYTES = 32 * 1024 * 1024;
|
||||
|
||||
describe("a PUT that stalls or fails on the network is sent again", () => {
|
||||
/** A server that answers its Nth request as `plan[n]` ("hang" never answers; a number is the status) and keeps
|
||||
* the last answer for any later one. */
|
||||
/** Answers its Nth request as `plan[n]`, the last one for any later: "hang" never answers, "cut" drops the
|
||||
* connection at 400 ms, "slow" drains slowly and answers 200, a number is the status. */
|
||||
async function withPlan(
|
||||
plan: Array<"hang" | number>,
|
||||
plan: Array<"hang" | "cut" | "slow" | number>,
|
||||
run: (url: string, requests: () => number) => Promise<void>,
|
||||
): Promise<void> {
|
||||
let n = 0;
|
||||
const open: Array<() => void> = [];
|
||||
const server = createServer(async (req, res) => {
|
||||
const answer = plan[Math.min(n++, plan.length - 1)]!;
|
||||
for await (const _ of req) void _;
|
||||
const born = Date.now();
|
||||
try {
|
||||
for await (const _ of req) {
|
||||
void _;
|
||||
if (answer !== "cut" && answer !== "slow") continue;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
if (answer === "cut" && Date.now() - born >= 400) return void req.socket.destroy();
|
||||
}
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (answer === "hang") return void open.push(() => res.destroy());
|
||||
if (answer === "cut" || answer === "slow") return void res.writeHead(200).end();
|
||||
res.writeHead(answer).end(answer === 200 ? "" : "refused");
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
@@ -396,12 +414,17 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
|
||||
}
|
||||
}
|
||||
|
||||
const upload = (url: string, asBytes = false) => {
|
||||
const upload = (
|
||||
url: string,
|
||||
asBytes = false,
|
||||
big?: { onProgress: (event: UploadProgressEvent) => void },
|
||||
) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-upload-retry-"));
|
||||
onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
|
||||
writeFileSync(join(dir, "clip.mp4"), "bytes");
|
||||
const content = big ? Buffer.alloc(BIG_BYTES, 1) : Buffer.from("bytes");
|
||||
writeFileSync(join(dir, "clip.mp4"), content);
|
||||
const was = { ...putTimings };
|
||||
Object.assign(putTimings, { stallMs: 150, backoffMs: 5 });
|
||||
Object.assign(putTimings, { stallMs: 150, backoffMs: 5, progressMs: 50 });
|
||||
onTestFinished(() => void Object.assign(putTimings, was));
|
||||
return uploadAssetViaDirectUpload({
|
||||
client: makeClient({
|
||||
@@ -414,9 +437,12 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
|
||||
status: "pending_upload" as const,
|
||||
})),
|
||||
} as Partial<HyperframesCloudClient>),
|
||||
...(asBytes ? { bytes: Buffer.from("bytes") } : { path: join(dir, "clip.mp4"), size: 5 }),
|
||||
...(asBytes
|
||||
? { bytes: Buffer.from("bytes") }
|
||||
: { path: join(dir, "clip.mp4"), size: content.byteLength }),
|
||||
filename: "clip.mp4",
|
||||
contentType: "video/mp4",
|
||||
onProgress: big?.onProgress,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -451,4 +477,35 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
|
||||
expect(requests()).toBe(1);
|
||||
});
|
||||
});
|
||||
it("stops after a second identical cut instead of repeating the doomed try", async () => {
|
||||
await withPlan(["cut"], async (url, requests) => {
|
||||
await expect(upload(url, false, { onProgress: () => undefined })).rejects.toThrow(
|
||||
/cut at \d+ s on two tries in a row.*Every try restarts the file/,
|
||||
);
|
||||
expect(requests()).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
it("reports the bytes sent while a slow PUT runs, and the restart of a second try", async () => {
|
||||
const events: UploadProgressEvent[] = [];
|
||||
await withPlan([503, "slow"], async (url) => {
|
||||
await upload(url, false, { onProgress: (event) => events.push(event) });
|
||||
});
|
||||
const sent = events.flatMap((e) => (e.phase === "upload" && e.attempt === 2 ? [e] : []));
|
||||
const live = sent.filter((e) => (e.sentBytes ?? 0) > 0);
|
||||
expect(live.length).toBeGreaterThan(1);
|
||||
expect(live.map((e) => e.sentBytes)).toEqual(
|
||||
[...live.map((e) => e.sentBytes)].sort((a, b) => a! - b!),
|
||||
);
|
||||
expect(live.every((e) => e.totalBytes === BIG_BYTES)).toBe(true);
|
||||
expect(events.some((e) => e.phase === "upload" && e.attempt === 2 && e.sentBytes === 0)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("says how many tries it made and how far the last got when the retries run out", async () => {
|
||||
await withPlan([503], async (url) => {
|
||||
await expect(upload(url)).rejects.toThrow(/503.*4 tries, each from the first byte/);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -41,7 +41,15 @@ export interface UploadZipViaDirectResult {
|
||||
|
||||
export type UploadProgressEvent =
|
||||
| { phase: "initialize" }
|
||||
| { phase: "upload"; percent: number }
|
||||
| {
|
||||
phase: "upload";
|
||||
percent: number;
|
||||
/** Bytes the connection has accepted so far in this try; the very last ones may still be in flight. */
|
||||
sentBytes?: number;
|
||||
totalBytes?: number;
|
||||
/** 1 for the first try; a higher number means the file restarted from its first byte. */
|
||||
attempt?: number;
|
||||
}
|
||||
| { phase: "complete" };
|
||||
|
||||
/** A file on disk, hashed and uploaded as a stream so it is never held in memory. */
|
||||
@@ -68,7 +76,43 @@ const COMPLETE_RETRY_BASE_MS = 500;
|
||||
const CONTENT_TYPE_ZIP = "application/zip";
|
||||
|
||||
/** A failed or stalled PUT is sent again to the same presigned URL (it stays valid and a PUT replaces the object). */
|
||||
export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000 };
|
||||
export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000, progressMs: 10_000 };
|
||||
|
||||
/** What one try of the PUT got through, read by the retry loop and the progress lines. */
|
||||
interface PutTry {
|
||||
attempt: number;
|
||||
startedAt: number;
|
||||
sent: number;
|
||||
}
|
||||
|
||||
const CUT_CODES = new Set(["ECONNRESET", "EPIPE", "ECONNABORTED"]);
|
||||
|
||||
function isConnectionCut(err: unknown): boolean {
|
||||
const code = (err as { code?: unknown }).code;
|
||||
return (typeof code === "string" && CUT_CODES.has(code)) || /socket hang up/.test(String(err));
|
||||
}
|
||||
|
||||
/** The same cut twice: a connection that dies at the same age with the same bytes sent dies the same way again. */
|
||||
function isSameCut(before: PutTry & { ageMs: number }, now: PutTry & { ageMs: number }): boolean {
|
||||
const near = (a: number, b: number) => Math.abs(a - b) <= 0.2 * Math.max(a, b);
|
||||
return before.sent > 0 && near(before.ageMs, now.ageMs) && near(before.sent, now.sent);
|
||||
}
|
||||
|
||||
function seconds(ms: number): string {
|
||||
return `${Math.round(ms / 1000)} s`;
|
||||
}
|
||||
|
||||
function exhausted(err: unknown, tries: number, last: PutTry & { ageMs: number }, total: number) {
|
||||
const base = err instanceof Error ? err.message : String(err);
|
||||
const percent = total > 0 ? Math.floor((100 * last.sent) / total) : 0;
|
||||
return Object.assign(
|
||||
new Error(
|
||||
`${base} (${tries} tries, each from the first byte; the last got ${percent}% of the file through in ${seconds(last.ageMs)})`,
|
||||
{ cause: err },
|
||||
),
|
||||
{ code: (err as { code?: unknown }).code },
|
||||
);
|
||||
}
|
||||
|
||||
/** S3's answer to a PUT it refused; a refusal below 500 is final (an expired signature stays expired). */
|
||||
class PutRefusedError extends Error {
|
||||
@@ -88,17 +132,59 @@ function isFinal(err: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
async function putWithRetry(attempt: () => Promise<void>, signal?: AbortSignal): Promise<void> {
|
||||
async function putWithRetry(
|
||||
attempt: (tried: PutTry) => Promise<void>,
|
||||
total: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
let before: (PutTry & { ageMs: number }) | undefined;
|
||||
let cutBefore = false;
|
||||
for (let tried = 0; ; tried++) {
|
||||
const current: PutTry = { attempt: tried + 1, startedAt: Date.now(), sent: 0 };
|
||||
try {
|
||||
return await attempt();
|
||||
return await attempt(current);
|
||||
} catch (err) {
|
||||
if (isFinal(err) || signal?.aborted || tried >= putTimings.retries) throw err;
|
||||
if (isFinal(err) || signal?.aborted) throw err;
|
||||
const now = { ...current, ageMs: Date.now() - current.startedAt };
|
||||
const cut = isConnectionCut(err);
|
||||
if (cut && cutBefore && before && isSameCut(before, now)) {
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
`The connection was cut at ${seconds(now.ageMs)} on two tries in a row, after ${Math.floor((100 * now.sent) / total)}% of the file was sent (${(err as { code?: string }).code ?? "connection reset"}). Every try restarts the file, so this link cannot finish it before the cut: use a faster connection or send less media.`,
|
||||
{ cause: err },
|
||||
),
|
||||
{ code: (err as { code?: unknown }).code },
|
||||
);
|
||||
}
|
||||
if (tried >= putTimings.retries) throw exhausted(err, tried + 1, now, total);
|
||||
[before, cutBefore] = [now, cut];
|
||||
await sleep(putTimings.backoffMs * 2 ** tried, signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Reports the bytes a try has sent at most every progressMs, or when it gains 5 points. */
|
||||
function progressReporter(
|
||||
total: number,
|
||||
emit: ((event: UploadProgressEvent) => void) | undefined,
|
||||
): (tried: PutTry) => void {
|
||||
let lastAt = Date.now();
|
||||
let lastPercent = 0;
|
||||
return (tried) => {
|
||||
const percent = total > 0 ? Math.min(99, Math.floor((100 * tried.sent) / total)) : 0;
|
||||
const now = Date.now();
|
||||
if (now - lastAt < putTimings.progressMs && percent < lastPercent + 5) return;
|
||||
[lastAt, lastPercent] = [now, percent];
|
||||
emit?.({
|
||||
phase: "upload",
|
||||
percent,
|
||||
sentBytes: Math.min(tried.sent, total),
|
||||
totalBytes: total,
|
||||
attempt: tried.attempt,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function sha256Hex(bytes: Uint8Array): string {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
@@ -141,16 +227,32 @@ async function putBytesToPresignedUrl(
|
||||
body: Uint8Array | UploadFileSource,
|
||||
contentType: string,
|
||||
signal?: AbortSignal,
|
||||
onProgress?: (event: UploadProgressEvent) => void,
|
||||
): Promise<void> {
|
||||
const total = body instanceof Uint8Array ? body.byteLength : body.size;
|
||||
const report = progressReporter(total, onProgress);
|
||||
const headers: Record<string, string> = {
|
||||
"content-type": contentType,
|
||||
...normalizeUploadHeaders(uploadHeaders),
|
||||
};
|
||||
return putWithRetry(
|
||||
() =>
|
||||
fetchImpl && body instanceof Uint8Array
|
||||
(tried) => {
|
||||
if (tried.attempt > 1)
|
||||
onProgress?.({
|
||||
phase: "upload",
|
||||
percent: 0,
|
||||
sentBytes: 0,
|
||||
totalBytes: total,
|
||||
attempt: tried.attempt,
|
||||
});
|
||||
return fetchImpl && body instanceof Uint8Array
|
||||
? putBytesOnce(fetchImpl, uploadUrl, headers, body, signal)
|
||||
: putStreamToPresignedUrl(uploadUrl, headers, body, signal),
|
||||
: putStreamToPresignedUrl(uploadUrl, headers, body, signal, (written) => {
|
||||
tried.sent = written;
|
||||
report(tried);
|
||||
});
|
||||
},
|
||||
total,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
@@ -198,12 +300,13 @@ function bodyStream(body: Uint8Array | UploadFileSource): Readable {
|
||||
|
||||
/** Ends `req` when no byte has been accepted by its socket for stallMs (its buffer is full and the link drains
|
||||
* nothing). Returns the function that stops the watch. */
|
||||
function watchStall(req: ClientRequest): () => void {
|
||||
function watchStall(req: ClientRequest, onSample: (written: number) => void): () => void {
|
||||
let written = -1;
|
||||
let movedAt = Date.now();
|
||||
const watch = setInterval(
|
||||
() => {
|
||||
const now = req.socket?.bytesWritten ?? 0;
|
||||
onSample(now);
|
||||
if (now !== written) [written, movedAt] = [now, Date.now()];
|
||||
else if (Date.now() - movedAt >= putTimings.stallMs)
|
||||
req.destroy(new Error(`Upload stalled: no data sent for ${putTimings.stallMs / 1000} s`));
|
||||
@@ -217,7 +320,8 @@ async function putStreamToPresignedUrl(
|
||||
uploadUrl: string,
|
||||
headers: Record<string, string>,
|
||||
body: Uint8Array | UploadFileSource,
|
||||
signal?: AbortSignal,
|
||||
signal: AbortSignal | undefined,
|
||||
onSample: (written: number) => void,
|
||||
): Promise<void> {
|
||||
const size = body instanceof Uint8Array ? body.byteLength : body.size;
|
||||
const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest;
|
||||
@@ -229,7 +333,7 @@ async function putStreamToPresignedUrl(
|
||||
const response = new Promise<IncomingMessage>((resolve, reject) => {
|
||||
req.once("response", resolve).once("error", reject);
|
||||
});
|
||||
const stopWatching = watchStall(req);
|
||||
const stopWatching = watchStall(req, onSample);
|
||||
const source = bodyStream(body);
|
||||
const sent = pipeline(source, req);
|
||||
let res: IncomingMessage;
|
||||
@@ -333,6 +437,7 @@ export async function uploadAssetViaDirectUpload(
|
||||
body,
|
||||
opts.contentType,
|
||||
opts.signal,
|
||||
opts.onProgress,
|
||||
);
|
||||
opts.onProgress?.({ phase: "upload", percent: 100 });
|
||||
|
||||
|
||||
@@ -75,7 +75,11 @@ import {
|
||||
import { reportApiError } from "../../cloud/errors.js";
|
||||
import { describeSwappedFonts, embedSystemFonts } from "../../cloud/fontPreflight.js";
|
||||
import { parseEnumFlag, parseIntFlag, parseNumericFlag } from "../../cloud/parsing.js";
|
||||
import { uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "../../cloud/upload.js";
|
||||
import {
|
||||
uploadAssetViaDirectUpload,
|
||||
uploadZipViaDirectUpload,
|
||||
type UploadProgressEvent,
|
||||
} from "../../cloud/upload.js";
|
||||
import {
|
||||
OFFLOAD_MIN_BYTES,
|
||||
offloadHeavyMedia,
|
||||
@@ -292,7 +296,12 @@ export default defineCommand({
|
||||
project,
|
||||
args.composition,
|
||||
asJson,
|
||||
(file) => uploadAssetViaDirectUpload({ client, ...file }),
|
||||
(file) =>
|
||||
uploadAssetViaDirectUpload({
|
||||
client,
|
||||
...file,
|
||||
onProgress: reportUploadProgress(file.filename, asJson),
|
||||
}),
|
||||
variables && JSON.stringify(variables),
|
||||
)
|
||||
: undefined;
|
||||
@@ -795,6 +804,33 @@ export function assertArchiveFits(sizeBytes: number, largestFiles: ArchiveFileSu
|
||||
failCommand();
|
||||
}
|
||||
|
||||
/** A line per PUT progress event: JSON on stderr under --json, so a caller can tell a slow upload from a dead one. */
|
||||
function reportUploadProgress(name: string, asJson: boolean): (event: UploadProgressEvent) => void {
|
||||
return (event) => {
|
||||
if (event.phase !== "upload" || event.sentBytes === undefined) return;
|
||||
const { percent, sentBytes, totalBytes, attempt = 1 } = event;
|
||||
if (asJson) {
|
||||
console.error(
|
||||
JSON.stringify({
|
||||
event: "upload_progress",
|
||||
file: name,
|
||||
attempt,
|
||||
percent,
|
||||
sent_bytes: sentBytes,
|
||||
total_bytes: totalBytes,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
const retry = attempt > 1 ? ` (try ${attempt}, from the first byte)` : "";
|
||||
console.log(
|
||||
c.dim(
|
||||
` ${name}: ${percent}% · ${formatBytes(sentBytes)} of ${formatBytes(totalBytes ?? 0)}${retry}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function describeOffload(item: OffloadedMedia, dryRun: boolean): string {
|
||||
const verb =
|
||||
item.action === "cached"
|
||||
@@ -880,17 +916,17 @@ async function maybeUploadProject(
|
||||
bytes: archive.buffer,
|
||||
filename: `${project.name}.zip`,
|
||||
idempotencyKey,
|
||||
onProgress: !asJson
|
||||
? (ev) => {
|
||||
if (ev.phase === "initialize") {
|
||||
console.log(c.dim(` initializing…`));
|
||||
} else if (ev.phase === "upload" && ev.percent === 0) {
|
||||
console.log(c.dim(` uploading to S3…`));
|
||||
} else if (ev.phase === "complete") {
|
||||
console.log(c.dim(` finalizing…`));
|
||||
}
|
||||
}
|
||||
: undefined,
|
||||
onProgress: (ev) => {
|
||||
reportUploadProgress(`${project.name}.zip`, asJson)(ev);
|
||||
if (asJson) return;
|
||||
if (ev.phase === "initialize") {
|
||||
console.log(c.dim(` initializing…`));
|
||||
} else if (ev.phase === "upload" && ev.percent === 0 && ev.attempt === undefined) {
|
||||
console.log(c.dim(` uploading to S3…`));
|
||||
} else if (ev.phase === "complete") {
|
||||
console.log(c.dim(` finalizing…`));
|
||||
}
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
reportApiError("Upload failed", err);
|
||||
|
||||
Reference in New Issue
Block a user