Files
hyperframes/packages/cli/src/cloud/upload.test.ts
T
Miguel Ángel 2ea78c0ab4 test(cli): the upload cut test stops failing on busy Windows runners (#7843)
* test(cli): the upload cut test stops failing on busy Windows runners

* test(cli): the upload cut test leaves room for a busy runner's slow first try
2026-11-15 14:12:04 +00:00

534 lines
19 KiB
TypeScript

import { describe, it, expect, onTestFinished, vi } from "vitest";
import { createHash } from "node:crypto";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { createServer, type IncomingHttpHeaders, type IncomingMessage } from "node:http";
import type { AddressInfo } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { HyperframesApiError } from "./_gen/client.js";
import type { HyperframesCloudClient } from "./_gen/client.js";
import {
putTimings,
uploadAssetViaDirectUpload,
uploadZipViaDirectUpload,
type UploadProgressEvent,
} from "./upload.js";
function sha256Hex(bytes: Uint8Array): string {
return createHash("sha256").update(bytes).digest("hex");
}
function makeClient(overrides: Partial<HyperframesCloudClient> = {}): HyperframesCloudClient {
return {
createAssetUpload: vi.fn(async () => ({
asset_id: "asset_xyz",
upload_url: "https://s3.example/asset_xyz?sig=abc",
upload_headers: { "x-amz-checksum-sha256-b64": "..." },
expires_in_seconds: 3600,
max_bytes: 200 * 1024 * 1024,
status: "pending_upload" as const,
})),
completeAssetUpload: vi.fn(async () => ({
asset_id: "asset_xyz",
url: "https://files.heygen.com/document/asset_xyz/original.zip",
mime_type: "application/zip",
size_bytes: 42,
status: "processing" as const,
})),
...overrides,
} as unknown as HyperframesCloudClient;
}
function makeFetchOk(): typeof fetch {
return vi.fn(async () => new Response("", { status: 200 })) as unknown as typeof fetch;
}
describe("uploadZipViaDirectUpload", () => {
it("sends the correct filename, content_type, size_bytes, and SHA256 to createAssetUpload", async () => {
const bytes = new TextEncoder().encode("hello-hyperframes-zip");
const expectedSha = sha256Hex(bytes);
const client = makeClient();
const fetchImpl = makeFetchOk();
await uploadZipViaDirectUpload({
client,
bytes,
filename: "my-comp.zip",
fetchImpl,
});
expect(client.createAssetUpload).toHaveBeenCalledOnce();
const arg = (client.createAssetUpload as ReturnType<typeof vi.fn>).mock.calls[0]![0];
expect(arg.body).toEqual({
filename: "my-comp.zip",
content_type: "application/zip",
size_bytes: bytes.byteLength,
checksum_sha256: expectedSha,
});
});
it("PUTs to the returned upload_url with the returned upload_headers verbatim + content-type", async () => {
const bytes = new Uint8Array([1, 2, 3, 4]);
const client = makeClient({
createAssetUpload: vi.fn(async () => ({
asset_id: "asset_xyz",
upload_url: "https://s3.example/target?sig=xxx",
upload_headers: { "x-signed-header": "signed-value", "x-other": "other-value" },
expires_in_seconds: 3600,
max_bytes: 200 * 1024 * 1024,
status: "pending_upload" as const,
})) as HyperframesCloudClient["createAssetUpload"],
});
const fetchImpl = makeFetchOk();
await uploadZipViaDirectUpload({ client, bytes, filename: "x.zip", fetchImpl });
expect(fetchImpl).toHaveBeenCalledOnce();
const [url, init] = (fetchImpl as ReturnType<typeof vi.fn>).mock.calls[0]!;
expect(url).toBe("https://s3.example/target?sig=xxx");
expect(init.method).toBe("PUT");
expect(init.body).toBe(bytes);
expect(init.headers).toEqual({
"content-type": "application/zip",
"x-signed-header": "signed-value",
"x-other": "other-value",
});
});
it("does NOT attach CLI auth headers to the S3 PUT — presigned URL carries auth", async () => {
const client = makeClient();
const fetchImpl = makeFetchOk();
await uploadZipViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "x.zip",
fetchImpl,
});
const init = (fetchImpl as ReturnType<typeof vi.fn>).mock.calls[0]![1];
// No Authorization / x-api-key / Bearer header should be present.
const headerKeys = Object.keys(init.headers as Record<string, string>).map((k) =>
k.toLowerCase(),
);
expect(headerKeys).not.toContain("authorization");
expect(headerKeys).not.toContain("x-api-key");
});
it("calls completeAssetUpload with the initialize's asset_id + same checksum", async () => {
const bytes = new TextEncoder().encode("determinism");
const expectedSha = sha256Hex(bytes);
const client = makeClient();
const fetchImpl = makeFetchOk();
const result = await uploadZipViaDirectUpload({
client,
bytes,
filename: "x.zip",
fetchImpl,
});
expect(client.completeAssetUpload).toHaveBeenCalledOnce();
const completeArg = (client.completeAssetUpload as ReturnType<typeof vi.fn>).mock.calls[0]![0];
expect(completeArg.asset_id).toBe("asset_xyz");
expect(completeArg.body).toEqual({ checksum_sha256: expectedSha });
expect(result.asset_id).toBe("asset_xyz");
expect(result.size_bytes).toBe(bytes.byteLength);
});
it("retries completeAssetUpload on 409 and succeeds on a later attempt", async () => {
let completeCalls = 0;
const completeAssetUpload = vi.fn(async () => {
completeCalls++;
if (completeCalls < 3) {
throw new HyperframesApiError({
status: 409,
message: "Uploaded object not found yet. Retry after upload PUT returns 200.",
code: "conflict",
});
}
return {
asset_id: "asset_xyz",
url: "u",
mime_type: "application/zip",
size_bytes: 1,
status: "processing" as const,
};
});
const client = makeClient({
completeAssetUpload:
completeAssetUpload as unknown as HyperframesCloudClient["completeAssetUpload"],
});
const result = await uploadZipViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "x.zip",
fetchImpl: makeFetchOk(),
});
expect(completeCalls).toBe(3);
expect(result.asset_id).toBe("asset_xyz");
});
it("surfaces non-409 errors from complete without retrying", async () => {
let completeCalls = 0;
const completeAssetUpload = vi.fn(async () => {
completeCalls++;
throw new HyperframesApiError({
status: 400,
message: "invalid checksum",
code: "invalid_parameter",
});
});
const client = makeClient({
completeAssetUpload:
completeAssetUpload as unknown as HyperframesCloudClient["completeAssetUpload"],
});
await expect(
uploadZipViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "x.zip",
fetchImpl: makeFetchOk(),
}),
).rejects.toThrow(/invalid checksum/);
expect(completeCalls).toBe(1);
});
it("names the file and keeps the server's words when complete refuses its type", async () => {
const completeAssetUpload = vi.fn(async () => {
throw new HyperframesApiError({
status: 400,
message: "Stored file type not supported: application/octet-stream",
code: "invalid_parameter",
param: "asset_id",
});
});
const client = makeClient({
completeAssetUpload:
completeAssetUpload as unknown as HyperframesCloudClient["completeAssetUpload"],
});
const err = await uploadAssetViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "song.mp3",
contentType: "audio/mpeg",
fetchImpl: makeFetchOk(),
}).catch((e: unknown) => e);
expect(err).toBeInstanceOf(HyperframesApiError);
expect((err as HyperframesApiError).message).toBe(
"The cloud couldn't read this audio file's type: song.mp3. It said: Stored file type not supported: application/octet-stream",
);
expect((err as HyperframesApiError).param).toBe("asset_id");
expect((err as HyperframesApiError).status).toBe(400);
});
it("does not blame the file's type when complete refuses an expired upload", async () => {
const completeAssetUpload = vi.fn(async () => {
throw new HyperframesApiError({
status: 400,
message: "This upload has expired; create a new upload.",
code: "invalid_parameter",
});
});
const client = makeClient({
completeAssetUpload:
completeAssetUpload as unknown as HyperframesCloudClient["completeAssetUpload"],
});
const err = await uploadAssetViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "song.mp3",
contentType: "audio/mpeg",
fetchImpl: makeFetchOk(),
}).catch((e: unknown) => e);
expect((err as HyperframesApiError).message).toBe(
"The cloud refused song.mp3: This upload has expired; create a new upload.",
);
});
it("surfaces PUT failures with response body detail", async () => {
const client = makeClient();
const fetchImpl = vi.fn(
async () =>
new Response("SignatureDoesNotMatch: request signature we calculated does not match", {
status: 403,
}),
) as unknown as typeof fetch;
await expect(
uploadZipViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "x.zip",
fetchImpl,
}),
).rejects.toThrow(/Direct upload PUT failed: 403.*SignatureDoesNotMatch/);
});
it("passes idempotencyKey through to createAssetUpload", async () => {
const client = makeClient();
await uploadZipViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "x.zip",
idempotencyKey: "test-key-123",
fetchImpl: makeFetchOk(),
});
const arg = (client.createAssetUpload as ReturnType<typeof vi.fn>).mock.calls[0]![0];
expect(arg.idempotencyKey).toBe("test-key-123");
});
it("emits progress events in order", async () => {
const client = makeClient();
const events: Array<{ phase: string }> = [];
await uploadZipViaDirectUpload({
client,
bytes: new Uint8Array([0]),
filename: "x.zip",
fetchImpl: makeFetchOk(),
onProgress: (e) => events.push({ phase: e.phase }),
});
expect(events.map((e) => e.phase)).toEqual(["initialize", "upload", "upload", "complete"]);
});
});
describe("uploadAssetViaDirectUpload from a file on disk", () => {
async function withServer(
status: number,
run: (url: string, got: { headers?: IncomingHttpHeaders; body?: Buffer }) => Promise<void>,
): Promise<void> {
const got: { headers?: IncomingHttpHeaders; body?: Buffer } = {};
const server = createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(chunk as Buffer);
Object.assign(got, { headers: req.headers, body: Buffer.concat(chunks) });
res.writeHead(status).end(status === 200 ? "" : "SignatureDoesNotMatch");
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
try {
await run(`http://127.0.0.1:${(server.address() as AddressInfo).port}/put`, got);
} finally {
server.close();
}
}
function fileOnDisk(bytes: Buffer): string {
const dir = mkdtempSync(join(tmpdir(), "hf-upload-test-"));
onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
writeFileSync(join(dir, "clip.mp4"), bytes);
return join(dir, "clip.mp4");
}
const clientFor = (url: string) =>
makeClient({
createAssetUpload: vi.fn(async () => ({
asset_id: "asset_xyz",
upload_url: url,
upload_headers: {},
expires_in_seconds: 3600,
max_bytes: 200 * 1024 * 1024,
status: "pending_upload" as const,
})),
} as Partial<HyperframesCloudClient>);
it("hashes the file and PUTs it as a stream with its exact length", async () => {
const bytes = Buffer.from("a file far too big to read whole, in miniature");
const path = fileOnDisk(bytes);
await withServer(200, async (url, got) => {
const client = clientFor(url);
await uploadAssetViaDirectUpload({
client,
path,
size: bytes.byteLength,
filename: "clip.mp4",
contentType: "video/mp4",
});
expect(client.createAssetUpload).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.objectContaining({ checksum_sha256: sha256Hex(bytes) }),
}),
);
expect(got.body?.equals(bytes)).toBe(true);
expect(got.headers?.["content-length"]).toBe(String(bytes.byteLength));
expect(got.headers?.["content-type"]).toBe("video/mp4");
});
});
it("reports the status of a refused PUT", async () => {
const path = fileOnDisk(Buffer.from("bytes"));
await withServer(403, async (url) => {
await expect(
uploadAssetViaDirectUpload({
client: clientFor(url),
path,
size: 5,
filename: "clip.mp4",
contentType: "video/mp4",
}),
).rejects.toThrow(/Direct upload PUT failed: 403.*SignatureDoesNotMatch/);
});
});
});
const BIG_BYTES = 16 * 1024 * 1024;
// Two cuts count as one within 20% of their age; a busy Windows runner adds up to ~150 ms to a first try, which
// overran 20% of a 400 ms cut. The 16 MB body takes 2 s at 8 MB/s, so the cut still comes first.
const CUT_MS = 1600;
/** Reads the body; "cut" and "slow" take 8 MB/s, "cut" drops the connection CUT_MS after it starts reading, and
* "late cut" is a "cut" that starts reading 150 ms late, like a first try still setting up. */
async function drain(req: IncomingMessage, answer: unknown): Promise<"read" | "dropped"> {
const cut = answer === "cut" || answer === "late cut";
const slow = cut || answer === "slow";
if (answer === "late cut") await new Promise((resolve) => setTimeout(resolve, 150));
const born = Date.now();
try {
for await (const chunk of req as AsyncIterable<Buffer>) {
if (!slow) continue;
await new Promise((resolve) => setTimeout(resolve, Math.ceil(chunk.length / 8192)));
if (cut && Date.now() - born >= CUT_MS) {
req.socket.destroy();
return "dropped";
}
}
} catch {
return "dropped";
}
return "read";
}
describe("a PUT that stalls or fails on the network is sent again", () => {
/** Answers its Nth request as `plan[n]`, the last one for any later: "hang" never answers, "cut" and "late cut"
* drop the connection (see drain), "slow" drains slowly and answers 200, a number is the status. */
async function withPlan(
plan: Array<"hang" | "cut" | "late cut" | "slow" | number>,
run: (url: string, requests: () => number) => Promise<void>,
): Promise<void> {
let n = 0;
const open: Array<() => void> = [];
const server = createServer(async (req, res) => {
const answer = plan[Math.min(n++, plan.length - 1)]!;
if ((await drain(req, answer)) === "dropped") return;
if (answer === "hang") return void open.push(() => res.destroy());
if (answer === "cut" || answer === "late cut" || answer === "slow")
return void res.writeHead(200).end();
res.writeHead(answer).end(answer === 200 ? "" : "refused");
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
try {
await run(`http://127.0.0.1:${(server.address() as AddressInfo).port}/put`, () => n);
} finally {
for (const close of open) close();
server.close();
}
}
const upload = (
url: string,
asBytes = false,
big?: { onProgress: (event: UploadProgressEvent) => void },
) => {
const dir = mkdtempSync(join(tmpdir(), "hf-upload-retry-"));
onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
const content = big ? Buffer.alloc(BIG_BYTES, 1) : Buffer.from("bytes");
writeFileSync(join(dir, "clip.mp4"), content);
const was = { ...putTimings };
Object.assign(putTimings, {
stallMs: big ? 5000 : 150,
backoffMs: 5,
progressMs: 50,
sampleMs: 50,
});
onTestFinished(() => void Object.assign(putTimings, was));
return uploadAssetViaDirectUpload({
client: makeClient({
createAssetUpload: vi.fn(async () => ({
asset_id: "asset_xyz",
upload_url: url,
upload_headers: {},
expires_in_seconds: 3600,
max_bytes: 200 * 1024 * 1024,
status: "pending_upload" as const,
})),
} as Partial<HyperframesCloudClient>),
...(asBytes
? { bytes: Buffer.from("bytes") }
: { path: join(dir, "clip.mp4"), size: content.byteLength }),
filename: "clip.mp4",
contentType: "video/mp4",
onProgress: big?.onProgress,
});
};
it("ends a try that moves no data and sends the file again", async () => {
await withPlan(["hang", 200], async (url, requests) => {
await upload(url);
expect(requests()).toBe(2);
});
});
it("bounds the project zip, which is sent from memory, the same way", async () => {
await withPlan(["hang", 200], async (url, requests) => {
await upload(url, true);
expect(requests()).toBe(2);
});
});
it("sends it again after a server error, and gives up after the retries with the last words", async () => {
await withPlan([503, 200], async (url, requests) => {
await upload(url);
expect(requests()).toBe(2);
});
await withPlan(["hang"], async (url, requests) => {
await expect(upload(url)).rejects.toThrow(/Upload stalled/);
expect(requests()).toBe(1 + putTimings.retries);
});
});
it("does not send it again when S3 refuses it", async () => {
await withPlan([403], async (url, requests) => {
await expect(upload(url)).rejects.toThrow(/PUT failed: 403/);
expect(requests()).toBe(1);
});
});
it("stops after a second identical cut, even when the first try was slow to start", async () => {
await withPlan(["late cut", "cut"], async (url, requests) => {
await expect(upload(url, false, { onProgress: () => undefined })).rejects.toThrow(
/cut at \d+ s on two tries in a row.*Every try restarts the file/,
);
expect(requests()).toBe(2);
});
}, 15_000);
it("reports the bytes sent while a slow PUT runs, and the restart of a second try", async () => {
const events: UploadProgressEvent[] = [];
await withPlan([503, "slow"], async (url) => {
await upload(url, false, { onProgress: (event) => events.push(event) });
});
const sent = events.flatMap((e) => (e.phase === "upload" && e.attempt === 2 ? [e] : []));
const live = sent.filter((e) => (e.sentBytes ?? 0) > 0);
expect(live.length).toBeGreaterThan(1);
expect(live.map((e) => e.sentBytes)).toEqual(
[...live.map((e) => e.sentBytes)].sort((a, b) => a! - b!),
);
expect(live.every((e) => e.totalBytes === BIG_BYTES)).toBe(true);
expect(events.some((e) => e.phase === "upload" && e.attempt === 2 && e.sentBytes === 0)).toBe(
true,
);
});
it("says how many tries it made and how far the last got when the retries run out", async () => {
await withPlan([503], async (url) => {
await expect(upload(url)).rejects.toThrow(/503.*4 tries, each from the first byte/);
});
});
});