Files
hyperframes/packages/cli/src/utils/download.test.ts
T
Miguel Ángel b501dc4e1b feat(cli): lambda deploy works from the npm install, no repo checkout (#5866)
* feat(cli): lambda deploy works from the npm install, no repo checkout

Each release builds the Lambda handler zip on Linux, attaches it to the
GitHub release, and pins its URL and sha256 (plus the SAM template) in
@hyperframes/aws-lambda before npm publish. Outside a checkout, lambda
deploy downloads that zip, refuses it on a sha256 mismatch, and deploys
it with the packaged template. With neither a checkout nor a pinned zip
it fails up front with one message naming every way out.

* fix(cli): lambda deploy test mocks the pinned-handler reader, comments fit the checks

* test(cli): sam tests deploy from a SAM directory, not the repo root

* chore(aws-lambda): raise the package size budget for the pinned handler template

Measured with pnpm@10.17.1 pack on Linux. Release state (prebuilt/ written by pin-handler-zip): 77,307 packed / 285,654 unpacked. CI state (no prebuilt/): 70,104 / 253,390. Packed is the check's 5% headroom over the release size. Unpacked is 292,000 (2.2% over release) because a 5% budget would put the CI pack under the 85% ratchet.

* test(cli): anchor the docs link check in the lambda deploy fallback test

* fix(cli): lambda deploy logs the verified handler sha256 and keeps a presigned query out of errors

* fix(cli): a redirect-limit download error keeps a signed query out of its message

* test(cli): the redirect-limit leak test types its rejection

* docs(aws-lambda): header comments cut to what the code cannot say

* chore(aws-lambda): size budget fits both the CI and the release package
2026-10-19 21:41:50 +00:00

360 lines
14 KiB
TypeScript

import { randomUUID } from "node:crypto";
import { MODELS_DIR, ensureModel } from "../whisper/manager.js";
import { EventEmitter } from "node:events";
import fs, { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { syncBuiltinESMExports } from "node:module";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { PassThrough } from "node:stream";
import { get as httpsGet } from "node:https";
import type { ClientRequest, IncomingMessage } from "node:http";
import { afterEach, describe, expect, it, vi } from "vitest";
import { downloadFile, REDIRECT_CODES, redirectTarget } from "./download.js";
vi.mock("node:https", () => ({ get: vi.fn() }));
const mockGet = vi.mocked(httpsGet);
const tempDirs: string[] = [];
function httpsResponse(
statusCode: number,
headers: Record<string, string> = {},
body?: string | ((url: string) => string),
onResponse?: (response: IncomingMessage) => void,
): typeof httpsGet {
return ((_url: string, callback: (response: IncomingMessage) => void) => {
const response = new PassThrough() as PassThrough & {
statusCode: number;
headers: Record<string, string>;
};
response.statusCode = statusCode;
response.headers = headers;
const request = new EventEmitter() as ClientRequest;
request.setTimeout = vi.fn();
const message = response as unknown as IncomingMessage;
onResponse?.(message);
callback(message);
if (body !== undefined) {
queueMicrotask(() => response.end(typeof body === "function" ? body(_url) : body));
}
return request;
}) as typeof httpsGet;
}
afterEach(() => {
vi.clearAllMocks();
vi.restoreAllMocks();
syncBuiltinESMExports();
for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
describe("downloadFile", () => {
it("hands https.get the caller's signal, so aborting a model download removes the partial file", async () => {
let seen: AbortSignal | undefined;
mockGet.mockImplementation(((
_url: string,
options: { signal?: AbortSignal },
callback: (response: IncomingMessage) => void,
) => {
seen = options.signal;
const response = Object.assign(new PassThrough(), { statusCode: 200, headers: {} });
const request = new EventEmitter() as ClientRequest;
request.setTimeout = vi.fn();
callback(response as unknown as IncomingMessage);
response.write("partial");
// What Node does with a request whose signal aborts.
options.signal?.addEventListener("abort", () =>
request.emit(
"error",
Object.assign(new Error("The operation was aborted"), { name: "AbortError" }),
),
);
return request;
}) as unknown as typeof httpsGet);
const controller = new AbortController();
const model = `test-${randomUUID()}`;
const download = ensureModel(model, { signal: controller.signal });
await new Promise((resolve) => setImmediate(resolve));
controller.abort();
await expect(download).rejects.toThrow("The operation was aborted");
expect(seen).toBe(controller.signal);
expect(readdirSync(MODELS_DIR).filter((name) => name.includes(model))).toEqual([]);
});
it("carries model download progress through the real pipeline and stays quiet on cache hits", async () => {
mockGet.mockImplementation(httpsResponse(200, { "content-length": "5" }, "hello"));
const events: unknown[] = [];
const model = `test-${randomUUID()}`;
const options = {
onDownloadProgress: (receivedBytes: number, totalBytes: number | null) =>
events.push({ receivedBytes, totalBytes }),
};
const modelPath = await ensureModel(model, options);
try {
expect(readFileSync(modelPath, "utf8")).toBe("hello");
expect(events).toEqual([
{ receivedBytes: 0, totalBytes: 5 },
{ receivedBytes: 5, totalBytes: 5 },
]);
events.length = 0;
expect(await ensureModel(model, options)).toBe(modelPath);
expect(events).toEqual([]);
expect(mockGet).toHaveBeenCalledTimes(1);
} finally {
rmSync(modelPath);
}
});
it.each([
[{ "content-length": "5" }, 5],
[{}, null],
[{ "content-length": "invalid" }, null],
])("reports actual response bytes with total %j", async (headers, totalBytes) => {
mockGet.mockImplementation(httpsResponse(200, headers as Record<string, string>, "hello"));
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
tempDirs.push(dir);
const events: unknown[] = [];
await downloadFile("https://example.test/model", join(dir, "model"), {
onProgress: (receivedBytes, total) => events.push({ receivedBytes, totalBytes: total }),
});
expect(events).toContainEqual({ receivedBytes: 5, totalBytes });
expect(readFileSync(join(dir, "model"), "utf8")).toBe("hello");
});
it("keeps concurrent partial downloads separate", async () => {
mockGet.mockImplementation(httpsResponse(200, {}, (url) => url));
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
tempDirs.push(dir);
const dest = join(dir, "model.onnx");
const first = "https://example.test/first";
const second = "https://example.test/second";
await Promise.all([downloadFile(first, dest), downloadFile(second, dest)]);
expect([first, second]).toContain(readFileSync(dest, "utf-8"));
});
it("rejects a response that exceeds its byte limit", async () => {
mockGet.mockImplementation(httpsResponse(200, {}, "too large"));
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
tempDirs.push(dir);
const dest = join(dir, "model.onnx");
await expect(
downloadFile("https://example.test/model.onnx", dest, { maxBytes: 3 }),
).rejects.toThrow("Download exceeded 3 bytes");
expect(existsSync(dest)).toBe(false);
expect(readdirSync(dir)).toEqual([]);
});
it("rejects an idle response and removes the partial file", async () => {
let responseClosed = false;
const realUnlinkSync = fs.unlinkSync;
vi.spyOn(fs, "unlinkSync").mockImplementation((path) => {
if (!responseClosed) {
const error = new Error("file is locked");
Object.assign(error, { code: "EBUSY" });
throw error;
}
realUnlinkSync(path);
});
// Node's named exports of a builtin follow its default export only once synced.
syncBuiltinESMExports();
mockGet.mockImplementation(((_url: string, callback: (response: IncomingMessage) => void) => {
const response = new PassThrough() as PassThrough & {
statusCode: number;
headers: Record<string, string>;
};
response.statusCode = 200;
response.headers = {};
response.once("close", () => {
responseClosed = true;
});
class FakeRequest extends EventEmitter {
private timeout: ReturnType<typeof setTimeout> | undefined;
private fallback = setTimeout(() => this.destroy(new Error("late shutdown")), 50);
private destroyed = false;
setTimeout(ms: number, onTimeout: () => void): this {
this.timeout = setTimeout(onTimeout, ms);
return this;
}
destroy(error: Error): void {
if (this.destroyed) return;
this.destroyed = true;
clearTimeout(this.timeout);
clearTimeout(this.fallback);
response.destroy(error);
this.emit("error", error);
}
}
const request = new FakeRequest();
callback(response as unknown as IncomingMessage);
return request as unknown as ClientRequest;
}) as typeof httpsGet);
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
tempDirs.push(dir);
const dest = join(dir, "model.onnx");
await expect(
downloadFile("https://example.test/model.onnx", dest, { timeoutMs: 10 }),
).rejects.toThrow("Download timed out after 10ms");
expect(existsSync(`${dest}.tmp`)).toBe(false);
expect(existsSync(dest)).toBe(false);
});
});
describe("redirect handling", () => {
it("closes each redirect response before opening the next request", async () => {
const responses: IncomingMessage[] = [];
const capture = (response: IncomingMessage) => responses.push(response);
mockGet
.mockImplementationOnce(httpsResponse(302, { location: "/second" }, undefined, capture))
.mockImplementationOnce(
httpsResponse(307, { location: "/model" }, undefined, (response) => {
expect(responses[0]?.destroyed).toBe(true);
capture(response);
}),
)
.mockImplementationOnce(
httpsResponse(200, {}, "hello", () => {
expect(responses[1]?.destroyed).toBe(true);
}),
);
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
tempDirs.push(dir);
await downloadFile("https://example.test/first", join(dir, "model"));
expect(responses).toHaveLength(2);
expect(responses.every((response) => response.destroyed)).toBe(true);
expect(mockGet).toHaveBeenCalledTimes(3);
expect(readFileSync(join(dir, "model"), "utf8")).toBe("hello");
});
it("does not let a discarded redirect request fail the active download", async () => {
mockGet
.mockImplementationOnce(httpsResponse(302, { location: "/model" }))
.mockImplementationOnce(httpsResponse(200, {}, "hello"));
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
tempDirs.push(dir);
const pending = downloadFile("https://example.test/first", join(dir, "model"));
const firstRequest = mockGet.mock.results[0]?.value;
firstRequest?.emit("error", new Error("old redirect socket failed"));
await expect(pending).resolves.toBeUndefined();
expect(readFileSync(join(dir, "model"), "utf8")).toBe("hello");
expect(readdirSync(dir)).toEqual(["model"]);
});
it("closes redirect loop responses when reaching the hop limit", async () => {
const responses: IncomingMessage[] = [];
mockGet.mockImplementation(
httpsResponse(302, { location: "/loop" }, undefined, (response) => responses.push(response)),
);
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
tempDirs.push(dir);
await expect(downloadFile("https://example.test/loop", join(dir, "model"))).rejects.toThrow(
"more than 10 redirects",
);
expect(responses).toHaveLength(11);
expect(responses.every((response) => response.destroyed)).toBe(true);
expect(readdirSync(dir)).toEqual([]);
});
it("keeps a signed query out of the redirect-limit error", async () => {
mockGet.mockImplementation(httpsResponse(302, { location: "/loop" }));
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-redirect-"));
tempDirs.push(dir);
const error = await downloadFile(
"https://example.test/handler.zip?X-Amz-Signature=secret",
join(dir, "handler.zip"),
).then(
() => {
throw new Error("expected the download to fail");
},
(caught: unknown) => caught as Error,
);
expect(error.message).toContain("https://example.test/handler.zip");
expect(error.message).not.toContain("secret");
});
it.each([302, 404, 503])("closes rejected HTTP %s responses", async (status) => {
const responses: IncomingMessage[] = [];
mockGet.mockImplementation(
httpsResponse(status, {}, undefined, (response) => responses.push(response)),
);
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-http-error-"));
tempDirs.push(dir);
await expect(downloadFile("https://example.test/model", join(dir, "model"))).rejects.toThrow(
`HTTP ${status}`,
);
expect(responses[0]?.destroyed).toBe(true);
expect(readdirSync(dir)).toEqual([]);
});
it("rejects a redirect that the HTTPS client cannot follow", async () => {
mockGet
.mockImplementationOnce(
httpsResponse(307, { location: "http://cdn.example.test/model.onnx" }),
)
.mockImplementationOnce(() => {
throw new TypeError('Protocol "http:" not supported');
});
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-"));
tempDirs.push(dir);
await expect(
downloadFile("https://example.test/model.onnx", join(dir, "model.onnx")),
).rejects.toThrow('Protocol "http:" not supported');
});
it("follows every redirect a host may answer with", () => {
// 307 is the one that broke: HuggingFace answers the tokenizer with it,
// and the original set stopped at 302, so the download fell through to the
// not-200 branch and hung.
for (const code of [301, 302, 303, 307, 308]) {
expect(REDIRECT_CODES.has(code)).toBe(true);
}
});
it("does not treat a success or an error as a redirect", () => {
for (const code of [200, 204, 404, 500]) {
expect(REDIRECT_CODES.has(code)).toBe(false);
}
});
it("resolves a relative location against the url that sent it", () => {
// The actual failure: handing this path back as a request target is not a
// valid URL, so nothing was ever fetched.
expect(
redirectTarget(
"/api/resolve-cache/models/x/tokenizer.json",
"https://huggingface.co/x/resolve/main/tokenizer.json",
),
).toBe("https://huggingface.co/api/resolve-cache/models/x/tokenizer.json");
});
it("keeps an absolute location as it is", () => {
expect(redirectTarget("https://cdn.example.com/blob", "https://huggingface.co/a/b")).toBe(
"https://cdn.example.com/blob",
);
});
it("resolves a sibling-relative location", () => {
expect(redirectTarget("other.json", "https://example.com/a/b/file.json")).toBe(
"https://example.com/a/b/other.json",
);
});
});