fix(engine): bound remote media downloads by size, encoding and throughput (#7786)

downloadToTemp now refuses any Content-Encoding other than identity, caps each
download at 20 GiB (HYPERFRAMES_MAX_DOWNLOAD_BYTES) on the declared Content-Length
and while streaming, and aborts a transfer whose bytes over the last 60 s average
below 32 KiB/s (HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC, 0 disables).

Compressed and over-limit responses are not retried. A throughput abort is a
retryable timeout like the stall timer, which is unchanged. A sized body that
trickles one chunk every 100 s is now cut by the floor instead of finishing.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
James Russo
2026-11-15 01:47:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 83eae240fb
commit 5111cf3840
3 changed files with 388 additions and 10 deletions
@@ -0,0 +1,133 @@
import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { _setDownloadFetchForTests, downloadToTemp, UrlDownloadError } from "./urlDownloader.js";
const stubFetch = (replacement: unknown): void =>
_setDownloadFetchForTests(replacement as Parameters<typeof _setDownloadFetchForTests>[0]);
const dirs: string[] = [];
afterEach(() => {
vi.unstubAllEnvs();
_setDownloadFetchForTests();
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
function tempDir(): string {
const dir = mkdtempSync(join(tmpdir(), "hf-url-limits-"));
dirs.push(dir);
return dir;
}
const MEDIA = "media-payload-bytes";
function mediaResponse(headers: Record<string, string> = {}): Response {
return new Response(MEDIA, { headers: { "content-length": `${MEDIA.length}`, ...headers } });
}
describe("downloadToTemp's Content-Encoding policy", () => {
it.each(["gzip", "br", " GZIP ", "identity, gzip"])(
"refuses a %j response without retrying or leaving a file",
async (encoding) => {
const fetchMock = vi.fn(async () => mediaResponse({ "content-encoding": encoding }));
stubFetch(fetchMock);
const dir = tempDir();
await expect(
downloadToTemp("https://cdn.example/encoded.mp4", dir, 1_000),
).rejects.toMatchObject({
kind: "http_rejected",
retryable: false,
message: "Download response is compressed; only identity Content-Encoding is accepted",
} satisfies Partial<UrlDownloadError>);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(readdirSync(dir)).toEqual([]);
},
);
it.each([
["absent", {}],
["identity", { "content-encoding": "identity" }],
["padded Identity", { "content-encoding": " Identity " }],
])("downloads a response whose Content-Encoding is %s", async (_case, headers) => {
stubFetch(vi.fn(async () => mediaResponse(headers)));
const path = await downloadToTemp("https://cdn.example/plain.mp4", tempDir(), 1_000);
expect(readFileSync(path, "utf8")).toBe(MEDIA);
});
});
describe("downloadToTemp's byte limit", () => {
it("refuses a declared Content-Length over the limit before reading the body", async () => {
vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", "16");
const cancel = vi.fn();
// The body never sends, so a reader would hang until the 1 s timeout instead.
const body = new ReadableStream<Uint8Array>({ cancel });
const fetchMock = vi.fn(
async () => new Response(body, { headers: { "content-length": "17" } }),
);
stubFetch(fetchMock);
const dir = tempDir();
await expect(downloadToTemp("https://cdn.example/big.mp4", dir, 1_000)).rejects.toMatchObject({
kind: "length_mismatch",
retryable: false,
message: "Download exceeded the 16-byte limit",
telemetry: expect.objectContaining({ expectedBytes: 17 }),
} satisfies Partial<UrlDownloadError>);
expect(cancel).toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(readdirSync(dir)).toEqual([]);
});
it("stops an endless body with no Content-Length at the limit and removes the partial", async () => {
const limit = 1024 * 1024;
const chunkBytes = 64 * 1024;
vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", `${limit}`);
const cancel = vi.fn();
const fetchMock = vi.fn(
async () =>
new Response(
new ReadableStream<Uint8Array>({
pull(controller) {
controller.enqueue(new Uint8Array(chunkBytes).fill(0x61));
},
cancel,
}),
),
);
stubFetch(fetchMock);
const dir = tempDir();
const error = await downloadToTemp("https://cdn.example/endless.mp4", dir, 10_000).catch(
(caught: unknown) => caught,
);
expect(error).toMatchObject({
kind: "length_mismatch",
retryable: false,
message: `Download exceeded the ${limit}-byte limit`,
});
const receivedBytes = (error as UrlDownloadError).telemetry?.receivedBytes ?? 0;
expect(receivedBytes).toBeGreaterThan(limit);
expect(receivedBytes).toBeLessThanOrEqual(limit + chunkBytes);
expect(cancel).toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(readdirSync(dir)).toEqual([]);
});
it.each(["", "abc", "0", "-5", "1.5"])(
"keeps the default limit when HYPERFRAMES_MAX_DOWNLOAD_BYTES is %j",
async (value) => {
vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", value);
stubFetch(vi.fn(async () => mediaResponse()));
const path = await downloadToTemp("https://cdn.example/default-limit.mp4", tempDir(), 1_000);
expect(readFileSync(path, "utf8")).toBe(MEDIA);
},
);
});
@@ -1,4 +1,4 @@
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
@@ -15,6 +15,7 @@ beforeEach(() => {
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
_setDownloadFetchForTests();
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
@@ -60,6 +61,48 @@ function trickle(
return new Response(body, { headers });
}
/**
* A sized body that sends `chunk` `everyMs` after each read, `count` times. Waiting for
* `readerWaiting()` before advancing the clock keeps real disk writes from lagging fake time.
*/
function pacedBody(chunk: Uint8Array, count: number, everyMs: number) {
let sent = 0;
let waiting = false;
const body = new ReadableStream<Uint8Array>(
{
pull(controller) {
waiting = true;
return new Promise<void>((resolve) => {
setTimeout(() => {
waiting = false;
controller.enqueue(chunk);
sent += 1;
if (sent === count) controller.close();
resolve();
}, everyMs);
});
},
},
{ highWaterMark: 0 },
);
const response = new Response(body, {
headers: { "content-length": `${chunk.length * count}` },
});
return {
response,
readerWaiting: () => waiting || sent === count,
finished: () => sent === count,
};
}
/** Like `waitForIo`, but bounded by real time so a loaded disk cannot fall behind the fake clock. */
async function waitForRealIo(done: () => boolean, budgetMs = 10_000): Promise<void> {
const deadline = performance.now() + budgetMs;
while (!done() && performance.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
}
/** Real I/O (cache lock, file writes) settles on the real event loop; the fake clock stays put. */
async function waitForIo(done: () => boolean, turns = 100_000): Promise<void> {
for (let i = 0; i < turns && !done(); i += 1) {
@@ -82,16 +125,44 @@ async function startDownload(response: () => Response) {
const fetchMock = vi.fn(async () => response());
stubFetch(fetchMock);
const outcome: { path?: string; error?: unknown } = {};
void downloadToTemp("https://cdn.example/media.mp4", tempDir()).then(
const dir = tempDir();
void downloadToTemp("https://cdn.example/media.mp4", dir).then(
(path) => (outcome.path = path),
(error: unknown) => (outcome.error = error),
);
await waitForIo(fetchCalls(fetchMock, 1));
return { fetchMock, outcome };
return { fetchMock, outcome, dir };
}
const disableThroughputFloor = (): void =>
vi.stubEnv("HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC", "0");
const tooSlow = {
kind: "timeout",
retryable: true,
message: "Download too slow: under 32768 bytes/s over the last 60s",
};
describe("downloadToTemp's timeout", () => {
it("lets a sized download that sends a chunk every 100 s for 400 s finish", async () => {
it("cuts a sized download that sends a chunk every 100 s at the throughput floor", async () => {
const chunks = ["chunk-1 ", "chunk-2 ", "chunk-3 ", "chunk-4"];
const { fetchMock, outcome, dir } = await startDownload(() =>
trickle(chunks, 100_000, { contentLength: chunks.join("").length }),
);
await advanceSeconds(159);
expect(fetchMock).toHaveBeenCalledTimes(1);
await advanceSeconds(1);
await waitForIo(fetchCalls(fetchMock, 2));
expect(fetchMock).toHaveBeenCalledTimes(2);
await advanceSeconds(160);
await waitForIo(() => "error" in outcome);
expect(outcome.error).toMatchObject(tooSlow);
expect(readdirSync(dir)).toEqual([]);
});
it("lets the same 100 s trickle finish when the floor is disabled with 0", async () => {
disableThroughputFloor();
const chunks = ["chunk-1 ", "chunk-2 ", "chunk-3 ", "chunk-4"];
const { fetchMock, outcome } = await startDownload(() =>
trickle(chunks, 100_000, { contentLength: chunks.join("").length }),
@@ -104,7 +175,42 @@ describe("downloadToTemp's timeout", () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("cuts a 1 KiB/s trickle that declares a huge Content-Length and removes the partial", async () => {
const { fetchMock, outcome, dir } = await startDownload(() =>
trickle(["x".repeat(1024)], 1_000, { contentLength: 10 * 1024 ** 3, then: "repeat" }),
);
await advanceSeconds(60);
expect(fetchMock).toHaveBeenCalledTimes(1);
await advanceSeconds(1);
await waitForIo(fetchCalls(fetchMock, 2));
expect(fetchMock).toHaveBeenCalledTimes(2);
await advanceSeconds(61);
await waitForIo(() => "error" in outcome);
expect(outcome.error).toMatchObject(tooSlow);
expect(readdirSync(dir)).toEqual([]);
});
it("lets a steady sized download above the floor run past 15 minutes", async () => {
const chunk = new Uint8Array(40 * 1024).fill(0x78);
const seconds = 15 * 60 + 20;
const paced = pacedBody(chunk, seconds, 1_000);
const startedAt = Date.now();
const { fetchMock, outcome } = await startDownload(() => paced.response);
while (!paced.finished() && !("error" in outcome)) {
await waitForRealIo(paced.readerWaiting);
await vi.advanceTimersByTimeAsync(1_000);
}
await waitForRealIo(() => "path" in outcome || "error" in outcome);
expect(outcome.error).toBeUndefined();
expect(Date.now() - startedAt).toBeGreaterThan(15 * 60_000);
expect(statSync(outcome.path!).size).toBe(chunk.length * seconds);
expect(fetchMock).toHaveBeenCalledTimes(1);
}, 120_000);
it("cuts a body with no Content-Length at the fixed 300 s cap even while it streams", async () => {
disableThroughputFloor();
const { fetchMock, outcome } = await startDownload(() =>
trickle(["tick "], 10_000, { then: "repeat" }),
);
@@ -141,6 +247,7 @@ describe("downloadToTemp's timeout", () => {
});
it("fails a sized body 300 s after its last chunk and says it stalled", async () => {
disableThroughputFloor();
const { fetchMock, outcome } = await startDownload(() =>
trickle(["part-1 "], 100_000, { contentLength: 1_000, then: "hang" }),
);
+144 -6
View File
@@ -796,6 +796,79 @@ function checksumMismatchError(
);
}
const DEFAULT_MAX_DOWNLOAD_BYTES = 20 * 1024 ** 3;
const DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC = 32 * 1024;
const THROUGHPUT_WINDOW_SECONDS = 60;
function integerFromEnv(name: string, fallback: number, minimum: number): number {
const raw = process.env[name]?.trim();
if (!raw || !/^\d+$/.test(raw)) return fallback;
const value = Number(raw);
return Number.isSafeInteger(value) && value >= minimum ? value : fallback;
}
const maxDownloadBytes = (): number =>
integerFromEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", DEFAULT_MAX_DOWNLOAD_BYTES, 1);
const minDownloadBytesPerSec = (): number =>
integerFromEnv("HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC", DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC, 0);
function isCompressedEncoding(contentEncoding: string | null): boolean {
const encoding = contentEncoding?.trim().toLowerCase();
return Boolean(encoding) && encoding !== "identity";
}
function overByteLimitError(
maxBytes: number,
status: number,
telemetry: Partial<UrlDownloadTelemetry>,
): UrlDownloadError {
return new UrlDownloadError(
"length_mismatch",
false,
`Download exceeded the ${maxBytes}-byte limit`,
status,
telemetry,
);
}
// One spare bucket so the summed span is never shorter than the window, which errs toward keeping
// a download that runs right at the floor.
class ThroughputWindow {
private readonly buckets = new Array<number>(THROUGHPUT_WINDOW_SECONDS + 1).fill(0);
private bytesInWindow = 0;
private currentSecond: number;
constructor(private readonly startedAtMs: number) {
this.currentSecond = Math.floor(startedAtMs / 1000);
}
record(byteCount: number, nowMs: number): void {
this.advanceTo(nowMs);
this.buckets[this.currentSecond % this.buckets.length]! += byteCount;
this.bytesInWindow += byteCount;
}
isBelow(minBytesPerSec: number, nowMs: number): boolean {
this.advanceTo(nowMs);
return (
nowMs - this.startedAtMs >= THROUGHPUT_WINDOW_SECONDS * 1000 &&
this.bytesInWindow < minBytesPerSec * THROUGHPUT_WINDOW_SECONDS
);
}
private advanceTo(nowMs: number): void {
const second = Math.floor(nowMs / 1000);
const expired = Math.min(second - this.currentSecond, this.buckets.length);
for (let step = 1; step <= expired; step += 1) {
const index = (this.currentSecond + step) % this.buckets.length;
this.bytesInWindow -= this.buckets[index]!;
this.buckets[index] = 0;
}
this.currentSecond = Math.max(second, this.currentSecond);
}
}
// Response protocol, streamed byte accounting, hashes, and payload validation
// share one lifecycle so no validation can happen after publication.
// fallow-ignore-next-line complexity
@@ -804,7 +877,7 @@ async function fetchToPartial(
partialPath: string,
controller: AbortController,
options: UrlDownloadOptions,
onSizedBodyBytes: () => void,
onBodyBytes: (byteCount: number, sized: boolean) => void,
): Promise<PartialIntegrity> {
const { response, finalUrl } = await fetchWithValidatedRedirects(url, controller);
const finalIdentity = safeDownloadUrlIdentity(finalUrl);
@@ -869,9 +942,26 @@ async function fetchToPartial(
}
throw error;
}
const contentEncoding = response.headers.get("content-encoding")?.trim().toLowerCase();
const expectedBytes =
!contentEncoding || contentEncoding === "identity" ? declaredLength : undefined;
const responseTelemetry = { finalHost: finalIdentity.host, status: response.status };
if (isCompressedEncoding(response.headers.get("content-encoding"))) {
await cancelResponseBody(response);
throw new UrlDownloadError(
"http_rejected",
false,
"Download response is compressed; only identity Content-Encoding is accepted",
response.status,
responseTelemetry,
);
}
const maxBytes = maxDownloadBytes();
if (declaredLength !== undefined && declaredLength > maxBytes) {
await cancelResponseBody(response);
throw overByteLimitError(maxBytes, response.status, {
...responseTelemetry,
expectedBytes: declaredLength,
});
}
const expectedBytes = declaredLength;
let receivedBytes = 0;
const sha256 = createHash("sha256");
@@ -881,8 +971,18 @@ async function fetchToPartial(
const inspector = new Transform({
transform(chunk: Buffer, _encoding, callback) {
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
if (declaredLength !== undefined) onSizedBodyBytes();
onBodyBytes(bytes.length, declaredLength !== undefined);
receivedBytes += bytes.length;
if (receivedBytes > maxBytes) {
callback(
overByteLimitError(maxBytes, response.status, {
...responseTelemetry,
expectedBytes,
receivedBytes,
}),
);
return;
}
sha256.update(bytes);
md5.update(bytes);
if (prefixBytes < 1024) {
@@ -1084,13 +1184,35 @@ async function runDownloadAttempt(
sizedBodyStarted = true;
armTimeout();
};
const minBytesPerSec = minDownloadBytesPerSec();
let throughput: ThroughputWindow | undefined;
let tooSlow = false;
let floorCheckId: ReturnType<typeof setTimeout> | undefined;
const checkThroughputFloor = (): void => {
if (throughput?.isBelow(minBytesPerSec, Date.now())) {
tooSlow = true;
controller.abort();
return;
}
floorCheckId = setTimeout(checkThroughputFloor, 1000);
};
const onBodyBytes = (byteCount: number, sized: boolean): void => {
if (sized) extendTimeout();
if (minBytesPerSec === 0) return;
if (!throughput) {
throughput = new ThroughputWindow(Date.now());
floorCheckId = setTimeout(checkThroughputFloor, 1000);
}
throughput.record(byteCount, Date.now());
};
const identity = safeDownloadUrlIdentity(url);
try {
if (callerAborted) {
throw new UrlDownloadError("cancelled", false, "Download cancelled");
}
const integrity = await fetchToPartial(url, partialPath, controller, options, extendTimeout);
const integrity = await fetchToPartial(url, partialPath, controller, options, onBodyBytes);
clearTimeout(floorCheckId);
if (await playlistLeavesItsFolder(partialPath, localPath)) {
throw new UrlDownloadError(
"invalid_payload",
@@ -1171,6 +1293,21 @@ async function runDownloadAttempt(
});
throw classified;
}
if (tooSlow) {
const classified = new UrlDownloadError(
"timeout",
true,
`Download too slow: under ${minBytesPerSec} bytes/s over the last ${THROUGHPUT_WINDOW_SECONDS}s`,
);
emitDownloadTelemetry(options, {
urlFingerprint: identity.urlFingerprint,
initialHost: identity.host,
attempt,
outcome: "attempt_failed",
failureKind: classified.kind,
});
throw classified;
}
const classified = classifyDownloadFailure(error);
emitDownloadTelemetry(options, {
urlFingerprint: identity.urlFingerprint,
@@ -1183,6 +1320,7 @@ async function runDownloadAttempt(
throw classified;
} finally {
clearTimeout(timeoutId);
clearTimeout(floorCheckId);
signal?.removeEventListener("abort", onCallerAbort);
controller.abort();
rmSync(attemptDir, { recursive: true, force: true });