From 5111cf3840c37267f954f4e7a4f64a99b1bc0fdc Mon Sep 17 00:00:00 2001 From: James Russo Date: Sun, 15 Nov 2026 01:47:10 +0000 Subject: [PATCH] fix(engine): bound remote media downloads by size, encoding and throughput (#7786) downloadToTemp now refuses any Content-Encoding other than identity, caps each download at 20 GiB (HYPERFRAMES_MAX_DOWNLOAD_BYTES) on the declared Content-Length and while streaming, and aborts a transfer whose bytes over the last 60 s average below 32 KiB/s (HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC, 0 disables). Compressed and over-limit responses are not retried. A throughput abort is a retryable timeout like the stall timer, which is unchanged. A sized body that trickles one chunk every 100 s is now cut by the floor instead of finishing. Co-authored-by: Claude Opus 5.5 --- .../src/utils/urlDownloader.limits.test.ts | 133 ++++++++++++++++ .../src/utils/urlDownloader.progress.test.ts | 115 +++++++++++++- packages/engine/src/utils/urlDownloader.ts | 150 +++++++++++++++++- 3 files changed, 388 insertions(+), 10 deletions(-) create mode 100644 packages/engine/src/utils/urlDownloader.limits.test.ts diff --git a/packages/engine/src/utils/urlDownloader.limits.test.ts b/packages/engine/src/utils/urlDownloader.limits.test.ts new file mode 100644 index 0000000000..708907c1b5 --- /dev/null +++ b/packages/engine/src/utils/urlDownloader.limits.test.ts @@ -0,0 +1,133 @@ +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { _setDownloadFetchForTests, downloadToTemp, UrlDownloadError } from "./urlDownloader.js"; + +const stubFetch = (replacement: unknown): void => + _setDownloadFetchForTests(replacement as Parameters[0]); + +const dirs: string[] = []; + +afterEach(() => { + vi.unstubAllEnvs(); + _setDownloadFetchForTests(); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "hf-url-limits-")); + dirs.push(dir); + return dir; +} + +const MEDIA = "media-payload-bytes"; + +function mediaResponse(headers: Record = {}): Response { + return new Response(MEDIA, { headers: { "content-length": `${MEDIA.length}`, ...headers } }); +} + +describe("downloadToTemp's Content-Encoding policy", () => { + it.each(["gzip", "br", " GZIP ", "identity, gzip"])( + "refuses a %j response without retrying or leaving a file", + async (encoding) => { + const fetchMock = vi.fn(async () => mediaResponse({ "content-encoding": encoding })); + stubFetch(fetchMock); + const dir = tempDir(); + + await expect( + downloadToTemp("https://cdn.example/encoded.mp4", dir, 1_000), + ).rejects.toMatchObject({ + kind: "http_rejected", + retryable: false, + message: "Download response is compressed; only identity Content-Encoding is accepted", + } satisfies Partial); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(readdirSync(dir)).toEqual([]); + }, + ); + + it.each([ + ["absent", {}], + ["identity", { "content-encoding": "identity" }], + ["padded Identity", { "content-encoding": " Identity " }], + ])("downloads a response whose Content-Encoding is %s", async (_case, headers) => { + stubFetch(vi.fn(async () => mediaResponse(headers))); + + const path = await downloadToTemp("https://cdn.example/plain.mp4", tempDir(), 1_000); + + expect(readFileSync(path, "utf8")).toBe(MEDIA); + }); +}); + +describe("downloadToTemp's byte limit", () => { + it("refuses a declared Content-Length over the limit before reading the body", async () => { + vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", "16"); + const cancel = vi.fn(); + // The body never sends, so a reader would hang until the 1 s timeout instead. + const body = new ReadableStream({ cancel }); + const fetchMock = vi.fn( + async () => new Response(body, { headers: { "content-length": "17" } }), + ); + stubFetch(fetchMock); + const dir = tempDir(); + + await expect(downloadToTemp("https://cdn.example/big.mp4", dir, 1_000)).rejects.toMatchObject({ + kind: "length_mismatch", + retryable: false, + message: "Download exceeded the 16-byte limit", + telemetry: expect.objectContaining({ expectedBytes: 17 }), + } satisfies Partial); + expect(cancel).toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(readdirSync(dir)).toEqual([]); + }); + + it("stops an endless body with no Content-Length at the limit and removes the partial", async () => { + const limit = 1024 * 1024; + const chunkBytes = 64 * 1024; + vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", `${limit}`); + const cancel = vi.fn(); + const fetchMock = vi.fn( + async () => + new Response( + new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(chunkBytes).fill(0x61)); + }, + cancel, + }), + ), + ); + stubFetch(fetchMock); + const dir = tempDir(); + + const error = await downloadToTemp("https://cdn.example/endless.mp4", dir, 10_000).catch( + (caught: unknown) => caught, + ); + + expect(error).toMatchObject({ + kind: "length_mismatch", + retryable: false, + message: `Download exceeded the ${limit}-byte limit`, + }); + const receivedBytes = (error as UrlDownloadError).telemetry?.receivedBytes ?? 0; + expect(receivedBytes).toBeGreaterThan(limit); + expect(receivedBytes).toBeLessThanOrEqual(limit + chunkBytes); + expect(cancel).toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(readdirSync(dir)).toEqual([]); + }); + + it.each(["", "abc", "0", "-5", "1.5"])( + "keeps the default limit when HYPERFRAMES_MAX_DOWNLOAD_BYTES is %j", + async (value) => { + vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", value); + stubFetch(vi.fn(async () => mediaResponse())); + + const path = await downloadToTemp("https://cdn.example/default-limit.mp4", tempDir(), 1_000); + + expect(readFileSync(path, "utf8")).toBe(MEDIA); + }, + ); +}); diff --git a/packages/engine/src/utils/urlDownloader.progress.test.ts b/packages/engine/src/utils/urlDownloader.progress.test.ts index 8de7604967..e81433f80e 100644 --- a/packages/engine/src/utils/urlDownloader.progress.test.ts +++ b/packages/engine/src/utils/urlDownloader.progress.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -15,6 +15,7 @@ beforeEach(() => { afterEach(() => { vi.useRealTimers(); + vi.unstubAllEnvs(); _setDownloadFetchForTests(); for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); @@ -60,6 +61,48 @@ function trickle( return new Response(body, { headers }); } +/** + * A sized body that sends `chunk` `everyMs` after each read, `count` times. Waiting for + * `readerWaiting()` before advancing the clock keeps real disk writes from lagging fake time. + */ +function pacedBody(chunk: Uint8Array, count: number, everyMs: number) { + let sent = 0; + let waiting = false; + const body = new ReadableStream( + { + pull(controller) { + waiting = true; + return new Promise((resolve) => { + setTimeout(() => { + waiting = false; + controller.enqueue(chunk); + sent += 1; + if (sent === count) controller.close(); + resolve(); + }, everyMs); + }); + }, + }, + { highWaterMark: 0 }, + ); + const response = new Response(body, { + headers: { "content-length": `${chunk.length * count}` }, + }); + return { + response, + readerWaiting: () => waiting || sent === count, + finished: () => sent === count, + }; +} + +/** Like `waitForIo`, but bounded by real time so a loaded disk cannot fall behind the fake clock. */ +async function waitForRealIo(done: () => boolean, budgetMs = 10_000): Promise { + const deadline = performance.now() + budgetMs; + while (!done() && performance.now() < deadline) { + await new Promise((resolve) => setImmediate(resolve)); + } +} + /** Real I/O (cache lock, file writes) settles on the real event loop; the fake clock stays put. */ async function waitForIo(done: () => boolean, turns = 100_000): Promise { for (let i = 0; i < turns && !done(); i += 1) { @@ -82,16 +125,44 @@ async function startDownload(response: () => Response) { const fetchMock = vi.fn(async () => response()); stubFetch(fetchMock); const outcome: { path?: string; error?: unknown } = {}; - void downloadToTemp("https://cdn.example/media.mp4", tempDir()).then( + const dir = tempDir(); + void downloadToTemp("https://cdn.example/media.mp4", dir).then( (path) => (outcome.path = path), (error: unknown) => (outcome.error = error), ); await waitForIo(fetchCalls(fetchMock, 1)); - return { fetchMock, outcome }; + return { fetchMock, outcome, dir }; } +const disableThroughputFloor = (): void => + vi.stubEnv("HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC", "0"); + +const tooSlow = { + kind: "timeout", + retryable: true, + message: "Download too slow: under 32768 bytes/s over the last 60s", +}; + describe("downloadToTemp's timeout", () => { - it("lets a sized download that sends a chunk every 100 s for 400 s finish", async () => { + it("cuts a sized download that sends a chunk every 100 s at the throughput floor", async () => { + const chunks = ["chunk-1 ", "chunk-2 ", "chunk-3 ", "chunk-4"]; + const { fetchMock, outcome, dir } = await startDownload(() => + trickle(chunks, 100_000, { contentLength: chunks.join("").length }), + ); + await advanceSeconds(159); + expect(fetchMock).toHaveBeenCalledTimes(1); + await advanceSeconds(1); + await waitForIo(fetchCalls(fetchMock, 2)); + expect(fetchMock).toHaveBeenCalledTimes(2); + await advanceSeconds(160); + await waitForIo(() => "error" in outcome); + + expect(outcome.error).toMatchObject(tooSlow); + expect(readdirSync(dir)).toEqual([]); + }); + + it("lets the same 100 s trickle finish when the floor is disabled with 0", async () => { + disableThroughputFloor(); const chunks = ["chunk-1 ", "chunk-2 ", "chunk-3 ", "chunk-4"]; const { fetchMock, outcome } = await startDownload(() => trickle(chunks, 100_000, { contentLength: chunks.join("").length }), @@ -104,7 +175,42 @@ describe("downloadToTemp's timeout", () => { expect(fetchMock).toHaveBeenCalledTimes(1); }); + it("cuts a 1 KiB/s trickle that declares a huge Content-Length and removes the partial", async () => { + const { fetchMock, outcome, dir } = await startDownload(() => + trickle(["x".repeat(1024)], 1_000, { contentLength: 10 * 1024 ** 3, then: "repeat" }), + ); + await advanceSeconds(60); + expect(fetchMock).toHaveBeenCalledTimes(1); + await advanceSeconds(1); + await waitForIo(fetchCalls(fetchMock, 2)); + expect(fetchMock).toHaveBeenCalledTimes(2); + await advanceSeconds(61); + await waitForIo(() => "error" in outcome); + + expect(outcome.error).toMatchObject(tooSlow); + expect(readdirSync(dir)).toEqual([]); + }); + + it("lets a steady sized download above the floor run past 15 minutes", async () => { + const chunk = new Uint8Array(40 * 1024).fill(0x78); + const seconds = 15 * 60 + 20; + const paced = pacedBody(chunk, seconds, 1_000); + const startedAt = Date.now(); + const { fetchMock, outcome } = await startDownload(() => paced.response); + while (!paced.finished() && !("error" in outcome)) { + await waitForRealIo(paced.readerWaiting); + await vi.advanceTimersByTimeAsync(1_000); + } + await waitForRealIo(() => "path" in outcome || "error" in outcome); + + expect(outcome.error).toBeUndefined(); + expect(Date.now() - startedAt).toBeGreaterThan(15 * 60_000); + expect(statSync(outcome.path!).size).toBe(chunk.length * seconds); + expect(fetchMock).toHaveBeenCalledTimes(1); + }, 120_000); + it("cuts a body with no Content-Length at the fixed 300 s cap even while it streams", async () => { + disableThroughputFloor(); const { fetchMock, outcome } = await startDownload(() => trickle(["tick "], 10_000, { then: "repeat" }), ); @@ -141,6 +247,7 @@ describe("downloadToTemp's timeout", () => { }); it("fails a sized body 300 s after its last chunk and says it stalled", async () => { + disableThroughputFloor(); const { fetchMock, outcome } = await startDownload(() => trickle(["part-1 "], 100_000, { contentLength: 1_000, then: "hang" }), ); diff --git a/packages/engine/src/utils/urlDownloader.ts b/packages/engine/src/utils/urlDownloader.ts index 6cc61c86c0..6f6b38a1b9 100644 --- a/packages/engine/src/utils/urlDownloader.ts +++ b/packages/engine/src/utils/urlDownloader.ts @@ -796,6 +796,79 @@ function checksumMismatchError( ); } +const DEFAULT_MAX_DOWNLOAD_BYTES = 20 * 1024 ** 3; +const DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC = 32 * 1024; +const THROUGHPUT_WINDOW_SECONDS = 60; + +function integerFromEnv(name: string, fallback: number, minimum: number): number { + const raw = process.env[name]?.trim(); + if (!raw || !/^\d+$/.test(raw)) return fallback; + const value = Number(raw); + return Number.isSafeInteger(value) && value >= minimum ? value : fallback; +} + +const maxDownloadBytes = (): number => + integerFromEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", DEFAULT_MAX_DOWNLOAD_BYTES, 1); + +const minDownloadBytesPerSec = (): number => + integerFromEnv("HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC", DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC, 0); + +function isCompressedEncoding(contentEncoding: string | null): boolean { + const encoding = contentEncoding?.trim().toLowerCase(); + return Boolean(encoding) && encoding !== "identity"; +} + +function overByteLimitError( + maxBytes: number, + status: number, + telemetry: Partial, +): UrlDownloadError { + return new UrlDownloadError( + "length_mismatch", + false, + `Download exceeded the ${maxBytes}-byte limit`, + status, + telemetry, + ); +} + +// One spare bucket so the summed span is never shorter than the window, which errs toward keeping +// a download that runs right at the floor. +class ThroughputWindow { + private readonly buckets = new Array(THROUGHPUT_WINDOW_SECONDS + 1).fill(0); + private bytesInWindow = 0; + private currentSecond: number; + + constructor(private readonly startedAtMs: number) { + this.currentSecond = Math.floor(startedAtMs / 1000); + } + + record(byteCount: number, nowMs: number): void { + this.advanceTo(nowMs); + this.buckets[this.currentSecond % this.buckets.length]! += byteCount; + this.bytesInWindow += byteCount; + } + + isBelow(minBytesPerSec: number, nowMs: number): boolean { + this.advanceTo(nowMs); + return ( + nowMs - this.startedAtMs >= THROUGHPUT_WINDOW_SECONDS * 1000 && + this.bytesInWindow < minBytesPerSec * THROUGHPUT_WINDOW_SECONDS + ); + } + + private advanceTo(nowMs: number): void { + const second = Math.floor(nowMs / 1000); + const expired = Math.min(second - this.currentSecond, this.buckets.length); + for (let step = 1; step <= expired; step += 1) { + const index = (this.currentSecond + step) % this.buckets.length; + this.bytesInWindow -= this.buckets[index]!; + this.buckets[index] = 0; + } + this.currentSecond = Math.max(second, this.currentSecond); + } +} + // Response protocol, streamed byte accounting, hashes, and payload validation // share one lifecycle so no validation can happen after publication. // fallow-ignore-next-line complexity @@ -804,7 +877,7 @@ async function fetchToPartial( partialPath: string, controller: AbortController, options: UrlDownloadOptions, - onSizedBodyBytes: () => void, + onBodyBytes: (byteCount: number, sized: boolean) => void, ): Promise { const { response, finalUrl } = await fetchWithValidatedRedirects(url, controller); const finalIdentity = safeDownloadUrlIdentity(finalUrl); @@ -869,9 +942,26 @@ async function fetchToPartial( } throw error; } - const contentEncoding = response.headers.get("content-encoding")?.trim().toLowerCase(); - const expectedBytes = - !contentEncoding || contentEncoding === "identity" ? declaredLength : undefined; + const responseTelemetry = { finalHost: finalIdentity.host, status: response.status }; + if (isCompressedEncoding(response.headers.get("content-encoding"))) { + await cancelResponseBody(response); + throw new UrlDownloadError( + "http_rejected", + false, + "Download response is compressed; only identity Content-Encoding is accepted", + response.status, + responseTelemetry, + ); + } + const maxBytes = maxDownloadBytes(); + if (declaredLength !== undefined && declaredLength > maxBytes) { + await cancelResponseBody(response); + throw overByteLimitError(maxBytes, response.status, { + ...responseTelemetry, + expectedBytes: declaredLength, + }); + } + const expectedBytes = declaredLength; let receivedBytes = 0; const sha256 = createHash("sha256"); @@ -881,8 +971,18 @@ async function fetchToPartial( const inspector = new Transform({ transform(chunk: Buffer, _encoding, callback) { const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - if (declaredLength !== undefined) onSizedBodyBytes(); + onBodyBytes(bytes.length, declaredLength !== undefined); receivedBytes += bytes.length; + if (receivedBytes > maxBytes) { + callback( + overByteLimitError(maxBytes, response.status, { + ...responseTelemetry, + expectedBytes, + receivedBytes, + }), + ); + return; + } sha256.update(bytes); md5.update(bytes); if (prefixBytes < 1024) { @@ -1084,13 +1184,35 @@ async function runDownloadAttempt( sizedBodyStarted = true; armTimeout(); }; + const minBytesPerSec = minDownloadBytesPerSec(); + let throughput: ThroughputWindow | undefined; + let tooSlow = false; + let floorCheckId: ReturnType | undefined; + const checkThroughputFloor = (): void => { + if (throughput?.isBelow(minBytesPerSec, Date.now())) { + tooSlow = true; + controller.abort(); + return; + } + floorCheckId = setTimeout(checkThroughputFloor, 1000); + }; + const onBodyBytes = (byteCount: number, sized: boolean): void => { + if (sized) extendTimeout(); + if (minBytesPerSec === 0) return; + if (!throughput) { + throughput = new ThroughputWindow(Date.now()); + floorCheckId = setTimeout(checkThroughputFloor, 1000); + } + throughput.record(byteCount, Date.now()); + }; const identity = safeDownloadUrlIdentity(url); try { if (callerAborted) { throw new UrlDownloadError("cancelled", false, "Download cancelled"); } - const integrity = await fetchToPartial(url, partialPath, controller, options, extendTimeout); + const integrity = await fetchToPartial(url, partialPath, controller, options, onBodyBytes); + clearTimeout(floorCheckId); if (await playlistLeavesItsFolder(partialPath, localPath)) { throw new UrlDownloadError( "invalid_payload", @@ -1171,6 +1293,21 @@ async function runDownloadAttempt( }); throw classified; } + if (tooSlow) { + const classified = new UrlDownloadError( + "timeout", + true, + `Download too slow: under ${minBytesPerSec} bytes/s over the last ${THROUGHPUT_WINDOW_SECONDS}s`, + ); + emitDownloadTelemetry(options, { + urlFingerprint: identity.urlFingerprint, + initialHost: identity.host, + attempt, + outcome: "attempt_failed", + failureKind: classified.kind, + }); + throw classified; + } const classified = classifyDownloadFailure(error); emitDownloadTelemetry(options, { urlFingerprint: identity.urlFingerprint, @@ -1183,6 +1320,7 @@ async function runDownloadAttempt( throw classified; } finally { clearTimeout(timeoutId); + clearTimeout(floorCheckId); signal?.removeEventListener("abort", onCallerAbort); controller.abort(); rmSync(attemptDir, { recursive: true, force: true });