fix(engine): remote downloads get an absolute deadline and stop before the disk fills (#7827)

A download whose response declares a Content-Length re-arms its attempt
timeout on every chunk, so a server that trickles just above the throughput
floor was never cut off by time. Parallel downloads also had no shared limit
on the disk they could fill.

- Each download attempt now has a deadline that incoming data never moves:
  30 minutes by default, HYPERFRAMES_MAX_DOWNLOAD_MS to change it. Passing it
  fails the download without a local retry.
- The default per-file cap drops from 20 GiB to 8 GiB
  (HYPERFRAMES_MAX_DOWNLOAD_BYTES still overrides it).
- Downloads keep 512 MiB free on their volume
  (HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES, 0 turns it off). A declared size is
  claimed before the body is read and counted against every parallel
  download on the same device, and free space is measured again every 4 MiB
  as bytes arrive. Free space that cannot be measured does not block.
- docs/packages/engine.mdx lists the download limits and their variables.
This commit is contained in:
Michael Wang
2026-11-15 17:17:44 +00:00
committed by GitHub
parent 3a4f572523
commit e1043659d4
4 changed files with 286 additions and 37 deletions
+20
View File
@@ -81,6 +81,26 @@ software WebGPU may use SwiftShader, but it must also create a device. The check
releases its temporary device; the composition still creates its own device
with any features or limits its renderer needs.
## Remote media downloads
The engine downloads remote `https://` video, audio, image, and font sources
into the render's work directory before it uses them. Each download has these
limits, and an environment variable changes each one:
| Limit | Default | Environment variable |
| --- | --- | --- |
| Largest file | 8 GiB | `HYPERFRAMES_MAX_DOWNLOAD_BYTES` |
| Longest download, however fast data arrives | 30 minutes | `HYPERFRAMES_MAX_DOWNLOAD_MS` |
| Slowest average speed over a minute | 32 KiB/s | `HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC` |
| Free space left on the download volume | 512 MiB | `HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES` |
Sizes are in bytes and the time limit is in milliseconds. A speed or free-space
value of `0` turns that check off.
A download that passes one of these limits stops with an error. The free-space
check counts the declared sizes of all downloads running at the same time, so
parallel downloads cannot fill the volume together.
## What else the package exposes
| Surface | Use it for |
@@ -1,15 +1,32 @@
import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { _setDownloadFetchForTests, downloadToTemp, UrlDownloadError } from "./urlDownloader.js";
const fakeDisk = vi.hoisted(() => {
const state: { freeBytes?: (path: string) => number | undefined } = {};
return state;
});
vi.mock("fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("fs")>();
return {
...actual,
statfsSync: (path: string) => {
const freeBytes = fakeDisk.freeBytes?.(path);
return freeBytes === undefined ? actual.statfsSync(path) : { bavail: freeBytes, bsize: 1 };
},
};
});
const stubFetch = (replacement: unknown): void =>
_setDownloadFetchForTests(replacement as Parameters<typeof _setDownloadFetchForTests>[0]);
const dirs: string[] = [];
afterEach(() => {
fakeDisk.freeBytes = undefined;
vi.unstubAllEnvs();
_setDownloadFetchForTests();
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
@@ -131,3 +148,87 @@ describe("downloadToTemp's byte limit", () => {
},
);
});
const MiB = 1024 ** 2;
function bytesUnder(dir: string): number {
return readdirSync(dir, { withFileTypes: true }).reduce((total, entry) => {
const path = join(dir, entry.name);
return total + (entry.isDirectory() ? bytesUnder(path) : statSync(path).size);
}, 0);
}
/** Reports `capacity` minus what is written under `dir` as free space there; returns the lowest seen. */
function fakeVolume(dir: string, capacity: number): { lowestFree: () => number } {
let lowest = capacity;
fakeDisk.freeBytes = (path) => {
if (!path.startsWith(dir)) return undefined;
const free = capacity - bytesUnder(dir);
lowest = Math.min(lowest, free);
return free;
};
return { lowestFree: () => lowest };
}
function mibBody(chunks: number | "endless", declare: boolean): Response {
let sent = 0;
const body = new ReadableStream<Uint8Array>({
pull(controller) {
controller.enqueue(new Uint8Array(MiB).fill(0x61));
sent += 1;
if (chunks !== "endless" && sent === chunks) controller.close();
},
});
const headers =
declare && chunks !== "endless" ? { "content-length": `${chunks * MiB}` } : undefined;
return new Response(body, { headers });
}
const lowDisk = {
kind: "filesystem",
retryable: false,
message: `Download stopped: it would leave less than ${32 * MiB} bytes free on the download volume`,
};
describe("downloadToTemp's free-space guard", () => {
it("refuses parallel declared bodies that together would cut into the free-space minimum", async () => {
vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", `${32 * MiB}`);
const dir = tempDir();
const volume = fakeVolume(dir, 96 * MiB);
const fetchMock = vi.fn(async () => mibBody(40, true));
stubFetch(fetchMock);
const results = await Promise.allSettled(
[1, 2, 3].map((n) => downloadToTemp(`https://cdn.example/large-${n}.mp4`, dir, 30_000)),
);
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
const refusals = results.flatMap((result) =>
result.status === "rejected" ? [result.reason] : [],
);
expect(refusals).toEqual([expect.objectContaining(lowDisk), expect.objectContaining(lowDisk)]);
expect(fetchMock).toHaveBeenCalledTimes(3);
expect(readdirSync(dir)).toHaveLength(1);
expect(volume.lowestFree()).toBeGreaterThanOrEqual(32 * MiB);
});
it("stops parallel endless bodies with no Content-Length before the volume fills", async () => {
vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", `${32 * MiB}`);
vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", `${128 * MiB}`);
const dir = tempDir();
const volume = fakeVolume(dir, 96 * MiB);
const fetchMock = vi.fn(async () => mibBody("endless", false));
stubFetch(fetchMock);
const results = await Promise.allSettled(
[1, 2, 3].map((n) => downloadToTemp(`https://cdn.example/endless-${n}.mp4`, dir, 30_000)),
);
expect(results).toEqual(
[1, 2, 3].map(() => ({ status: "rejected", reason: expect.objectContaining(lowDisk) })),
);
expect(fetchMock).toHaveBeenCalledTimes(3);
expect(readdirSync(dir)).toEqual([]);
expect(volume.lowestFree()).toBeGreaterThan(0);
});
});
@@ -68,12 +68,13 @@ function trickle(
function pacedBody(chunk: Uint8Array, count: number, everyMs: number) {
let sent = 0;
let waiting = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const body = new ReadableStream<Uint8Array>(
{
pull(controller) {
waiting = true;
return new Promise<void>((resolve) => {
setTimeout(() => {
timer = setTimeout(() => {
waiting = false;
controller.enqueue(chunk);
sent += 1;
@@ -82,6 +83,9 @@ function pacedBody(chunk: Uint8Array, count: number, everyMs: number) {
}, everyMs);
});
},
cancel() {
clearTimeout(timer);
},
},
{ highWaterMark: 0 },
);
@@ -176,8 +180,9 @@ describe("downloadToTemp's timeout", () => {
});
it("cuts a 1 KiB/s trickle that declares a huge Content-Length and removes the partial", async () => {
vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", "0");
const { fetchMock, outcome, dir } = await startDownload(() =>
trickle(["x".repeat(1024)], 1_000, { contentLength: 10 * 1024 ** 3, then: "repeat" }),
trickle(["x".repeat(1024)], 1_000, { contentLength: 6 * 1024 ** 3, then: "repeat" }),
);
await advanceSeconds(60);
expect(fetchMock).toHaveBeenCalledTimes(1);
@@ -209,6 +214,26 @@ describe("downloadToTemp's timeout", () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
}, 120_000);
it("stops a steady sized download above the floor at the absolute deadline without retrying", async () => {
vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_MS", "120000");
const chunk = new Uint8Array(40 * 1024).fill(0x78);
const paced = pacedBody(chunk, 10_000, 1_000);
const { fetchMock, outcome, dir } = await startDownload(() => paced.response);
for (let s = 0; s < 200 && !("error" in outcome); s += 1) {
await waitForRealIo(paced.readerWaiting);
await vi.advanceTimersByTimeAsync(1_000);
}
await waitForRealIo(() => "error" in outcome);
expect(outcome.error).toMatchObject({
kind: "timeout",
retryable: false,
message: "Download exceeded the 120s time limit",
});
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(readdirSync(dir)).toEqual([]);
}, 60_000);
it("cuts a body with no Content-Length at the fixed 300 s cap even while it streams", async () => {
disableThroughputFloor();
const { fetchMock, outcome } = await startDownload(() =>
+137 -34
View File
@@ -12,6 +12,7 @@ import {
rmdirSync,
rmSync,
statSync,
statfsSync,
type Stats,
unlinkSync,
} from "fs";
@@ -847,7 +848,10 @@ function checksumMismatchError(
);
}
const DEFAULT_MAX_DOWNLOAD_BYTES = 20 * 1024 ** 3;
const DEFAULT_MAX_DOWNLOAD_BYTES = 8 * 1024 ** 3;
const DEFAULT_MAX_DOWNLOAD_MS = 30 * 60 * 1000;
const DEFAULT_DOWNLOAD_MIN_FREE_BYTES = 512 * 1024 ** 2;
const FREE_SPACE_CHECK_INTERVAL_BYTES = 4 * 1024 ** 2;
const DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC = 32 * 1024;
const THROUGHPUT_WINDOW_SECONDS = 60;
@@ -864,6 +868,87 @@ const maxDownloadBytes = (): number =>
const minDownloadBytesPerSec = (): number =>
integerFromEnv("HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC", DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC, 0);
const maxDownloadMs = (): number =>
integerFromEnv("HYPERFRAMES_MAX_DOWNLOAD_MS", DEFAULT_MAX_DOWNLOAD_MS, 1);
const downloadMinFreeBytes = (): number =>
integerFromEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", DEFAULT_DOWNLOAD_MIN_FREE_BYTES, 0);
function freeDiskBytes(path: string): number | null {
try {
const stat = statfsSync(path);
return stat.bavail * stat.bsize;
} catch {
return null;
}
}
// Bytes that in-flight downloads declared but have not received yet, per device.
const unreceivedBytesByDevice = new Map<number, number>();
/**
* Keeps `minFreeBytes` free on the download volume across parallel downloads: a declared size
* is claimed up front, and free space is measured again every few MiB as bytes arrive. Free
* space that cannot be measured counts as enough, and a minimum of 0 turns the guard off.
*/
class DiskHeadroomGuard {
private unreceivedBytes = 0;
private bytesSinceCheck = 0;
private device = 0;
constructor(
private readonly dir: string,
readonly minFreeBytes: number,
) {}
claim(declaredBytes: number): boolean {
if (this.minFreeBytes === 0) return true;
this.device = statSync(this.dir).dev;
this.adjustUnreceived(declaredBytes);
return this.keepsMinimum();
}
receive(byteCount: number): boolean {
if (this.minFreeBytes === 0) return true;
this.adjustUnreceived(-Math.min(byteCount, this.unreceivedBytes));
this.bytesSinceCheck += byteCount;
if (this.bytesSinceCheck < FREE_SPACE_CHECK_INTERVAL_BYTES) return true;
this.bytesSinceCheck = 0;
return this.keepsMinimum();
}
release(): void {
this.adjustUnreceived(-this.unreceivedBytes);
}
private keepsMinimum(): boolean {
const free = freeDiskBytes(this.dir);
if (free === null) return true;
return free - (unreceivedBytesByDevice.get(this.device) ?? 0) >= this.minFreeBytes;
}
private adjustUnreceived(delta: number): void {
this.unreceivedBytes += delta;
const total = (unreceivedBytesByDevice.get(this.device) ?? 0) + delta;
if (total > 0) unreceivedBytesByDevice.set(this.device, total);
else unreceivedBytesByDevice.delete(this.device);
}
}
function lowDiskSpaceError(
minFreeBytes: number,
status: number,
telemetry: Partial<UrlDownloadTelemetry>,
): UrlDownloadError {
return new UrlDownloadError(
"filesystem",
false,
`Download stopped: it would leave less than ${minFreeBytes} bytes free on the download volume`,
status,
telemetry,
);
}
function isCompressedEncoding(contentEncoding: string | null): boolean {
const encoding = contentEncoding?.trim().toLowerCase();
return Boolean(encoding) && encoding !== "identity";
@@ -929,6 +1014,7 @@ async function fetchToPartial(
controller: AbortController,
options: UrlDownloadOptions,
onBodyBytes: (byteCount: number, sized: boolean) => void,
headroom: DiskHeadroomGuard,
): Promise<PartialIntegrity> {
const { response, finalUrl } = await fetchWithValidatedRedirects(url, controller);
const finalIdentity = safeDownloadUrlIdentity(finalUrl);
@@ -1012,6 +1098,13 @@ async function fetchToPartial(
expectedBytes: declaredLength,
});
}
if (!headroom.claim(declaredLength ?? 0)) {
await cancelResponseBody(response);
throw lowDiskSpaceError(headroom.minFreeBytes, response.status, {
...responseTelemetry,
expectedBytes: declaredLength,
});
}
const expectedBytes = declaredLength;
let receivedBytes = 0;
@@ -1034,6 +1127,16 @@ async function fetchToPartial(
);
return;
}
if (!headroom.receive(bytes.length)) {
callback(
lowDiskSpaceError(headroom.minFreeBytes, response.status, {
...responseTelemetry,
expectedBytes,
receivedBytes,
}),
);
return;
}
sha256.update(bytes);
md5.update(bytes);
if (prefixBytes < 1024) {
@@ -1230,6 +1333,13 @@ async function runDownloadAttempt(
}, timeoutMs);
};
armTimeout();
// Unlike the attempt timeout, incoming data never moves this deadline.
const deadlineMs = maxDownloadMs();
let pastDeadline = false;
const deadlineId = setTimeout(() => {
pastDeadline = true;
controller.abort();
}, deadlineMs);
let sizedBodyStarted = false;
const extendTimeout = (): void => {
sizedBodyStarted = true;
@@ -1257,12 +1367,21 @@ async function runDownloadAttempt(
throughput.record(byteCount, Date.now());
};
const identity = safeDownloadUrlIdentity(url);
const headroom = new DiskHeadroomGuard(attemptDir, downloadMinFreeBytes());
try {
if (callerAborted) {
throw new UrlDownloadError("cancelled", false, "Download cancelled");
}
const integrity = await fetchToPartial(url, partialPath, controller, options, onBodyBytes);
const integrity = await fetchToPartial(
url,
partialPath,
controller,
options,
onBodyBytes,
headroom,
);
clearTimeout(deadlineId);
clearTimeout(floorCheckId);
if (await playlistLeavesItsFolder(partialPath, localPath)) {
throw new UrlDownloadError(
@@ -1316,50 +1435,32 @@ async function runDownloadAttempt(
});
return localPath;
} catch (error) {
let classified: UrlDownloadError;
if (callerAborted) {
const classified = new UrlDownloadError("cancelled", false, "Download cancelled");
emitDownloadTelemetry(options, {
urlFingerprint: identity.urlFingerprint,
initialHost: identity.host,
attempt,
outcome: "attempt_failed",
failureKind: classified.kind,
});
throw classified;
}
if (timedOut) {
const classified = new UrlDownloadError(
classified = new UrlDownloadError("cancelled", false, "Download cancelled");
} else if (pastDeadline) {
classified = new UrlDownloadError(
"timeout",
false,
`Download exceeded the ${deadlineMs / 1000}s time limit`,
);
} else if (timedOut) {
classified = new UrlDownloadError(
"timeout",
true,
sizedBodyStarted
? `Download stalled: no data for ${timeoutMs / 1000}s`
: `Download timeout after ${timeoutMs / 1000}s`,
);
emitDownloadTelemetry(options, {
urlFingerprint: identity.urlFingerprint,
initialHost: identity.host,
attempt,
outcome: "attempt_failed",
failureKind: classified.kind,
});
throw classified;
}
if (tooSlow) {
const classified = new UrlDownloadError(
} else if (tooSlow) {
classified = new UrlDownloadError(
"timeout",
true,
`Download too slow: under ${minBytesPerSec} bytes/s over the last ${THROUGHPUT_WINDOW_SECONDS}s`,
);
emitDownloadTelemetry(options, {
urlFingerprint: identity.urlFingerprint,
initialHost: identity.host,
attempt,
outcome: "attempt_failed",
failureKind: classified.kind,
});
throw classified;
} else {
classified = classifyDownloadFailure(error);
}
const classified = classifyDownloadFailure(error);
emitDownloadTelemetry(options, {
urlFingerprint: identity.urlFingerprint,
initialHost: identity.host,
@@ -1371,7 +1472,9 @@ async function runDownloadAttempt(
throw classified;
} finally {
clearTimeout(timeoutId);
clearTimeout(deadlineId);
clearTimeout(floorCheckId);
headroom.release();
signal?.removeEventListener("abort", onCallerAbort);
controller.abort();
rmSync(attemptDir, { recursive: true, force: true });