diff --git a/docs/packages/engine.mdx b/docs/packages/engine.mdx index c344b2c8c2..dbf77134cb 100644 --- a/docs/packages/engine.mdx +++ b/docs/packages/engine.mdx @@ -81,6 +81,26 @@ software WebGPU may use SwiftShader, but it must also create a device. The check releases its temporary device; the composition still creates its own device with any features or limits its renderer needs. +## Remote media downloads + +The engine downloads remote `https://` video, audio, image, and font sources +into the render's work directory before it uses them. Each download has these +limits, and an environment variable changes each one: + +| Limit | Default | Environment variable | +| --- | --- | --- | +| Largest file | 8 GiB | `HYPERFRAMES_MAX_DOWNLOAD_BYTES` | +| Longest download, however fast data arrives | 30 minutes | `HYPERFRAMES_MAX_DOWNLOAD_MS` | +| Slowest average speed over a minute | 32 KiB/s | `HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC` | +| Free space left on the download volume | 512 MiB | `HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES` | + +Sizes are in bytes and the time limit is in milliseconds. A speed or free-space +value of `0` turns that check off. + +A download that passes one of these limits stops with an error. The free-space +check counts the declared sizes of all downloads running at the same time, so +parallel downloads cannot fill the volume together. + ## What else the package exposes | Surface | Use it for | diff --git a/packages/engine/src/utils/urlDownloader.limits.test.ts b/packages/engine/src/utils/urlDownloader.limits.test.ts index 708907c1b5..ad28bd188f 100644 --- a/packages/engine/src/utils/urlDownloader.limits.test.ts +++ b/packages/engine/src/utils/urlDownloader.limits.test.ts @@ -1,15 +1,32 @@ -import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { _setDownloadFetchForTests, downloadToTemp, UrlDownloadError } from "./urlDownloader.js"; +const fakeDisk = vi.hoisted(() => { + const state: { freeBytes?: (path: string) => number | undefined } = {}; + return state; +}); + +vi.mock("fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + statfsSync: (path: string) => { + const freeBytes = fakeDisk.freeBytes?.(path); + return freeBytes === undefined ? actual.statfsSync(path) : { bavail: freeBytes, bsize: 1 }; + }, + }; +}); + const stubFetch = (replacement: unknown): void => _setDownloadFetchForTests(replacement as Parameters[0]); const dirs: string[] = []; afterEach(() => { + fakeDisk.freeBytes = undefined; vi.unstubAllEnvs(); _setDownloadFetchForTests(); for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); @@ -131,3 +148,87 @@ describe("downloadToTemp's byte limit", () => { }, ); }); + +const MiB = 1024 ** 2; + +function bytesUnder(dir: string): number { + return readdirSync(dir, { withFileTypes: true }).reduce((total, entry) => { + const path = join(dir, entry.name); + return total + (entry.isDirectory() ? bytesUnder(path) : statSync(path).size); + }, 0); +} + +/** Reports `capacity` minus what is written under `dir` as free space there; returns the lowest seen. */ +function fakeVolume(dir: string, capacity: number): { lowestFree: () => number } { + let lowest = capacity; + fakeDisk.freeBytes = (path) => { + if (!path.startsWith(dir)) return undefined; + const free = capacity - bytesUnder(dir); + lowest = Math.min(lowest, free); + return free; + }; + return { lowestFree: () => lowest }; +} + +function mibBody(chunks: number | "endless", declare: boolean): Response { + let sent = 0; + const body = new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(MiB).fill(0x61)); + sent += 1; + if (chunks !== "endless" && sent === chunks) controller.close(); + }, + }); + const headers = + declare && chunks !== "endless" ? { "content-length": `${chunks * MiB}` } : undefined; + return new Response(body, { headers }); +} + +const lowDisk = { + kind: "filesystem", + retryable: false, + message: `Download stopped: it would leave less than ${32 * MiB} bytes free on the download volume`, +}; + +describe("downloadToTemp's free-space guard", () => { + it("refuses parallel declared bodies that together would cut into the free-space minimum", async () => { + vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", `${32 * MiB}`); + const dir = tempDir(); + const volume = fakeVolume(dir, 96 * MiB); + const fetchMock = vi.fn(async () => mibBody(40, true)); + stubFetch(fetchMock); + + const results = await Promise.allSettled( + [1, 2, 3].map((n) => downloadToTemp(`https://cdn.example/large-${n}.mp4`, dir, 30_000)), + ); + + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + const refusals = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + expect(refusals).toEqual([expect.objectContaining(lowDisk), expect.objectContaining(lowDisk)]); + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(readdirSync(dir)).toHaveLength(1); + expect(volume.lowestFree()).toBeGreaterThanOrEqual(32 * MiB); + }); + + it("stops parallel endless bodies with no Content-Length before the volume fills", async () => { + vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", `${32 * MiB}`); + vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", `${128 * MiB}`); + const dir = tempDir(); + const volume = fakeVolume(dir, 96 * MiB); + const fetchMock = vi.fn(async () => mibBody("endless", false)); + stubFetch(fetchMock); + + const results = await Promise.allSettled( + [1, 2, 3].map((n) => downloadToTemp(`https://cdn.example/endless-${n}.mp4`, dir, 30_000)), + ); + + expect(results).toEqual( + [1, 2, 3].map(() => ({ status: "rejected", reason: expect.objectContaining(lowDisk) })), + ); + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(readdirSync(dir)).toEqual([]); + expect(volume.lowestFree()).toBeGreaterThan(0); + }); +}); diff --git a/packages/engine/src/utils/urlDownloader.progress.test.ts b/packages/engine/src/utils/urlDownloader.progress.test.ts index e81433f80e..a48683bea4 100644 --- a/packages/engine/src/utils/urlDownloader.progress.test.ts +++ b/packages/engine/src/utils/urlDownloader.progress.test.ts @@ -68,12 +68,13 @@ function trickle( function pacedBody(chunk: Uint8Array, count: number, everyMs: number) { let sent = 0; let waiting = false; + let timer: ReturnType | undefined; const body = new ReadableStream( { pull(controller) { waiting = true; return new Promise((resolve) => { - setTimeout(() => { + timer = setTimeout(() => { waiting = false; controller.enqueue(chunk); sent += 1; @@ -82,6 +83,9 @@ function pacedBody(chunk: Uint8Array, count: number, everyMs: number) { }, everyMs); }); }, + cancel() { + clearTimeout(timer); + }, }, { highWaterMark: 0 }, ); @@ -176,8 +180,9 @@ describe("downloadToTemp's timeout", () => { }); it("cuts a 1 KiB/s trickle that declares a huge Content-Length and removes the partial", async () => { + vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", "0"); const { fetchMock, outcome, dir } = await startDownload(() => - trickle(["x".repeat(1024)], 1_000, { contentLength: 10 * 1024 ** 3, then: "repeat" }), + trickle(["x".repeat(1024)], 1_000, { contentLength: 6 * 1024 ** 3, then: "repeat" }), ); await advanceSeconds(60); expect(fetchMock).toHaveBeenCalledTimes(1); @@ -209,6 +214,26 @@ describe("downloadToTemp's timeout", () => { expect(fetchMock).toHaveBeenCalledTimes(1); }, 120_000); + it("stops a steady sized download above the floor at the absolute deadline without retrying", async () => { + vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_MS", "120000"); + const chunk = new Uint8Array(40 * 1024).fill(0x78); + const paced = pacedBody(chunk, 10_000, 1_000); + const { fetchMock, outcome, dir } = await startDownload(() => paced.response); + for (let s = 0; s < 200 && !("error" in outcome); s += 1) { + await waitForRealIo(paced.readerWaiting); + await vi.advanceTimersByTimeAsync(1_000); + } + await waitForRealIo(() => "error" in outcome); + + expect(outcome.error).toMatchObject({ + kind: "timeout", + retryable: false, + message: "Download exceeded the 120s time limit", + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(readdirSync(dir)).toEqual([]); + }, 60_000); + it("cuts a body with no Content-Length at the fixed 300 s cap even while it streams", async () => { disableThroughputFloor(); const { fetchMock, outcome } = await startDownload(() => diff --git a/packages/engine/src/utils/urlDownloader.ts b/packages/engine/src/utils/urlDownloader.ts index c21219bdfb..5b33383d5c 100644 --- a/packages/engine/src/utils/urlDownloader.ts +++ b/packages/engine/src/utils/urlDownloader.ts @@ -12,6 +12,7 @@ import { rmdirSync, rmSync, statSync, + statfsSync, type Stats, unlinkSync, } from "fs"; @@ -847,7 +848,10 @@ function checksumMismatchError( ); } -const DEFAULT_MAX_DOWNLOAD_BYTES = 20 * 1024 ** 3; +const DEFAULT_MAX_DOWNLOAD_BYTES = 8 * 1024 ** 3; +const DEFAULT_MAX_DOWNLOAD_MS = 30 * 60 * 1000; +const DEFAULT_DOWNLOAD_MIN_FREE_BYTES = 512 * 1024 ** 2; +const FREE_SPACE_CHECK_INTERVAL_BYTES = 4 * 1024 ** 2; const DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC = 32 * 1024; const THROUGHPUT_WINDOW_SECONDS = 60; @@ -864,6 +868,87 @@ const maxDownloadBytes = (): number => const minDownloadBytesPerSec = (): number => integerFromEnv("HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC", DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC, 0); +const maxDownloadMs = (): number => + integerFromEnv("HYPERFRAMES_MAX_DOWNLOAD_MS", DEFAULT_MAX_DOWNLOAD_MS, 1); + +const downloadMinFreeBytes = (): number => + integerFromEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", DEFAULT_DOWNLOAD_MIN_FREE_BYTES, 0); + +function freeDiskBytes(path: string): number | null { + try { + const stat = statfsSync(path); + return stat.bavail * stat.bsize; + } catch { + return null; + } +} + +// Bytes that in-flight downloads declared but have not received yet, per device. +const unreceivedBytesByDevice = new Map(); + +/** + * Keeps `minFreeBytes` free on the download volume across parallel downloads: a declared size + * is claimed up front, and free space is measured again every few MiB as bytes arrive. Free + * space that cannot be measured counts as enough, and a minimum of 0 turns the guard off. + */ +class DiskHeadroomGuard { + private unreceivedBytes = 0; + private bytesSinceCheck = 0; + private device = 0; + + constructor( + private readonly dir: string, + readonly minFreeBytes: number, + ) {} + + claim(declaredBytes: number): boolean { + if (this.minFreeBytes === 0) return true; + this.device = statSync(this.dir).dev; + this.adjustUnreceived(declaredBytes); + return this.keepsMinimum(); + } + + receive(byteCount: number): boolean { + if (this.minFreeBytes === 0) return true; + this.adjustUnreceived(-Math.min(byteCount, this.unreceivedBytes)); + this.bytesSinceCheck += byteCount; + if (this.bytesSinceCheck < FREE_SPACE_CHECK_INTERVAL_BYTES) return true; + this.bytesSinceCheck = 0; + return this.keepsMinimum(); + } + + release(): void { + this.adjustUnreceived(-this.unreceivedBytes); + } + + private keepsMinimum(): boolean { + const free = freeDiskBytes(this.dir); + if (free === null) return true; + return free - (unreceivedBytesByDevice.get(this.device) ?? 0) >= this.minFreeBytes; + } + + private adjustUnreceived(delta: number): void { + this.unreceivedBytes += delta; + const total = (unreceivedBytesByDevice.get(this.device) ?? 0) + delta; + if (total > 0) unreceivedBytesByDevice.set(this.device, total); + else unreceivedBytesByDevice.delete(this.device); + } +} + +function lowDiskSpaceError( + minFreeBytes: number, + status: number, + telemetry: Partial, +): UrlDownloadError { + return new UrlDownloadError( + "filesystem", + false, + `Download stopped: it would leave less than ${minFreeBytes} bytes free on the download volume`, + status, + telemetry, + ); +} + function isCompressedEncoding(contentEncoding: string | null): boolean { const encoding = contentEncoding?.trim().toLowerCase(); return Boolean(encoding) && encoding !== "identity"; @@ -929,6 +1014,7 @@ async function fetchToPartial( controller: AbortController, options: UrlDownloadOptions, onBodyBytes: (byteCount: number, sized: boolean) => void, + headroom: DiskHeadroomGuard, ): Promise { const { response, finalUrl } = await fetchWithValidatedRedirects(url, controller); const finalIdentity = safeDownloadUrlIdentity(finalUrl); @@ -1012,6 +1098,13 @@ async function fetchToPartial( expectedBytes: declaredLength, }); } + if (!headroom.claim(declaredLength ?? 0)) { + await cancelResponseBody(response); + throw lowDiskSpaceError(headroom.minFreeBytes, response.status, { + ...responseTelemetry, + expectedBytes: declaredLength, + }); + } const expectedBytes = declaredLength; let receivedBytes = 0; @@ -1034,6 +1127,16 @@ async function fetchToPartial( ); return; } + if (!headroom.receive(bytes.length)) { + callback( + lowDiskSpaceError(headroom.minFreeBytes, response.status, { + ...responseTelemetry, + expectedBytes, + receivedBytes, + }), + ); + return; + } sha256.update(bytes); md5.update(bytes); if (prefixBytes < 1024) { @@ -1230,6 +1333,13 @@ async function runDownloadAttempt( }, timeoutMs); }; armTimeout(); + // Unlike the attempt timeout, incoming data never moves this deadline. + const deadlineMs = maxDownloadMs(); + let pastDeadline = false; + const deadlineId = setTimeout(() => { + pastDeadline = true; + controller.abort(); + }, deadlineMs); let sizedBodyStarted = false; const extendTimeout = (): void => { sizedBodyStarted = true; @@ -1257,12 +1367,21 @@ async function runDownloadAttempt( throughput.record(byteCount, Date.now()); }; const identity = safeDownloadUrlIdentity(url); + const headroom = new DiskHeadroomGuard(attemptDir, downloadMinFreeBytes()); try { if (callerAborted) { throw new UrlDownloadError("cancelled", false, "Download cancelled"); } - const integrity = await fetchToPartial(url, partialPath, controller, options, onBodyBytes); + const integrity = await fetchToPartial( + url, + partialPath, + controller, + options, + onBodyBytes, + headroom, + ); + clearTimeout(deadlineId); clearTimeout(floorCheckId); if (await playlistLeavesItsFolder(partialPath, localPath)) { throw new UrlDownloadError( @@ -1316,50 +1435,32 @@ async function runDownloadAttempt( }); return localPath; } catch (error) { + let classified: UrlDownloadError; if (callerAborted) { - const classified = new UrlDownloadError("cancelled", false, "Download cancelled"); - emitDownloadTelemetry(options, { - urlFingerprint: identity.urlFingerprint, - initialHost: identity.host, - attempt, - outcome: "attempt_failed", - failureKind: classified.kind, - }); - throw classified; - } - if (timedOut) { - const classified = new UrlDownloadError( + classified = new UrlDownloadError("cancelled", false, "Download cancelled"); + } else if (pastDeadline) { + classified = new UrlDownloadError( + "timeout", + false, + `Download exceeded the ${deadlineMs / 1000}s time limit`, + ); + } else if (timedOut) { + classified = new UrlDownloadError( "timeout", true, sizedBodyStarted ? `Download stalled: no data for ${timeoutMs / 1000}s` : `Download timeout after ${timeoutMs / 1000}s`, ); - emitDownloadTelemetry(options, { - urlFingerprint: identity.urlFingerprint, - initialHost: identity.host, - attempt, - outcome: "attempt_failed", - failureKind: classified.kind, - }); - throw classified; - } - if (tooSlow) { - const classified = new UrlDownloadError( + } else if (tooSlow) { + classified = new UrlDownloadError( "timeout", true, `Download too slow: under ${minBytesPerSec} bytes/s over the last ${THROUGHPUT_WINDOW_SECONDS}s`, ); - emitDownloadTelemetry(options, { - urlFingerprint: identity.urlFingerprint, - initialHost: identity.host, - attempt, - outcome: "attempt_failed", - failureKind: classified.kind, - }); - throw classified; + } else { + classified = classifyDownloadFailure(error); } - const classified = classifyDownloadFailure(error); emitDownloadTelemetry(options, { urlFingerprint: identity.urlFingerprint, initialHost: identity.host, @@ -1371,7 +1472,9 @@ async function runDownloadAttempt( throw classified; } finally { clearTimeout(timeoutId); + clearTimeout(deadlineId); clearTimeout(floorCheckId); + headroom.release(); signal?.removeEventListener("abort", onCallerAbort); controller.abort(); rmSync(attemptDir, { recursive: true, force: true });