mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(engine): remote downloads get an absolute deadline and stop before the disk fills (#7827)
A download whose response declares a Content-Length re-arms its attempt timeout on every chunk, so a server that trickles just above the throughput floor was never cut off by time. Parallel downloads also had no shared limit on the disk they could fill. - Each download attempt now has a deadline that incoming data never moves: 30 minutes by default, HYPERFRAMES_MAX_DOWNLOAD_MS to change it. Passing it fails the download without a local retry. - The default per-file cap drops from 20 GiB to 8 GiB (HYPERFRAMES_MAX_DOWNLOAD_BYTES still overrides it). - Downloads keep 512 MiB free on their volume (HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES, 0 turns it off). A declared size is claimed before the body is read and counted against every parallel download on the same device, and free space is measured again every 4 MiB as bytes arrive. Free space that cannot be measured does not block. - docs/packages/engine.mdx lists the download limits and their variables.
This commit is contained in:
@@ -81,6 +81,26 @@ software WebGPU may use SwiftShader, but it must also create a device. The check
|
||||
releases its temporary device; the composition still creates its own device
|
||||
with any features or limits its renderer needs.
|
||||
|
||||
## Remote media downloads
|
||||
|
||||
The engine downloads remote `https://` video, audio, image, and font sources
|
||||
into the render's work directory before it uses them. Each download has these
|
||||
limits, and an environment variable changes each one:
|
||||
|
||||
| Limit | Default | Environment variable |
|
||||
| --- | --- | --- |
|
||||
| Largest file | 8 GiB | `HYPERFRAMES_MAX_DOWNLOAD_BYTES` |
|
||||
| Longest download, however fast data arrives | 30 minutes | `HYPERFRAMES_MAX_DOWNLOAD_MS` |
|
||||
| Slowest average speed over a minute | 32 KiB/s | `HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC` |
|
||||
| Free space left on the download volume | 512 MiB | `HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES` |
|
||||
|
||||
Sizes are in bytes and the time limit is in milliseconds. A speed or free-space
|
||||
value of `0` turns that check off.
|
||||
|
||||
A download that passes one of these limits stops with an error. The free-space
|
||||
check counts the declared sizes of all downloads running at the same time, so
|
||||
parallel downloads cannot fill the volume together.
|
||||
|
||||
## What else the package exposes
|
||||
|
||||
| Surface | Use it for |
|
||||
|
||||
@@ -1,15 +1,32 @@
|
||||
import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
||||
import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { _setDownloadFetchForTests, downloadToTemp, UrlDownloadError } from "./urlDownloader.js";
|
||||
|
||||
const fakeDisk = vi.hoisted(() => {
|
||||
const state: { freeBytes?: (path: string) => number | undefined } = {};
|
||||
return state;
|
||||
});
|
||||
|
||||
vi.mock("fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("fs")>();
|
||||
return {
|
||||
...actual,
|
||||
statfsSync: (path: string) => {
|
||||
const freeBytes = fakeDisk.freeBytes?.(path);
|
||||
return freeBytes === undefined ? actual.statfsSync(path) : { bavail: freeBytes, bsize: 1 };
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const stubFetch = (replacement: unknown): void =>
|
||||
_setDownloadFetchForTests(replacement as Parameters<typeof _setDownloadFetchForTests>[0]);
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
fakeDisk.freeBytes = undefined;
|
||||
vi.unstubAllEnvs();
|
||||
_setDownloadFetchForTests();
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
@@ -131,3 +148,87 @@ describe("downloadToTemp's byte limit", () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
const MiB = 1024 ** 2;
|
||||
|
||||
function bytesUnder(dir: string): number {
|
||||
return readdirSync(dir, { withFileTypes: true }).reduce((total, entry) => {
|
||||
const path = join(dir, entry.name);
|
||||
return total + (entry.isDirectory() ? bytesUnder(path) : statSync(path).size);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
/** Reports `capacity` minus what is written under `dir` as free space there; returns the lowest seen. */
|
||||
function fakeVolume(dir: string, capacity: number): { lowestFree: () => number } {
|
||||
let lowest = capacity;
|
||||
fakeDisk.freeBytes = (path) => {
|
||||
if (!path.startsWith(dir)) return undefined;
|
||||
const free = capacity - bytesUnder(dir);
|
||||
lowest = Math.min(lowest, free);
|
||||
return free;
|
||||
};
|
||||
return { lowestFree: () => lowest };
|
||||
}
|
||||
|
||||
function mibBody(chunks: number | "endless", declare: boolean): Response {
|
||||
let sent = 0;
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
controller.enqueue(new Uint8Array(MiB).fill(0x61));
|
||||
sent += 1;
|
||||
if (chunks !== "endless" && sent === chunks) controller.close();
|
||||
},
|
||||
});
|
||||
const headers =
|
||||
declare && chunks !== "endless" ? { "content-length": `${chunks * MiB}` } : undefined;
|
||||
return new Response(body, { headers });
|
||||
}
|
||||
|
||||
const lowDisk = {
|
||||
kind: "filesystem",
|
||||
retryable: false,
|
||||
message: `Download stopped: it would leave less than ${32 * MiB} bytes free on the download volume`,
|
||||
};
|
||||
|
||||
describe("downloadToTemp's free-space guard", () => {
|
||||
it("refuses parallel declared bodies that together would cut into the free-space minimum", async () => {
|
||||
vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", `${32 * MiB}`);
|
||||
const dir = tempDir();
|
||||
const volume = fakeVolume(dir, 96 * MiB);
|
||||
const fetchMock = vi.fn(async () => mibBody(40, true));
|
||||
stubFetch(fetchMock);
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
[1, 2, 3].map((n) => downloadToTemp(`https://cdn.example/large-${n}.mp4`, dir, 30_000)),
|
||||
);
|
||||
|
||||
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
|
||||
const refusals = results.flatMap((result) =>
|
||||
result.status === "rejected" ? [result.reason] : [],
|
||||
);
|
||||
expect(refusals).toEqual([expect.objectContaining(lowDisk), expect.objectContaining(lowDisk)]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
expect(readdirSync(dir)).toHaveLength(1);
|
||||
expect(volume.lowestFree()).toBeGreaterThanOrEqual(32 * MiB);
|
||||
});
|
||||
|
||||
it("stops parallel endless bodies with no Content-Length before the volume fills", async () => {
|
||||
vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", `${32 * MiB}`);
|
||||
vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_BYTES", `${128 * MiB}`);
|
||||
const dir = tempDir();
|
||||
const volume = fakeVolume(dir, 96 * MiB);
|
||||
const fetchMock = vi.fn(async () => mibBody("endless", false));
|
||||
stubFetch(fetchMock);
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
[1, 2, 3].map((n) => downloadToTemp(`https://cdn.example/endless-${n}.mp4`, dir, 30_000)),
|
||||
);
|
||||
|
||||
expect(results).toEqual(
|
||||
[1, 2, 3].map(() => ({ status: "rejected", reason: expect.objectContaining(lowDisk) })),
|
||||
);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
expect(volume.lowestFree()).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -68,12 +68,13 @@ function trickle(
|
||||
function pacedBody(chunk: Uint8Array, count: number, everyMs: number) {
|
||||
let sent = 0;
|
||||
let waiting = false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const body = new ReadableStream<Uint8Array>(
|
||||
{
|
||||
pull(controller) {
|
||||
waiting = true;
|
||||
return new Promise<void>((resolve) => {
|
||||
setTimeout(() => {
|
||||
timer = setTimeout(() => {
|
||||
waiting = false;
|
||||
controller.enqueue(chunk);
|
||||
sent += 1;
|
||||
@@ -82,6 +83,9 @@ function pacedBody(chunk: Uint8Array, count: number, everyMs: number) {
|
||||
}, everyMs);
|
||||
});
|
||||
},
|
||||
cancel() {
|
||||
clearTimeout(timer);
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 0 },
|
||||
);
|
||||
@@ -176,8 +180,9 @@ describe("downloadToTemp's timeout", () => {
|
||||
});
|
||||
|
||||
it("cuts a 1 KiB/s trickle that declares a huge Content-Length and removes the partial", async () => {
|
||||
vi.stubEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", "0");
|
||||
const { fetchMock, outcome, dir } = await startDownload(() =>
|
||||
trickle(["x".repeat(1024)], 1_000, { contentLength: 10 * 1024 ** 3, then: "repeat" }),
|
||||
trickle(["x".repeat(1024)], 1_000, { contentLength: 6 * 1024 ** 3, then: "repeat" }),
|
||||
);
|
||||
await advanceSeconds(60);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
@@ -209,6 +214,26 @@ describe("downloadToTemp's timeout", () => {
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
}, 120_000);
|
||||
|
||||
it("stops a steady sized download above the floor at the absolute deadline without retrying", async () => {
|
||||
vi.stubEnv("HYPERFRAMES_MAX_DOWNLOAD_MS", "120000");
|
||||
const chunk = new Uint8Array(40 * 1024).fill(0x78);
|
||||
const paced = pacedBody(chunk, 10_000, 1_000);
|
||||
const { fetchMock, outcome, dir } = await startDownload(() => paced.response);
|
||||
for (let s = 0; s < 200 && !("error" in outcome); s += 1) {
|
||||
await waitForRealIo(paced.readerWaiting);
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
}
|
||||
await waitForRealIo(() => "error" in outcome);
|
||||
|
||||
expect(outcome.error).toMatchObject({
|
||||
kind: "timeout",
|
||||
retryable: false,
|
||||
message: "Download exceeded the 120s time limit",
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
it("cuts a body with no Content-Length at the fixed 300 s cap even while it streams", async () => {
|
||||
disableThroughputFloor();
|
||||
const { fetchMock, outcome } = await startDownload(() =>
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
rmdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
statfsSync,
|
||||
type Stats,
|
||||
unlinkSync,
|
||||
} from "fs";
|
||||
@@ -847,7 +848,10 @@ function checksumMismatchError(
|
||||
);
|
||||
}
|
||||
|
||||
const DEFAULT_MAX_DOWNLOAD_BYTES = 20 * 1024 ** 3;
|
||||
const DEFAULT_MAX_DOWNLOAD_BYTES = 8 * 1024 ** 3;
|
||||
const DEFAULT_MAX_DOWNLOAD_MS = 30 * 60 * 1000;
|
||||
const DEFAULT_DOWNLOAD_MIN_FREE_BYTES = 512 * 1024 ** 2;
|
||||
const FREE_SPACE_CHECK_INTERVAL_BYTES = 4 * 1024 ** 2;
|
||||
const DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC = 32 * 1024;
|
||||
const THROUGHPUT_WINDOW_SECONDS = 60;
|
||||
|
||||
@@ -864,6 +868,87 @@ const maxDownloadBytes = (): number =>
|
||||
const minDownloadBytesPerSec = (): number =>
|
||||
integerFromEnv("HYPERFRAMES_MIN_DOWNLOAD_BYTES_PER_SEC", DEFAULT_MIN_DOWNLOAD_BYTES_PER_SEC, 0);
|
||||
|
||||
const maxDownloadMs = (): number =>
|
||||
integerFromEnv("HYPERFRAMES_MAX_DOWNLOAD_MS", DEFAULT_MAX_DOWNLOAD_MS, 1);
|
||||
|
||||
const downloadMinFreeBytes = (): number =>
|
||||
integerFromEnv("HYPERFRAMES_DOWNLOAD_MIN_FREE_BYTES", DEFAULT_DOWNLOAD_MIN_FREE_BYTES, 0);
|
||||
|
||||
function freeDiskBytes(path: string): number | null {
|
||||
try {
|
||||
const stat = statfsSync(path);
|
||||
return stat.bavail * stat.bsize;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Bytes that in-flight downloads declared but have not received yet, per device.
|
||||
const unreceivedBytesByDevice = new Map<number, number>();
|
||||
|
||||
/**
|
||||
* Keeps `minFreeBytes` free on the download volume across parallel downloads: a declared size
|
||||
* is claimed up front, and free space is measured again every few MiB as bytes arrive. Free
|
||||
* space that cannot be measured counts as enough, and a minimum of 0 turns the guard off.
|
||||
*/
|
||||
class DiskHeadroomGuard {
|
||||
private unreceivedBytes = 0;
|
||||
private bytesSinceCheck = 0;
|
||||
private device = 0;
|
||||
|
||||
constructor(
|
||||
private readonly dir: string,
|
||||
readonly minFreeBytes: number,
|
||||
) {}
|
||||
|
||||
claim(declaredBytes: number): boolean {
|
||||
if (this.minFreeBytes === 0) return true;
|
||||
this.device = statSync(this.dir).dev;
|
||||
this.adjustUnreceived(declaredBytes);
|
||||
return this.keepsMinimum();
|
||||
}
|
||||
|
||||
receive(byteCount: number): boolean {
|
||||
if (this.minFreeBytes === 0) return true;
|
||||
this.adjustUnreceived(-Math.min(byteCount, this.unreceivedBytes));
|
||||
this.bytesSinceCheck += byteCount;
|
||||
if (this.bytesSinceCheck < FREE_SPACE_CHECK_INTERVAL_BYTES) return true;
|
||||
this.bytesSinceCheck = 0;
|
||||
return this.keepsMinimum();
|
||||
}
|
||||
|
||||
release(): void {
|
||||
this.adjustUnreceived(-this.unreceivedBytes);
|
||||
}
|
||||
|
||||
private keepsMinimum(): boolean {
|
||||
const free = freeDiskBytes(this.dir);
|
||||
if (free === null) return true;
|
||||
return free - (unreceivedBytesByDevice.get(this.device) ?? 0) >= this.minFreeBytes;
|
||||
}
|
||||
|
||||
private adjustUnreceived(delta: number): void {
|
||||
this.unreceivedBytes += delta;
|
||||
const total = (unreceivedBytesByDevice.get(this.device) ?? 0) + delta;
|
||||
if (total > 0) unreceivedBytesByDevice.set(this.device, total);
|
||||
else unreceivedBytesByDevice.delete(this.device);
|
||||
}
|
||||
}
|
||||
|
||||
function lowDiskSpaceError(
|
||||
minFreeBytes: number,
|
||||
status: number,
|
||||
telemetry: Partial<UrlDownloadTelemetry>,
|
||||
): UrlDownloadError {
|
||||
return new UrlDownloadError(
|
||||
"filesystem",
|
||||
false,
|
||||
`Download stopped: it would leave less than ${minFreeBytes} bytes free on the download volume`,
|
||||
status,
|
||||
telemetry,
|
||||
);
|
||||
}
|
||||
|
||||
function isCompressedEncoding(contentEncoding: string | null): boolean {
|
||||
const encoding = contentEncoding?.trim().toLowerCase();
|
||||
return Boolean(encoding) && encoding !== "identity";
|
||||
@@ -929,6 +1014,7 @@ async function fetchToPartial(
|
||||
controller: AbortController,
|
||||
options: UrlDownloadOptions,
|
||||
onBodyBytes: (byteCount: number, sized: boolean) => void,
|
||||
headroom: DiskHeadroomGuard,
|
||||
): Promise<PartialIntegrity> {
|
||||
const { response, finalUrl } = await fetchWithValidatedRedirects(url, controller);
|
||||
const finalIdentity = safeDownloadUrlIdentity(finalUrl);
|
||||
@@ -1012,6 +1098,13 @@ async function fetchToPartial(
|
||||
expectedBytes: declaredLength,
|
||||
});
|
||||
}
|
||||
if (!headroom.claim(declaredLength ?? 0)) {
|
||||
await cancelResponseBody(response);
|
||||
throw lowDiskSpaceError(headroom.minFreeBytes, response.status, {
|
||||
...responseTelemetry,
|
||||
expectedBytes: declaredLength,
|
||||
});
|
||||
}
|
||||
const expectedBytes = declaredLength;
|
||||
|
||||
let receivedBytes = 0;
|
||||
@@ -1034,6 +1127,16 @@ async function fetchToPartial(
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!headroom.receive(bytes.length)) {
|
||||
callback(
|
||||
lowDiskSpaceError(headroom.minFreeBytes, response.status, {
|
||||
...responseTelemetry,
|
||||
expectedBytes,
|
||||
receivedBytes,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
sha256.update(bytes);
|
||||
md5.update(bytes);
|
||||
if (prefixBytes < 1024) {
|
||||
@@ -1230,6 +1333,13 @@ async function runDownloadAttempt(
|
||||
}, timeoutMs);
|
||||
};
|
||||
armTimeout();
|
||||
// Unlike the attempt timeout, incoming data never moves this deadline.
|
||||
const deadlineMs = maxDownloadMs();
|
||||
let pastDeadline = false;
|
||||
const deadlineId = setTimeout(() => {
|
||||
pastDeadline = true;
|
||||
controller.abort();
|
||||
}, deadlineMs);
|
||||
let sizedBodyStarted = false;
|
||||
const extendTimeout = (): void => {
|
||||
sizedBodyStarted = true;
|
||||
@@ -1257,12 +1367,21 @@ async function runDownloadAttempt(
|
||||
throughput.record(byteCount, Date.now());
|
||||
};
|
||||
const identity = safeDownloadUrlIdentity(url);
|
||||
const headroom = new DiskHeadroomGuard(attemptDir, downloadMinFreeBytes());
|
||||
|
||||
try {
|
||||
if (callerAborted) {
|
||||
throw new UrlDownloadError("cancelled", false, "Download cancelled");
|
||||
}
|
||||
const integrity = await fetchToPartial(url, partialPath, controller, options, onBodyBytes);
|
||||
const integrity = await fetchToPartial(
|
||||
url,
|
||||
partialPath,
|
||||
controller,
|
||||
options,
|
||||
onBodyBytes,
|
||||
headroom,
|
||||
);
|
||||
clearTimeout(deadlineId);
|
||||
clearTimeout(floorCheckId);
|
||||
if (await playlistLeavesItsFolder(partialPath, localPath)) {
|
||||
throw new UrlDownloadError(
|
||||
@@ -1316,50 +1435,32 @@ async function runDownloadAttempt(
|
||||
});
|
||||
return localPath;
|
||||
} catch (error) {
|
||||
let classified: UrlDownloadError;
|
||||
if (callerAborted) {
|
||||
const classified = new UrlDownloadError("cancelled", false, "Download cancelled");
|
||||
emitDownloadTelemetry(options, {
|
||||
urlFingerprint: identity.urlFingerprint,
|
||||
initialHost: identity.host,
|
||||
attempt,
|
||||
outcome: "attempt_failed",
|
||||
failureKind: classified.kind,
|
||||
});
|
||||
throw classified;
|
||||
}
|
||||
if (timedOut) {
|
||||
const classified = new UrlDownloadError(
|
||||
classified = new UrlDownloadError("cancelled", false, "Download cancelled");
|
||||
} else if (pastDeadline) {
|
||||
classified = new UrlDownloadError(
|
||||
"timeout",
|
||||
false,
|
||||
`Download exceeded the ${deadlineMs / 1000}s time limit`,
|
||||
);
|
||||
} else if (timedOut) {
|
||||
classified = new UrlDownloadError(
|
||||
"timeout",
|
||||
true,
|
||||
sizedBodyStarted
|
||||
? `Download stalled: no data for ${timeoutMs / 1000}s`
|
||||
: `Download timeout after ${timeoutMs / 1000}s`,
|
||||
);
|
||||
emitDownloadTelemetry(options, {
|
||||
urlFingerprint: identity.urlFingerprint,
|
||||
initialHost: identity.host,
|
||||
attempt,
|
||||
outcome: "attempt_failed",
|
||||
failureKind: classified.kind,
|
||||
});
|
||||
throw classified;
|
||||
}
|
||||
if (tooSlow) {
|
||||
const classified = new UrlDownloadError(
|
||||
} else if (tooSlow) {
|
||||
classified = new UrlDownloadError(
|
||||
"timeout",
|
||||
true,
|
||||
`Download too slow: under ${minBytesPerSec} bytes/s over the last ${THROUGHPUT_WINDOW_SECONDS}s`,
|
||||
);
|
||||
emitDownloadTelemetry(options, {
|
||||
urlFingerprint: identity.urlFingerprint,
|
||||
initialHost: identity.host,
|
||||
attempt,
|
||||
outcome: "attempt_failed",
|
||||
failureKind: classified.kind,
|
||||
});
|
||||
throw classified;
|
||||
} else {
|
||||
classified = classifyDownloadFailure(error);
|
||||
}
|
||||
const classified = classifyDownloadFailure(error);
|
||||
emitDownloadTelemetry(options, {
|
||||
urlFingerprint: identity.urlFingerprint,
|
||||
initialHost: identity.host,
|
||||
@@ -1371,7 +1472,9 @@ async function runDownloadAttempt(
|
||||
throw classified;
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
clearTimeout(deadlineId);
|
||||
clearTimeout(floorCheckId);
|
||||
headroom.release();
|
||||
signal?.removeEventListener("abort", onCallerAbort);
|
||||
controller.abort();
|
||||
rmSync(attemptDir, { recursive: true, force: true });
|
||||
|
||||
Reference in New Issue
Block a user