Files
hyperframes/packages/cli/src/cloud/download.ts
T
Michael Wang 9701f0639c fix(cli): the audio caches never write or read through links in .hyperframes (#7825)
* fix(cli): the audio caches never write or read through links in .hyperframes

The read-audio cache, its stems, mix and descriptions, and the audio-kind cache
now use only real folders under the project's .hyperframes. A link or a file in
place of a cache folder stops the write with a clear message. Every cache file is
written to an exclusive temp file with a random name and renamed into place, so a
link at a temp or final name is replaced, never written through. ffmpeg writes the
voice-free mix into a private staging folder. Stem downloads use the new
downloadToFile replaceLink option. Cached files are read only when they are plain
files, and a stem list may only name files in its own folder.

* fix(cli): the kind cache reads no entry through a linked cache folder

readKindCache now checks that .hyperframes and audio-kind are real folders before
it reads an entry, as writeKindCache already does before it writes one.

* test(cli): read the replaced download before checking its type and mode

The replaceLink test now reads the file's bytes and lstat result once each, so
no read follows a check on the same path.
2026-11-15 17:31:16 +00:00

188 lines
7.4 KiB
TypeScript

/**
* Stream a presigned `video_url` (or any HTTPS URL) into a local file.
*
* The presigned URLs returned by `GET /v3/hyperframes/renders/{id}` are
* S3 URLs scoped per-request — they don't take any HeyGen auth header.
* That's why this lives separate from the cloud client: the client
* threads auth headers, the download path explicitly does NOT.
*
* Failure behavior is "all or nothing": on any error we (1) listen for
* stream errors / aborts so awaits resolve promptly instead of hanging,
* (2) verify the final byte count matches `content-length` when the
* server supplied one, and (3) discard the private staged output. An
* existing destination is replaced only after the download succeeds.
*/
import { chmodSync, lstatSync, renameSync, rmSync, statSync } from "node:fs";
import { makeOutputFolder } from "../utils/outputFolder.js";
import { open } from "node:fs/promises";
import { dirname, join } from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { createStagingDir, resolveWritePath } from "@hyperframes/core/atomic-file";
// The first request asks for PART_BYTES; a larger file is split into up to
// CONNECTIONS ranged GETs. A server that ignores Range answers 200 and is streamed.
const PART_BYTES = 8 * 1024 * 1024;
const CONNECTIONS = 8;
// A ranged part whose connection drops resumes where it stopped, after 0.25, 0.5 and 1 s.
const RETRIES = 3;
export interface DownloadOptions {
signal?: AbortSignal;
/** Inject fetch (used by tests). */
fetchImpl?: typeof fetch;
/** Called with (bytes downloaded, total or undefined). */
onProgress?: (bytes: number, total: number | undefined) => void;
partBytes?: number;
/** Replace a link at `destPath` with the download instead of writing to the file it points at. */
replaceLink?: boolean;
}
export interface DownloadResult {
path: string;
bytes: number;
}
/**
* Stream `url` into `destPath`. Creates the parent directory if needed,
* replaces an existing file only after the complete response is written,
* and removes staged bytes on failure while preserving the old output.
* Atomic replacement requires a writable parent directory and creates a new
* inode: mode is retained, owner/group are not, and hard links keep old bytes.
* A read-only file can be replaced when its parent permits the rename.
*/
// fallow-ignore-next-line complexity
export async function downloadToFile(
url: string,
destPath: string,
options: DownloadOptions = {},
): Promise<DownloadResult> {
const fetchImpl = options.fetchImpl ?? fetch;
const partBytes = options.partBytes ?? PART_BYTES;
// Cancels the sibling parts when one fails.
const failFast = new AbortController();
const signal = options.signal
? AbortSignal.any([options.signal, failFast.signal])
: failFast.signal;
const get = async (range?: string) => fetchImpl(url, { signal, headers: range ? { range } : {} });
let first = await get(`bytes=0-${partBytes - 1}`);
// 416 (empty file) or a 206 without a total cannot be split: fetch it whole.
if (first.status === 416 || (first.status === 206 && contentRangeTotal(first) === undefined)) {
await first.body?.cancel();
first = await get();
}
assertOk(first, url);
const ranged = first.status === 206;
const total = ranged ? contentRangeTotal(first) : contentLength(first);
makeOutputFolder(dirname(destPath));
const destination = options.replaceLink ? destPath : resolveWritePath(destPath);
const previous = options.replaceLink
? lstatSync(destination, { throwIfNoEntry: false })
: statSync(destination, { throwIfNoEntry: false });
const stage = createStagingDir(dirname(destination), ".hf-download-");
const stagedFile = join(stage, "download");
let bytes = 0;
let firstError: unknown;
try {
const file = await open(stagedFile, "wx");
// Writes the body at `part.at` and advances it, so a dropped range resumes there.
const pump = async (res: Response, part: { at: number }) => {
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
await file.write(chunk, 0, chunk.byteLength, part.at);
part.at += chunk.byteLength;
bytes += chunk.byteLength;
signal.throwIfAborted();
options.onProgress?.(bytes, total);
}
};
const fetchRange = async (start: number, end: number, res?: Response) => {
const part = { at: start };
for (let attempt = 0; ; attempt++) {
try {
res ??= await get(`bytes=${part.at}-${end}`);
assertOk(res, url);
if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);
await pump(res, part);
if (part.at > end) return;
throw new Error(`connection closed at byte ${part.at}`);
} catch (error) {
if (signal.aborted) throw error;
if (attempt === RETRIES) {
throw new Error(
`Failed to download ${url}: bytes ${start}-${end} dropped ${RETRIES + 1} times`,
{ cause: error },
);
}
res?.body?.cancel().catch(() => {});
res = undefined;
await sleep(250 * 2 ** attempt, undefined, { signal });
}
}
};
const parts: Promise<void>[] = [];
if (ranged && total !== undefined) {
parts.push(fetchRange(0, Math.min(partBytes, total) - 1, first));
const rest = total - partBytes;
const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / partBytes)));
for (let start = partBytes; start < total; start += size) {
parts.push(fetchRange(start, Math.min(start + size, total) - 1));
}
} else {
// No Range support means nothing to resume from: fail fast.
const part = { at: 0 };
parts.push(
pump(first, part).then(() => {
if (total !== undefined && part.at !== total) {
throw new Error(
`Truncated download: got ${part.at} bytes, expected ${total} (content-length). ` +
`The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
);
}
}),
);
}
await Promise.all(
parts.map((part) =>
part.catch((error: unknown) => {
firstError ??= error;
failFast.abort(error);
}),
),
);
await file.close();
if (firstError !== undefined) throw firstError;
options.signal?.throwIfAborted();
if (previous?.isFile()) chmodSync(stagedFile, previous.mode & 0o777);
renameSync(stagedFile, destination);
} catch (error) {
const reason = options.signal?.reason;
if (options.signal?.aborted && reason instanceof Error) throw reason;
throw error;
} finally {
rmSync(stage, { recursive: true, force: true });
}
return { path: destPath, bytes };
}
function assertOk(res: Response, url: string): void {
if (!res.ok) {
throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`);
}
if (!res.body) {
throw new Error(`Failed to download ${url}: empty response body`);
}
}
function contentLength(res: Response): number | undefined {
const total = Number.parseInt(res.headers.get("content-length") ?? "", 10);
return Number.isFinite(total) ? total : undefined;
}
function contentRangeTotal(res: Response): number | undefined {
const match = /^bytes \d+-\d+\/(\d+)$/.exec(res.headers.get("content-range") ?? "");
return match ? Number(match[1]) : undefined;
}