mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
* fix(cli): the audio caches never write or read through links in .hyperframes The read-audio cache, its stems, mix and descriptions, and the audio-kind cache now use only real folders under the project's .hyperframes. A link or a file in place of a cache folder stops the write with a clear message. Every cache file is written to an exclusive temp file with a random name and renamed into place, so a link at a temp or final name is replaced, never written through. ffmpeg writes the voice-free mix into a private staging folder. Stem downloads use the new downloadToFile replaceLink option. Cached files are read only when they are plain files, and a stem list may only name files in its own folder. * fix(cli): the kind cache reads no entry through a linked cache folder readKindCache now checks that .hyperframes and audio-kind are real folders before it reads an entry, as writeKindCache already does before it writes one. * test(cli): read the replaced download before checking its type and mode The replaceLink test now reads the file's bytes and lstat result once each, so no read follows a check on the same path.
188 lines
7.4 KiB
TypeScript
188 lines
7.4 KiB
TypeScript
/**
|
|
* Stream a presigned `video_url` (or any HTTPS URL) into a local file.
|
|
*
|
|
* The presigned URLs returned by `GET /v3/hyperframes/renders/{id}` are
|
|
* S3 URLs scoped per-request — they don't take any HeyGen auth header.
|
|
* That's why this lives separate from the cloud client: the client
|
|
* threads auth headers, the download path explicitly does NOT.
|
|
*
|
|
* Failure behavior is "all or nothing": on any error we (1) listen for
|
|
* stream errors / aborts so awaits resolve promptly instead of hanging,
|
|
* (2) verify the final byte count matches `content-length` when the
|
|
* server supplied one, and (3) discard the private staged output. An
|
|
* existing destination is replaced only after the download succeeds.
|
|
*/
|
|
|
|
import { chmodSync, lstatSync, renameSync, rmSync, statSync } from "node:fs";
|
|
import { makeOutputFolder } from "../utils/outputFolder.js";
|
|
import { open } from "node:fs/promises";
|
|
import { dirname, join } from "node:path";
|
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
import { createStagingDir, resolveWritePath } from "@hyperframes/core/atomic-file";
|
|
|
|
// The first request asks for PART_BYTES; a larger file is split into up to
|
|
// CONNECTIONS ranged GETs. A server that ignores Range answers 200 and is streamed.
|
|
const PART_BYTES = 8 * 1024 * 1024;
|
|
const CONNECTIONS = 8;
|
|
// A ranged part whose connection drops resumes where it stopped, after 0.25, 0.5 and 1 s.
|
|
const RETRIES = 3;
|
|
|
|
export interface DownloadOptions {
|
|
signal?: AbortSignal;
|
|
/** Inject fetch (used by tests). */
|
|
fetchImpl?: typeof fetch;
|
|
/** Called with (bytes downloaded, total or undefined). */
|
|
onProgress?: (bytes: number, total: number | undefined) => void;
|
|
partBytes?: number;
|
|
/** Replace a link at `destPath` with the download instead of writing to the file it points at. */
|
|
replaceLink?: boolean;
|
|
}
|
|
|
|
export interface DownloadResult {
|
|
path: string;
|
|
bytes: number;
|
|
}
|
|
|
|
/**
|
|
* Stream `url` into `destPath`. Creates the parent directory if needed,
|
|
* replaces an existing file only after the complete response is written,
|
|
* and removes staged bytes on failure while preserving the old output.
|
|
* Atomic replacement requires a writable parent directory and creates a new
|
|
* inode: mode is retained, owner/group are not, and hard links keep old bytes.
|
|
* A read-only file can be replaced when its parent permits the rename.
|
|
*/
|
|
// fallow-ignore-next-line complexity
|
|
export async function downloadToFile(
|
|
url: string,
|
|
destPath: string,
|
|
options: DownloadOptions = {},
|
|
): Promise<DownloadResult> {
|
|
const fetchImpl = options.fetchImpl ?? fetch;
|
|
const partBytes = options.partBytes ?? PART_BYTES;
|
|
// Cancels the sibling parts when one fails.
|
|
const failFast = new AbortController();
|
|
const signal = options.signal
|
|
? AbortSignal.any([options.signal, failFast.signal])
|
|
: failFast.signal;
|
|
const get = async (range?: string) => fetchImpl(url, { signal, headers: range ? { range } : {} });
|
|
|
|
let first = await get(`bytes=0-${partBytes - 1}`);
|
|
// 416 (empty file) or a 206 without a total cannot be split: fetch it whole.
|
|
if (first.status === 416 || (first.status === 206 && contentRangeTotal(first) === undefined)) {
|
|
await first.body?.cancel();
|
|
first = await get();
|
|
}
|
|
assertOk(first, url);
|
|
const ranged = first.status === 206;
|
|
const total = ranged ? contentRangeTotal(first) : contentLength(first);
|
|
|
|
makeOutputFolder(dirname(destPath));
|
|
|
|
const destination = options.replaceLink ? destPath : resolveWritePath(destPath);
|
|
const previous = options.replaceLink
|
|
? lstatSync(destination, { throwIfNoEntry: false })
|
|
: statSync(destination, { throwIfNoEntry: false });
|
|
const stage = createStagingDir(dirname(destination), ".hf-download-");
|
|
const stagedFile = join(stage, "download");
|
|
let bytes = 0;
|
|
let firstError: unknown;
|
|
try {
|
|
const file = await open(stagedFile, "wx");
|
|
// Writes the body at `part.at` and advances it, so a dropped range resumes there.
|
|
const pump = async (res: Response, part: { at: number }) => {
|
|
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
|
|
await file.write(chunk, 0, chunk.byteLength, part.at);
|
|
part.at += chunk.byteLength;
|
|
bytes += chunk.byteLength;
|
|
signal.throwIfAborted();
|
|
options.onProgress?.(bytes, total);
|
|
}
|
|
};
|
|
const fetchRange = async (start: number, end: number, res?: Response) => {
|
|
const part = { at: start };
|
|
for (let attempt = 0; ; attempt++) {
|
|
try {
|
|
res ??= await get(`bytes=${part.at}-${end}`);
|
|
assertOk(res, url);
|
|
if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);
|
|
await pump(res, part);
|
|
if (part.at > end) return;
|
|
throw new Error(`connection closed at byte ${part.at}`);
|
|
} catch (error) {
|
|
if (signal.aborted) throw error;
|
|
if (attempt === RETRIES) {
|
|
throw new Error(
|
|
`Failed to download ${url}: bytes ${start}-${end} dropped ${RETRIES + 1} times`,
|
|
{ cause: error },
|
|
);
|
|
}
|
|
res?.body?.cancel().catch(() => {});
|
|
res = undefined;
|
|
await sleep(250 * 2 ** attempt, undefined, { signal });
|
|
}
|
|
}
|
|
};
|
|
const parts: Promise<void>[] = [];
|
|
if (ranged && total !== undefined) {
|
|
parts.push(fetchRange(0, Math.min(partBytes, total) - 1, first));
|
|
const rest = total - partBytes;
|
|
const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / partBytes)));
|
|
for (let start = partBytes; start < total; start += size) {
|
|
parts.push(fetchRange(start, Math.min(start + size, total) - 1));
|
|
}
|
|
} else {
|
|
// No Range support means nothing to resume from: fail fast.
|
|
const part = { at: 0 };
|
|
parts.push(
|
|
pump(first, part).then(() => {
|
|
if (total !== undefined && part.at !== total) {
|
|
throw new Error(
|
|
`Truncated download: got ${part.at} bytes, expected ${total} (content-length). ` +
|
|
`The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
|
|
);
|
|
}
|
|
}),
|
|
);
|
|
}
|
|
await Promise.all(
|
|
parts.map((part) =>
|
|
part.catch((error: unknown) => {
|
|
firstError ??= error;
|
|
failFast.abort(error);
|
|
}),
|
|
),
|
|
);
|
|
await file.close();
|
|
if (firstError !== undefined) throw firstError;
|
|
options.signal?.throwIfAborted();
|
|
if (previous?.isFile()) chmodSync(stagedFile, previous.mode & 0o777);
|
|
renameSync(stagedFile, destination);
|
|
} catch (error) {
|
|
const reason = options.signal?.reason;
|
|
if (options.signal?.aborted && reason instanceof Error) throw reason;
|
|
throw error;
|
|
} finally {
|
|
rmSync(stage, { recursive: true, force: true });
|
|
}
|
|
return { path: destPath, bytes };
|
|
}
|
|
|
|
function assertOk(res: Response, url: string): void {
|
|
if (!res.ok) {
|
|
throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`);
|
|
}
|
|
if (!res.body) {
|
|
throw new Error(`Failed to download ${url}: empty response body`);
|
|
}
|
|
}
|
|
|
|
function contentLength(res: Response): number | undefined {
|
|
const total = Number.parseInt(res.headers.get("content-length") ?? "", 10);
|
|
return Number.isFinite(total) ? total : undefined;
|
|
}
|
|
|
|
function contentRangeTotal(res: Response): number | undefined {
|
|
const match = /^bytes \d+-\d+\/(\d+)$/.exec(res.headers.get("content-range") ?? "");
|
|
return match ? Number(match[1]) : undefined;
|
|
}
|