fix(cli): the audio caches never write or read through links in .hyperframes (#7825)

* fix(cli): the audio caches never write or read through links in .hyperframes

The read-audio cache, its stems, mix and descriptions, and the audio-kind cache
now use only real folders under the project's .hyperframes. A link or a file in
place of a cache folder stops the write with a clear message. Every cache file is
written to an exclusive temp file with a random name and renamed into place, so a
link at a temp or final name is replaced, never written through. ffmpeg writes the
voice-free mix into a private staging folder. Stem downloads use the new
downloadToFile replaceLink option. Cached files are read only when they are plain
files, and a stem list may only name files in its own folder.

* fix(cli): the kind cache reads no entry through a linked cache folder

readKindCache now checks that .hyperframes and audio-kind are real folders before
it reads an entry, as writeKindCache already does before it writes one.

* test(cli): read the replaced download before checking its type and mode

The replaceLink test now reads the file's bytes and lstat result once each, so
no read follows a check on the same path.
This commit is contained in:
Michael Wang
2026-11-15 17:31:16 +00:00
committed by GitHub
parent e1043659d4
commit 9701f0639c
10 changed files with 383 additions and 60 deletions
@@ -1,4 +1,13 @@
import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import {
lstatSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
symlinkSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
@@ -41,4 +50,51 @@ describe("kind cache", () => {
writeFileSync(blocked, "file");
expect(writeKindCache(join(blocked, "entry.json"), kind)).toMatch(/could not be cached/);
});
it.skipIf(process.platform === "win32")(
"never writes through a link at the entry or in place of its folders, and reads no linked entry",
() => {
const project = join(dir, "project");
const outside = join(dir, "outside");
const cache = join(project, ".hyperframes", "audio-kind");
mkdirSync(cache, { recursive: true });
mkdirSync(outside);
const victim = join(outside, "victim.json");
writeFileSync(victim, JSON.stringify(kind));
const entry = join(cache, "entry.json");
symlinkSync(victim, entry);
expect(readKindCache(entry)).toBeNull();
expect(writeKindCache(entry, { ...kind, value: "voice" })).toBeNull();
expect(JSON.parse(readFileSync(victim, "utf8")).value).toBe("music");
expect(lstatSync(entry).isFile()).toBe(true);
rmSync(cache, { recursive: true });
symlinkSync(outside, cache);
expect(writeKindCache(join(cache, "other.json"), kind)).toMatch(/could not be cached/);
expect(readdirSync(outside)).toEqual(["victim.json"]);
},
);
it.skipIf(process.platform === "win32")(
"reads no entry through a link in place of .hyperframes or audio-kind",
() => {
const outside = join(dir, "linked-outside");
mkdirSync(join(outside, "audio-kind"), { recursive: true });
writeFileSync(join(outside, "entry.json"), JSON.stringify(kind));
writeFileSync(join(outside, "audio-kind", "entry.json"), JSON.stringify(kind));
expect(readKindCache(join(outside, "entry.json"))).not.toBeNull();
const linkedKind = join(dir, "linked-kind", ".hyperframes");
mkdirSync(linkedKind, { recursive: true });
symlinkSync(outside, join(linkedKind, "audio-kind"));
expect(readKindCache(join(linkedKind, "audio-kind", "entry.json"))).toBeNull();
const linkedRecords = join(dir, "linked-records");
mkdirSync(linkedRecords);
symlinkSync(outside, join(linkedRecords, ".hyperframes"));
expect(
readKindCache(join(linkedRecords, ".hyperframes", "audio-kind", "entry.json")),
).toBeNull();
},
);
});
+13 -13
View File
@@ -1,11 +1,15 @@
import { createHash, randomUUID } from "node:crypto";
import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import { createHash } from "node:crypto";
import { statSync } from "node:fs";
import { dirname, join } from "node:path";
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
import { readPlainFile } from "../../utils/projectRecords.js";
import { isPlainFile, isRealFolder, makeRealFolder } from "../cacheFolder.js";
import type { KindFeatures, KindGuess } from "@hyperframes/core/audio-analysis";
import type { AnalyzeOptions, Analysis } from "./analyze.js";
import type { AudioTarget } from "./targets.js";
const CACHE_VERSION = 1;
const MAX_ENTRY_BYTES = 64 * 1024;
const record = (value: unknown): value is Record<string, unknown> =>
value !== null && typeof value === "object" && !Array.isArray(value);
@@ -68,7 +72,10 @@ export function readKindCache(file: string | null): Analysis | null {
if (file === null) return null;
let value: unknown;
try {
value = JSON.parse(readFileSync(file, "utf8"));
value =
isRealFolder(dirname(file), 2) && isPlainFile(file)
? JSON.parse(readPlainFile(file, MAX_ENTRY_BYTES))
: null;
} catch {
return null;
}
@@ -82,21 +89,14 @@ export function readKindCache(file: string | null): Analysis | null {
};
}
/** Saves a guess in `<project>/.hyperframes/audio-kind`, both real folders; a link there is replaced, not followed. */
export function writeKindCache(file: string, value: KindGuess): string | null {
const temporary = `${file}.${randomUUID()}.tmp`;
try {
mkdirSync(dirname(file), { recursive: true });
writeFileSync(temporary, JSON.stringify(value));
renameSync(temporary, file);
makeRealFolder(dirname(file), 2);
replaceFileAtomically(file, JSON.stringify(value));
return null;
} catch {
return "kind could not be cached; another query will analyze the clip again";
} finally {
try {
rmSync(temporary, { force: true });
} catch {
// A failed cache cleanup must not discard measured audio.
}
}
}
+44
View File
@@ -0,0 +1,44 @@
import { lstatSync, mkdirSync } from "node:fs";
import { basename, dirname } from "node:path";
// The audio caches live in the project's .hyperframes/. A project can hold links someone else planted there, so
// a cache folder, and every file in it, is used only when it is real: a link in its place is never followed.
function foldersOf(path: string, depth: number): string[] {
const folders: string[] = [];
for (let folder = path; folders.length < depth; folder = dirname(folder)) folders.unshift(folder);
return folders;
}
const isFolder = (path: string) =>
lstatSync(path, { throwIfNoEntry: false })?.isDirectory() === true;
export function isRealFolder(path: string, depth: number): boolean {
return foldersOf(path, depth).every(isFolder);
}
/** Makes the last `depth` folders of `path` where missing. Throws when a link or a file holds one's name. */
export function makeRealFolder(path: string, depth: number): string {
for (const folder of foldersOf(path, depth)) {
try {
mkdirSync(folder);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
}
if (!isFolder(folder))
throw new Error(
`${basename(folder)} in ${dirname(folder)} is a link or a file, not a folder; remove it and run again`,
);
}
return path;
}
export const isPlainFile = (path: string) =>
lstatSync(path, { throwIfNoEntry: false })?.isFile() === true;
export const isPlainName = (name: unknown): name is string =>
typeof name === "string" &&
name !== "." &&
name !== ".." &&
!/[/\\]/.test(name) &&
basename(name) === name;
+41 -1
View File
@@ -1,4 +1,11 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import {
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
symlinkSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
@@ -302,6 +309,39 @@ describe("describeStems", () => {
expect(warnings).toEqual([]);
});
it.skipIf(process.platform === "win32")(
"neither reads nor writes a cached answer through a link",
async () => {
vi.stubEnv("HEYGEN_API_KEY", "signed-in");
const fetch = mockMessages();
const warnings: string[] = [];
await describeAudio(warnings);
const outside = mkdtempSync(join(tmpdir(), "hf-describe-outside-"));
try {
const secret = join(outside, "secret.txt");
writeFileSync(secret, "outside words");
const answers = join(dir, "describe");
for (const name of readdirSync(answers)) {
rmSync(join(answers, name));
symlinkSync(secret, join(answers, name));
}
const again = await describeAudio(warnings);
expect(fetch).toHaveBeenCalledTimes(6);
expect(JSON.stringify(again)).not.toContain("outside words");
expect(readFileSync(secret, "utf8")).toBe("outside words");
rmSync(answers, { recursive: true });
symlinkSync(outside, answers);
const linked: string[] = [];
expect((await describeAudio(linked)).description).toBeNull();
expect(linked).toEqual([expect.stringMatching(/is a link or a file, not a folder/)]);
expect(readdirSync(outside)).toEqual(["secret.txt"]);
} finally {
rmSync(outside, { recursive: true, force: true });
}
},
);
it("does not describe silent stems", async () => {
const fetch = vi.spyOn(globalThis, "fetch");
const warnings: string[] = [];
+9 -6
View File
@@ -1,10 +1,13 @@
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { readFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Span } from "@hyperframes/core/audio-analysis";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
import { readPlainFile } from "../../utils/projectRecords.js";
import { isPlainFile, makeRealFolder } from "../cacheFolder.js";
import {
runFile,
serviceAccess,
@@ -155,6 +158,8 @@ async function generate(
}
}
const MAX_ANSWER_BYTES = 1024 * 1024;
interface Ask {
path: string | null;
prompt: string;
@@ -162,8 +167,7 @@ interface Ask {
}
async function answers(asks: Ask[], cacheDir: string, access: ServiceAccess) {
const dir = join(cacheDir, "describe");
mkdirSync(dir, { recursive: true });
const dir = makeRealFolder(join(cacheDir, "describe"), 1);
const scratch = await mkdtemp(join(tmpdir(), "hf-describe-"));
try {
// Wait for every conversion before removing their shared scratch directory.
@@ -176,11 +180,10 @@ async function answers(asks: Ask[], cacheDir: string, access: ServiceAccess) {
.update(structured ? JSON.stringify(OVERALL_SCHEMA) : "plain");
const id = (audio ? hash.update(audio) : hash).digest("hex");
const file = join(dir, `${id.slice(0, 24)}.txt`);
if (existsSync(file)) return readFileSync(file, "utf8");
if (isPlainFile(file)) return readPlainFile(file, MAX_ANSWER_BYTES);
const text = await generate(access, audio, prompt, structured);
if (structured) descriptionOf(text);
writeFileSync(`${file}.tmp`, text);
renameSync(`${file}.tmp`, file);
replaceFileAtomically(file, text);
return text;
}),
);
+135 -1
View File
@@ -1,6 +1,16 @@
import { createServer, type IncomingMessage, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import {
existsSync,
lstatSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
symlinkSync,
writeFileSync,
} from "node:fs";
import { createHash } from "node:crypto";
import { execFileSync } from "node:child_process";
import { tmpdir } from "node:os";
@@ -552,3 +562,127 @@ describe.skipIf(!ffmpeg)("readAudio with a declared layer", () => {
expect("layer" in analysis).toBe(false);
});
});
// Links a project can hold in its .hyperframes/: the reader runs outside any sandbox, so none may be followed.
describe.skipIf(!ffmpeg || process.platform === "win32")("readAudio cache links", () => {
function project(name: string) {
const cacheDir = join(dir, name, ".hyperframes", "read-audio");
const outside = join(dir, `${name}-outside`);
mkdirSync(cacheDir, { recursive: true });
mkdirSync(outside);
const victim = join(outside, "victim.txt");
writeFileSync(victim, "keep");
return { cacheDir, outside, victim };
}
const shaOf = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex");
const isPlain = (path: string) => lstatSync(path).isFile();
it("never writes a cache entry through a link at its temp or final name", async () => {
const { cacheDir, victim } = project("entry-links");
const path = wav("entry-links.wav", 2, voiced);
const entry = join(cacheDir, `entry-links-${shaOf(path).slice(0, 12)}`);
for (const ext of ["json", "txt"]) symlinkSync(victim, `${entry}.${ext}.${process.pid}.tmp`);
await readAudio(path, "entry-links.wav", { cacheDir });
expect(readFileSync(victim, "utf8")).toBe("keep");
rmSync(`${entry}.json`);
rmSync(`${entry}.txt`);
symlinkSync(victim, `${entry}.json`);
symlinkSync(victim, `${entry}.txt`);
const again = await readAudio(path, "entry-links.wav", { cacheDir });
expect(again.cached).toBe(false);
expect(readFileSync(victim, "utf8")).toBe("keep");
expect(isPlain(`${entry}.json`) && isPlain(`${entry}.txt`)).toBe(true);
});
it.each([".hyperframes", join(".hyperframes", "read-audio")])(
"refuses a link in place of %s and writes nothing outside the project",
async (folder) => {
const name = `folder-link-${folder.length}`;
const { cacheDir, outside } = project(name);
const linked = join(dir, name, folder);
rmSync(linked, { recursive: true });
symlinkSync(outside, linked);
const path = wav(`${name}.wav`, 2, voiced);
await expect(readAudio(path, `${name}.wav`, { cacheDir })).rejects.toThrow(
/is a link or a file, not a folder/,
);
expect(readdirSync(outside)).toEqual(["victim.txt"]);
},
);
it("saves stems and their list in place of links, never through them", async () => {
for (const name of Object.keys(stemFiles)) delete stemFiles[name];
const silence = wav("stem-links-silence.wav", 1, () => 0);
for (const name of ["vocals", "drums", "bass", "guitar", "piano", "other"])
stemFiles[name] = silence;
const { cacheDir, outside, victim } = project("stem-links");
const stemDir = join(cacheDir, "stems", "0123456789ab");
mkdirSync(stemDir, { recursive: true });
symlinkSync(victim, join(stemDir, "vocals.audio"));
symlinkSync(join(outside, "planted.json"), join(stemDir, "stems.json"));
const access = {
url: `${origin()}/v3/audio/stems`,
headers: { "x-api-key": "test-key" },
timeoutMs: 60_000,
};
const stems = await fetchStems(wav("stem-links.wav", 1, voiced), stemDir, access);
expect(readFileSync(victim, "utf8")).toBe("keep");
expect(existsSync(join(outside, "planted.json"))).toBe(false);
expect(isPlain(stems.vocals!) && isPlain(join(stemDir, "stems.json"))).toBe(true);
});
it("reads no stems through a linked folder or a name that leaves it, and refuses to fetch into a link", async () => {
const { cacheDir, outside } = project("stem-folder-link");
const silence = wav("stem-folder-silence.wav", 1, () => 0);
const names = ["vocals", "drums", "bass", "guitar", "piano", "other"];
writeFileSync(join(outside, "stem.wav"), readFileSync(silence));
writeFileSync(
join(outside, "stems.json"),
JSON.stringify(Object.fromEntries(names.map((n) => [n, "stem.wav"]))),
);
mkdirSync(join(cacheDir, "stems"));
symlinkSync(outside, join(cacheDir, "stems", "0123456789ab"));
const linkedDir = join(cacheDir, "stems", "0123456789ab");
expect(cachedStems(linkedDir)).toBeNull();
const realDir = join(cacheDir, "stems", "ba9876543210");
mkdirSync(realDir);
writeFileSync(
join(realDir, "stems.json"),
JSON.stringify(
Object.fromEntries(names.map((n) => [n, "../../../../stem-folder-link-outside/stem.wav"])),
),
);
expect(cachedStems(realDir)).toBeNull();
for (const name of names) stemFiles[name] = silence;
await expect(
fetchStems(wav("stem-folder.wav", 1, voiced), linkedDir, {
url: `${origin()}/v3/audio/stems`,
headers: { "x-api-key": "test-key" },
timeoutMs: 60_000,
}),
).rejects.toThrow(/is a link or a file, not a folder/);
expect(readdirSync(outside).sort()).toEqual(["stem.wav", "stems.json", "victim.txt"]);
});
it("mixes the voice-free stems into a staged file, never through a link at a temp name", async () => {
const { cacheDir, victim } = project("mix-links");
const path = wav("mix-links.wav", 2, (t) => voiced(t) + bed(t));
const stemDir = join(cacheDir, "stems", shaOf(path).slice(0, 12));
mkdirSync(stemDir, { recursive: true });
writeFileSync(join(stemDir, "vocals.wav"), readFileSync(wav("mix-vocals.wav", 2, voiced)));
writeFileSync(join(stemDir, "bed.wav"), readFileSync(wav("mix-bed.wav", 2, bed)));
const names = { vocals: "vocals.wav", drums: "bed.wav", bass: "bed.wav" };
writeFileSync(
join(stemDir, "stems.json"),
JSON.stringify({ ...names, guitar: "bed.wav", piano: "bed.wav", other: "bed.wav" }),
);
symlinkSync(victim, join(stemDir, `no-vocals.wav.${process.pid}.tmp.wav`));
const { analysis } = await readAudio(path, "mix-links.wav", { cacheDir, quality: "fast" });
expect(analysis.voiceFreeMix).toMatch(/no-vocals\.wav$/);
expect(readFileSync(victim, "utf8")).toBe("keep");
expect(isPlain(join(stemDir, "no-vocals.wav"))).toBe(true);
});
});
+29 -25
View File
@@ -1,17 +1,13 @@
import { createHash } from "node:crypto";
import {
createReadStream,
existsSync,
mkdirSync,
readFileSync,
renameSync,
writeFileSync,
} from "node:fs";
import { createReadStream, existsSync, readFileSync, renameSync, rmSync } from "node:fs";
import { basename, dirname, extname, join, relative } from "node:path";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { createStagingDir, replaceFileAtomically } from "@hyperframes/core/atomic-file";
import { loudness, type AnalysisQuality, type Span } from "@hyperframes/core/audio-analysis";
import { loadTranscript } from "../../whisper/normalize.js";
import { readPlainFile } from "../../utils/projectRecords.js";
import { runFile } from "../analysis/providers/shared.js";
import { isPlainFile, makeRealFolder } from "../cacheFolder.js";
import { describeAccess, describeStems } from "./describe.js";
import { findEffects, type EffectSource } from "./effects.js";
import { formatRead } from "./format.js";
@@ -74,13 +70,8 @@ function wordsOf(transcript: string): TimedWord[] {
const fingerprint = (path: string | null) =>
path ? createHash("sha256").update(readFileSync(path)).digest("hex").slice(0, 16) : "none";
function writeAtomic(path: string, content: string) {
const temporary = `${path}.${process.pid}.tmp`;
writeFileSync(temporary, content);
renameSync(temporary, path);
}
const CACHE_REVISION = 1;
const MAX_ENTRY_BYTES = 256 * 1024 * 1024;
interface CacheEntry {
revision: number;
@@ -88,6 +79,15 @@ interface CacheEntry {
transcriptKey: string;
}
function cachedEntry(jsonPath: string, textPath: string): CacheEntry | null {
if (!isPlainFile(jsonPath) || !isPlainFile(textPath)) return null;
try {
return JSON.parse(readPlainFile(jsonPath, MAX_ENTRY_BYTES)) as CacheEntry;
} catch {
return null;
}
}
async function stemsFor(
path: string,
dir: string,
@@ -130,14 +130,16 @@ const VOICE_FREE = "no-vocals.wav";
/** Stems summed at full quality into one file next to them, made once. */
async function mixedFile(paths: string[], out: string, warnings: string[]) {
if (existsSync(out)) return out;
if (isPlainFile(out)) return out;
const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true });
if (!ffmpeg) {
warnings.push(`no ${basename(out)}: ffmpeg is required; run: npx hyperframes doctor`);
return null;
}
const temporary = `${out}.${process.pid}.tmp.wav`;
let stage: string | undefined;
try {
stage = createStagingDir(dirname(out), ".hf-mix-");
const temporary = join(stage, VOICE_FREE);
await runFile(
ffmpeg,
[
@@ -153,6 +155,8 @@ async function mixedFile(paths: string[], out: string, warnings: string[]) {
} catch (error) {
warnings.push(`no ${basename(out)}: ${error instanceof Error ? error.message : error}`);
return null;
} finally {
if (stage) rmSync(stage, { recursive: true, force: true });
}
}
@@ -461,12 +465,12 @@ export async function readAudio(
const transcript = transcriptFor(path, options.transcript);
const transcriptKey = fingerprint(transcript);
const stem = `${basename(file, extname(file)).replace(/[^\w.-]/g, "_")}-${sha.slice(0, 12)}`;
mkdirSync(options.cacheDir, { recursive: true });
const jsonPath = join(options.cacheDir, `${stem}.json`);
const textPath = join(options.cacheDir, `${stem}.txt`);
const stemDir = join(options.cacheDir, "stems", sha.slice(0, 12));
if (existsSync(jsonPath) && existsSync(textPath)) {
const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry;
const cacheDir = makeRealFolder(options.cacheDir, 2);
const jsonPath = join(cacheDir, `${stem}.json`);
const textPath = join(cacheDir, `${stem}.txt`);
const stemDir = join(cacheDir, "stems", sha.slice(0, 12));
const entry = cachedEntry(jsonPath, textPath);
if (entry) {
const stemsNowPossible =
!layer &&
entry.analysis.separation === "mix" &&
@@ -490,11 +494,11 @@ export async function readAudio(
}
const analysis = layer
? await analyzeLayer(path, file, sha, transcript, layer)
: await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
writeAtomic(
: await analyze(path, file, sha, transcript, stemDir, quality, cacheDir);
replaceFileAtomically(
jsonPath,
JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry),
);
writeAtomic(textPath, formatRead(analysis));
replaceFileAtomically(textPath, formatRead(analysis));
return { analysis, textPath, jsonPath, cached: false };
}
+27 -9
View File
@@ -1,9 +1,10 @@
import { randomUUID } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { statSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
import { setTimeout as sleep } from "node:timers/promises";
import { downloadToFile } from "../../cloud/download.js";
import { HyperframesCloudClient } from "../../cloud/_gen/client.js";
@@ -15,6 +16,8 @@ import {
type ServiceAccess,
} from "../analysis/providers/shared.js";
import { probeAudio } from "../analysis/decode.js";
import { readPlainFile } from "../../utils/projectRecords.js";
import { isPlainFile, isPlainName, isRealFolder, makeRealFolder } from "../cacheFolder.js";
/**
* Upload 192 kb/s stereo MP3 through the service origin's direct-upload routes, then POST
@@ -36,6 +39,7 @@ const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS";
const DEFAULT_TIMEOUT_MS = 30 * 60_000;
const RETRY_MS = 5000;
const DONE = "stems.json";
const MAX_DONE_BYTES = 64 * 1024;
const STEM_NAMES = ["vocals", "drums", "bass", "guitar", "piano", "other"] as const;
export const VOICE_STEM = "vocals";
@@ -49,14 +53,28 @@ export const stemsAccess = () =>
DEFAULT_TIMEOUT_MS,
);
/** Stems already fetched into `dir`, by name; null when there are none yet. */
function namedStems(text: string): Record<string, string> | null {
let names: unknown;
try {
names = JSON.parse(text);
} catch {
return null;
}
if (typeof names !== "object" || names === null) return null;
const files = names as Record<string, unknown>;
return STEM_NAMES.every((name) => isPlainName(files[name]))
? Object.fromEntries(STEM_NAMES.map((name) => [name, files[name] as string]))
: null;
}
/** Stems already fetched into `dir`, by name; null when there are none yet or a link stands in for any of it. */
export function cachedStems(dir: string): Record<string, string> | null {
const done = join(dir, DONE);
if (!existsSync(done)) return null;
const names = JSON.parse(readFileSync(done, "utf8")) as Record<string, string>;
if (!STEM_NAMES.every((name) => typeof names[name] === "string")) return null;
if (!isRealFolder(dir, 2) || !isPlainFile(done)) return null;
const names = namedStems(readPlainFile(done, MAX_DONE_BYTES));
if (!names) return null;
const paths = Object.fromEntries(STEM_NAMES.map((name) => [name, join(dir, names[name]!)]));
return Object.values(paths).every((p) => existsSync(p)) ? paths : null;
return Object.values(paths).every(isPlainFile) ? paths : null;
}
class StillRunning extends Error {}
@@ -165,16 +183,16 @@ export async function fetchStems(
throw new Error(
`the stems service returned an incomplete separation: missing ${missing.join(", ")}`,
);
mkdirSync(dir, { recursive: true });
makeRealFolder(dir, 2);
const names: Record<string, string> = {};
for (const name of STEM_NAMES) {
const stem = stems.find((candidate) => candidate.name === name);
if (!stem) continue;
const file = `${name}.audio`;
await downloadToFile(stem.url, join(dir, file), { signal });
await downloadToFile(stem.url, join(dir, file), { signal, replaceLink: true });
names[name] = file;
}
writeFileSync(join(dir, DONE), JSON.stringify(names));
replaceFileAtomically(join(dir, DONE), JSON.stringify(names));
return cachedStems(dir)!;
} finally {
await rm(scratch, { recursive: true, force: true });
+21
View File
@@ -262,6 +262,27 @@ describe("cloud/download", () => {
},
);
it.skipIf(process.platform === "win32")(
"with replaceLink, replaces an output link instead of writing to its target",
async () => {
const target = join(dir, "target.mp4");
writeFileSync(target, "keep");
chmodSync(target, 0o755);
const alias = join(dir, "alias.mp4");
symlinkSync("target.mp4", alias);
await downloadToFile("https://example/x", alias, {
fetchImpl: makeBytesFetch(new Uint8Array([7])),
replaceLink: true,
});
const downloaded = readFileSync(alias);
const written = lstatSync(alias);
expect(readFileSync(target, "utf8")).toBe("keep");
expect(written.isFile()).toBe(true);
expect(written.mode & 0o111).toBe(0);
expect(downloaded).toEqual(Buffer.from([7]));
},
);
it.skipIf(process.platform === "win32")(
"follows a relative output link inside a linked folder to the real target",
async () => {
+7 -4
View File
@@ -13,7 +13,7 @@
* existing destination is replaced only after the download succeeds.
*/
import { chmodSync, renameSync, rmSync, statSync } from "node:fs";
import { chmodSync, lstatSync, renameSync, rmSync, statSync } from "node:fs";
import { makeOutputFolder } from "../utils/outputFolder.js";
import { open } from "node:fs/promises";
import { dirname, join } from "node:path";
@@ -34,6 +34,8 @@ export interface DownloadOptions {
/** Called with (bytes downloaded, total or undefined). */
onProgress?: (bytes: number, total: number | undefined) => void;
partBytes?: number;
/** Replace a link at `destPath` with the download instead of writing to the file it points at. */
replaceLink?: boolean;
}
export interface DownloadResult {
@@ -76,9 +78,10 @@ export async function downloadToFile(
makeOutputFolder(dirname(destPath));
// Writes through a caller-selected symlink, as before.
const destination = resolveWritePath(destPath);
const previous = statSync(destination, { throwIfNoEntry: false });
const destination = options.replaceLink ? destPath : resolveWritePath(destPath);
const previous = options.replaceLink
? lstatSync(destination, { throwIfNoEntry: false })
: statSync(destination, { throwIfNoEntry: false });
const stage = createStagingDir(dirname(destination), ".hf-download-");
const stagedFile = join(stage, "download");
let bytes = 0;