From 9701f0639c3614f4fb33f2a83973adafd179ef08 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Sun, 15 Nov 2026 17:31:16 +0000 Subject: [PATCH] fix(cli): the audio caches never write or read through links in .hyperframes (#7825) * fix(cli): the audio caches never write or read through links in .hyperframes The read-audio cache, its stems, mix and descriptions, and the audio-kind cache now use only real folders under the project's .hyperframes. A link or a file in place of a cache folder stops the write with a clear message. Every cache file is written to an exclusive temp file with a random name and renamed into place, so a link at a temp or final name is replaced, never written through. ffmpeg writes the voice-free mix into a private staging folder. Stem downloads use the new downloadToFile replaceLink option. Cached files are read only when they are plain files, and a stem list may only name files in its own folder. * fix(cli): the kind cache reads no entry through a linked cache folder readKindCache now checks that .hyperframes and audio-kind are real folders before it reads an entry, as writeKindCache already does before it writes one. * test(cli): read the replaced download before checking its type and mode The replaceLink test now reads the file's bytes and lstat result once each, so no read follows a check on the same path. --- .../cli/src/audio/analysis/kindCache.test.ts | 58 +++++++- packages/cli/src/audio/analysis/kindCache.ts | 26 ++-- packages/cli/src/audio/cacheFolder.ts | 44 ++++++ packages/cli/src/audio/read/describe.test.ts | 42 +++++- packages/cli/src/audio/read/describe.ts | 15 +- packages/cli/src/audio/read/read.test.ts | 136 +++++++++++++++++- packages/cli/src/audio/read/read.ts | 54 +++---- packages/cli/src/audio/read/stems.ts | 36 +++-- packages/cli/src/cloud/download.test.ts | 21 +++ packages/cli/src/cloud/download.ts | 11 +- 10 files changed, 383 insertions(+), 60 deletions(-) create mode 100644 packages/cli/src/audio/cacheFolder.ts diff --git a/packages/cli/src/audio/analysis/kindCache.test.ts b/packages/cli/src/audio/analysis/kindCache.test.ts index e46d71fc0f..187654cbc5 100644 --- a/packages/cli/src/audio/analysis/kindCache.test.ts +++ b/packages/cli/src/audio/analysis/kindCache.test.ts @@ -1,4 +1,13 @@ -import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, describe, expect, it } from "vitest"; @@ -41,4 +50,51 @@ describe("kind cache", () => { writeFileSync(blocked, "file"); expect(writeKindCache(join(blocked, "entry.json"), kind)).toMatch(/could not be cached/); }); + + it.skipIf(process.platform === "win32")( + "never writes through a link at the entry or in place of its folders, and reads no linked entry", + () => { + const project = join(dir, "project"); + const outside = join(dir, "outside"); + const cache = join(project, ".hyperframes", "audio-kind"); + mkdirSync(cache, { recursive: true }); + mkdirSync(outside); + const victim = join(outside, "victim.json"); + writeFileSync(victim, JSON.stringify(kind)); + const entry = join(cache, "entry.json"); + symlinkSync(victim, entry); + expect(readKindCache(entry)).toBeNull(); + expect(writeKindCache(entry, { ...kind, value: "voice" })).toBeNull(); + expect(JSON.parse(readFileSync(victim, "utf8")).value).toBe("music"); + expect(lstatSync(entry).isFile()).toBe(true); + + rmSync(cache, { recursive: true }); + symlinkSync(outside, cache); + expect(writeKindCache(join(cache, "other.json"), kind)).toMatch(/could not be cached/); + expect(readdirSync(outside)).toEqual(["victim.json"]); + }, + ); + + it.skipIf(process.platform === "win32")( + "reads no entry through a link in place of .hyperframes or audio-kind", + () => { + const outside = join(dir, "linked-outside"); + mkdirSync(join(outside, "audio-kind"), { recursive: true }); + writeFileSync(join(outside, "entry.json"), JSON.stringify(kind)); + writeFileSync(join(outside, "audio-kind", "entry.json"), JSON.stringify(kind)); + expect(readKindCache(join(outside, "entry.json"))).not.toBeNull(); + + const linkedKind = join(dir, "linked-kind", ".hyperframes"); + mkdirSync(linkedKind, { recursive: true }); + symlinkSync(outside, join(linkedKind, "audio-kind")); + expect(readKindCache(join(linkedKind, "audio-kind", "entry.json"))).toBeNull(); + + const linkedRecords = join(dir, "linked-records"); + mkdirSync(linkedRecords); + symlinkSync(outside, join(linkedRecords, ".hyperframes")); + expect( + readKindCache(join(linkedRecords, ".hyperframes", "audio-kind", "entry.json")), + ).toBeNull(); + }, + ); }); diff --git a/packages/cli/src/audio/analysis/kindCache.ts b/packages/cli/src/audio/analysis/kindCache.ts index 434992a0d6..465d31587d 100644 --- a/packages/cli/src/audio/analysis/kindCache.ts +++ b/packages/cli/src/audio/analysis/kindCache.ts @@ -1,11 +1,15 @@ -import { createHash, randomUUID } from "node:crypto"; -import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { statSync } from "node:fs"; import { dirname, join } from "node:path"; +import { replaceFileAtomically } from "@hyperframes/core/atomic-file"; +import { readPlainFile } from "../../utils/projectRecords.js"; +import { isPlainFile, isRealFolder, makeRealFolder } from "../cacheFolder.js"; import type { KindFeatures, KindGuess } from "@hyperframes/core/audio-analysis"; import type { AnalyzeOptions, Analysis } from "./analyze.js"; import type { AudioTarget } from "./targets.js"; const CACHE_VERSION = 1; +const MAX_ENTRY_BYTES = 64 * 1024; const record = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); @@ -68,7 +72,10 @@ export function readKindCache(file: string | null): Analysis | null { if (file === null) return null; let value: unknown; try { - value = JSON.parse(readFileSync(file, "utf8")); + value = + isRealFolder(dirname(file), 2) && isPlainFile(file) + ? JSON.parse(readPlainFile(file, MAX_ENTRY_BYTES)) + : null; } catch { return null; } @@ -82,21 +89,14 @@ export function readKindCache(file: string | null): Analysis | null { }; } +/** Saves a guess in `/.hyperframes/audio-kind`, both real folders; a link there is replaced, not followed. */ export function writeKindCache(file: string, value: KindGuess): string | null { - const temporary = `${file}.${randomUUID()}.tmp`; try { - mkdirSync(dirname(file), { recursive: true }); - writeFileSync(temporary, JSON.stringify(value)); - renameSync(temporary, file); + makeRealFolder(dirname(file), 2); + replaceFileAtomically(file, JSON.stringify(value)); return null; } catch { return "kind could not be cached; another query will analyze the clip again"; - } finally { - try { - rmSync(temporary, { force: true }); - } catch { - // A failed cache cleanup must not discard measured audio. - } } } diff --git a/packages/cli/src/audio/cacheFolder.ts b/packages/cli/src/audio/cacheFolder.ts new file mode 100644 index 0000000000..17ef932967 --- /dev/null +++ b/packages/cli/src/audio/cacheFolder.ts @@ -0,0 +1,44 @@ +import { lstatSync, mkdirSync } from "node:fs"; +import { basename, dirname } from "node:path"; + +// The audio caches live in the project's .hyperframes/. A project can hold links someone else planted there, so +// a cache folder, and every file in it, is used only when it is real: a link in its place is never followed. + +function foldersOf(path: string, depth: number): string[] { + const folders: string[] = []; + for (let folder = path; folders.length < depth; folder = dirname(folder)) folders.unshift(folder); + return folders; +} + +const isFolder = (path: string) => + lstatSync(path, { throwIfNoEntry: false })?.isDirectory() === true; + +export function isRealFolder(path: string, depth: number): boolean { + return foldersOf(path, depth).every(isFolder); +} + +/** Makes the last `depth` folders of `path` where missing. Throws when a link or a file holds one's name. */ +export function makeRealFolder(path: string, depth: number): string { + for (const folder of foldersOf(path, depth)) { + try { + mkdirSync(folder); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + if (!isFolder(folder)) + throw new Error( + `${basename(folder)} in ${dirname(folder)} is a link or a file, not a folder; remove it and run again`, + ); + } + return path; +} + +export const isPlainFile = (path: string) => + lstatSync(path, { throwIfNoEntry: false })?.isFile() === true; + +export const isPlainName = (name: unknown): name is string => + typeof name === "string" && + name !== "." && + name !== ".." && + !/[/\\]/.test(name) && + basename(name) === name; diff --git a/packages/cli/src/audio/read/describe.test.ts b/packages/cli/src/audio/read/describe.test.ts index 5c394fbc3a..b658c00994 100644 --- a/packages/cli/src/audio/read/describe.test.ts +++ b/packages/cli/src/audio/read/describe.test.ts @@ -1,4 +1,11 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -302,6 +309,39 @@ describe("describeStems", () => { expect(warnings).toEqual([]); }); + it.skipIf(process.platform === "win32")( + "neither reads nor writes a cached answer through a link", + async () => { + vi.stubEnv("HEYGEN_API_KEY", "signed-in"); + const fetch = mockMessages(); + const warnings: string[] = []; + await describeAudio(warnings); + const outside = mkdtempSync(join(tmpdir(), "hf-describe-outside-")); + try { + const secret = join(outside, "secret.txt"); + writeFileSync(secret, "outside words"); + const answers = join(dir, "describe"); + for (const name of readdirSync(answers)) { + rmSync(join(answers, name)); + symlinkSync(secret, join(answers, name)); + } + const again = await describeAudio(warnings); + expect(fetch).toHaveBeenCalledTimes(6); + expect(JSON.stringify(again)).not.toContain("outside words"); + expect(readFileSync(secret, "utf8")).toBe("outside words"); + + rmSync(answers, { recursive: true }); + symlinkSync(outside, answers); + const linked: string[] = []; + expect((await describeAudio(linked)).description).toBeNull(); + expect(linked).toEqual([expect.stringMatching(/is a link or a file, not a folder/)]); + expect(readdirSync(outside)).toEqual(["secret.txt"]); + } finally { + rmSync(outside, { recursive: true, force: true }); + } + }, + ); + it("does not describe silent stems", async () => { const fetch = vi.spyOn(globalThis, "fetch"); const warnings: string[] = []; diff --git a/packages/cli/src/audio/read/describe.ts b/packages/cli/src/audio/read/describe.ts index bf27881e0c..a7831e538f 100644 --- a/packages/cli/src/audio/read/describe.ts +++ b/packages/cli/src/audio/read/describe.ts @@ -1,10 +1,13 @@ import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Span } from "@hyperframes/core/audio-analysis"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { replaceFileAtomically } from "@hyperframes/core/atomic-file"; +import { readPlainFile } from "../../utils/projectRecords.js"; +import { isPlainFile, makeRealFolder } from "../cacheFolder.js"; import { runFile, serviceAccess, @@ -155,6 +158,8 @@ async function generate( } } +const MAX_ANSWER_BYTES = 1024 * 1024; + interface Ask { path: string | null; prompt: string; @@ -162,8 +167,7 @@ interface Ask { } async function answers(asks: Ask[], cacheDir: string, access: ServiceAccess) { - const dir = join(cacheDir, "describe"); - mkdirSync(dir, { recursive: true }); + const dir = makeRealFolder(join(cacheDir, "describe"), 1); const scratch = await mkdtemp(join(tmpdir(), "hf-describe-")); try { // Wait for every conversion before removing their shared scratch directory. @@ -176,11 +180,10 @@ async function answers(asks: Ask[], cacheDir: string, access: ServiceAccess) { .update(structured ? JSON.stringify(OVERALL_SCHEMA) : "plain"); const id = (audio ? hash.update(audio) : hash).digest("hex"); const file = join(dir, `${id.slice(0, 24)}.txt`); - if (existsSync(file)) return readFileSync(file, "utf8"); + if (isPlainFile(file)) return readPlainFile(file, MAX_ANSWER_BYTES); const text = await generate(access, audio, prompt, structured); if (structured) descriptionOf(text); - writeFileSync(`${file}.tmp`, text); - renameSync(`${file}.tmp`, file); + replaceFileAtomically(file, text); return text; }), ); diff --git a/packages/cli/src/audio/read/read.test.ts b/packages/cli/src/audio/read/read.test.ts index 346760e1e4..3f1f7e3f02 100644 --- a/packages/cli/src/audio/read/read.test.ts +++ b/packages/cli/src/audio/read/read.test.ts @@ -1,6 +1,16 @@ import { createServer, type IncomingMessage, type Server } from "node:http"; import type { AddressInfo } from "node:net"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; import { tmpdir } from "node:os"; @@ -552,3 +562,127 @@ describe.skipIf(!ffmpeg)("readAudio with a declared layer", () => { expect("layer" in analysis).toBe(false); }); }); + +// Links a project can hold in its .hyperframes/: the reader runs outside any sandbox, so none may be followed. +describe.skipIf(!ffmpeg || process.platform === "win32")("readAudio cache links", () => { + function project(name: string) { + const cacheDir = join(dir, name, ".hyperframes", "read-audio"); + const outside = join(dir, `${name}-outside`); + mkdirSync(cacheDir, { recursive: true }); + mkdirSync(outside); + const victim = join(outside, "victim.txt"); + writeFileSync(victim, "keep"); + return { cacheDir, outside, victim }; + } + const shaOf = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex"); + const isPlain = (path: string) => lstatSync(path).isFile(); + + it("never writes a cache entry through a link at its temp or final name", async () => { + const { cacheDir, victim } = project("entry-links"); + const path = wav("entry-links.wav", 2, voiced); + const entry = join(cacheDir, `entry-links-${shaOf(path).slice(0, 12)}`); + for (const ext of ["json", "txt"]) symlinkSync(victim, `${entry}.${ext}.${process.pid}.tmp`); + await readAudio(path, "entry-links.wav", { cacheDir }); + expect(readFileSync(victim, "utf8")).toBe("keep"); + + rmSync(`${entry}.json`); + rmSync(`${entry}.txt`); + symlinkSync(victim, `${entry}.json`); + symlinkSync(victim, `${entry}.txt`); + const again = await readAudio(path, "entry-links.wav", { cacheDir }); + expect(again.cached).toBe(false); + expect(readFileSync(victim, "utf8")).toBe("keep"); + expect(isPlain(`${entry}.json`) && isPlain(`${entry}.txt`)).toBe(true); + }); + + it.each([".hyperframes", join(".hyperframes", "read-audio")])( + "refuses a link in place of %s and writes nothing outside the project", + async (folder) => { + const name = `folder-link-${folder.length}`; + const { cacheDir, outside } = project(name); + const linked = join(dir, name, folder); + rmSync(linked, { recursive: true }); + symlinkSync(outside, linked); + const path = wav(`${name}.wav`, 2, voiced); + await expect(readAudio(path, `${name}.wav`, { cacheDir })).rejects.toThrow( + /is a link or a file, not a folder/, + ); + expect(readdirSync(outside)).toEqual(["victim.txt"]); + }, + ); + + it("saves stems and their list in place of links, never through them", async () => { + for (const name of Object.keys(stemFiles)) delete stemFiles[name]; + const silence = wav("stem-links-silence.wav", 1, () => 0); + for (const name of ["vocals", "drums", "bass", "guitar", "piano", "other"]) + stemFiles[name] = silence; + const { cacheDir, outside, victim } = project("stem-links"); + const stemDir = join(cacheDir, "stems", "0123456789ab"); + mkdirSync(stemDir, { recursive: true }); + symlinkSync(victim, join(stemDir, "vocals.audio")); + symlinkSync(join(outside, "planted.json"), join(stemDir, "stems.json")); + const access = { + url: `${origin()}/v3/audio/stems`, + headers: { "x-api-key": "test-key" }, + timeoutMs: 60_000, + }; + const stems = await fetchStems(wav("stem-links.wav", 1, voiced), stemDir, access); + expect(readFileSync(victim, "utf8")).toBe("keep"); + expect(existsSync(join(outside, "planted.json"))).toBe(false); + expect(isPlain(stems.vocals!) && isPlain(join(stemDir, "stems.json"))).toBe(true); + }); + + it("reads no stems through a linked folder or a name that leaves it, and refuses to fetch into a link", async () => { + const { cacheDir, outside } = project("stem-folder-link"); + const silence = wav("stem-folder-silence.wav", 1, () => 0); + const names = ["vocals", "drums", "bass", "guitar", "piano", "other"]; + writeFileSync(join(outside, "stem.wav"), readFileSync(silence)); + writeFileSync( + join(outside, "stems.json"), + JSON.stringify(Object.fromEntries(names.map((n) => [n, "stem.wav"]))), + ); + mkdirSync(join(cacheDir, "stems")); + symlinkSync(outside, join(cacheDir, "stems", "0123456789ab")); + const linkedDir = join(cacheDir, "stems", "0123456789ab"); + expect(cachedStems(linkedDir)).toBeNull(); + + const realDir = join(cacheDir, "stems", "ba9876543210"); + mkdirSync(realDir); + writeFileSync( + join(realDir, "stems.json"), + JSON.stringify( + Object.fromEntries(names.map((n) => [n, "../../../../stem-folder-link-outside/stem.wav"])), + ), + ); + expect(cachedStems(realDir)).toBeNull(); + + for (const name of names) stemFiles[name] = silence; + await expect( + fetchStems(wav("stem-folder.wav", 1, voiced), linkedDir, { + url: `${origin()}/v3/audio/stems`, + headers: { "x-api-key": "test-key" }, + timeoutMs: 60_000, + }), + ).rejects.toThrow(/is a link or a file, not a folder/); + expect(readdirSync(outside).sort()).toEqual(["stem.wav", "stems.json", "victim.txt"]); + }); + + it("mixes the voice-free stems into a staged file, never through a link at a temp name", async () => { + const { cacheDir, victim } = project("mix-links"); + const path = wav("mix-links.wav", 2, (t) => voiced(t) + bed(t)); + const stemDir = join(cacheDir, "stems", shaOf(path).slice(0, 12)); + mkdirSync(stemDir, { recursive: true }); + writeFileSync(join(stemDir, "vocals.wav"), readFileSync(wav("mix-vocals.wav", 2, voiced))); + writeFileSync(join(stemDir, "bed.wav"), readFileSync(wav("mix-bed.wav", 2, bed))); + const names = { vocals: "vocals.wav", drums: "bed.wav", bass: "bed.wav" }; + writeFileSync( + join(stemDir, "stems.json"), + JSON.stringify({ ...names, guitar: "bed.wav", piano: "bed.wav", other: "bed.wav" }), + ); + symlinkSync(victim, join(stemDir, `no-vocals.wav.${process.pid}.tmp.wav`)); + const { analysis } = await readAudio(path, "mix-links.wav", { cacheDir, quality: "fast" }); + expect(analysis.voiceFreeMix).toMatch(/no-vocals\.wav$/); + expect(readFileSync(victim, "utf8")).toBe("keep"); + expect(isPlain(join(stemDir, "no-vocals.wav"))).toBe(true); + }); +}); diff --git a/packages/cli/src/audio/read/read.ts b/packages/cli/src/audio/read/read.ts index ee37907810..59b9c4e494 100644 --- a/packages/cli/src/audio/read/read.ts +++ b/packages/cli/src/audio/read/read.ts @@ -1,17 +1,13 @@ import { createHash } from "node:crypto"; -import { - createReadStream, - existsSync, - mkdirSync, - readFileSync, - renameSync, - writeFileSync, -} from "node:fs"; +import { createReadStream, existsSync, readFileSync, renameSync, rmSync } from "node:fs"; import { basename, dirname, extname, join, relative } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { createStagingDir, replaceFileAtomically } from "@hyperframes/core/atomic-file"; import { loudness, type AnalysisQuality, type Span } from "@hyperframes/core/audio-analysis"; import { loadTranscript } from "../../whisper/normalize.js"; +import { readPlainFile } from "../../utils/projectRecords.js"; import { runFile } from "../analysis/providers/shared.js"; +import { isPlainFile, makeRealFolder } from "../cacheFolder.js"; import { describeAccess, describeStems } from "./describe.js"; import { findEffects, type EffectSource } from "./effects.js"; import { formatRead } from "./format.js"; @@ -74,13 +70,8 @@ function wordsOf(transcript: string): TimedWord[] { const fingerprint = (path: string | null) => path ? createHash("sha256").update(readFileSync(path)).digest("hex").slice(0, 16) : "none"; -function writeAtomic(path: string, content: string) { - const temporary = `${path}.${process.pid}.tmp`; - writeFileSync(temporary, content); - renameSync(temporary, path); -} - const CACHE_REVISION = 1; +const MAX_ENTRY_BYTES = 256 * 1024 * 1024; interface CacheEntry { revision: number; @@ -88,6 +79,15 @@ interface CacheEntry { transcriptKey: string; } +function cachedEntry(jsonPath: string, textPath: string): CacheEntry | null { + if (!isPlainFile(jsonPath) || !isPlainFile(textPath)) return null; + try { + return JSON.parse(readPlainFile(jsonPath, MAX_ENTRY_BYTES)) as CacheEntry; + } catch { + return null; + } +} + async function stemsFor( path: string, dir: string, @@ -130,14 +130,16 @@ const VOICE_FREE = "no-vocals.wav"; /** Stems summed at full quality into one file next to them, made once. */ async function mixedFile(paths: string[], out: string, warnings: string[]) { - if (existsSync(out)) return out; + if (isPlainFile(out)) return out; const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true }); if (!ffmpeg) { warnings.push(`no ${basename(out)}: ffmpeg is required; run: npx hyperframes doctor`); return null; } - const temporary = `${out}.${process.pid}.tmp.wav`; + let stage: string | undefined; try { + stage = createStagingDir(dirname(out), ".hf-mix-"); + const temporary = join(stage, VOICE_FREE); await runFile( ffmpeg, [ @@ -153,6 +155,8 @@ async function mixedFile(paths: string[], out: string, warnings: string[]) { } catch (error) { warnings.push(`no ${basename(out)}: ${error instanceof Error ? error.message : error}`); return null; + } finally { + if (stage) rmSync(stage, { recursive: true, force: true }); } } @@ -461,12 +465,12 @@ export async function readAudio( const transcript = transcriptFor(path, options.transcript); const transcriptKey = fingerprint(transcript); const stem = `${basename(file, extname(file)).replace(/[^\w.-]/g, "_")}-${sha.slice(0, 12)}`; - mkdirSync(options.cacheDir, { recursive: true }); - const jsonPath = join(options.cacheDir, `${stem}.json`); - const textPath = join(options.cacheDir, `${stem}.txt`); - const stemDir = join(options.cacheDir, "stems", sha.slice(0, 12)); - if (existsSync(jsonPath) && existsSync(textPath)) { - const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry; + const cacheDir = makeRealFolder(options.cacheDir, 2); + const jsonPath = join(cacheDir, `${stem}.json`); + const textPath = join(cacheDir, `${stem}.txt`); + const stemDir = join(cacheDir, "stems", sha.slice(0, 12)); + const entry = cachedEntry(jsonPath, textPath); + if (entry) { const stemsNowPossible = !layer && entry.analysis.separation === "mix" && @@ -490,11 +494,11 @@ export async function readAudio( } const analysis = layer ? await analyzeLayer(path, file, sha, transcript, layer) - : await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir); - writeAtomic( + : await analyze(path, file, sha, transcript, stemDir, quality, cacheDir); + replaceFileAtomically( jsonPath, JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry), ); - writeAtomic(textPath, formatRead(analysis)); + replaceFileAtomically(textPath, formatRead(analysis)); return { analysis, textPath, jsonPath, cached: false }; } diff --git a/packages/cli/src/audio/read/stems.ts b/packages/cli/src/audio/read/stems.ts index 868b79edcf..1ce4e739e2 100644 --- a/packages/cli/src/audio/read/stems.ts +++ b/packages/cli/src/audio/read/stems.ts @@ -1,9 +1,10 @@ import { randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { statSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { replaceFileAtomically } from "@hyperframes/core/atomic-file"; import { setTimeout as sleep } from "node:timers/promises"; import { downloadToFile } from "../../cloud/download.js"; import { HyperframesCloudClient } from "../../cloud/_gen/client.js"; @@ -15,6 +16,8 @@ import { type ServiceAccess, } from "../analysis/providers/shared.js"; import { probeAudio } from "../analysis/decode.js"; +import { readPlainFile } from "../../utils/projectRecords.js"; +import { isPlainFile, isPlainName, isRealFolder, makeRealFolder } from "../cacheFolder.js"; /** * Upload 192 kb/s stereo MP3 through the service origin's direct-upload routes, then POST @@ -36,6 +39,7 @@ const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS"; const DEFAULT_TIMEOUT_MS = 30 * 60_000; const RETRY_MS = 5000; const DONE = "stems.json"; +const MAX_DONE_BYTES = 64 * 1024; const STEM_NAMES = ["vocals", "drums", "bass", "guitar", "piano", "other"] as const; export const VOICE_STEM = "vocals"; @@ -49,14 +53,28 @@ export const stemsAccess = () => DEFAULT_TIMEOUT_MS, ); -/** Stems already fetched into `dir`, by name; null when there are none yet. */ +function namedStems(text: string): Record | null { + let names: unknown; + try { + names = JSON.parse(text); + } catch { + return null; + } + if (typeof names !== "object" || names === null) return null; + const files = names as Record; + return STEM_NAMES.every((name) => isPlainName(files[name])) + ? Object.fromEntries(STEM_NAMES.map((name) => [name, files[name] as string])) + : null; +} + +/** Stems already fetched into `dir`, by name; null when there are none yet or a link stands in for any of it. */ export function cachedStems(dir: string): Record | null { const done = join(dir, DONE); - if (!existsSync(done)) return null; - const names = JSON.parse(readFileSync(done, "utf8")) as Record; - if (!STEM_NAMES.every((name) => typeof names[name] === "string")) return null; + if (!isRealFolder(dir, 2) || !isPlainFile(done)) return null; + const names = namedStems(readPlainFile(done, MAX_DONE_BYTES)); + if (!names) return null; const paths = Object.fromEntries(STEM_NAMES.map((name) => [name, join(dir, names[name]!)])); - return Object.values(paths).every((p) => existsSync(p)) ? paths : null; + return Object.values(paths).every(isPlainFile) ? paths : null; } class StillRunning extends Error {} @@ -165,16 +183,16 @@ export async function fetchStems( throw new Error( `the stems service returned an incomplete separation: missing ${missing.join(", ")}`, ); - mkdirSync(dir, { recursive: true }); + makeRealFolder(dir, 2); const names: Record = {}; for (const name of STEM_NAMES) { const stem = stems.find((candidate) => candidate.name === name); if (!stem) continue; const file = `${name}.audio`; - await downloadToFile(stem.url, join(dir, file), { signal }); + await downloadToFile(stem.url, join(dir, file), { signal, replaceLink: true }); names[name] = file; } - writeFileSync(join(dir, DONE), JSON.stringify(names)); + replaceFileAtomically(join(dir, DONE), JSON.stringify(names)); return cachedStems(dir)!; } finally { await rm(scratch, { recursive: true, force: true }); diff --git a/packages/cli/src/cloud/download.test.ts b/packages/cli/src/cloud/download.test.ts index 95daf89479..57daa704cf 100644 --- a/packages/cli/src/cloud/download.test.ts +++ b/packages/cli/src/cloud/download.test.ts @@ -262,6 +262,27 @@ describe("cloud/download", () => { }, ); + it.skipIf(process.platform === "win32")( + "with replaceLink, replaces an output link instead of writing to its target", + async () => { + const target = join(dir, "target.mp4"); + writeFileSync(target, "keep"); + chmodSync(target, 0o755); + const alias = join(dir, "alias.mp4"); + symlinkSync("target.mp4", alias); + await downloadToFile("https://example/x", alias, { + fetchImpl: makeBytesFetch(new Uint8Array([7])), + replaceLink: true, + }); + const downloaded = readFileSync(alias); + const written = lstatSync(alias); + expect(readFileSync(target, "utf8")).toBe("keep"); + expect(written.isFile()).toBe(true); + expect(written.mode & 0o111).toBe(0); + expect(downloaded).toEqual(Buffer.from([7])); + }, + ); + it.skipIf(process.platform === "win32")( "follows a relative output link inside a linked folder to the real target", async () => { diff --git a/packages/cli/src/cloud/download.ts b/packages/cli/src/cloud/download.ts index 7992474a03..3e05b9ce6c 100644 --- a/packages/cli/src/cloud/download.ts +++ b/packages/cli/src/cloud/download.ts @@ -13,7 +13,7 @@ * existing destination is replaced only after the download succeeds. */ -import { chmodSync, renameSync, rmSync, statSync } from "node:fs"; +import { chmodSync, lstatSync, renameSync, rmSync, statSync } from "node:fs"; import { makeOutputFolder } from "../utils/outputFolder.js"; import { open } from "node:fs/promises"; import { dirname, join } from "node:path"; @@ -34,6 +34,8 @@ export interface DownloadOptions { /** Called with (bytes downloaded, total or undefined). */ onProgress?: (bytes: number, total: number | undefined) => void; partBytes?: number; + /** Replace a link at `destPath` with the download instead of writing to the file it points at. */ + replaceLink?: boolean; } export interface DownloadResult { @@ -76,9 +78,10 @@ export async function downloadToFile( makeOutputFolder(dirname(destPath)); - // Writes through a caller-selected symlink, as before. - const destination = resolveWritePath(destPath); - const previous = statSync(destination, { throwIfNoEntry: false }); + const destination = options.replaceLink ? destPath : resolveWritePath(destPath); + const previous = options.replaceLink + ? lstatSync(destination, { throwIfNoEntry: false }) + : statSync(destination, { throwIfNoEntry: false }); const stage = createStagingDir(dirname(destination), ".hf-download-"); const stagedFile = join(stage, "download"); let bytes = 0;