mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): the audio caches never write or read through links in .hyperframes (#7825)
* fix(cli): the audio caches never write or read through links in .hyperframes The read-audio cache, its stems, mix and descriptions, and the audio-kind cache now use only real folders under the project's .hyperframes. A link or a file in place of a cache folder stops the write with a clear message. Every cache file is written to an exclusive temp file with a random name and renamed into place, so a link at a temp or final name is replaced, never written through. ffmpeg writes the voice-free mix into a private staging folder. Stem downloads use the new downloadToFile replaceLink option. Cached files are read only when they are plain files, and a stem list may only name files in its own folder. * fix(cli): the kind cache reads no entry through a linked cache folder readKindCache now checks that .hyperframes and audio-kind are real folders before it reads an entry, as writeKindCache already does before it writes one. * test(cli): read the replaced download before checking its type and mode The replaceLink test now reads the file's bytes and lstat result once each, so no read follows a check on the same path.
This commit is contained in:
@@ -1,4 +1,13 @@
|
||||
import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
@@ -41,4 +50,51 @@ describe("kind cache", () => {
|
||||
writeFileSync(blocked, "file");
|
||||
expect(writeKindCache(join(blocked, "entry.json"), kind)).toMatch(/could not be cached/);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"never writes through a link at the entry or in place of its folders, and reads no linked entry",
|
||||
() => {
|
||||
const project = join(dir, "project");
|
||||
const outside = join(dir, "outside");
|
||||
const cache = join(project, ".hyperframes", "audio-kind");
|
||||
mkdirSync(cache, { recursive: true });
|
||||
mkdirSync(outside);
|
||||
const victim = join(outside, "victim.json");
|
||||
writeFileSync(victim, JSON.stringify(kind));
|
||||
const entry = join(cache, "entry.json");
|
||||
symlinkSync(victim, entry);
|
||||
expect(readKindCache(entry)).toBeNull();
|
||||
expect(writeKindCache(entry, { ...kind, value: "voice" })).toBeNull();
|
||||
expect(JSON.parse(readFileSync(victim, "utf8")).value).toBe("music");
|
||||
expect(lstatSync(entry).isFile()).toBe(true);
|
||||
|
||||
rmSync(cache, { recursive: true });
|
||||
symlinkSync(outside, cache);
|
||||
expect(writeKindCache(join(cache, "other.json"), kind)).toMatch(/could not be cached/);
|
||||
expect(readdirSync(outside)).toEqual(["victim.json"]);
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"reads no entry through a link in place of .hyperframes or audio-kind",
|
||||
() => {
|
||||
const outside = join(dir, "linked-outside");
|
||||
mkdirSync(join(outside, "audio-kind"), { recursive: true });
|
||||
writeFileSync(join(outside, "entry.json"), JSON.stringify(kind));
|
||||
writeFileSync(join(outside, "audio-kind", "entry.json"), JSON.stringify(kind));
|
||||
expect(readKindCache(join(outside, "entry.json"))).not.toBeNull();
|
||||
|
||||
const linkedKind = join(dir, "linked-kind", ".hyperframes");
|
||||
mkdirSync(linkedKind, { recursive: true });
|
||||
symlinkSync(outside, join(linkedKind, "audio-kind"));
|
||||
expect(readKindCache(join(linkedKind, "audio-kind", "entry.json"))).toBeNull();
|
||||
|
||||
const linkedRecords = join(dir, "linked-records");
|
||||
mkdirSync(linkedRecords);
|
||||
symlinkSync(outside, join(linkedRecords, ".hyperframes"));
|
||||
expect(
|
||||
readKindCache(join(linkedRecords, ".hyperframes", "audio-kind", "entry.json")),
|
||||
).toBeNull();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { statSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { readPlainFile } from "../../utils/projectRecords.js";
|
||||
import { isPlainFile, isRealFolder, makeRealFolder } from "../cacheFolder.js";
|
||||
import type { KindFeatures, KindGuess } from "@hyperframes/core/audio-analysis";
|
||||
import type { AnalyzeOptions, Analysis } from "./analyze.js";
|
||||
import type { AudioTarget } from "./targets.js";
|
||||
|
||||
const CACHE_VERSION = 1;
|
||||
const MAX_ENTRY_BYTES = 64 * 1024;
|
||||
|
||||
const record = (value: unknown): value is Record<string, unknown> =>
|
||||
value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
@@ -68,7 +72,10 @@ export function readKindCache(file: string | null): Analysis | null {
|
||||
if (file === null) return null;
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(readFileSync(file, "utf8"));
|
||||
value =
|
||||
isRealFolder(dirname(file), 2) && isPlainFile(file)
|
||||
? JSON.parse(readPlainFile(file, MAX_ENTRY_BYTES))
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@@ -82,21 +89,14 @@ export function readKindCache(file: string | null): Analysis | null {
|
||||
};
|
||||
}
|
||||
|
||||
/** Saves a guess in `<project>/.hyperframes/audio-kind`, both real folders; a link there is replaced, not followed. */
|
||||
export function writeKindCache(file: string, value: KindGuess): string | null {
|
||||
const temporary = `${file}.${randomUUID()}.tmp`;
|
||||
try {
|
||||
mkdirSync(dirname(file), { recursive: true });
|
||||
writeFileSync(temporary, JSON.stringify(value));
|
||||
renameSync(temporary, file);
|
||||
makeRealFolder(dirname(file), 2);
|
||||
replaceFileAtomically(file, JSON.stringify(value));
|
||||
return null;
|
||||
} catch {
|
||||
return "kind could not be cached; another query will analyze the clip again";
|
||||
} finally {
|
||||
try {
|
||||
rmSync(temporary, { force: true });
|
||||
} catch {
|
||||
// A failed cache cleanup must not discard measured audio.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { lstatSync, mkdirSync } from "node:fs";
|
||||
import { basename, dirname } from "node:path";
|
||||
|
||||
// The audio caches live in the project's .hyperframes/. A project can hold links someone else planted there, so
|
||||
// a cache folder, and every file in it, is used only when it is real: a link in its place is never followed.
|
||||
|
||||
function foldersOf(path: string, depth: number): string[] {
|
||||
const folders: string[] = [];
|
||||
for (let folder = path; folders.length < depth; folder = dirname(folder)) folders.unshift(folder);
|
||||
return folders;
|
||||
}
|
||||
|
||||
const isFolder = (path: string) =>
|
||||
lstatSync(path, { throwIfNoEntry: false })?.isDirectory() === true;
|
||||
|
||||
export function isRealFolder(path: string, depth: number): boolean {
|
||||
return foldersOf(path, depth).every(isFolder);
|
||||
}
|
||||
|
||||
/** Makes the last `depth` folders of `path` where missing. Throws when a link or a file holds one's name. */
|
||||
export function makeRealFolder(path: string, depth: number): string {
|
||||
for (const folder of foldersOf(path, depth)) {
|
||||
try {
|
||||
mkdirSync(folder);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
||||
}
|
||||
if (!isFolder(folder))
|
||||
throw new Error(
|
||||
`${basename(folder)} in ${dirname(folder)} is a link or a file, not a folder; remove it and run again`,
|
||||
);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
export const isPlainFile = (path: string) =>
|
||||
lstatSync(path, { throwIfNoEntry: false })?.isFile() === true;
|
||||
|
||||
export const isPlainName = (name: unknown): name is string =>
|
||||
typeof name === "string" &&
|
||||
name !== "." &&
|
||||
name !== ".." &&
|
||||
!/[/\\]/.test(name) &&
|
||||
basename(name) === name;
|
||||
@@ -1,4 +1,11 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
@@ -302,6 +309,39 @@ describe("describeStems", () => {
|
||||
expect(warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"neither reads nor writes a cached answer through a link",
|
||||
async () => {
|
||||
vi.stubEnv("HEYGEN_API_KEY", "signed-in");
|
||||
const fetch = mockMessages();
|
||||
const warnings: string[] = [];
|
||||
await describeAudio(warnings);
|
||||
const outside = mkdtempSync(join(tmpdir(), "hf-describe-outside-"));
|
||||
try {
|
||||
const secret = join(outside, "secret.txt");
|
||||
writeFileSync(secret, "outside words");
|
||||
const answers = join(dir, "describe");
|
||||
for (const name of readdirSync(answers)) {
|
||||
rmSync(join(answers, name));
|
||||
symlinkSync(secret, join(answers, name));
|
||||
}
|
||||
const again = await describeAudio(warnings);
|
||||
expect(fetch).toHaveBeenCalledTimes(6);
|
||||
expect(JSON.stringify(again)).not.toContain("outside words");
|
||||
expect(readFileSync(secret, "utf8")).toBe("outside words");
|
||||
|
||||
rmSync(answers, { recursive: true });
|
||||
symlinkSync(outside, answers);
|
||||
const linked: string[] = [];
|
||||
expect((await describeAudio(linked)).description).toBeNull();
|
||||
expect(linked).toEqual([expect.stringMatching(/is a link or a file, not a folder/)]);
|
||||
expect(readdirSync(outside)).toEqual(["secret.txt"]);
|
||||
} finally {
|
||||
rmSync(outside, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("does not describe silent stems", async () => {
|
||||
const fetch = vi.spyOn(globalThis, "fetch");
|
||||
const warnings: string[] = [];
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Span } from "@hyperframes/core/audio-analysis";
|
||||
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
|
||||
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { readPlainFile } from "../../utils/projectRecords.js";
|
||||
import { isPlainFile, makeRealFolder } from "../cacheFolder.js";
|
||||
import {
|
||||
runFile,
|
||||
serviceAccess,
|
||||
@@ -155,6 +158,8 @@ async function generate(
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_ANSWER_BYTES = 1024 * 1024;
|
||||
|
||||
interface Ask {
|
||||
path: string | null;
|
||||
prompt: string;
|
||||
@@ -162,8 +167,7 @@ interface Ask {
|
||||
}
|
||||
|
||||
async function answers(asks: Ask[], cacheDir: string, access: ServiceAccess) {
|
||||
const dir = join(cacheDir, "describe");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const dir = makeRealFolder(join(cacheDir, "describe"), 1);
|
||||
const scratch = await mkdtemp(join(tmpdir(), "hf-describe-"));
|
||||
try {
|
||||
// Wait for every conversion before removing their shared scratch directory.
|
||||
@@ -176,11 +180,10 @@ async function answers(asks: Ask[], cacheDir: string, access: ServiceAccess) {
|
||||
.update(structured ? JSON.stringify(OVERALL_SCHEMA) : "plain");
|
||||
const id = (audio ? hash.update(audio) : hash).digest("hex");
|
||||
const file = join(dir, `${id.slice(0, 24)}.txt`);
|
||||
if (existsSync(file)) return readFileSync(file, "utf8");
|
||||
if (isPlainFile(file)) return readPlainFile(file, MAX_ANSWER_BYTES);
|
||||
const text = await generate(access, audio, prompt, structured);
|
||||
if (structured) descriptionOf(text);
|
||||
writeFileSync(`${file}.tmp`, text);
|
||||
renameSync(`${file}.tmp`, file);
|
||||
replaceFileAtomically(file, text);
|
||||
return text;
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -1,6 +1,16 @@
|
||||
import { createServer, type IncomingMessage, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -552,3 +562,127 @@ describe.skipIf(!ffmpeg)("readAudio with a declared layer", () => {
|
||||
expect("layer" in analysis).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// Links a project can hold in its .hyperframes/: the reader runs outside any sandbox, so none may be followed.
|
||||
describe.skipIf(!ffmpeg || process.platform === "win32")("readAudio cache links", () => {
|
||||
function project(name: string) {
|
||||
const cacheDir = join(dir, name, ".hyperframes", "read-audio");
|
||||
const outside = join(dir, `${name}-outside`);
|
||||
mkdirSync(cacheDir, { recursive: true });
|
||||
mkdirSync(outside);
|
||||
const victim = join(outside, "victim.txt");
|
||||
writeFileSync(victim, "keep");
|
||||
return { cacheDir, outside, victim };
|
||||
}
|
||||
const shaOf = (path: string) => createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
const isPlain = (path: string) => lstatSync(path).isFile();
|
||||
|
||||
it("never writes a cache entry through a link at its temp or final name", async () => {
|
||||
const { cacheDir, victim } = project("entry-links");
|
||||
const path = wav("entry-links.wav", 2, voiced);
|
||||
const entry = join(cacheDir, `entry-links-${shaOf(path).slice(0, 12)}`);
|
||||
for (const ext of ["json", "txt"]) symlinkSync(victim, `${entry}.${ext}.${process.pid}.tmp`);
|
||||
await readAudio(path, "entry-links.wav", { cacheDir });
|
||||
expect(readFileSync(victim, "utf8")).toBe("keep");
|
||||
|
||||
rmSync(`${entry}.json`);
|
||||
rmSync(`${entry}.txt`);
|
||||
symlinkSync(victim, `${entry}.json`);
|
||||
symlinkSync(victim, `${entry}.txt`);
|
||||
const again = await readAudio(path, "entry-links.wav", { cacheDir });
|
||||
expect(again.cached).toBe(false);
|
||||
expect(readFileSync(victim, "utf8")).toBe("keep");
|
||||
expect(isPlain(`${entry}.json`) && isPlain(`${entry}.txt`)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([".hyperframes", join(".hyperframes", "read-audio")])(
|
||||
"refuses a link in place of %s and writes nothing outside the project",
|
||||
async (folder) => {
|
||||
const name = `folder-link-${folder.length}`;
|
||||
const { cacheDir, outside } = project(name);
|
||||
const linked = join(dir, name, folder);
|
||||
rmSync(linked, { recursive: true });
|
||||
symlinkSync(outside, linked);
|
||||
const path = wav(`${name}.wav`, 2, voiced);
|
||||
await expect(readAudio(path, `${name}.wav`, { cacheDir })).rejects.toThrow(
|
||||
/is a link or a file, not a folder/,
|
||||
);
|
||||
expect(readdirSync(outside)).toEqual(["victim.txt"]);
|
||||
},
|
||||
);
|
||||
|
||||
it("saves stems and their list in place of links, never through them", async () => {
|
||||
for (const name of Object.keys(stemFiles)) delete stemFiles[name];
|
||||
const silence = wav("stem-links-silence.wav", 1, () => 0);
|
||||
for (const name of ["vocals", "drums", "bass", "guitar", "piano", "other"])
|
||||
stemFiles[name] = silence;
|
||||
const { cacheDir, outside, victim } = project("stem-links");
|
||||
const stemDir = join(cacheDir, "stems", "0123456789ab");
|
||||
mkdirSync(stemDir, { recursive: true });
|
||||
symlinkSync(victim, join(stemDir, "vocals.audio"));
|
||||
symlinkSync(join(outside, "planted.json"), join(stemDir, "stems.json"));
|
||||
const access = {
|
||||
url: `${origin()}/v3/audio/stems`,
|
||||
headers: { "x-api-key": "test-key" },
|
||||
timeoutMs: 60_000,
|
||||
};
|
||||
const stems = await fetchStems(wav("stem-links.wav", 1, voiced), stemDir, access);
|
||||
expect(readFileSync(victim, "utf8")).toBe("keep");
|
||||
expect(existsSync(join(outside, "planted.json"))).toBe(false);
|
||||
expect(isPlain(stems.vocals!) && isPlain(join(stemDir, "stems.json"))).toBe(true);
|
||||
});
|
||||
|
||||
it("reads no stems through a linked folder or a name that leaves it, and refuses to fetch into a link", async () => {
|
||||
const { cacheDir, outside } = project("stem-folder-link");
|
||||
const silence = wav("stem-folder-silence.wav", 1, () => 0);
|
||||
const names = ["vocals", "drums", "bass", "guitar", "piano", "other"];
|
||||
writeFileSync(join(outside, "stem.wav"), readFileSync(silence));
|
||||
writeFileSync(
|
||||
join(outside, "stems.json"),
|
||||
JSON.stringify(Object.fromEntries(names.map((n) => [n, "stem.wav"]))),
|
||||
);
|
||||
mkdirSync(join(cacheDir, "stems"));
|
||||
symlinkSync(outside, join(cacheDir, "stems", "0123456789ab"));
|
||||
const linkedDir = join(cacheDir, "stems", "0123456789ab");
|
||||
expect(cachedStems(linkedDir)).toBeNull();
|
||||
|
||||
const realDir = join(cacheDir, "stems", "ba9876543210");
|
||||
mkdirSync(realDir);
|
||||
writeFileSync(
|
||||
join(realDir, "stems.json"),
|
||||
JSON.stringify(
|
||||
Object.fromEntries(names.map((n) => [n, "../../../../stem-folder-link-outside/stem.wav"])),
|
||||
),
|
||||
);
|
||||
expect(cachedStems(realDir)).toBeNull();
|
||||
|
||||
for (const name of names) stemFiles[name] = silence;
|
||||
await expect(
|
||||
fetchStems(wav("stem-folder.wav", 1, voiced), linkedDir, {
|
||||
url: `${origin()}/v3/audio/stems`,
|
||||
headers: { "x-api-key": "test-key" },
|
||||
timeoutMs: 60_000,
|
||||
}),
|
||||
).rejects.toThrow(/is a link or a file, not a folder/);
|
||||
expect(readdirSync(outside).sort()).toEqual(["stem.wav", "stems.json", "victim.txt"]);
|
||||
});
|
||||
|
||||
it("mixes the voice-free stems into a staged file, never through a link at a temp name", async () => {
|
||||
const { cacheDir, victim } = project("mix-links");
|
||||
const path = wav("mix-links.wav", 2, (t) => voiced(t) + bed(t));
|
||||
const stemDir = join(cacheDir, "stems", shaOf(path).slice(0, 12));
|
||||
mkdirSync(stemDir, { recursive: true });
|
||||
writeFileSync(join(stemDir, "vocals.wav"), readFileSync(wav("mix-vocals.wav", 2, voiced)));
|
||||
writeFileSync(join(stemDir, "bed.wav"), readFileSync(wav("mix-bed.wav", 2, bed)));
|
||||
const names = { vocals: "vocals.wav", drums: "bed.wav", bass: "bed.wav" };
|
||||
writeFileSync(
|
||||
join(stemDir, "stems.json"),
|
||||
JSON.stringify({ ...names, guitar: "bed.wav", piano: "bed.wav", other: "bed.wav" }),
|
||||
);
|
||||
symlinkSync(victim, join(stemDir, `no-vocals.wav.${process.pid}.tmp.wav`));
|
||||
const { analysis } = await readAudio(path, "mix-links.wav", { cacheDir, quality: "fast" });
|
||||
expect(analysis.voiceFreeMix).toMatch(/no-vocals\.wav$/);
|
||||
expect(readFileSync(victim, "utf8")).toBe("keep");
|
||||
expect(isPlain(join(stemDir, "no-vocals.wav"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,17 +1,13 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
createReadStream,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { createReadStream, existsSync, readFileSync, renameSync, rmSync } from "node:fs";
|
||||
import { basename, dirname, extname, join, relative } from "node:path";
|
||||
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
|
||||
import { createStagingDir, replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { loudness, type AnalysisQuality, type Span } from "@hyperframes/core/audio-analysis";
|
||||
import { loadTranscript } from "../../whisper/normalize.js";
|
||||
import { readPlainFile } from "../../utils/projectRecords.js";
|
||||
import { runFile } from "../analysis/providers/shared.js";
|
||||
import { isPlainFile, makeRealFolder } from "../cacheFolder.js";
|
||||
import { describeAccess, describeStems } from "./describe.js";
|
||||
import { findEffects, type EffectSource } from "./effects.js";
|
||||
import { formatRead } from "./format.js";
|
||||
@@ -74,13 +70,8 @@ function wordsOf(transcript: string): TimedWord[] {
|
||||
const fingerprint = (path: string | null) =>
|
||||
path ? createHash("sha256").update(readFileSync(path)).digest("hex").slice(0, 16) : "none";
|
||||
|
||||
function writeAtomic(path: string, content: string) {
|
||||
const temporary = `${path}.${process.pid}.tmp`;
|
||||
writeFileSync(temporary, content);
|
||||
renameSync(temporary, path);
|
||||
}
|
||||
|
||||
const CACHE_REVISION = 1;
|
||||
const MAX_ENTRY_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
interface CacheEntry {
|
||||
revision: number;
|
||||
@@ -88,6 +79,15 @@ interface CacheEntry {
|
||||
transcriptKey: string;
|
||||
}
|
||||
|
||||
function cachedEntry(jsonPath: string, textPath: string): CacheEntry | null {
|
||||
if (!isPlainFile(jsonPath) || !isPlainFile(textPath)) return null;
|
||||
try {
|
||||
return JSON.parse(readPlainFile(jsonPath, MAX_ENTRY_BYTES)) as CacheEntry;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function stemsFor(
|
||||
path: string,
|
||||
dir: string,
|
||||
@@ -130,14 +130,16 @@ const VOICE_FREE = "no-vocals.wav";
|
||||
|
||||
/** Stems summed at full quality into one file next to them, made once. */
|
||||
async function mixedFile(paths: string[], out: string, warnings: string[]) {
|
||||
if (existsSync(out)) return out;
|
||||
if (isPlainFile(out)) return out;
|
||||
const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true });
|
||||
if (!ffmpeg) {
|
||||
warnings.push(`no ${basename(out)}: ffmpeg is required; run: npx hyperframes doctor`);
|
||||
return null;
|
||||
}
|
||||
const temporary = `${out}.${process.pid}.tmp.wav`;
|
||||
let stage: string | undefined;
|
||||
try {
|
||||
stage = createStagingDir(dirname(out), ".hf-mix-");
|
||||
const temporary = join(stage, VOICE_FREE);
|
||||
await runFile(
|
||||
ffmpeg,
|
||||
[
|
||||
@@ -153,6 +155,8 @@ async function mixedFile(paths: string[], out: string, warnings: string[]) {
|
||||
} catch (error) {
|
||||
warnings.push(`no ${basename(out)}: ${error instanceof Error ? error.message : error}`);
|
||||
return null;
|
||||
} finally {
|
||||
if (stage) rmSync(stage, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -461,12 +465,12 @@ export async function readAudio(
|
||||
const transcript = transcriptFor(path, options.transcript);
|
||||
const transcriptKey = fingerprint(transcript);
|
||||
const stem = `${basename(file, extname(file)).replace(/[^\w.-]/g, "_")}-${sha.slice(0, 12)}`;
|
||||
mkdirSync(options.cacheDir, { recursive: true });
|
||||
const jsonPath = join(options.cacheDir, `${stem}.json`);
|
||||
const textPath = join(options.cacheDir, `${stem}.txt`);
|
||||
const stemDir = join(options.cacheDir, "stems", sha.slice(0, 12));
|
||||
if (existsSync(jsonPath) && existsSync(textPath)) {
|
||||
const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry;
|
||||
const cacheDir = makeRealFolder(options.cacheDir, 2);
|
||||
const jsonPath = join(cacheDir, `${stem}.json`);
|
||||
const textPath = join(cacheDir, `${stem}.txt`);
|
||||
const stemDir = join(cacheDir, "stems", sha.slice(0, 12));
|
||||
const entry = cachedEntry(jsonPath, textPath);
|
||||
if (entry) {
|
||||
const stemsNowPossible =
|
||||
!layer &&
|
||||
entry.analysis.separation === "mix" &&
|
||||
@@ -490,11 +494,11 @@ export async function readAudio(
|
||||
}
|
||||
const analysis = layer
|
||||
? await analyzeLayer(path, file, sha, transcript, layer)
|
||||
: await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
|
||||
writeAtomic(
|
||||
: await analyze(path, file, sha, transcript, stemDir, quality, cacheDir);
|
||||
replaceFileAtomically(
|
||||
jsonPath,
|
||||
JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry),
|
||||
);
|
||||
writeAtomic(textPath, formatRead(analysis));
|
||||
replaceFileAtomically(textPath, formatRead(analysis));
|
||||
return { analysis, textPath, jsonPath, cached: false };
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { statSync } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
|
||||
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { downloadToFile } from "../../cloud/download.js";
|
||||
import { HyperframesCloudClient } from "../../cloud/_gen/client.js";
|
||||
@@ -15,6 +16,8 @@ import {
|
||||
type ServiceAccess,
|
||||
} from "../analysis/providers/shared.js";
|
||||
import { probeAudio } from "../analysis/decode.js";
|
||||
import { readPlainFile } from "../../utils/projectRecords.js";
|
||||
import { isPlainFile, isPlainName, isRealFolder, makeRealFolder } from "../cacheFolder.js";
|
||||
|
||||
/**
|
||||
* Upload 192 kb/s stereo MP3 through the service origin's direct-upload routes, then POST
|
||||
@@ -36,6 +39,7 @@ const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_STEMS_TIMEOUT_MS";
|
||||
const DEFAULT_TIMEOUT_MS = 30 * 60_000;
|
||||
const RETRY_MS = 5000;
|
||||
const DONE = "stems.json";
|
||||
const MAX_DONE_BYTES = 64 * 1024;
|
||||
const STEM_NAMES = ["vocals", "drums", "bass", "guitar", "piano", "other"] as const;
|
||||
|
||||
export const VOICE_STEM = "vocals";
|
||||
@@ -49,14 +53,28 @@ export const stemsAccess = () =>
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
/** Stems already fetched into `dir`, by name; null when there are none yet. */
|
||||
function namedStems(text: string): Record<string, string> | null {
|
||||
let names: unknown;
|
||||
try {
|
||||
names = JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof names !== "object" || names === null) return null;
|
||||
const files = names as Record<string, unknown>;
|
||||
return STEM_NAMES.every((name) => isPlainName(files[name]))
|
||||
? Object.fromEntries(STEM_NAMES.map((name) => [name, files[name] as string]))
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Stems already fetched into `dir`, by name; null when there are none yet or a link stands in for any of it. */
|
||||
export function cachedStems(dir: string): Record<string, string> | null {
|
||||
const done = join(dir, DONE);
|
||||
if (!existsSync(done)) return null;
|
||||
const names = JSON.parse(readFileSync(done, "utf8")) as Record<string, string>;
|
||||
if (!STEM_NAMES.every((name) => typeof names[name] === "string")) return null;
|
||||
if (!isRealFolder(dir, 2) || !isPlainFile(done)) return null;
|
||||
const names = namedStems(readPlainFile(done, MAX_DONE_BYTES));
|
||||
if (!names) return null;
|
||||
const paths = Object.fromEntries(STEM_NAMES.map((name) => [name, join(dir, names[name]!)]));
|
||||
return Object.values(paths).every((p) => existsSync(p)) ? paths : null;
|
||||
return Object.values(paths).every(isPlainFile) ? paths : null;
|
||||
}
|
||||
|
||||
class StillRunning extends Error {}
|
||||
@@ -165,16 +183,16 @@ export async function fetchStems(
|
||||
throw new Error(
|
||||
`the stems service returned an incomplete separation: missing ${missing.join(", ")}`,
|
||||
);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
makeRealFolder(dir, 2);
|
||||
const names: Record<string, string> = {};
|
||||
for (const name of STEM_NAMES) {
|
||||
const stem = stems.find((candidate) => candidate.name === name);
|
||||
if (!stem) continue;
|
||||
const file = `${name}.audio`;
|
||||
await downloadToFile(stem.url, join(dir, file), { signal });
|
||||
await downloadToFile(stem.url, join(dir, file), { signal, replaceLink: true });
|
||||
names[name] = file;
|
||||
}
|
||||
writeFileSync(join(dir, DONE), JSON.stringify(names));
|
||||
replaceFileAtomically(join(dir, DONE), JSON.stringify(names));
|
||||
return cachedStems(dir)!;
|
||||
} finally {
|
||||
await rm(scratch, { recursive: true, force: true });
|
||||
|
||||
@@ -262,6 +262,27 @@ describe("cloud/download", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"with replaceLink, replaces an output link instead of writing to its target",
|
||||
async () => {
|
||||
const target = join(dir, "target.mp4");
|
||||
writeFileSync(target, "keep");
|
||||
chmodSync(target, 0o755);
|
||||
const alias = join(dir, "alias.mp4");
|
||||
symlinkSync("target.mp4", alias);
|
||||
await downloadToFile("https://example/x", alias, {
|
||||
fetchImpl: makeBytesFetch(new Uint8Array([7])),
|
||||
replaceLink: true,
|
||||
});
|
||||
const downloaded = readFileSync(alias);
|
||||
const written = lstatSync(alias);
|
||||
expect(readFileSync(target, "utf8")).toBe("keep");
|
||||
expect(written.isFile()).toBe(true);
|
||||
expect(written.mode & 0o111).toBe(0);
|
||||
expect(downloaded).toEqual(Buffer.from([7]));
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"follows a relative output link inside a linked folder to the real target",
|
||||
async () => {
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
* existing destination is replaced only after the download succeeds.
|
||||
*/
|
||||
|
||||
import { chmodSync, renameSync, rmSync, statSync } from "node:fs";
|
||||
import { chmodSync, lstatSync, renameSync, rmSync, statSync } from "node:fs";
|
||||
import { makeOutputFolder } from "../utils/outputFolder.js";
|
||||
import { open } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
@@ -34,6 +34,8 @@ export interface DownloadOptions {
|
||||
/** Called with (bytes downloaded, total or undefined). */
|
||||
onProgress?: (bytes: number, total: number | undefined) => void;
|
||||
partBytes?: number;
|
||||
/** Replace a link at `destPath` with the download instead of writing to the file it points at. */
|
||||
replaceLink?: boolean;
|
||||
}
|
||||
|
||||
export interface DownloadResult {
|
||||
@@ -76,9 +78,10 @@ export async function downloadToFile(
|
||||
|
||||
makeOutputFolder(dirname(destPath));
|
||||
|
||||
// Writes through a caller-selected symlink, as before.
|
||||
const destination = resolveWritePath(destPath);
|
||||
const previous = statSync(destination, { throwIfNoEntry: false });
|
||||
const destination = options.replaceLink ? destPath : resolveWritePath(destPath);
|
||||
const previous = options.replaceLink
|
||||
? lstatSync(destination, { throwIfNoEntry: false })
|
||||
: statSync(destination, { throwIfNoEntry: false });
|
||||
const stage = createStagingDir(dirname(destination), ".hf-download-");
|
||||
const stagedFile = join(stage, "download");
|
||||
let bytes = 0;
|
||||
|
||||
Reference in New Issue
Block a user