diff --git a/packages/cli/src/commands/media-use.test.ts b/packages/cli/src/commands/media-use.test.ts index 07cacacdeb..0f80ce1d7d 100644 --- a/packages/cli/src/commands/media-use.test.ts +++ b/packages/cli/src/commands/media-use.test.ts @@ -29,6 +29,14 @@ describe("mediaUseEngineEnv", () => { }); }); + // fails without the fix: the engine loaded a key the CLI had unset again from the same .env. + it("names each key the CLI unset from a project .env, so the engine leaves it unset", () => { + const env = mediaUseEngineEnv({}, "", "linux", {}, [["GEMINI_API_KEY", "/work/project"]]); + expect(JSON.parse(env.HYPERFRAMES_PROJECT_ENV_YIELDED ?? "[]")).toEqual([ + ["GEMINI_API_KEY", "/work/project"], + ]); + }); + it("hands nothing over from a TypeScript source entry, which plain node cannot run", () => { expect(mediaUseEngineEnv({}, "src/cli.ts", "linux")).toEqual({}); expect(mediaUseEngineEnv({}, "", "linux")).toEqual({}); diff --git a/packages/cli/src/commands/media-use.ts b/packages/cli/src/commands/media-use.ts index d37d6068e5..16a3c58489 100644 --- a/packages/cli/src/commands/media-use.ts +++ b/packages/cli/src/commands/media-use.ts @@ -5,7 +5,12 @@ import { spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; import { finishCommand } from "../utils/commandResult.js"; import { withMediaAuthEnvironment } from "../auth/media-environment.js"; -import { PROJECT_ENV_ORIGINS, projectEnvOrigins } from "../utils/dotEnv.js"; +import { + PROJECT_ENV_ORIGINS, + PROJECT_ENV_YIELDED, + projectEnvOrigins, + projectEnvYielded, +} from "../utils/dotEnv.js"; import { skipWorkingDirectoryInExeSearch, withOnlyTrustedPathFolders, @@ -85,10 +90,12 @@ export function mediaUseEngineEnv( entry: string | undefined = process.argv[1], platform: NodeJS.Platform = process.platform, origins: Record = projectEnvOrigins(), + yielded: [string, string][] = projectEnvYielded(), ): NodeJS.ProcessEnv { const scoped = withOnlyTrustedPathFolders(env, platform); skipWorkingDirectoryInExeSearch(scoped, platform); if (Object.keys(origins).length > 0) scoped[PROJECT_ENV_ORIGINS] = JSON.stringify(origins); + if (yielded.length > 0) scoped[PROJECT_ENV_YIELDED] = JSON.stringify(yielded); if (!entry || /\.[cm]?ts$/.test(entry)) return scoped; return { ...scoped, HYPERFRAMES_CLI_ENTRY: resolve(entry) }; } diff --git a/packages/cli/src/media-use/lib/received-env.d.mts b/packages/cli/src/media-use/lib/received-env.d.mts new file mode 100644 index 0000000000..a9caeb8abe --- /dev/null +++ b/packages/cli/src/media-use/lib/received-env.d.mts @@ -0,0 +1,5 @@ +export function receivedFileState( + projectDir: string, + path: string, + options?: { atOrAboveOnly?: boolean }, +): "received" | "unchecked" | null; diff --git a/packages/cli/src/media-use/lib/received-env.mjs b/packages/cli/src/media-use/lib/received-env.mjs new file mode 100644 index 0000000000..f761c33a1c --- /dev/null +++ b/packages/cli/src/media-use/lib/received-env.mjs @@ -0,0 +1,486 @@ +// The media-use scripts run without the CLI's packages installed, so they keep this copy of the CLI's received-files +// rule (receivedFiles and receivedRecordStatus in packages/cli/src/utils/privateProjectFile.ts), read-only and for one +// file. receivedEnv.parity.test.ts runs both over the same projects. +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readdirSync, + readFileSync, + readSync, + realpathSync, + statSync, +} from "node:fs"; +import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; + +const RECEIVED_RECORD = join(".hyperframes", "received-files.json"); +const RECEIVED_CONTENT = "received-content.json"; +const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; +const MAX_RECORD_BYTES = 16 * 1024 * 1024; +const SHA256 = /^[0-9a-f]{64}$/; +const WALK_MAX_ENTRIES = 20_000; +const WALK_MAX_MS = 2_000; +const NO_FOLLOW = constants.O_NOFOLLOW ?? 0; +const NON_BLOCK = constants.O_NONBLOCK ?? 0; +const CODE_FILE = new Set([".html", ".htm", ".xhtml", ".css", ".js", ".mjs", ".cjs", ".svg"]); + +function realpath(path) { + try { + return realpathSync.native(path); + } catch (error) { + if (error.code === "EISDIR") return realpathSync(path); + throw error; + } +} + +function walkStart(projectDir) { + try { + return realpath(projectDir); + } catch { + return resolve(projectDir); + } +} + +function realFilePath(filePath) { + try { + return realpath(filePath); + } catch { + const dir = dirname(filePath); + return dir === filePath ? filePath : join(realFilePath(dir), basename(filePath)); + } +} + +const isAtOrUnder = (path, folder) => { + const inside = relative(folder, path); + return ( + inside === "" || (inside !== ".." && !inside.startsWith(`..${sep}`) && !isAbsolute(inside)) + ); +}; + +const isOutside = (fromDir) => + fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir); + +function fileIdentity(path) { + try { + const stat = lstatSync(path, { bigint: true }); + return stat.ino === 0n ? null : `${stat.dev}:${stat.ino}`; + } catch { + return null; + } +} + +function recordKind(dir) { + const path = join(dir, RECEIVED_RECORD); + let found; + try { + found = lstatSync(path, { throwIfNoEntry: false }); + } catch (error) { + return error.code === "ENOTDIR" ? "absent" : "unknown"; + } + if (!found) return "absent"; + try { + return statSync(path).isFile() ? "file" : "other"; + } catch { + return "other"; + } +} + +const strongest = (a, b) => (a === "all" ? a : b); + +function hostEntry(value) { + const { path, received, dev, ino } = value ?? {}; + if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute"); + if (received !== "expected" && received !== "all") throw new Error("unknown state"); + const identity = typeof dev === "string" && typeof ino === "string" && ino !== "0"; + return { path, received, ...(identity ? { dev, ino } : {}) }; +} + +function parseHostStates(text) { + const projects = JSON.parse(text)?.projects; + if (!Array.isArray(projects)) throw new Error("projects is not a list"); + const states = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + for (const value of projects) { + const { path, received, dev, ino } = hostEntry(value); + const key = walkStart(path); + states.byPath.set(key, strongest(states.byPath.get(key), received)); + if (dev !== undefined && ino !== undefined) { + const identity = `${dev}:${ino}`; + states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); + } + } + return states; +} + +function hostStates() { + const file = process.env[RECEIVED_STATE_ENV]; + if (!file) return null; + try { + return parseHostStates(readFileSync(file, "utf-8")); + } catch { + return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + } +} + +function hostState(dir, states) { + if (!states) return undefined; + if (states.unreadable) return "all"; + const named = states.byPath.get(dir); + if (named || states.byIdentity.size === 0) return named; + const identity = fileIdentity(dir); + return identity === null ? undefined : states.byIdentity.get(identity); +} + +function allReceivedAtOrAbove(start, states) { + for (let dir = start; ; dir = dirname(dir)) { + if (hostState(dir, states) === "all") return true; + if (dirname(dir) === dir) return false; + } +} + +function allReceivedAround(start, states) { + if (!states) return false; + if (states.unreadable) return true; + const overlaps = [...states.byPath].some( + ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ); + return overlaps || allReceivedAtOrAbove(start, states); +} + +function readRecord(dir, ownFolder, states) { + const kind = recordKind(dir); + if (kind === "absent" && hostState(dir, states) !== undefined) return { listed: undefined }; + if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; + if (kind !== "file") return { listed: undefined }; + try { + return { listed: JSON.parse(readFileSync(join(dir, RECEIVED_RECORD), "utf-8")).files }; + } catch { + return { listed: undefined }; + } +} + +function recordsAtOrAbove(start, states) { + const records = []; + for (let dir = start; ; ) { + const record = readRecord(dir, dir === start, states); + if (record) + records.push({ + file: join(dir, RECEIVED_RECORD), + root: dir, + projectDirFromRoot: relative(dir, start).split(sep).join("/"), + ...record, + }); + const parent = dirname(dir); + if (parent === dir) return records; + dir = parent; + } +} + +function nestedRecord(start, dir, record) { + const folder = relative(start, dir).split(sep).join("/"); + const listed = Array.isArray(record.listed) + ? record.listed.map((entry) => (folder ? `${folder}/${String(entry)}` : String(entry))) + : undefined; + return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; +} + +// The parsers' projectPathSegments: a segment as a case-insensitive file system or Windows opens it. +const pathSegments = (relPath) => + relPath + .normalize("NFKC") + .toUpperCase() + .toLowerCase() + .replace(/\p{Default_Ignorable_Code_Point}/gu, "") + .split(/[\\/]/) + .map((segment) => (segment.split(":", 1)[0] ?? "").replace(/[. ]+$/, "")); + +// The parsers' isStudioPrivatePath (packages/parsers/src/privatePath.ts), which the CLI's walk uses to skip folders. +const STUDIO_FILES = new Set([ + ".hyperframes/studio-motion.json", + ".hyperframes/studio-manual-edits.json", +]); +const STUDIO_FOLDERS = [".hyperframes/prepared-assets", ".hyperframes/requests"]; +const UNCONVERTED_PREVIEW_DOWNLOADS = ".hyperframes/prepared-assets/downloads"; +const CREDENTIAL_FILES = new Set([ + ".npmrc", + ".yarnrc", + ".yarnrc.yml", + ".pypirc", + "pip.conf", + "pip.ini", + ".netrc", + "_netrc", + ".git-credentials", + ".gitconfig", + ".mcp.json", + "credentials.json", + "serviceaccountkey.json", + "service-account.json", + "service_account.json", +]); +const CREDENTIAL_FOLDERS = new Set([ + ".ssh", + ".aws", + ".gnupg", + ".docker", + ".kube", + ".azure", + ".claude", + ".codex", + ".cursor", + ".agents", + ".gemini", + ".vscode", + ".idea", +]); +const CREDENTIAL_CONFIG_FOLDERS = new Set(["gcloud", "gh"]); +const KEY_FILE = + /^(?:id_(?:rsa|dsa|ecdsa|ed25519).*|.*\.(?:pem|key|p12|pfx|p8|ppk|jks|keystore|kdbx))$/; +const SECRET_STORE_FILE = /\.(?:tfstate(?:\.backup)?|tfvars(?:\.json)?)$/; +const MEDIA_FOLDER = ".media"; +const BUILD_OUTPUT_FOLDERS = new Set([ + ".next", + ".nuxt", + ".output", + ".svelte-kit", + ".vercel", + ".netlify", + ".wrangler", + ".astro", + ".turbo", +]); +const LOADED_ASSET_TYPE = + /.\.(?:png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|heic|heif|mp4|m4v|mov|webm|mkv|mxf|avi|mp3|wav|m4a|aac|ogg|oga|opus|flac|woff2?|ttf|otf|css|m?js|cjs)$/; + +const isVcsOrSecretPath = (relPath) => + pathSegments(relPath).some((segment) => segment === ".git" || segment.startsWith(".env")); + +function isCredentialOrToolStatePath(relPath) { + const segments = pathSegments(relPath); + const name = segments.at(-1) ?? ""; + return ( + CREDENTIAL_FILES.has(name) || + KEY_FILE.test(name) || + SECRET_STORE_FILE.test(name) || + segments.some( + (segment, at) => + CREDENTIAL_FOLDERS.has(segment) || + (segment === ".config" && CREDENTIAL_CONFIG_FOLDERS.has(segments[at + 1] ?? "")), + ) + ); +} + +function isStudioOwnFile(path) { + if ( + path === UNCONVERTED_PREVIEW_DOWNLOADS || + path.startsWith(`${UNCONVERTED_PREVIEW_DOWNLOADS}/`) + ) + return false; + return ( + STUDIO_FILES.has(path) || + STUDIO_FOLDERS.some((folder) => path === folder || path.startsWith(`${folder}/`)) + ); +} + +function isStudioPrivatePath(relPath) { + if (isVcsOrSecretPath(relPath) || isCredentialOrToolStatePath(relPath)) return true; + const segments = pathSegments(relPath); + const inside = !isAbsolute(relPath) && !/^\.\.[. ]*$/.test(relPath.split(/[\\/]/, 1)[0] ?? ""); + const name = segments.at(-1) ?? ""; + const loadedType = !name.startsWith(".") && LOADED_ASSET_TYPE.test(name); + return segments.some((segment, at) => { + const atRoot = segments.slice(0, at).every((before) => before === ""); + if (segment === ".hyperframes") + return !atRoot || !isStudioOwnFile(segments.slice(at).join("/")); + if (BUILD_OUTPUT_FOLDERS.has(segment)) return inside; + const mediaFolder = atRoot && segment === MEDIA_FOLDER; + return inside && segment.startsWith(".") && !mediaFolder && !loadedType; + }); +} + +const GIT_AND_PACKAGE_FOLDERS = new Set([".git", "node_modules"]); + +function mayHoldLoadedFiles(start, folder) { + const rel = relative(start, folder).split(sep).join("/"); + const segments = pathSegments(rel); + if (GIT_AND_PACKAGE_FOLDERS.has(segments.at(-1) ?? "")) return false; + if (!segments.includes(".hyperframes")) return true; + return segments.length === 1 || !isStudioPrivatePath(rel); +} + +function recordsBelow(start, states) { + const deadline = Date.now() + WALK_MAX_MS; + let seen = 0; + const records = []; + const pending = [start]; + for (let dir = pending.pop(); dir !== undefined; dir = pending.pop()) { + let folders; + try { + folders = readdirSync(dir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => join(dir, entry.name)) + .filter((folder) => mayHoldLoadedFiles(start, folder)); + } catch { + return null; + } + seen += folders.length; + if (seen > WALK_MAX_ENTRIES || Date.now() > deadline) return null; + const record = dir === start ? null : readRecord(dir, true, states); + if (record) records.push(nestedRecord(start, dir, record)); + pending.push(...folders); + } + return records; +} + +function receivedRecords(start, atOrAboveOnly) { + const states = hostStates(); + const below = atOrAboveOnly ? [] : recordsBelow(start, states); + if (!below) return null; + const records = [...recordsAtOrAbove(start, states), ...below]; + const all = atOrAboveOnly + ? states !== null && (states.unreadable || allReceivedAtOrAbove(start, states)) + : allReceivedAround(start, states); + if (all) + records.push({ file: process.env[RECEIVED_STATE_ENV] ?? "", root: start, listed: undefined }); + return records; +} + +// A plain file's text, or null for a link, a device or a missing file; `whole` is false past `maxBytes`. +function readPlainTail(path, maxBytes) { + let fd; + try { + fd = openSync(path, constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); + } catch { + return null; + } + try { + const stats = fstatSync(fd); + const named = lstatSync(path); + const same = named.ino === stats.ino && named.dev === stats.dev; + if (!stats.isFile() || !named.isFile() || !same) return null; + const length = Math.min(stats.size, maxBytes); + const buffer = Buffer.alloc(length); + readSync(fd, buffer, 0, length, stats.size - length); + return { text: buffer.toString("utf8"), whole: length === stats.size }; + } catch { + return null; + } finally { + closeSync(fd); + } +} + +function contentRecordKind(dir) { + let path = dir; + for (const [index, part] of [".hyperframes", RECEIVED_CONTENT].entries()) { + path = join(path, part); + try { + const stats = lstatSync(path, { throwIfNoEntry: false }); + if (!stats) return "absent"; + if (!(index === 1 ? stats.isFile() : stats.isDirectory())) return "other"; + } catch { + return "other"; + } + } + return "file"; +} + +// The received content hashes kept beside the record in `dir`; null when that file cannot be read. +function receivedContentHashes(dir) { + let records; + try { + records = lstatSync(join(dir, ".hyperframes")).isDirectory() ? join(dir, ".hyperframes") : null; + } catch { + records = null; + } + const read = records ? readPlainTail(join(records, RECEIVED_CONTENT), MAX_RECORD_BYTES) : null; + if (read === null) return contentRecordKind(dir) === "absent" ? [] : null; + let saved; + try { + saved = read.whole ? JSON.parse(read.text) : null; + } catch { + saved = null; + } + if (saved === null || typeof saved !== "object" || Array.isArray(saved)) return null; + const listed = Array.isArray(saved.hashes) ? saved.hashes : []; + return listed.filter((hash) => typeof hash === "string" && SHA256.test(hash)); +} + +function contentHash(path) { + let fd; + try { + fd = openSync(path, constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); + } catch { + return null; + } + try { + const stat = fstatSync(fd); + const hashable = + stat.isFile() && + (stat.size <= MAX_RECORD_BYTES || CODE_FILE.has(extname(path).toLowerCase())); + return hashable ? createHash("sha256").update(readFileSync(fd)).digest("hex") : null; + } catch { + return null; + } finally { + closeSync(fd); + } +} + +function listedIn(record, projectDir, start, path) { + const listed = record.listed.map((entry) => String(entry).normalize("NFC")); + const key = (record.projectDirFromRoot ? `${record.projectDirFromRoot}/${path}` : path).normalize( + "NFC", + ); + if (listed.includes(key)) return true; + const identity = fileIdentity(join(projectDir, path)); + if (identity === null) return false; + return listed + .map((entry) => join(record.root, entry)) + .some((file) => isAtOrUnder(file, start) && fileIdentity(file) === identity); +} + +function onDisk(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }) !== undefined; + } catch { + return false; + } +} + +// Every received content hash beside the records; null when one of those stores cannot be read. +function receivedHashes(records) { + const hashes = new Set(); + for (const folder of new Set(records.map((record) => dirname(dirname(record.file))))) { + const found = receivedContentHashes(folder); + if (found === null) return null; + for (const hash of found) hashes.add(hash); + } + return hashes; +} + +/** "unchecked" when the project's folders are too many to scan, "received" when the CLI's rule counts + * `/` as received, else null. With `atOrAboveOnly`, for a file in a folder above the one a script + * works in, only records and host state at or above that folder count: never a sibling's or a subfolder's. */ +export function receivedFileState(projectDir, path, { atOrAboveOnly = false } = {}) { + const start = walkStart(projectDir); + const records = receivedRecords(start, atOrAboveOnly); + if (records === null) return "unchecked"; + if (records.some((record) => !Array.isArray(record.listed))) return "received"; + const hashes = receivedHashes(records); + if (hashes === null) return "received"; + if (records.some((record) => listedIn(record, projectDir, start, path))) return "received"; + const file = join(projectDir, path); + if (hashes.size > 0 && hashes.has(contentHash(file) ?? "")) return "received"; + return !atOrAboveOnly && mayBeGoneFile(records, projectDir, start, path) ? "received" : null; +} + +// The CLI's rule for a listed file that is gone: a page, script or a link out of the project may have taken its place. +function mayBeGoneFile(records, projectDir, start, path) { + const listedInProject = records + .flatMap((record) => record.listed.map((entry) => join(record.root, String(entry)))) + .filter((file) => isAtOrUnder(file, start)); + if (!listedInProject.some((listed) => !onDisk(listed))) return false; + if (CODE_FILE.has(extname(path).toLowerCase())) return true; + return isOutside(relative(realFilePath(projectDir), realFilePath(join(projectDir, path)))); +} diff --git a/packages/cli/src/media-use/receivedEnv.parity.test.ts b/packages/cli/src/media-use/receivedEnv.parity.test.ts new file mode 100644 index 0000000000..4b7d0da57f --- /dev/null +++ b/packages/cli/src/media-use/receivedEnv.parity.test.ts @@ -0,0 +1,214 @@ +import { createHash } from "node:crypto"; +import { lstatSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { + RECEIVED_STATE_ENV, + receivedFiles, + receivedRecordStatus, +} from "../utils/privateProjectFile.js"; +import { receivedFileState } from "./lib/received-env.mjs"; + +const ENV_TEXT = "ELEVENLABS_API_KEY=from-env\n"; +const sha256 = (text: string) => createHash("sha256").update(text).digest("hex"); + +type Arrange = (root: string, project: string) => void; + +const record = (folder: string, files: unknown) => { + mkdirSync(join(folder, ".hyperframes"), { recursive: true }); + writeFileSync(join(folder, ".hyperframes", "received-files.json"), JSON.stringify({ files })); +}; + +const hostSays = (root: string, projects: unknown) => { + const file = join(root, "state.json"); + writeFileSync(file, typeof projects === "string" ? projects : JSON.stringify({ projects })); + vi.stubEnv(RECEIVED_STATE_ENV, file); +}; + +const unreadableRecord = (folder: string) => { + mkdirSync(join(folder, ".hyperframes"), { recursive: true }); + writeFileSync(join(folder, ".hyperframes", "received-files.json"), "{oops"); +}; + +const FIXTURES: [string, Arrange, "received" | "unchecked" | null][] = [ + ["a folder with no record", () => {}, null], + ["a record listing the .env", (_root, project) => record(project, [".env"]), "received"], + ["a record listing other files", (_root, project) => record(project, ["index.html"]), null], + ["a record above listing the .env", (root) => record(root, ["app/.env"]), "received"], + [ + "an unreadable record", + (_root, project) => { + mkdirSync(join(project, ".hyperframes")); + writeFileSync(join(project, ".hyperframes", "received-files.json"), "{oops"); + }, + "received", + ], + [ + "an unreadable nested record", + (_root, project) => { + mkdirSync(join(project, "scenes", ".hyperframes"), { recursive: true }); + writeFileSync(join(project, "scenes", ".hyperframes", "received-files.json"), "{oops"); + }, + "received", + ], + [ + "a nested record that lists the .env", + (_root, project) => record(join(project, "scenes"), ["../.env"]), + "received", + ], + [ + "an unreadable record in a Studio folder whose hidden name looks like an asset, which the walk skips", + (_root, project) => unreadableRecord(join(project, ".hyperframes", "requests", ".clip.png")), + null, + ], + [ + "a record in a Studio folder whose hidden name looks like an asset, listing the .env, which the walk skips", + (_root, project) => + record(join(project, ".hyperframes", "requests", ".clip.png"), ["../../../.env"]), + null, + ], + [ + "an unreadable record in a Studio folder named like a key file, which the walk skips", + (_root, project) => unreadableRecord(join(project, ".hyperframes", "requests", "id_rsa")), + null, + ], + [ + "more folders than the scan allows", + (_root, project) => { + for (let i = 0; i <= 20_000; i++) mkdirSync(join(project, `d${i}`)); + }, + "unchecked", + ], + [ + "a .env with the bytes of received content", + (_root, project) => { + record(project, []); + writeFileSync( + join(project, ".hyperframes", "received-content.json"), + JSON.stringify({ hashes: [sha256(ENV_TEXT)] }), + ); + }, + "received", + ], + [ + "received content with other bytes", + (_root, project) => { + record(project, []); + writeFileSync( + join(project, ".hyperframes", "received-content.json"), + JSON.stringify({ hashes: [sha256("other")] }), + ); + }, + null, + ], + [ + "an unreadable received-content store", + (_root, project) => { + record(project, []); + writeFileSync(join(project, ".hyperframes", "received-content.json"), "{oops"); + }, + "received", + ], + [ + "host state all at the folder", + (root, project) => hostSays(root, [{ path: project, received: "all" }]), + "received", + ], + [ + "host state all above the folder", + (root) => hostSays(root, [{ path: root, received: "all" }]), + "received", + ], + [ + "host state all below the folder", + (root, project) => { + mkdirSync(join(project, "refs")); + hostSays(root, [{ path: join(project, "refs"), received: "all" }]); + }, + "received", + ], + [ + "host state expected with its record missing", + (root, project) => hostSays(root, [{ path: project, received: "expected" }]), + "received", + ], + [ + "host state expected with a record that leaves the .env out", + (root, project) => { + record(project, ["index.html"]); + hostSays(root, [{ path: project, received: "expected" }]); + }, + null, + ], + [ + "host state expected for the folder by its identity", + (root, project) => { + const { dev, ino } = lstatSync(project, { bigint: true }); + const elsewhere = join(root, "renamed-away"); + hostSays(root, [ + { path: elsewhere, dev: String(dev), ino: String(ino), received: "expected" }, + ]); + }, + "received", + ], + ["an unreadable host state file", (root) => hostSays(root, "{not json"), "received"], + [ + "a host state file that is missing", + (root) => vi.stubEnv(RECEIVED_STATE_ENV, join(root, "missing.json")), + "received", + ], + [ + "an unknown host state", + (root, project) => hostSays(root, [{ path: project, received: "maybe" }]), + "received", + ], + [ + "host state naming another folder", + (root) => hostSays(root, [{ path: join(root, "other"), received: "all" }]), + null, + ], +]; + +const dirs: string[] = []; + +afterEach(() => { + vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function cliState(project: string): "received" | "unchecked" | null { + if (receivedRecordStatus(project) === "unchecked") return "unchecked"; + return receivedFiles(project)(".env") ? "received" : null; +} + +it.each(FIXTURES)( + "the media-use copy of the received rule reads %s as the CLI does", + (_, arrange, expected) => { + vi.stubEnv(RECEIVED_STATE_ENV, undefined); + const root = mkdtempSync(join(tmpdir(), "hf-received-env-")); + dirs.push(root); + const project = join(root, "app"); + mkdirSync(project); + writeFileSync(join(project, ".env"), ENV_TEXT); + arrange(root, project); + expect(cliState(project)).toBe(expected); + expect(receivedFileState(project, ".env")).toBe(expected); + }, + 30_000, +); + +it.skipIf(process.platform === "win32")( + "the media-use copy reads a .env linked outside the project as the CLI does once a listed file is gone", + () => { + const root = mkdtempSync(join(tmpdir(), "hf-received-env-")); + dirs.push(root); + const project = join(root, "app"); + mkdirSync(project); + writeFileSync(join(root, "outside.env"), ENV_TEXT); + symlinkSync(join(root, "outside.env"), join(project, ".env")); + record(project, ["gone.html"]); + expect(cliState(project)).toBe("received"); + expect(receivedFileState(project, ".env")).toBe("received"); + }, +); diff --git a/packages/cli/src/utils/dotEnv.ts b/packages/cli/src/utils/dotEnv.ts index 9686574462..f9b34b9fd3 100644 --- a/packages/cli/src/utils/dotEnv.ts +++ b/packages/cli/src/utils/dotEnv.ts @@ -223,6 +223,13 @@ export async function noteProjectProviderKey( } export const PROJECT_ENV_ORIGINS = "HYPERFRAMES_PROJECT_ENV_ORIGINS"; +export const PROJECT_ENV_YIELDED = "HYPERFRAMES_PROJECT_ENV_YIELDED"; + +export const projectEnvYielded = (): [string, string][] => + [...yieldedKeys].map((id) => { + const [dir = "", name = ""] = id.split("\0"); + return [name, dir]; + }); /** The folder of each key a project `.env` set, for the scripts the CLI launches to weigh the same way. */ export function projectEnvOrigins(): Record { diff --git a/scripts/check-media-use-copy-parity.test.mjs b/scripts/check-media-use-copy-parity.test.mjs index 3ae0ced458..a154a91c77 100644 --- a/scripts/check-media-use-copy-parity.test.mjs +++ b/scripts/check-media-use-copy-parity.test.mjs @@ -29,6 +29,7 @@ export const MEDIA_USE_COPY_NAMES = [ "npx-sync.mjs", "parakeet-words.mjs", "prefs-store.mjs", + "received-env.mjs", "recipe-store.mjs", "telemetry.mjs", "terminal-text.mjs", diff --git a/scripts/generate-skill-module-copies.mjs b/scripts/generate-skill-module-copies.mjs index 6161be6da8..437b6b3bed 100644 --- a/scripts/generate-skill-module-copies.mjs +++ b/scripts/generate-skill-module-copies.mjs @@ -12,6 +12,10 @@ export const skillModuleCopies = [ "packages/cli/src/media-use/lib/terminal-text.mjs", ["skills/media-use/scripts/lib/terminal-text.mjs"], ], + [ + "packages/cli/src/media-use/lib/received-env.mjs", + ["skills/media-use/scripts/lib/received-env.mjs"], + ], [ "packages/cli/src/media-use/lib/openverse.mjs", ["skills/media-use/audio/scripts/lib/openverse.mjs"], diff --git a/skills-manifest.json b/skills-manifest.json index c98a005674..9263002ddc 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,8 +18,8 @@ "files": 13 }, "media-use": { - "hash": "f4a4eb3dd71ad671", - "files": 129 + "hash": "abbb676657aa5be4", + "files": 130 }, "remotion-to-hyperframes": { "hash": "27f757cb659e1aaa", diff --git a/skills/media-use/audio/scripts/lib/heygen.mjs b/skills/media-use/audio/scripts/lib/heygen.mjs index d487b003c1..fe83dfc99d 100644 --- a/skills/media-use/audio/scripts/lib/heygen.mjs +++ b/skills/media-use/audio/scripts/lib/heygen.mjs @@ -1,4 +1,5 @@ import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs"; +import { receivedFileState } from "../../../scripts/lib/received-env.mjs"; import { terminalText } from "../../../scripts/lib/terminal-text.mjs"; // heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio // pipeline. The credential resolver matches the hyperframes CLI auth: first @@ -12,9 +13,9 @@ import { terminalText } from "../../../scripts/lib/terminal-text.mjs"; // heygen CLI; plain HTTP needs $HEYGEN_ALLOW_HTTP=1, as there. // Vendored so the skill ships standalone. Pure node. -import { lstatSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; -import { dirname, join, relative, resolve, sep } from "node:path"; +import { dirname, join, resolve } from "node:path"; export const HEYGEN_BASE = "https://api.heygen.com/v3"; @@ -111,67 +112,46 @@ function inheritProjectKeyOrigins() { } } -function isFile(path) { +function realFolder(dir) { try { - return statSync(path).isFile(); + return realpathSync(dir); } catch { - return false; + return resolve(dir); } } -function fileIdentity(path) { +// The keys the CLI unset from a folder's .env, as `\0`, so loading that .env again under any +// spelling of the folder leaves them out. +function inheritedYieldedKeys() { try { - const stat = lstatSync(path, { bigint: true }); - return stat.ino === 0n ? null : `${stat.dev}:${stat.ino}`; + const yielded = JSON.parse(process.env.HYPERFRAMES_PROJECT_ENV_YIELDED || "[]"); + if (!Array.isArray(yielded)) return new Set(); + return new Set( + yielded.map((entry) => `${realFolder(String(entry?.[1]))}\0${String(entry?.[0])}`), + ); } catch { - return null; - } -} - -// The CLI's received-files rule: a record at or above `dir` lists its .env (by path or as the same file on disk), or a -// record there cannot be read. → "listed" | "unreadable" | null. -function receivedEnvFile(dir) { - let start; - try { - start = realpathSync(dir); - } catch { - start = resolve(dir); - } - const envIdentity = fileIdentity(join(dir, ".env")); - for (let root = start; ; root = dirname(root)) { - const record = join(root, ".hyperframes", "received-files.json"); - if (root === start || isFile(record)) { - let files; - try { - files = JSON.parse(readFileSync(record, "utf8")).files; - } catch (error) { - if (root !== start || !["ENOENT", "ENOTDIR"].includes(error.code)) return "unreadable"; - files = []; - } - if (!Array.isArray(files)) return "unreadable"; - const key = relative(root, join(start, ".env")).split(sep).join("/").normalize("NFC"); - const listed = files.some((file) => { - const entry = String(file).normalize("NFC"); - return ( - entry === key || (envIdentity !== null && fileIdentity(join(root, entry)) === envIdentity) - ); - }); - if (listed) return "listed"; - } - if (dirname(root) === root) return null; + return new Set(); } } const RECEIVED_ORIGIN = { - listed: "a received project", - unreadable: "a project whose received-files record could not be read", + received: "a received project", + unchecked: "a project whose origin could not be checked (too many folders to scan)", }; +const envFileStates = new Map(); -// How a project key's .env is not the person's own, or null when it is theirs or the key is not from a .env. +// How a project key's .env is not the person's own by the CLI's received-files rule: received, or unchecked when its +// folders are too many to scan. Null when it is theirs or the key is not from a .env. A .env found in a folder above +// the working one is judged only by records and host state at or above that folder. function untrustedOrigin(name) { const dir = projectKeyDir(name); - const kind = dir === undefined ? null : receivedEnvFile(dir); - return kind === null ? null : RECEIVED_ORIGIN[kind]; + if (dir === undefined) return null; + const atOrAboveOnly = projectEnvValues.get(name)?.parent === true; + const id = `${dir}\0${atOrAboveOnly}`; + if (!envFileStates.has(id)) + envFileStates.set(id, receivedFileState(dir, ".env", { atOrAboveOnly })); + const state = envFileStates.get(id); + return state === null ? null : RECEIVED_ORIGIN[state]; } // 0: set outside any project .env; 1: the person's own project .env; 2: a received or unreadable one. @@ -236,10 +216,13 @@ export function loadEnvFromDir(startDir) { } function loadFirstEnvFile(startDir) { - let dir = resolve(startDir); + const yielded = inheritedYieldedKeys(); + const start = resolve(startDir); + let dir = start; for (let i = 0; i < 5; i++) { const text = envFileText(join(dir, ".env")); if (text != null) { + const real = realFolder(dir); for (const raw of text.split("\n")) { let line = raw.trim(); if (!line || line.startsWith("#")) continue; @@ -253,9 +236,10 @@ function loadFirstEnvFile(startDir) { const end = val.indexOf(q, 1); val = end > 0 ? val.slice(1, end) : val.slice(1); } - if (PROJECT_ENV_KEYS.has(key) && !(key in process.env)) { + const skip = key in process.env || yielded.has(`${real}\0${key}`); + if (PROJECT_ENV_KEYS.has(key) && !skip) { process.env[key] = val; - projectEnvValues.set(key, { val, dir }); + projectEnvValues.set(key, { val, dir, parent: dir !== start }); } } return; diff --git a/skills/media-use/audio/scripts/lib/heygen.test.mjs b/skills/media-use/audio/scripts/lib/heygen.test.mjs index 8f9f64c73b..2e31482aae 100644 --- a/skills/media-use/audio/scripts/lib/heygen.test.mjs +++ b/skills/media-use/audio/scripts/lib/heygen.test.mjs @@ -1,4 +1,5 @@ import { test } from "node:test"; +import { createHash } from "node:crypto"; import assert from "node:assert/strict"; import { linkSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { createServer } from "node:http"; @@ -772,3 +773,142 @@ test("a received .env's ElevenLabs key with no competing credential is used, wit }, ); }); + +// fails without the fix: the scripts read only received-files.json, so a folder the host named was the person's own. +test("a .env in a folder the host names as all received gives way like a listed one", () => { + withProviderProject( + { shell: { HEYGEN_API_KEY: "own" }, env: "ELEVENLABS_API_KEY=received\n" }, + (project) => { + const state = join(project, "..", `${project.split(/[\\/]/).at(-1)}-state.json`); + writeFileSync(state, JSON.stringify({ projects: [{ path: project, received: "all" }] })); + process.env.HYPERFRAMES_RECEIVED_STATE_FILE = state; + try { + loadEnvFromDir(project); + assert.equal(process.env.ELEVENLABS_API_KEY, undefined); + } finally { + delete process.env.HYPERFRAMES_RECEIVED_STATE_FILE; + rmSync(state, { force: true }); + } + }, + ); +}); + +// fails without the fix: a .env holding received bytes under a new name was the person's own. +test("a .env with the bytes of received content gives way like a listed one", () => { + const text = "GEMINI_API_KEY=received\n"; + withProviderProject( + { shell: { GOOGLE_APPLICATION_CREDENTIALS: "/own/sa.json" }, env: text }, + (project) => { + mkdirSync(join(project, ".hyperframes")); + writeFileSync(join(project, ".hyperframes", "received-files.json"), '{"files":[]}'); + const hash = createHash("sha256").update(text).digest("hex"); + writeFileSync( + join(project, ".hyperframes", "received-content.json"), + JSON.stringify({ hashes: [hash] }), + ); + loadEnvFromDir(project); + assert.equal(process.env.GEMINI_API_KEY, undefined); + }, + ); +}); + +// fails without the fix: the script loaded the key the CLI had unset again from the same .env. +test("a key the CLI unset from a .env stays unset when the script reads that .env", () => { + withProviderProject({ env: "GEMINI_API_KEY=received\nGOOGLE_API_KEY=mine\n" }, (project) => { + process.env.HYPERFRAMES_PROJECT_ENV_YIELDED = JSON.stringify([["GEMINI_API_KEY", project]]); + try { + loadEnvFromDir(project); + assert.equal(process.env.GEMINI_API_KEY, undefined); + assert.equal(process.env.GOOGLE_API_KEY, "mine"); + } finally { + delete process.env.HYPERFRAMES_PROJECT_ENV_YIELDED; + } + }); +}); + +// fails without the fix: a linked spelling of the folder loaded the key the CLI had unset. +test( + "a key the CLI unset stays unset when the script reaches the folder through a link", + { skip: process.platform === "win32" }, + () => { + withProviderProject({ env: "GEMINI_API_KEY=received\n" }, (project) => { + const link = `${project}-link`; + symlinkSync(project, link); + process.env.HYPERFRAMES_PROJECT_ENV_YIELDED = JSON.stringify([["GEMINI_API_KEY", project]]); + try { + loadEnvFromDir(link); + assert.equal(process.env.GEMINI_API_KEY, undefined); + } finally { + delete process.env.HYPERFRAMES_PROJECT_ENV_YIELDED; + rmSync(link, { force: true }); + } + }); + }, +); + +test("a key the CLI unset from another folder's .env still loads from this one", () => { + withProviderProject({ env: "GEMINI_API_KEY=mine\n" }, (project) => { + const other = join(project, "elsewhere"); + process.env.HYPERFRAMES_PROJECT_ENV_YIELDED = JSON.stringify([["GEMINI_API_KEY", other]]); + try { + loadEnvFromDir(project); + assert.equal(process.env.GEMINI_API_KEY, "mine"); + } finally { + delete process.env.HYPERFRAMES_PROJECT_ENV_YIELDED; + } + }); +}); + +// fails without the fix: a .env above the working folder was judged by every project under it. +test("a .env in a parent folder stays the person's own next to a received sibling project", () => { + withCleanHeygenEnv(() => { + const home = mkdtempSync(join(tmpdir(), "hf-media-home-")); + const previous = process.env.ELEVENLABS_API_KEY; + try { + delete process.env.ELEVENLABS_API_KEY; + process.env.HYPERFRAMES_CONFIG_DIR = join(home, "no-config"); + process.env.HEYGEN_API_KEY = "own"; + writeFileSync(join(home, ".env"), "ELEVENLABS_API_KEY=mine\n"); + mkdirSync(join(home, "shared", ".hyperframes"), { recursive: true }); + writeFileSync(join(home, "shared", ".hyperframes", "received-files.json"), "{oops"); + mkdirSync(join(home, "work")); + const state = join(home, "state.json"); + writeFileSync( + state, + JSON.stringify({ projects: [{ path: join(home, "shared"), received: "all" }] }), + ); + process.env.HYPERFRAMES_RECEIVED_STATE_FILE = state; + loadEnvFromDir(join(home, "work")); + assert.equal(process.env.ELEVENLABS_API_KEY, "mine"); + } finally { + delete process.env.HYPERFRAMES_RECEIVED_STATE_FILE; + if (previous === undefined) delete process.env.ELEVENLABS_API_KEY; + else process.env.ELEVENLABS_API_KEY = previous; + rmSync(home, { recursive: true, force: true }); + } + }); +}); + +test("a .env in a parent folder the host names as received still gives way", () => { + withCleanHeygenEnv(() => { + const home = mkdtempSync(join(tmpdir(), "hf-media-home-")); + const previous = process.env.ELEVENLABS_API_KEY; + try { + delete process.env.ELEVENLABS_API_KEY; + process.env.HYPERFRAMES_CONFIG_DIR = join(home, "no-config"); + process.env.HEYGEN_API_KEY = "own"; + writeFileSync(join(home, ".env"), "ELEVENLABS_API_KEY=received\n"); + mkdirSync(join(home, "work")); + const state = join(home, "state.json"); + writeFileSync(state, JSON.stringify({ projects: [{ path: home, received: "all" }] })); + process.env.HYPERFRAMES_RECEIVED_STATE_FILE = state; + loadEnvFromDir(join(home, "work")); + assert.equal(process.env.ELEVENLABS_API_KEY, undefined); + } finally { + delete process.env.HYPERFRAMES_RECEIVED_STATE_FILE; + if (previous === undefined) delete process.env.ELEVENLABS_API_KEY; + else process.env.ELEVENLABS_API_KEY = previous; + rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/skills/media-use/scripts/lib/received-env.mjs b/skills/media-use/scripts/lib/received-env.mjs new file mode 100644 index 0000000000..f761c33a1c --- /dev/null +++ b/skills/media-use/scripts/lib/received-env.mjs @@ -0,0 +1,486 @@ +// The media-use scripts run without the CLI's packages installed, so they keep this copy of the CLI's received-files +// rule (receivedFiles and receivedRecordStatus in packages/cli/src/utils/privateProjectFile.ts), read-only and for one +// file. receivedEnv.parity.test.ts runs both over the same projects. +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readdirSync, + readFileSync, + readSync, + realpathSync, + statSync, +} from "node:fs"; +import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; + +const RECEIVED_RECORD = join(".hyperframes", "received-files.json"); +const RECEIVED_CONTENT = "received-content.json"; +const RECEIVED_STATE_ENV = "HYPERFRAMES_RECEIVED_STATE_FILE"; +const MAX_RECORD_BYTES = 16 * 1024 * 1024; +const SHA256 = /^[0-9a-f]{64}$/; +const WALK_MAX_ENTRIES = 20_000; +const WALK_MAX_MS = 2_000; +const NO_FOLLOW = constants.O_NOFOLLOW ?? 0; +const NON_BLOCK = constants.O_NONBLOCK ?? 0; +const CODE_FILE = new Set([".html", ".htm", ".xhtml", ".css", ".js", ".mjs", ".cjs", ".svg"]); + +function realpath(path) { + try { + return realpathSync.native(path); + } catch (error) { + if (error.code === "EISDIR") return realpathSync(path); + throw error; + } +} + +function walkStart(projectDir) { + try { + return realpath(projectDir); + } catch { + return resolve(projectDir); + } +} + +function realFilePath(filePath) { + try { + return realpath(filePath); + } catch { + const dir = dirname(filePath); + return dir === filePath ? filePath : join(realFilePath(dir), basename(filePath)); + } +} + +const isAtOrUnder = (path, folder) => { + const inside = relative(folder, path); + return ( + inside === "" || (inside !== ".." && !inside.startsWith(`..${sep}`) && !isAbsolute(inside)) + ); +}; + +const isOutside = (fromDir) => + fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir); + +function fileIdentity(path) { + try { + const stat = lstatSync(path, { bigint: true }); + return stat.ino === 0n ? null : `${stat.dev}:${stat.ino}`; + } catch { + return null; + } +} + +function recordKind(dir) { + const path = join(dir, RECEIVED_RECORD); + let found; + try { + found = lstatSync(path, { throwIfNoEntry: false }); + } catch (error) { + return error.code === "ENOTDIR" ? "absent" : "unknown"; + } + if (!found) return "absent"; + try { + return statSync(path).isFile() ? "file" : "other"; + } catch { + return "other"; + } +} + +const strongest = (a, b) => (a === "all" ? a : b); + +function hostEntry(value) { + const { path, received, dev, ino } = value ?? {}; + if (typeof path !== "string" || !isAbsolute(path)) throw new Error("path is not absolute"); + if (received !== "expected" && received !== "all") throw new Error("unknown state"); + const identity = typeof dev === "string" && typeof ino === "string" && ino !== "0"; + return { path, received, ...(identity ? { dev, ino } : {}) }; +} + +function parseHostStates(text) { + const projects = JSON.parse(text)?.projects; + if (!Array.isArray(projects)) throw new Error("projects is not a list"); + const states = { unreadable: false, byPath: new Map(), byIdentity: new Map() }; + for (const value of projects) { + const { path, received, dev, ino } = hostEntry(value); + const key = walkStart(path); + states.byPath.set(key, strongest(states.byPath.get(key), received)); + if (dev !== undefined && ino !== undefined) { + const identity = `${dev}:${ino}`; + states.byIdentity.set(identity, strongest(states.byIdentity.get(identity), received)); + } + } + return states; +} + +function hostStates() { + const file = process.env[RECEIVED_STATE_ENV]; + if (!file) return null; + try { + return parseHostStates(readFileSync(file, "utf-8")); + } catch { + return { unreadable: true, byPath: new Map(), byIdentity: new Map() }; + } +} + +function hostState(dir, states) { + if (!states) return undefined; + if (states.unreadable) return "all"; + const named = states.byPath.get(dir); + if (named || states.byIdentity.size === 0) return named; + const identity = fileIdentity(dir); + return identity === null ? undefined : states.byIdentity.get(identity); +} + +function allReceivedAtOrAbove(start, states) { + for (let dir = start; ; dir = dirname(dir)) { + if (hostState(dir, states) === "all") return true; + if (dirname(dir) === dir) return false; + } +} + +function allReceivedAround(start, states) { + if (!states) return false; + if (states.unreadable) return true; + const overlaps = [...states.byPath].some( + ([path, state]) => state === "all" && (isAtOrUnder(path, start) || isAtOrUnder(start, path)), + ); + return overlaps || allReceivedAtOrAbove(start, states); +} + +function readRecord(dir, ownFolder, states) { + const kind = recordKind(dir); + if (kind === "absent" && hostState(dir, states) !== undefined) return { listed: undefined }; + if (kind === "absent" || (kind === "unknown" && !ownFolder)) return null; + if (kind !== "file") return { listed: undefined }; + try { + return { listed: JSON.parse(readFileSync(join(dir, RECEIVED_RECORD), "utf-8")).files }; + } catch { + return { listed: undefined }; + } +} + +function recordsAtOrAbove(start, states) { + const records = []; + for (let dir = start; ; ) { + const record = readRecord(dir, dir === start, states); + if (record) + records.push({ + file: join(dir, RECEIVED_RECORD), + root: dir, + projectDirFromRoot: relative(dir, start).split(sep).join("/"), + ...record, + }); + const parent = dirname(dir); + if (parent === dir) return records; + dir = parent; + } +} + +function nestedRecord(start, dir, record) { + const folder = relative(start, dir).split(sep).join("/"); + const listed = Array.isArray(record.listed) + ? record.listed.map((entry) => (folder ? `${folder}/${String(entry)}` : String(entry))) + : undefined; + return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed }; +} + +// The parsers' projectPathSegments: a segment as a case-insensitive file system or Windows opens it. +const pathSegments = (relPath) => + relPath + .normalize("NFKC") + .toUpperCase() + .toLowerCase() + .replace(/\p{Default_Ignorable_Code_Point}/gu, "") + .split(/[\\/]/) + .map((segment) => (segment.split(":", 1)[0] ?? "").replace(/[. ]+$/, "")); + +// The parsers' isStudioPrivatePath (packages/parsers/src/privatePath.ts), which the CLI's walk uses to skip folders. +const STUDIO_FILES = new Set([ + ".hyperframes/studio-motion.json", + ".hyperframes/studio-manual-edits.json", +]); +const STUDIO_FOLDERS = [".hyperframes/prepared-assets", ".hyperframes/requests"]; +const UNCONVERTED_PREVIEW_DOWNLOADS = ".hyperframes/prepared-assets/downloads"; +const CREDENTIAL_FILES = new Set([ + ".npmrc", + ".yarnrc", + ".yarnrc.yml", + ".pypirc", + "pip.conf", + "pip.ini", + ".netrc", + "_netrc", + ".git-credentials", + ".gitconfig", + ".mcp.json", + "credentials.json", + "serviceaccountkey.json", + "service-account.json", + "service_account.json", +]); +const CREDENTIAL_FOLDERS = new Set([ + ".ssh", + ".aws", + ".gnupg", + ".docker", + ".kube", + ".azure", + ".claude", + ".codex", + ".cursor", + ".agents", + ".gemini", + ".vscode", + ".idea", +]); +const CREDENTIAL_CONFIG_FOLDERS = new Set(["gcloud", "gh"]); +const KEY_FILE = + /^(?:id_(?:rsa|dsa|ecdsa|ed25519).*|.*\.(?:pem|key|p12|pfx|p8|ppk|jks|keystore|kdbx))$/; +const SECRET_STORE_FILE = /\.(?:tfstate(?:\.backup)?|tfvars(?:\.json)?)$/; +const MEDIA_FOLDER = ".media"; +const BUILD_OUTPUT_FOLDERS = new Set([ + ".next", + ".nuxt", + ".output", + ".svelte-kit", + ".vercel", + ".netlify", + ".wrangler", + ".astro", + ".turbo", +]); +const LOADED_ASSET_TYPE = + /.\.(?:png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|heic|heif|mp4|m4v|mov|webm|mkv|mxf|avi|mp3|wav|m4a|aac|ogg|oga|opus|flac|woff2?|ttf|otf|css|m?js|cjs)$/; + +const isVcsOrSecretPath = (relPath) => + pathSegments(relPath).some((segment) => segment === ".git" || segment.startsWith(".env")); + +function isCredentialOrToolStatePath(relPath) { + const segments = pathSegments(relPath); + const name = segments.at(-1) ?? ""; + return ( + CREDENTIAL_FILES.has(name) || + KEY_FILE.test(name) || + SECRET_STORE_FILE.test(name) || + segments.some( + (segment, at) => + CREDENTIAL_FOLDERS.has(segment) || + (segment === ".config" && CREDENTIAL_CONFIG_FOLDERS.has(segments[at + 1] ?? "")), + ) + ); +} + +function isStudioOwnFile(path) { + if ( + path === UNCONVERTED_PREVIEW_DOWNLOADS || + path.startsWith(`${UNCONVERTED_PREVIEW_DOWNLOADS}/`) + ) + return false; + return ( + STUDIO_FILES.has(path) || + STUDIO_FOLDERS.some((folder) => path === folder || path.startsWith(`${folder}/`)) + ); +} + +function isStudioPrivatePath(relPath) { + if (isVcsOrSecretPath(relPath) || isCredentialOrToolStatePath(relPath)) return true; + const segments = pathSegments(relPath); + const inside = !isAbsolute(relPath) && !/^\.\.[. ]*$/.test(relPath.split(/[\\/]/, 1)[0] ?? ""); + const name = segments.at(-1) ?? ""; + const loadedType = !name.startsWith(".") && LOADED_ASSET_TYPE.test(name); + return segments.some((segment, at) => { + const atRoot = segments.slice(0, at).every((before) => before === ""); + if (segment === ".hyperframes") + return !atRoot || !isStudioOwnFile(segments.slice(at).join("/")); + if (BUILD_OUTPUT_FOLDERS.has(segment)) return inside; + const mediaFolder = atRoot && segment === MEDIA_FOLDER; + return inside && segment.startsWith(".") && !mediaFolder && !loadedType; + }); +} + +const GIT_AND_PACKAGE_FOLDERS = new Set([".git", "node_modules"]); + +function mayHoldLoadedFiles(start, folder) { + const rel = relative(start, folder).split(sep).join("/"); + const segments = pathSegments(rel); + if (GIT_AND_PACKAGE_FOLDERS.has(segments.at(-1) ?? "")) return false; + if (!segments.includes(".hyperframes")) return true; + return segments.length === 1 || !isStudioPrivatePath(rel); +} + +function recordsBelow(start, states) { + const deadline = Date.now() + WALK_MAX_MS; + let seen = 0; + const records = []; + const pending = [start]; + for (let dir = pending.pop(); dir !== undefined; dir = pending.pop()) { + let folders; + try { + folders = readdirSync(dir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => join(dir, entry.name)) + .filter((folder) => mayHoldLoadedFiles(start, folder)); + } catch { + return null; + } + seen += folders.length; + if (seen > WALK_MAX_ENTRIES || Date.now() > deadline) return null; + const record = dir === start ? null : readRecord(dir, true, states); + if (record) records.push(nestedRecord(start, dir, record)); + pending.push(...folders); + } + return records; +} + +function receivedRecords(start, atOrAboveOnly) { + const states = hostStates(); + const below = atOrAboveOnly ? [] : recordsBelow(start, states); + if (!below) return null; + const records = [...recordsAtOrAbove(start, states), ...below]; + const all = atOrAboveOnly + ? states !== null && (states.unreadable || allReceivedAtOrAbove(start, states)) + : allReceivedAround(start, states); + if (all) + records.push({ file: process.env[RECEIVED_STATE_ENV] ?? "", root: start, listed: undefined }); + return records; +} + +// A plain file's text, or null for a link, a device or a missing file; `whole` is false past `maxBytes`. +function readPlainTail(path, maxBytes) { + let fd; + try { + fd = openSync(path, constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); + } catch { + return null; + } + try { + const stats = fstatSync(fd); + const named = lstatSync(path); + const same = named.ino === stats.ino && named.dev === stats.dev; + if (!stats.isFile() || !named.isFile() || !same) return null; + const length = Math.min(stats.size, maxBytes); + const buffer = Buffer.alloc(length); + readSync(fd, buffer, 0, length, stats.size - length); + return { text: buffer.toString("utf8"), whole: length === stats.size }; + } catch { + return null; + } finally { + closeSync(fd); + } +} + +function contentRecordKind(dir) { + let path = dir; + for (const [index, part] of [".hyperframes", RECEIVED_CONTENT].entries()) { + path = join(path, part); + try { + const stats = lstatSync(path, { throwIfNoEntry: false }); + if (!stats) return "absent"; + if (!(index === 1 ? stats.isFile() : stats.isDirectory())) return "other"; + } catch { + return "other"; + } + } + return "file"; +} + +// The received content hashes kept beside the record in `dir`; null when that file cannot be read. +function receivedContentHashes(dir) { + let records; + try { + records = lstatSync(join(dir, ".hyperframes")).isDirectory() ? join(dir, ".hyperframes") : null; + } catch { + records = null; + } + const read = records ? readPlainTail(join(records, RECEIVED_CONTENT), MAX_RECORD_BYTES) : null; + if (read === null) return contentRecordKind(dir) === "absent" ? [] : null; + let saved; + try { + saved = read.whole ? JSON.parse(read.text) : null; + } catch { + saved = null; + } + if (saved === null || typeof saved !== "object" || Array.isArray(saved)) return null; + const listed = Array.isArray(saved.hashes) ? saved.hashes : []; + return listed.filter((hash) => typeof hash === "string" && SHA256.test(hash)); +} + +function contentHash(path) { + let fd; + try { + fd = openSync(path, constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); + } catch { + return null; + } + try { + const stat = fstatSync(fd); + const hashable = + stat.isFile() && + (stat.size <= MAX_RECORD_BYTES || CODE_FILE.has(extname(path).toLowerCase())); + return hashable ? createHash("sha256").update(readFileSync(fd)).digest("hex") : null; + } catch { + return null; + } finally { + closeSync(fd); + } +} + +function listedIn(record, projectDir, start, path) { + const listed = record.listed.map((entry) => String(entry).normalize("NFC")); + const key = (record.projectDirFromRoot ? `${record.projectDirFromRoot}/${path}` : path).normalize( + "NFC", + ); + if (listed.includes(key)) return true; + const identity = fileIdentity(join(projectDir, path)); + if (identity === null) return false; + return listed + .map((entry) => join(record.root, entry)) + .some((file) => isAtOrUnder(file, start) && fileIdentity(file) === identity); +} + +function onDisk(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }) !== undefined; + } catch { + return false; + } +} + +// Every received content hash beside the records; null when one of those stores cannot be read. +function receivedHashes(records) { + const hashes = new Set(); + for (const folder of new Set(records.map((record) => dirname(dirname(record.file))))) { + const found = receivedContentHashes(folder); + if (found === null) return null; + for (const hash of found) hashes.add(hash); + } + return hashes; +} + +/** "unchecked" when the project's folders are too many to scan, "received" when the CLI's rule counts + * `/` as received, else null. With `atOrAboveOnly`, for a file in a folder above the one a script + * works in, only records and host state at or above that folder count: never a sibling's or a subfolder's. */ +export function receivedFileState(projectDir, path, { atOrAboveOnly = false } = {}) { + const start = walkStart(projectDir); + const records = receivedRecords(start, atOrAboveOnly); + if (records === null) return "unchecked"; + if (records.some((record) => !Array.isArray(record.listed))) return "received"; + const hashes = receivedHashes(records); + if (hashes === null) return "received"; + if (records.some((record) => listedIn(record, projectDir, start, path))) return "received"; + const file = join(projectDir, path); + if (hashes.size > 0 && hashes.has(contentHash(file) ?? "")) return "received"; + return !atOrAboveOnly && mayBeGoneFile(records, projectDir, start, path) ? "received" : null; +} + +// The CLI's rule for a listed file that is gone: a page, script or a link out of the project may have taken its place. +function mayBeGoneFile(records, projectDir, start, path) { + const listedInProject = records + .flatMap((record) => record.listed.map((entry) => join(record.root, String(entry)))) + .filter((file) => isAtOrUnder(file, start)); + if (!listedInProject.some((listed) => !onDisk(listed))) return false; + if (CODE_FILE.has(extname(path).toLowerCase())) return true; + return isOutside(relative(realFilePath(projectDir), realFilePath(join(projectDir, path)))); +}