diff --git a/docs/developers/share-source.mdx b/docs/developers/share-source.mdx index 0b41a593a6..48b557f129 100644 --- a/docs/developers/share-source.mdx +++ b/docs/developers/share-source.mdx @@ -56,9 +56,10 @@ These rules can only exclude files, never re-include credentials or other files blocked by the portable project policy. Cloud render reads the same file as gitignore syntax, so it may hold rules this -reader rejects, such as `/exports/` or `**/*.mov`. Source capture then stops with -an error. Change detection keeps working and ignores the file's rules, so those -files still count as changes. +reader rejects, such as `/exports/` or `**/*.mov`. Source capture, and anything +else that picks project files to upload, then stops with an error. Change +detection keeps working and ignores the file's rules, so those files still count +as changes. The upload uses the workspace's file-count, per-file, project-size and manifest limits. Large media uploads in parts. Only a verified published snapshot that diff --git a/packages/share/src/projectSnapshot.mjs b/packages/share/src/projectSnapshot.mjs index 09976350a8..8a9d898e45 100644 --- a/packages/share/src/projectSnapshot.mjs +++ b/packages/share/src/projectSnapshot.mjs @@ -1,7 +1,7 @@ // @ts-check // A project's files as change detection sees them: every file the film can use (isOutsideFilm), keyed by mtime and // size, except html, keyed by content without the data-hf-id stamps. A run's change set and the watcher both read it. -import { readChangeIgnore } from "./sourceIgnore.mjs"; +import { readChangeIgnore, readSourceIgnore } from "./sourceIgnore.mjs"; import fs from "node:fs/promises"; import { constants } from "node:fs"; import path from "node:path"; @@ -57,11 +57,18 @@ async function entryKind(entry, full, followSymlinks) { // What a run is judged against: each file's mtime and size, except html, whose key ignores the data-hf-id stamps an // open or a save writes. Files the film cannot use are not part of the snapshot. /** @typedef {Map} Snapshot */ -/** @param {string} dir @param {{ byContent?: boolean, followSymlinks?: boolean }} [options] @returns {Promise} */ -export async function snapshotProject(dir, { byContent = false, followSymlinks = false } = {}) { +// A `.hyperframesignore` this package can't read makes the snapshot throw, so a caller that picks files to send never +// sends one the file meant to exclude. Change detection passes `lenientIgnore`: it then sees more files, never fewer. +/** @typedef {{ lenientIgnore?: boolean }} IgnoreOptions */ +/** @param {string} dir @param {{ byContent?: boolean, followSymlinks?: boolean } & IgnoreOptions} [options] + * @returns {Promise} */ +export async function snapshotProject( + dir, + { byContent = false, followSymlinks = false, lenientIgnore = false } = {}, +) { /** @type {Snapshot} */ const result = new Map(); - const ignored = await readChangeIgnore(dir); + const ignored = await ignoreFor(dir, lenientIgnore); /** @param {string} current @param {Set} ancestors */ async function walk(current, ancestors) { const real = await fs.realpath(current).catch(() => null); @@ -98,11 +105,11 @@ export function snapshotFingerprint(snapshot) { /** Just these project files, keyed as snapshotProject keys them (undefined: not a file the snapshot holds); null when * one is a folder, which only a whole snapshot covers. @param {string} dir @param {Iterable} files - * @returns {Promise | null>} */ -export async function snapshotFiles(dir, files) { + * @param {IgnoreOptions} [options] @returns {Promise | null>} */ +export async function snapshotFiles(dir, files, { lenientIgnore = false } = {}) { /** @type {Map} */ const result = new Map(); - const ignored = await readChangeIgnore(dir); + const ignored = await ignoreFor(dir, lenientIgnore); for (const file of files) { const full = path.join(dir, file); const stat = await fs.lstat(full).catch(() => null); @@ -125,9 +132,15 @@ export function diffChangedFiles(before, after) { return changed; } -/** @param {string} projectPath @param {{ byContent?: boolean }} [options] */ +/** Change detection only: a fingerprint names no files to send, so it reads the ignore file leniently. + * @param {string} projectPath @param {{ byContent?: boolean }} [options] */ export const projectFingerprint = async (projectPath, options) => - snapshotFingerprint(await snapshotProject(projectPath, { ...options, followSymlinks: true })); + snapshotFingerprint( + await snapshotProject(projectPath, { ...options, followSymlinks: true, lenientIgnore: true }), + ); + +/** @param {string} dir @param {boolean} lenient */ +const ignoreFor = (dir, lenient) => (lenient ? readChangeIgnore(dir) : readSourceIgnore(dir)); /** @param {string} dir @param {string} full @param {(path: string) => boolean} ignored */ function excludedFromSnapshot(dir, full, ignored) { diff --git a/packages/share/src/sourceIgnore.mjs b/packages/share/src/sourceIgnore.mjs index 82953e6943..f30daff5a2 100644 --- a/packages/share/src/sourceIgnore.mjs +++ b/packages/share/src/sourceIgnore.mjs @@ -65,8 +65,9 @@ export async function readSourceIgnore(root) { /** The exclusions change detection applies. Change detection must keep working whatever the file holds: the CLI * reads `.hyperframesignore` as gitignore syntax (`/renders/`, `**`, `!keep`), which this reader refuses. An - * unreadable file then excludes only scratch files, so a scan sees more files, never fewer. Source capture keeps - * the strict reader and still refuses the file. @param {string} root @returns {Promise<(path: string) => boolean>} */ + * unreadable file then excludes only scratch files, so a scan sees more files, never fewer. Never use it to pick + * files to send: it drops the file's valid exclusions too. @param {string} root + * @returns {Promise<(path: string) => boolean>} */ export async function readChangeIgnore(root) { return readSourceIgnore(root).catch(() => scratch); } diff --git a/packages/share/tests/kernel.test.mjs b/packages/share/tests/kernel.test.mjs index dceaaee6ac..367ad086a1 100644 --- a/packages/share/tests/kernel.test.mjs +++ b/packages/share/tests/kernel.test.mjs @@ -649,6 +649,7 @@ test("a video-only retry never replays an unanswered source-bearing export", asy ); assert.equal(creates, 1); }); +const lenient = { lenientIgnore: true }; test("change detection keeps working when .hyperframesignore uses gitignore syntax", async (t) => { for (const rule of ["/renders/", "**/*.mov", "!keep.mp4", "assets/*/raw-*.mp4", "a[bc].png"]) { const dir = await folder(t); @@ -657,11 +658,11 @@ test("change detection keeps working when .hyperframesignore uses gitignore synt await writeFile(join(dir, "assets/clip.mov"), "clip"); await writeFile(join(dir, ".hyperframesignore"), `${rule}\n`); // The CLI's syntax is not this reader's: nothing is excluded beyond scratch, so a change still counts. - const files = await snapshotProject(dir); + const files = await snapshotProject(dir, lenient); assert.deepEqual([...files.keys()].sort(), ["assets/clip.mov", "index.html"], rule); // snapshotFiles names every file asked for; an excluded one is keyed undefined. assert.notEqual( - (await snapshotFiles(dir, ["assets/clip.mov"])).get("assets/clip.mov"), + (await snapshotFiles(dir, ["assets/clip.mov"], lenient)).get("assets/clip.mov"), undefined, rule, ); @@ -674,7 +675,24 @@ test("change detection keeps working when .hyperframesignore is a link", async ( const dir = await folder(t); await writeFile(join(dir, "index.html"), "
Hello
"); await symlink(join(dir, "index.html"), join(dir, ".hyperframesignore")); - assert.deepEqual([...(await snapshotProject(dir)).keys()], ["index.html"]); + assert.deepEqual([...(await snapshotProject(dir, lenient)).keys()], ["index.html"]); + await projectFingerprint(dir); +}); +test("a snapshot that picks files to send refuses a .hyperframesignore it can't read", async (t) => { + const dir = await folder(t); + await writeFile(join(dir, "index.html"), "
Hello
"); + await writeFile(join(dir, "private-notes.txt"), "private"); + // One valid exclusion and one rule this reader refuses: the whole file is unreadable here. + await writeFile(join(dir, ".hyperframesignore"), "private-notes.txt\n/renders/\n"); + await assert.rejects(snapshotProject(dir), { code: "invalid_source_ignore" }); + await assert.rejects(snapshotFiles(dir, ["private-notes.txt"]), { + code: "invalid_source_ignore", + }); + // Change detection still runs, and sees the file the valid line meant to exclude. + assert.equal((await snapshotProject(dir, lenient)).has("private-notes.txt"), true); + await rm(join(dir, ".hyperframesignore")); + await symlink(join(dir, "index.html"), join(dir, ".hyperframesignore")); + await assert.rejects(snapshotProject(dir), { code: "invalid_source_ignore" }); }); test("change detection still applies a .hyperframesignore it can read", async (t) => { const dir = await folder(t); @@ -683,5 +701,6 @@ test("change detection still applies a .hyperframesignore it can read", async (t await writeFile(join(dir, "assets/raw.mov"), "raw"); await writeFile(join(dir, "debug.log"), "log"); await writeFile(join(dir, ".hyperframesignore"), "assets/raw.mov\n"); - assert.deepEqual([...(await snapshotProject(dir)).keys()], ["index.html"]); + for (const options of [undefined, lenient]) + assert.deepEqual([...(await snapshotProject(dir, options)).keys()], ["index.html"]); });