diff --git a/packages/producer/src/regression-harness-lambda-local.ts b/packages/producer/src/regression-harness-lambda-local.ts index c971aebece..b218013d66 100644 --- a/packages/producer/src/regression-harness-lambda-local.ts +++ b/packages/producer/src/regression-harness-lambda-local.ts @@ -79,7 +79,11 @@ export async function runLambdaLocalRender( const projectKey = `sites/harness/${Date.now()}/project.tar.gz`; const projectS3Path = join(s3Root, projectKey); mkdirSync(dirname(projectS3Path), { recursive: true }); - await tarDirectory(input.projectDir, projectS3Path, listSiteFiles(input.projectDir)); + await tarDirectory( + input.projectDir, + projectS3Path, + listSiteFiles(input.projectDir, undefined, "include"), + ); const fakeS3 = new FilesystemBackedFakeS3(s3Root); const deps: HandlerDeps = { diff --git a/packages/producer/src/services/distributed/projectHash.test.ts b/packages/producer/src/services/distributed/projectHash.test.ts index e40fa72704..afbc170fb1 100644 --- a/packages/producer/src/services/distributed/projectHash.test.ts +++ b/packages/producer/src/services/distributed/projectHash.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "bun:test"; +import { afterEach, describe, expect, it, spyOn } from "bun:test"; import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, rmSync, symlinkSync, truncateSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -29,7 +29,7 @@ describe("listSiteFiles / hashProjectDir", () => { expect(hashProjectDir(link)).toBe(hashProjectDir(real)); }); - it("lists symlinked files and folders, and hashes their targets' bytes", () => { + it("lists symlinked files and folders when asked to include them, and hashes their targets' bytes", () => { const dir = project(); const outside = tempDir(); mkdirSync(join(outside, "media")); @@ -37,11 +37,38 @@ describe("listSiteFiles / hashProjectDir", () => { writeFileSync(join(outside, "media", "clip.txt"), "v1"); symlinkSync(join(outside, "shared.css"), join(dir, "shared.css")); symlinkSync(join(outside, "media"), join(dir, "media")); - expect(listSiteFiles(dir)).toEqual(["index.html", "media/clip.txt", "shared.css"]); + const list = () => listSiteFiles(dir, undefined, "include"); + expect(list()).toEqual(["index.html", "media/clip.txt", "shared.css"]); - const before = hashProjectDir(dir); + const before = hashProjectDir(dir, list()); writeFileSync(join(outside, "media", "clip.txt"), "v2"); - expect(hashProjectDir(dir)).not.toBe(before); + expect(hashProjectDir(dir, list())).not.toBe(before); + }); + + // fails without the fix: a bare listSiteFiles followed links out of the project. + it("leaves out files and folders linked in from outside the project unless asked to include them", () => { + const dir = project(); + const outside = tempDir(); + mkdirSync(join(outside, "media")); + writeFileSync(join(outside, "shared.css"), "a{}"); + writeFileSync(join(outside, "media", "clip.txt"), "v1"); + symlinkSync(join(outside, "shared.css"), join(dir, "shared.css")); + symlinkSync(join(outside, "media"), join(dir, "media")); + expect(listSiteFiles(dir)).toEqual(["index.html"]); + expect(hashProjectDir(dir)).toBe(hashProjectDir(project())); + }); + + it("treats a sibling folder whose name starts with the project's as outside it", () => { + const parent = tempDir(); + const dir = join(parent, "proj"); + mkdirSync(dir); + writeFileSync(join(dir, "index.html"), "hi"); + mkdirSync(join(parent, "proj-other")); + writeFileSync(join(parent, "proj-other", "a.css"), "a{}"); + symlinkSync(join(parent, "proj-other", "a.css"), join(dir, "a.css")); + const outside: string[] = []; + expect(listSiteFiles(dir, undefined, (path) => outside.push(path))).toEqual(["index.html"]); + expect(outside).toEqual(["a.css"]); }); it("skips top-level skip-list folders", () => { @@ -149,6 +176,22 @@ describe("listSiteFiles / hashProjectDir", () => { expect(notices[0]).toContain("--include-outside-files"); }); + // fails without the fix: an SDK caller with no onNotice lost these files without a word. + it("warns on the console about files an upload left out when the caller passes no onNotice", () => { + const dir = project(); + const outside = tempDir(); + writeFileSync(join(outside, "shared.css"), "a{}"); + symlinkSync(join(outside, "shared.css"), join(dir, "shared.css")); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + expect(listSiteFilesForUpload(dir)).toEqual(["index.html"]); + expect(warn).toHaveBeenCalledTimes(1); + expect(String(warn.mock.calls[0]?.[0])).toContain("outside the project: shared.css."); + } finally { + warn.mockRestore(); + } + }); + it("uploads files linked in from outside the project when the caller includes them", () => { const dir = project(); const outside = tempDir(); diff --git a/packages/producer/src/services/distributed/projectHash.ts b/packages/producer/src/services/distributed/projectHash.ts index f8a2af9d38..464ffb794e 100644 --- a/packages/producer/src/services/distributed/projectHash.ts +++ b/packages/producer/src/services/distributed/projectHash.ts @@ -32,16 +32,15 @@ function statEntry(full: string, relPath: string) { /** * The files a site upload ships, as `/`-separated paths under `projectDir`. Symlinks are followed - * (root included), top-level {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} are left out as the plan stage - * does, and a dangling symlink throws. Private files (env, Git, credentials, tool and agent state, - * app records) are left out at every depth, by name or by where a link points; `onPrivate` hears - * each one left out, a folder once. With `onOutside`, a file or folder whose link points outside - * the project is left out too, and `onOutside` hears it, a folder once. + * (root included), out of the project only with `onOutside` `"include"`; top-level + * {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} are left out as the plan stage does, and a dangling link + * throws. Private files (env, Git, credentials, tool and agent state, app records) are left out at + * every depth, by name or by link target. `onPrivate` and `onOutside` hear each, a folder once. */ export function listSiteFiles( projectDir: string, onPrivate?: (relPath: string) => void, - onOutside?: (relPath: string) => void, + onOutside: ((relPath: string) => void) | "include" = () => {}, ): string[] { const files: string[] = []; const root = realpathSync(projectDir); @@ -50,7 +49,9 @@ export function listSiteFiles( if (isPrivateFileUnder(projectDir, full, isStudioPrivatePath)) return void onPrivate?.(relPath); const stat = statEntry(full, relPath); if (!stat.isFile() && !stat.isDirectory()) return; - if (onOutside && outsideFolder(root, realpathSync(full))) return void onOutside(relPath); + if (onOutside !== "include" && outsideFolder(root, realpathSync(full))) { + return void onOutside(relPath); + } if (stat.isFile()) return void files.push(relPath); const real = realpathSync(full); // A folder linking back to an ancestor is skipped rather than walked forever. @@ -83,12 +84,12 @@ const printablePath = (path: string): string => /** * {@link listSiteFiles} for an upload: one line to `onNotice` names the private files left out. - * Files linked in from outside the project ship only with `includeOutsideFiles`; otherwise a second - * line names each one, since the person may not know a link put them in the project. + * Files linked in from outside ship only with `includeOutsideFiles`, else a second line names them. + * The upload reads each path again after this check, so a link swapped in meanwhile still ships. */ export function listSiteFilesForUpload( projectDir: string, - onNotice?: (message: string) => void, + onNotice: (message: string) => void = (message) => console.warn(message), includeOutsideFiles = false, ): string[] { const leftOut: string[] = []; @@ -96,16 +97,16 @@ export function listSiteFilesForUpload( const files = listSiteFiles( projectDir, (path) => leftOut.push(path), - includeOutsideFiles ? undefined : (path) => outside.push(path), + includeOutsideFiles ? "include" : (path) => outside.push(path), ); if (leftOut.length > 0) { const more = leftOut.length > 5 ? ` and ${leftOut.length - 5} more` : ""; - onNotice?.( + onNotice( `[deploySite] left out private files no composition loads (env, Git, credentials, tool state): ${leftOut.slice(0, 5).map(printablePath).join(", ")}${more}`, ); } if (outside.length > 0) { - onNotice?.( + onNotice( `[deploySite] left out files linked in from outside the project: ${outside.map(printablePath).join(", ")}. Pass --include-outside-files (includeOutsideFiles in the SDK) to upload them.`, ); }