diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 9e2a87ad34..8007370d87 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2660,6 +2660,9 @@ match. Those rules use gitignore syntax and apply to `hyperframes publish` too, the project lint behind `lint`, `check`, `snapshot` and `render` does not read compositions they match. Keep them narrow — a dynamically selected asset may have no static reference to protect it. +Env files (`.env`, `.env.*`, `.envrc`, at any depth) and Studio's private records +under `.hyperframes/` are always left out, even when a rule re-includes them, and +each one is reported as left out with the reason `private`. Media files of 5 MB or more that the project names by path go up as HeyGen assets before zipping, and the zip's copy of each HTML, CSS, script and JSON @@ -2693,7 +2696,7 @@ imported project are left out, with a warning naming each. To include such a file, move it into the project folder. Under `--json`, the result lists those as `left_out`, each with the `file` that points at it, the `reference` as written, its `path` from the project folder, the `reason` (`blocklisted`, -`received` or `file_type`) and `levels_up`, how many folders above the project +`received`, `file_type` or `private`) and `levels_up`, how many folders above the project it is. A full run also prints each as a `{"event":"left_out",…}` line on stderr, so a wrapper can ask with `--dry-run --json` before uploading and still hear about each one during the run. diff --git a/packages/cli/src/utils/publishProject.privateFiles.test.ts b/packages/cli/src/utils/publishProject.privateFiles.test.ts new file mode 100644 index 0000000000..d1145cd460 --- /dev/null +++ b/packages/cli/src/utils/publishProject.privateFiles.test.ts @@ -0,0 +1,104 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import AdmZip from "adm-zip"; +import type { LeftOutReference } from "./leftOutReference.js"; +import { buildPublishFileMap, createPublishArchive, stagePublishFiles } from "./publishProject.js"; + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "hf-publish-private-")); + writeFileSync(join(dir, "index.html"), "", "utf-8"); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +function write(path: string, content = "x"): void { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + writeFileSync(join(dir, path), content, "utf-8"); +} + +function stage(): { keys: string[]; leftOut: LeftOutReference[]; warnings: string[] } { + const leftOut: LeftOutReference[] = []; + const warnings: string[] = []; + const staged = stagePublishFiles(dir, Infinity, (warning, reference) => { + warnings.push(warning); + leftOut.push(reference); + }); + return { keys: [...staged.keys()].sort(), leftOut, warnings }; +} + +const SECRETS = [".env", ".env.local", ".env.production", "assets/.env", ".envrc"]; + +describe("upload archives leave out env files and private data whatever .hyperframesignore says", () => { + it.each([ + ["re-includes .env files", "!.env\n!.env.*\n!.envrc\n!**/.env\n"], + ["re-includes every dotfile", "!.*\n"], + ["re-includes every path", "!*\n!**/*\n"], + ])("when it %s", (_, rules) => { + for (const path of SECRETS) write(path, "HEYGEN_API_KEY=secret"); + write(".hyperframesignore", rules); + + const { keys, leftOut, warnings } = stage(); + + for (const path of SECRETS) expect(keys).not.toContain(path); + expect(leftOut.map((one) => one.file).sort()).toEqual([...SECRETS].sort()); + expect(new Set(leftOut.map((one) => one.reason))).toEqual(new Set(["private"])); + expect(warnings.every((warning) => warning.endsWith("it was left out of the upload."))).toBe( + true, + ); + }); + + it("leaves Git data out when a hidden folder is re-included recursively", () => { + write(".git/config", "[remote]"); + write(".media/.git/HEAD", "ref"); + write(".media/.env", "KEY=1"); + write(".media/logo.png", "logo"); + write(".hyperframesignore", "!/.git/\n!/.git/**\n!/.media/\n!/.media/**\n"); + + const { keys, leftOut } = stage(); + + expect(keys).toContain(".media/logo.png"); + expect(keys.filter((key) => key.includes(".git") || key.endsWith(".env"))).toEqual([]); + expect(leftOut.map((one) => one.file)).toEqual([".media/.env"]); + }); + + it("leaves Studio's private records out but keeps its shared ones", () => { + write(".hyperframes/received-files.json", "{}"); + write(".hyperframesignore", "!/.hyperframes/\n!/.hyperframes/**\n"); + + const { keys, leftOut } = stage(); + + expect(keys).not.toContain(".hyperframes/received-files.json"); + expect(leftOut.map((one) => one.file)).toEqual([".hyperframes/received-files.json"]); + }); + + it("applies to the publish map and the feedback repro archive too", () => { + write(".env", "HEYGEN_API_KEY=secret"); + write(".hyperframesignore", "!.*\n"); + + const publishLeftOut: string[] = []; + const publishMap = buildPublishFileMap(dir, (_, one) => publishLeftOut.push(one.file)); + expect([...publishMap.keys()]).not.toContain(".env"); + expect(publishLeftOut).toEqual([".env"]); + + const reproLeftOut: string[] = []; + const repro = createPublishArchive(dir, (_, one) => reproLeftOut.push(one.file)); + const entries = new AdmZip(repro.buffer).getEntries().map((entry) => entry.entryName); + expect(entries).not.toContain(".env"); + expect(reproLeftOut).toEqual([".env"]); + }); + + it("says nothing when the ignore rules already leave the env file out", () => { + write(".env", "HEYGEN_API_KEY=secret"); + + const { keys, leftOut } = stage(); + + expect(keys).toEqual(["index.html"]); + expect(leftOut).toEqual([]); + }); +}); diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index 5d7d164b40..79d595de35 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -760,6 +760,26 @@ function composeNamesInText(fileContents: Map, filePaths: st } } +type WalkedLeftOutReason = Exclude; + +const WALKED_LEFT_OUT: Record = { + private: "holds secrets or tool data (an env file, Git data or Studio records)", + blocklisted: "links to a file outside the project in a private location", + received: "links to a file outside the project and came in with an imported project", +}; + +function walkedFileLeftOut( + absProjectDir: string, + filePath: string, + absPath: string, + received: (path: string) => boolean, +): WalkedLeftOutReason | null { + if (isPrivateCompositionFile(absProjectDir, absPath)) return "private"; + if (linksToPrivateOutsideFile(absProjectDir, absPath)) return "blocklisted"; + if (received(filePath) && linksOutside(absProjectDir, absPath)) return "received"; + return null; +} + /** {@link buildPublishFileMap}, but non-text files of `onDiskMinBytes` or more stay on disk. */ export function stagePublishFiles( projectDir: string, @@ -780,16 +800,10 @@ export function stagePublishFiles( const received = receivedFiles(absProjectDir); for (const filePath of filePaths) { const absPath = join(absProjectDir, filePath); - const reason = linksToPrivateOutsideFile(absProjectDir, absPath) - ? "blocklisted" - : received(filePath) && linksOutside(absProjectDir, absPath) - ? "received" - : null; + const reason = walkedFileLeftOut(absProjectDir, filePath, absPath, received); if (reason) { - const why = - reason === "received" ? "and came in with an imported project" : "in a private location"; onLeftOut?.( - `${filePath} links to a file outside the project ${why}; it was left out of the upload.`, + `${filePath} ${WALKED_LEFT_OUT[reason]}; it was left out of the upload.`, leftOutReference(absProjectDir, filePath, filePath, absPath, reason), ); continue; diff --git a/packages/studio-server/src/helpers/outsideFile.ts b/packages/studio-server/src/helpers/outsideFile.ts index 9f262f5ebb..a1dc324005 100644 --- a/packages/studio-server/src/helpers/outsideFile.ts +++ b/packages/studio-server/src/helpers/outsideFile.ts @@ -90,9 +90,10 @@ export const linksOutside = (projectDir: string, filePath: string): boolean => isOutside(relative(realFilePath(projectDir), realFilePath(filePath))); /** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */ -export type LeftOutReason = "blocklisted" | "received" | "file_type"; +export type LeftOutReason = "blocklisted" | "received" | "file_type" | "private"; const WHY: Record = { + private: "which holds secrets or tool data", blocklisted: "which is outside the project in a private location", received: "which leads outside the project and came in with an imported project", file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file",