diff --git a/docs/guides/export-and-share.mdx b/docs/guides/export-and-share.mdx index 797d4447a7..5b8efd1e1e 100644 --- a/docs/guides/export-and-share.mdx +++ b/docs/guides/export-and-share.mdx @@ -129,21 +129,29 @@ publish creates a claimable link. Sign in when the CLI should own the project immediately, update an existing link, or publish into a shared space. A local video poster referenced with a relative path, such as -`../shared/poster.png`, is copied into the published project even when it lives -outside the project directory. Percent-encoded paths work too, such as +`../shared/poster.png`, can be copied into the published project even when it +lives outside the project directory, once you approve it (see below). +Percent-encoded paths work too, such as `../shared/poster%20art.svg` for `poster art.svg` or `../shared/%E5%B0%81%E9%9D%A2.svg` for `封面.svg`. Local images outside the project referenced by an `` or `` element's -`srcset` are included too. Their size or pixel-density descriptors are preserved, +`srcset` can be included the same way. Their size or pixel-density descriptors are preserved, and your source files stay unchanged. Only images, video, audio, fonts and CSS files are copied from outside the project. Any other outside file, such as a script, a Lottie `.json` or a `.pdf`, is left out with a warning naming it; move it into the project folder to include -it. Before uploading, `publish` lists every file it is about to copy from outside -the project folder, including files the project only links to, so you can see -what leaves your computer. +it. + +Nothing from outside the project folder is uploaded without your say. Before +uploading, `publish` lists each such file, including files the project only +links to, and asks whether to upload them. The answer defaults to no, which +leaves them out. A yes is remembered for that project in your user config +folder (`~/.hyperframes/outside-files.json`), never in the project, so the next +publish asks only about new files. With `--yes`, or when nobody can answer the +prompt, outside files are left out with a warning naming each one. Pass +`--include-outside-files` to upload them without asking. Before sharing source, remove secrets and confirm that every included media file can be shared. Tell a reviewer whether you need feedback on story, visuals, diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 7fd0660503..5a4cd8a121 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1174,6 +1174,7 @@ npx hyperframes publish --space | `--update ` | Update an existing project in place. Requires sign-in. | | `--space ` | Publish into a shared team space. Requires sign-in. | | `--proxy` / `--no-proxy` | Bake H.264 proxies for browser-hostile codecs, or skip them | +| `--include-outside-files` | Upload files from outside the project folder without asking. Otherwise they need your approval, and `--yes` leaves them out | `publish` zips the project, uploads it, and prints a URL that resolves to stored content — so it keeps working after the CLI exits. No local server stays alive, @@ -2823,14 +2824,22 @@ same share as the project is copied like any other. To include such a file, move it into the project folder. Under `--json`, the result lists those as `left_out`, each with the `file` that points at it, the `reference` as written, its `path` from the project folder, the `reason` (`blocklisted`, -`received`, `file_type`, `private` or `network_share`) and `levels_up`, how many folders above the project +`received`, `file_type`, `private`, `network_share` or `not_approved`) and `levels_up`, how many folders above the project it is. A full run also prints each as a `{"event":"left_out",…}` line on stderr, so a wrapper can ask with `--dry-run --json` before uploading and still hear about each one during the run. -Before uploading, the command also names every file it carries from outside the -project folder: each file a composition points at, and each project file that is -a link to a file outside the project. It prints the list and carries on. Under +A file from outside the project folder that would be copied (each file a +composition points at, and each project file that is a link to a file outside +the project) is uploaded only when you approve it. Run in a terminal, the +command lists those files and asks, defaulting to no. A yes is remembered for +that project in `~/.hyperframes/outside-files.json`, never in the project. Under +`--json` or `--dry-run`, or when nobody can answer, they are left out with the +`not_approved` reason and a warning naming each one. `--include-outside-files` +uploads them without asking; `hyperframes publish` and `hyperframes feedback +--file-issue` take the same flag and ask the same way unless given `--yes`. + +Before uploading, the command names every outside file it carries. Under `--json`, the result lists them as `outside_files`, with the same `file`, `reference`, `path` and `levels_up` fields, and each is printed as a `{"event":"outside_file",…}` line on stderr. For a project link, `file` and @@ -2852,6 +2861,7 @@ Render parameters mirror local `hyperframes render` where they overlap: | `--title` | — | Free-text label echoed back in detail responses | | `--output` / `-o` | `renders/.` | Where the downloaded video lands | | `--dry-run` | off | Build and inspect the zip without authenticating, uploading, or rendering | +| `--include-outside-files` | off | Upload files from outside the project folder without asking. Otherwise only approved ones go | Lifecycle and control: diff --git a/packages/cli/src/commands/cloud/render.outsideFiles.test.ts b/packages/cli/src/commands/cloud/render.outsideFiles.test.ts index da19ddbbb5..baf2838067 100644 --- a/packages/cli/src/commands/cloud/render.outsideFiles.test.ts +++ b/packages/cli/src/commands/cloud/render.outsideFiles.test.ts @@ -22,23 +22,20 @@ function projectWithOutsideImage(): string { return dir; } -function dryRun(dir: string, json: boolean) { - return spawnSync( - "bun", - ["run", cliEntry, "cloud", "render", dir, "--dry-run", ...(json ? ["--json"] : [])], - { - encoding: "utf-8", - timeout: 30_000, - env: { - ...process.env, - CI: "1", - HEYGEN_API_URL: "http://127.0.0.1:1", - HYPERFRAMES_NO_UPDATE_CHECK: "1", - HYPERFRAMES_NO_TELEMETRY: "1", - HOME: dir, - }, +function dryRun(dir: string, json: boolean, include = true) { + const flags = [...(json ? ["--json"] : []), ...(include ? ["--include-outside-files"] : [])]; + return spawnSync("bun", ["run", cliEntry, "cloud", "render", dir, "--dry-run", ...flags], { + encoding: "utf-8", + timeout: 30_000, + env: { + ...process.env, + CI: "1", + HEYGEN_API_URL: "http://127.0.0.1:1", + HYPERFRAMES_NO_UPDATE_CHECK: "1", + HYPERFRAMES_NO_TELEMETRY: "1", + HOME: dir, }, - ); + }); } const expected = { @@ -48,7 +45,27 @@ const expected = { levels_up: 1, }; -describe("cloud render --dry-run names each file it uploads from outside the project", () => { +describe("cloud render --dry-run without approval leaves out files from outside the project", () => { + it("and names each one with how to include it, in --json and in print", () => { + const json = dryRun(projectWithOutsideImage(), true, false); + expect(json.status).toBe(0); + const { archive } = JSON.parse(json.stdout) as { archive: { outside_files: unknown[] } }; + expect(archive.outside_files).toEqual([]); + const events = json.stderr + .split("\n") + .filter((line) => line.startsWith('{"event":"left_out"')) + .map((line) => JSON.parse(line) as Record); + expect(events).toEqual([{ event: "left_out", ...expected, reason: "not_approved" }]); + + const printed = dryRun(projectWithOutsideImage(), false, false); + expect(printed.status).toBe(0); + expect(printed.stdout).toContain("index.html references ../shared/logo.png"); + expect(printed.stdout).toContain("Pass --include-outside-files to upload it."); + expect(printed.stdout).not.toContain("Uploading 1 file from outside"); + }); +}); + +describe("cloud render --dry-run --include-outside-files names each file it uploads from outside the project", () => { it("in its --json result, and as an outside_file event per file on stderr", () => { const result = dryRun(projectWithOutsideImage(), true); expect(result.status).toBe(0); diff --git a/packages/cli/src/commands/cloud/render.ts b/packages/cli/src/commands/cloud/render.ts index 613f5f5bd4..9249b50c33 100644 --- a/packages/cli/src/commands/cloud/render.ts +++ b/packages/cli/src/commands/cloud/render.ts @@ -71,6 +71,11 @@ import { outsideFilesNotice, } from "../../utils/leftOutReference.js"; import { trackUploadLeftOut } from "../../telemetry/uploadLeftOut.js"; +import { isUnattended } from "../../utils/attendedTerminal.js"; +import { + confirmOutsideFiles, + stageWithOutsideFileApproval, +} from "../../utils/outsideFileApproval.js"; import type { Example } from "../_examples.js"; import { @@ -236,6 +241,12 @@ export default defineCommand({ description: "Build and inspect the project zip without authenticating or uploading", default: false, }, + "include-outside-files": { + type: "boolean", + description: + "Upload the files the project uses from outside its folder without asking. Otherwise they are left out unless you approve them.", + default: false, + }, "idempotency-key": { type: "string", description: "Optional Idempotency-Key for safe retries (1-255 chars from [A-Za-z0-9_:.-])", @@ -285,9 +296,13 @@ export default defineCommand({ resolution, ); + const includeOutsideFiles = args["include-outside-files"] === true; if (args["dry-run"]) { if (project.kind !== "dir") throw new Error("Dry-run project must be a local directory"); - reportDryRun(await prepareLocalArchive(project, args.composition, asJson), asJson); + reportDryRun( + await prepareLocalArchive(project, args.composition, asJson, includeOutsideFiles), + asJson, + ); return; } @@ -306,6 +321,7 @@ export default defineCommand({ project, args.composition, asJson, + includeOutsideFiles, (file) => uploadAssetViaDirectUpload({ client, @@ -692,6 +708,7 @@ async function prepareLocalArchive( source: ProjectInputSource, composition: string | undefined, asJson: boolean, + includeOutsideFiles: boolean, upload?: OffloadOptions["upload"], variablesText?: string, ): Promise { @@ -700,12 +717,11 @@ async function prepareLocalArchive( console.log(""); console.log(`${c.accent("◆")} Zipping ${c.accent(project.name)}`); } - const { fileMap, entryPath, leftOut, outsideFiles } = stageAndReport( - project, - composition, + const { fileMap, entryPath, leftOut, outsideFiles } = await stageAndReport(project, composition, { asJson, - !upload, - ); + dryRun: !upload, + includeOutsideFiles, + }); const swappedFonts = await embedSystemFonts({ dir: project.dir, entryFile: composition, @@ -715,15 +731,7 @@ async function prepareLocalArchive( if (swappedFonts.length > 0 && !asJson) console.log(c.dim(describeSwappedFonts(swappedFonts))); const offloaded = await offloadMedia(project, fileMap, entryPath, asJson, upload, variablesText); - // Media left on disk barely compresses: past the limit, its size is the report and nothing is read. - const onDiskBytes = [...fileMap.values()] - .filter((file) => !Buffer.isBuffer(file)) - .reduce((sum, file) => sum + stagedSize(file), 0); - const archive = - onDiskBytes > DIRECT_UPLOAD_LIMIT_BYTES - ? undefined - : zipPublishFileMap(new Map([...fileMap].map(([path, file]) => [path, readStaged(file)]))); - const sizeBytes = archive?.buffer.byteLength ?? onDiskBytes; + const { archive, sizeBytes } = zipWithinUploadLimit(fileMap); if (!asJson) { console.log(c.dim(` ${fileMap.size} files · ${formatBytes(sizeBytes)}`)); } @@ -746,32 +754,48 @@ async function prepareLocalArchive( return prepared; } -function warnAndListLeftOut(leftOut: LeftOutReference[], asJson: boolean): OnLeftOut { - return (warning, reference) => { - leftOut.push(reference); - if (!asJson) return console.log(c.warn(` ${warning}`)); - console.error(c.warn(` ${warning}`)); - console.error(JSON.stringify({ event: "left_out", ...reference })); - }; +// Media left on disk barely compresses: past the limit, its size is the report and nothing is read. +function zipWithinUploadLimit(fileMap: Map) { + const onDiskBytes = [...fileMap.values()] + .filter((file) => !Buffer.isBuffer(file)) + .reduce((sum, file) => sum + stagedSize(file), 0); + const archive = + onDiskBytes > DIRECT_UPLOAD_LIMIT_BYTES + ? undefined + : zipPublishFileMap(new Map([...fileMap].map(([path, file]) => [path, readStaged(file)]))); + return { archive, sizeBytes: archive?.buffer.byteLength ?? onDiskBytes }; } -function stageAndReport( +function reportLeftOut(warning: string, reference: LeftOutReference, asJson: boolean): void { + if (!asJson) return console.log(c.warn(` ${warning}`)); + console.error(c.warn(` ${warning}`)); + console.error(JSON.stringify({ event: "left_out", ...reference })); +} + +/** Asks about unapproved outside files only on a real upload a person watches; otherwise they stay out. */ +async function stageAndReport( project: ProjectDir, composition: string | undefined, - asJson: boolean, - dryRun: boolean, + options: { asJson: boolean; dryRun: boolean; includeOutsideFiles: boolean }, ) { - const leftOut: LeftOutReference[] = []; - const outsideFiles: OutsideFileReference[] = []; - const staged = stageProjectFiles( - project, - composition, - warnAndListLeftOut(leftOut, asJson), - (reference) => outsideFiles.push(reference), + const { asJson, dryRun } = options; + const { staged, outsideFiles } = await stageWithOutsideFileApproval( + project.dir, + { + includeAll: options.includeOutsideFiles, + ask: asJson || dryRun || isUnattended() ? undefined : confirmOutsideFiles, + }, + (onOutsideFile) => { + const warnings: { warning: string; reference: LeftOutReference }[] = []; + const onLeftOut: OnLeftOut = (warning, reference) => warnings.push({ warning, reference }); + return { ...stageProjectFiles(project, composition, onLeftOut, onOutsideFile), warnings }; + }, ); + for (const { warning, reference } of staged.warnings) reportLeftOut(warning, reference, asJson); + const leftOut = staged.warnings.map(({ reference }) => reference); trackUploadLeftOut("cloud_render", leftOut, dryRun); listOutsideFiles(outsideFiles, asJson); - return { ...staged, leftOut, outsideFiles }; + return { fileMap: staged.fileMap, entryPath: staged.entryPath, leftOut, outsideFiles }; } function listOutsideFiles(outsideFiles: OutsideFileReference[], asJson: boolean): void { diff --git a/packages/cli/src/commands/feedback.ts b/packages/cli/src/commands/feedback.ts index 51e03cee88..90aab00cb6 100644 --- a/packages/cli/src/commands/feedback.ts +++ b/packages/cli/src/commands/feedback.ts @@ -22,12 +22,13 @@ import { isUnattended } from "../utils/attendedTerminal.js"; import { parseFeedbackRating } from "../utils/feedbackRating.js"; import { feedbackEmail, parseFeedbackSource } from "../utils/feedbackSource.js"; import { lintFeedbackComment, type FeedbackLintInput } from "../utils/feedbackLint.js"; -import { - type LeftOutReference, - type OutsideFileReference, - outsideFilesNotice, -} from "../utils/leftOutReference.js"; +import { type LeftOutReference, outsideFilesNotice } from "../utils/leftOutReference.js"; import { trackUploadLeftOut } from "../telemetry/uploadLeftOut.js"; +import { + confirmOutsideFiles, + type OutsideFileChoice, + stageWithOutsideFileApproval, +} from "../utils/outsideFileApproval.js"; export const examples: Example[] = [ ["Submit render feedback", 'hyperframes feedback --rating 8 --comment "fast but font missing"'], @@ -122,21 +123,40 @@ async function confirmFileIssue(dir: string, yes: boolean): Promise { * Publish a minimal repro and return its public URL. Degrades gracefully: * on failure it returns undefined so the issue still opens without a link. */ -async function publishRepro(dir: string): Promise { +async function publishRepro( + dir: string, + outsideFileChoice: OutsideFileChoice, +): Promise { const spinner = clack.spinner(); spinner.start("Publishing minimal repro..."); try { - const leftOut: string[] = []; - const references: LeftOutReference[] = []; - const outsideFiles: OutsideFileReference[] = []; - const archive = createPublishArchive( + const ask = outsideFileChoice.ask; + const { staged, outsideFiles } = await stageWithOutsideFileApproval( dir, - (warning, reference) => { - leftOut.push(warning); - references.push(reference); + { + ...outsideFileChoice, + ask: + ask && + ((files) => { + spinner.stop("Repro prepared"); + return ask(files).finally(() => spinner.start("Publishing minimal repro...")); + }), + }, + (onOutsideFile) => { + const leftOut: string[] = []; + const references: LeftOutReference[] = []; + const archive = createPublishArchive( + dir, + (warning, reference) => { + leftOut.push(warning); + references.push(reference); + }, + onOutsideFile, + ); + return { archive, leftOut, references }; }, - (reference) => outsideFiles.push(reference), ); + const { archive, leftOut, references } = staged; trackUploadLeftOut("feedback", references); if (outsideFiles.length > 0) { spinner.stop("Repro prepared"); @@ -186,11 +206,15 @@ async function fileGithubIssue(opts: { comment?: string; rawDir?: string; yes: boolean; + includeOutsideFiles: boolean; doctorSummary: string; }): Promise { const dir = resolve(opts.rawDir ?? "."); if (!(await confirmFileIssue(dir, opts.yes))) return; - const repoPublicUrl = await publishRepro(dir); + const repoPublicUrl = await publishRepro(dir, { + includeAll: opts.includeOutsideFiles, + ask: opts.yes || isUnattended() ? undefined : confirmOutsideFiles, + }); const url = buildIssueUrl({ repoUrl: HYPERFRAMES_REPO_URL, rating: opts.rating, @@ -290,6 +314,12 @@ export default defineCommand({ description: "Skip the publish + file-issue consent prompt", default: false, }, + "include-outside-files": { + type: "boolean", + description: + "Upload the files the repro uses from outside its folder without asking. Otherwise they are left out unless you approve them.", + default: false, + }, }, async run({ args }) { // A search miss is a different report from a rating: it names a gap in the @@ -366,6 +396,7 @@ export default defineCommand({ comment, rawDir: args.dir, yes: args.yes === true, + includeOutsideFiles: args["include-outside-files"] === true, doctorSummary, }); } diff --git a/packages/cli/src/commands/publish.ts b/packages/cli/src/commands/publish.ts index 09438f4f26..0a2acc5c52 100644 --- a/packages/cli/src/commands/publish.ts +++ b/packages/cli/src/commands/publish.ts @@ -31,12 +31,9 @@ import { writeTeamProject, } from "../utils/projectLink.js"; import { cliSourced } from "../utils/sourceTag.js"; -import { - type LeftOutReference, - type OutsideFileReference, - outsideFilesNotice, -} from "../utils/leftOutReference.js"; +import { type LeftOutReference, outsideFilesNotice } from "../utils/leftOutReference.js"; import { trackUploadLeftOut } from "../telemetry/uploadLeftOut.js"; +import { confirmOutsideFiles, stageWithOutsideFileApproval } from "../utils/outsideFileApproval.js"; export const examples: Example[] = [ ["Publish the current project privately to a stable URL", "hyperframes publish"], @@ -101,6 +98,12 @@ export default defineCommand({ description: "Show full lint findings before publishing instead of a one-line summary", default: false, }, + "include-outside-files": { + type: "boolean", + description: + "Upload the files the project uses from outside its folder without asking. Otherwise they are left out unless you approve them.", + default: false, + }, }, async run({ args }) { const publishPhase = checkPublishTransition(args); @@ -256,17 +259,35 @@ export default defineCommand({ // committed config, else on by default. const proxyFlagValue = typeof args.proxy === "boolean" ? args.proxy : undefined; const autoProxy = resolveAutoProxy(dir, proxyFlagValue); - const leftOut: string[] = []; - const references: LeftOutReference[] = []; - const outsideFiles: OutsideFileReference[] = []; - const fileMap = buildPublishFileMap( + const { staged, outsideFiles } = await stageWithOutsideFileApproval( dir, - (warning, reference) => { - leftOut.push(warning); - references.push(reference); + { + includeAll: args["include-outside-files"] === true, + ask: + args.yes === true + ? undefined + : (files) => { + publishSpinner.stop("Project prepared"); + return confirmOutsideFiles(files).finally(() => + publishSpinner.start("Preparing project..."), + ); + }, + }, + (onOutsideFile) => { + const leftOut: string[] = []; + const references: LeftOutReference[] = []; + const fileMap = buildPublishFileMap( + dir, + (warning, reference) => { + leftOut.push(warning); + references.push(reference); + }, + onOutsideFile, + ); + return { fileMap, leftOut, references }; }, - (reference) => outsideFiles.push(reference), ); + const { fileMap, leftOut, references } = staged; trackUploadLeftOut("publish", references); let proxyBakeManifest: Awaited> | undefined; if (autoProxy) { diff --git a/packages/cli/src/telemetry/uploadLeftOut.test.ts b/packages/cli/src/telemetry/uploadLeftOut.test.ts index 0098d5d2cc..41d32b858a 100644 --- a/packages/cli/src/telemetry/uploadLeftOut.test.ts +++ b/packages/cli/src/telemetry/uploadLeftOut.test.ts @@ -38,6 +38,7 @@ describe("cli_upload_left_out", () => { received_count: 4, blocklisted_count: 1, file_type_count: 2, + not_approved_count: 0, max_levels_up: 5, levels_up_1: 3, levels_up_2: 2, diff --git a/packages/cli/src/telemetry/uploadLeftOut.ts b/packages/cli/src/telemetry/uploadLeftOut.ts index b41f9573a5..aea7801594 100644 --- a/packages/cli/src/telemetry/uploadLeftOut.ts +++ b/packages/cli/src/telemetry/uploadLeftOut.ts @@ -12,6 +12,8 @@ function leftOutCounts(references: readonly LeftOutReference[]) { received_count: references.filter((reference) => reference.reason === "received").length, blocklisted_count: references.filter((reference) => reference.reason === "blocklisted").length, file_type_count: references.filter((reference) => reference.reason === "file_type").length, + not_approved_count: references.filter((reference) => reference.reason === "not_approved") + .length, max_levels_up: Math.max(0, ...levels.map((n) => n ?? 0)), levels_up_1: counted((n) => n === 1), levels_up_2: counted((n) => n === 2), diff --git a/packages/cli/src/utils/leftOutReference.ts b/packages/cli/src/utils/leftOutReference.ts index d15102b37a..b1fa6a9927 100644 --- a/packages/cli/src/utils/leftOutReference.ts +++ b/packages/cli/src/utils/leftOutReference.ts @@ -1,12 +1,14 @@ import { terminalText } from "@hyperframes/core"; import { isAbsolute, relative } from "node:path"; import { + INCLUDE_OUTSIDE_FILES_HINT, type LeftOutReason, leftOutWarning as leftOutText, } from "@hyperframes/studio-server/outside-file"; import { realFilePath } from "@hyperframes/studio-server/safe-path"; import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution"; +export { INCLUDE_OUTSIDE_FILES_HINT }; export type { LeftOutReason }; export const leftOutWarning = (file: string, reference: string, reason: LeftOutReason): string => @@ -25,7 +27,7 @@ export type OnLeftOut = (warning: string, reference: LeftOutReference) => void; export type OutsideFileReference = Omit; -export type OnOutsideFile = (reference: OutsideFileReference) => void; +export type OnOutsideFile = (reference: OutsideFileReference) => boolean | void; export function outsideFileReference( projectDir: string, @@ -51,12 +53,26 @@ export function leftOutReference( return { file, reference, path, reason, levels_up }; } -export function outsideFilesNotice(references: OutsideFileReference[]): string { - const count = references.length === 1 ? "1 file" : `${references.length} files`; - const lines = references.map(({ file, reference, path }) => +function outsideFileLines(references: OutsideFileReference[]): string[] { + return references.map(({ file, reference, path }) => terminalText( file === reference ? ` ${path} (linked from ${file})` : ` ${path} (referenced by ${file})`, ), ); - return [`Uploading ${count} from outside the project folder:`, ...lines].join("\n"); +} + +const fileCount = (references: OutsideFileReference[]) => + references.length === 1 ? "1 file" : `${references.length} files`; + +export function outsideFilesNotice(references: OutsideFileReference[]): string { + const heading = `Uploading ${fileCount(references)} from outside the project folder:`; + return [heading, ...outsideFileLines(references)].join("\n"); +} + +export function outsideFilesQuestion(references: OutsideFileReference[]): string { + return [ + `The project uses ${fileCount(references)} from outside its folder:`, + ...outsideFileLines(references), + "Upload them too? Yes is remembered for this project; No leaves them out.", + ].join("\n"); } diff --git a/packages/cli/src/utils/outsideFileApproval.test.ts b/packages/cli/src/utils/outsideFileApproval.test.ts new file mode 100644 index 0000000000..d971086589 --- /dev/null +++ b/packages/cli/src/utils/outsideFileApproval.test.ts @@ -0,0 +1,115 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { approvedOutsideFiles, stageWithOutsideFileApproval } from "./outsideFileApproval.js"; +import { stagePublishFiles } from "./publishProject.js"; + +let root: string; +let project: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "hf-outside-approval-")); + project = join(root, "p"); + mkdirSync(join(root, "shared"), { recursive: true }); + mkdirSync(project); + writeFileSync(join(root, "shared", "logo.png"), "png"); + writeFileSync(join(project, "index.html"), ``); + vi.stubEnv("HYPERFRAMES_CONFIG_DIR", join(root, "config")); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); +}); + +const stage = (onOutsideFile: Parameters[3]) => + stagePublishFiles(project, Infinity, undefined, onOutsideFile); +const extKeys = (staged: Map) => + [...staged.keys()].filter((key) => key.startsWith("_ext/")); + +describe("uploading files from outside the project", () => { + it("leaves them out when nobody can be asked", async () => { + const { staged, outsideFiles } = await stageWithOutsideFileApproval( + project, + { includeAll: false }, + stage, + ); + expect(extKeys(staged)).toEqual([]); + expect(outsideFiles).toEqual([]); + }); + + it("asks once, defaulting to no, and leaves them out on no", async () => { + const ask = vi.fn(async () => false); + const { staged } = await stageWithOutsideFileApproval( + project, + { includeAll: false, ask }, + stage, + ); + expect(ask).toHaveBeenCalledTimes(1); + expect(ask).toHaveBeenCalledWith([expect.objectContaining({ path: "../shared/logo.png" })]); + expect(extKeys(staged)).toEqual([]); + expect(approvedOutsideFiles(project).size).toBe(0); + }); + + it("uploads them on yes and remembers that in the config folder, not the project", async () => { + const ask = vi.fn(async () => true); + const { staged, outsideFiles } = await stageWithOutsideFileApproval( + project, + { includeAll: false, ask }, + stage, + ); + expect(extKeys(staged)).toEqual(["_ext/shared/logo.png"]); + expect(outsideFiles).toHaveLength(1); + expect(existsSync(join(project, ".hyperframes"))).toBe(false); + const saved = join(root, "config", "outside-files.json"); + if (process.platform !== "win32") expect(statSync(saved).mode & 0o777).toBe(0o600); + expect(Object.values(JSON.parse(readFileSync(saved, "utf-8")))).toEqual([ + ["../shared/logo.png"], + ]); + + const again = vi.fn(async () => false); + const next = await stageWithOutsideFileApproval( + project, + { includeAll: false, ask: again }, + stage, + ); + expect(again).not.toHaveBeenCalled(); + expect(extKeys(next.staged)).toEqual(["_ext/shared/logo.png"]); + }); + + it("asks again about a file approved for another project", async () => { + await stageWithOutsideFileApproval( + project, + { includeAll: false, ask: async () => true }, + stage, + ); + const other = join(root, "q"); + mkdirSync(other); + writeFileSync(join(other, "index.html"), ``); + const { staged } = await stageWithOutsideFileApproval(other, { includeAll: false }, (gate) => + stagePublishFiles(other, Infinity, undefined, gate), + ); + expect(extKeys(staged)).toEqual([]); + }); + + it("uploads them without asking or remembering with --include-outside-files", async () => { + const ask = vi.fn(async () => false); + const { staged } = await stageWithOutsideFileApproval( + project, + { includeAll: true, ask }, + stage, + ); + expect(ask).not.toHaveBeenCalled(); + expect(extKeys(staged)).toEqual(["_ext/shared/logo.png"]); + expect(approvedOutsideFiles(project).size).toBe(0); + }); +}); diff --git a/packages/cli/src/utils/outsideFileApproval.ts b/packages/cli/src/utils/outsideFileApproval.ts new file mode 100644 index 0000000000..fa05cb92a1 --- /dev/null +++ b/packages/cli/src/utils/outsideFileApproval.ts @@ -0,0 +1,96 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { realFilePath } from "@hyperframes/studio-server/safe-path"; +import * as clack from "../ui/prompts.js"; +import { hyperframesConfigDir } from "./configDir.js"; +import { + type OnOutsideFile, + type OutsideFileReference, + outsideFilesQuestion, +} from "./leftOutReference.js"; + +/** Outside files a person approved, per project folder. Kept in the CLI's config folder, never in a project. */ +const approvalsFile = () => hyperframesConfigDir("outside-files.json"); + +type Approvals = Record; + +function readApprovals(): Approvals { + try { + const parsed: unknown = JSON.parse(readFileSync(approvalsFile(), "utf-8")); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Approvals) + : {}; + } catch { + // expected: no approvals yet, or a file that cannot be read, which approves nothing + return {}; + } +} + +const projectKey = (projectDir: string) => realFilePath(resolve(projectDir)); + +export function approvedOutsideFiles(projectDir: string): Set { + const listed = readApprovals()[projectKey(projectDir)]; + return new Set(Array.isArray(listed) ? listed.map(String) : []); +} + +function rememberOutsideFiles(projectDir: string, paths: Iterable): void { + const approvals = readApprovals(); + const key = projectKey(projectDir); + approvals[key] = [...new Set([...approvedOutsideFiles(projectDir), ...paths])].sort(); + const file = approvalsFile(); + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + const staging = `${file}.${process.pid}.tmp`; + writeFileSync(staging, `${JSON.stringify(approvals, null, 2)}\n`, { mode: 0o600 }); + renameSync(staging, file); +} + +export async function confirmOutsideFiles(references: OutsideFileReference[]): Promise { + const answer = await clack.confirm({ + message: outsideFilesQuestion(references), + initialValue: false, + }); + return answer === true; +} + +export interface OutsideFileChoice { + /** `--include-outside-files`: every outside file goes, approved or not. */ + includeAll: boolean; + /** Asks a person about the files not yet approved; absent when nobody can answer. */ + ask?: (references: OutsideFileReference[]) => Promise; +} + +/** + * Runs `stage` with outside files left out unless approved for this project or `includeAll`, and returns what it + * staged with the outside files it copied. When `ask` approves the rest, they are remembered and `stage` runs again. + */ +export async function stageWithOutsideFileApproval( + projectDir: string, + choice: OutsideFileChoice, + stage: (onOutsideFile: OnOutsideFile) => T, +): Promise<{ staged: T; outsideFiles: OutsideFileReference[] }> { + const approved = approvedOutsideFiles(projectDir); + const run = () => { + const outsideFiles: OutsideFileReference[] = []; + const notApproved = new Map(); + const staged = stage((reference) => { + if (!choice.includeAll && !approved.has(reference.path)) { + notApproved.set(reference.path, reference); + return false; + } + outsideFiles.push(reference); + return true; + }); + return { staged, outsideFiles, notApproved: [...notApproved.values()] }; + }; + const first = run(); + if (first.notApproved.length === 0 || !choice.ask || !(await choice.ask(first.notApproved))) + return first; + const paths = first.notApproved.map((reference) => reference.path); + for (const path of paths) approved.add(path); + try { + rememberOutsideFiles(projectDir, paths); + } catch (error) { + clack.log.warn(`Could not remember the approval for next time: ${(error as Error).message}`); + } + return run(); +} diff --git a/packages/cli/src/utils/publishProject.outsideList.test.ts b/packages/cli/src/utils/publishProject.outsideList.test.ts index 22f8d94565..86b04420d3 100644 --- a/packages/cli/src/utils/publishProject.outsideList.test.ts +++ b/packages/cli/src/utils/publishProject.outsideList.test.ts @@ -2,14 +2,22 @@ import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { type OutsideFileReference, outsideFilesNotice } from "./leftOutReference.js"; +import { + type LeftOutReference, + type OutsideFileReference, + outsideFilesNotice, +} from "./leftOutReference.js"; import { stagePublishFiles } from "./publishProject.js"; const dirs: string[] = []; afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))); /** Writes `files` under a fresh folder, makes `links` (link to target), and stages its `p` project. */ -function stage(files: Record, links: Record = {}) { +function stage( + files: Record, + links: Record = {}, + approve = (_reference: OutsideFileReference) => true, +) { const root = mkdtempSync(join(tmpdir(), "hf-outside-list-")); dirs.push(root); for (const [name, body] of Object.entries(files)) { @@ -21,10 +29,17 @@ function stage(files: Record, links: Record = {} symlinkSync(join(root, target), join(root, link)); } const outside: OutsideFileReference[] = []; - const staged = stagePublishFiles(join(root, "p"), Infinity, undefined, (reference) => - outside.push(reference), + const leftOut: { warning: string; reference: LeftOutReference }[] = []; + const staged = stagePublishFiles( + join(root, "p"), + Infinity, + (warning, reference) => leftOut.push({ warning, reference }), + (reference) => { + outside.push(reference); + return approve(reference); + }, ); - return { root, staged, outside }; + return { root, staged, outside, leftOut }; } describe("files from outside the project that an upload carries", () => { @@ -95,6 +110,66 @@ describe("files from outside the project that an upload carries", () => { }); }); +describe("files from outside the project nobody approved", () => { + const refuse = () => false; + + it("stay out of the upload for ../ and absolute references, each named with how to include it", () => { + const { staged, leftOut } = stage( + { + "shared/logo.png": "png", + "shared/brand.woff2": "woff2", + "p/style.css": `@font-face { font-family: B; src: url("../shared/brand.woff2"); }`, + "p/index.html": [ + ``, + ``, + ].join(""), + }, + {}, + refuse, + ); + expect([...staged.keys()].filter((key) => key.startsWith("_ext/"))).toEqual([]); + expect(String(staged.get("index.html"))).toContain(`src="../shared/logo.png"`); + expect(leftOut.map(({ reference }) => [reference.reference, reference.reason])).toEqual( + expect.arrayContaining([ + ["../shared/logo.png", "not_approved"], + ["../shared/brand.woff2", "not_approved"], + ]), + ); + expect(leftOut.map(({ reference }) => reference.path)).not.toContain("logo.png"); + for (const { warning } of leftOut) expect(warning).toContain("--include-outside-files"); + }); + + it("leave out project files that are links to them", () => { + const { staged, leftOut } = stage( + { "shared/logo.png": "png", "p/index.html": `` }, + { "p/logo.png": "shared/logo.png" }, + refuse, + ); + expect(staged.has("logo.png")).toBe(false); + expect(leftOut).toEqual([ + expect.objectContaining({ + warning: expect.stringContaining("--include-outside-files"), + reference: expect.objectContaining({ file: "logo.png", reason: "not_approved" }), + }), + ]); + }); + + it("go when approved one by one", () => { + const { staged } = stage( + { + "shared/a.png": "a", + "shared/b.png": "b", + "p/index.html": ``, + }, + {}, + (reference) => reference.path.endsWith("a.png"), + ); + expect([...staged.keys()].filter((key) => key.startsWith("_ext/"))).toEqual([ + "_ext/shared/a.png", + ]); + }); +}); + describe("the notice that names them", () => { it("says how many there are and where each one comes from", () => { expect( diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index 3783d3f402..be8be29b13 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -32,6 +32,7 @@ import { receivedFiles, } from "./privateProjectFile.js"; import { + INCLUDE_OUTSIDE_FILES_HINT, leftOutReference, leftOutWarning, type LeftOutReason, @@ -415,6 +416,7 @@ interface ExternalAssetContext { externalMap: Map; usedArchivePaths: Set; leftOut: Set; + notApproved: Set; onLeftOut?: OnLeftOut; onOutsideFile?: OnOutsideFile; received: (path: string) => boolean; @@ -427,10 +429,15 @@ function addExternalAsset( absPath: string, entryPath: string, rawPath: string, -): string { +): string | null { const existing = ctx.externalMap.get(absPath); if (existing) return existing; - ctx.onOutsideFile?.(outsideFileReference(ctx.absProjectDir, entryPath, rawPath, absPath)); + const reference = outsideFileReference(ctx.absProjectDir, entryPath, rawPath, absPath); + if (ctx.notApproved.has(absPath) || ctx.onOutsideFile?.(reference) === false) { + ctx.notApproved.add(absPath); + leaveOut(ctx, entryPath, rawPath, absPath, "not_approved"); + return null; + } const rel = relative(ctx.absProjectDir, absPath).replaceAll("\\", "/"); const stripped = rel.replace(/^(?:\.\.\/)+/, ""); @@ -465,24 +472,27 @@ const loadableReference = (absProjectDir: string, absPath: string): boolean => function referencedFileLeftOut( ctx: ExternalAssetContext, entryPath: string, + rel: string, absPath: string, ): LeftOutReason | null { if (linksToPrivateOutsideFile(ctx.absProjectDir, absPath)) return "blocklisted"; if (isPrivateCompositionFile(ctx.absProjectDir, absPath)) return "private"; if (ctx.received(entryPath) && linksOutside(ctx.absProjectDir, absPath)) return "received"; if (!loadableReference(ctx.absProjectDir, absPath)) return "private"; + if (!linkOutsideApproved(ctx.absProjectDir, rel, absPath, ctx.onOutsideFile)) + return "not_approved"; return null; } -function reportProjectLinkOutside( +/** Whether a project file may be uploaded as far as where it links goes: false when it links outside, unapproved. */ +function linkOutsideApproved( absProjectDir: string, file: string, absPath: string, onOutsideFile: OnOutsideFile | undefined, -): void { - if (onOutsideFile && linksOutside(absProjectDir, absPath)) { - onOutsideFile(outsideFileReference(absProjectDir, file, file, absPath)); - } +): boolean { + if (!onOutsideFile || !linksOutside(absProjectDir, absPath)) return true; + return onOutsideFile(outsideFileReference(absProjectDir, file, file, absPath)) !== false; } /** Ships a referenced in-project file the ignore rules skipped, resolved from its folder or the root. */ @@ -504,9 +514,8 @@ function includeReferencedProjectFile( continue; } if (!statSync(absPath).isFile()) return; - const reason = referencedFileLeftOut(ctx, entryPath, absPath); + const reason = referencedFileLeftOut(ctx, entryPath, rel, absPath); if (reason) return leaveOut(ctx, entryPath, rawPath, absPath, reason); - reportProjectLinkOutside(ctx.absProjectDir, rel, absPath, ctx.onOutsideFile); ctx.fileContents.set(rel, stageFile(absPath, ctx.onDiskMinBytes)); return; } @@ -580,7 +589,17 @@ function archiveTargetOf( } catch { return undefined; } - return encodeUrlPath(addExternalAsset(ctx, absPath, options.entryPath, rawPath)); + return externalArchiveTarget(ctx, absPath, options.entryPath, rawPath); +} + +function externalArchiveTarget( + ctx: ExternalAssetContext, + absPath: string, + entryPath: string, + rawPath: string, +): string | undefined { + const archivePath = addExternalAsset(ctx, absPath, entryPath, rawPath); + return archivePath === null ? undefined : encodeUrlPath(archivePath); } // The ? of a \\?\ device path is part of the path, not a query. @@ -806,6 +825,7 @@ export function localizeExternalAssets( externalMap: new Map(), usedArchivePaths: new Set(), leftOut: new Set(), + notApproved: new Set(), onLeftOut, onOutsideFile, received: receivedFiles(absProjectDir), @@ -867,6 +887,7 @@ const WALKED_LEFT_OUT: Record = { "holds secrets or tool data (an env file, Git data, credentials, agent or editor state, or Studio records)", blocklisted: "links to a file outside the project in a private location", received: "links to a file outside the project and came in with an imported project", + not_approved: "links to a file outside the project that was not approved for upload", }; function walkedFileLeftOut( @@ -924,7 +945,14 @@ export function stagePublishFiles( ); } diskPathOf.set(key, filePath); - reportProjectLinkOutside(absProjectDir, filePath, absPath, onOutsideFile); + if (!linkOutsideApproved(absProjectDir, filePath, absPath, onOutsideFile)) { + onLeftOut?.( + `${filePath} ${WALKED_LEFT_OUT.not_approved}; it was left out of the upload. ${INCLUDE_OUTSIDE_FILES_HINT}`, + leftOutReference(absProjectDir, filePath, filePath, absPath, "not_approved"), + ); + walkedLeftOut.add(key); + continue; + } fileContents.set(key, stageFile(absPath, onDiskMinBytes)); } diff --git a/packages/studio-server/src/helpers/outsideFile.ts b/packages/studio-server/src/helpers/outsideFile.ts index 35269e3c1e..0a5a7cfa79 100644 --- a/packages/studio-server/src/helpers/outsideFile.ts +++ b/packages/studio-server/src/helpers/outsideFile.ts @@ -146,14 +146,15 @@ export function isPrivateOutsideFile(projectDir: string, absPath: string): boole export const linksOutside = (projectDir: string, filePath: string): boolean => isOutside(relative(realFilePath(projectDir), realFilePath(filePath))); -/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy; `network_share`: on another computer; `script`: a script a local render was not told to run. */ +/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy; `network_share`: on another computer; `script`: a script a local render was not told to run; `not_approved`: the person has not approved uploading it. */ export type LeftOutReason = | "blocklisted" | "received" | "file_type" | "private" | "network_share" - | "script"; + | "script" + | "not_approved"; const WHY: Record, string> = { private: "which holds secrets or tool data", @@ -161,8 +162,11 @@ const WHY: Record, string> = { received: "which leads outside the project and came in with an imported project", network_share: "which is on another computer's network share or is a Windows device path", script: "which is a script outside the project", + not_approved: "which is outside the project and was not approved for upload", }; +export const INCLUDE_OUTSIDE_FILES_HINT = "Pass --include-outside-files to upload it."; + const COPIED_TYPES = { upload: "an image, video, audio, font or CSS file", render: "an image, video, audio, font, CSS or JS file", @@ -180,6 +184,7 @@ export function leftOutWarning( : WHY[reason]; const warning = `${file} references ${reference}, ${why}; it was left out of the ${from}.`; if (reason === "file_type") return `${warning} Move it into the project to include it.`; + if (reason === "not_approved") return `${warning} ${INCLUDE_OUTSIDE_FILES_HINT}`; if (reason !== "script") return warning; return `${warning} Move it into the project, or pass --include-outside-files to hyperframes render, to run it.`; }