diff --git a/packages/cli/src/commands/cloud/render.ts b/packages/cli/src/commands/cloud/render.ts index 0b7de37747..dc15fb027e 100644 --- a/packages/cli/src/commands/cloud/render.ts +++ b/packages/cli/src/commands/cloud/render.ts @@ -685,7 +685,9 @@ async function prepareLocalArchive( console.log(""); console.log(`${c.accent("◆")} Zipping ${c.accent(project.name)}`); } - const { fileMap, entryPath } = stageProjectFiles(project, composition); + const { fileMap, entryPath } = stageProjectFiles(project, composition, (warning) => + (asJson ? console.error : console.log)(c.warn(` ${warning}`)), + ); const swappedFonts = await embedSystemFonts({ dir: project.dir, entryFile: composition, @@ -727,9 +729,10 @@ async function prepareLocalArchive( function stageProjectFiles( project: ProjectDir, composition: string | undefined, + onLeftOut: (warning: string) => void, ): { fileMap: Map; entryPath: string } { try { - const fileMap = stagePublishFiles(project.dir, OFFLOAD_MIN_BYTES); + const fileMap = stagePublishFiles(project.dir, OFFLOAD_MIN_BYTES, onLeftOut); const entryPath = archiveKey( relative(project.dir, resolvePath(project.dir, composition ?? "index.html")).replaceAll( "\\", diff --git a/packages/cli/src/commands/feedback.ts b/packages/cli/src/commands/feedback.ts index cf29c43c6e..48b1ad05df 100644 --- a/packages/cli/src/commands/feedback.ts +++ b/packages/cli/src/commands/feedback.ts @@ -13,7 +13,7 @@ import { import { shouldTrack } from "../telemetry/client.js"; import { getDoctorSummary } from "../telemetry/feedback.js"; import { readConfig, type LastCheckRecord, type RecentRenderRecord } from "../telemetry/config.js"; -import { publishProjectArchive } from "../utils/publishProject.js"; +import { createPublishArchive, publishProjectArchive } from "../utils/publishProject.js"; import { submitCatalogSearchMiss, submitFeedback } from "../utils/submitFeedback.js"; import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js"; import { VERSION } from "../version.js"; @@ -120,8 +120,11 @@ async function publishRepro(dir: string): Promise { const spinner = clack.spinner(); spinner.start("Publishing minimal repro..."); try { - const published = await publishProjectArchive(dir); + const leftOut: string[] = []; + const archive = createPublishArchive(dir, (warning) => leftOut.push(warning)); + const published = await publishProjectArchive(dir, { archive }); spinner.stop(c.success("Repro published")); + for (const warning of leftOut) clack.log.warn(c.warn(warning)); return published.url; } catch (err: unknown) { spinner.stop(c.error("Publish failed")); diff --git a/packages/cli/src/commands/publish.ts b/packages/cli/src/commands/publish.ts index 6da11d3710..14415efb38 100644 --- a/packages/cli/src/commands/publish.ts +++ b/packages/cli/src/commands/publish.ts @@ -233,7 +233,8 @@ export default defineCommand({ // committed config, else on by default. const proxyFlagValue = typeof args.proxy === "boolean" ? args.proxy : undefined; const autoProxy = resolveAutoProxy(dir, proxyFlagValue); - const fileMap = buildPublishFileMap(dir); + const leftOut: string[] = []; + const fileMap = buildPublishFileMap(dir, (warning) => leftOut.push(warning)); let proxyBakeManifest: Awaited> | undefined; if (autoProxy) { proxyBakeManifest = await bakeMediaProxies(dir, fileMap); @@ -249,6 +250,7 @@ export default defineCommand({ credential, }); publishSpinner.stop(c.success("Project published")); + for (const warning of leftOut) clack.log.warn(c.warn(warning)); console.log(); console.log(` ${c.dim("Project")} ${c.accent(published.title)}`); diff --git a/packages/cli/src/utils/privateProjectFile.ts b/packages/cli/src/utils/privateProjectFile.ts index 2d94361507..bd6c1c71fc 100644 --- a/packages/cli/src/utils/privateProjectFile.ts +++ b/packages/cli/src/utils/privateProjectFile.ts @@ -1,4 +1,5 @@ -import { relative } from "node:path"; +import { homedir } from "node:os"; +import { isAbsolute, join, relative, sep } from "node:path"; import { isPrivateProjectFile, realFilePath } from "@hyperframes/studio-server/safe-path"; const ENV_FILE = /^\.env([._-].*|rc)?$/; @@ -8,6 +9,55 @@ export function isEnvOrGitPath(path: string): boolean { return segments.includes(".git") || ENV_FILE.test(segments.at(-1) ?? ""); } +const SYSTEM_DIRS = ["/etc", "/proc", "/sys", "/dev", "/private/etc"]; + +function inPrivateLocation(projectDir: string, path: string): boolean { + const posixPath = path.replaceAll("\\", "/"); + if (SYSTEM_DIRS.some((dir) => posixPath === dir || posixPath.startsWith(`${dir}/`))) return true; + const segmentsPastProjectFolders = relative(projectDir, path) + .split(/[\\/]/) + .filter((segment) => segment !== ".."); + return ( + segmentsPastProjectFolders.some((segment) => segment.startsWith(".")) || + inHomeAppData(projectDir, path) + ); +} + +function isUnder(path: string, ...dir: string[]): boolean { + const lowerPath = path.toLowerCase(); + const lowerDir = join(homedir(), ...dir).toLowerCase(); + return lowerPath === lowerDir || lowerPath.startsWith(`${lowerDir}${sep}`); +} + +function homeAppDataFolder(path: string): string | null { + if (isUnder(path, "AppData")) return "AppData"; + if (isUnder(path, "Library") && !isUnder(path, "Library", "Mobile Documents")) return "Library"; + return null; +} + +function inHomeAppData(projectDir: string, path: string): boolean { + const folder = homeAppDataFolder(path); + return folder !== null && !isUnder(projectDir, folder); +} + +/** An existing file outside the project that publish leaves out: env files, Git data, hidden, system and app-data folders. */ +export function isPrivateOutsideFile(projectDir: string, absPath: string): boolean { + const realPath = realFilePath(absPath); + return ( + isEnvOrGitPath(absPath) || + isEnvOrGitPath(realPath) || + inPrivateLocation(projectDir, absPath) || + inPrivateLocation(realFilePath(projectDir), realPath) + ); +} + +export function linksToPrivateOutsideFile(projectDir: string, filePath: string): boolean { + const fromProject = relative(realFilePath(projectDir), realFilePath(filePath)); + const outside = + fromProject === ".." || fromProject.startsWith(`..${sep}`) || isAbsolute(fromProject); + return outside && isPrivateOutsideFile(projectDir, filePath); +} + /** Project files no composition loads, so none is served or uploaded for one: env files, Git data, tool records. */ export function isPrivateCompositionFile(projectDir: string, filePath: string): boolean { return ( diff --git a/packages/cli/src/utils/publishProject.absolutePath.test.ts b/packages/cli/src/utils/publishProject.absolutePath.test.ts index 54ecd335ae..a8ac045d6e 100644 --- a/packages/cli/src/utils/publishProject.absolutePath.test.ts +++ b/packages/cli/src/utils/publishProject.absolutePath.test.ts @@ -1,7 +1,7 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { basename, dirname, join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { stagePublishFiles } from "./publishProject.js"; const dirs: string[] = []; @@ -81,3 +81,208 @@ describe("an env file outside the project", () => { ]); }); }); + +describe("a reference to a file outside the project in a private location", () => { + /** Writes `files` under a fresh folder, stages `/`, and collects the warnings. */ + function stageIn(projectPath: string, files: Record) { + const root = temp(); + for (const [name, body] of Object.entries(files)) { + mkdirSync(dirname(join(root, name)), { recursive: true }); + writeFileSync(join(root, name), body.replaceAll("", root)); + } + const warnings: string[] = []; + const staged = stagePublishFiles(join(root, projectPath), Infinity, (w) => warnings.push(w)); + const ext = [...staged.keys()].filter((key) => key.startsWith("_ext/")); + return { root, warnings, ext }; + } + + it("leaves out a ../ reference into a hidden folder and names the file and reference", () => { + const { warnings, ext } = stageIn("p", { + ".ssh/id_ed25519": "KEY", + "p/index.html": "", + "p/compositions/intro.html": ``, + }); + expect(ext).toEqual([]); + expect(warnings).toEqual([ + "compositions/intro.html references ../../.ssh/id_ed25519, which is outside the project in a private location; it was left out of the upload.", + ]); + }); + + it("leaves out an absolute reference under a hidden folder", () => { + const { root, warnings, ext } = stageIn("p", { + ".aws/credentials": "KEY", + "p/index.html": ``, + }); + expect(ext).toEqual([]); + expect(warnings).toEqual([ + `index.html references ${root}/.aws/credentials, which is outside the project in a private location; it was left out of the upload.`, + ]); + }); + + it("leaves out an ordinary-looking outside file that links into a hidden folder", () => { + const root = temp(); + mkdirSync(join(root, ".aws")); + mkdirSync(join(root, "p")); + writeFileSync(join(root, ".aws", "credentials"), "KEY"); + symlinkSync(join(root, ".aws", "credentials"), join(root, "link.png")); + writeFileSync(join(root, "p", "index.html"), ``); + const warnings: string[] = []; + const staged = stagePublishFiles(join(root, "p"), Infinity, (w) => warnings.push(w)); + expect([...staged.keys()].some((key) => key.startsWith("_ext/"))).toBe(false); + expect(warnings).toEqual([expect.stringMatching(/^index\.html references \.\.\/link\.png,/)]); + }); + + it.skipIf(process.platform === "win32")("leaves out a file under /etc", () => { + const { warnings, ext } = stageIn("p", { "p/index.html": `` }); + expect(ext).toEqual([]); + expect(warnings).toHaveLength(1); + }); + + it("warns when it leaves out an env file outside the project", () => { + const { warnings, ext } = stageIn("p", { + "shared/.env.production": "SECRET=1", + "p/index.html": ``, + }); + expect(ext).toEqual([]); + expect(warnings).toEqual([ + expect.stringMatching(/^index\.html references \.\.\/shared\/\.env\.production,/), + ]); + }); + + it("still copies an ordinary outside file, without a warning", () => { + const { warnings, ext } = stageIn("p", { + "shared/logo.png": "png", + "p/index.html": ``, + }); + expect(ext).toEqual(["_ext/shared/logo.png"]); + expect(warnings).toEqual([]); + }); + + it("does not count hidden folders the project itself sits in", () => { + const { warnings, ext } = stageIn(".work/p", { + ".work/shared/logo.png": "png", + ".work/p/index.html": ``, + }); + expect(ext).toEqual(["_ext/shared/logo.png"]); + expect(warnings).toEqual([]); + }); + + it("keeps a referenced file under a hidden folder inside the project, without a warning", () => { + const root = temp(); + mkdirSync(join(root, ".media")); + writeFileSync(join(root, ".media", "clip.mp4"), "v"); + writeFileSync(join(root, "index.html"), ``); + const warnings: string[] = []; + const staged = stagePublishFiles(root, Infinity, (w) => warnings.push(w)); + expect(staged.has(".media/clip.mp4")).toBe(true); + expect(warnings).toEqual([]); + }); + + it("leaves out a project file that links to a private file outside the project", () => { + const root = temp(); + mkdirSync(join(root, ".ssh")); + mkdirSync(join(root, "p")); + writeFileSync(join(root, ".ssh", "id_ed25519"), "KEY"); + symlinkSync(join(root, ".ssh", "id_ed25519"), join(root, "p", "logo.png")); + writeFileSync(join(root, "p", "index.html"), ""); + const warnings: string[] = []; + const staged = stagePublishFiles(join(root, "p"), Infinity, (w) => warnings.push(w)); + expect([...staged.keys()]).toEqual(["index.html"]); + expect(warnings).toEqual([ + "logo.png links to a file outside the project in a private location; it was left out of the upload.", + ]); + }); + + it("leaves out a referenced project file that links to a private file outside the project", () => { + const root = temp(); + mkdirSync(join(root, ".ssh")); + mkdirSync(join(root, "p")); + writeFileSync(join(root, ".ssh", "id_ed25519"), "KEY"); + symlinkSync(join(root, ".ssh", "id_ed25519"), join(root, "p", "logo.png")); + writeFileSync(join(root, "p", "index.html"), ``); + const warnings: string[] = []; + const staged = stagePublishFiles(join(root, "p"), Infinity, (w) => warnings.push(w)); + expect([...staged.keys()]).toEqual(["index.html"]); + expect(warnings).toEqual([ + "logo.png links to a file outside the project in a private location; it was left out of the upload.", + ]); + }); + + it("still uploads a project file that links to an ordinary file outside the project", () => { + const root = temp(); + mkdirSync(join(root, "shared")); + mkdirSync(join(root, "p")); + writeFileSync(join(root, "shared", "logo.png"), "png"); + symlinkSync(join(root, "shared", "logo.png"), join(root, "p", "logo.png")); + writeFileSync(join(root, "p", "index.html"), ""); + const warnings: string[] = []; + const staged = stagePublishFiles(join(root, "p"), Infinity, (w) => warnings.push(w)); + expect([...staged.keys()].sort()).toEqual(["index.html", "logo.png"]); + expect(warnings).toEqual([]); + }); + + it("leaves out references into the home AppData and Library, but not iCloud Drive", () => { + const home = temp(); + vi.stubEnv("HOME", home); + vi.stubEnv("USERPROFILE", home); + try { + const files = [ + "AppData/Roaming/token.json", + "Library/Cookies/c.binarycookies", + "Library/Mobile Documents/shared/logo.png", + ]; + for (const file of files) { + mkdirSync(dirname(join(home, file)), { recursive: true }); + writeFileSync(join(home, file), "x"); + } + const project = temp(); + writeFileSync( + join(project, "index.html"), + files.map((file) => ``).join(""), + ); + const warnings: string[] = []; + const staged = stagePublishFiles(project, Infinity, (w) => warnings.push(w)); + const ext = [...staged.keys()].filter((key) => key.startsWith("_ext/")); + expect(ext).toEqual([expect.stringMatching(/Mobile Documents\/shared\/logo\.png$/)]); + expect(warnings).toHaveLength(2); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("still copies outside assets for a project that itself sits in the home AppData", () => { + const home = temp(); + vi.stubEnv("HOME", home); + vi.stubEnv("USERPROFILE", home); + try { + const temps = join(home, "AppData", "Local", "Temp"); + mkdirSync(join(temps, "shared"), { recursive: true }); + mkdirSync(join(temps, "p")); + writeFileSync(join(temps, "shared", "logo.png"), "png"); + writeFileSync(join(temps, "p", "index.html"), ``); + const warnings: string[] = []; + const staged = stagePublishFiles(join(temps, "p"), Infinity, (w) => warnings.push(w)); + expect([...staged.keys()].filter((key) => key.startsWith("_ext/"))).toEqual([ + "_ext/shared/logo.png", + ]); + expect(warnings).toEqual([]); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("still copies a sibling asset folder named library or AppData outside the home folder", () => { + const { ext, warnings } = stageIn("p", { + "library/logo.png": "png", + "AppData/clip.png": "png", + "p/index.html": ``, + }); + expect(ext.sort()).toEqual(["_ext/AppData/clip.png", "_ext/library/logo.png"]); + expect(warnings).toEqual([]); + }); + + it("says nothing about a reference into a hidden folder that does not exist", () => { + const { warnings } = stageIn("p", { "p/index.html": `x` }); + expect(warnings).toEqual([]); + }); +}); diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index f70d6e000d..a0f9242ed5 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -7,7 +7,6 @@ import { openSync, readdirSync, readFileSync, - realpathSync, type Stats, statSync, } from "node:fs"; @@ -23,7 +22,11 @@ import { isAuthError } from "../auth/errors.js"; import { envCredentialVar, isTokenExpired, type ResolvedCredential } from "../auth/resolver.js"; import { refreshIfNeeded } from "../cloud/auth.js"; import { writeProjectLink } from "./projectLink.js"; -import { isEnvOrGitPath, isPrivateCompositionFile } from "./privateProjectFile.js"; +import { + isPrivateCompositionFile, + isPrivateOutsideFile, + linksToPrivateOutsideFile, +} from "./privateProjectFile.js"; const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]); const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]); @@ -392,6 +395,8 @@ interface ExternalAssetContext { onDiskMinBytes: number; externalMap: Map; usedArchivePaths: Set; + leftOut: Set; + onLeftOut?: (warning: string) => void; } function addExternalAsset(ctx: ExternalAssetContext, absPath: string): string { @@ -418,7 +423,10 @@ function addExternalAsset(ctx: ExternalAssetContext, absPath: string): string { function staysOut(ctx: ExternalAssetContext, rel: string, absPath: string): boolean { if (!rel || ctx.fileContents.has(rel) || rel.split("/").some(shouldIgnoreSegment)) return true; - return isPrivateCompositionFile(ctx.absProjectDir, absPath); + return ( + isPrivateCompositionFile(ctx.absProjectDir, absPath) || + (existsSync(absPath) && linksToPrivateOutsideFile(ctx.absProjectDir, absPath)) + ); } /** Ships a referenced in-project file the ignore rules skipped, resolved from its folder or the root. */ @@ -450,11 +458,18 @@ const HARD_REFERENCES = new Set(["video[src]", "audio[src]", "img[src]", "video[ const relativeToProject = (ctx: ExternalAssetContext, absPath: string) => encodeUrlPath(relative(ctx.absProjectDir, absPath).replaceAll("\\", "/")); +function leaveOut(ctx: ExternalAssetContext, entryPath: string, rawPath: string): void { + const warning = `${entryPath} references ${rawPath}, which is outside the project in a private location; it was left out of the upload.`; + if (ctx.leftOut.has(warning)) return; + ctx.leftOut.add(warning); + ctx.onLeftOut?.(warning); +} + function archiveTargetOf( ctx: ExternalAssetContext, absPath: string, rawPath: string, - options: { requireOnDisk: boolean; filesystemPath: boolean }, + options: { entryPath: string; requireOnDisk: boolean; filesystemPath: boolean }, ): string | null | undefined { if (isPathInside(absPath, ctx.absProjectDir)) { includeReferencedProjectFile(ctx, absPath, rawPath, options.requireOnDisk); @@ -462,7 +477,10 @@ function archiveTargetOf( } try { if (!existsSync(absPath) || !statSync(absPath).isFile()) return undefined; - if (isEnvOrGitPath(absPath) || isEnvOrGitPath(realpathSync(absPath))) return undefined; + if (isPrivateOutsideFile(ctx.absProjectDir, absPath)) { + leaveOut(ctx, options.entryPath, rawPath); + return undefined; + } } catch { return undefined; } @@ -473,6 +491,7 @@ function tryResolveExternal( ctx: ExternalAssetContext, rawPath: string, referrerAbsDir: string, + entryPath: string, requireOnDisk = false, ): string | null { const filesystemPath = isAbsolute(rawPath) && !rawPath.startsWith("//"); @@ -481,6 +500,7 @@ function tryResolveExternal( const [path, suffix] = cut < 0 ? [rawPath, ""] : [rawPath.slice(0, cut), rawPath.slice(cut)]; for (const variant of decodeUrlPathVariants(path)) { const target = archiveTargetOf(ctx, resolve(referrerAbsDir, variant), rawPath, { + entryPath, requireOnDisk, filesystemPath, }); @@ -616,6 +636,7 @@ function localizeHtmlEntry(ctx: ExternalAssetContext, entryPath: string, content ctx, val, dir, + entryPath, ref !== undefined && HARD_REFERENCES.has(`${ref.tag}[${ref.attr}]`), ); const attrsChanged = rewriteHtmlAttributes(document, referrerAbsDir, entryPath, resolveTarget, { @@ -633,7 +654,7 @@ function localizeCssEntry(ctx: ExternalAssetContext, entryPath: string, content: const css = content.toString("utf-8"); if (!css.includes("url(")) return; const result = rewriteCssUrls(css, referrerAbsDir, entryPath, (val, dir) => - tryResolveExternal(ctx, val, dir), + tryResolveExternal(ctx, val, dir, entryPath), ); if (result.modified) { ctx.fileContents.set(entryPath, Buffer.from(result.css, "utf-8")); @@ -649,6 +670,7 @@ export function localizeExternalAssets( absProjectDir: string, fileContents: Map, onDiskMinBytes = Infinity, + onLeftOut?: (warning: string) => void, ): number { const ctx: ExternalAssetContext = { absProjectDir, @@ -656,6 +678,8 @@ export function localizeExternalAssets( onDiskMinBytes, externalMap: new Map(), usedArchivePaths: new Set(), + leftOut: new Set(), + onLeftOut, }; for (const [entryPath, content] of [...fileContents.entries()]) { @@ -677,8 +701,11 @@ export function localizeExternalAssets( * and `zipPublishFileMap` below. `cloud render` never sees this seam: it * keeps calling `createPublishArchive` directly. */ -export function buildPublishFileMap(projectDir: string): Map { - return stagePublishFiles(projectDir, Infinity) as Map; +export function buildPublishFileMap( + projectDir: string, + onLeftOut?: (warning: string) => void, +): Map { + return stagePublishFiles(projectDir, Infinity, onLeftOut) as Map; } export const archiveKey = (path: string): string => path.normalize("NFC"); @@ -705,6 +732,7 @@ function composeNamesInText(fileContents: Map, filePaths: st export function stagePublishFiles( projectDir: string, onDiskMinBytes: number, + onLeftOut?: (warning: string) => void, ): Map { const absProjectDir = resolve(projectDir); const filePaths: string[] = []; @@ -718,6 +746,12 @@ export function stagePublishFiles( const fileContents = new Map(); const diskPathOf = new Map(); for (const filePath of filePaths) { + if (linksToPrivateOutsideFile(absProjectDir, join(absProjectDir, filePath))) { + onLeftOut?.( + `${filePath} links to a file outside the project in a private location; it was left out of the upload.`, + ); + continue; + } const key = archiveKey(filePath); const clash = diskPathOf.get(key); if (clash !== undefined) { @@ -730,7 +764,7 @@ export function stagePublishFiles( } composeNamesInText(fileContents, filePaths); - localizeExternalAssets(absProjectDir, fileContents, onDiskMinBytes); + localizeExternalAssets(absProjectDir, fileContents, onDiskMinBytes, onLeftOut); return fileContents; } @@ -776,8 +810,11 @@ export function zipPublishFileMap(fileContents: Map): PublishArc * byte-identical (never see baked proxies); only `publish.ts` inserts a baking * transform between them. */ -export function createPublishArchive(projectDir: string): PublishArchiveResult { - return zipPublishFileMap(buildPublishFileMap(projectDir)); +export function createPublishArchive( + projectDir: string, + onLeftOut?: (warning: string) => void, +): PublishArchiveResult { + return zipPublishFileMap(buildPublishFileMap(projectDir, onLeftOut)); } export function getPublishApiBaseUrl(): string {