diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 744be6a89f..50a3185371 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -2731,11 +2731,15 @@ an MPEG transport stream, so TypeScript source is not copied. Any other outside file (a script, a Lottie `.json`, a `.pdf`, a file with no extension), one in a private location (env files, Git data, hidden, system and app-data folders), and one reached from a file that came in with an -imported project are left out, with a warning naming each. To include such a +imported project are left out, with a warning naming each. A reference to another +computer's network share or to a Windows device path (`\\server\share\clip.mp4`, +`\\?\…`, `\\.\…`) is left out the same way, and the command never opens it, because on +Windows even checking such a path sends that computer your sign-in. A file on the +same share as the project is copied like any other. To include such a file, move it into the project folder. Under `--json`, the result lists those as `left_out`, each with the `file` that points at it, the `reference` as written, its `path` from the project folder, the `reason` (`blocklisted`, -`received`, `file_type` or `private`) and `levels_up`, how many folders above the project +`received`, `file_type`, `private` or `network_share`) and `levels_up`, how many folders above the project it is. A full run also prints each as a `{"event":"left_out",…}` line on stderr, so a wrapper can ask with `--dry-run --json` before uploading and still hear about each one during the run. diff --git a/packages/cli/src/utils/leftOutReference.ts b/packages/cli/src/utils/leftOutReference.ts index 17ca76708f..efb72a6cfe 100644 --- a/packages/cli/src/utils/leftOutReference.ts +++ b/packages/cli/src/utils/leftOutReference.ts @@ -4,6 +4,7 @@ import { leftOutWarning as leftOutText, } from "@hyperframes/studio-server/outside-file"; import { realFilePath } from "@hyperframes/studio-server/safe-path"; +import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution"; import { terminalText } from "./terminalText.js"; export type { LeftOutReason }; @@ -32,7 +33,9 @@ export function outsideFileReference( reference: string, absPath: string, ): OutsideFileReference { - const path = relative(realFilePath(projectDir), realFilePath(absPath)).replaceAll("\\", "/"); + const onAnotherShare = namesAnotherNetworkRoot(projectDir, absPath); + const target = onAnotherShare ? absPath : realFilePath(absPath); + const path = relative(realFilePath(projectDir), target).replaceAll("\\", "/"); const levels_up = isAbsolute(path) ? null : path.split("/").filter((s) => s === "..").length; return { file, reference, path, levels_up }; } diff --git a/packages/cli/src/utils/publishProject.networkShare.win32.test.ts b/packages/cli/src/utils/publishProject.networkShare.win32.test.ts new file mode 100644 index 0000000000..f9f4823a11 --- /dev/null +++ b/packages/cli/src/utils/publishProject.networkShare.win32.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, it, vi } from "vitest"; + +// Windows path rules on any host, and a disk that records every path it is asked about. +const disk = vi.hoisted(() => ({ + files: new Map(), + touched: [] as { op: string; path: string }[], + open: new Map(), +})); +vi.mock("node:path", async () => { + const path = await vi.importActual("node:path"); + return { ...path.win32, default: path.win32 }; +}); +vi.mock("node:fs", async () => { + const fs = await vi.importActual("node:fs"); + const missing = (path: string) => Object.assign(new Error(`ENOENT: ${path}`), { code: "ENOENT" }); + const look = (op: string, path: string) => { + disk.touched.push({ op, path }); + return disk.files.get(path.toLowerCase()); + }; + const found = (op: string, path: string) => { + const file = look(op, path); + if (file === undefined) throw missing(path); + return file; + }; + const inode = (path: string) => [...disk.files.keys()].indexOf(path.toLowerCase()) + 1; + const fileStat = (text: string, path: string) => ({ + isFile: () => true, + isDirectory: () => false, + isSymbolicLink: () => false, + size: text.length, + mtimeMs: 0, + dev: 1n, + ino: BigInt(inode(path)), + }); + const statSync = (path: string, options?: { throwIfNoEntry?: boolean }) => { + const file = look("stat", path); + if (file === undefined && options?.throwIfNoEntry === false) return undefined; + if (file === undefined) throw missing(path); + return fileStat(file.text, path); + }; + const lstatSync = (path: string, options?: { throwIfNoEntry?: boolean }) => { + const file = look("lstat", path); + if (file === undefined && options?.throwIfNoEntry === false) return undefined; + if (file === undefined) throw missing(path); + return fileStat(file.text, path); + }; + const existsSync = (path: string) => { + disk.touched.push({ op: "exists", path }); + const folder = `${path.toLowerCase()}\\`; + const keys = [...disk.files.keys()]; + return disk.files.has(path.toLowerCase()) || keys.some((key) => key.startsWith(folder)); + }; + const realpath = (path: string) => (disk.touched.push({ op: "realpath", path }), path); + const realpathSync = Object.assign(realpath, { native: realpath }); + const readdirSync = (dir: string) => { + disk.touched.push({ op: "readdir", path: dir }); + const names = new Map(); + for (const { path } of disk.files.values()) { + if (!path.toLowerCase().startsWith(`${dir.toLowerCase()}\\`)) continue; + const [name, ...rest] = path.slice(dir.length + 1).split("\\"); + names.set(name!, rest.length > 0); + } + return [...names].map(([name, folder]) => ({ + name, + isDirectory: () => folder, + isFile: () => !folder, + })); + }; + const openSync = (path: string) => { + found("open", path); + const fd = disk.open.size + 100; + disk.open.set(fd, path); + return fd; + }; + const textOf = (from: number | string) => + typeof from === "number" + ? disk.files.get(disk.open.get(from)!.toLowerCase())!.text + : found("read", from).text; + const readFileSync = (from: number | string, encoding?: unknown) => + encoding ? textOf(from) : Buffer.from(textOf(from)); + const fstatSync = (fd: number) => fileStat(textOf(fd), disk.open.get(fd)!); + const readSync = (fd: number, buffer: Buffer, offset: number, length: number) => + Buffer.from(textOf(fd)).copy(buffer, offset, 0, length); + const closeSync = (fd: number) => disk.open.delete(fd); + const overrides = { + statSync, + lstatSync, + existsSync, + realpathSync, + readdirSync, + openSync, + readFileSync, + fstatSync, + readSync, + closeSync, + }; + return { ...fs, ...overrides, default: { ...fs, ...overrides } }; +}); + +const { buildPublishFileMap, createPublishArchive, stagePublishFiles } = + await import("./publishProject.js"); + +type OnLeftOut = Parameters[2]; + +// Each spelling of another computer's share or a device path that a page might hold. +const SHARES = [ + "\\\\host\\share\\a.png", + "/\\host\\share\\b.png", + "\\/host/share/c.png", + "\\\\?\\UNC\\host\\share\\d.png", + "\\\\.\\pipe\\e", + "\\\\.\\UNC\\host\\share\\j.png", + "\\\\?\\D:\\media\\f.png", + "%5C%5Chost%5Cshare%5Ch.png", + "%2F%5Chost%5Cshare%5Ck.png", + "\\\\host\\share\\i.png?v=2", +]; +const onAShare = (path: string) => /^[\\/]{2}/.test(path); + +function setUp(project: string, files: Record) { + disk.files.clear(); + disk.touched.length = 0; + disk.open.clear(); + for (const [path, text] of Object.entries(files)) { + const full = path.includes(":") || onAShare(path) ? path : `${project}\\${path}`; + disk.files.set(full.toLowerCase(), { path: full, text }); + } + for (const share of ["\\\\host\\share\\a.png", "\\\\host\\share\\bg.png"]) { + disk.files.set(share.toLowerCase(), { path: share, text: "png" }); + } +} + +// `publish` stages through buildPublishFileMap; `cloud render`, with or without --dry-run, calls +// stagePublishFiles with a size above which files stay on disk and the entry page. +const UPLOADS = { + publish: (project: string, onLeftOut: OnLeftOut) => buildPublishFileMap(project, onLeftOut), + "cloud render": (project: string, onLeftOut: OnLeftOut) => + stagePublishFiles(project, 1, onLeftOut, undefined, "index.html"), + "the published archive": (project: string, onLeftOut: OnLeftOut) => { + createPublishArchive(project, onLeftOut); + return buildPublishFileMap(project); + }, +}; + +function stage(upload: keyof typeof UPLOADS, project: string) { + const leftOut: { reference: string; reason: string }[] = []; + const files = UPLOADS[upload](project, (_warning, { reference, reason }) => + leftOut.push({ reference, reason }), + ); + return { files, leftOut }; +} + +const PROJECT = "C:\\project"; +const PAGE = + `` + + SHARES.map((src) => ``).join("") + + `` + + ``; +const LOCAL = { + "index.html": PAGE, + "style.css": `body { background: url(\\\\host\\share\\bg.png) }`, + "D:\\media\\pic.png": "png", + "C:\\outside\\near.png": "png", +}; + +describe.each(Object.keys(UPLOADS) as (keyof typeof UPLOADS)[])( + "%s under Windows rules", + (upload) => { + it("never looks at another computer's share or a device path, however it is written", () => { + setUp(PROJECT, LOCAL); + stage(upload, PROJECT); + expect(disk.touched.filter(({ path }) => onAShare(path))).toEqual([]); + }); + + it("reports each of those references as left out", () => { + setUp(PROJECT, LOCAL); + const { leftOut } = stage(upload, PROJECT); + const shared = leftOut.filter(({ reason }) => reason === "network_share"); + expect(shared.map(({ reference }) => reference).sort()).toEqual( + [...SHARES, "\\\\host\\share\\s.png", "\\\\host\\share\\bg.png"].sort(), + ); + }); + + it("still uploads local files outside the project, on this drive or another", () => { + setUp(PROJECT, LOCAL); + const { files } = stage(upload, PROJECT); + const ext = [...files.keys()].filter((key) => key.startsWith("_ext/")); + expect(ext.some((key) => key.endsWith("pic.png"))).toBe(true); + expect(ext.some((key) => key.endsWith("near.png"))).toBe(true); + }); + + it("still uploads a file on the share the project itself lives on", () => { + const project = "\\\\nas\\films\\project"; + setUp(project, { + "index.html": ``, + "\\\\nas\\films\\shared\\pic.png": "png", + }); + const { files, leftOut } = stage(upload, project); + expect(leftOut).toEqual([]); + expect([...files.keys()].some((key) => key.endsWith("pic.png"))).toBe(true); + }); + }, +); + +describe("a page that came in with an imported project", () => { + it("has its outside references left out before any of them is opened", () => { + setUp(PROJECT, { + ...LOCAL, + ".hyperframes\\received-files.json": JSON.stringify({ files: ["index.html"] }), + }); + const { files, leftOut } = stage("cloud render", PROJECT); + expect(leftOut).toContainEqual({ reference: "../outside/near.png", reason: "received" }); + expect(leftOut).toContainEqual({ + reference: "\\\\host\\share\\a.png", + reason: "network_share", + }); + expect([...files.keys()].filter((key) => key.startsWith("_ext/"))).toEqual([]); + const probes = disk.touched.filter(({ op }) => op !== "realpath"); + expect(probes.filter(({ path }) => /outside|media/i.test(path))).toEqual([]); + expect(disk.touched.filter(({ path }) => onAShare(path))).toEqual([]); + }); +}); diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index 1eb52ce0dc..61eed12285 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -44,6 +44,7 @@ import { decodeTextHonoringBom, resolveCompositionSrc, } from "@hyperframes/parsers/asset-resolution"; +import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution"; const IGNORED_DIRS = new Set([".git", "node_modules", "dist", ".next", "coverage"]); const IGNORED_FILES = new Set([".DS_Store", "Thumbs.db"]); @@ -516,28 +517,45 @@ function leaveOut( ); } +function leftOnAnotherNetworkRoot( + ctx: ExternalAssetContext, + entryPath: string, + rawPath: string, + absPath: string, +): boolean { + if (!namesAnotherNetworkRoot(ctx.absProjectDir, absPath)) return false; + leaveOut(ctx, entryPath, rawPath, absPath, "network_share"); + return true; +} + +function outsideFileLeftOutReason( + ctx: ExternalAssetContext, + absPath: string, +): LeftOutReason | null { + if (isPrivateOutsideFile(ctx.absProjectDir, absPath)) return "blocklisted"; + return isUploadedOutsideFileType(absPath) ? null : "file_type"; +} + function archiveTargetOf( ctx: ExternalAssetContext, absPath: string, rawPath: string, options: { entryPath: string; requireOnDisk: boolean; filesystemPath: boolean }, ): string | null | undefined { + if (leftOnAnotherNetworkRoot(ctx, options.entryPath, rawPath, absPath)) return null; if (isPathInside(absPath, ctx.absProjectDir)) { includeReferencedProjectFile(ctx, absPath, rawPath, options.requireOnDisk, options.entryPath); return options.filesystemPath ? relativeToProject(ctx, absPath) : null; } + if (ctx.received(options.entryPath)) { + leaveOut(ctx, options.entryPath, rawPath, absPath, "received"); + return undefined; + } try { if (!existsSync(absPath) || !statSync(absPath).isFile()) return undefined; - if (ctx.received(options.entryPath)) { - leaveOut(ctx, options.entryPath, rawPath, absPath, "received"); - return undefined; - } - if (isPrivateOutsideFile(ctx.absProjectDir, absPath)) { - leaveOut(ctx, options.entryPath, rawPath, absPath); - return undefined; - } - if (!isUploadedOutsideFileType(absPath)) { - leaveOut(ctx, options.entryPath, rawPath, absPath, "file_type"); + const reason = outsideFileLeftOutReason(ctx, absPath); + if (reason) { + leaveOut(ctx, options.entryPath, rawPath, absPath, reason); return undefined; } } catch { @@ -546,6 +564,12 @@ function archiveTargetOf( return encodeUrlPath(addExternalAsset(ctx, absPath, options.entryPath, rawPath)); } +// The ? of a \\?\ device path is part of the path, not a query. +function splitPathSuffix(rawPath: string): [string, string] { + const cut = rawPath.search(/(?, filePaths: st } } -type WalkedLeftOutReason = Exclude; +type WalkedLeftOutReason = Exclude; const WALKED_LEFT_OUT: Record = { private: "holds secrets or tool data (an env file, Git data or Studio records)", diff --git a/packages/studio-server/src/helpers/outsideFile.ts b/packages/studio-server/src/helpers/outsideFile.ts index 2f6726586a..c639c13158 100644 --- a/packages/studio-server/src/helpers/outsideFile.ts +++ b/packages/studio-server/src/helpers/outsideFile.ts @@ -146,13 +146,14 @@ export function isPrivateOutsideFile(projectDir: string, absPath: string): boole export const linksOutside = (projectDir: string, filePath: string): boolean => isOutside(relative(realFilePath(projectDir), realFilePath(filePath))); -/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */ -export type LeftOutReason = "blocklisted" | "received" | "file_type" | "private"; +/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy; `network_share`: on another computer. */ +export type LeftOutReason = "blocklisted" | "received" | "file_type" | "private" | "network_share"; const WHY: Record, string> = { private: "which holds secrets or tool data", blocklisted: "which is outside the project in a private location", received: "which leads outside the project and came in with an imported project", + network_share: "which is on another computer's network share or is a Windows device path", }; const COPIED_TYPES = {