diff --git a/packages/cli/src/cloud/offload.ts b/packages/cli/src/cloud/offload.ts index a655cb503b..cb9952135d 100644 --- a/packages/cli/src/cloud/offload.ts +++ b/packages/cli/src/cloud/offload.ts @@ -33,7 +33,11 @@ import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg import { probeAudio } from "../audio/analysis/decode.js"; import { formatBytes } from "../ui/format.js"; import { bytesMatchExtension } from "./offloadSniff.js"; -import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "./mediaExtensions.js"; +import { + AUDIO_EXTS, + CONVERTIBLE_IMAGE_EXTS, + VIDEO_EXTS, +} from "@hyperframes/studio-server/outside-file"; import { type PlayedSegment, planTrim } from "./offloadTrim.js"; import { archiveKey, diff --git a/packages/cli/src/utils/leftOutReference.ts b/packages/cli/src/utils/leftOutReference.ts index efe7835dc1..47ff0765d6 100644 --- a/packages/cli/src/utils/leftOutReference.ts +++ b/packages/cli/src/utils/leftOutReference.ts @@ -1,8 +1,8 @@ import { isAbsolute, relative } from "node:path"; +import { type LeftOutReason, leftOutWarning } from "@hyperframes/studio-server/outside-file"; import { realFilePath } from "@hyperframes/studio-server/safe-path"; -/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */ -export type LeftOutReason = "blocklisted" | "received" | "file_type"; +export { type LeftOutReason, leftOutWarning }; /** One file an upload left out. `path` is from the project folder; `levels_up` is null on another drive. */ export interface LeftOutReference { @@ -15,17 +15,6 @@ export interface LeftOutReference { export type OnLeftOut = (warning: string, reference: LeftOutReference) => void; -const WHY: Record = { - blocklisted: "which is outside the project in a private location", - received: "which leads outside the project and came in with an imported project", - file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file", -}; - -export function leftOutWarning(file: string, reference: string, reason: LeftOutReason): string { - const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the upload.`; - return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning; -} - export function leftOutReference( projectDir: string, file: string, diff --git a/packages/cli/src/utils/outsideFileType.ts b/packages/cli/src/utils/outsideFileType.ts index b4f4123e9d..ba4f8e20ca 100644 --- a/packages/cli/src/utils/outsideFileType.ts +++ b/packages/cli/src/utils/outsideFileType.ts @@ -1,22 +1,4 @@ -import { extname } from "node:path"; -import { MIME_TYPES } from "@hyperframes/studio-server/mime"; -import { realFilePath } from "@hyperframes/studio-server/safe-path"; -import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "../cloud/mediaExtensions.js"; - -const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/; - -/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types, - * and the media cloud render re-encodes or converts. */ -export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([ - ...Object.entries(MIME_TYPES) - .filter(([, type]) => COPIED_TYPE.test(type)) - .map(([extension]) => extension), - ...VIDEO_EXTS, - ...AUDIO_EXTS, - ...CONVERTIBLE_IMAGE_EXTS, -]); - -const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase()); - -export const isCopiedOutsideFileType = (absPath: string): boolean => - hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath)); +export { + isCopiedOutsideFileType, + OUTSIDE_FILE_EXTENSIONS, +} from "@hyperframes/studio-server/outside-file"; diff --git a/packages/cli/src/utils/privateProjectFile.ts b/packages/cli/src/utils/privateProjectFile.ts index a7131da848..0059aa4ac7 100644 --- a/packages/cli/src/utils/privateProjectFile.ts +++ b/packages/cli/src/utils/privateProjectFile.ts @@ -1,66 +1,13 @@ import { readFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { dirname, isAbsolute, join, relative, sep } from "node:path"; +import { join, relative } from "node:path"; +import { + isEnvOrGitPath, + isPrivateOutsideFile, + linksOutside, +} from "@hyperframes/studio-server/outside-file"; import { isPrivateProjectFile, realFilePath } from "@hyperframes/studio-server/safe-path"; -const ENV_FILE = /^\.env([._-].*|rc)?$/; - -export function isEnvOrGitPath(path: string): boolean { - const segments = path.toLowerCase().split(/[\\/]/); - return segments.includes(".git") || ENV_FILE.test(segments.at(-1) ?? ""); -} - -const SYSTEM_DIRS = ["/etc", "/proc", "/sys", "/dev", "/private/etc"]; - -const isOutside = (fromDir: string) => - fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir); - -function inPrivateLocation(projectDir: string, path: string): boolean { - const posixPath = path.replaceAll("\\", "/"); - if (SYSTEM_DIRS.some((dir) => posixPath === dir || posixPath.startsWith(`${dir}/`))) return true; - const fromParent = relative(dirname(projectDir), path); - return ( - (isOutside(fromParent) ? path : fromParent).split(/[\\/]/).some((s) => s.startsWith(".")) || - inHomeAppData(projectDir, path) - ); -} - -function isUnder(path: string, ...dir: string[]): boolean { - const lowerPath = path.toLowerCase(); - const lowerDir = join(homedir(), ...dir).toLowerCase(); - return lowerPath === lowerDir || lowerPath.startsWith(`${lowerDir}${sep}`); -} - -const CLOUD_DRIVE_FOLDERS = ["Mobile Documents", "CloudStorage"]; - -function homeAppDataFolder(path: string): string | null { - if (isUnder(path, "AppData")) return "AppData"; - const cloudDrive = CLOUD_DRIVE_FOLDERS.some((folder) => isUnder(path, "Library", folder)); - if (isUnder(path, "Library") && !cloudDrive) return "Library"; - return null; -} - -function inHomeAppData(projectDir: string, path: string): boolean { - const folder = homeAppDataFolder(path); - const parent = dirname(projectDir); - return ( - folder !== null && (homeAppDataFolder(parent) !== folder || isOutside(relative(parent, path))) - ); -} - -/** An existing file outside the project that publish leaves out: env files, Git data, hidden, system and app-data folders. */ -export function isPrivateOutsideFile(projectDir: string, absPath: string): boolean { - const realPath = realFilePath(absPath); - return ( - isEnvOrGitPath(absPath) || - isEnvOrGitPath(realPath) || - inPrivateLocation(projectDir, absPath) || - inPrivateLocation(realFilePath(projectDir), realPath) - ); -} - -export const linksOutside = (projectDir: string, filePath: string): boolean => - isOutside(relative(realFilePath(projectDir), realFilePath(filePath))); +export { isEnvOrGitPath, isPrivateOutsideFile, linksOutside }; export function linksToPrivateOutsideFile(projectDir: string, filePath: string): boolean { return linksOutside(projectDir, filePath) && isPrivateOutsideFile(projectDir, filePath); diff --git a/packages/producer/src/services/animatedGifPrep.ts b/packages/producer/src/services/animatedGifPrep.ts index 7bce6639a6..6c5583bcc0 100644 --- a/packages/producer/src/services/animatedGifPrep.ts +++ b/packages/producer/src/services/animatedGifPrep.ts @@ -7,7 +7,7 @@ import { parseAnimatedGifMetadata, type AnimatedGifMetadata } from "@hyperframes import { DEFAULT_VP9_CPU_USED, runFfmpeg, renameWithRetry } from "@hyperframes/engine"; import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution"; import { isHttpUrl } from "../utils/urlDownloader.js"; -import { escapesAssetRoot } from "../utils/paths.js"; +import { escapesAssetRoot, outsideAssetRefusal } from "../utils/paths.js"; import { encoderFailureError } from "./render/encoderInterruption.js"; const PREPARED_GIF_SUBDIR = "_animated_gif"; @@ -109,7 +109,8 @@ export function resolveLocalImageSourcePath( ]; const allowed = (candidate: string) => - !escapesAssetRoot(candidate, options.assetRoot) || + (!escapesAssetRoot(candidate, options.assetRoot) && + outsideAssetRefusal(options.projectDir, candidate) !== "blocklisted") || !escapesAssetRoot(candidate, options.downloadDir); return candidates.find((candidate) => existsSync(candidate) && allowed(candidate)) ?? null; } diff --git a/packages/producer/src/services/assetMediaType.ts b/packages/producer/src/services/assetMediaType.ts index 7073ff09b1..5428149749 100644 --- a/packages/producer/src/services/assetMediaType.ts +++ b/packages/producer/src/services/assetMediaType.ts @@ -12,7 +12,7 @@ import { type VideoElement, } from "@hyperframes/engine"; import { withMediaProbeSlot } from "../utils/mediaProbeConcurrency.js"; -import { escapesAssetRoot } from "../utils/paths.js"; +import { escapesAssetRoot, leftOutOfRender, outsideAssetRefusal } from "../utils/paths.js"; import type { ProducerLogger } from "../logger.js"; export const ASSET_MEDIA_TYPE_MISMATCH = "ASSET_MEDIA_TYPE_MISMATCH" as const; @@ -97,6 +97,7 @@ export function dropMediaOutsideProject(input: { workDir: string; composition: { videos: VideoElement[]; audios: AudioElement[]; images: ImageElement[] }; log: ProducerLogger; + allowOutsideProjectAssets?: boolean; }): void { const { projectDir, workDir, composition, log } = input; const compiledDir = join(workDir, "compiled"); @@ -105,6 +106,11 @@ export function dropMediaOutsideProject(input: { if (!src || isRemoteOrInlineSource(src)) return true; const path = resolveProjectRelativeSrc(src, projectDir, compiledDir); if (roots.some((root) => !escapesAssetRoot(path, root))) return true; + if (input.allowOutsideProjectAssets) { + const refusal = outsideAssetRefusal(projectDir, path); + if (refusal) log.warn(`[Render] ${leftOutOfRender(`Media "${id}"`, src, refusal)}`); + return refusal === null; + } log.warn(`[Render] Media "${id}" source is missing: it is outside the project folder`, { src }); return false; }; diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 75fda52e57..db85d37f41 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -64,6 +64,7 @@ import { declareProbeFactsUrl, } from "@hyperframes/engine"; import { defaultLogger, type ProducerLogger } from "../../logger.js"; +import { refusesOutsideLink } from "../../utils/paths.js"; import { applyRenderWarningPolicy, type RenderJob, @@ -1277,7 +1278,9 @@ async function compilePlanComposition( // false-positive every descendant. const rel = relative(projectDir, src); if (rel === "" || rel.startsWith("..")) return true; - if (!job.config.allowOutsideProjectAssets && !isSafePath(projectDir, src)) return false; + if (job.config.allowOutsideProjectAssets) { + if (refusesOutsideLink(projectDir, src, (line) => log.warn(`[Plan] ${line}`))) return false; + } else if (!isSafePath(projectDir, src)) return false; const firstSegment = rel.split(sep, 1)[0]; return firstSegment === undefined || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment); }, diff --git a/packages/producer/src/services/fileServer.ts b/packages/producer/src/services/fileServer.ts index bac1d66f1f..2ba63b715c 100644 --- a/packages/producer/src/services/fileServer.ts +++ b/packages/producer/src/services/fileServer.ts @@ -34,6 +34,7 @@ import { openRangeEnd } from "@hyperframes/studio-server/media-range"; import { getVerifiedHyperframeRuntimeSource } from "./hyperframeRuntimeLoader.js"; import { getHfEarlyStub } from "../generated/hf-early-stub-inline.js"; import { defaultLogger, type ProducerLogger } from "../logger.js"; +import { refusesOutsideLink } from "../utils/paths.js"; const readFileAsync = promisify(readFile); @@ -812,6 +813,7 @@ export function createFileServer(options: FileServerOptions): Promise console.warn(`[FileServer] ${line}`); app.get("/*", async (c) => { let requestPath = c.req.path; @@ -840,7 +842,10 @@ export function createFileServer(options: FileServerOptions): Promise { export function collectExternalAssets( html: string, projectDir: string, + log: ProducerLogger = defaultLogger, + file = "index.html", ): { html: string; externalAssets: Map } { const absProjectDir = resolve(projectDir); const externalAssets = new Map(); + const leftOut = new Set(); function processPath(rawPath: string): string | null { const trimmed = rawPath.trim(); @@ -1415,6 +1429,12 @@ export function collectExternalAssets( return null; // inside projectDir, file server handles this } if (!existsSync(absPath)) return null; + const refusal = outsideAssetRefusal(absProjectDir, absPath); + if (refusal) { + if (!leftOut.has(trimmed)) log.warn(`[Compiler] ${leftOutOfRender(file, trimmed, refusal)}`); + leftOut.add(trimmed); + return null; + } // resolve() already canonicalises the path (no .. components remain); // toExternalAssetKey() produces a cross-platform relative key that // `path.join(compileDir, key)` cannot escape on any OS. @@ -1957,6 +1977,7 @@ async function embedLocalFontFaces( html: string, projectDir: string, assetRoot: string | undefined, + log: ProducerLogger = defaultLogger, ): Promise { const { fontToDataUri: toDataUri } = await import("@hyperframes/core/fonts/embed"); const styleBlockRe = /]*>([\s\S]*?)<\/style>/gi; @@ -1980,6 +2001,11 @@ async function embedLocalFontFaces( if (!localPath || embeddedPaths.has(localPath)) continue; const absPath = localPath.startsWith("/") ? localPath : resolve(projectDir, localPath); if (!isPathInside(absPath, projectDir) || escapesAssetRoot(absPath, assetRoot)) continue; + // A project font that is a link leading outside obeys the same policy as the file server. + if (refusesOutsideLink(projectDir, absPath, (line) => log.warn(`[Compiler] ${line}`))) { + embeddedPaths.add(localPath); + continue; + } const ext = absPath.match(/\.(woff2?|ttf|otf|ttc)$/i)?.[1]?.toLowerCase() ?? "ttf"; try { if (fileBackedAbsolutePaths.has(absPath)) { @@ -2166,6 +2192,11 @@ function bindVariableMediaSources(html: string, variables: Record + isPathInside(htmlPath, projectDir) + ? relative(projectDir, htmlPath).split(sep).join("/") + : basename(htmlPath); + /** * Compile an HTML composition project into a single self-contained HTML string * with all media metadata resolved. @@ -2356,13 +2387,23 @@ export async function compileForRender( defaultLogger.debug(`[Compiler] Prepared ${preparedGifs.length} animated GIF input(s) as WebM`); } - const embeddedHtml = await embedLocalFontFaces(htmlWithPreparedGifs, projectDir, assetRoot); + const embeddedHtml = await embedLocalFontFaces( + htmlWithPreparedGifs, + projectDir, + assetRoot, + options.log, + ); // Collect assets that resolve outside projectDir (e.g. ../shared-assets/hero.png). // These can't be served by the file server, so we map them to paths the // orchestrator will copy into the compiled output directory. const { html, externalAssets } = options.allowOutsideProjectAssets - ? collectExternalAssets(embeddedHtml, projectDir) + ? collectExternalAssets( + embeddedHtml, + projectDir, + options.log, + entryFileName(projectDir, htmlPath), + ) : { html: embeddedHtml, externalAssets: new Map() }; for (const [relPath, absPath] of remoteMediaAssets) { diff --git a/packages/producer/src/services/outsideProjectAssets.test.ts b/packages/producer/src/services/outsideProjectAssets.test.ts index 77bb12bf90..484701da35 100644 --- a/packages/producer/src/services/outsideProjectAssets.test.ts +++ b/packages/producer/src/services/outsideProjectAssets.test.ts @@ -7,12 +7,13 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, rmSync, symlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join, relative, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { resolveConfig } from "@hyperframes/engine"; import { runCompileStage } from "./render/stages/compileStage.js"; @@ -163,6 +164,54 @@ async function compileInProcess(projectDir: string, allowOutsideProjectAssets?: return { result, compiledDir, extract, serve, warnings }; } +function privateOutsideProject() { + const root = mkdtempSync(join(tmpdir(), "hf-private-outside-")); + roots.push(root); + const projectDir = join(root, "work", "project"); + const [ssh, shared] = [join(root, ".ssh"), join(root, "work", "shared")]; + for (const dir of [projectDir, ssh, shared]) mkdirSync(dir, { recursive: true }); + writeFileSync(join(ssh, "id_key"), "PRIVATE"); + writeFileSync(join(ssh, "pic.png"), PNG_1X1); + copyFileSync(tool.clip, join(ssh, "clip.mp4")); + const gif = ["-v", "error", "-f", "lavfi", "-i", "testsrc=size=16x16:rate=10", "-t", "0.3"]; + expect(spawnSync("ffmpeg", [...gif, join(ssh, "anim.gif")]).status).toBe(0); + writeFileSync(join(ssh, "scene.html"), scene); + writeFileSync(join(shared, "logo.png"), PNG_1X1); + writeFileSync(join(shared, "notes.txt"), "notes"); + symlinkSync(join(ssh, "pic.png"), join(projectDir, "link.png")); + symlinkSync(join(shared, "logo.png"), join(projectDir, "keep.png")); + symlinkSync(join(ssh, "id_key"), join(projectDir, "brand.ttf")); + symlinkSync(join(shared, "notes.txt"), join(projectDir, "notes.ttf")); + const pic = join(ssh, "pic.png"); + const video = (id: string, src: string) => + ``; + writeFileSync( + join(projectDir, "index.html"), + ` + +
+ +
+ keynotes + ${video("dotdot", "../../.ssh/clip.mp4")} + ${video("fileurl", pathToFileURL(join(ssh, "clip.mp4")).href)} +
`, + ); + const key = (path: string) => toExternalAssetKey(resolve(path)); + return { + root, + projectDir, + privateKeys: [key(pic), key(join(ssh, "id_key"))], + logo: key(join(shared, "logo.png")), + }; +} + +const filesUnder = (dir: string): string[] => + readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((entry) => !entry.isDirectory()) + .map((entry) => relative(dir, join(entry.parentPath, entry.name)).replaceAll("\\", "/")); + const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); const config = { fps: 30 as const, width: 320, height: 240, format: "mp4" as const }; @@ -225,6 +274,71 @@ describe.skipIf(!HAS_FFMPEG)("render servers keep markup inside the project", () }, ); + it("trusted local renders leave out private and non-media files outside the project", async () => { + const { root, projectDir, privateKeys, logo } = privateOutsideProject(); + tool.calls.length = 0; + const { result, compiledDir, serve, warnings } = await compileInProcess(projectDir, true); + + expect(filesUnder(compiledDir).sort()).toEqual([logo, "index.html"]); + expect(result.composition.videos).toEqual([]); + expect(tool.calls.filter((call) => call.includes(".ssh") && !call.includes("ffprobe"))).toEqual( + [], + ); + const warned = warnings.join("\n"); + const references = ["id_key", "pic.png", "clip.mp4", "anim.gif"].map((f) => `../../.ssh/${f}`); + for (const reference of references) { + expect(warned).toContain( + `index.html references ${reference}, which is outside the project in a private location; it was left out of the render.`, + ); + } + expect(warned).toMatch( + /Media "fileurl" references file:.*\.ssh\/clip\.mp4, which is outside the project in a private/, + ); + expect(warned).toContain( + "../shared/notes.txt, which is outside the project and is not an image", + ); + expect(warned).toMatch( + /brand\.ttf references .*\.ssh\/id_key, which is outside the project in a private/, + ); + expect(warned).toMatch( + /notes\.ttf references .*notes\.txt, which is outside the project and is not an image/, + ); + const compiledHtml = readFileSync(join(compiledDir, "index.html"), "utf-8"); + expect(compiledHtml).not.toContain(Buffer.from("PRIVATE").toString("base64")); + expect(compiledHtml).not.toContain(Buffer.from("notes").toString("base64")); + expect(compiledHtml).toContain('url("brand.ttf")'); + expect( + await statusOf(await serve(), [ + ...privateKeys.map((k) => `/${k}`), + "/link.png", + `/${logo}`, + "/keep.png", + "/brand.ttf", + ]), + ).toEqual([404, 404, 404, 200, 200, 404]); + + const planDir = join(root, "plan"); + await plan(projectDir, config, planDir, { allowOutsideProjectAssets: true }); + const planned = filesUnder(join(planDir, "compiled")); + expect(planned.filter((file) => /\.ssh|notes|link\.png|brand\.ttf/.test(file))).toEqual([]); + expect(planned).toEqual(expect.arrayContaining([logo, "keep.png"])); + }); + + it("trusted local renders refuse a linked composition in a private location", async () => { + const { root, projectDir } = privateOutsideProject(); + mkdirSync(join(projectDir, "scenes")); + symlinkSync(join(root, ".ssh", "scene.html"), join(projectDir, "scenes", "a.html")); + const host = `
`; + const html = readFileSync(join(projectDir, "index.html"), "utf-8").replace( + "", + `${host}`, + ); + writeFileSync(join(projectDir, "index.html"), html); + await expect(compileInProcess(projectDir, true)).rejects.toThrow( + /outside the project in a private location/, + ); + }); + it("trusted local renders still import outside files", async () => { const { root, projectDir, secretKey } = outsideProject(false); const { compiledDir, serve } = await compileInProcess(projectDir, true); diff --git a/packages/producer/src/services/render/stages/compileStage.ts b/packages/producer/src/services/render/stages/compileStage.ts index f34afbfc97..c4f6fe1818 100644 --- a/packages/producer/src/services/render/stages/compileStage.ts +++ b/packages/producer/src/services/render/stages/compileStage.ts @@ -377,9 +377,13 @@ export async function runCompileStage(input: CompileStageInput): Promise + leftOutWarning(file, reference, reason, "render"); + +/** True, after a warning, when a trusted local render must not serve or copy `path`, a project file that leads outside. */ +export function refusesOutsideLink(projectDir: string, path: string, warn: (line: string) => void) { + const refusal = outsideAssetRefusal(projectDir, path); + if (!refusal) return false; + const file = nodeRelative(projectDir, path).split(sep).join("/"); + warn(leftOutOfRender(file, realFilePath(path), refusal)); + return true; +} + /** * Build a safe, cross-platform relative key for an absolute asset path * that lives outside the project directory. diff --git a/packages/studio-server/package-subpaths.json b/packages/studio-server/package-subpaths.json index b2e2f91fc9..668643255c 100644 --- a/packages/studio-server/package-subpaths.json +++ b/packages/studio-server/package-subpaths.json @@ -110,6 +110,12 @@ "types": "./dist/helpers/mime.d.ts", "environments": ["bun", "node"] }, + "./outside-file": { + "source": "./src/helpers/outsideFile.ts", + "runtime": "./dist/helpers/outsideFile.js", + "types": "./dist/helpers/outsideFile.d.ts", + "environments": ["bun", "node"] + }, "./media-range": { "source": "./src/helpers/mediaRange.ts", "runtime": "./dist/helpers/mediaRange.js", diff --git a/packages/studio-server/package.json b/packages/studio-server/package.json index 9777ab8c34..8c829b63c7 100644 --- a/packages/studio-server/package.json +++ b/packages/studio-server/package.json @@ -117,6 +117,12 @@ "import": "./src/helpers/mime.ts", "types": "./src/helpers/mime.ts" }, + "./outside-file": { + "bun": "./src/helpers/outsideFile.ts", + "node": "./dist/helpers/outsideFile.js", + "import": "./src/helpers/outsideFile.ts", + "types": "./src/helpers/outsideFile.ts" + }, "./media-range": { "bun": "./src/helpers/mediaRange.ts", "node": "./dist/helpers/mediaRange.js", @@ -196,6 +202,10 @@ "import": "./dist/helpers/mime.js", "types": "./dist/helpers/mime.d.ts" }, + "./outside-file": { + "import": "./dist/helpers/outsideFile.js", + "types": "./dist/helpers/outsideFile.d.ts" + }, "./media-range": { "import": "./dist/helpers/mediaRange.js", "types": "./dist/helpers/mediaRange.d.ts" diff --git a/packages/cli/src/cloud/mediaExtensions.ts b/packages/studio-server/src/helpers/mediaExtensions.ts similarity index 100% rename from packages/cli/src/cloud/mediaExtensions.ts rename to packages/studio-server/src/helpers/mediaExtensions.ts diff --git a/packages/studio-server/src/helpers/outsideFile.ts b/packages/studio-server/src/helpers/outsideFile.ts new file mode 100644 index 0000000000..1c0729bb43 --- /dev/null +++ b/packages/studio-server/src/helpers/outsideFile.ts @@ -0,0 +1,103 @@ +import { homedir } from "node:os"; +import { dirname, extname, isAbsolute, join, relative, sep } from "node:path"; +import { MIME_TYPES } from "./mime.js"; +import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "./mediaExtensions.js"; +import { realFilePath } from "./safePath.js"; + +export { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS }; + +const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/; + +/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types, + * and the media cloud render re-encodes or converts. */ +export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([ + ...Object.entries(MIME_TYPES) + .filter(([, type]) => COPIED_TYPE.test(type)) + .map(([extension]) => extension), + ...VIDEO_EXTS, + ...AUDIO_EXTS, + ...CONVERTIBLE_IMAGE_EXTS, +]); + +const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase()); + +export const isCopiedOutsideFileType = (absPath: string): boolean => + hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath)); + +const ENV_FILE = /^\.env([._-].*|rc)?$/; + +export function isEnvOrGitPath(path: string): boolean { + const segments = path.toLowerCase().split(/[\\/]/); + return segments.includes(".git") || ENV_FILE.test(segments.at(-1) ?? ""); +} + +const SYSTEM_DIRS = ["/etc", "/proc", "/sys", "/dev", "/private/etc"]; + +const isOutside = (fromDir: string) => + fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir); + +function inPrivateLocation(projectDir: string, path: string): boolean { + const posixPath = path.replaceAll("\\", "/"); + if (SYSTEM_DIRS.some((dir) => posixPath === dir || posixPath.startsWith(`${dir}/`))) return true; + const fromParent = relative(dirname(projectDir), path); + return ( + (isOutside(fromParent) ? path : fromParent).split(/[\\/]/).some((s) => s.startsWith(".")) || + inHomeAppData(projectDir, path) + ); +} + +function isUnder(path: string, ...dir: string[]): boolean { + const lowerPath = path.toLowerCase(); + const lowerDir = join(homedir(), ...dir).toLowerCase(); + return lowerPath === lowerDir || lowerPath.startsWith(`${lowerDir}${sep}`); +} + +const CLOUD_DRIVE_FOLDERS = ["Mobile Documents", "CloudStorage"]; + +function homeAppDataFolder(path: string): string | null { + if (isUnder(path, "AppData")) return "AppData"; + const cloudDrive = CLOUD_DRIVE_FOLDERS.some((folder) => isUnder(path, "Library", folder)); + if (isUnder(path, "Library") && !cloudDrive) return "Library"; + return null; +} + +function inHomeAppData(projectDir: string, path: string): boolean { + const folder = homeAppDataFolder(path); + const parent = dirname(projectDir); + return ( + folder !== null && (homeAppDataFolder(parent) !== folder || isOutside(relative(parent, path))) + ); +} + +/** An existing file outside the project that publish leaves out: env files, Git data, hidden, system and app-data folders. */ +export function isPrivateOutsideFile(projectDir: string, absPath: string): boolean { + const realPath = realFilePath(absPath); + return ( + isEnvOrGitPath(absPath) || + isEnvOrGitPath(realPath) || + inPrivateLocation(projectDir, absPath) || + inPrivateLocation(realFilePath(projectDir), realPath) + ); +} + +export const linksOutside = (projectDir: string, filePath: string): boolean => + isOutside(relative(realFilePath(projectDir), realFilePath(filePath))); + +/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */ +export type LeftOutReason = "blocklisted" | "received" | "file_type"; + +const WHY: Record = { + blocklisted: "which is outside the project in a private location", + received: "which leads outside the project and came in with an imported project", + file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file", +}; + +export function leftOutWarning( + file: string, + reference: string, + reason: LeftOutReason, + from: "upload" | "render" = "upload", +): string { + const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the ${from}.`; + return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning; +} diff --git a/packages/studio-server/tsup.config.ts b/packages/studio-server/tsup.config.ts index 7e11506a19..7e4e98156c 100644 --- a/packages/studio-server/tsup.config.ts +++ b/packages/studio-server/tsup.config.ts @@ -19,6 +19,7 @@ export default defineConfig({ "helpers/encoderOptions": "src/helpers/encoderOptions.ts", "helpers/mediaRange": "src/helpers/mediaRange.ts", "helpers/mime": "src/helpers/mime.ts", + "helpers/outsideFile": "src/helpers/outsideFile.ts", "history/index": "src/history/index.ts", }, format: ["esm"], diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index df18990183..5bc0964512 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -37,7 +37,7 @@ }, "@hyperframes/producer": { "packed": 890000, - "unpacked": 3210000 + "unpacked": 3245000 }, "@hyperframes/sdk": { "packed": 90662, @@ -57,6 +57,6 @@ }, "@hyperframes/studio-server": { "packed": 168500, - "unpacked": 646000 + "unpacked": 654000 } }