From 2c2b399bfa229d759d560aaf63e9717afed9805c Mon Sep 17 00:00:00 2001 From: James Russo Date: Fri, 13 Nov 2026 16:26:19 +0000 Subject: [PATCH] fix(producer): local renders bring in only media, font, CSS and JS from outside the project (#7647) * fix(producer): local renders bring in only media, font, CSS and JS from outside the project Local renders (`render`, range render, `benchmark`, the Studio server and the Vite Studio adapter) run with allowOutsideProjectAssets, which copied every existing file that markup named outside the project into compiled/hf-ext, of any type and from any folder, and served project links that lead outside without looking at where they lead. Publish already applies a policy here: outside references are kept, private locations are left out, and only image, video, audio, font, CSS and JS files are copied. Local renders now follow the same policy. The publish predicates (private location, copied file type, left-out wording) move from the CLI to a new `@hyperframes/studio-server/outside-file` subpath, since the producer cannot import the CLI and already depends on studio-server. The CLI re-exports them, so publish behaves as before. The media extension sets move with them because the copied-type list includes them. The producer decides with one predicate, `outsideAssetRefusal` in utils/paths.ts, and applies it wherever allowOutsideProjectAssets lets an outside file in, so no entry point can skip it: - collectExternalAssets: src/href/url() references, which covers and - dropMediaOutsideProject: video/audio/image sources in any spelling the media resolver accepts (file: URLs, percent-encoding), so nothing probes or extracts them - the render file server: project links that lead outside - the distributed plan's project copy: the same links (range render) - GIF/HEIC prep and sub-composition links: private locations only, since those inputs are converted or compiled rather than copied as they are Each left-out file logs the publish wording with "left out of the render". To reproduce on main: a project whose index.html references ../../.ssh/id_key through , and url(), with a project link to a file in .ssh, rendered locally, gets those files copied into compiled/hf-ext and served. Comment lines: the four in studio-server's outsideFile.ts moved verbatim from the CLI files they replace. Two are new, in utils/paths.ts. The one on outsideAssetRefusal states the policy once, so its six call sites carry no comment. The one on refusesOutsideLink says its boolean means "refused, and already warned", which the name alone does not say. Comment ratchet is clean. Co-Authored-By: Claude Opus 5.5 * fix(cli): keep outsideFileType as a re-export of the shared module The base repository's deletion guard rejects removing a file that exists on main unless git pairs it as a rename. outsideFileType.ts moved into studio-server/outside-file with enough added around it that git no longer pairs the two, so CI's "Reject accidental file deletions" step failed. The file stays as a two-name re-export and publish keeps importing from it, so publishProject.ts and its file-type test are back to main. Co-Authored-By: Claude Opus 5.5 * chore(producer): raise the unpacked size budgets for the outside-file policy CI's packed-manifest check measured @hyperframes/producer at 3,244,665 bytes unpacked (budget 3,210,000) and @hyperframes/studio-server at 653,506 (budget 646,000), both just past the 1% tolerance. Both packages were already at the edge on main. Measured locally with npm pack --dry-run, main is 3,237,162 for producer and 652,502 for studio-server, so studio-server on main is already at its ceiling. This branch adds about 7.9 KB unpacked to producer, because the new checks sit in the compiler, render stages and file server, which each producer entry bundles. It adds about 5.5 KB to studio-server for the new outside-file entry and its declarations. No source maps, tests or assets were added to either tarball. The budgets go to 3,245,000 and 654,000, just above CI's measurements, the same way #7579 raised producer's budget for the project-folder checks. Co-Authored-By: Claude Opus 5.5 * fix(producer): apply the outside-file policy to embedded local fonts A project font file that links outside the project is now left out of font embedding with the same warning the file server gives, instead of being inlined into the compiled HTML. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- packages/cli/src/cloud/offload.ts | 6 +- packages/cli/src/utils/leftOutReference.ts | 15 +-- packages/cli/src/utils/outsideFileType.ts | 26 +--- packages/cli/src/utils/privateProjectFile.ts | 67 ++-------- .../producer/src/services/animatedGifPrep.ts | 5 +- .../producer/src/services/assetMediaType.ts | 8 +- .../producer/src/services/distributed/plan.ts | 5 +- packages/producer/src/services/fileServer.ts | 7 +- .../producer/src/services/htmlCompiler.ts | 47 ++++++- .../src/services/outsideProjectAssets.test.ts | 116 +++++++++++++++++- .../services/render/stages/compileStage.ts | 10 +- .../src/services/render/stages/probeStage.ts | 10 +- packages/producer/src/utils/paths.ts | 30 +++++ packages/studio-server/package-subpaths.json | 6 + packages/studio-server/package.json | 10 ++ .../src/helpers}/mediaExtensions.ts | 0 .../studio-server/src/helpers/outsideFile.ts | 103 ++++++++++++++++ packages/studio-server/tsup.config.ts | 1 + scripts/package-size-budgets.json | 4 +- 19 files changed, 363 insertions(+), 113 deletions(-) rename packages/{cli/src/cloud => studio-server/src/helpers}/mediaExtensions.ts (100%) create mode 100644 packages/studio-server/src/helpers/outsideFile.ts diff --git a/packages/cli/src/cloud/offload.ts b/packages/cli/src/cloud/offload.ts index a655cb503b..cb9952135d 100644 --- a/packages/cli/src/cloud/offload.ts +++ b/packages/cli/src/cloud/offload.ts @@ -33,7 +33,11 @@ import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg import { probeAudio } from "../audio/analysis/decode.js"; import { formatBytes } from "../ui/format.js"; import { bytesMatchExtension } from "./offloadSniff.js"; -import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "./mediaExtensions.js"; +import { + AUDIO_EXTS, + CONVERTIBLE_IMAGE_EXTS, + VIDEO_EXTS, +} from "@hyperframes/studio-server/outside-file"; import { type PlayedSegment, planTrim } from "./offloadTrim.js"; import { archiveKey, diff --git a/packages/cli/src/utils/leftOutReference.ts b/packages/cli/src/utils/leftOutReference.ts index efe7835dc1..47ff0765d6 100644 --- a/packages/cli/src/utils/leftOutReference.ts +++ b/packages/cli/src/utils/leftOutReference.ts @@ -1,8 +1,8 @@ import { isAbsolute, relative } from "node:path"; +import { type LeftOutReason, leftOutWarning } from "@hyperframes/studio-server/outside-file"; import { realFilePath } from "@hyperframes/studio-server/safe-path"; -/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */ -export type LeftOutReason = "blocklisted" | "received" | "file_type"; +export { type LeftOutReason, leftOutWarning }; /** One file an upload left out. `path` is from the project folder; `levels_up` is null on another drive. */ export interface LeftOutReference { @@ -15,17 +15,6 @@ export interface LeftOutReference { export type OnLeftOut = (warning: string, reference: LeftOutReference) => void; -const WHY: Record = { - blocklisted: "which is outside the project in a private location", - received: "which leads outside the project and came in with an imported project", - file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file", -}; - -export function leftOutWarning(file: string, reference: string, reason: LeftOutReason): string { - const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the upload.`; - return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning; -} - export function leftOutReference( projectDir: string, file: string, diff --git a/packages/cli/src/utils/outsideFileType.ts b/packages/cli/src/utils/outsideFileType.ts index b4f4123e9d..ba4f8e20ca 100644 --- a/packages/cli/src/utils/outsideFileType.ts +++ b/packages/cli/src/utils/outsideFileType.ts @@ -1,22 +1,4 @@ -import { extname } from "node:path"; -import { MIME_TYPES } from "@hyperframes/studio-server/mime"; -import { realFilePath } from "@hyperframes/studio-server/safe-path"; -import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "../cloud/mediaExtensions.js"; - -const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/; - -/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types, - * and the media cloud render re-encodes or converts. */ -export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([ - ...Object.entries(MIME_TYPES) - .filter(([, type]) => COPIED_TYPE.test(type)) - .map(([extension]) => extension), - ...VIDEO_EXTS, - ...AUDIO_EXTS, - ...CONVERTIBLE_IMAGE_EXTS, -]); - -const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase()); - -export const isCopiedOutsideFileType = (absPath: string): boolean => - hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath)); +export { + isCopiedOutsideFileType, + OUTSIDE_FILE_EXTENSIONS, +} from "@hyperframes/studio-server/outside-file"; diff --git a/packages/cli/src/utils/privateProjectFile.ts b/packages/cli/src/utils/privateProjectFile.ts index a7131da848..0059aa4ac7 100644 --- a/packages/cli/src/utils/privateProjectFile.ts +++ b/packages/cli/src/utils/privateProjectFile.ts @@ -1,66 +1,13 @@ import { readFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { dirname, isAbsolute, join, relative, sep } from "node:path"; +import { join, relative } from "node:path"; +import { + isEnvOrGitPath, + isPrivateOutsideFile, + linksOutside, +} from "@hyperframes/studio-server/outside-file"; import { isPrivateProjectFile, realFilePath } from "@hyperframes/studio-server/safe-path"; -const ENV_FILE = /^\.env([._-].*|rc)?$/; - -export function isEnvOrGitPath(path: string): boolean { - const segments = path.toLowerCase().split(/[\\/]/); - return segments.includes(".git") || ENV_FILE.test(segments.at(-1) ?? ""); -} - -const SYSTEM_DIRS = ["/etc", "/proc", "/sys", "/dev", "/private/etc"]; - -const isOutside = (fromDir: string) => - fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir); - -function inPrivateLocation(projectDir: string, path: string): boolean { - const posixPath = path.replaceAll("\\", "/"); - if (SYSTEM_DIRS.some((dir) => posixPath === dir || posixPath.startsWith(`${dir}/`))) return true; - const fromParent = relative(dirname(projectDir), path); - return ( - (isOutside(fromParent) ? path : fromParent).split(/[\\/]/).some((s) => s.startsWith(".")) || - inHomeAppData(projectDir, path) - ); -} - -function isUnder(path: string, ...dir: string[]): boolean { - const lowerPath = path.toLowerCase(); - const lowerDir = join(homedir(), ...dir).toLowerCase(); - return lowerPath === lowerDir || lowerPath.startsWith(`${lowerDir}${sep}`); -} - -const CLOUD_DRIVE_FOLDERS = ["Mobile Documents", "CloudStorage"]; - -function homeAppDataFolder(path: string): string | null { - if (isUnder(path, "AppData")) return "AppData"; - const cloudDrive = CLOUD_DRIVE_FOLDERS.some((folder) => isUnder(path, "Library", folder)); - if (isUnder(path, "Library") && !cloudDrive) return "Library"; - return null; -} - -function inHomeAppData(projectDir: string, path: string): boolean { - const folder = homeAppDataFolder(path); - const parent = dirname(projectDir); - return ( - folder !== null && (homeAppDataFolder(parent) !== folder || isOutside(relative(parent, path))) - ); -} - -/** An existing file outside the project that publish leaves out: env files, Git data, hidden, system and app-data folders. */ -export function isPrivateOutsideFile(projectDir: string, absPath: string): boolean { - const realPath = realFilePath(absPath); - return ( - isEnvOrGitPath(absPath) || - isEnvOrGitPath(realPath) || - inPrivateLocation(projectDir, absPath) || - inPrivateLocation(realFilePath(projectDir), realPath) - ); -} - -export const linksOutside = (projectDir: string, filePath: string): boolean => - isOutside(relative(realFilePath(projectDir), realFilePath(filePath))); +export { isEnvOrGitPath, isPrivateOutsideFile, linksOutside }; export function linksToPrivateOutsideFile(projectDir: string, filePath: string): boolean { return linksOutside(projectDir, filePath) && isPrivateOutsideFile(projectDir, filePath); diff --git a/packages/producer/src/services/animatedGifPrep.ts b/packages/producer/src/services/animatedGifPrep.ts index 7bce6639a6..6c5583bcc0 100644 --- a/packages/producer/src/services/animatedGifPrep.ts +++ b/packages/producer/src/services/animatedGifPrep.ts @@ -7,7 +7,7 @@ import { parseAnimatedGifMetadata, type AnimatedGifMetadata } from "@hyperframes import { DEFAULT_VP9_CPU_USED, runFfmpeg, renameWithRetry } from "@hyperframes/engine"; import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution"; import { isHttpUrl } from "../utils/urlDownloader.js"; -import { escapesAssetRoot } from "../utils/paths.js"; +import { escapesAssetRoot, outsideAssetRefusal } from "../utils/paths.js"; import { encoderFailureError } from "./render/encoderInterruption.js"; const PREPARED_GIF_SUBDIR = "_animated_gif"; @@ -109,7 +109,8 @@ export function resolveLocalImageSourcePath( ]; const allowed = (candidate: string) => - !escapesAssetRoot(candidate, options.assetRoot) || + (!escapesAssetRoot(candidate, options.assetRoot) && + outsideAssetRefusal(options.projectDir, candidate) !== "blocklisted") || !escapesAssetRoot(candidate, options.downloadDir); return candidates.find((candidate) => existsSync(candidate) && allowed(candidate)) ?? null; } diff --git a/packages/producer/src/services/assetMediaType.ts b/packages/producer/src/services/assetMediaType.ts index 7073ff09b1..5428149749 100644 --- a/packages/producer/src/services/assetMediaType.ts +++ b/packages/producer/src/services/assetMediaType.ts @@ -12,7 +12,7 @@ import { type VideoElement, } from "@hyperframes/engine"; import { withMediaProbeSlot } from "../utils/mediaProbeConcurrency.js"; -import { escapesAssetRoot } from "../utils/paths.js"; +import { escapesAssetRoot, leftOutOfRender, outsideAssetRefusal } from "../utils/paths.js"; import type { ProducerLogger } from "../logger.js"; export const ASSET_MEDIA_TYPE_MISMATCH = "ASSET_MEDIA_TYPE_MISMATCH" as const; @@ -97,6 +97,7 @@ export function dropMediaOutsideProject(input: { workDir: string; composition: { videos: VideoElement[]; audios: AudioElement[]; images: ImageElement[] }; log: ProducerLogger; + allowOutsideProjectAssets?: boolean; }): void { const { projectDir, workDir, composition, log } = input; const compiledDir = join(workDir, "compiled"); @@ -105,6 +106,11 @@ export function dropMediaOutsideProject(input: { if (!src || isRemoteOrInlineSource(src)) return true; const path = resolveProjectRelativeSrc(src, projectDir, compiledDir); if (roots.some((root) => !escapesAssetRoot(path, root))) return true; + if (input.allowOutsideProjectAssets) { + const refusal = outsideAssetRefusal(projectDir, path); + if (refusal) log.warn(`[Render] ${leftOutOfRender(`Media "${id}"`, src, refusal)}`); + return refusal === null; + } log.warn(`[Render] Media "${id}" source is missing: it is outside the project folder`, { src }); return false; }; diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 75fda52e57..db85d37f41 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -64,6 +64,7 @@ import { declareProbeFactsUrl, } from "@hyperframes/engine"; import { defaultLogger, type ProducerLogger } from "../../logger.js"; +import { refusesOutsideLink } from "../../utils/paths.js"; import { applyRenderWarningPolicy, type RenderJob, @@ -1277,7 +1278,9 @@ async function compilePlanComposition( // false-positive every descendant. const rel = relative(projectDir, src); if (rel === "" || rel.startsWith("..")) return true; - if (!job.config.allowOutsideProjectAssets && !isSafePath(projectDir, src)) return false; + if (job.config.allowOutsideProjectAssets) { + if (refusesOutsideLink(projectDir, src, (line) => log.warn(`[Plan] ${line}`))) return false; + } else if (!isSafePath(projectDir, src)) return false; const firstSegment = rel.split(sep, 1)[0]; return firstSegment === undefined || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment); }, diff --git a/packages/producer/src/services/fileServer.ts b/packages/producer/src/services/fileServer.ts index bac1d66f1f..2ba63b715c 100644 --- a/packages/producer/src/services/fileServer.ts +++ b/packages/producer/src/services/fileServer.ts @@ -34,6 +34,7 @@ import { openRangeEnd } from "@hyperframes/studio-server/media-range"; import { getVerifiedHyperframeRuntimeSource } from "./hyperframeRuntimeLoader.js"; import { getHfEarlyStub } from "../generated/hf-early-stub-inline.js"; import { defaultLogger, type ProducerLogger } from "../logger.js"; +import { refusesOutsideLink } from "../utils/paths.js"; const readFileAsync = promisify(readFile); @@ -812,6 +813,7 @@ export function createFileServer(options: FileServerOptions): Promise console.warn(`[FileServer] ${line}`); app.get("/*", async (c) => { let requestPath = c.req.path; @@ -840,7 +842,10 @@ export function createFileServer(options: FileServerOptions): Promise { export function collectExternalAssets( html: string, projectDir: string, + log: ProducerLogger = defaultLogger, + file = "index.html", ): { html: string; externalAssets: Map } { const absProjectDir = resolve(projectDir); const externalAssets = new Map(); + const leftOut = new Set(); function processPath(rawPath: string): string | null { const trimmed = rawPath.trim(); @@ -1415,6 +1429,12 @@ export function collectExternalAssets( return null; // inside projectDir, file server handles this } if (!existsSync(absPath)) return null; + const refusal = outsideAssetRefusal(absProjectDir, absPath); + if (refusal) { + if (!leftOut.has(trimmed)) log.warn(`[Compiler] ${leftOutOfRender(file, trimmed, refusal)}`); + leftOut.add(trimmed); + return null; + } // resolve() already canonicalises the path (no .. components remain); // toExternalAssetKey() produces a cross-platform relative key that // `path.join(compileDir, key)` cannot escape on any OS. @@ -1957,6 +1977,7 @@ async function embedLocalFontFaces( html: string, projectDir: string, assetRoot: string | undefined, + log: ProducerLogger = defaultLogger, ): Promise { const { fontToDataUri: toDataUri } = await import("@hyperframes/core/fonts/embed"); const styleBlockRe = /]*>([\s\S]*?)<\/style>/gi; @@ -1980,6 +2001,11 @@ async function embedLocalFontFaces( if (!localPath || embeddedPaths.has(localPath)) continue; const absPath = localPath.startsWith("/") ? localPath : resolve(projectDir, localPath); if (!isPathInside(absPath, projectDir) || escapesAssetRoot(absPath, assetRoot)) continue; + // A project font that is a link leading outside obeys the same policy as the file server. + if (refusesOutsideLink(projectDir, absPath, (line) => log.warn(`[Compiler] ${line}`))) { + embeddedPaths.add(localPath); + continue; + } const ext = absPath.match(/\.(woff2?|ttf|otf|ttc)$/i)?.[1]?.toLowerCase() ?? "ttf"; try { if (fileBackedAbsolutePaths.has(absPath)) { @@ -2166,6 +2192,11 @@ function bindVariableMediaSources(html: string, variables: Record + isPathInside(htmlPath, projectDir) + ? relative(projectDir, htmlPath).split(sep).join("/") + : basename(htmlPath); + /** * Compile an HTML composition project into a single self-contained HTML string * with all media metadata resolved. @@ -2356,13 +2387,23 @@ export async function compileForRender( defaultLogger.debug(`[Compiler] Prepared ${preparedGifs.length} animated GIF input(s) as WebM`); } - const embeddedHtml = await embedLocalFontFaces(htmlWithPreparedGifs, projectDir, assetRoot); + const embeddedHtml = await embedLocalFontFaces( + htmlWithPreparedGifs, + projectDir, + assetRoot, + options.log, + ); // Collect assets that resolve outside projectDir (e.g. ../shared-assets/hero.png). // These can't be served by the file server, so we map them to paths the // orchestrator will copy into the compiled output directory. const { html, externalAssets } = options.allowOutsideProjectAssets - ? collectExternalAssets(embeddedHtml, projectDir) + ? collectExternalAssets( + embeddedHtml, + projectDir, + options.log, + entryFileName(projectDir, htmlPath), + ) : { html: embeddedHtml, externalAssets: new Map() }; for (const [relPath, absPath] of remoteMediaAssets) { diff --git a/packages/producer/src/services/outsideProjectAssets.test.ts b/packages/producer/src/services/outsideProjectAssets.test.ts index 77bb12bf90..484701da35 100644 --- a/packages/producer/src/services/outsideProjectAssets.test.ts +++ b/packages/producer/src/services/outsideProjectAssets.test.ts @@ -7,12 +7,13 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, rmSync, symlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join, relative, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { resolveConfig } from "@hyperframes/engine"; import { runCompileStage } from "./render/stages/compileStage.js"; @@ -163,6 +164,54 @@ async function compileInProcess(projectDir: string, allowOutsideProjectAssets?: return { result, compiledDir, extract, serve, warnings }; } +function privateOutsideProject() { + const root = mkdtempSync(join(tmpdir(), "hf-private-outside-")); + roots.push(root); + const projectDir = join(root, "work", "project"); + const [ssh, shared] = [join(root, ".ssh"), join(root, "work", "shared")]; + for (const dir of [projectDir, ssh, shared]) mkdirSync(dir, { recursive: true }); + writeFileSync(join(ssh, "id_key"), "PRIVATE"); + writeFileSync(join(ssh, "pic.png"), PNG_1X1); + copyFileSync(tool.clip, join(ssh, "clip.mp4")); + const gif = ["-v", "error", "-f", "lavfi", "-i", "testsrc=size=16x16:rate=10", "-t", "0.3"]; + expect(spawnSync("ffmpeg", [...gif, join(ssh, "anim.gif")]).status).toBe(0); + writeFileSync(join(ssh, "scene.html"), scene); + writeFileSync(join(shared, "logo.png"), PNG_1X1); + writeFileSync(join(shared, "notes.txt"), "notes"); + symlinkSync(join(ssh, "pic.png"), join(projectDir, "link.png")); + symlinkSync(join(shared, "logo.png"), join(projectDir, "keep.png")); + symlinkSync(join(ssh, "id_key"), join(projectDir, "brand.ttf")); + symlinkSync(join(shared, "notes.txt"), join(projectDir, "notes.ttf")); + const pic = join(ssh, "pic.png"); + const video = (id: string, src: string) => + ``; + writeFileSync( + join(projectDir, "index.html"), + ` + +
+ +
+
keynotes + ${video("dotdot", "../../.ssh/clip.mp4")} + ${video("fileurl", pathToFileURL(join(ssh, "clip.mp4")).href)} +
`, + ); + const key = (path: string) => toExternalAssetKey(resolve(path)); + return { + root, + projectDir, + privateKeys: [key(pic), key(join(ssh, "id_key"))], + logo: key(join(shared, "logo.png")), + }; +} + +const filesUnder = (dir: string): string[] => + readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((entry) => !entry.isDirectory()) + .map((entry) => relative(dir, join(entry.parentPath, entry.name)).replaceAll("\\", "/")); + const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); const config = { fps: 30 as const, width: 320, height: 240, format: "mp4" as const }; @@ -225,6 +274,71 @@ describe.skipIf(!HAS_FFMPEG)("render servers keep markup inside the project", () }, ); + it("trusted local renders leave out private and non-media files outside the project", async () => { + const { root, projectDir, privateKeys, logo } = privateOutsideProject(); + tool.calls.length = 0; + const { result, compiledDir, serve, warnings } = await compileInProcess(projectDir, true); + + expect(filesUnder(compiledDir).sort()).toEqual([logo, "index.html"]); + expect(result.composition.videos).toEqual([]); + expect(tool.calls.filter((call) => call.includes(".ssh") && !call.includes("ffprobe"))).toEqual( + [], + ); + const warned = warnings.join("\n"); + const references = ["id_key", "pic.png", "clip.mp4", "anim.gif"].map((f) => `../../.ssh/${f}`); + for (const reference of references) { + expect(warned).toContain( + `index.html references ${reference}, which is outside the project in a private location; it was left out of the render.`, + ); + } + expect(warned).toMatch( + /Media "fileurl" references file:.*\.ssh\/clip\.mp4, which is outside the project in a private/, + ); + expect(warned).toContain( + "../shared/notes.txt, which is outside the project and is not an image", + ); + expect(warned).toMatch( + /brand\.ttf references .*\.ssh\/id_key, which is outside the project in a private/, + ); + expect(warned).toMatch( + /notes\.ttf references .*notes\.txt, which is outside the project and is not an image/, + ); + const compiledHtml = readFileSync(join(compiledDir, "index.html"), "utf-8"); + expect(compiledHtml).not.toContain(Buffer.from("PRIVATE").toString("base64")); + expect(compiledHtml).not.toContain(Buffer.from("notes").toString("base64")); + expect(compiledHtml).toContain('url("brand.ttf")'); + expect( + await statusOf(await serve(), [ + ...privateKeys.map((k) => `/${k}`), + "/link.png", + `/${logo}`, + "/keep.png", + "/brand.ttf", + ]), + ).toEqual([404, 404, 404, 200, 200, 404]); + + const planDir = join(root, "plan"); + await plan(projectDir, config, planDir, { allowOutsideProjectAssets: true }); + const planned = filesUnder(join(planDir, "compiled")); + expect(planned.filter((file) => /\.ssh|notes|link\.png|brand\.ttf/.test(file))).toEqual([]); + expect(planned).toEqual(expect.arrayContaining([logo, "keep.png"])); + }); + + it("trusted local renders refuse a linked composition in a private location", async () => { + const { root, projectDir } = privateOutsideProject(); + mkdirSync(join(projectDir, "scenes")); + symlinkSync(join(root, ".ssh", "scene.html"), join(projectDir, "scenes", "a.html")); + const host = `
`; + const html = readFileSync(join(projectDir, "index.html"), "utf-8").replace( + "", + `${host}`, + ); + writeFileSync(join(projectDir, "index.html"), html); + await expect(compileInProcess(projectDir, true)).rejects.toThrow( + /outside the project in a private location/, + ); + }); + it("trusted local renders still import outside files", async () => { const { root, projectDir, secretKey } = outsideProject(false); const { compiledDir, serve } = await compileInProcess(projectDir, true); diff --git a/packages/producer/src/services/render/stages/compileStage.ts b/packages/producer/src/services/render/stages/compileStage.ts index f34afbfc97..c4f6fe1818 100644 --- a/packages/producer/src/services/render/stages/compileStage.ts +++ b/packages/producer/src/services/render/stages/compileStage.ts @@ -377,9 +377,13 @@ export async function runCompileStage(input: CompileStageInput): Promise + leftOutWarning(file, reference, reason, "render"); + +/** True, after a warning, when a trusted local render must not serve or copy `path`, a project file that leads outside. */ +export function refusesOutsideLink(projectDir: string, path: string, warn: (line: string) => void) { + const refusal = outsideAssetRefusal(projectDir, path); + if (!refusal) return false; + const file = nodeRelative(projectDir, path).split(sep).join("/"); + warn(leftOutOfRender(file, realFilePath(path), refusal)); + return true; +} + /** * Build a safe, cross-platform relative key for an absolute asset path * that lives outside the project directory. diff --git a/packages/studio-server/package-subpaths.json b/packages/studio-server/package-subpaths.json index b2e2f91fc9..668643255c 100644 --- a/packages/studio-server/package-subpaths.json +++ b/packages/studio-server/package-subpaths.json @@ -110,6 +110,12 @@ "types": "./dist/helpers/mime.d.ts", "environments": ["bun", "node"] }, + "./outside-file": { + "source": "./src/helpers/outsideFile.ts", + "runtime": "./dist/helpers/outsideFile.js", + "types": "./dist/helpers/outsideFile.d.ts", + "environments": ["bun", "node"] + }, "./media-range": { "source": "./src/helpers/mediaRange.ts", "runtime": "./dist/helpers/mediaRange.js", diff --git a/packages/studio-server/package.json b/packages/studio-server/package.json index 9777ab8c34..8c829b63c7 100644 --- a/packages/studio-server/package.json +++ b/packages/studio-server/package.json @@ -117,6 +117,12 @@ "import": "./src/helpers/mime.ts", "types": "./src/helpers/mime.ts" }, + "./outside-file": { + "bun": "./src/helpers/outsideFile.ts", + "node": "./dist/helpers/outsideFile.js", + "import": "./src/helpers/outsideFile.ts", + "types": "./src/helpers/outsideFile.ts" + }, "./media-range": { "bun": "./src/helpers/mediaRange.ts", "node": "./dist/helpers/mediaRange.js", @@ -196,6 +202,10 @@ "import": "./dist/helpers/mime.js", "types": "./dist/helpers/mime.d.ts" }, + "./outside-file": { + "import": "./dist/helpers/outsideFile.js", + "types": "./dist/helpers/outsideFile.d.ts" + }, "./media-range": { "import": "./dist/helpers/mediaRange.js", "types": "./dist/helpers/mediaRange.d.ts" diff --git a/packages/cli/src/cloud/mediaExtensions.ts b/packages/studio-server/src/helpers/mediaExtensions.ts similarity index 100% rename from packages/cli/src/cloud/mediaExtensions.ts rename to packages/studio-server/src/helpers/mediaExtensions.ts diff --git a/packages/studio-server/src/helpers/outsideFile.ts b/packages/studio-server/src/helpers/outsideFile.ts new file mode 100644 index 0000000000..1c0729bb43 --- /dev/null +++ b/packages/studio-server/src/helpers/outsideFile.ts @@ -0,0 +1,103 @@ +import { homedir } from "node:os"; +import { dirname, extname, isAbsolute, join, relative, sep } from "node:path"; +import { MIME_TYPES } from "./mime.js"; +import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "./mediaExtensions.js"; +import { realFilePath } from "./safePath.js"; + +export { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS }; + +const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/; + +/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types, + * and the media cloud render re-encodes or converts. */ +export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet = new Set([ + ...Object.entries(MIME_TYPES) + .filter(([, type]) => COPIED_TYPE.test(type)) + .map(([extension]) => extension), + ...VIDEO_EXTS, + ...AUDIO_EXTS, + ...CONVERTIBLE_IMAGE_EXTS, +]); + +const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase()); + +export const isCopiedOutsideFileType = (absPath: string): boolean => + hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath)); + +const ENV_FILE = /^\.env([._-].*|rc)?$/; + +export function isEnvOrGitPath(path: string): boolean { + const segments = path.toLowerCase().split(/[\\/]/); + return segments.includes(".git") || ENV_FILE.test(segments.at(-1) ?? ""); +} + +const SYSTEM_DIRS = ["/etc", "/proc", "/sys", "/dev", "/private/etc"]; + +const isOutside = (fromDir: string) => + fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir); + +function inPrivateLocation(projectDir: string, path: string): boolean { + const posixPath = path.replaceAll("\\", "/"); + if (SYSTEM_DIRS.some((dir) => posixPath === dir || posixPath.startsWith(`${dir}/`))) return true; + const fromParent = relative(dirname(projectDir), path); + return ( + (isOutside(fromParent) ? path : fromParent).split(/[\\/]/).some((s) => s.startsWith(".")) || + inHomeAppData(projectDir, path) + ); +} + +function isUnder(path: string, ...dir: string[]): boolean { + const lowerPath = path.toLowerCase(); + const lowerDir = join(homedir(), ...dir).toLowerCase(); + return lowerPath === lowerDir || lowerPath.startsWith(`${lowerDir}${sep}`); +} + +const CLOUD_DRIVE_FOLDERS = ["Mobile Documents", "CloudStorage"]; + +function homeAppDataFolder(path: string): string | null { + if (isUnder(path, "AppData")) return "AppData"; + const cloudDrive = CLOUD_DRIVE_FOLDERS.some((folder) => isUnder(path, "Library", folder)); + if (isUnder(path, "Library") && !cloudDrive) return "Library"; + return null; +} + +function inHomeAppData(projectDir: string, path: string): boolean { + const folder = homeAppDataFolder(path); + const parent = dirname(projectDir); + return ( + folder !== null && (homeAppDataFolder(parent) !== folder || isOutside(relative(parent, path))) + ); +} + +/** An existing file outside the project that publish leaves out: env files, Git data, hidden, system and app-data folders. */ +export function isPrivateOutsideFile(projectDir: string, absPath: string): boolean { + const realPath = realFilePath(absPath); + return ( + isEnvOrGitPath(absPath) || + isEnvOrGitPath(realPath) || + inPrivateLocation(projectDir, absPath) || + inPrivateLocation(realFilePath(projectDir), realPath) + ); +} + +export const linksOutside = (projectDir: string, filePath: string): boolean => + isOutside(relative(realFilePath(projectDir), realFilePath(filePath))); + +/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */ +export type LeftOutReason = "blocklisted" | "received" | "file_type"; + +const WHY: Record = { + blocklisted: "which is outside the project in a private location", + received: "which leads outside the project and came in with an imported project", + file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file", +}; + +export function leftOutWarning( + file: string, + reference: string, + reason: LeftOutReason, + from: "upload" | "render" = "upload", +): string { + const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the ${from}.`; + return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning; +} diff --git a/packages/studio-server/tsup.config.ts b/packages/studio-server/tsup.config.ts index 7e11506a19..7e4e98156c 100644 --- a/packages/studio-server/tsup.config.ts +++ b/packages/studio-server/tsup.config.ts @@ -19,6 +19,7 @@ export default defineConfig({ "helpers/encoderOptions": "src/helpers/encoderOptions.ts", "helpers/mediaRange": "src/helpers/mediaRange.ts", "helpers/mime": "src/helpers/mime.ts", + "helpers/outsideFile": "src/helpers/outsideFile.ts", "history/index": "src/history/index.ts", }, format: ["esm"], diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index df18990183..5bc0964512 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -37,7 +37,7 @@ }, "@hyperframes/producer": { "packed": 890000, - "unpacked": 3210000 + "unpacked": 3245000 }, "@hyperframes/sdk": { "packed": 90662, @@ -57,6 +57,6 @@ }, "@hyperframes/studio-server": { "packed": 168500, - "unpacked": 646000 + "unpacked": 654000 } }