mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(producer): local renders bring in only media, font, CSS and JS from outside the project (#7647)
* fix(producer): local renders bring in only media, font, CSS and JS from outside the project Local renders (`render`, range render, `benchmark`, the Studio server and the Vite Studio adapter) run with allowOutsideProjectAssets, which copied every existing file that markup named outside the project into compiled/hf-ext, of any type and from any folder, and served project links that lead outside without looking at where they lead. Publish already applies a policy here: outside references are kept, private locations are left out, and only image, video, audio, font, CSS and JS files are copied. Local renders now follow the same policy. The publish predicates (private location, copied file type, left-out wording) move from the CLI to a new `@hyperframes/studio-server/outside-file` subpath, since the producer cannot import the CLI and already depends on studio-server. The CLI re-exports them, so publish behaves as before. The media extension sets move with them because the copied-type list includes them. The producer decides with one predicate, `outsideAssetRefusal` in utils/paths.ts, and applies it wherever allowOutsideProjectAssets lets an outside file in, so no entry point can skip it: - collectExternalAssets: src/href/url() references, which covers <a href> and <link href> - dropMediaOutsideProject: video/audio/image sources in any spelling the media resolver accepts (file: URLs, percent-encoding), so nothing probes or extracts them - the render file server: project links that lead outside - the distributed plan's project copy: the same links (range render) - GIF/HEIC prep and sub-composition links: private locations only, since those inputs are converted or compiled rather than copied as they are Each left-out file logs the publish wording with "left out of the render". To reproduce on main: a project whose index.html references ../../.ssh/id_key through <link href>, <a href> and url(), with a project link to a file in .ssh, rendered locally, gets those files copied into compiled/hf-ext and served. Comment lines: the four in studio-server's outsideFile.ts moved verbatim from the CLI files they replace. Two are new, in utils/paths.ts. The one on outsideAssetRefusal states the policy once, so its six call sites carry no comment. The one on refusesOutsideLink says its boolean means "refused, and already warned", which the name alone does not say. Comment ratchet is clean. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): keep outsideFileType as a re-export of the shared module The base repository's deletion guard rejects removing a file that exists on main unless git pairs it as a rename. outsideFileType.ts moved into studio-server/outside-file with enough added around it that git no longer pairs the two, so CI's "Reject accidental file deletions" step failed. The file stays as a two-name re-export and publish keeps importing from it, so publishProject.ts and its file-type test are back to main. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore(producer): raise the unpacked size budgets for the outside-file policy CI's packed-manifest check measured @hyperframes/producer at 3,244,665 bytes unpacked (budget 3,210,000) and @hyperframes/studio-server at 653,506 (budget 646,000), both just past the 1% tolerance. Both packages were already at the edge on main. Measured locally with npm pack --dry-run, main is 3,237,162 for producer and 652,502 for studio-server, so studio-server on main is already at its ceiling. This branch adds about 7.9 KB unpacked to producer, because the new checks sit in the compiler, render stages and file server, which each producer entry bundles. It adds about 5.5 KB to studio-server for the new outside-file entry and its declarations. No source maps, tests or assets were added to either tarball. The budgets go to 3,245,000 and 654,000, just above CI's measurements, the same way #7579 raised producer's budget for the project-folder checks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(producer): apply the outside-file policy to embedded local fonts A project font file that links outside the project is now left out of font embedding with the same warning the file server gives, instead of being inlined into the compiled HTML. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0bce3315e6
commit
2c2b399bfa
@@ -33,7 +33,11 @@ import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg
|
||||
import { probeAudio } from "../audio/analysis/decode.js";
|
||||
import { formatBytes } from "../ui/format.js";
|
||||
import { bytesMatchExtension } from "./offloadSniff.js";
|
||||
import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "./mediaExtensions.js";
|
||||
import {
|
||||
AUDIO_EXTS,
|
||||
CONVERTIBLE_IMAGE_EXTS,
|
||||
VIDEO_EXTS,
|
||||
} from "@hyperframes/studio-server/outside-file";
|
||||
import { type PlayedSegment, planTrim } from "./offloadTrim.js";
|
||||
import {
|
||||
archiveKey,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { isAbsolute, relative } from "node:path";
|
||||
import { type LeftOutReason, leftOutWarning } from "@hyperframes/studio-server/outside-file";
|
||||
import { realFilePath } from "@hyperframes/studio-server/safe-path";
|
||||
|
||||
/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */
|
||||
export type LeftOutReason = "blocklisted" | "received" | "file_type";
|
||||
export { type LeftOutReason, leftOutWarning };
|
||||
|
||||
/** One file an upload left out. `path` is from the project folder; `levels_up` is null on another drive. */
|
||||
export interface LeftOutReference {
|
||||
@@ -15,17 +15,6 @@ export interface LeftOutReference {
|
||||
|
||||
export type OnLeftOut = (warning: string, reference: LeftOutReference) => void;
|
||||
|
||||
const WHY: Record<LeftOutReason, string> = {
|
||||
blocklisted: "which is outside the project in a private location",
|
||||
received: "which leads outside the project and came in with an imported project",
|
||||
file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file",
|
||||
};
|
||||
|
||||
export function leftOutWarning(file: string, reference: string, reason: LeftOutReason): string {
|
||||
const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the upload.`;
|
||||
return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning;
|
||||
}
|
||||
|
||||
export function leftOutReference(
|
||||
projectDir: string,
|
||||
file: string,
|
||||
|
||||
@@ -1,22 +1,4 @@
|
||||
import { extname } from "node:path";
|
||||
import { MIME_TYPES } from "@hyperframes/studio-server/mime";
|
||||
import { realFilePath } from "@hyperframes/studio-server/safe-path";
|
||||
import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "../cloud/mediaExtensions.js";
|
||||
|
||||
const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/;
|
||||
|
||||
/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types,
|
||||
* and the media cloud render re-encodes or converts. */
|
||||
export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set([
|
||||
...Object.entries(MIME_TYPES)
|
||||
.filter(([, type]) => COPIED_TYPE.test(type))
|
||||
.map(([extension]) => extension),
|
||||
...VIDEO_EXTS,
|
||||
...AUDIO_EXTS,
|
||||
...CONVERTIBLE_IMAGE_EXTS,
|
||||
]);
|
||||
|
||||
const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase());
|
||||
|
||||
export const isCopiedOutsideFileType = (absPath: string): boolean =>
|
||||
hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath));
|
||||
export {
|
||||
isCopiedOutsideFileType,
|
||||
OUTSIDE_FILE_EXTENSIONS,
|
||||
} from "@hyperframes/studio-server/outside-file";
|
||||
|
||||
@@ -1,66 +1,13 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, isAbsolute, join, relative, sep } from "node:path";
|
||||
import { join, relative } from "node:path";
|
||||
import {
|
||||
isEnvOrGitPath,
|
||||
isPrivateOutsideFile,
|
||||
linksOutside,
|
||||
} from "@hyperframes/studio-server/outside-file";
|
||||
import { isPrivateProjectFile, realFilePath } from "@hyperframes/studio-server/safe-path";
|
||||
|
||||
const ENV_FILE = /^\.env([._-].*|rc)?$/;
|
||||
|
||||
export function isEnvOrGitPath(path: string): boolean {
|
||||
const segments = path.toLowerCase().split(/[\\/]/);
|
||||
return segments.includes(".git") || ENV_FILE.test(segments.at(-1) ?? "");
|
||||
}
|
||||
|
||||
const SYSTEM_DIRS = ["/etc", "/proc", "/sys", "/dev", "/private/etc"];
|
||||
|
||||
const isOutside = (fromDir: string) =>
|
||||
fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir);
|
||||
|
||||
function inPrivateLocation(projectDir: string, path: string): boolean {
|
||||
const posixPath = path.replaceAll("\\", "/");
|
||||
if (SYSTEM_DIRS.some((dir) => posixPath === dir || posixPath.startsWith(`${dir}/`))) return true;
|
||||
const fromParent = relative(dirname(projectDir), path);
|
||||
return (
|
||||
(isOutside(fromParent) ? path : fromParent).split(/[\\/]/).some((s) => s.startsWith(".")) ||
|
||||
inHomeAppData(projectDir, path)
|
||||
);
|
||||
}
|
||||
|
||||
function isUnder(path: string, ...dir: string[]): boolean {
|
||||
const lowerPath = path.toLowerCase();
|
||||
const lowerDir = join(homedir(), ...dir).toLowerCase();
|
||||
return lowerPath === lowerDir || lowerPath.startsWith(`${lowerDir}${sep}`);
|
||||
}
|
||||
|
||||
const CLOUD_DRIVE_FOLDERS = ["Mobile Documents", "CloudStorage"];
|
||||
|
||||
function homeAppDataFolder(path: string): string | null {
|
||||
if (isUnder(path, "AppData")) return "AppData";
|
||||
const cloudDrive = CLOUD_DRIVE_FOLDERS.some((folder) => isUnder(path, "Library", folder));
|
||||
if (isUnder(path, "Library") && !cloudDrive) return "Library";
|
||||
return null;
|
||||
}
|
||||
|
||||
function inHomeAppData(projectDir: string, path: string): boolean {
|
||||
const folder = homeAppDataFolder(path);
|
||||
const parent = dirname(projectDir);
|
||||
return (
|
||||
folder !== null && (homeAppDataFolder(parent) !== folder || isOutside(relative(parent, path)))
|
||||
);
|
||||
}
|
||||
|
||||
/** An existing file outside the project that publish leaves out: env files, Git data, hidden, system and app-data folders. */
|
||||
export function isPrivateOutsideFile(projectDir: string, absPath: string): boolean {
|
||||
const realPath = realFilePath(absPath);
|
||||
return (
|
||||
isEnvOrGitPath(absPath) ||
|
||||
isEnvOrGitPath(realPath) ||
|
||||
inPrivateLocation(projectDir, absPath) ||
|
||||
inPrivateLocation(realFilePath(projectDir), realPath)
|
||||
);
|
||||
}
|
||||
|
||||
export const linksOutside = (projectDir: string, filePath: string): boolean =>
|
||||
isOutside(relative(realFilePath(projectDir), realFilePath(filePath)));
|
||||
export { isEnvOrGitPath, isPrivateOutsideFile, linksOutside };
|
||||
|
||||
export function linksToPrivateOutsideFile(projectDir: string, filePath: string): boolean {
|
||||
return linksOutside(projectDir, filePath) && isPrivateOutsideFile(projectDir, filePath);
|
||||
|
||||
@@ -7,7 +7,7 @@ import { parseAnimatedGifMetadata, type AnimatedGifMetadata } from "@hyperframes
|
||||
import { DEFAULT_VP9_CPU_USED, runFfmpeg, renameWithRetry } from "@hyperframes/engine";
|
||||
import { namesAnotherNetworkRoot } from "@hyperframes/parsers/asset-resolution";
|
||||
import { isHttpUrl } from "../utils/urlDownloader.js";
|
||||
import { escapesAssetRoot } from "../utils/paths.js";
|
||||
import { escapesAssetRoot, outsideAssetRefusal } from "../utils/paths.js";
|
||||
import { encoderFailureError } from "./render/encoderInterruption.js";
|
||||
|
||||
const PREPARED_GIF_SUBDIR = "_animated_gif";
|
||||
@@ -109,7 +109,8 @@ export function resolveLocalImageSourcePath(
|
||||
];
|
||||
|
||||
const allowed = (candidate: string) =>
|
||||
!escapesAssetRoot(candidate, options.assetRoot) ||
|
||||
(!escapesAssetRoot(candidate, options.assetRoot) &&
|
||||
outsideAssetRefusal(options.projectDir, candidate) !== "blocklisted") ||
|
||||
!escapesAssetRoot(candidate, options.downloadDir);
|
||||
return candidates.find((candidate) => existsSync(candidate) && allowed(candidate)) ?? null;
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
type VideoElement,
|
||||
} from "@hyperframes/engine";
|
||||
import { withMediaProbeSlot } from "../utils/mediaProbeConcurrency.js";
|
||||
import { escapesAssetRoot } from "../utils/paths.js";
|
||||
import { escapesAssetRoot, leftOutOfRender, outsideAssetRefusal } from "../utils/paths.js";
|
||||
import type { ProducerLogger } from "../logger.js";
|
||||
|
||||
export const ASSET_MEDIA_TYPE_MISMATCH = "ASSET_MEDIA_TYPE_MISMATCH" as const;
|
||||
@@ -97,6 +97,7 @@ export function dropMediaOutsideProject(input: {
|
||||
workDir: string;
|
||||
composition: { videos: VideoElement[]; audios: AudioElement[]; images: ImageElement[] };
|
||||
log: ProducerLogger;
|
||||
allowOutsideProjectAssets?: boolean;
|
||||
}): void {
|
||||
const { projectDir, workDir, composition, log } = input;
|
||||
const compiledDir = join(workDir, "compiled");
|
||||
@@ -105,6 +106,11 @@ export function dropMediaOutsideProject(input: {
|
||||
if (!src || isRemoteOrInlineSource(src)) return true;
|
||||
const path = resolveProjectRelativeSrc(src, projectDir, compiledDir);
|
||||
if (roots.some((root) => !escapesAssetRoot(path, root))) return true;
|
||||
if (input.allowOutsideProjectAssets) {
|
||||
const refusal = outsideAssetRefusal(projectDir, path);
|
||||
if (refusal) log.warn(`[Render] ${leftOutOfRender(`Media "${id}"`, src, refusal)}`);
|
||||
return refusal === null;
|
||||
}
|
||||
log.warn(`[Render] Media "${id}" source is missing: it is outside the project folder`, { src });
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -64,6 +64,7 @@ import {
|
||||
declareProbeFactsUrl,
|
||||
} from "@hyperframes/engine";
|
||||
import { defaultLogger, type ProducerLogger } from "../../logger.js";
|
||||
import { refusesOutsideLink } from "../../utils/paths.js";
|
||||
import {
|
||||
applyRenderWarningPolicy,
|
||||
type RenderJob,
|
||||
@@ -1277,7 +1278,9 @@ async function compilePlanComposition(
|
||||
// false-positive every descendant.
|
||||
const rel = relative(projectDir, src);
|
||||
if (rel === "" || rel.startsWith("..")) return true;
|
||||
if (!job.config.allowOutsideProjectAssets && !isSafePath(projectDir, src)) return false;
|
||||
if (job.config.allowOutsideProjectAssets) {
|
||||
if (refusesOutsideLink(projectDir, src, (line) => log.warn(`[Plan] ${line}`))) return false;
|
||||
} else if (!isSafePath(projectDir, src)) return false;
|
||||
const firstSegment = rel.split(sep, 1)[0];
|
||||
return firstSegment === undefined || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment);
|
||||
},
|
||||
|
||||
@@ -34,6 +34,7 @@ import { openRangeEnd } from "@hyperframes/studio-server/media-range";
|
||||
import { getVerifiedHyperframeRuntimeSource } from "./hyperframeRuntimeLoader.js";
|
||||
import { getHfEarlyStub } from "../generated/hf-early-stub-inline.js";
|
||||
import { defaultLogger, type ProducerLogger } from "../logger.js";
|
||||
import { refusesOutsideLink } from "../utils/paths.js";
|
||||
|
||||
const readFileAsync = promisify(readFile);
|
||||
|
||||
@@ -812,6 +813,7 @@ export function createFileServer(options: FileServerOptions): Promise<FileServer
|
||||
const bodyScripts = options.bodyScripts ?? [buildRenderModeScript(options.fps), HF_BRIDGE_SCRIPT];
|
||||
|
||||
const app = new Hono();
|
||||
const warnLeftOut = (line: string) => console.warn(`[FileServer] ${line}`);
|
||||
|
||||
app.get("/*", async (c) => {
|
||||
let requestPath = c.req.path;
|
||||
@@ -840,7 +842,10 @@ export function createFileServer(options: FileServerOptions): Promise<FileServer
|
||||
for (const root of compiledDir ? [compiledDir, projectDir] : [projectDir]) {
|
||||
const candidate = join(root, relativePath);
|
||||
const resolveSymlinks = !allowOutsideProjectAssets && root !== compiledDir;
|
||||
if (isPathInside(candidate, root, { resolveSymlinks })) file = openRegularFile(candidate);
|
||||
if (!isPathInside(candidate, root, { resolveSymlinks })) continue;
|
||||
const checksLinks = allowOutsideProjectAssets && root !== compiledDir;
|
||||
if (checksLinks && refusesOutsideLink(root, candidate, warnLeftOut)) continue;
|
||||
file = openRegularFile(candidate);
|
||||
if (file) break;
|
||||
}
|
||||
|
||||
|
||||
@@ -80,7 +80,14 @@ import {
|
||||
} from "@hyperframes/parsers/asset-resolution";
|
||||
import { isPrivateFileUnder, isVcsOrSecretPath } from "@hyperframes/core/safe-path";
|
||||
import { extractMediaMetadata, extractAudioMetadata } from "../utils/ffprobe.js";
|
||||
import { escapesAssetRoot, isPathInside, toExternalAssetKey } from "../utils/paths.js";
|
||||
import {
|
||||
escapesAssetRoot,
|
||||
isPathInside,
|
||||
leftOutOfRender,
|
||||
outsideAssetRefusal,
|
||||
refusesOutsideLink,
|
||||
toExternalAssetKey,
|
||||
} from "../utils/paths.js";
|
||||
import { collectHiddenVideoIds, collectRenderMedia } from "./renderMediaCollector.js";
|
||||
import {
|
||||
type VideoElement,
|
||||
@@ -277,6 +284,10 @@ function assertSubCompositionsUsable(
|
||||
problems.push({ srcPath, detail: "it is outside the project folder" });
|
||||
continue;
|
||||
}
|
||||
if (outsideAssetRefusal(projectDir, filePath) === "blocklisted") {
|
||||
problems.push({ srcPath, detail: "it is outside the project in a private location" });
|
||||
continue;
|
||||
}
|
||||
if (isPrivateCompositionFile(projectDir, filePath)) {
|
||||
problems.push({ srcPath, detail: "it is git metadata or an env file" });
|
||||
continue;
|
||||
@@ -1403,9 +1414,12 @@ export async function inlineExternalScripts(html: string): Promise<string> {
|
||||
export function collectExternalAssets(
|
||||
html: string,
|
||||
projectDir: string,
|
||||
log: ProducerLogger = defaultLogger,
|
||||
file = "index.html",
|
||||
): { html: string; externalAssets: Map<string, string> } {
|
||||
const absProjectDir = resolve(projectDir);
|
||||
const externalAssets = new Map<string, string>();
|
||||
const leftOut = new Set<string>();
|
||||
|
||||
function processPath(rawPath: string): string | null {
|
||||
const trimmed = rawPath.trim();
|
||||
@@ -1415,6 +1429,12 @@ export function collectExternalAssets(
|
||||
return null; // inside projectDir, file server handles this
|
||||
}
|
||||
if (!existsSync(absPath)) return null;
|
||||
const refusal = outsideAssetRefusal(absProjectDir, absPath);
|
||||
if (refusal) {
|
||||
if (!leftOut.has(trimmed)) log.warn(`[Compiler] ${leftOutOfRender(file, trimmed, refusal)}`);
|
||||
leftOut.add(trimmed);
|
||||
return null;
|
||||
}
|
||||
// resolve() already canonicalises the path (no .. components remain);
|
||||
// toExternalAssetKey() produces a cross-platform relative key that
|
||||
// `path.join(compileDir, key)` cannot escape on any OS.
|
||||
@@ -1957,6 +1977,7 @@ async function embedLocalFontFaces(
|
||||
html: string,
|
||||
projectDir: string,
|
||||
assetRoot: string | undefined,
|
||||
log: ProducerLogger = defaultLogger,
|
||||
): Promise<string> {
|
||||
const { fontToDataUri: toDataUri } = await import("@hyperframes/core/fonts/embed");
|
||||
const styleBlockRe = /<style\b[^>]*>([\s\S]*?)<\/style>/gi;
|
||||
@@ -1980,6 +2001,11 @@ async function embedLocalFontFaces(
|
||||
if (!localPath || embeddedPaths.has(localPath)) continue;
|
||||
const absPath = localPath.startsWith("/") ? localPath : resolve(projectDir, localPath);
|
||||
if (!isPathInside(absPath, projectDir) || escapesAssetRoot(absPath, assetRoot)) continue;
|
||||
// A project font that is a link leading outside obeys the same policy as the file server.
|
||||
if (refusesOutsideLink(projectDir, absPath, (line) => log.warn(`[Compiler] ${line}`))) {
|
||||
embeddedPaths.add(localPath);
|
||||
continue;
|
||||
}
|
||||
const ext = absPath.match(/\.(woff2?|ttf|otf|ttc)$/i)?.[1]?.toLowerCase() ?? "ttf";
|
||||
try {
|
||||
if (fileBackedAbsolutePaths.has(absPath)) {
|
||||
@@ -2166,6 +2192,11 @@ function bindVariableMediaSources(html: string, variables: Record<string, unknow
|
||||
return changed ? document.toString() : html;
|
||||
}
|
||||
|
||||
const entryFileName = (projectDir: string, htmlPath: string) =>
|
||||
isPathInside(htmlPath, projectDir)
|
||||
? relative(projectDir, htmlPath).split(sep).join("/")
|
||||
: basename(htmlPath);
|
||||
|
||||
/**
|
||||
* Compile an HTML composition project into a single self-contained HTML string
|
||||
* with all media metadata resolved.
|
||||
@@ -2356,13 +2387,23 @@ export async function compileForRender(
|
||||
defaultLogger.debug(`[Compiler] Prepared ${preparedGifs.length} animated GIF input(s) as WebM`);
|
||||
}
|
||||
|
||||
const embeddedHtml = await embedLocalFontFaces(htmlWithPreparedGifs, projectDir, assetRoot);
|
||||
const embeddedHtml = await embedLocalFontFaces(
|
||||
htmlWithPreparedGifs,
|
||||
projectDir,
|
||||
assetRoot,
|
||||
options.log,
|
||||
);
|
||||
|
||||
// Collect assets that resolve outside projectDir (e.g. ../shared-assets/hero.png).
|
||||
// These can't be served by the file server, so we map them to paths the
|
||||
// orchestrator will copy into the compiled output directory.
|
||||
const { html, externalAssets } = options.allowOutsideProjectAssets
|
||||
? collectExternalAssets(embeddedHtml, projectDir)
|
||||
? collectExternalAssets(
|
||||
embeddedHtml,
|
||||
projectDir,
|
||||
options.log,
|
||||
entryFileName(projectDir, htmlPath),
|
||||
)
|
||||
: { html: embeddedHtml, externalAssets: new Map<string, string>() };
|
||||
|
||||
for (const [relPath, absPath] of remoteMediaAssets) {
|
||||
|
||||
@@ -7,12 +7,13 @@ import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { resolveConfig } from "@hyperframes/engine";
|
||||
import { runCompileStage } from "./render/stages/compileStage.js";
|
||||
@@ -163,6 +164,54 @@ async function compileInProcess(projectDir: string, allowOutsideProjectAssets?:
|
||||
return { result, compiledDir, extract, serve, warnings };
|
||||
}
|
||||
|
||||
function privateOutsideProject() {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-private-outside-"));
|
||||
roots.push(root);
|
||||
const projectDir = join(root, "work", "project");
|
||||
const [ssh, shared] = [join(root, ".ssh"), join(root, "work", "shared")];
|
||||
for (const dir of [projectDir, ssh, shared]) mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(ssh, "id_key"), "PRIVATE");
|
||||
writeFileSync(join(ssh, "pic.png"), PNG_1X1);
|
||||
copyFileSync(tool.clip, join(ssh, "clip.mp4"));
|
||||
const gif = ["-v", "error", "-f", "lavfi", "-i", "testsrc=size=16x16:rate=10", "-t", "0.3"];
|
||||
expect(spawnSync("ffmpeg", [...gif, join(ssh, "anim.gif")]).status).toBe(0);
|
||||
writeFileSync(join(ssh, "scene.html"), scene);
|
||||
writeFileSync(join(shared, "logo.png"), PNG_1X1);
|
||||
writeFileSync(join(shared, "notes.txt"), "notes");
|
||||
symlinkSync(join(ssh, "pic.png"), join(projectDir, "link.png"));
|
||||
symlinkSync(join(shared, "logo.png"), join(projectDir, "keep.png"));
|
||||
symlinkSync(join(ssh, "id_key"), join(projectDir, "brand.ttf"));
|
||||
symlinkSync(join(shared, "notes.txt"), join(projectDir, "notes.ttf"));
|
||||
const pic = join(ssh, "pic.png");
|
||||
const video = (id: string, src: string) =>
|
||||
`<video id="${id}" src="${src}" data-start="0" data-duration="1" muted></video>`;
|
||||
writeFileSync(
|
||||
join(projectDir, "index.html"),
|
||||
`<!doctype html><html><head><link rel="stylesheet" href="../../.ssh/id_key">
|
||||
<style>.bg { background-image: url("../../.ssh/pic.png"); }
|
||||
@font-face { font-family: B; src: url("brand.ttf"); } @font-face { font-family: N; src: url("notes.ttf"); }</style></head><body>
|
||||
<div data-composition-id="root" data-width="320" data-height="240" data-duration="1">
|
||||
<img src="../../.ssh/pic.png"><img src="${pic}"><img src="${pathToFileURL(pic).href}">
|
||||
<img src="..%2F..%2F.ssh%2Fpic.png"><div style="background: url(${pic})"></div>
|
||||
<a href="../../.ssh/id_key">key</a><a href="../shared/notes.txt">notes</a>
|
||||
<img src="../shared/logo.png"><img src="../../.ssh/anim.gif">${video("dotdot", "../../.ssh/clip.mp4")}
|
||||
${video("fileurl", pathToFileURL(join(ssh, "clip.mp4")).href)}
|
||||
</div></body></html>`,
|
||||
);
|
||||
const key = (path: string) => toExternalAssetKey(resolve(path));
|
||||
return {
|
||||
root,
|
||||
projectDir,
|
||||
privateKeys: [key(pic), key(join(ssh, "id_key"))],
|
||||
logo: key(join(shared, "logo.png")),
|
||||
};
|
||||
}
|
||||
|
||||
const filesUnder = (dir: string): string[] =>
|
||||
readdirSync(dir, { recursive: true, withFileTypes: true })
|
||||
.filter((entry) => !entry.isDirectory())
|
||||
.map((entry) => relative(dir, join(entry.parentPath, entry.name)).replaceAll("\\", "/"));
|
||||
|
||||
const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
|
||||
const config = { fps: 30 as const, width: 320, height: 240, format: "mp4" as const };
|
||||
|
||||
@@ -225,6 +274,71 @@ describe.skipIf(!HAS_FFMPEG)("render servers keep markup inside the project", ()
|
||||
},
|
||||
);
|
||||
|
||||
it("trusted local renders leave out private and non-media files outside the project", async () => {
|
||||
const { root, projectDir, privateKeys, logo } = privateOutsideProject();
|
||||
tool.calls.length = 0;
|
||||
const { result, compiledDir, serve, warnings } = await compileInProcess(projectDir, true);
|
||||
|
||||
expect(filesUnder(compiledDir).sort()).toEqual([logo, "index.html"]);
|
||||
expect(result.composition.videos).toEqual([]);
|
||||
expect(tool.calls.filter((call) => call.includes(".ssh") && !call.includes("ffprobe"))).toEqual(
|
||||
[],
|
||||
);
|
||||
const warned = warnings.join("\n");
|
||||
const references = ["id_key", "pic.png", "clip.mp4", "anim.gif"].map((f) => `../../.ssh/${f}`);
|
||||
for (const reference of references) {
|
||||
expect(warned).toContain(
|
||||
`index.html references ${reference}, which is outside the project in a private location; it was left out of the render.`,
|
||||
);
|
||||
}
|
||||
expect(warned).toMatch(
|
||||
/Media "fileurl" references file:.*\.ssh\/clip\.mp4, which is outside the project in a private/,
|
||||
);
|
||||
expect(warned).toContain(
|
||||
"../shared/notes.txt, which is outside the project and is not an image",
|
||||
);
|
||||
expect(warned).toMatch(
|
||||
/brand\.ttf references .*\.ssh\/id_key, which is outside the project in a private/,
|
||||
);
|
||||
expect(warned).toMatch(
|
||||
/notes\.ttf references .*notes\.txt, which is outside the project and is not an image/,
|
||||
);
|
||||
const compiledHtml = readFileSync(join(compiledDir, "index.html"), "utf-8");
|
||||
expect(compiledHtml).not.toContain(Buffer.from("PRIVATE").toString("base64"));
|
||||
expect(compiledHtml).not.toContain(Buffer.from("notes").toString("base64"));
|
||||
expect(compiledHtml).toContain('url("brand.ttf")');
|
||||
expect(
|
||||
await statusOf(await serve(), [
|
||||
...privateKeys.map((k) => `/${k}`),
|
||||
"/link.png",
|
||||
`/${logo}`,
|
||||
"/keep.png",
|
||||
"/brand.ttf",
|
||||
]),
|
||||
).toEqual([404, 404, 404, 200, 200, 404]);
|
||||
|
||||
const planDir = join(root, "plan");
|
||||
await plan(projectDir, config, planDir, { allowOutsideProjectAssets: true });
|
||||
const planned = filesUnder(join(planDir, "compiled"));
|
||||
expect(planned.filter((file) => /\.ssh|notes|link\.png|brand\.ttf/.test(file))).toEqual([]);
|
||||
expect(planned).toEqual(expect.arrayContaining([logo, "keep.png"]));
|
||||
});
|
||||
|
||||
it("trusted local renders refuse a linked composition in a private location", async () => {
|
||||
const { root, projectDir } = privateOutsideProject();
|
||||
mkdirSync(join(projectDir, "scenes"));
|
||||
symlinkSync(join(root, ".ssh", "scene.html"), join(projectDir, "scenes", "a.html"));
|
||||
const host = `<div data-composition-src="scenes/a.html" data-start="0" data-duration="1"></div>`;
|
||||
const html = readFileSync(join(projectDir, "index.html"), "utf-8").replace(
|
||||
"</div></body>",
|
||||
`${host}</div></body>`,
|
||||
);
|
||||
writeFileSync(join(projectDir, "index.html"), html);
|
||||
await expect(compileInProcess(projectDir, true)).rejects.toThrow(
|
||||
/outside the project in a private location/,
|
||||
);
|
||||
});
|
||||
|
||||
it("trusted local renders still import outside files", async () => {
|
||||
const { root, projectDir, secretKey } = outsideProject(false);
|
||||
const { compiledDir, serve } = await compileInProcess(projectDir, true);
|
||||
|
||||
@@ -377,9 +377,13 @@ export async function runCompileStage(input: CompileStageInput): Promise<Compile
|
||||
width: compiled.width,
|
||||
height: compiled.height,
|
||||
};
|
||||
if (!job.config.allowOutsideProjectAssets) {
|
||||
dropMediaOutsideProject({ projectDir, workDir, composition, log });
|
||||
}
|
||||
dropMediaOutsideProject({
|
||||
projectDir,
|
||||
workDir,
|
||||
composition,
|
||||
log,
|
||||
allowOutsideProjectAssets: job.config.allowOutsideProjectAssets,
|
||||
});
|
||||
await preflightCompositionAssetMediaTypes({
|
||||
projectDir,
|
||||
compiledDir: join(workDir, "compiled"),
|
||||
|
||||
@@ -842,9 +842,13 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
}
|
||||
}
|
||||
|
||||
if (!job.config.allowOutsideProjectAssets) {
|
||||
dropMediaOutsideProject({ projectDir, workDir, composition, log });
|
||||
}
|
||||
dropMediaOutsideProject({
|
||||
projectDir,
|
||||
workDir,
|
||||
composition,
|
||||
log,
|
||||
allowOutsideProjectAssets: job.config.allowOutsideProjectAssets,
|
||||
});
|
||||
await preflightCompositionAssetMediaTypes({
|
||||
projectDir,
|
||||
compiledDir: join(workDir, "compiled"),
|
||||
|
||||
@@ -5,12 +5,22 @@
|
||||
import {
|
||||
basename,
|
||||
join,
|
||||
sep,
|
||||
resolve as nodeResolve,
|
||||
relative as nodeRelative,
|
||||
isAbsolute as nodeIsAbsolute,
|
||||
} from "node:path";
|
||||
import { statSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { isSafePath } from "@hyperframes/core/safe-path";
|
||||
import {
|
||||
isCopiedOutsideFileType,
|
||||
isPrivateOutsideFile,
|
||||
type LeftOutReason,
|
||||
leftOutWarning,
|
||||
linksOutside,
|
||||
} from "@hyperframes/studio-server/outside-file";
|
||||
import { realFilePath } from "@hyperframes/studio-server/safe-path";
|
||||
|
||||
export interface RenderPaths {
|
||||
absoluteProjectDir: string;
|
||||
@@ -70,6 +80,26 @@ export function escapesAssetRoot(path: string, assetRoot: string | undefined): b
|
||||
return assetRoot !== undefined && !isSafePath(assetRoot, path);
|
||||
}
|
||||
|
||||
/** Why a trusted local render leaves out `path`, which leads outside `projectDir`, as publish does; null to bring it in. */
|
||||
export function outsideAssetRefusal(projectDir: string, path: string): LeftOutReason | null {
|
||||
if (!linksOutside(projectDir, path)) return null;
|
||||
if (isPrivateOutsideFile(projectDir, path)) return "blocklisted";
|
||||
const folder = statSync(path, { throwIfNoEntry: false })?.isDirectory();
|
||||
return folder || isCopiedOutsideFileType(path) ? null : "file_type";
|
||||
}
|
||||
|
||||
export const leftOutOfRender = (file: string, reference: string, reason: LeftOutReason): string =>
|
||||
leftOutWarning(file, reference, reason, "render");
|
||||
|
||||
/** True, after a warning, when a trusted local render must not serve or copy `path`, a project file that leads outside. */
|
||||
export function refusesOutsideLink(projectDir: string, path: string, warn: (line: string) => void) {
|
||||
const refusal = outsideAssetRefusal(projectDir, path);
|
||||
if (!refusal) return false;
|
||||
const file = nodeRelative(projectDir, path).split(sep).join("/");
|
||||
warn(leftOutOfRender(file, realFilePath(path), refusal));
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a safe, cross-platform relative key for an absolute asset path
|
||||
* that lives outside the project directory.
|
||||
|
||||
@@ -110,6 +110,12 @@
|
||||
"types": "./dist/helpers/mime.d.ts",
|
||||
"environments": ["bun", "node"]
|
||||
},
|
||||
"./outside-file": {
|
||||
"source": "./src/helpers/outsideFile.ts",
|
||||
"runtime": "./dist/helpers/outsideFile.js",
|
||||
"types": "./dist/helpers/outsideFile.d.ts",
|
||||
"environments": ["bun", "node"]
|
||||
},
|
||||
"./media-range": {
|
||||
"source": "./src/helpers/mediaRange.ts",
|
||||
"runtime": "./dist/helpers/mediaRange.js",
|
||||
|
||||
@@ -117,6 +117,12 @@
|
||||
"import": "./src/helpers/mime.ts",
|
||||
"types": "./src/helpers/mime.ts"
|
||||
},
|
||||
"./outside-file": {
|
||||
"bun": "./src/helpers/outsideFile.ts",
|
||||
"node": "./dist/helpers/outsideFile.js",
|
||||
"import": "./src/helpers/outsideFile.ts",
|
||||
"types": "./src/helpers/outsideFile.ts"
|
||||
},
|
||||
"./media-range": {
|
||||
"bun": "./src/helpers/mediaRange.ts",
|
||||
"node": "./dist/helpers/mediaRange.js",
|
||||
@@ -196,6 +202,10 @@
|
||||
"import": "./dist/helpers/mime.js",
|
||||
"types": "./dist/helpers/mime.d.ts"
|
||||
},
|
||||
"./outside-file": {
|
||||
"import": "./dist/helpers/outsideFile.js",
|
||||
"types": "./dist/helpers/outsideFile.d.ts"
|
||||
},
|
||||
"./media-range": {
|
||||
"import": "./dist/helpers/mediaRange.js",
|
||||
"types": "./dist/helpers/mediaRange.d.ts"
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, extname, isAbsolute, join, relative, sep } from "node:path";
|
||||
import { MIME_TYPES } from "./mime.js";
|
||||
import { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS } from "./mediaExtensions.js";
|
||||
import { realFilePath } from "./safePath.js";
|
||||
|
||||
export { AUDIO_EXTS, CONVERTIBLE_IMAGE_EXTS, VIDEO_EXTS };
|
||||
|
||||
const COPIED_TYPE = /^(?:image|video|audio|font)\/|^text\/(?:css|javascript);/;
|
||||
|
||||
/** What an upload copies from outside the project: the preview server's image, video, audio, font, CSS and JS types,
|
||||
* and the media cloud render re-encodes or converts. */
|
||||
export const OUTSIDE_FILE_EXTENSIONS: ReadonlySet<string> = new Set([
|
||||
...Object.entries(MIME_TYPES)
|
||||
.filter(([, type]) => COPIED_TYPE.test(type))
|
||||
.map(([extension]) => extension),
|
||||
...VIDEO_EXTS,
|
||||
...AUDIO_EXTS,
|
||||
...CONVERTIBLE_IMAGE_EXTS,
|
||||
]);
|
||||
|
||||
const hasCopiedType = (path: string) => OUTSIDE_FILE_EXTENSIONS.has(extname(path).toLowerCase());
|
||||
|
||||
export const isCopiedOutsideFileType = (absPath: string): boolean =>
|
||||
hasCopiedType(absPath) && hasCopiedType(realFilePath(absPath));
|
||||
|
||||
const ENV_FILE = /^\.env([._-].*|rc)?$/;
|
||||
|
||||
export function isEnvOrGitPath(path: string): boolean {
|
||||
const segments = path.toLowerCase().split(/[\\/]/);
|
||||
return segments.includes(".git") || ENV_FILE.test(segments.at(-1) ?? "");
|
||||
}
|
||||
|
||||
const SYSTEM_DIRS = ["/etc", "/proc", "/sys", "/dev", "/private/etc"];
|
||||
|
||||
const isOutside = (fromDir: string) =>
|
||||
fromDir === ".." || fromDir.startsWith(`..${sep}`) || isAbsolute(fromDir);
|
||||
|
||||
function inPrivateLocation(projectDir: string, path: string): boolean {
|
||||
const posixPath = path.replaceAll("\\", "/");
|
||||
if (SYSTEM_DIRS.some((dir) => posixPath === dir || posixPath.startsWith(`${dir}/`))) return true;
|
||||
const fromParent = relative(dirname(projectDir), path);
|
||||
return (
|
||||
(isOutside(fromParent) ? path : fromParent).split(/[\\/]/).some((s) => s.startsWith(".")) ||
|
||||
inHomeAppData(projectDir, path)
|
||||
);
|
||||
}
|
||||
|
||||
function isUnder(path: string, ...dir: string[]): boolean {
|
||||
const lowerPath = path.toLowerCase();
|
||||
const lowerDir = join(homedir(), ...dir).toLowerCase();
|
||||
return lowerPath === lowerDir || lowerPath.startsWith(`${lowerDir}${sep}`);
|
||||
}
|
||||
|
||||
const CLOUD_DRIVE_FOLDERS = ["Mobile Documents", "CloudStorage"];
|
||||
|
||||
function homeAppDataFolder(path: string): string | null {
|
||||
if (isUnder(path, "AppData")) return "AppData";
|
||||
const cloudDrive = CLOUD_DRIVE_FOLDERS.some((folder) => isUnder(path, "Library", folder));
|
||||
if (isUnder(path, "Library") && !cloudDrive) return "Library";
|
||||
return null;
|
||||
}
|
||||
|
||||
function inHomeAppData(projectDir: string, path: string): boolean {
|
||||
const folder = homeAppDataFolder(path);
|
||||
const parent = dirname(projectDir);
|
||||
return (
|
||||
folder !== null && (homeAppDataFolder(parent) !== folder || isOutside(relative(parent, path)))
|
||||
);
|
||||
}
|
||||
|
||||
/** An existing file outside the project that publish leaves out: env files, Git data, hidden, system and app-data folders. */
|
||||
export function isPrivateOutsideFile(projectDir: string, absPath: string): boolean {
|
||||
const realPath = realFilePath(absPath);
|
||||
return (
|
||||
isEnvOrGitPath(absPath) ||
|
||||
isEnvOrGitPath(realPath) ||
|
||||
inPrivateLocation(projectDir, absPath) ||
|
||||
inPrivateLocation(realFilePath(projectDir), realPath)
|
||||
);
|
||||
}
|
||||
|
||||
export const linksOutside = (projectDir: string, filePath: string): boolean =>
|
||||
isOutside(relative(realFilePath(projectDir), realFilePath(filePath)));
|
||||
|
||||
/** `blocklisted`: a private location; `received`: reached from an imported file; `file_type`: not a type uploads copy. */
|
||||
export type LeftOutReason = "blocklisted" | "received" | "file_type";
|
||||
|
||||
const WHY: Record<LeftOutReason, string> = {
|
||||
blocklisted: "which is outside the project in a private location",
|
||||
received: "which leads outside the project and came in with an imported project",
|
||||
file_type: "which is outside the project and is not an image, video, audio, font, CSS or JS file",
|
||||
};
|
||||
|
||||
export function leftOutWarning(
|
||||
file: string,
|
||||
reference: string,
|
||||
reason: LeftOutReason,
|
||||
from: "upload" | "render" = "upload",
|
||||
): string {
|
||||
const warning = `${file} references ${reference}, ${WHY[reason]}; it was left out of the ${from}.`;
|
||||
return reason === "file_type" ? `${warning} Move it into the project to include it.` : warning;
|
||||
}
|
||||
@@ -19,6 +19,7 @@ export default defineConfig({
|
||||
"helpers/encoderOptions": "src/helpers/encoderOptions.ts",
|
||||
"helpers/mediaRange": "src/helpers/mediaRange.ts",
|
||||
"helpers/mime": "src/helpers/mime.ts",
|
||||
"helpers/outsideFile": "src/helpers/outsideFile.ts",
|
||||
"history/index": "src/history/index.ts",
|
||||
},
|
||||
format: ["esm"],
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
},
|
||||
"@hyperframes/producer": {
|
||||
"packed": 890000,
|
||||
"unpacked": 3210000
|
||||
"unpacked": 3245000
|
||||
},
|
||||
"@hyperframes/sdk": {
|
||||
"packed": 90662,
|
||||
@@ -57,6 +57,6 @@
|
||||
},
|
||||
"@hyperframes/studio-server": {
|
||||
"packed": 168500,
|
||||
"unpacked": 646000
|
||||
"unpacked": 654000
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user