mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): upload archives never include credential or agent-state files (#7830)
* fix(cli): upload archives never include credential or agent-state files Publish, cloud render and feedback stage the project through one path. It now always leaves out package-manager and Git credentials (.npmrc, .yarnrc, .pypirc, pip.conf, .netrc, .git-credentials, .gitconfig), key files (id_rsa*, id_ed25519*, *.pem, *.key, *.p12, *.pfx), .ssh, .aws, .gnupg, .docker, .kube, .azure, .config/gcloud and .config/gh, agent and editor folders (.claude, .codex, .cursor, .agents, .gemini, .vscode, .idea) and .mcp.json, at any depth and in any letter case. This holds whatever .hyperframesignore says and whatever the HTML or CSS references. A reference can bring back a file the ignore rules skipped only when it is an image, video, audio, font, CSS or JS file, or when it is not in a hidden folder. Every file left out this way is reported, and a file the folder walk already reported is not reported again for each reference. * fix(cli): anchor both alternatives of the key-file pattern * fix(cli): a page's own scripts in a hidden folder still upload
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
export {
|
||||
isCopiedOutsideFileType,
|
||||
isUploadedOutsideFileType,
|
||||
UPLOADED_OUTSIDE_FILE_EXTENSIONS,
|
||||
} from "@hyperframes/studio-server/outside-file";
|
||||
|
||||
@@ -13,6 +13,7 @@ import { spawn } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import {
|
||||
isCredentialOrToolStatePath,
|
||||
isEnvOrGitPath,
|
||||
isPrivateCompositionFile,
|
||||
receivedFiles,
|
||||
@@ -52,6 +53,44 @@ describe("isEnvOrGitPath", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("isCredentialOrToolStatePath", () => {
|
||||
it("names logins, keys and agent or editor state at any depth and in any case", () => {
|
||||
for (const path of [
|
||||
".npmrc",
|
||||
"a/b/.YARNRC.yml",
|
||||
"_netrc",
|
||||
"pip.ini",
|
||||
".git-credentials",
|
||||
".ssh/config",
|
||||
"x\\.AWS\\credentials",
|
||||
".config/gcloud/application_default_credentials.json",
|
||||
".config/gh/hosts.yml",
|
||||
"keys/id_ecdsa.pub",
|
||||
"certs/site.PEM",
|
||||
".claude/settings.json",
|
||||
".idea/workspace.xml",
|
||||
"sub/.mcp.json",
|
||||
]) {
|
||||
expect(isCredentialOrToolStatePath(path), path).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves ordinary composition files alone", () => {
|
||||
for (const path of [
|
||||
"assets/keyframes.json",
|
||||
"monkey.png",
|
||||
"id.png",
|
||||
"fonts/key.woff2",
|
||||
".config/theme.json",
|
||||
"claude/logo.svg",
|
||||
"scenes/mcp.json",
|
||||
".media/clip.mp4",
|
||||
]) {
|
||||
expect(isCredentialOrToolStatePath(path), path).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("isPrivateCompositionFile", () => {
|
||||
it("covers env files, Git data and tool records, not composition assets", () => {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-private-file-"));
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
rmSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { dirname, join, relative, resolve, sep } from "node:path";
|
||||
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import {
|
||||
isEnvOrGitPath,
|
||||
isPrivateOutsideFile,
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from "@hyperframes/studio-server/outside-file";
|
||||
import { isPrivateProjectFile, realFilePath } from "@hyperframes/studio-server/safe-path";
|
||||
import { mkdirWithinProject } from "@hyperframes/core";
|
||||
import { projectPathSegments } from "@hyperframes/core/safe-path";
|
||||
import { publishRegistryFile, registryRoot, registryTargetPath } from "../registry/publication.js";
|
||||
|
||||
export { isEnvOrGitPath, isPrivateOutsideFile, linksOutside };
|
||||
@@ -202,11 +203,71 @@ export function receivedRecordStatus(
|
||||
return { path: join(nearest.root, RECEIVED_RECORD), listed: records.flatMap(listedUnder) };
|
||||
}
|
||||
|
||||
/** Project files no composition loads, so none is served or uploaded for one: env files, Git data, tool records. */
|
||||
export function isPrivateCompositionFile(projectDir: string, filePath: string): boolean {
|
||||
const CREDENTIAL_FILES = new Set([
|
||||
".npmrc",
|
||||
".yarnrc",
|
||||
".yarnrc.yml",
|
||||
".pypirc",
|
||||
"pip.conf",
|
||||
"pip.ini",
|
||||
".netrc",
|
||||
"_netrc",
|
||||
".git-credentials",
|
||||
".gitconfig",
|
||||
".mcp.json",
|
||||
]);
|
||||
const CREDENTIAL_FOLDERS = new Set([
|
||||
".ssh",
|
||||
".aws",
|
||||
".gnupg",
|
||||
".docker",
|
||||
".kube",
|
||||
".azure",
|
||||
".claude",
|
||||
".codex",
|
||||
".cursor",
|
||||
".agents",
|
||||
".gemini",
|
||||
".vscode",
|
||||
".idea",
|
||||
]);
|
||||
const CREDENTIAL_CONFIG_FOLDERS = new Set(["gcloud", "gh"]);
|
||||
const KEY_FILE = /^(?:id_(?:rsa|dsa|ecdsa|ed25519).*|.*\.(?:pem|key|p12|pfx))$/;
|
||||
|
||||
export function isCredentialOrToolStatePath(relPath: string): boolean {
|
||||
const segments = projectPathSegments(relPath);
|
||||
const name = segments.at(-1) ?? "";
|
||||
return (
|
||||
isEnvOrGitPath(relative(projectDir, filePath)) ||
|
||||
isEnvOrGitPath(relative(realFilePath(projectDir), realFilePath(filePath))) ||
|
||||
CREDENTIAL_FILES.has(name) ||
|
||||
KEY_FILE.test(name) ||
|
||||
segments.some(
|
||||
(segment, at) =>
|
||||
CREDENTIAL_FOLDERS.has(segment) ||
|
||||
(segment === ".config" && CREDENTIAL_CONFIG_FOLDERS.has(segments[at + 1] ?? "")),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const leavesProject = (path: string): boolean =>
|
||||
isAbsolute(path) || path.split(/[\\/]/)[0] === "..";
|
||||
|
||||
const projectRelatives = (projectDir: string, filePath: string): string[] => [
|
||||
relative(projectDir, filePath),
|
||||
relative(realFilePath(projectDir), realFilePath(filePath)),
|
||||
];
|
||||
|
||||
/** Project files no composition loads, so none is served or uploaded: env, Git, credentials, tool and agent state. */
|
||||
export function isPrivateCompositionFile(projectDir: string, filePath: string): boolean {
|
||||
const paths = projectRelatives(projectDir, filePath);
|
||||
return (
|
||||
paths.some(isEnvOrGitPath) ||
|
||||
paths.some((path) => !leavesProject(path) && isCredentialOrToolStatePath(path)) ||
|
||||
isPrivateProjectFile(projectDir, filePath)
|
||||
);
|
||||
}
|
||||
|
||||
export function isHiddenProjectFile(projectDir: string, filePath: string): boolean {
|
||||
return projectRelatives(projectDir, filePath).some((path) =>
|
||||
path.split(/[\\/]/).some((segment) => segment.startsWith(".") && segment !== ".."),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -102,3 +102,141 @@ describe("upload archives leave out env files and private data whatever .hyperfr
|
||||
expect(leftOut).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
const CREDENTIALS = [
|
||||
".npmrc",
|
||||
".yarnrc.yml",
|
||||
".pypirc",
|
||||
"pip.conf",
|
||||
".netrc",
|
||||
".git-credentials",
|
||||
".gitconfig",
|
||||
".mcp.json",
|
||||
".ssh/id_ed25519",
|
||||
".aws/credentials",
|
||||
".gnupg/private-keys-v1.d/key.key",
|
||||
".docker/config.json",
|
||||
".kube/config",
|
||||
".azure/accessTokens.json",
|
||||
".config/gcloud/credentials.db",
|
||||
".claude/settings.local.json",
|
||||
".codex/auth.json",
|
||||
".cursor/mcp.json",
|
||||
".agents/state.json",
|
||||
".gemini/settings.json",
|
||||
".vscode/settings.json",
|
||||
".idea/workspace.xml",
|
||||
"keys/id_rsa",
|
||||
"keys/server.pem",
|
||||
"keys/tls.key",
|
||||
"keys/cert.p12",
|
||||
"keys/cert.pfx",
|
||||
"scenes/_netrc",
|
||||
"scenes/.NPMRC",
|
||||
"scenes/.Claude/settings.json",
|
||||
];
|
||||
|
||||
const HTML_REFERENCED = CREDENTIALS.filter((_, at) => at % 2 === 0);
|
||||
const CSS_REFERENCED = CREDENTIALS.filter((_, at) => at % 2 === 1);
|
||||
|
||||
function credentialProject(): void {
|
||||
for (const path of CREDENTIALS) write(path, "token=secret");
|
||||
write("assets/logo.png", "logo");
|
||||
write(".media/clip.mp4", "clip");
|
||||
write(
|
||||
"style.css",
|
||||
CSS_REFERENCED.map((path, at) => `.c${at}{background:url("${path}")}`).join("\n") +
|
||||
`\n.logo{background:url("assets/logo.png")}`,
|
||||
);
|
||||
write(
|
||||
"index.html",
|
||||
`<link rel="stylesheet" href="style.css"><video src=".media/clip.mp4"></video>` +
|
||||
HTML_REFERENCED.map((path) => `<img src="${path}"><link href="${path}">`).join(""),
|
||||
);
|
||||
}
|
||||
|
||||
function leftOutPaths(leftOut: LeftOutReference[]): Set<string> {
|
||||
return new Set(leftOut.filter((one) => one.reason === "private").map((one) => one.path));
|
||||
}
|
||||
|
||||
describe("upload archives leave out credentials and agent or editor state", () => {
|
||||
it.each([
|
||||
["re-includes every dotfile", "!.*\n"],
|
||||
["re-includes every path", "!*\n!**/*\n"],
|
||||
["is absent", null],
|
||||
])("when .hyperframesignore %s and the page and a stylesheet name them", (_, rules) => {
|
||||
credentialProject();
|
||||
if (rules) write(".hyperframesignore", rules);
|
||||
|
||||
const { keys, leftOut } = stage();
|
||||
|
||||
for (const path of CREDENTIALS) expect(keys).not.toContain(path);
|
||||
const reported = leftOutPaths(leftOut);
|
||||
for (const path of CREDENTIALS) expect(reported).toContain(path);
|
||||
expect(keys).toEqual(expect.arrayContaining(["assets/logo.png", ".media/clip.mp4"]));
|
||||
});
|
||||
|
||||
it("keeps them out of the publish map, the cloud render staging and the feedback archive", () => {
|
||||
credentialProject();
|
||||
write(".hyperframesignore", "!.*\n");
|
||||
|
||||
const publishMap = buildPublishFileMap(dir);
|
||||
const cloudStaged = stagePublishFiles(dir, 1);
|
||||
const feedback = new AdmZip(createPublishArchive(dir).buffer)
|
||||
.getEntries()
|
||||
.map((entry) => entry.entryName);
|
||||
|
||||
for (const entries of [[...publishMap.keys()], [...cloudStaged.keys()], feedback]) {
|
||||
expect(entries.filter((entry) => CREDENTIALS.includes(entry))).toEqual([]);
|
||||
expect(entries).toEqual(expect.arrayContaining(["assets/logo.png", ".media/clip.mp4"]));
|
||||
}
|
||||
});
|
||||
|
||||
// fails without the fix: the reference check used the outside-upload type list, which leaves out scripts, so a
|
||||
// page's own .js and .mjs files in a hidden folder were dropped from every archive while the stylesheet stayed.
|
||||
it("brings back scripts and stylesheets a page names from a hidden folder in every archive", () => {
|
||||
write(".assets/animation.js", "console.log(1)");
|
||||
write(".assets/module.mjs", "export {}");
|
||||
write(".assets/style.css", "body{}");
|
||||
write(
|
||||
"index.html",
|
||||
`<link rel="stylesheet" href=".assets/style.css">` +
|
||||
`<script src=".assets/animation.js"></script>` +
|
||||
`<script type="module" src=".assets/module.mjs"></script>`,
|
||||
);
|
||||
const wanted = [".assets/animation.js", ".assets/module.mjs", ".assets/style.css"];
|
||||
|
||||
const publishMap = [...buildPublishFileMap(dir).keys()];
|
||||
const cloudStaged = [...stagePublishFiles(dir, 1).keys()];
|
||||
const feedback = new AdmZip(createPublishArchive(dir).buffer)
|
||||
.getEntries()
|
||||
.map((entry) => entry.entryName);
|
||||
|
||||
for (const entries of [publishMap, cloudStaged, feedback])
|
||||
expect(entries).toEqual(expect.arrayContaining(wanted));
|
||||
});
|
||||
|
||||
it("brings back a skipped file a page names only when it is an asset or not hidden", () => {
|
||||
write("renders/data.json", "{}");
|
||||
write("snapshots/frame.png", "png");
|
||||
write(".cache/notes.yaml", "token: secret");
|
||||
write(".cache/still.webp", "webp");
|
||||
write(
|
||||
"index.html",
|
||||
`<a href="renders/data.json">d</a><img src="snapshots/frame.png">` +
|
||||
`<a href=".cache/notes.yaml">n</a><img src=".cache/still.webp">`,
|
||||
);
|
||||
|
||||
const { keys, leftOut } = stage();
|
||||
|
||||
expect(keys).toEqual([
|
||||
".cache/still.webp",
|
||||
"index.html",
|
||||
"renders/data.json",
|
||||
"snapshots/frame.png",
|
||||
]);
|
||||
expect(leftOut.map(({ path, reason }) => ({ path, reason }))).toEqual([
|
||||
{ path: ".cache/notes.yaml", reason: "private" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,6 +24,7 @@ import { envCredentialVar, isTokenExpired, type ResolvedCredential } from "../au
|
||||
import { refreshIfNeeded } from "../cloud/auth.js";
|
||||
import { writeProjectLink } from "./projectLink.js";
|
||||
import {
|
||||
isHiddenProjectFile,
|
||||
isPrivateCompositionFile,
|
||||
isPrivateOutsideFile,
|
||||
linksOutside,
|
||||
@@ -38,7 +39,7 @@ import {
|
||||
type OnOutsideFile,
|
||||
outsideFileReference,
|
||||
} from "./leftOutReference.js";
|
||||
import { isUploadedOutsideFileType } from "./outsideFileType.js";
|
||||
import { isCopiedOutsideFileType, isUploadedOutsideFileType } from "./outsideFileType.js";
|
||||
import {
|
||||
collectSubCompositionSrcs,
|
||||
decodeTextHonoringBom,
|
||||
@@ -417,6 +418,7 @@ interface ExternalAssetContext {
|
||||
onLeftOut?: OnLeftOut;
|
||||
onOutsideFile?: OnOutsideFile;
|
||||
received: (path: string) => boolean;
|
||||
walkedLeftOut: ReadonlySet<string>;
|
||||
filmPages: Set<string> | null;
|
||||
}
|
||||
|
||||
@@ -448,14 +450,30 @@ function addExternalAsset(
|
||||
return archivePath;
|
||||
}
|
||||
|
||||
function staysOut(ctx: ExternalAssetContext, rel: string, absPath: string): boolean {
|
||||
if (!rel || ctx.fileContents.has(rel) || rel.split("/").some(shouldIgnoreSegment)) return true;
|
||||
function staysOut(ctx: ExternalAssetContext, rel: string): boolean {
|
||||
return (
|
||||
isPrivateCompositionFile(ctx.absProjectDir, absPath) ||
|
||||
(existsSync(absPath) && linksToPrivateOutsideFile(ctx.absProjectDir, absPath))
|
||||
!rel ||
|
||||
ctx.fileContents.has(rel) ||
|
||||
ctx.walkedLeftOut.has(rel) ||
|
||||
rel.split("/").some(shouldIgnoreSegment)
|
||||
);
|
||||
}
|
||||
|
||||
const loadableReference = (absProjectDir: string, absPath: string): boolean =>
|
||||
isCopiedOutsideFileType(absPath) || !isHiddenProjectFile(absProjectDir, absPath);
|
||||
|
||||
function referencedFileLeftOut(
|
||||
ctx: ExternalAssetContext,
|
||||
entryPath: string,
|
||||
absPath: string,
|
||||
): LeftOutReason | null {
|
||||
if (isPrivateCompositionFile(ctx.absProjectDir, absPath)) return "private";
|
||||
if (linksToPrivateOutsideFile(ctx.absProjectDir, absPath)) return "blocklisted";
|
||||
if (ctx.received(entryPath) && linksOutside(ctx.absProjectDir, absPath)) return "received";
|
||||
if (!loadableReference(ctx.absProjectDir, absPath)) return "private";
|
||||
return null;
|
||||
}
|
||||
|
||||
function reportProjectLinkOutside(
|
||||
absProjectDir: string,
|
||||
file: string,
|
||||
@@ -480,15 +498,16 @@ function includeReferencedProjectFile(
|
||||
const absPath = candidate.replace(/[?#].*$/, "");
|
||||
if (!isPathInside(absPath, ctx.absProjectDir)) continue;
|
||||
const rel = archiveKey(relative(ctx.absProjectDir, absPath).replaceAll("\\", "/"));
|
||||
if (staysOut(ctx, rel, absPath)) return;
|
||||
if (!existsSync(absPath)) continue;
|
||||
if (ctx.received(entryPath) && linksOutside(ctx.absProjectDir, absPath)) {
|
||||
return leaveOut(ctx, entryPath, rawPath, absPath, "received");
|
||||
}
|
||||
if (statSync(absPath).isFile()) {
|
||||
reportProjectLinkOutside(ctx.absProjectDir, rel, absPath, ctx.onOutsideFile);
|
||||
ctx.fileContents.set(rel, stageFile(absPath, ctx.onDiskMinBytes));
|
||||
if (staysOut(ctx, rel)) return;
|
||||
if (!existsSync(absPath)) {
|
||||
if (isPrivateCompositionFile(ctx.absProjectDir, absPath)) return;
|
||||
continue;
|
||||
}
|
||||
if (!statSync(absPath).isFile()) return;
|
||||
const reason = referencedFileLeftOut(ctx, entryPath, absPath);
|
||||
if (reason) return leaveOut(ctx, entryPath, rawPath, absPath, reason);
|
||||
reportProjectLinkOutside(ctx.absProjectDir, rel, absPath, ctx.onOutsideFile);
|
||||
ctx.fileContents.set(rel, stageFile(absPath, ctx.onDiskMinBytes));
|
||||
return;
|
||||
}
|
||||
if (!requireOnDisk) return;
|
||||
@@ -778,6 +797,7 @@ export function localizeExternalAssets(
|
||||
onLeftOut?: OnLeftOut,
|
||||
onOutsideFile?: OnOutsideFile,
|
||||
entry = "index.html",
|
||||
walkedLeftOut: ReadonlySet<string> = new Set(),
|
||||
): number {
|
||||
const ctx: ExternalAssetContext = {
|
||||
absProjectDir,
|
||||
@@ -789,6 +809,7 @@ export function localizeExternalAssets(
|
||||
onLeftOut,
|
||||
onOutsideFile,
|
||||
received: receivedFiles(absProjectDir),
|
||||
walkedLeftOut,
|
||||
filmPages: filmPagesOf(absProjectDir, fileContents, entry),
|
||||
};
|
||||
|
||||
@@ -842,7 +863,8 @@ function composeNamesInText(fileContents: Map<string, StagedFile>, filePaths: st
|
||||
type WalkedLeftOutReason = Exclude<LeftOutReason, "file_type" | "network_share">;
|
||||
|
||||
const WALKED_LEFT_OUT: Record<WalkedLeftOutReason, string> = {
|
||||
private: "holds secrets or tool data (an env file, Git data or Studio records)",
|
||||
private:
|
||||
"holds secrets or tool data (an env file, Git data, credentials, agent or editor state, or Studio records)",
|
||||
blocklisted: "links to a file outside the project in a private location",
|
||||
received: "links to a file outside the project and came in with an imported project",
|
||||
};
|
||||
@@ -882,6 +904,7 @@ export function stagePublishFiles(
|
||||
const fileContents = new Map<string, StagedFile>();
|
||||
const diskPathOf = new Map<string, string>();
|
||||
const received = receivedFiles(absProjectDir);
|
||||
const walkedLeftOut = new Set<string>();
|
||||
for (const filePath of filePaths) {
|
||||
const absPath = join(absProjectDir, filePath);
|
||||
const reason = walkedFileLeftOut(absProjectDir, filePath, absPath, received);
|
||||
@@ -890,6 +913,7 @@ export function stagePublishFiles(
|
||||
`${filePath} ${WALKED_LEFT_OUT[reason]}; it was left out of the upload.`,
|
||||
leftOutReference(absProjectDir, filePath, filePath, absPath, reason),
|
||||
);
|
||||
walkedLeftOut.add(archiveKey(filePath));
|
||||
continue;
|
||||
}
|
||||
const key = archiveKey(filePath);
|
||||
@@ -912,6 +936,7 @@ export function stagePublishFiles(
|
||||
onLeftOut,
|
||||
onOutsideFile,
|
||||
entry,
|
||||
walkedLeftOut,
|
||||
);
|
||||
return fileContents;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user