mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): a received-files record in a folder under the project counts for a render (#7900)
* wip: partial security fix, paused before tests finished * fix(cli): a received-files record in a folder under the project counts for a render The received-files lookup now also walks the project's folders for records, skipping tool and output folders and never following links. A nested record's entries are received files named from the project folder, so a received scene copied in with its own record makes the render leave out outside files, as a record at or above the project does. A walk that cannot list every folder fails closed.
This commit is contained in:
@@ -137,6 +137,36 @@ describe("receivedFiles", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("receivedFiles for a received project copied into a folder", () => {
|
||||
function withNested(record?: string): string {
|
||||
dir = mkdtempSync(join(tmpdir(), "hf-received-nested-"));
|
||||
mkdirSync(join(dir, "compositions", "x"), { recursive: true });
|
||||
writeFileSync(join(dir, "index.html"), "");
|
||||
writeFileSync(join(dir, "compositions", "x", "index.html"), "");
|
||||
if (record !== undefined) {
|
||||
mkdirSync(join(dir, "compositions", "x", ".hyperframes"));
|
||||
writeFileSync(join(dir, "compositions", "x", ".hyperframes", "received-files.json"), record);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
it("names the copy's files from the project folder", () => {
|
||||
const received = receivedFiles(withNested(JSON.stringify({ files: ["index.html"] })));
|
||||
expect(received("compositions/x/index.html")).toBe(true);
|
||||
expect(received("index.html")).toBe(false);
|
||||
});
|
||||
|
||||
it("counts every file as received when the copy's record cannot be read", () => {
|
||||
expect(receivedFiles(withNested("{not json"))("index.html")).toBe(true);
|
||||
});
|
||||
|
||||
it("counts no file as received when the folder has no record", () => {
|
||||
const received = receivedFiles(withNested());
|
||||
expect(received("compositions/x/index.html")).toBe(false);
|
||||
expect(received("index.html")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordReceivedFiles", () => {
|
||||
const record = (root: string) => join(root, ".hyperframes", "received-files.json");
|
||||
const lock = (root: string) => `${record(root)}.lock`;
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
closeSync,
|
||||
lstatSync,
|
||||
openSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
realpathSync,
|
||||
rmSync,
|
||||
@@ -17,6 +18,7 @@ import { isPrivateProjectFile, realFilePath } from "@hyperframes/studio-server/s
|
||||
import { mkdirWithinProject } from "@hyperframes/core";
|
||||
import { isCredentialOrToolStatePath } from "@hyperframes/core/safe-path";
|
||||
import { publishRegistryFile, registryRoot, registryTargetPath } from "../registry/publication.js";
|
||||
import { walkBudget } from "./walkBudget.js";
|
||||
|
||||
export { isEnvOrGitPath, isPrivateOutsideFile, linksOutside };
|
||||
|
||||
@@ -30,6 +32,7 @@ const LOCK_WAIT_MS = 2_000;
|
||||
const LOCK_STALE_MS = 10_000;
|
||||
|
||||
interface ReceivedRecord {
|
||||
file: string;
|
||||
root: string;
|
||||
projectDirFromRoot: string;
|
||||
listed: unknown;
|
||||
@@ -118,18 +121,13 @@ export function forgetReceivedFiles(root: string, paths: string[]): void {
|
||||
changeReceivedRecord(root, (files) => paths.map((path) => files.delete(path)).some(Boolean));
|
||||
}
|
||||
|
||||
function receivedRecordsAtOrAbove(projectDir: string): ReceivedRecord[] {
|
||||
let start: string;
|
||||
try {
|
||||
start = realpathSync(projectDir);
|
||||
} catch {
|
||||
start = resolve(projectDir);
|
||||
}
|
||||
function receivedRecordsAtOrAbove(start: string): ReceivedRecord[] {
|
||||
const records: ReceivedRecord[] = [];
|
||||
for (let dir = start; ; ) {
|
||||
const record = readRecord(dir, dir === start);
|
||||
if (record)
|
||||
records.push({
|
||||
file: join(dir, RECEIVED_RECORD),
|
||||
root: dir,
|
||||
projectDirFromRoot: relative(dir, start).split(sep).join("/"),
|
||||
...record,
|
||||
@@ -140,6 +138,56 @@ function receivedRecordsAtOrAbove(projectDir: string): ReceivedRecord[] {
|
||||
}
|
||||
}
|
||||
|
||||
function nestedRecord(start: string, dir: string, record: { listed: unknown }): ReceivedRecord {
|
||||
const folder = relative(start, dir).split(sep).join("/");
|
||||
const listed = Array.isArray(record.listed)
|
||||
? record.listed.map((entry) => `${folder}/${String(entry)}`)
|
||||
: undefined;
|
||||
return { file: join(dir, RECEIVED_RECORD), root: start, projectDirFromRoot: "", listed };
|
||||
}
|
||||
|
||||
const NOT_WALKED = new Set([
|
||||
".git",
|
||||
".hyperframes",
|
||||
"node_modules",
|
||||
"renders",
|
||||
"dist",
|
||||
".next",
|
||||
"coverage",
|
||||
]);
|
||||
|
||||
function receivedRecordsBelow(start: string): ReceivedRecord[] | null {
|
||||
const withinBudget = walkBudget();
|
||||
const records: ReceivedRecord[] = [];
|
||||
const pending = [start];
|
||||
for (let dir = pending.pop(); dir !== undefined; dir = pending.pop()) {
|
||||
let folders: string[];
|
||||
try {
|
||||
folders = readdirSync(dir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory() && !NOT_WALKED.has(entry.name.toLowerCase()))
|
||||
.map((entry) => join(dir, entry.name));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!withinBudget(folders.length)) return null;
|
||||
const record = dir === start ? null : readRecord(dir, true);
|
||||
if (record) records.push(nestedRecord(start, dir, record));
|
||||
pending.push(...folders);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
function receivedRecords(projectDir: string): ReceivedRecord[] | null {
|
||||
let start: string;
|
||||
try {
|
||||
start = realpathSync(projectDir);
|
||||
} catch {
|
||||
start = resolve(projectDir);
|
||||
}
|
||||
const below = receivedRecordsBelow(start);
|
||||
return below && [...receivedRecordsAtOrAbove(start), ...below];
|
||||
}
|
||||
|
||||
function fileIdentity(path: string): string | null {
|
||||
try {
|
||||
const stat = lstatSync(path, { bigint: true });
|
||||
@@ -166,8 +214,9 @@ function listedIn(record: ReceivedRecord, projectDir: string): (path: string) =>
|
||||
}
|
||||
|
||||
export function receivedFiles(projectDir: string): (path: string) => boolean {
|
||||
const records = receivedRecordsAtOrAbove(projectDir);
|
||||
if (records.some((record) => !Array.isArray(record.listed))) return () => true;
|
||||
const records = receivedRecords(projectDir);
|
||||
if (records === null || records.some((record) => !Array.isArray(record.listed)))
|
||||
return () => true;
|
||||
const listedInAnyRecord = records.map((record) => listedIn(record, projectDir));
|
||||
return (path) => listedInAnyRecord.some((listed) => listed(path));
|
||||
}
|
||||
@@ -194,13 +243,14 @@ function listedUnder(record: ReceivedRecord): string[] {
|
||||
|
||||
export function receivedRecordStatus(
|
||||
projectDir: string,
|
||||
): { path: string; listed: string[] | null } | null {
|
||||
const records = receivedRecordsAtOrAbove(projectDir);
|
||||
): { path: string; listed: string[] | null } | "unchecked" | null {
|
||||
const records = receivedRecords(projectDir);
|
||||
if (records === null) return "unchecked";
|
||||
const [nearest] = records;
|
||||
if (!nearest) return null;
|
||||
const unreadable = records.find((record) => !Array.isArray(record.listed));
|
||||
if (unreadable) return { path: join(unreadable.root, RECEIVED_RECORD), listed: null };
|
||||
return { path: join(nearest.root, RECEIVED_RECORD), listed: records.flatMap(listedUnder) };
|
||||
if (unreadable) return { path: unreadable.file, listed: null };
|
||||
return { path: nearest.file, listed: records.flatMap(listedUnder) };
|
||||
}
|
||||
|
||||
const leavesProject = (path: string): boolean =>
|
||||
|
||||
@@ -248,6 +248,80 @@ describe("outsideAssetsAllowed", () => {
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe("a received project copied into a folder of the project", () => {
|
||||
const NESTED = join("compositions", "x");
|
||||
function withNested(record?: string, files: Record<string, string | Buffer> = {}): string {
|
||||
const dir = project();
|
||||
write(
|
||||
join(dir, "index.html"),
|
||||
`<div data-composition-src="compositions/x/index.html"></div>`,
|
||||
);
|
||||
write(join(dir, NESTED, "index.html"), `<img src="../../../outside/pic.png">`);
|
||||
for (const [name, content] of Object.entries(files)) write(join(dir, NESTED, name), content);
|
||||
if (record !== undefined)
|
||||
write(join(dir, NESTED, ".hyperframes", "received-files.json"), record);
|
||||
return dir;
|
||||
}
|
||||
|
||||
it("refuses outside files and names what it leaves out", () => {
|
||||
const { allowed, warn, line } = decide(withNested(received("index.html")));
|
||||
expect(allowed).toBe(false);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(line).toContain(`${join(NESTED, "index.html")} ${SOMEONE_ELSE}`);
|
||||
expect(line).toContain("../../../outside/pic.png");
|
||||
});
|
||||
|
||||
it("refuses outside files when the copy's entry is the rendered composition", () => {
|
||||
const entry = "compositions/x/index.html";
|
||||
const { allowed, line } = decide(withNested(received("index.html")), entry);
|
||||
expect(allowed).toBe(false);
|
||||
expect(line).toContain(SOMEONE_ELSE);
|
||||
expect(line).toContain("../../../outside/pic.png");
|
||||
});
|
||||
|
||||
it("keeps outside files when the copy's record lists only media", () => {
|
||||
const dir = withNested(received("clip.mp4"), { "clip.mp4": SIGNATURES.mp4 });
|
||||
const { allowed, warn } = decide(dir);
|
||||
expect(allowed).toBe(true);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses outside files and names the copy's record when it cannot be read", () => {
|
||||
const dir = withNested("{not json");
|
||||
const { allowed, line } = decide(dir);
|
||||
expect(allowed).toBe(false);
|
||||
const path = join(realpathSync(dir), NESTED, ".hyperframes", "received-files.json");
|
||||
expect(line).toContain(`Could not read ${path}`);
|
||||
});
|
||||
|
||||
it("keeps outside files for a folder with no record of its own", () => {
|
||||
const { allowed, warn } = decide(withNested());
|
||||
expect(allowed).toBe(true);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps outside files for a project whose tool folders hold very many folders", () => {
|
||||
const dir = withNested();
|
||||
for (const tool of ["node_modules", "renders", ".git"]) {
|
||||
mkdirSync(join(dir, tool));
|
||||
for (let i = 0; i <= 7_000; i++) mkdirSync(join(dir, tool, `d${i}`));
|
||||
}
|
||||
const { allowed, warn } = decide(dir);
|
||||
expect(allowed).toBe(true);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses outside files when the project has too many folders to search", () => {
|
||||
const dir = withNested();
|
||||
const many = join(dir, "folders");
|
||||
mkdirSync(many);
|
||||
for (let i = 0; i <= 20_000; i++) mkdirSync(join(many, `d${i}`));
|
||||
const { allowed, line } = decide(dir);
|
||||
expect(allowed).toBe(false);
|
||||
expect(line).toContain("Could not check every project folder");
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps outside files for a project with no received-files record", () => {
|
||||
const { allowed, warn } = decide(project());
|
||||
expect(allowed).toBe(true);
|
||||
@@ -587,7 +661,7 @@ describe("outsideAssetsAllowed", () => {
|
||||
try {
|
||||
const { allowed, line } = decide(dir);
|
||||
expect(allowed).toBe(false);
|
||||
expect(line).toContain("Could not check every project file");
|
||||
expect(line).toMatch(/Could not check every project (file|folder)/);
|
||||
} finally {
|
||||
chmodSync(join(dir, "locked"), 0o755);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
} from "@hyperframes/parsers/asset-resolution";
|
||||
import { linksOutside, receivedRecordStatus } from "./privateProjectFile.js";
|
||||
import { terminalText } from "./terminalText.js";
|
||||
import { walkBudget } from "./walkBudget.js";
|
||||
|
||||
const REFERENCE_ATTRS = ["src", "href", "poster"];
|
||||
const MAX_SCANNED_BYTES = 2 * 1024 * 1024;
|
||||
@@ -291,9 +292,6 @@ function isMediaFontOrImage(path: string): boolean {
|
||||
return bytes !== null && matches !== undefined && matches(bytes);
|
||||
}
|
||||
|
||||
const ALIAS_WALK_MAX_ENTRIES = 20_000;
|
||||
const ALIAS_WALK_MAX_MS = 2_000;
|
||||
|
||||
interface Target {
|
||||
identity: string;
|
||||
isDirectory: boolean;
|
||||
@@ -354,15 +352,6 @@ function receivedAlias(
|
||||
return !isMediaName(path) && received.has(target.identity) ? path : null;
|
||||
}
|
||||
|
||||
function walkBudget(): (entries: number) => boolean {
|
||||
const deadline = Date.now() + ALIAS_WALK_MAX_MS;
|
||||
let seen = 0;
|
||||
return (entries) => {
|
||||
seen += entries;
|
||||
return seen <= ALIAS_WALK_MAX_ENTRIES && Date.now() <= deadline;
|
||||
};
|
||||
}
|
||||
|
||||
interface AliasWalk {
|
||||
received: ReadonlySet<string>;
|
||||
realProjectDir: string;
|
||||
@@ -406,9 +395,13 @@ const READS_NOTHING_OUTSIDE = "so the render reads nothing outside the project."
|
||||
|
||||
function receivedRefusal(
|
||||
projectDir: string,
|
||||
record: { path: string; listed: string[] | null },
|
||||
record: { path: string; listed: string[] | null } | "unchecked",
|
||||
): Refusal | null {
|
||||
const shown = (path: string) => relative(realOrNull(projectDir) ?? projectDir, path);
|
||||
if (record === "unchecked") {
|
||||
const reason = `Could not check every project folder for a received-files record, ${READS_NOTHING_OUTSIDE}`;
|
||||
return { reason, alwaysWarn: true };
|
||||
}
|
||||
if (record.listed === null) {
|
||||
return { reason: `Could not read ${record.path}, ${READS_NOTHING_OUTSIDE}`, alwaysWarn: true };
|
||||
}
|
||||
@@ -430,7 +423,7 @@ function receivedRefusal(
|
||||
return { reason, alwaysWarn: false };
|
||||
}
|
||||
|
||||
/** False when the received-files record cannot be read, lists a received file that is not positively media, a font or an image by extension and leading bytes, or a project file reaches a received file under another kind of name; `warn` then says why and names the references outside the project it refuses. */
|
||||
/** False when a received-files record of the project, an ancestor or a folder under the project cannot be read or found, lists a received file that is not positively media, a font or an image by extension and leading bytes, or a project file reaches a received file under another kind of name; `warn` then says why and names the references outside the project it refuses. */
|
||||
export function outsideAssetsAllowed(
|
||||
projectDir: string,
|
||||
entryFile: string | undefined,
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
const WALK_MAX_ENTRIES = 20_000;
|
||||
const WALK_MAX_MS = 2_000;
|
||||
|
||||
/** Counts what a project walk lists; false once it passes the entry or time limit, so the walk stops unfinished. */
|
||||
export function walkBudget(): (entries: number) => boolean {
|
||||
const deadline = Date.now() + WALK_MAX_MS;
|
||||
let seen = 0;
|
||||
return (entries) => {
|
||||
seen += entries;
|
||||
return seen <= WALK_MAX_ENTRIES && Date.now() <= deadline;
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user