feat(cli): uploads list every file they carry from outside the project (#7831)

* fix(cli): uploads leave out outside scripts; .ts counts as video only when it is one

* chore(studio-server): raise unpacked size budget for the upload file-type rule

* fix(studio-server): transport stream check never waits on a pipe and needs binary bytes

* feat(cli): uploads list every file they carry from outside the project
This commit is contained in:
Michael Wang
2026-11-15 10:31:13 +00:00
committed by GitHub
parent bcf3d177d3
commit e426ce4eb4
9 changed files with 339 additions and 25 deletions
+3 -1
View File
@@ -141,7 +141,9 @@ and your source files stay unchanged.
Only images, video, audio, fonts and CSS files are copied from outside the
project. Any other outside file, such as a script, a Lottie `.json` or a `.pdf`,
is left out with a warning naming it; move it into the project folder to include
it.
it. Before uploading, `publish` lists every file it is about to copy from outside
the project folder, including files the project only links to, so you can see
what leaves your computer.
Before sharing source, remove secrets and confirm that every included media file
can be shared. Tell a reviewer whether you need feedback on story, visuals,
+8
View File
@@ -2735,6 +2735,14 @@ it is. A full run also prints each as a `{"event":"left_out",…}`
line on stderr, so a wrapper can ask with `--dry-run --json` before uploading
and still hear about each one during the run.
Before uploading, the command also names every file it carries from outside the
project folder: each file a composition points at, and each project file that is
a link to a file outside the project. It prints the list and carries on. Under
`--json`, the result lists them as `outside_files`, with the same `file`,
`reference`, `path` and `levels_up` fields, and each is printed as a
`{"event":"outside_file",…}` line on stderr. For a project link, `file` and
`reference` are the link's own path.
Render parameters mirror local `hyperframes render` where they overlap:
| Flag | Default | Description |
@@ -0,0 +1,70 @@
import { describe, expect, it, onTestFinished } from "vitest";
import { spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
const cliEntry = resolve(fileURLToPath(import.meta.url), "..", "..", "..", "cli.ts");
/** `<root>/p`, whose page shows `<root>/shared/logo.png`. */
function projectWithOutsideImage(): string {
const root = mkdtempSync(join(tmpdir(), "hf-cloud-outside-"));
onTestFinished(() => rmSync(root, { recursive: true, force: true }));
const dir = join(root, "p");
mkdirSync(dir, { recursive: true });
mkdirSync(join(root, "shared"));
writeFileSync(join(root, "shared", "logo.png"), "png");
writeFileSync(
join(dir, "index.html"),
`<!doctype html><html><body><div data-composition-id="main" data-width="1920" data-height="1080"><img src="../shared/logo.png"></div></body></html>`,
);
return dir;
}
function dryRun(dir: string, json: boolean) {
return spawnSync(
"bun",
["run", cliEntry, "cloud", "render", dir, "--dry-run", ...(json ? ["--json"] : [])],
{
encoding: "utf-8",
timeout: 30_000,
env: {
...process.env,
CI: "1",
HEYGEN_API_URL: "http://127.0.0.1:1",
HYPERFRAMES_NO_UPDATE_CHECK: "1",
HYPERFRAMES_NO_TELEMETRY: "1",
HOME: dir,
},
},
);
}
const expected = {
file: "index.html",
reference: "../shared/logo.png",
path: "../shared/logo.png",
levels_up: 1,
};
describe("cloud render --dry-run names each file it uploads from outside the project", () => {
it("in its --json result, and as an outside_file event per file on stderr", () => {
const result = dryRun(projectWithOutsideImage(), true);
expect(result.status).toBe(0);
const { archive } = JSON.parse(result.stdout) as { archive: { outside_files: unknown[] } };
expect(archive.outside_files).toEqual([expected]);
const events = result.stderr
.split("\n")
.filter((line) => line.startsWith('{"event":"outside_file"'))
.map((line) => JSON.parse(line) as Record<string, unknown>);
expect(events).toEqual([{ event: "outside_file", ...expected }]);
});
it("in its printed output", () => {
const result = dryRun(projectWithOutsideImage(), false);
expect(result.status).toBe(0);
expect(result.stdout).toContain("Uploading 1 file from outside the project folder:");
expect(result.stdout).toContain("../shared/logo.png (referenced by index.html)");
});
});
+46 -6
View File
@@ -63,7 +63,13 @@ import {
validateVariablesAgainstProject,
} from "../../utils/variables.js";
import { withMeta } from "../../utils/updateCheck.js";
import type { LeftOutReference, OnLeftOut } from "../../utils/leftOutReference.js";
import {
type LeftOutReference,
type OnLeftOut,
type OnOutsideFile,
type OutsideFileReference,
outsideFilesNotice,
} from "../../utils/leftOutReference.js";
import { trackUploadLeftOut } from "../../telemetry/uploadLeftOut.js";
import type { Example } from "../_examples.js";
@@ -341,6 +347,7 @@ export default defineCommand({
render: { render_id: renderId, status: "queued" as const },
swapped_fonts: preparedArchive?.swappedFonts ?? [],
left_out: preparedArchive?.leftOut ?? [],
outside_files: preparedArchive?.outsideFiles ?? [],
}),
null,
2,
@@ -388,6 +395,7 @@ export default defineCommand({
bytes_written: downloadResult.bytes,
swapped_fonts: preparedArchive?.swappedFonts ?? [],
left_out: preparedArchive?.leftOut ?? [],
outside_files: preparedArchive?.outsideFiles ?? [],
}),
null,
2,
@@ -669,6 +677,7 @@ interface PreparedLocalArchive {
offloaded: OffloadedMedia[];
swappedFonts: string[];
leftOut: LeftOutReference[];
outsideFiles: OutsideFileReference[];
}
function summarizeLargestFiles(fileMap: Map<string, StagedFile>): ArchiveFileSummary[] {
@@ -690,13 +699,12 @@ async function prepareLocalArchive(
console.log("");
console.log(`${c.accent("◆")} Zipping ${c.accent(project.name)}`);
}
const leftOut: LeftOutReference[] = [];
const { fileMap, entryPath } = stageProjectFiles(
const { fileMap, entryPath, leftOut, outsideFiles } = stageAndReport(
project,
composition,
warnAndListLeftOut(leftOut, asJson),
asJson,
!upload,
);
trackUploadLeftOut("cloud_render", leftOut, !upload);
const swappedFonts = await embedSystemFonts({
dir: project.dir,
entryFile: composition,
@@ -728,6 +736,7 @@ async function prepareLocalArchive(
offloaded,
swappedFonts,
leftOut,
outsideFiles,
};
if (upload) assertArchiveFits(sizeBytes, largestFiles);
if (!asJson && sizeBytes >= ARCHIVE_DIAGNOSTIC_THRESHOLD_BYTES) {
@@ -745,13 +754,43 @@ function warnAndListLeftOut(leftOut: LeftOutReference[], asJson: boolean): OnLef
};
}
function stageAndReport(
project: ProjectDir,
composition: string | undefined,
asJson: boolean,
dryRun: boolean,
) {
const leftOut: LeftOutReference[] = [];
const outsideFiles: OutsideFileReference[] = [];
const staged = stageProjectFiles(
project,
composition,
warnAndListLeftOut(leftOut, asJson),
(reference) => outsideFiles.push(reference),
);
trackUploadLeftOut("cloud_render", leftOut, dryRun);
listOutsideFiles(outsideFiles, asJson);
return { ...staged, leftOut, outsideFiles };
}
function listOutsideFiles(outsideFiles: OutsideFileReference[], asJson: boolean): void {
if (outsideFiles.length === 0) return;
if (!asJson) return console.log(c.dim(indentLines(outsideFilesNotice(outsideFiles))));
for (const reference of outsideFiles) {
console.error(JSON.stringify({ event: "outside_file", ...reference }));
}
}
const indentLines = (text: string) => text.replace(/^/gm, " ");
function stageProjectFiles(
project: ProjectDir,
composition: string | undefined,
onLeftOut: OnLeftOut,
onOutsideFile: OnOutsideFile,
): { fileMap: Map<string, StagedFile>; entryPath: string } {
try {
const fileMap = stagePublishFiles(project.dir, OFFLOAD_MIN_BYTES, onLeftOut);
const fileMap = stagePublishFiles(project.dir, OFFLOAD_MIN_BYTES, onLeftOut, onOutsideFile);
const entryPath = archiveKey(
relative(project.dir, resolvePath(project.dir, composition ?? "index.html")).replaceAll(
"\\",
@@ -894,6 +933,7 @@ function reportDryRun(prepared: PreparedLocalArchive, asJson: boolean): void {
offloaded_media: prepared.offloaded,
swapped_fonts: prepared.swappedFonts,
left_out: prepared.leftOut,
outside_files: prepared.outsideFiles,
},
}),
null,
+19 -5
View File
@@ -22,7 +22,11 @@ import { isUnattended } from "../utils/attendedTerminal.js";
import { parseFeedbackRating } from "../utils/feedbackRating.js";
import { feedbackEmail, parseFeedbackSource } from "../utils/feedbackSource.js";
import { lintFeedbackComment, type FeedbackLintInput } from "../utils/feedbackLint.js";
import type { LeftOutReference } from "../utils/leftOutReference.js";
import {
type LeftOutReference,
type OutsideFileReference,
outsideFilesNotice,
} from "../utils/leftOutReference.js";
import { trackUploadLeftOut } from "../telemetry/uploadLeftOut.js";
export const examples: Example[] = [
@@ -124,11 +128,21 @@ async function publishRepro(dir: string): Promise<string | undefined> {
try {
const leftOut: string[] = [];
const references: LeftOutReference[] = [];
const archive = createPublishArchive(dir, (warning, reference) => {
leftOut.push(warning);
references.push(reference);
});
const outsideFiles: OutsideFileReference[] = [];
const archive = createPublishArchive(
dir,
(warning, reference) => {
leftOut.push(warning);
references.push(reference);
},
(reference) => outsideFiles.push(reference),
);
trackUploadLeftOut("feedback", references);
if (outsideFiles.length > 0) {
spinner.stop("Repro prepared");
clack.log.info(outsideFilesNotice(outsideFiles));
spinner.start("Publishing minimal repro...");
}
const published = await publishProjectArchive(dir, { archive });
spinner.stop(c.success("Repro published"));
for (const warning of leftOut) clack.log.warn(c.warn(warning));
+21 -6
View File
@@ -31,7 +31,11 @@ import {
writeTeamProject,
} from "../utils/projectLink.js";
import { cliSourced } from "../utils/sourceTag.js";
import type { LeftOutReference } from "../utils/leftOutReference.js";
import {
type LeftOutReference,
type OutsideFileReference,
outsideFilesNotice,
} from "../utils/leftOutReference.js";
import { trackUploadLeftOut } from "../telemetry/uploadLeftOut.js";
export const examples: Example[] = [
@@ -237,17 +241,28 @@ export default defineCommand({
const autoProxy = resolveAutoProxy(dir, proxyFlagValue);
const leftOut: string[] = [];
const references: LeftOutReference[] = [];
const fileMap = buildPublishFileMap(dir, (warning, reference) => {
leftOut.push(warning);
references.push(reference);
});
const outsideFiles: OutsideFileReference[] = [];
const fileMap = buildPublishFileMap(
dir,
(warning, reference) => {
leftOut.push(warning);
references.push(reference);
},
(reference) => outsideFiles.push(reference),
);
trackUploadLeftOut("publish", references);
let proxyBakeManifest: Awaited<ReturnType<typeof bakeMediaProxies>> | undefined;
if (autoProxy) {
proxyBakeManifest = await bakeMediaProxies(dir, fileMap);
}
const archive = zipPublishFileMap(fileMap);
publishSpinner.message("Uploading project...");
if (outsideFiles.length > 0) {
publishSpinner.stop("Project prepared");
clack.log.info(outsideFilesNotice(outsideFiles));
publishSpinner.start("Uploading project...");
} else {
publishSpinner.message("Uploading project...");
}
const published = await publishProjectArchive(dir, {
public: args.public === true,
+24 -2
View File
@@ -15,6 +15,21 @@ export interface LeftOutReference {
export type OnLeftOut = (warning: string, reference: LeftOutReference) => void;
export type OutsideFileReference = Omit<LeftOutReference, "reason">;
export type OnOutsideFile = (reference: OutsideFileReference) => void;
export function outsideFileReference(
projectDir: string,
file: string,
reference: string,
absPath: string,
): OutsideFileReference {
const path = relative(realFilePath(projectDir), realFilePath(absPath)).replaceAll("\\", "/");
const levels_up = isAbsolute(path) ? null : path.split("/").filter((s) => s === "..").length;
return { file, reference, path, levels_up };
}
export function leftOutReference(
projectDir: string,
file: string,
@@ -22,7 +37,14 @@ export function leftOutReference(
absPath: string,
reason: LeftOutReason,
): LeftOutReference {
const path = relative(realFilePath(projectDir), realFilePath(absPath)).replaceAll("\\", "/");
const levels_up = isAbsolute(path) ? null : path.split("/").filter((s) => s === "..").length;
const { path, levels_up } = outsideFileReference(projectDir, file, reference, absPath);
return { file, reference, path, reason, levels_up };
}
export function outsideFilesNotice(references: OutsideFileReference[]): string {
const count = references.length === 1 ? "1 file" : `${references.length} files`;
const lines = references.map(({ file, reference, path }) =>
file === reference ? ` ${path} (linked from ${file})` : ` ${path} (referenced by ${file})`,
);
return [`Uploading ${count} from outside the project folder:`, ...lines].join("\n");
}
@@ -0,0 +1,116 @@
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { type OutsideFileReference, outsideFilesNotice } from "./leftOutReference.js";
import { stagePublishFiles } from "./publishProject.js";
const dirs: string[] = [];
afterEach(() => dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true })));
/** Writes `files` under a fresh folder, makes `links` (link to target), and stages its `p` project. */
function stage(files: Record<string, string>, links: Record<string, string> = {}) {
const root = mkdtempSync(join(tmpdir(), "hf-outside-list-"));
dirs.push(root);
for (const [name, body] of Object.entries(files)) {
mkdirSync(dirname(join(root, name)), { recursive: true });
writeFileSync(join(root, name), body.replaceAll("<root>", root));
}
for (const [link, target] of Object.entries(links)) {
mkdirSync(dirname(join(root, link)), { recursive: true });
symlinkSync(join(root, target), join(root, link));
}
const outside: OutsideFileReference[] = [];
const staged = stagePublishFiles(join(root, "p"), Infinity, undefined, (reference) =>
outside.push(reference),
);
return { root, staged, outside };
}
describe("files from outside the project that an upload carries", () => {
it("are each named once, for ../, absolute and CSS url() references", () => {
const { root, staged, outside } = stage({
"shared/logo.png": "png",
"shared/brand.woff2": "woff2",
"p/style.css": `@font-face { font-family: B; src: url("../shared/brand.woff2"); }`,
"p/index.html": [
`<img src="../shared/logo.png"><img src="<root>/shared/logo.png">`,
`<link rel="stylesheet" href="style.css">`,
].join(""),
});
expect(staged.has("_ext/shared/logo.png")).toBe(true);
expect(outside).toEqual([
{
file: "index.html",
reference: "../shared/logo.png",
path: "../shared/logo.png",
levels_up: 1,
},
{
file: "style.css",
reference: "../shared/brand.woff2",
path: "../shared/brand.woff2",
levels_up: 1,
},
]);
expect(outside.some((one) => one.reference === `${root}/shared/logo.png`)).toBe(false);
});
it("include project files that are links to files outside the project", () => {
const { staged, outside } = stage(
{
"shared/logo.png": "png",
"shared/pack/scene.json": "{}",
"p/index.html": `<img src="logo.png">`,
},
{ "p/logo.png": "shared/logo.png", "p/data/scene.json": "shared/pack/scene.json" },
);
expect(staged.has("logo.png")).toBe(true);
expect(staged.has("data/scene.json")).toBe(true);
expect(outside).toEqual(
expect.arrayContaining([
{ file: "logo.png", reference: "logo.png", path: "../shared/logo.png", levels_up: 1 },
{
file: "data/scene.json",
reference: "data/scene.json",
path: "../shared/pack/scene.json",
levels_up: 1,
},
]),
);
expect(outside).toHaveLength(2);
});
it("leave out project files and files the upload does not carry", () => {
const { outside } = stage({
"shared/data.json": "{}",
"shared/.cache/logo.png": "png",
"p/media/clip.mp4": "mp4",
"p/index.html": [
`<video src="media/clip.mp4"></video><link href="../shared/data.json">`,
`<img src="../shared/.cache/logo.png"><img src="../shared/missing.png">`,
].join(""),
});
expect(outside).toEqual([]);
});
});
describe("the notice that names them", () => {
it("says how many there are and where each one comes from", () => {
expect(
outsideFilesNotice([
{ file: "index.html", reference: "../a.png", path: "../a.png", levels_up: 1 },
{ file: "logo.png", reference: "logo.png", path: "../../b.png", levels_up: 2 },
]),
).toBe(
[
"Uploading 2 files from outside the project folder:",
" ../a.png (referenced by index.html)",
" ../../b.png (linked from logo.png)",
].join("\n"),
);
expect(
outsideFilesNotice([{ file: "a.html", reference: "/x.png", path: "../x.png", levels_up: 1 }]),
).toMatch(/^Uploading 1 file from outside/);
});
});
+32 -5
View File
@@ -35,6 +35,8 @@ import {
leftOutWarning,
type LeftOutReason,
type OnLeftOut,
type OnOutsideFile,
outsideFileReference,
} from "./leftOutReference.js";
import { isUploadedOutsideFileType } from "./outsideFileType.js";
@@ -407,12 +409,19 @@ interface ExternalAssetContext {
usedArchivePaths: Set<string>;
leftOut: Set<string>;
onLeftOut?: OnLeftOut;
onOutsideFile?: OnOutsideFile;
received: (path: string) => boolean;
}
function addExternalAsset(ctx: ExternalAssetContext, absPath: string): string {
function addExternalAsset(
ctx: ExternalAssetContext,
absPath: string,
entryPath: string,
rawPath: string,
): string {
const existing = ctx.externalMap.get(absPath);
if (existing) return existing;
ctx.onOutsideFile?.(outsideFileReference(ctx.absProjectDir, entryPath, rawPath, absPath));
const rel = relative(ctx.absProjectDir, absPath).replaceAll("\\", "/");
const stripped = rel.replace(/^(?:\.\.\/)+/, "");
@@ -440,6 +449,17 @@ function staysOut(ctx: ExternalAssetContext, rel: string, absPath: string): bool
);
}
function reportProjectLinkOutside(
absProjectDir: string,
file: string,
absPath: string,
onOutsideFile: OnOutsideFile | undefined,
): void {
if (onOutsideFile && linksOutside(absProjectDir, absPath)) {
onOutsideFile(outsideFileReference(absProjectDir, file, file, absPath));
}
}
/** Ships a referenced in-project file the ignore rules skipped, resolved from its folder or the root. */
function includeReferencedProjectFile(
ctx: ExternalAssetContext,
@@ -459,6 +479,7 @@ function includeReferencedProjectFile(
return leaveOut(ctx, entryPath, rawPath, absPath, "received");
}
if (statSync(absPath).isFile()) {
reportProjectLinkOutside(ctx.absProjectDir, rel, absPath, ctx.onOutsideFile);
ctx.fileContents.set(rel, stageFile(absPath, ctx.onDiskMinBytes));
}
return;
@@ -516,7 +537,7 @@ function archiveTargetOf(
} catch {
return undefined;
}
return encodeUrlPath(addExternalAsset(ctx, absPath));
return encodeUrlPath(addExternalAsset(ctx, absPath, options.entryPath, rawPath));
}
function tryResolveExternal(
@@ -703,6 +724,7 @@ export function localizeExternalAssets(
fileContents: Map<string, StagedFile>,
onDiskMinBytes = Infinity,
onLeftOut?: OnLeftOut,
onOutsideFile?: OnOutsideFile,
): number {
const ctx: ExternalAssetContext = {
absProjectDir,
@@ -712,6 +734,7 @@ export function localizeExternalAssets(
usedArchivePaths: new Set(),
leftOut: new Set(),
onLeftOut,
onOutsideFile,
received: receivedFiles(absProjectDir),
};
@@ -737,8 +760,9 @@ export function localizeExternalAssets(
export function buildPublishFileMap(
projectDir: string,
onLeftOut?: OnLeftOut,
onOutsideFile?: OnOutsideFile,
): Map<string, Buffer> {
return stagePublishFiles(projectDir, Infinity, onLeftOut) as Map<string, Buffer>;
return stagePublishFiles(projectDir, Infinity, onLeftOut, onOutsideFile) as Map<string, Buffer>;
}
export const archiveKey = (path: string): string => path.normalize("NFC");
@@ -786,6 +810,7 @@ export function stagePublishFiles(
projectDir: string,
onDiskMinBytes: number,
onLeftOut?: OnLeftOut,
onOutsideFile?: OnOutsideFile,
): Map<string, StagedFile> {
const absProjectDir = resolve(projectDir);
const filePaths: string[] = [];
@@ -817,11 +842,12 @@ export function stagePublishFiles(
);
}
diskPathOf.set(key, filePath);
reportProjectLinkOutside(absProjectDir, filePath, absPath, onOutsideFile);
fileContents.set(key, stageFile(absPath, onDiskMinBytes));
}
composeNamesInText(fileContents, filePaths);
localizeExternalAssets(absProjectDir, fileContents, onDiskMinBytes, onLeftOut);
localizeExternalAssets(absProjectDir, fileContents, onDiskMinBytes, onLeftOut, onOutsideFile);
return fileContents;
}
@@ -870,8 +896,9 @@ export function zipPublishFileMap(fileContents: Map<string, Buffer>): PublishArc
export function createPublishArchive(
projectDir: string,
onLeftOut?: OnLeftOut,
onOutsideFile?: OnOutsideFile,
): PublishArchiveResult {
return zipPublishFileMap(buildPublishFileMap(projectDir, onLeftOut));
return zipPublishFileMap(buildPublishFileMap(projectDir, onLeftOut, onOutsideFile));
}
export function getPublishApiBaseUrl(): string {