fix(cli): leave files from outside the project out of Lambda and Cloud Run site uploads (#8060)

* fix(cli): leave files from outside the project out of Lambda and Cloud Run site uploads

A site upload for `lambda` or `cloudrun` followed every symlink in the project and
shipped what it pointed at, wherever that was on the machine. `publish` and `cloud
render` already ask first. Site uploads now leave a file or folder linked in from
outside the project out and name each one; `--include-outside-files` (or
`includeOutsideFiles` in the SDK's deploySite, renderToLambda and renderToCloudRun)
uploads them. publishProjectArchive called without a prepared archive now leaves
outside files out too, since nothing approved them.

— Rames

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(producer): drop the new option comments the comment ratchet refuses

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
James Russo
2026-11-17 09:25:25 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent efc91fc877
commit 39347da9de
16 changed files with 169 additions and 26 deletions
+5
View File
@@ -3031,6 +3031,7 @@ And these drive `render` and `render-batch`:
| ----------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
| `--width`, `--height` | required | Render dimensions in pixels |
| `--site-id` | content hash | Reuse an uploaded site instead of uploading again |
| `--include-outside-files` | off | Upload the files a link brings in from outside the project folder. Without it they are left out and named |
| `--fps` | `30` | `24`, `25`, `30`, or `60` |
| `--format` | `mp4` | `mp4`, `mov`, `webm`, or `png-sequence` |
| `--codec` | `h264` | `h264` or `h265`, MP4 only |
@@ -3083,6 +3084,10 @@ hyperframes lambda sites create ./my-project
hyperframes lambda render ./my-project --site-id=abc1234deadbeef0 --width 1920 --height 1080
```
A file or folder that a symlink brings in from outside the project is left out
of the upload, and a notice names each one. Pass `--include-outside-files` (or
`includeOutsideFiles: true` to the SDK's `deploySite`) to upload them too.
### `lambda render <projectDir>`
Starts a Step Functions execution and returns a `renderId` immediately — poll it
+14 -3
View File
@@ -202,7 +202,7 @@ describe("deploySite", () => {
expect(after.siteId).toBe(before.siteId);
});
it("ships a symlinked project and its symlinked files and folders as content", async () => {
it("ships a symlinked project, and its files linked in from outside only when included", async () => {
mkdirSync(join(scratch, "media"));
writeFileSync(join(scratch, "media", "clip.txt"), "clip");
writeFileSync(join(scratch, "logo.txt"), "logo");
@@ -210,12 +210,23 @@ describe("deploySite", () => {
symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt"));
symlinkSync(projectDir, join(scratch, "fx"));
const s3 = new FakeS3();
const result = await deploySite({
const notices: string[] = [];
const leftOut = await deploySite({
projectDir: join(scratch, "fx"),
bucketName: "test-bucket",
onNotice: (line) => notices.push(line),
s3: asS3Client(s3),
});
expect(sitePaths(s3, result.siteId)).toEqual([
expect(sitePaths(s3, leftOut.siteId)).toEqual(["assets/style.css", "index.html"]);
expect(notices.join("\n")).toContain("outside the project: logo.txt, media.");
const included = await deploySite({
projectDir: join(scratch, "fx"),
bucketName: "test-bucket",
includeOutsideFiles: true,
s3: asS3Client(s3),
});
expect(sitePaths(s3, included.siteId)).toEqual([
"assets/style.css",
"index.html",
"logo.txt",
+2 -1
View File
@@ -64,6 +64,7 @@ export interface DeploySiteOptions {
siteId?: string;
/** Called with a line a user should see, such as audio sidecars skipped for want of ffmpeg. */
onNotice?: (message: string) => void;
includeOutsideFiles?: boolean;
expectedBucketOwner?: string;
/** Injection seam for tests. Production callers leave unset. */
s3?: S3Client;
@@ -162,7 +163,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
if (!statSync(opts.projectDir).isDirectory()) {
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
}
const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice);
const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice, opts.includeOutsideFiles);
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
const projectS3Uri = siteManifestUri(opts.bucketName, siteId);
@@ -38,6 +38,7 @@ export interface RenderToLambdaOptions {
/** Re-use an existing `deploySite` upload (skips tar+S3 PUT). */
siteHandle?: SiteHandle;
onNotice?: (message: string) => void;
includeOutsideFiles?: boolean;
/** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */
config: SerializableDistributedRenderConfig;
/** S3 bucket from the SAM stack output (`RenderBucketName`). */
@@ -123,6 +124,7 @@ export async function renderToLambda(opts: RenderToLambdaOptions): Promise<Rende
bucketName: opts.bucketName,
region: opts.region,
onNotice: opts.onNotice,
includeOutsideFiles: opts.includeOutsideFiles,
s3,
});
+10
View File
@@ -159,6 +159,12 @@ export default defineCommand({
// sites / render
"site-id": { type: "string", description: "Explicit site id (overrides content hash)" },
"include-outside-files": {
type: "boolean",
description:
"Upload the files a link brings in from outside the project folder. Otherwise they are left out and named.",
default: false,
},
width: { type: "string", description: "Render width in pixels" },
height: { type: "string", description: "Render height in pixels" },
fps: { type: "string", description: `Render fps (${DISTRIBUTED_FPS.join(" | ")})` },
@@ -512,6 +518,7 @@ async function runSites(args: Record<string, unknown>): Promise<void> {
projectDir: resolve(projectDir),
bucketName: state.bucketName,
siteId: args["site-id"] as string | undefined,
includeOutsideFiles: args["include-outside-files"] === true,
onNotice: (message) => console.error(message),
});
if (args.json) {
@@ -563,6 +570,8 @@ async function runRender(args: Record<string, unknown>): Promise<void> {
const { renderToCloudRun, getRenderProgress } = await loadCloudRunAdapter();
const handle = await renderToCloudRun({
projectDir: resolve(projectDir),
includeOutsideFiles: args["include-outside-files"] === true,
onNotice: (message) => console.error(message),
config: config as Parameters<typeof renderToCloudRun>[0]["config"],
bucketName: state.bucketName,
projectId: state.projectId,
@@ -694,6 +703,7 @@ async function runRenderBatch(args: Record<string, unknown>): Promise<void> {
projectDir: resolve(projectDir),
bucketName: state.bucketName,
siteId: args["site-id"] as string | undefined,
includeOutsideFiles: args["include-outside-files"] === true,
onNotice: (message) => console.error(message),
});
+9
View File
@@ -136,6 +136,12 @@ export default defineCommand({
// sites / render
"site-id": { type: "string", description: "Explicit site id (overrides content hash)" },
"include-outside-files": {
type: "boolean",
description:
"Upload the files a link brings in from outside the project folder. Otherwise they are left out and named.",
default: false,
},
width: {
type: "string",
description:
@@ -303,6 +309,7 @@ export default defineCommand({
projectDir,
stackName,
siteId: args["site-id"] as string | undefined,
includeOutsideFiles: args["include-outside-files"] === true,
json: Boolean(args.json),
});
return;
@@ -334,6 +341,7 @@ export default defineCommand({
projectDir,
stackName,
siteId: args["site-id"] as string | undefined,
includeOutsideFiles: args["include-outside-files"] === true,
fps: fpsRaw,
width,
height,
@@ -392,6 +400,7 @@ export default defineCommand({
stackName,
batch,
siteId: args["site-id"] as string | undefined,
includeOutsideFiles: args["include-outside-files"] === true,
fps: fpsRaw,
width,
height,
@@ -62,6 +62,7 @@ export interface RenderBatchArgs extends DeliveryFlagValues {
* separate `sites create` step (CI / cross-machine).
*/
siteId?: string;
includeOutsideFiles?: boolean;
/** Composition config — fps/width/height/format required, rest optional. */
fps: DistributedFps;
width: number;
@@ -244,6 +245,7 @@ export async function runRenderBatch(args: RenderBatchArgs): Promise<void> {
bucketName: stack.bucketName,
expectedBucketOwner: parseStateMachineArn(stack.stateMachineArn).account,
region: stack.region,
includeOutsideFiles: args.includeOutsideFiles,
onNotice: (message) => console.error(c.dim(message)),
});
if (!args.json) {
@@ -32,6 +32,7 @@ export interface RenderArgs extends DeliveryFlagValues {
projectDir: string;
stackName: string;
siteId?: string;
includeOutsideFiles?: boolean;
/** Composition config — fps/width/height/format required, rest optional. */
fps: DistributedFps;
width: number;
@@ -146,6 +147,7 @@ export async function runRender(args: RenderArgs): Promise<void> {
config,
executionName: args.executionName,
outputKey: args.outputKey,
includeOutsideFiles: args.includeOutsideFiles,
onNotice: (message) => console.error(c.dim(message)),
});
@@ -22,6 +22,7 @@ export interface SitesCreateArgs {
projectDir: string;
stackName: string;
siteId?: string;
includeOutsideFiles?: boolean;
/** Print machine-readable JSON instead of the human-friendly summary. */
json: boolean;
}
@@ -38,6 +39,7 @@ export async function runSitesCreate(args: SitesCreateArgs): Promise<void> {
expectedBucketOwner: parseStateMachineArn(stack.stateMachineArn).account,
region: stack.region,
siteId: args.siteId,
includeOutsideFiles: args.includeOutsideFiles,
onNotice: (message) => console.error(c.dim(message)),
});
@@ -751,6 +751,31 @@ const signedStagedS3Url =
"https://s3.example.com/upload?X-Amz-SignedHeaders=content-length;content-type;host;x-amz-server-side-encryption";
describe("publishProjectArchive", () => {
// fails without the fix: with no archive passed in, files from outside the project were uploaded unasked.
it("leaves files from outside the project out of an archive it builds itself", async () => {
const dir = makeProjectDir();
const outside = mkdtempSync(join(tmpdir(), "hf-outside-"));
const fetchMock = directFetch();
vi.stubGlobal("fetch", fetchMock);
try {
writeFileSync(join(outside, "logo.png"), "png", "utf-8");
writeFileSync(join(dir, "index.html"), `<img src="${join(outside, "logo.png")}">`, "utf-8");
writeFileSync(join(dir, "styles.css"), "body {}", "utf-8");
await publishProjectArchive(dir);
const file = (fetchMock.mock.calls[1]![1].body as FormData).get("file") as File;
const entries = new AdmZip(Buffer.from(await file.arrayBuffer()))
.getEntries()
.map((entry) => entry.entryName);
expect(entries).toContain("styles.css");
expect(entries.filter((name) => name.includes("logo.png"))).toEqual([]);
} finally {
rmSync(dir, { recursive: true, force: true });
rmSync(outside, { recursive: true, force: true });
}
});
it("takes the publish service from the environment under HYPERFRAMES_ENV=dev", () => {
vi.stubEnv("HYPERFRAMES_ENV", "dev");
expect(getPublishApiBaseUrl()).toBe("https://api2.heygen.com");
+3 -3
View File
@@ -1186,8 +1186,8 @@ export interface PublishOptions {
* Pre-built archive to upload instead of building one fresh from
* `projectDir` via `createPublishArchive`. `publish.ts` passes this so it
* can bake proxies into the file map between `buildPublishFileMap` and
* `zipPublishFileMap` (U6); callers that omit it (e.g. `feedback`'s
* minimal-repro publish) keep today's behavior unchanged.
* `zipPublishFileMap` (U6). Without it, the archive built here leaves out
* every file from outside the project, since no one approved those.
*/
archive?: PublishArchiveResult;
/** The credential the caller already checked; `null` publishes anonymously. Resolved here when omitted. */
@@ -1200,7 +1200,7 @@ export async function publishProjectArchive(
): Promise<PublishedProjectResponse> {
const isPublic = opts.public === true;
const title = basename(projectDir);
const archive = opts.archive ?? createPublishArchive(projectDir);
const archive = opts.archive ?? createPublishArchive(projectDir, undefined, () => false);
const apiBaseUrl = getPublishApiBaseUrl();
const credential = await resolvePublishCredential(opts.credential);
const authHeaders = credential ? buildAuthHeaders(credential) : {};
@@ -85,7 +85,7 @@ describe("deploySite", () => {
expect(handle.projectGcsUri).toBe("gs://b/sites/my-git-sha/project.tar.gz");
});
it("packs a symlinked project and its symlinked files as content", async () => {
it("packs a symlinked project, and its files linked in from outside only when included", async () => {
const gcs = new FakeGcs();
const scratch = mkdtempSync(join(tmpdir(), "hf-site-scratch-"));
tmpDirs.push(scratch);
@@ -93,15 +93,20 @@ describe("deploySite", () => {
writeFileSync(join(scratch, "logo.txt"), "logo");
symlinkSync(join(scratch, "logo.txt"), join(real, "logo.txt"));
symlinkSync(real, join(scratch, "fx"));
const handle = await deploySite({
projectDir: join(scratch, "fx"),
bucketName: "b",
storage: asStorage(gcs),
});
const file = join(scratch, "uploaded.tar.gz");
writeFileSync(file, gcs.objects.get(handle.projectGcsUri) ?? "");
const entries: string[] = [];
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
expect(entries).toEqual(["File index.html", "File logo.txt"]);
async function packed(includeOutsideFiles: boolean): Promise<string[]> {
const handle = await deploySite({
projectDir: join(scratch, "fx"),
bucketName: "b",
includeOutsideFiles,
storage: asStorage(gcs),
});
const file = join(scratch, "uploaded.tar.gz");
writeFileSync(file, gcs.objects.get(handle.projectGcsUri) ?? "");
const entries: string[] = [];
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
return entries;
}
expect(await packed(false)).toEqual(["File index.html"]);
expect(await packed(true)).toEqual(["File index.html", "File logo.txt"]);
});
});
+2 -1
View File
@@ -33,6 +33,7 @@ export interface DeploySiteOptions {
*/
siteId?: string;
onNotice?: (message: string) => void;
includeOutsideFiles?: boolean;
/** Injection seam for tests. Production callers leave unset. */
storage?: Storage;
}
@@ -66,7 +67,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
}
const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice);
const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice, opts.includeOutsideFiles);
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
const key = `sites/${siteId}/project.tar.gz`;
const projectGcsUri = formatGcsUri({ bucket: opts.bucketName, key });
@@ -50,6 +50,9 @@ export interface RenderToCloudRunOptions {
projectDir?: string;
/** Re-use an existing `deploySite` upload (skips tar+GCS upload). */
siteHandle?: SiteHandle;
/** Passed to {@link deploySite} when this call uploads the project, with its left-out notices. */
includeOutsideFiles?: boolean;
onNotice?: (message: string) => void;
/** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */
config: SerializableDistributedRenderConfig;
/** GCS bucket from the Terraform output (`render_bucket_name`). */
@@ -146,6 +149,8 @@ export async function renderToCloudRun(opts: RenderToCloudRunOptions): Promise<R
(await deploySite({
projectDir: opts.projectDir as string,
bucketName: opts.bucketName,
onNotice: opts.onNotice,
includeOutsideFiles: opts.includeOutsideFiles,
storage: opts.storage,
}));
@@ -125,6 +125,44 @@ describe("listSiteFiles / hashProjectDir", () => {
expect(listSiteFiles(dir)).toEqual(["index.html"]);
});
// fails without the fix: an upload shipped whatever a link inside the project pointed at.
it("leaves out an upload's files and folders linked in from outside the project, naming each", () => {
const dir = project();
const outside = tempDir();
mkdirSync(join(outside, "media"));
mkdirSync(join(dir, "assets"));
writeFileSync(join(outside, "shared.css"), "a{}");
writeFileSync(join(outside, "media", "clip.txt"), "v1");
writeFileSync(join(dir, "assets", "logo.png"), "png");
symlinkSync(join(outside, "shared.css"), join(dir, "shared.css"));
symlinkSync(join(outside, "media"), join(dir, "media"));
symlinkSync(join(dir, "assets", "logo.png"), join(dir, "logo.png"));
const notices: string[] = [];
expect(listSiteFilesForUpload(dir, (line) => notices.push(line))).toEqual([
"assets/logo.png",
"index.html",
"logo.png",
]);
expect(notices).toHaveLength(1);
expect(notices[0]).toContain("outside the project: media, shared.css.");
expect(notices[0]).toContain("--include-outside-files");
});
it("uploads files linked in from outside the project when the caller includes them", () => {
const dir = project();
const outside = tempDir();
writeFileSync(join(outside, "shared.css"), "a{}");
symlinkSync(join(outside, "shared.css"), join(dir, "shared.css"));
const notices: string[] = [];
expect(listSiteFilesForUpload(dir, (line) => notices.push(line), true)).toEqual([
"index.html",
"shared.css",
]);
expect(notices).toEqual([]);
});
it("does not loop on a folder that links back to an ancestor", () => {
const dir = project();
symlinkSync(dir, join(dir, "self"));
@@ -11,7 +11,7 @@
import { closeSync, openSync, readdirSync, readSync, realpathSync, statSync } from "node:fs";
import { createHash } from "node:crypto";
import { join } from "node:path";
import { join, sep } from "node:path";
import { isPrivateFileUnder, isStudioPrivatePath } from "@hyperframes/core/safe-path";
import { PLAN_PROJECT_DIR_SKIP_SEGMENTS } from "./plan.js";
@@ -35,16 +35,23 @@ function statEntry(full: string, relPath: string) {
* (root included), top-level {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} are left out as the plan stage
* does, and a dangling symlink throws. Private files (env, Git, credentials, tool and agent state,
* app records) are left out at every depth, by name or by where a link points; `onPrivate` hears
* each one left out, a folder once.
* each one left out, a folder once. With `onOutside`, a file or folder whose link points outside
* the project is left out too, and `onOutside` hears it, a folder once.
*/
export function listSiteFiles(projectDir: string, onPrivate?: (relPath: string) => void): string[] {
export function listSiteFiles(
projectDir: string,
onPrivate?: (relPath: string) => void,
onOutside?: (relPath: string) => void,
): string[] {
const files: string[] = [];
const ancestors = new Set([realpathSync(projectDir)]);
const root = realpathSync(projectDir);
const ancestors = new Set([root]);
function visit(full: string, relPath: string): void {
if (isPrivateFileUnder(projectDir, full, isStudioPrivatePath)) return void onPrivate?.(relPath);
const stat = statEntry(full, relPath);
if (!stat.isFile() && !stat.isDirectory()) return;
if (onOutside && outsideFolder(root, realpathSync(full))) return void onOutside(relPath);
if (stat.isFile()) return void files.push(relPath);
if (!stat.isDirectory()) return;
const real = realpathSync(full);
// A folder linking back to an ancestor is skipped rather than walked forever.
if (ancestors.has(real)) return;
@@ -62,6 +69,9 @@ export function listSiteFiles(projectDir: string, onPrivate?: (relPath: string)
return files;
}
const outsideFolder = (root: string, real: string): boolean =>
real !== root && !real.startsWith(root + sep);
const printablePath = (path: string): string =>
[...path]
.map((ch) => {
@@ -71,19 +81,34 @@ const printablePath = (path: string): string =>
})
.join("");
/** {@link listSiteFiles} for an upload: one line to `onNotice` names the private files left out. */
/**
* {@link listSiteFiles} for an upload: one line to `onNotice` names the private files left out.
* Files linked in from outside the project ship only with `includeOutsideFiles`; otherwise a second
* line names each one, since the person may not know a link put them in the project.
*/
export function listSiteFilesForUpload(
projectDir: string,
onNotice?: (message: string) => void,
includeOutsideFiles = false,
): string[] {
const leftOut: string[] = [];
const files = listSiteFiles(projectDir, (path) => leftOut.push(path));
const outside: string[] = [];
const files = listSiteFiles(
projectDir,
(path) => leftOut.push(path),
includeOutsideFiles ? undefined : (path) => outside.push(path),
);
if (leftOut.length > 0) {
const more = leftOut.length > 5 ? ` and ${leftOut.length - 5} more` : "";
onNotice?.(
`[deploySite] left out private files no composition loads (env, Git, credentials, tool state): ${leftOut.slice(0, 5).map(printablePath).join(", ")}${more}`,
);
}
if (outside.length > 0) {
onNotice?.(
`[deploySite] left out files linked in from outside the project: ${outside.map(printablePath).join(", ")}. Pass --include-outside-files (includeOutsideFiles in the SDK) to upload them.`,
);
}
return files;
}