diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index f8b308d473..edb5e9899e 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -3031,6 +3031,7 @@ And these drive `render` and `render-batch`: | ----------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------------- | | `--width`, `--height` | required | Render dimensions in pixels | | `--site-id` | content hash | Reuse an uploaded site instead of uploading again | +| `--include-outside-files` | off | Upload the files a link brings in from outside the project folder. Without it they are left out and named | | `--fps` | `30` | `24`, `25`, `30`, or `60` | | `--format` | `mp4` | `mp4`, `mov`, `webm`, or `png-sequence` | | `--codec` | `h264` | `h264` or `h265`, MP4 only | @@ -3083,6 +3084,10 @@ hyperframes lambda sites create ./my-project hyperframes lambda render ./my-project --site-id=abc1234deadbeef0 --width 1920 --height 1080 ``` +A file or folder that a symlink brings in from outside the project is left out +of the upload, and a notice names each one. Pass `--include-outside-files` (or +`includeOutsideFiles: true` to the SDK's `deploySite`) to upload them too. + ### `lambda render ` Starts a Step Functions execution and returns a `renderId` immediately — poll it diff --git a/packages/aws-lambda/src/sdk/deploySite.test.ts b/packages/aws-lambda/src/sdk/deploySite.test.ts index 9ea662a7c9..f7fbe05e34 100644 --- a/packages/aws-lambda/src/sdk/deploySite.test.ts +++ b/packages/aws-lambda/src/sdk/deploySite.test.ts @@ -202,7 +202,7 @@ describe("deploySite", () => { expect(after.siteId).toBe(before.siteId); }); - it("ships a symlinked project and its symlinked files and folders as content", async () => { + it("ships a symlinked project, and its files linked in from outside only when included", async () => { mkdirSync(join(scratch, "media")); writeFileSync(join(scratch, "media", "clip.txt"), "clip"); writeFileSync(join(scratch, "logo.txt"), "logo"); @@ -210,12 +210,23 @@ describe("deploySite", () => { symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt")); symlinkSync(projectDir, join(scratch, "fx")); const s3 = new FakeS3(); - const result = await deploySite({ + const notices: string[] = []; + const leftOut = await deploySite({ projectDir: join(scratch, "fx"), bucketName: "test-bucket", + onNotice: (line) => notices.push(line), s3: asS3Client(s3), }); - expect(sitePaths(s3, result.siteId)).toEqual([ + expect(sitePaths(s3, leftOut.siteId)).toEqual(["assets/style.css", "index.html"]); + expect(notices.join("\n")).toContain("outside the project: logo.txt, media."); + + const included = await deploySite({ + projectDir: join(scratch, "fx"), + bucketName: "test-bucket", + includeOutsideFiles: true, + s3: asS3Client(s3), + }); + expect(sitePaths(s3, included.siteId)).toEqual([ "assets/style.css", "index.html", "logo.txt", diff --git a/packages/aws-lambda/src/sdk/deploySite.ts b/packages/aws-lambda/src/sdk/deploySite.ts index 8f965b8b33..bb41ace264 100644 --- a/packages/aws-lambda/src/sdk/deploySite.ts +++ b/packages/aws-lambda/src/sdk/deploySite.ts @@ -64,6 +64,7 @@ export interface DeploySiteOptions { siteId?: string; /** Called with a line a user should see, such as audio sidecars skipped for want of ffmpeg. */ onNotice?: (message: string) => void; + includeOutsideFiles?: boolean; expectedBucketOwner?: string; /** Injection seam for tests. Production callers leave unset. */ s3?: S3Client; @@ -162,7 +163,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise { if (!statSync(opts.projectDir).isDirectory()) { throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`); } - const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice); + const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice, opts.includeOutsideFiles); const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files); const projectS3Uri = siteManifestUri(opts.bucketName, siteId); diff --git a/packages/aws-lambda/src/sdk/renderToLambda.ts b/packages/aws-lambda/src/sdk/renderToLambda.ts index 47ab02e0a2..04f0fcd403 100644 --- a/packages/aws-lambda/src/sdk/renderToLambda.ts +++ b/packages/aws-lambda/src/sdk/renderToLambda.ts @@ -38,6 +38,7 @@ export interface RenderToLambdaOptions { /** Re-use an existing `deploySite` upload (skips tar+S3 PUT). */ siteHandle?: SiteHandle; onNotice?: (message: string) => void; + includeOutsideFiles?: boolean; /** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */ config: SerializableDistributedRenderConfig; /** S3 bucket from the SAM stack output (`RenderBucketName`). */ @@ -123,6 +124,7 @@ export async function renderToLambda(opts: RenderToLambdaOptions): Promise): Promise { projectDir: resolve(projectDir), bucketName: state.bucketName, siteId: args["site-id"] as string | undefined, + includeOutsideFiles: args["include-outside-files"] === true, onNotice: (message) => console.error(message), }); if (args.json) { @@ -563,6 +570,8 @@ async function runRender(args: Record): Promise { const { renderToCloudRun, getRenderProgress } = await loadCloudRunAdapter(); const handle = await renderToCloudRun({ projectDir: resolve(projectDir), + includeOutsideFiles: args["include-outside-files"] === true, + onNotice: (message) => console.error(message), config: config as Parameters[0]["config"], bucketName: state.bucketName, projectId: state.projectId, @@ -694,6 +703,7 @@ async function runRenderBatch(args: Record): Promise { projectDir: resolve(projectDir), bucketName: state.bucketName, siteId: args["site-id"] as string | undefined, + includeOutsideFiles: args["include-outside-files"] === true, onNotice: (message) => console.error(message), }); diff --git a/packages/cli/src/commands/lambda.ts b/packages/cli/src/commands/lambda.ts index d90c755acc..0f069ce326 100644 --- a/packages/cli/src/commands/lambda.ts +++ b/packages/cli/src/commands/lambda.ts @@ -136,6 +136,12 @@ export default defineCommand({ // sites / render "site-id": { type: "string", description: "Explicit site id (overrides content hash)" }, + "include-outside-files": { + type: "boolean", + description: + "Upload the files a link brings in from outside the project folder. Otherwise they are left out and named.", + default: false, + }, width: { type: "string", description: @@ -303,6 +309,7 @@ export default defineCommand({ projectDir, stackName, siteId: args["site-id"] as string | undefined, + includeOutsideFiles: args["include-outside-files"] === true, json: Boolean(args.json), }); return; @@ -334,6 +341,7 @@ export default defineCommand({ projectDir, stackName, siteId: args["site-id"] as string | undefined, + includeOutsideFiles: args["include-outside-files"] === true, fps: fpsRaw, width, height, @@ -392,6 +400,7 @@ export default defineCommand({ stackName, batch, siteId: args["site-id"] as string | undefined, + includeOutsideFiles: args["include-outside-files"] === true, fps: fpsRaw, width, height, diff --git a/packages/cli/src/commands/lambda/render-batch.ts b/packages/cli/src/commands/lambda/render-batch.ts index 43ac73f101..77637f7641 100644 --- a/packages/cli/src/commands/lambda/render-batch.ts +++ b/packages/cli/src/commands/lambda/render-batch.ts @@ -62,6 +62,7 @@ export interface RenderBatchArgs extends DeliveryFlagValues { * separate `sites create` step (CI / cross-machine). */ siteId?: string; + includeOutsideFiles?: boolean; /** Composition config — fps/width/height/format required, rest optional. */ fps: DistributedFps; width: number; @@ -244,6 +245,7 @@ export async function runRenderBatch(args: RenderBatchArgs): Promise { bucketName: stack.bucketName, expectedBucketOwner: parseStateMachineArn(stack.stateMachineArn).account, region: stack.region, + includeOutsideFiles: args.includeOutsideFiles, onNotice: (message) => console.error(c.dim(message)), }); if (!args.json) { diff --git a/packages/cli/src/commands/lambda/render.ts b/packages/cli/src/commands/lambda/render.ts index 9f0b657c86..59b126ec2b 100644 --- a/packages/cli/src/commands/lambda/render.ts +++ b/packages/cli/src/commands/lambda/render.ts @@ -32,6 +32,7 @@ export interface RenderArgs extends DeliveryFlagValues { projectDir: string; stackName: string; siteId?: string; + includeOutsideFiles?: boolean; /** Composition config — fps/width/height/format required, rest optional. */ fps: DistributedFps; width: number; @@ -146,6 +147,7 @@ export async function runRender(args: RenderArgs): Promise { config, executionName: args.executionName, outputKey: args.outputKey, + includeOutsideFiles: args.includeOutsideFiles, onNotice: (message) => console.error(c.dim(message)), }); diff --git a/packages/cli/src/commands/lambda/sites.ts b/packages/cli/src/commands/lambda/sites.ts index b0132d99e8..d4042f8a0a 100644 --- a/packages/cli/src/commands/lambda/sites.ts +++ b/packages/cli/src/commands/lambda/sites.ts @@ -22,6 +22,7 @@ export interface SitesCreateArgs { projectDir: string; stackName: string; siteId?: string; + includeOutsideFiles?: boolean; /** Print machine-readable JSON instead of the human-friendly summary. */ json: boolean; } @@ -38,6 +39,7 @@ export async function runSitesCreate(args: SitesCreateArgs): Promise { expectedBucketOwner: parseStateMachineArn(stack.stateMachineArn).account, region: stack.region, siteId: args.siteId, + includeOutsideFiles: args.includeOutsideFiles, onNotice: (message) => console.error(c.dim(message)), }); diff --git a/packages/cli/src/utils/publishProject.test.ts b/packages/cli/src/utils/publishProject.test.ts index d869305a57..db2a86f37f 100644 --- a/packages/cli/src/utils/publishProject.test.ts +++ b/packages/cli/src/utils/publishProject.test.ts @@ -751,6 +751,31 @@ const signedStagedS3Url = "https://s3.example.com/upload?X-Amz-SignedHeaders=content-length;content-type;host;x-amz-server-side-encryption"; describe("publishProjectArchive", () => { + // fails without the fix: with no archive passed in, files from outside the project were uploaded unasked. + it("leaves files from outside the project out of an archive it builds itself", async () => { + const dir = makeProjectDir(); + const outside = mkdtempSync(join(tmpdir(), "hf-outside-")); + const fetchMock = directFetch(); + vi.stubGlobal("fetch", fetchMock); + try { + writeFileSync(join(outside, "logo.png"), "png", "utf-8"); + writeFileSync(join(dir, "index.html"), ``, "utf-8"); + writeFileSync(join(dir, "styles.css"), "body {}", "utf-8"); + + await publishProjectArchive(dir); + + const file = (fetchMock.mock.calls[1]![1].body as FormData).get("file") as File; + const entries = new AdmZip(Buffer.from(await file.arrayBuffer())) + .getEntries() + .map((entry) => entry.entryName); + expect(entries).toContain("styles.css"); + expect(entries.filter((name) => name.includes("logo.png"))).toEqual([]); + } finally { + rmSync(dir, { recursive: true, force: true }); + rmSync(outside, { recursive: true, force: true }); + } + }); + it("takes the publish service from the environment under HYPERFRAMES_ENV=dev", () => { vi.stubEnv("HYPERFRAMES_ENV", "dev"); expect(getPublishApiBaseUrl()).toBe("https://api2.heygen.com"); diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index d516c1d4ca..43bee211bf 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -1186,8 +1186,8 @@ export interface PublishOptions { * Pre-built archive to upload instead of building one fresh from * `projectDir` via `createPublishArchive`. `publish.ts` passes this so it * can bake proxies into the file map between `buildPublishFileMap` and - * `zipPublishFileMap` (U6); callers that omit it (e.g. `feedback`'s - * minimal-repro publish) keep today's behavior unchanged. + * `zipPublishFileMap` (U6). Without it, the archive built here leaves out + * every file from outside the project, since no one approved those. */ archive?: PublishArchiveResult; /** The credential the caller already checked; `null` publishes anonymously. Resolved here when omitted. */ @@ -1200,7 +1200,7 @@ export async function publishProjectArchive( ): Promise { const isPublic = opts.public === true; const title = basename(projectDir); - const archive = opts.archive ?? createPublishArchive(projectDir); + const archive = opts.archive ?? createPublishArchive(projectDir, undefined, () => false); const apiBaseUrl = getPublishApiBaseUrl(); const credential = await resolvePublishCredential(opts.credential); const authHeaders = credential ? buildAuthHeaders(credential) : {}; diff --git a/packages/gcp-cloud-run/src/sdk/deploySite.test.ts b/packages/gcp-cloud-run/src/sdk/deploySite.test.ts index b7156b421e..5ca91dd7d5 100644 --- a/packages/gcp-cloud-run/src/sdk/deploySite.test.ts +++ b/packages/gcp-cloud-run/src/sdk/deploySite.test.ts @@ -85,7 +85,7 @@ describe("deploySite", () => { expect(handle.projectGcsUri).toBe("gs://b/sites/my-git-sha/project.tar.gz"); }); - it("packs a symlinked project and its symlinked files as content", async () => { + it("packs a symlinked project, and its files linked in from outside only when included", async () => { const gcs = new FakeGcs(); const scratch = mkdtempSync(join(tmpdir(), "hf-site-scratch-")); tmpDirs.push(scratch); @@ -93,15 +93,20 @@ describe("deploySite", () => { writeFileSync(join(scratch, "logo.txt"), "logo"); symlinkSync(join(scratch, "logo.txt"), join(real, "logo.txt")); symlinkSync(real, join(scratch, "fx")); - const handle = await deploySite({ - projectDir: join(scratch, "fx"), - bucketName: "b", - storage: asStorage(gcs), - }); - const file = join(scratch, "uploaded.tar.gz"); - writeFileSync(file, gcs.objects.get(handle.projectGcsUri) ?? ""); - const entries: string[] = []; - await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) }); - expect(entries).toEqual(["File index.html", "File logo.txt"]); + async function packed(includeOutsideFiles: boolean): Promise { + const handle = await deploySite({ + projectDir: join(scratch, "fx"), + bucketName: "b", + includeOutsideFiles, + storage: asStorage(gcs), + }); + const file = join(scratch, "uploaded.tar.gz"); + writeFileSync(file, gcs.objects.get(handle.projectGcsUri) ?? ""); + const entries: string[] = []; + await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) }); + return entries; + } + expect(await packed(false)).toEqual(["File index.html"]); + expect(await packed(true)).toEqual(["File index.html", "File logo.txt"]); }); }); diff --git a/packages/gcp-cloud-run/src/sdk/deploySite.ts b/packages/gcp-cloud-run/src/sdk/deploySite.ts index 7bf63cca76..a098eebf62 100644 --- a/packages/gcp-cloud-run/src/sdk/deploySite.ts +++ b/packages/gcp-cloud-run/src/sdk/deploySite.ts @@ -33,6 +33,7 @@ export interface DeploySiteOptions { */ siteId?: string; onNotice?: (message: string) => void; + includeOutsideFiles?: boolean; /** Injection seam for tests. Production callers leave unset. */ storage?: Storage; } @@ -66,7 +67,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise { throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`); } - const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice); + const files = listSiteFilesForUpload(opts.projectDir, opts.onNotice, opts.includeOutsideFiles); const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files); const key = `sites/${siteId}/project.tar.gz`; const projectGcsUri = formatGcsUri({ bucket: opts.bucketName, key }); diff --git a/packages/gcp-cloud-run/src/sdk/renderToCloudRun.ts b/packages/gcp-cloud-run/src/sdk/renderToCloudRun.ts index ac02060a13..a200bfb3e7 100644 --- a/packages/gcp-cloud-run/src/sdk/renderToCloudRun.ts +++ b/packages/gcp-cloud-run/src/sdk/renderToCloudRun.ts @@ -50,6 +50,9 @@ export interface RenderToCloudRunOptions { projectDir?: string; /** Re-use an existing `deploySite` upload (skips tar+GCS upload). */ siteHandle?: SiteHandle; + /** Passed to {@link deploySite} when this call uploads the project, with its left-out notices. */ + includeOutsideFiles?: boolean; + onNotice?: (message: string) => void; /** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */ config: SerializableDistributedRenderConfig; /** GCS bucket from the Terraform output (`render_bucket_name`). */ @@ -146,6 +149,8 @@ export async function renderToCloudRun(opts: RenderToCloudRunOptions): Promise { expect(listSiteFiles(dir)).toEqual(["index.html"]); }); + // fails without the fix: an upload shipped whatever a link inside the project pointed at. + it("leaves out an upload's files and folders linked in from outside the project, naming each", () => { + const dir = project(); + const outside = tempDir(); + mkdirSync(join(outside, "media")); + mkdirSync(join(dir, "assets")); + writeFileSync(join(outside, "shared.css"), "a{}"); + writeFileSync(join(outside, "media", "clip.txt"), "v1"); + writeFileSync(join(dir, "assets", "logo.png"), "png"); + symlinkSync(join(outside, "shared.css"), join(dir, "shared.css")); + symlinkSync(join(outside, "media"), join(dir, "media")); + symlinkSync(join(dir, "assets", "logo.png"), join(dir, "logo.png")); + const notices: string[] = []; + + expect(listSiteFilesForUpload(dir, (line) => notices.push(line))).toEqual([ + "assets/logo.png", + "index.html", + "logo.png", + ]); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain("outside the project: media, shared.css."); + expect(notices[0]).toContain("--include-outside-files"); + }); + + it("uploads files linked in from outside the project when the caller includes them", () => { + const dir = project(); + const outside = tempDir(); + writeFileSync(join(outside, "shared.css"), "a{}"); + symlinkSync(join(outside, "shared.css"), join(dir, "shared.css")); + const notices: string[] = []; + + expect(listSiteFilesForUpload(dir, (line) => notices.push(line), true)).toEqual([ + "index.html", + "shared.css", + ]); + expect(notices).toEqual([]); + }); + it("does not loop on a folder that links back to an ancestor", () => { const dir = project(); symlinkSync(dir, join(dir, "self")); diff --git a/packages/producer/src/services/distributed/projectHash.ts b/packages/producer/src/services/distributed/projectHash.ts index 091b8b4807..f8a2af9d38 100644 --- a/packages/producer/src/services/distributed/projectHash.ts +++ b/packages/producer/src/services/distributed/projectHash.ts @@ -11,7 +11,7 @@ import { closeSync, openSync, readdirSync, readSync, realpathSync, statSync } from "node:fs"; import { createHash } from "node:crypto"; -import { join } from "node:path"; +import { join, sep } from "node:path"; import { isPrivateFileUnder, isStudioPrivatePath } from "@hyperframes/core/safe-path"; import { PLAN_PROJECT_DIR_SKIP_SEGMENTS } from "./plan.js"; @@ -35,16 +35,23 @@ function statEntry(full: string, relPath: string) { * (root included), top-level {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} are left out as the plan stage * does, and a dangling symlink throws. Private files (env, Git, credentials, tool and agent state, * app records) are left out at every depth, by name or by where a link points; `onPrivate` hears - * each one left out, a folder once. + * each one left out, a folder once. With `onOutside`, a file or folder whose link points outside + * the project is left out too, and `onOutside` hears it, a folder once. */ -export function listSiteFiles(projectDir: string, onPrivate?: (relPath: string) => void): string[] { +export function listSiteFiles( + projectDir: string, + onPrivate?: (relPath: string) => void, + onOutside?: (relPath: string) => void, +): string[] { const files: string[] = []; - const ancestors = new Set([realpathSync(projectDir)]); + const root = realpathSync(projectDir); + const ancestors = new Set([root]); function visit(full: string, relPath: string): void { if (isPrivateFileUnder(projectDir, full, isStudioPrivatePath)) return void onPrivate?.(relPath); const stat = statEntry(full, relPath); + if (!stat.isFile() && !stat.isDirectory()) return; + if (onOutside && outsideFolder(root, realpathSync(full))) return void onOutside(relPath); if (stat.isFile()) return void files.push(relPath); - if (!stat.isDirectory()) return; const real = realpathSync(full); // A folder linking back to an ancestor is skipped rather than walked forever. if (ancestors.has(real)) return; @@ -62,6 +69,9 @@ export function listSiteFiles(projectDir: string, onPrivate?: (relPath: string) return files; } +const outsideFolder = (root: string, real: string): boolean => + real !== root && !real.startsWith(root + sep); + const printablePath = (path: string): string => [...path] .map((ch) => { @@ -71,19 +81,34 @@ const printablePath = (path: string): string => }) .join(""); -/** {@link listSiteFiles} for an upload: one line to `onNotice` names the private files left out. */ +/** + * {@link listSiteFiles} for an upload: one line to `onNotice` names the private files left out. + * Files linked in from outside the project ship only with `includeOutsideFiles`; otherwise a second + * line names each one, since the person may not know a link put them in the project. + */ export function listSiteFilesForUpload( projectDir: string, onNotice?: (message: string) => void, + includeOutsideFiles = false, ): string[] { const leftOut: string[] = []; - const files = listSiteFiles(projectDir, (path) => leftOut.push(path)); + const outside: string[] = []; + const files = listSiteFiles( + projectDir, + (path) => leftOut.push(path), + includeOutsideFiles ? undefined : (path) => outside.push(path), + ); if (leftOut.length > 0) { const more = leftOut.length > 5 ? ` and ${leftOut.length - 5} more` : ""; onNotice?.( `[deploySite] left out private files no composition loads (env, Git, credentials, tool state): ${leftOut.slice(0, 5).map(printablePath).join(", ")}${more}`, ); } + if (outside.length > 0) { + onNotice?.( + `[deploySite] left out files linked in from outside the project: ${outside.map(printablePath).join(", ")}. Pass --include-outside-files (includeOutsideFiles in the SDK) to upload them.`, + ); + } return files; }