diff --git a/packages/aws-lambda/src/s3Transport.ts b/packages/aws-lambda/src/s3Transport.ts index 5efbac7be1..3a0ec18619 100644 --- a/packages/aws-lambda/src/s3Transport.ts +++ b/packages/aws-lambda/src/s3Transport.ts @@ -267,12 +267,20 @@ function isRecord(value: unknown): value is Record { * binary — the AWS Lambda Node 22 base image ships a minimal set of * userland tools and does NOT include `tar` in `/usr/bin`. */ -export async function tarDirectory(sourceDir: string, destTarball: string): Promise { +export async function tarDirectory( + sourceDir: string, + destTarball: string, + files?: string[], +): Promise { if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) { throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`); } mkdirSync(dirname(destTarball), { recursive: true }); - await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]); + // `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed. + await tar.create( + { gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined }, + files ?? ["."], + ); } /** diff --git a/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts b/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts index 8221b9c797..036abc711b 100644 --- a/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts +++ b/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts @@ -8,6 +8,7 @@ * `deploySite`'s `finally` block has rmSync'd it. */ +import { buffer } from "node:stream/consumers"; import type { S3Client } from "@aws-sdk/client-s3"; export interface FakeS3Op { @@ -19,6 +20,7 @@ export interface FakeS3Op { export class FakeS3 { ops: FakeS3Op[] = []; existing = new Set(); + bodies = new Map(); async send(command: unknown): Promise { const cmdName = (command as { constructor: { name: string } }).constructor.name; @@ -39,7 +41,12 @@ export class FakeS3 { if (cmdName === "PutObjectCommand") { this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key }); this.existing.add(`${input.Bucket}/${input.Key}`); - await drainBody((command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body); + this.bodies.set( + `${input.Bucket}/${input.Key}`, + await drainBody( + (command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body, + ), + ); return {}; } throw new Error(`FakeS3: unexpected command ${cmdName}`); @@ -58,12 +65,6 @@ export function asS3Client(fake: FakeS3): S3Client { * the workdir — surfacing as an "Unhandled error between tests" * teardown noise that bun's runner attributes to the next test. */ -async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise { - if (Buffer.isBuffer(body)) return; - await new Promise((resolve, reject) => { - body.on("data", () => {}); - body.on("end", () => resolve()); - body.on("close", () => resolve()); - body.on("error", reject); - }); +async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise { + return Buffer.isBuffer(body) ? body : buffer(body); } diff --git a/packages/aws-lambda/src/sdk/deploySite.test.ts b/packages/aws-lambda/src/sdk/deploySite.test.ts index 9863b05187..bd65557fe5 100644 --- a/packages/aws-lambda/src/sdk/deploySite.test.ts +++ b/packages/aws-lambda/src/sdk/deploySite.test.ts @@ -1,15 +1,18 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { S3Client } from "@aws-sdk/client-s3"; +import * as tar from "tar"; import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js"; import { deploySite } from "./deploySite.js"; let projectDir: string; +let scratch: string; beforeEach(() => { projectDir = mkdtempSync(join(tmpdir(), "hf-deploy-site-test-")); + scratch = mkdtempSync(join(tmpdir(), "hf-deploy-site-scratch-")); mkdirSync(join(projectDir, "assets")); writeFileSync(join(projectDir, "index.html"), "hi"); writeFileSync(join(projectDir, "assets", "style.css"), "body { color: red; }"); @@ -17,8 +20,17 @@ beforeEach(() => { afterEach(() => { rmSync(projectDir, { recursive: true, force: true }); + rmSync(scratch, { recursive: true, force: true }); }); +async function uploadedEntries(s3: FakeS3, siteId: string): Promise { + const file = join(scratch, "uploaded.tar.gz"); + writeFileSync(file, s3.bodies.get(`test-bucket/sites/${siteId}/project.tar.gz`) ?? ""); + const entries: string[] = []; + await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) }); + return entries; +} + describe("deploySite", () => { it("uploads the tarball when no matching object exists", async () => { const s3 = new FakeS3(); @@ -145,4 +157,25 @@ describe("deploySite", () => { // node_modules contents shouldn't move the hash. expect(after.siteId).toBe(before.siteId); }); + + it("packs a symlinked project and its symlinked files and folders as content", async () => { + mkdirSync(join(scratch, "media")); + writeFileSync(join(scratch, "media", "clip.txt"), "clip"); + writeFileSync(join(scratch, "logo.txt"), "logo"); + symlinkSync(join(scratch, "media"), join(projectDir, "media")); + symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt")); + symlinkSync(projectDir, join(scratch, "fx")); + const s3 = new FakeS3(); + const result = await deploySite({ + projectDir: join(scratch, "fx"), + bucketName: "test-bucket", + s3: asS3Client(s3), + }); + expect(await uploadedEntries(s3, result.siteId)).toEqual([ + "File assets/style.css", + "File index.html", + "File logo.txt", + "File media/clip.txt", + ]); + }); }); diff --git a/packages/aws-lambda/src/sdk/deploySite.ts b/packages/aws-lambda/src/sdk/deploySite.ts index c325a0b6a1..1a5ab49e25 100644 --- a/packages/aws-lambda/src/sdk/deploySite.ts +++ b/packages/aws-lambda/src/sdk/deploySite.ts @@ -16,7 +16,7 @@ import { mkdtempSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3"; -import { hashProjectDir } from "@hyperframes/producer/distributed"; +import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed"; import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js"; /** Options for {@link deploySite}. */ @@ -65,7 +65,8 @@ export async function deploySite(opts: DeploySiteOptions): Promise { throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`); } - const siteId = opts.siteId ?? hashProjectDir(opts.projectDir); + const files = listSiteFiles(opts.projectDir); + const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files); const key = `sites/${siteId}/project.tar.gz`; const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key }); const s3 = opts.s3 ?? new S3Client({ region: opts.region }); @@ -88,12 +89,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise { const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-")); try { const tarball = join(workdir, "project.tar.gz"); - await tarDirectory(opts.projectDir, tarball); - // Note: tarDirectory packs *everything* under `cwd`. We don't need to - // re-implement the skip list inside the tar pack because the - // producer's plan stage applies the same skip during its copy; the - // archive is slightly bigger than the planDir's compiled/ subtree - // but the cost is bounded by the project's user-authored content. + await tarDirectory(opts.projectDir, tarball, files); const size = statSync(tarball).size; await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip"); return { diff --git a/packages/gcp-cloud-run/src/gcsTransport.ts b/packages/gcp-cloud-run/src/gcsTransport.ts index 4147ad09ae..32055ca568 100644 --- a/packages/gcp-cloud-run/src/gcsTransport.ts +++ b/packages/gcp-cloud-run/src/gcsTransport.ts @@ -241,12 +241,20 @@ function isGcsPreconditionFailed(error: unknown): boolean { * package (pure JS over `node:zlib`) rather than spawning a system tar * binary so the archive format is independent of the container's userland. */ -export async function tarDirectory(sourceDir: string, destTarball: string): Promise { +export async function tarDirectory( + sourceDir: string, + destTarball: string, + files?: string[], +): Promise { if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) { throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`); } mkdirSync(dirname(destTarball), { recursive: true }); - await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]); + // `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed. + await tar.create( + { gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined }, + files ?? ["."], + ); } /** diff --git a/packages/gcp-cloud-run/src/sdk/deploySite.test.ts b/packages/gcp-cloud-run/src/sdk/deploySite.test.ts index 6c631ef6c9..b7156b421e 100644 --- a/packages/gcp-cloud-run/src/sdk/deploySite.test.ts +++ b/packages/gcp-cloud-run/src/sdk/deploySite.test.ts @@ -4,9 +4,10 @@ */ import { afterEach, describe, expect, it } from "bun:test"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import * as tar from "tar"; import { asStorage, FakeGcs } from "../__fixtures__/fakeGcs.js"; import { deploySite } from "./deploySite.js"; @@ -83,4 +84,24 @@ describe("deploySite", () => { expect(handle.siteId).toBe("my-git-sha"); expect(handle.projectGcsUri).toBe("gs://b/sites/my-git-sha/project.tar.gz"); }); + + it("packs a symlinked project and its symlinked files as content", async () => { + const gcs = new FakeGcs(); + const scratch = mkdtempSync(join(tmpdir(), "hf-site-scratch-")); + tmpDirs.push(scratch); + const real = mkProject("linked"); + writeFileSync(join(scratch, "logo.txt"), "logo"); + symlinkSync(join(scratch, "logo.txt"), join(real, "logo.txt")); + symlinkSync(real, join(scratch, "fx")); + const handle = await deploySite({ + projectDir: join(scratch, "fx"), + bucketName: "b", + storage: asStorage(gcs), + }); + const file = join(scratch, "uploaded.tar.gz"); + writeFileSync(file, gcs.objects.get(handle.projectGcsUri) ?? ""); + const entries: string[] = []; + await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) }); + expect(entries).toEqual(["File index.html", "File logo.txt"]); + }); }); diff --git a/packages/gcp-cloud-run/src/sdk/deploySite.ts b/packages/gcp-cloud-run/src/sdk/deploySite.ts index dfd9f63ab6..75e0aab3f3 100644 --- a/packages/gcp-cloud-run/src/sdk/deploySite.ts +++ b/packages/gcp-cloud-run/src/sdk/deploySite.ts @@ -17,7 +17,7 @@ import { mkdtempSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Storage } from "@google-cloud/storage"; -import { hashProjectDir } from "@hyperframes/producer/distributed"; +import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed"; import { formatGcsUri, tarDirectory, uploadFileToGcs } from "../gcsTransport.js"; /** Options for {@link deploySite}. */ @@ -65,7 +65,8 @@ export async function deploySite(opts: DeploySiteOptions): Promise { throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`); } - const siteId = opts.siteId ?? hashProjectDir(opts.projectDir); + const files = listSiteFiles(opts.projectDir); + const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files); const key = `sites/${siteId}/project.tar.gz`; const projectGcsUri = formatGcsUri({ bucket: opts.bucketName, key }); const storage = opts.storage ?? new Storage(); @@ -89,7 +90,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise { const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-")); try { const tarball = join(workdir, "project.tar.gz"); - await tarDirectory(opts.projectDir, tarball); + await tarDirectory(opts.projectDir, tarball, files); const size = statSync(tarball).size; await uploadFileToGcs(storage, tarball, projectGcsUri, "application/gzip"); return { diff --git a/packages/producer/src/distributed.ts b/packages/producer/src/distributed.ts index 0a1366a41e..aaa60d1e19 100644 --- a/packages/producer/src/distributed.ts +++ b/packages/producer/src/distributed.ts @@ -115,7 +115,7 @@ export { validateDistributedRenderConfig, validateVariablesPayload, } from "./services/distributed/renderConfigValidation.js"; -export { hashProjectDir } from "./services/distributed/projectHash.js"; +export { hashProjectDir, listSiteFiles } from "./services/distributed/projectHash.js"; // ── Plan protocol compatibility ──────────────────────────────────────────── // Workers validate this descriptor before consuming layout-specific diff --git a/packages/producer/src/regression-harness-lambda-local.ts b/packages/producer/src/regression-harness-lambda-local.ts index 88a2da4127..e8817f5332 100644 --- a/packages/producer/src/regression-harness-lambda-local.ts +++ b/packages/producer/src/regression-harness-lambda-local.ts @@ -45,6 +45,7 @@ import type { RenderChunkLambdaResult, SerializableDistributedRenderConfig, } from "@hyperframes/aws-lambda"; +import { listSiteFiles } from "./services/distributed/projectHash.js"; export type { LambdaLocalRenderResult, @@ -78,7 +79,7 @@ export async function runLambdaLocalRender( const projectKey = `sites/harness/${Date.now()}/project.tar.gz`; const projectS3Path = join(s3Root, projectKey); mkdirSync(dirname(projectS3Path), { recursive: true }); - await tarDirectory(input.projectDir, projectS3Path); + await tarDirectory(input.projectDir, projectS3Path, listSiteFiles(input.projectDir)); const fakeS3 = new FilesystemBackedFakeS3(s3Root); const deps: HandlerDeps = { diff --git a/packages/producer/src/services/distributed/projectHash.test.ts b/packages/producer/src/services/distributed/projectHash.test.ts new file mode 100644 index 0000000000..7833f98aaa --- /dev/null +++ b/packages/producer/src/services/distributed/projectHash.test.ts @@ -0,0 +1,74 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { hashProjectDir, listSiteFiles } from "./projectHash.js"; + +const dirs: string[] = []; +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "hf-project-hash-")); + dirs.push(dir); + return dir; +} +function project(): string { + const dir = tempDir(); + writeFileSync(join(dir, "index.html"), "hi"); + return dir; +} +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("listSiteFiles / hashProjectDir", () => { + it("follows a symlinked project root", () => { + const real = project(); + const link = join(tempDir(), "fx"); + symlinkSync(real, link); + expect(listSiteFiles(link)).toEqual(["index.html"]); + expect(hashProjectDir(link)).toBe(hashProjectDir(real)); + }); + + it("lists symlinked files and folders, and hashes their targets' bytes", () => { + const dir = project(); + const outside = tempDir(); + mkdirSync(join(outside, "media")); + writeFileSync(join(outside, "shared.css"), "a{}"); + writeFileSync(join(outside, "media", "clip.txt"), "v1"); + symlinkSync(join(outside, "shared.css"), join(dir, "shared.css")); + symlinkSync(join(outside, "media"), join(dir, "media")); + expect(listSiteFiles(dir)).toEqual(["index.html", "media/clip.txt", "shared.css"]); + + const before = hashProjectDir(dir); + writeFileSync(join(outside, "media", "clip.txt"), "v2"); + expect(hashProjectDir(dir)).not.toBe(before); + }); + + it("skips top-level skip-list folders", () => { + const dir = project(); + mkdirSync(join(dir, "node_modules")); + writeFileSync(join(dir, "node_modules", "junk.js"), "x"); + expect(listSiteFiles(dir)).toEqual(["index.html"]); + }); + + it("does not loop on a folder that links back to an ancestor", () => { + const dir = project(); + symlinkSync(dir, join(dir, "self")); + expect(listSiteFiles(dir)).toEqual(["index.html"]); + }); + + it("throws on a dangling symlink, naming it", () => { + const dir = project(); + symlinkSync(join(dir, "missing.png"), join(dir, "broken.png")); + expect(() => listSiteFiles(dir)).toThrow(/cannot read broken\.png/); + }); + + it("never reuses an id from the unversioned scheme", () => { + const dir = project(); + const unversioned = createHash("sha256") + .update("index.html\0hi") + .digest("hex") + .slice(0, 16); + expect(hashProjectDir(dir)).not.toBe(unversioned); + }); +}); diff --git a/packages/producer/src/services/distributed/projectHash.ts b/packages/producer/src/services/distributed/projectHash.ts index bc64c03157..baf5246485 100644 --- a/packages/producer/src/services/distributed/projectHash.ts +++ b/packages/producer/src/services/distributed/projectHash.ts @@ -9,44 +9,73 @@ * exactly why it lives here rather than being copy-pasted per adapter. */ -import { readdirSync, readFileSync } from "node:fs"; +import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs"; import { createHash } from "node:crypto"; -import { join, relative } from "node:path"; +import { join } from "node:path"; import { PLAN_PROJECT_DIR_SKIP_SEGMENTS } from "./plan.js"; +// Bump when the hashed set changes so uploads cached under an older scheme are never reused. +const SITE_ID_SCHEME = "site-v2\0"; + /** - * SHA-256 over every regular file under `projectDir` (sorted by relative - * path) → 16-character hex prefix. The prefix is the `siteId`. + * The files a site upload ships, as `/`-separated paths under `projectDir`. Symlinks are followed + * (root included), top-level {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} are left out as the plan stage + * does, and a dangling symlink throws. + */ +export function listSiteFiles(projectDir: string): string[] { + const files: string[] = []; + const ancestors = new Set([realpathSync(projectDir)]); + function walk(dir: string, rel: string): void { + for (const name of readdirSync(dir).sort()) { + if (rel === "" && PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(name)) continue; + const full = join(dir, name); + const relPath = rel === "" ? name : `${rel}/${name}`; + let stat; + try { + stat = statSync(full); + } catch (err) { + throw new Error(`[listSiteFiles] cannot read ${relPath} (dangling symlink?)`, { + cause: err, + }); + } + if (stat.isFile()) { + files.push(relPath); + } else if (stat.isDirectory()) { + const real = realpathSync(full); + // A folder linking back to an ancestor is skipped rather than walked forever. + if (ancestors.has(real)) continue; + ancestors.add(real); + walk(full, relPath); + ancestors.delete(real); + } + } + } + walk(projectDir, ""); + return files; +} + +/** + * SHA-256 over a scheme tag plus each of {@link listSiteFiles}'s paths and + * bytes → 16-character hex prefix. The prefix is the `siteId`. * * The hash includes the relative path plus every byte of each file, so a * same-bytes rename still yields a fresh id. We trim to 16 chars because the - * full 64 isn't useful in an object key for legibility. Top-level segments in - * {@link PLAN_PROJECT_DIR_SKIP_SEGMENTS} (e.g. `node_modules`) are skipped to - * match what the plan stage copies. + * full 64 isn't useful in an object key for legibility. Pass `files` when the + * caller already listed them, so the hashed set is the shipped set. * * Reads are synchronous: project trees are typically tens of MB at most * (HTML/CSS/JS plus a few composition assets), so the simpler shape wins over * a streaming pipeline. */ -export function hashProjectDir(projectDir: string): string { - const hash = createHash("sha256"); - const files: string[] = []; - function walk(dir: string, isRoot: boolean): void { - for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => - a.name < b.name ? -1 : a.name > b.name ? 1 : 0, - )) { - if (isRoot && PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(entry.name)) continue; - const full = join(dir, entry.name); - if (entry.isDirectory()) walk(full, false); - else if (entry.isFile()) files.push(full); - } - } - walk(projectDir, true); - for (const file of files) { - const rel = relative(projectDir, file).replaceAll("\\", "/"); +export function hashProjectDir( + projectDir: string, + files: readonly string[] = listSiteFiles(projectDir), +): string { + const hash = createHash("sha256").update(SITE_ID_SCHEME); + for (const rel of files) { hash.update(rel); hash.update("\0"); - hash.update(readFileSync(file)); + hash.update(readFileSync(join(projectDir, rel))); } return hash.digest("hex").slice(0, 16); }