fix(producer): cloud site uploads pack the files their id hashes, symlinks included

This commit is contained in:
Miguel Angel Simon Sierra
2026-10-17 00:36:01 -07:00
parent cc9e1bb33a
commit 57df7e88b5
11 changed files with 224 additions and 52 deletions
+10 -2
View File
@@ -267,12 +267,20 @@ function isRecord(value: unknown): value is Record<string, unknown> {
* binary — the AWS Lambda Node 22 base image ships a minimal set of
* userland tools and does NOT include `tar` in `/usr/bin`.
*/
export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {
export async function tarDirectory(
sourceDir: string,
destTarball: string,
files?: string[],
): Promise<void> {
if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
}
mkdirSync(dirname(destTarball), { recursive: true });
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
// `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed.
await tar.create(
{ gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined },
files ?? ["."],
);
}
/**
@@ -8,6 +8,7 @@
* `deploySite`'s `finally` block has rmSync'd it.
*/
import { buffer } from "node:stream/consumers";
import type { S3Client } from "@aws-sdk/client-s3";
export interface FakeS3Op {
@@ -19,6 +20,7 @@ export interface FakeS3Op {
export class FakeS3 {
ops: FakeS3Op[] = [];
existing = new Set<string>();
bodies = new Map<string, Buffer>();
async send(command: unknown): Promise<unknown> {
const cmdName = (command as { constructor: { name: string } }).constructor.name;
@@ -39,7 +41,12 @@ export class FakeS3 {
if (cmdName === "PutObjectCommand") {
this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });
this.existing.add(`${input.Bucket}/${input.Key}`);
await drainBody((command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body);
this.bodies.set(
`${input.Bucket}/${input.Key}`,
await drainBody(
(command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body,
),
);
return {};
}
throw new Error(`FakeS3: unexpected command ${cmdName}`);
@@ -58,12 +65,6 @@ export function asS3Client(fake: FakeS3): S3Client {
* the workdir — surfacing as an "Unhandled error between tests"
* teardown noise that bun's runner attributes to the next test.
*/
async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<void> {
if (Buffer.isBuffer(body)) return;
await new Promise<void>((resolve, reject) => {
body.on("data", () => {});
body.on("end", () => resolve());
body.on("close", () => resolve());
body.on("error", reject);
});
async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<Buffer> {
return Buffer.isBuffer(body) ? body : buffer(body);
}
+34 -1
View File
@@ -1,15 +1,18 @@
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { S3Client } from "@aws-sdk/client-s3";
import * as tar from "tar";
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
import { deploySite } from "./deploySite.js";
let projectDir: string;
let scratch: string;
beforeEach(() => {
projectDir = mkdtempSync(join(tmpdir(), "hf-deploy-site-test-"));
scratch = mkdtempSync(join(tmpdir(), "hf-deploy-site-scratch-"));
mkdirSync(join(projectDir, "assets"));
writeFileSync(join(projectDir, "index.html"), "<html><body>hi</body></html>");
writeFileSync(join(projectDir, "assets", "style.css"), "body { color: red; }");
@@ -17,8 +20,17 @@ beforeEach(() => {
afterEach(() => {
rmSync(projectDir, { recursive: true, force: true });
rmSync(scratch, { recursive: true, force: true });
});
async function uploadedEntries(s3: FakeS3, siteId: string): Promise<string[]> {
const file = join(scratch, "uploaded.tar.gz");
writeFileSync(file, s3.bodies.get(`test-bucket/sites/${siteId}/project.tar.gz`) ?? "");
const entries: string[] = [];
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
return entries;
}
describe("deploySite", () => {
it("uploads the tarball when no matching object exists", async () => {
const s3 = new FakeS3();
@@ -145,4 +157,25 @@ describe("deploySite", () => {
// node_modules contents shouldn't move the hash.
expect(after.siteId).toBe(before.siteId);
});
it("packs a symlinked project and its symlinked files and folders as content", async () => {
mkdirSync(join(scratch, "media"));
writeFileSync(join(scratch, "media", "clip.txt"), "clip");
writeFileSync(join(scratch, "logo.txt"), "logo");
symlinkSync(join(scratch, "media"), join(projectDir, "media"));
symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt"));
symlinkSync(projectDir, join(scratch, "fx"));
const s3 = new FakeS3();
const result = await deploySite({
projectDir: join(scratch, "fx"),
bucketName: "test-bucket",
s3: asS3Client(s3),
});
expect(await uploadedEntries(s3, result.siteId)).toEqual([
"File assets/style.css",
"File index.html",
"File logo.txt",
"File media/clip.txt",
]);
});
});
+4 -8
View File
@@ -16,7 +16,7 @@ import { mkdtempSync, rmSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { hashProjectDir } from "@hyperframes/producer/distributed";
import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";
import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js";
/** Options for {@link deploySite}. */
@@ -65,7 +65,8 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
}
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
const files = listSiteFiles(opts.projectDir);
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
const key = `sites/${siteId}/project.tar.gz`;
const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key });
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
@@ -88,12 +89,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
try {
const tarball = join(workdir, "project.tar.gz");
await tarDirectory(opts.projectDir, tarball);
// Note: tarDirectory packs *everything* under `cwd`. We don't need to
// re-implement the skip list inside the tar pack because the
// producer's plan stage applies the same skip during its copy; the
// archive is slightly bigger than the planDir's compiled/ subtree
// but the cost is bounded by the project's user-authored content.
await tarDirectory(opts.projectDir, tarball, files);
const size = statSync(tarball).size;
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
return {