mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-20 03:47:38 +00:00
fix(producer): cloud site uploads pack the files their id hashes, symlinks included
This commit is contained in:
@@ -267,12 +267,20 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
* binary — the AWS Lambda Node 22 base image ships a minimal set of
|
||||
* userland tools and does NOT include `tar` in `/usr/bin`.
|
||||
*/
|
||||
export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {
|
||||
export async function tarDirectory(
|
||||
sourceDir: string,
|
||||
destTarball: string,
|
||||
files?: string[],
|
||||
): Promise<void> {
|
||||
if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
||||
throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
|
||||
}
|
||||
mkdirSync(dirname(destTarball), { recursive: true });
|
||||
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
|
||||
// `files` (relative to `sourceDir`) packs exactly those paths, with symlinks followed.
|
||||
await tar.create(
|
||||
{ gzip: true, file: destTarball, cwd: sourceDir, follow: files !== undefined },
|
||||
files ?? ["."],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
* `deploySite`'s `finally` block has rmSync'd it.
|
||||
*/
|
||||
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
|
||||
export interface FakeS3Op {
|
||||
@@ -19,6 +20,7 @@ export interface FakeS3Op {
|
||||
export class FakeS3 {
|
||||
ops: FakeS3Op[] = [];
|
||||
existing = new Set<string>();
|
||||
bodies = new Map<string, Buffer>();
|
||||
|
||||
async send(command: unknown): Promise<unknown> {
|
||||
const cmdName = (command as { constructor: { name: string } }).constructor.name;
|
||||
@@ -39,7 +41,12 @@ export class FakeS3 {
|
||||
if (cmdName === "PutObjectCommand") {
|
||||
this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });
|
||||
this.existing.add(`${input.Bucket}/${input.Key}`);
|
||||
await drainBody((command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body);
|
||||
this.bodies.set(
|
||||
`${input.Bucket}/${input.Key}`,
|
||||
await drainBody(
|
||||
(command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body,
|
||||
),
|
||||
);
|
||||
return {};
|
||||
}
|
||||
throw new Error(`FakeS3: unexpected command ${cmdName}`);
|
||||
@@ -58,12 +65,6 @@ export function asS3Client(fake: FakeS3): S3Client {
|
||||
* the workdir — surfacing as an "Unhandled error between tests"
|
||||
* teardown noise that bun's runner attributes to the next test.
|
||||
*/
|
||||
async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<void> {
|
||||
if (Buffer.isBuffer(body)) return;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
body.on("data", () => {});
|
||||
body.on("end", () => resolve());
|
||||
body.on("close", () => resolve());
|
||||
body.on("error", reject);
|
||||
});
|
||||
async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<Buffer> {
|
||||
return Buffer.isBuffer(body) ? body : buffer(body);
|
||||
}
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import * as tar from "tar";
|
||||
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
|
||||
import { deploySite } from "./deploySite.js";
|
||||
|
||||
let projectDir: string;
|
||||
let scratch: string;
|
||||
|
||||
beforeEach(() => {
|
||||
projectDir = mkdtempSync(join(tmpdir(), "hf-deploy-site-test-"));
|
||||
scratch = mkdtempSync(join(tmpdir(), "hf-deploy-site-scratch-"));
|
||||
mkdirSync(join(projectDir, "assets"));
|
||||
writeFileSync(join(projectDir, "index.html"), "<html><body>hi</body></html>");
|
||||
writeFileSync(join(projectDir, "assets", "style.css"), "body { color: red; }");
|
||||
@@ -17,8 +20,17 @@ beforeEach(() => {
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(projectDir, { recursive: true, force: true });
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function uploadedEntries(s3: FakeS3, siteId: string): Promise<string[]> {
|
||||
const file = join(scratch, "uploaded.tar.gz");
|
||||
writeFileSync(file, s3.bodies.get(`test-bucket/sites/${siteId}/project.tar.gz`) ?? "");
|
||||
const entries: string[] = [];
|
||||
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
|
||||
return entries;
|
||||
}
|
||||
|
||||
describe("deploySite", () => {
|
||||
it("uploads the tarball when no matching object exists", async () => {
|
||||
const s3 = new FakeS3();
|
||||
@@ -145,4 +157,25 @@ describe("deploySite", () => {
|
||||
// node_modules contents shouldn't move the hash.
|
||||
expect(after.siteId).toBe(before.siteId);
|
||||
});
|
||||
|
||||
it("packs a symlinked project and its symlinked files and folders as content", async () => {
|
||||
mkdirSync(join(scratch, "media"));
|
||||
writeFileSync(join(scratch, "media", "clip.txt"), "clip");
|
||||
writeFileSync(join(scratch, "logo.txt"), "logo");
|
||||
symlinkSync(join(scratch, "media"), join(projectDir, "media"));
|
||||
symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt"));
|
||||
symlinkSync(projectDir, join(scratch, "fx"));
|
||||
const s3 = new FakeS3();
|
||||
const result = await deploySite({
|
||||
projectDir: join(scratch, "fx"),
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(await uploadedEntries(s3, result.siteId)).toEqual([
|
||||
"File assets/style.css",
|
||||
"File index.html",
|
||||
"File logo.txt",
|
||||
"File media/clip.txt",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,7 +16,7 @@ import { mkdtempSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import { hashProjectDir } from "@hyperframes/producer/distributed";
|
||||
import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";
|
||||
import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js";
|
||||
|
||||
/** Options for {@link deploySite}. */
|
||||
@@ -65,7 +65,8 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
|
||||
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
||||
}
|
||||
|
||||
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
|
||||
const files = listSiteFiles(opts.projectDir);
|
||||
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
|
||||
const key = `sites/${siteId}/project.tar.gz`;
|
||||
const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key });
|
||||
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
|
||||
@@ -88,12 +89,7 @@ export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
|
||||
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
|
||||
try {
|
||||
const tarball = join(workdir, "project.tar.gz");
|
||||
await tarDirectory(opts.projectDir, tarball);
|
||||
// Note: tarDirectory packs *everything* under `cwd`. We don't need to
|
||||
// re-implement the skip list inside the tar pack because the
|
||||
// producer's plan stage applies the same skip during its copy; the
|
||||
// archive is slightly bigger than the planDir's compiled/ subtree
|
||||
// but the cost is bounded by the project's user-authored content.
|
||||
await tarDirectory(opts.projectDir, tarball, files);
|
||||
const size = statSync(tarball).size;
|
||||
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user