perf(aws-lambda): upload project files once to a shared content-addressed area

This commit is contained in:
Miguel Angel Simon Sierra
2026-10-17 00:53:33 -07:00
parent 57df7e88b5
commit abb8f6bd08
21 changed files with 607 additions and 223 deletions
@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { App, Stack } from "aws-cdk-lib";
import { Template } from "aws-cdk-lib/assertions";
import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js";
import { HyperframesRenderStack } from "./HyperframesRenderStack.js";
// CDK synth is slow on cold start and reached 32s under full-workspace CI
@@ -61,7 +62,7 @@ describe("HyperframesRenderStack — contract", () => {
});
});
it("provisions exactly one S3 bucket with PublicAccessBlockConfiguration and a 7-day intermediates lifecycle", () => {
it("provisions exactly one S3 bucket with PublicAccessBlockConfiguration and a 7-day intermediates and 30-day shared content lifecycle", () => {
const t = DEFAULT_TEMPLATE;
t.resourceCountIs("AWS::S3::Bucket", 1);
t.hasResourceProperties("AWS::S3::Bucket", {
@@ -79,6 +80,12 @@ describe("HyperframesRenderStack — contract", () => {
Prefix: "renders/",
ExpirationInDays: 7,
},
{
Id: "ExpireSharedContent",
Status: "Enabled",
Prefix: `${CONTENT_STORE_PREFIX}/`,
ExpirationInDays: CONTENT_STORE_RETENTION_DAYS,
},
],
},
});
@@ -26,6 +26,11 @@ import { join } from "node:path";
import { App, Stack } from "aws-cdk-lib";
import { Template } from "aws-cdk-lib/assertions";
import { parse as parseYaml } from "yaml";
import {
CONTENT_STORE_PREFIX,
CONTENT_STORE_REFRESH_AFTER_DAYS,
CONTENT_STORE_RETENTION_DAYS,
} from "../contentStore.js";
import { HyperframesRenderStack } from "./HyperframesRenderStack.js";
// CDK synth + Template.fromStack is slow on cold start in CI (~5-8s on
@@ -264,6 +269,26 @@ describe("HyperframesRenderStack — snapshot", () => {
expect(v2).not.toContain("PlanS3Uri");
});
it("keeps the SAM bucket lifecycle identical to CDK's, shared content on the transport's terms", () => {
const cdkBucket = Object.values(SYNTHED.template.findResources("AWS::S3::Bucket"))[0] as {
Properties: { LifecycleConfiguration: unknown };
};
const samBucket = requireRecordProperty(readSamResources(), "RenderBucket", "SAM RenderBucket");
const samProperties = requireRecordProperty(samBucket, "Properties", "SAM bucket properties");
expect(samProperties.LifecycleConfiguration).toEqual(
cdkBucket.Properties.LifecycleConfiguration,
);
expect(CONTENT_STORE_REFRESH_AFTER_DAYS * 2).toBeLessThan(CONTENT_STORE_RETENTION_DAYS);
expect(cdkBucket.Properties.LifecycleConfiguration).toMatchObject({
Rules: expect.arrayContaining([
expect.objectContaining({
Prefix: `${CONTENT_STORE_PREFIX}/`,
ExpirationInDays: CONTENT_STORE_RETENTION_DAYS,
}),
]),
});
});
it("sends the same payload from every Lambda task as the SAM template", () => {
// CDK drops null-valued payload fields when it renders the definition (#5259), so compare
// what each deployed task sends rather than the construct source.
@@ -433,7 +458,7 @@ function getV1TaskStates(definition: {
};
}
function readSamDefinition(): { States: Record<string, unknown> } {
function readSamResources(): Record<string, unknown> {
const source = readFileSync(
new URL("../../../../examples/aws-lambda/template.yaml", import.meta.url),
"utf8",
@@ -443,7 +468,11 @@ function readSamDefinition(): { States: Record<string, unknown> } {
// warnings for the intentionally unresolved `!Ref`/`!GetAtt` tags.
const parsed: unknown = parseYaml(source, { logLevel: "silent" });
const root = requireRecord(parsed, "SAM template");
const resources = requireRecordProperty(root, "Resources", "SAM resources");
return requireRecordProperty(root, "Resources", "SAM resources");
}
function readSamDefinition(): { States: Record<string, unknown> } {
const resources = readSamResources();
const stateMachine = requireRecordProperty(
resources,
"RenderStateMachine",
@@ -29,6 +29,7 @@ import * as s3 from "aws-cdk-lib/aws-s3";
import * as sfn from "aws-cdk-lib/aws-stepfunctions";
import * as tasks from "aws-cdk-lib/aws-stepfunctions-tasks";
import { Construct } from "constructs";
import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js";
/** Construction-time props for {@link HyperframesRenderStack}. */
export interface HyperframesRenderStackProps {
@@ -92,6 +93,12 @@ export class HyperframesRenderStack extends Construct {
prefix: "renders/",
expiration: Duration.days(7),
},
{
id: "ExpireSharedContent",
enabled: true,
prefix: `${CONTENT_STORE_PREFIX}/`,
expiration: Duration.days(CONTENT_STORE_RETENTION_DAYS),
},
],
});
+15
View File
@@ -0,0 +1,15 @@
// Shared by every render: project files and plan-v2 artifacts by SHA-256. Dependency-free for CDK.
export const CONTENT_STORE_PREFIX = "cas/sha256";
export const CONTENT_STORE_RETENTION_DAYS = 30;
/** Reusing an object at least this old rewrites it, so a plan's objects outlive it by 23+ days. */
export const CONTENT_STORE_REFRESH_AFTER_DAYS = 7;
export function contentStoreUri(bucket: string, sha256: string): string {
if (!/^[a-f0-9]{64}$/.test(sha256)) {
throw new Error(
`[contentStore] expected lowercase SHA-256 digest, got ${JSON.stringify(sha256)}`,
);
}
return `s3://${bucket}/${CONTENT_STORE_PREFIX}/${sha256.slice(0, 2)}/${sha256}`;
}
+106 -1
View File
@@ -631,7 +631,7 @@ describe("handler dispatch", () => {
expect(planned).toMatchObject({
PlanProtocol: "v2",
PlanV2ManifestS3Uri: "s3://bucket/renders/v2/v2/manifest.json",
PlanV2ArtifactS3Prefix: "s3://bucket/renders/v2/v2/artifacts/sha256",
PlanV2ArtifactS3Prefix: "s3://bucket/cas/sha256",
});
expect("PlanS3Uri" in planned).toBe(false);
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2") {
@@ -744,6 +744,111 @@ describe("handler dispatch", () => {
});
});
describe("handler — plan from a deployed site", () => {
function storeSiteFile(s3: FakeS3Client, bytes: string): string {
const sha256 = createHash("sha256").update(bytes).digest("hex");
const uri = `s3://bucket/cas/sha256/${sha256.slice(0, 2)}/${sha256}`;
s3.objects.set(uri, Buffer.from(bytes));
s3.metadata.set(uri, { sha256 });
return sha256;
}
function planEvent(): PlanEvent {
return {
Action: "plan",
ProjectS3Uri: "s3://bucket/sites/site1/files.json",
PlanOutputS3Prefix: "s3://bucket/renders/r1/",
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
};
}
it("rebuilds the project from the content store and reuses its media without uploading it", async () => {
const s3 = new FakeS3Client();
const html = storeSiteFile(s3, "<html></html>");
const media = storeSiteFile(s3, "VIDEO-BYTES");
s3.objects.set(
"s3://bucket/sites/site1/files.json",
Buffer.from(
JSON.stringify({
files: [
{ path: "assets/bg.mp4", sha256: media, sizeBytes: 11 },
{ path: "index.html", sha256: html, sizeBytes: 13 },
],
}),
),
);
const planV2WithPublisher = mock(
async (projectDir: string, _config: unknown, publisher: PlanV2ArtifactPublisher) => {
expect(readFileSync(join(projectDir, "index.html"), "utf-8")).toBe("<html></html>");
const mediaPath = join(projectDir, "assets", "bg.mp4");
await publisher.putBlob({ sourcePath: mediaPath, sha256: media, sizeBytes: 11 });
const manifest = {
planHash: "c".repeat(64),
chunkCount: 1,
totalFrames: 30,
fps: 30,
width: 640,
height: 360,
format: "mp4",
ffmpegVersion: "test",
producerVersion: "test",
artifacts: [{ path: "compiled/assets/bg.mp4", sha256: media, sizeBytes: 11 }],
};
await publisher.commitManifest(JSON.stringify(manifest));
return manifest as unknown as PlanV2Manifest;
},
);
const result = await handler(planEvent(), {
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
primitives: {
plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,
planV2WithPublisher:
planV2WithPublisher as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,
renderChunk:
mock() as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
assemble: mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble,
},
tmpRoot: makeTmpRoot(),
skipChromeResolution: true,
});
expect(result).toMatchObject({ PlanV2ArtifactS3Prefix: "s3://bucket/cas/sha256" });
expect(planV2WithPublisher).toHaveBeenCalledTimes(1);
expect(s3.ops.filter((op) => op.kind === "upload").map((op) => op.uri)).toEqual([
"s3://bucket/renders/r1/v2/manifest.json",
]);
expect(s3.ops.some((op) => op.uri.endsWith(".tar.gz"))).toBe(false);
});
it("refuses a site manifest path that escapes the project", async () => {
const s3 = new FakeS3Client();
const sha256 = storeSiteFile(s3, "x");
s3.objects.set(
"s3://bucket/sites/site1/files.json",
Buffer.from(JSON.stringify({ files: [{ path: "../escape", sha256, sizeBytes: 1 }] })),
);
const planV2WithPublisher = mock();
await expect(
handler(planEvent(), {
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
primitives: {
plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,
planV2WithPublisher:
planV2WithPublisher as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,
renderChunk:
mock() as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
assemble:
mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble,
},
tmpRoot: makeTmpRoot(),
skipChromeResolution: true,
}),
).rejects.toMatchObject({ name: "SITE_MANIFEST_INVALID" });
expect(planV2WithPublisher).not.toHaveBeenCalled();
});
});
describe("handler — S3 URI allowlist (security: F-004)", () => {
let prevBucket: string | undefined;
+20 -36
View File
@@ -54,12 +54,14 @@ import type {
import {
downloadS3ObjectToFile,
downloadS3ObjectToFileVerified,
mapConcurrent,
parseS3Uri,
tarDirectory,
untarDirectory,
uploadFileToS3,
} from "./s3Transport.js";
import { S3PlanV2ArtifactPublisher } from "./s3PlanV2Publisher.js";
import { downloadSiteFiles, isSiteManifestUri } from "./siteManifest.js";
/**
* Lazily-constructed S3 client. Cached at module scope so warm Lambda
@@ -337,17 +339,10 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam
}
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-"));
// We use `.tar.gz` (not `.zip`) as the project archive's on-the-wire
// format because Lambda's Amazon Linux base image ships GNU `tar` but
// not `unzip` in `/usr/bin`. The smoke script + future CLI both
// produce tar.gz uploads.
const projectArchive = join(work, "project.tar.gz");
const projectDir = join(work, "project");
const planDir = join(work, "plan");
try {
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
await untarDirectory(projectArchive, projectDir);
const projectDir = await fetchProject(s3, event.ProjectS3Uri, work);
const config: DistributedRenderConfig = {
...event.Config,
@@ -395,6 +390,22 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam
}
}
/**
* `ProjectS3Uri` is a site manifest from `deploySite`, or a `.tar.gz` of the project
* (hand-built uploads such as `examples/aws-lambda/scripts/smoke.sh`).
*/
async function fetchProject(s3: S3Client, projectS3Uri: string, work: string): Promise<string> {
const projectDir = join(work, "project");
if (isSiteManifestUri(projectS3Uri)) {
await downloadSiteFiles(s3, projectS3Uri, projectDir);
} else {
const projectArchive = join(work, "project.tar.gz");
await downloadS3ObjectToFile(s3, projectS3Uri, projectArchive);
await untarDirectory(projectArchive, projectDir);
}
return projectDir;
}
// Plan v2 orchestration is kept as one transactional boundary: stage, CAS upload,
// and manifest-last publication must stay ordered and fail together.
// fallow-ignore-next-line complexity
@@ -410,11 +421,8 @@ async function handlePlanV2(
}
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-v2-"));
const projectArchive = join(work, "project.tar.gz");
const projectDir = join(work, "project");
try {
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
await untarDirectory(projectArchive, projectDir);
const projectDir = await fetchProject(s3, event.ProjectS3Uri, work);
const publisher = new S3PlanV2ArtifactPublisher({
s3,
planOutputS3Prefix: event.PlanOutputS3Prefix,
@@ -749,30 +757,6 @@ function throwPlanHashMismatch(expected: string, actual: string): never {
throw error;
}
async function mapConcurrent<T>(
values: readonly T[],
concurrency: number,
fn: (value: T) => Promise<void>,
): Promise<void> {
let cursor = 0;
async function worker(): Promise<void> {
while (cursor < values.length) {
const index = cursor++;
await fn(values[index]!);
}
}
const results = await Promise.allSettled(
Array.from({ length: Math.min(concurrency, values.length) }, () => worker()),
);
const failure = results.find(
(result): result is PromiseRejectedResult => result.status === "rejected",
);
// Do not reject while sibling workers may still be writing into invocation
// scratch. The caller removes that directory in `finally`; draining the pool
// first prevents late S3 streams from racing cleanup after another GET fails.
if (failure) throw failure.reason;
}
async function downloadChunkObjects(
s3: S3Client,
uris: string[],
@@ -110,14 +110,37 @@ describe("S3PlanV2ArtifactPublisher", () => {
planOutputS3Prefix: `s3://bucket/render${"/".repeat(10_000)}`,
});
expect(publisher.artifactPrefix).toBe("s3://bucket/render/v2/artifacts/sha256");
expect(publisher.artifactPrefix).toBe("s3://bucket/cas/sha256");
expect(publisher.manifestUri).toBe("s3://bucket/render/v2/manifest.json");
});
it("reuses a blob the SDK or another render already stored instead of uploading it", async () => {
const source = makeSource("shared media");
const s3 = new FakeS3();
const blobUri = `s3://bucket/cas/sha256/${source.digest.slice(0, 2)}/${source.digest}`;
s3.objects.set(blobUri, { bytes: Buffer.from("shared media"), sha256: source.digest });
const publisher = new S3PlanV2ArtifactPublisher({
s3: s3.asClient(),
planOutputS3Prefix: "s3://bucket/renders/second",
temporaryRoot: source.root,
});
await publisher.putBlob({
sourcePath: source.path,
sha256: source.digest,
sizeBytes: source.sizeBytes,
});
await publisher.commitManifest(manifestFor(source.digest));
expect(s3.puts.map((operation) => operation.uri)).toEqual([
"s3://bucket/renders/second/v2/manifest.json",
]);
});
it("publishes immutable blobs before the fixed-key manifest", async () => {
const source = makeSource("hello");
const s3 = new FakeS3();
const artifactPrefix = "s3://bucket/render/v2/artifacts/sha256";
const artifactPrefix = "s3://bucket/cas/sha256";
const manifestUri = "s3://bucket/render/v2/manifest.json";
const publisher = new S3PlanV2ArtifactPublisher({
s3: s3.asClient(),
+15 -8
View File
@@ -8,7 +8,13 @@ import {
type PlanV2ArtifactPublisher,
type PlanV2PublishBlob,
} from "@hyperframes/producer/distributed";
import { parseS3Uri, uploadContentAddressedFileToS3 } from "./s3Transport.js";
import { CONTENT_STORE_PREFIX } from "./contentStore.js";
import {
formatS3Uri,
parseS3Uri,
putContentStoreFile,
uploadContentAddressedFileToS3,
} from "./s3Transport.js";
export interface S3PlanV2ArtifactPublisherOptions {
readonly s3: S3Client;
@@ -66,15 +72,18 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
readonly artifactPrefix: string;
readonly manifestUri: string;
readonly #s3: S3Client;
readonly #bucket: string;
readonly #temporaryRoot: string;
readonly #publishedDigests = new Set<string>();
#state: "open" | "committed" | "aborted" = "open";
constructor(options: Readonly<S3PlanV2ArtifactPublisherOptions>) {
const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;
parseS3Uri(outputPrefix);
const { bucket } = parseS3Uri(outputPrefix);
this.#s3 = options.s3;
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
this.#bucket = bucket;
// Shared by every render, so a blob the SDK or an earlier render uploaded is reused, not re-sent.
this.artifactPrefix = formatS3Uri({ bucket, key: CONTENT_STORE_PREFIX });
this.manifestUri = `${outputPrefix}/manifest.json`;
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
mkdirSync(this.#temporaryRoot, { recursive: true });
@@ -89,8 +98,7 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`,
);
}
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);
await putContentStoreFile(this.#s3, blob.sourcePath, this.#bucket, digest);
this.#publishedDigests.add(digest);
}
@@ -124,9 +132,8 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
async abort(): Promise<void> {
if (this.#state === "open") this.#state = "aborted";
// Remote CAS blobs are immutable and may already be reused by a retry.
// Without a committed manifest they are unreachable and expire under the
// render bucket's intermediate-object lifecycle policy.
// Remote CAS blobs are immutable and may already be reused by a retry or
// another render. They expire under the shared content store's lifecycle.
}
#assertOpen(operation: string): void {
+51 -1
View File
@@ -9,6 +9,7 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js";
import {
downloadS3ObjectToFileVerified,
formatS3Uri,
@@ -16,6 +17,7 @@ import {
sha256File,
tarDirectory,
untarDirectory,
putContentStoreFile,
uploadContentAddressedFileToS3,
} from "./s3Transport.js";
@@ -162,6 +164,37 @@ describe("content-addressed v2 artifacts", () => {
expect(s3.putCount).toBe(0);
});
it("rewrites a reused object at the refresh age so its lifecycle expiry restarts", async () => {
const source = join(scratchRoot, "artifact-aging.bin");
writeFileSync(source, "aging bytes");
const digest = await sha256File(source);
const uri = contentStoreUri("bucket", digest);
const s3 = new ContentAddressedFakeS3();
const days = (n: number) => new Date(Date.now() - n * 86_400_000);
s3.objects.set(uri, {
bytes: Buffer.from("aging bytes"),
sha256: digest,
lastModified: days(CONTENT_STORE_REFRESH_AFTER_DAYS - 1),
});
expect(await putContentStoreFile(s3.asClient(), source, "bucket", digest)).toBe("reused");
expect(s3.copies).toEqual([]);
s3.objects.get(uri)!.lastModified = days(CONTENT_STORE_REFRESH_AFTER_DAYS);
expect(await putContentStoreFile(s3.asClient(), source, "bucket", digest)).toBe("reused");
expect(s3.copies).toEqual([
{
uri,
CopySource: `bucket/cas/sha256/${digest.slice(0, 2)}/${digest}`,
MetadataDirective: "REPLACE",
Metadata: { sha256: digest },
ChecksumAlgorithm: "SHA256",
},
]);
expect(s3.objects.get(uri)).toMatchObject({ sha256: digest });
expect(s3.putCount).toBe(0);
});
it("deletes a downloaded artifact when digest verification fails", async () => {
const expectedSource = join(scratchRoot, "artifact-expected.bin");
const destination = join(scratchRoot, "artifact-download.bin");
@@ -180,7 +213,8 @@ describe("content-addressed v2 artifacts", () => {
});
class ContentAddressedFakeS3 {
readonly objects = new Map<string, { bytes: Buffer; sha256: string }>();
readonly objects = new Map<string, { bytes: Buffer; sha256: string; lastModified?: Date }>();
readonly copies: Record<string, unknown>[] = [];
putCount = 0;
raceOnNextPut: { bytes: Buffer; sha256: string } | undefined;
@@ -199,9 +233,24 @@ class ContentAddressedFakeS3 {
Key: string;
Body?: NodeJS.ReadableStream;
Metadata?: Record<string, string>;
CopySource?: string;
MetadataDirective?: string;
ChecksumAlgorithm?: string;
};
};
const uri = `s3://${value.input.Bucket}/${value.input.Key}`;
if (value.constructor.name === "CopyObjectCommand") {
const { CopySource, MetadataDirective, Metadata, ChecksumAlgorithm } = value.input;
this.copies.push({ uri, CopySource, MetadataDirective, Metadata, ChecksumAlgorithm });
const source = this.objects.get(`s3://${CopySource}`);
if (!source) throw new Error("missing copy source");
this.objects.set(uri, {
bytes: source.bytes,
sha256: Metadata?.sha256 ?? "",
lastModified: new Date(),
});
return {};
}
if (value.constructor.name === "HeadObjectCommand") {
const object = this.objects.get(uri);
if (!object) {
@@ -215,6 +264,7 @@ class ContentAddressedFakeS3 {
return {
ContentLength: object.bytes.length,
Metadata: { sha256: object.sha256 },
LastModified: object.lastModified,
};
}
if (value.constructor.name === "GetObjectCommand") {
+83 -9
View File
@@ -29,12 +29,14 @@ import { createHash } from "node:crypto";
import { dirname } from "node:path";
import { pipeline } from "node:stream/promises";
import {
CopyObjectCommand,
GetObjectCommand,
HeadObjectCommand,
PutObjectCommand,
type S3Client,
} from "@aws-sdk/client-s3";
import * as tar from "tar";
import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js";
/** Parsed `s3://bucket/key` URI. */
export interface S3Location {
@@ -144,6 +146,16 @@ export async function uploadContentAddressedFileToS3(
expectedSha256: string,
contentType?: string,
): Promise<"uploaded" | "reused"> {
return (await putImmutableFile(client, localPath, uri, expectedSha256, contentType)).outcome;
}
async function putImmutableFile(
client: S3Client,
localPath: string,
uri: string,
expectedSha256: string,
contentType?: string,
): Promise<{ outcome: "uploaded" | "reused"; lastModified?: Date }> {
assertSha256(expectedSha256);
if (!existsSync(localPath)) {
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
@@ -160,8 +172,9 @@ export async function uploadContentAddressedFileToS3(
const { bucket, key } = parseS3Uri(uri);
const size = statSync(localPath).size;
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
if (existing === "matching") return "reused";
if (existing === "conflict") throwImmutableObjectConflict(uri);
if (existing.state === "matching")
return { outcome: "reused", lastModified: existing.lastModified };
if (existing.state === "conflict") throwImmutableObjectConflict(uri);
const body = createReadStream(localPath);
try {
@@ -180,12 +193,12 @@ export async function uploadContentAddressedFileToS3(
IfNoneMatch: "*",
}),
);
return "uploaded";
return { outcome: "uploaded" };
} catch (error) {
if (!isS3PreconditionFailed(error)) throw error;
const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
if (raced === "matching") return "reused";
if (raced === "conflict") throwImmutableObjectConflict(uri);
if (raced.state === "matching") return { outcome: "reused" };
if (raced.state === "conflict") throwImmutableObjectConflict(uri);
// The winning object was deleted between the conditional failure and
// verification. Preserve the service error so the orchestrator may retry.
throw error;
@@ -197,6 +210,30 @@ export async function uploadContentAddressedFileToS3(
}
}
export async function mapConcurrent<T>(
values: readonly T[],
concurrency: number,
fn: (value: T) => Promise<void>,
): Promise<void> {
let cursor = 0;
async function worker(): Promise<void> {
while (cursor < values.length) {
const index = cursor++;
await fn(values[index]!);
}
}
const results = await Promise.allSettled(
Array.from({ length: Math.min(concurrency, values.length) }, () => worker()),
);
const failure = results.find(
(result): result is PromiseRejectedResult => result.status === "rejected",
);
// Do not reject while sibling workers may still be writing into invocation
// scratch. The caller removes that directory in `finally`; draining the pool
// first prevents late S3 streams from racing cleanup after another GET fails.
if (failure) throw failure.reason;
}
export async function sha256File(path: string): Promise<string> {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) {
@@ -213,7 +250,9 @@ function assertSha256(value: string): void {
}
}
type ContentAddressedObjectState = "missing" | "matching" | "conflict";
type ContentAddressedObjectState =
| { state: "missing" | "conflict" }
| { state: "matching"; lastModified?: Date };
async function inspectContentAddressedObject(
client: S3Client,
@@ -227,14 +266,49 @@ async function inspectContentAddressedObject(
new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" }),
);
return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256
? "matching"
: "conflict";
? { state: "matching", lastModified: existing.LastModified }
: { state: "conflict" };
} catch (error) {
if (isS3NotFound(error)) return "missing";
if (isS3NotFound(error)) return { state: "missing" };
throw error;
}
}
/** Put a file into the shared content store once; reusing an aging object restarts its expiry. */
export async function putContentStoreFile(
client: S3Client,
localPath: string,
bucket: string,
sha256: string,
): Promise<"uploaded" | "reused"> {
const uri = contentStoreUri(bucket, sha256);
const { outcome, lastModified } = await putImmutableFile(client, localPath, uri, sha256);
if (lastModified) await refreshAgingObject(client, uri, sha256, lastModified);
return outcome;
}
// S3 lifecycle expiry counts from the last write, not the last read, so reuse has to write.
async function refreshAgingObject(
client: S3Client,
uri: string,
sha256: string,
lastModified: Date,
): Promise<void> {
const { bucket, key } = parseS3Uri(uri);
const ageMs = Date.now() - lastModified.getTime();
if (ageMs < CONTENT_STORE_REFRESH_AFTER_DAYS * 86_400_000) return;
await client.send(
new CopyObjectCommand({
Bucket: bucket,
Key: key,
CopySource: `${bucket}/${key}`,
MetadataDirective: "REPLACE",
Metadata: { sha256 },
ChecksumAlgorithm: "SHA256",
}),
);
}
function throwImmutableObjectConflict(uri: string): never {
const error = new Error(
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`,
@@ -21,28 +21,24 @@ export class FakeS3 {
ops: FakeS3Op[] = [];
existing = new Set<string>();
bodies = new Map<string, Buffer>();
metadata = new Map<string, Record<string, string> | undefined>();
async send(command: unknown): Promise<unknown> {
const cmdName = (command as { constructor: { name: string } }).constructor.name;
const input = (command as { input: { Bucket: string; Key: string } }).input;
const input = (
command as { input: { Bucket: string; Key: string; Metadata?: Record<string, string> } }
).input;
const id = `${input.Bucket}/${input.Key}`;
if (cmdName === "HeadObjectCommand") {
this.ops.push({ kind: "head", bucket: input.Bucket, key: input.Key });
if (this.existing.has(`${input.Bucket}/${input.Key}`)) {
return { ContentLength: 1, LastModified: new Date("2026-05-16T00:00:00Z") };
}
const err = new Error("Not Found") as Error & {
$metadata: { httpStatusCode: number };
name: string;
};
err.name = "NotFound";
err.$metadata = { httpStatusCode: 404 };
throw err;
return this.head(id);
}
if (cmdName === "PutObjectCommand") {
this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });
this.existing.add(`${input.Bucket}/${input.Key}`);
this.existing.add(id);
this.metadata.set(id, input.Metadata);
this.bodies.set(
`${input.Bucket}/${input.Key}`,
id,
await drainBody(
(command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body,
),
@@ -51,6 +47,19 @@ export class FakeS3 {
}
throw new Error(`FakeS3: unexpected command ${cmdName}`);
}
private head(id: string): unknown {
if (this.existing.has(id)) {
return { ContentLength: this.bodies.get(id)?.length, Metadata: this.metadata.get(id) };
}
const err = new Error("Not Found") as Error & {
$metadata: { httpStatusCode: number };
name: string;
};
err.name = "NotFound";
err.$metadata = { httpStatusCode: 404 };
throw err;
}
}
/** Convenience cast so `deploySite({ s3: makeFakeS3() })` reads cleanly. */
+53 -24
View File
@@ -2,8 +2,9 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createHash } from "node:crypto";
import type { S3Client } from "@aws-sdk/client-s3";
import * as tar from "tar";
import { parseSiteManifest } from "../siteManifest.js";
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
import { deploySite } from "./deploySite.js";
@@ -23,16 +24,25 @@ afterEach(() => {
rmSync(scratch, { recursive: true, force: true });
});
async function uploadedEntries(s3: FakeS3, siteId: string): Promise<string[]> {
const file = join(scratch, "uploaded.tar.gz");
writeFileSync(file, s3.bodies.get(`test-bucket/sites/${siteId}/project.tar.gz`) ?? "");
const entries: string[] = [];
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
return entries;
function sha256(text: string): string {
return createHash("sha256").update(text).digest("hex");
}
function storeKey(text: string): string {
const digest = sha256(text);
return `cas/sha256/${digest.slice(0, 2)}/${digest}`;
}
function sitePaths(s3: FakeS3, siteId: string): string[] {
const manifest = s3.bodies.get(`test-bucket/sites/${siteId}/files.json`)?.toString() ?? "";
return parseSiteManifest(manifest).map((file) => file.path);
}
const HTML = "<html><body>hi</body></html>";
const CSS = "body { color: red; }";
describe("deploySite", () => {
it("uploads the tarball when no matching object exists", async () => {
it("uploads each file once under its content hash, then the site manifest", async () => {
const s3 = new FakeS3();
const result = await deploySite({
projectDir,
@@ -43,14 +53,33 @@ describe("deploySite", () => {
expect(result.uploaded).toBe(true);
expect(result.siteId).toMatch(/^[0-9a-f]{16}$/);
expect(result.bucketName).toBe("test-bucket");
expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/project.tar.gz`);
expect(result.bytes).toBeGreaterThan(0);
expect(s3.ops).toEqual([
{ kind: "head", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },
{ kind: "put", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },
expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/files.json`);
expect(result.bytes).toBe(HTML.length + CSS.length);
const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);
expect(puts.slice(0, 2).sort()).toEqual([storeKey(HTML), storeKey(CSS)].sort());
expect(puts[2]).toBe(`sites/${result.siteId}/files.json`);
expect(s3.bodies.get(`test-bucket/${storeKey(CSS)}`)?.toString()).toBe(CSS);
expect(parseSiteManifest(s3.bodies.get(`test-bucket/${puts[2]}`)?.toString() ?? "")).toEqual([
{ path: "assets/style.css", sha256: sha256(CSS), sizeBytes: CSS.length },
{ path: "index.html", sha256: sha256(HTML), sizeBytes: HTML.length },
]);
});
it("skips a file another site already stored", async () => {
const s3 = new FakeS3();
const cssId = `test-bucket/${storeKey(CSS)}`;
s3.existing.add(cssId);
s3.bodies.set(cssId, Buffer.from(CSS));
s3.metadata.set(cssId, { sha256: sha256(CSS) });
await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) });
const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);
expect(puts).toHaveLength(2);
expect(puts).not.toContain(storeKey(CSS));
expect(puts).toContain(storeKey(HTML));
});
it("yields a stable siteId across re-runs of the same tree", async () => {
const s3a = new FakeS3();
const a = await deploySite({
@@ -85,7 +114,7 @@ describe("deploySite", () => {
expect(after.siteId).not.toBe(before.siteId);
});
it("short-circuits on HEAD 200 (skips PUT)", async () => {
it("uploads nothing when the same tree is deployed again", async () => {
const s3 = new FakeS3();
const first = await deploySite({
projectDir,
@@ -101,9 +130,9 @@ describe("deploySite", () => {
expect(second.uploaded).toBe(false);
expect(second.siteId).toBe(first.siteId);
// Only one PUT total, plus two HEADs.
expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(1);
expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(2);
// Two files plus the manifest, each put once and HEAD-checked on both deploys.
expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(3);
expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(6);
});
it("honours a caller-supplied siteId", async () => {
@@ -115,7 +144,7 @@ describe("deploySite", () => {
s3: asS3Client(s3),
});
expect(result.siteId).toBe("release-v1.2.3");
expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/project.tar.gz");
expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/files.json");
});
it("propagates non-404 S3 errors", async () => {
@@ -158,7 +187,7 @@ describe("deploySite", () => {
expect(after.siteId).toBe(before.siteId);
});
it("packs a symlinked project and its symlinked files and folders as content", async () => {
it("ships a symlinked project and its symlinked files and folders as content", async () => {
mkdirSync(join(scratch, "media"));
writeFileSync(join(scratch, "media", "clip.txt"), "clip");
writeFileSync(join(scratch, "logo.txt"), "logo");
@@ -171,11 +200,11 @@ describe("deploySite", () => {
bucketName: "test-bucket",
s3: asS3Client(s3),
});
expect(await uploadedEntries(s3, result.siteId)).toEqual([
"File assets/style.css",
"File index.html",
"File logo.txt",
"File media/clip.txt",
expect(sitePaths(s3, result.siteId)).toEqual([
"assets/style.css",
"index.html",
"logo.txt",
"media/clip.txt",
]);
});
});
+57 -77
View File
@@ -1,23 +1,28 @@
/**
* `deploySite` — upload a project directory to S3 once per content hash
* and return a reusable handle.
* `deploySite` — upload a project directory to S3 and return a reusable handle.
*
* `renderToLambda` calls this implicitly when no `siteHandle` is passed,
* but exposing it as a standalone verb lets adopters bundle a project
* ahead of time and reuse the handle across many renders without
* re-tarring the project tree on every call.
* but exposing it as a standalone verb lets adopters upload a project
* ahead of time and reuse the handle across many renders.
*
* The handle is **content-addressed**: `siteId` is derived from a SHA-256
* over the project files. Two `deploySite` calls on an unchanged tree
* produce the same `siteId` and `HeadObject`-short-circuit the upload.
* Every file goes into the bucket's shared content store under its SHA-256,
* once: a file any earlier site or render already stored is skipped. The
* site itself is a small `sites/<siteId>/files.json` listing path → hash,
* where `siteId` is derived from a SHA-256 over the project files.
*/
import { mkdtempSync, rmSync, statSync } from "node:fs";
import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { S3Client } from "@aws-sdk/client-s3";
import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";
import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js";
import {
mapConcurrent,
putContentStoreFile,
sha256File,
uploadContentAddressedFileToS3,
} from "../s3Transport.js";
import { serializeSiteManifest, siteManifestUri, type SiteFile } from "../siteManifest.js";
/** Options for {@link deploySite}. */
export interface DeploySiteOptions {
@@ -43,90 +48,65 @@ export interface SiteHandle {
siteId: string;
/** Bucket the site landed in. Surfaced separately so callers don't have to re-parse `projectS3Uri`. */
bucketName: string;
/** Full `s3://bucket/sites/<siteId>/project.tar.gz` URI; pass through to `renderToLambda`. */
/** Full `s3://bucket/sites/<siteId>/files.json` URI; pass through to `renderToLambda`. */
projectS3Uri: string;
/** Tarball size in bytes; useful for "did we actually skip the upload?" assertions. */
/** Total size of the project's files in bytes. */
bytes: number;
/** ISO timestamp of the most recent upload OR the existing object the short-circuit found. */
/** ISO timestamp of this `deploySite` call. */
uploadedAt: string;
/** `false` if the object already existed and we skipped the PUT. */
/** `false` if every file and the site manifest were already in the bucket. */
uploaded: boolean;
}
/**
* Upload `projectDir` to `s3://bucketName/sites/<siteId>/project.tar.gz`.
*
* Short-circuits when an object with the same key already exists in the
* bucket — `siteId` derives from the project's content hash, so the same
* bytes produce the same key, and re-uploading would be redundant.
*/
/** Handle for a site an earlier {@link deploySite} uploaded; `bytes` and `uploadedAt` are unknown. */
export function siteHandleFromId(bucketName: string, siteId: string): SiteHandle {
const projectS3Uri = siteManifestUri(bucketName, siteId);
return { siteId, bucketName, projectS3Uri, bytes: 0, uploadedAt: "", uploaded: false };
}
/** Upload `projectDir`'s files to the content store and write the site manifest. */
export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
if (!statSync(opts.projectDir).isDirectory()) {
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
}
const files = listSiteFiles(opts.projectDir);
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
const key = `sites/${siteId}/project.tar.gz`;
const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key });
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
const projectS3Uri = siteManifestUri(opts.bucketName, siteId);
// HeadObject short-circuit. Adopters re-rendering the same project on
// a tight inner loop (CI smoke, demo flows) save the tar+gzip+PUT pass
// on every iteration.
const existing = await headObject(s3, opts.bucketName, key);
if (existing) {
return {
siteId,
bucketName: opts.bucketName,
projectS3Uri,
bytes: existing.bytes,
uploadedAt: existing.lastModified,
uploaded: false,
};
}
const stored: SiteFile[] = [];
let uploaded = false;
await mapConcurrent(files, 8, async (path) => {
const localPath = join(opts.projectDir, path);
const sha256 = await sha256File(localPath);
stored.push({ path, sha256, sizeBytes: statSync(localPath).size });
const outcome = await putContentStoreFile(s3, localPath, opts.bucketName, sha256);
uploaded ||= outcome === "uploaded";
});
stored.sort((a, b) => (a.path < b.path ? -1 : 1));
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
try {
const tarball = join(workdir, "project.tar.gz");
await tarDirectory(opts.projectDir, tarball, files);
const size = statSync(tarball).size;
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
return {
siteId,
bucketName: opts.bucketName,
const manifestPath = join(workdir, "files.json");
const manifest = serializeSiteManifest(stored);
writeFileSync(manifestPath, manifest);
const outcome = await uploadContentAddressedFileToS3(
s3,
manifestPath,
projectS3Uri,
bytes: size,
uploadedAt: new Date().toISOString(),
uploaded: true,
};
await sha256File(manifestPath),
"application/json",
);
uploaded ||= outcome === "uploaded";
} finally {
rmSync(workdir, { recursive: true, force: true });
}
}
async function headObject(
s3: S3Client,
bucket: string,
key: string,
): Promise<{ bytes: number; lastModified: string } | null> {
try {
const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
return {
bytes: typeof res.ContentLength === "number" ? res.ContentLength : 0,
lastModified:
res.LastModified instanceof Date
? res.LastModified.toISOString()
: new Date().toISOString(),
};
} catch (err) {
// The SDK throws different error shapes for 404 vs 403 vs network;
// a 404 means "needs upload" and is the most common case. Anything
// else propagates so callers see auth / network failures.
const status = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
if (status === 404) return null;
const name = (err as { name?: string }).name;
if (name === "NotFound" || name === "NoSuchKey") return null;
throw err;
}
return {
siteId,
bucketName: opts.bucketName,
projectS3Uri,
bytes: stored.reduce((sum, file) => sum + file.sizeBytes, 0),
uploadedAt: new Date().toISOString(),
uploaded,
};
}
+6 -1
View File
@@ -8,7 +8,12 @@
* AWS-SDK only — safe to load in any Node environment.
*/
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./deploySite.js";
export {
deploySite,
type DeploySiteOptions,
type SiteHandle,
siteHandleFromId,
} from "./deploySite.js";
export { renderToLambda, type RenderHandle, type RenderToLambdaOptions } from "./renderToLambda.js";
export {
getRenderProgress,
@@ -76,9 +76,7 @@ describe("renderToLambda", () => {
expect(handle.executionArn).toContain("smoke-1");
expect(handle.bucketName).toBe("test-bucket");
expect(handle.outputS3Uri).toBe("s3://test-bucket/renders/smoke-1/output.mp4");
expect(handle.projectS3Uri).toMatch(
/^s3:\/\/test-bucket\/sites\/[0-9a-f]{16}\/project\.tar\.gz$/,
);
expect(handle.projectS3Uri).toMatch(/^s3:\/\/test-bucket\/sites\/[0-9a-f]{16}\/files\.json$/);
expect(Object.keys(handle)).toEqual([
"renderId",
"executionArn",
+90
View File
@@ -0,0 +1,90 @@
/**
* A deployed site is `sites/<siteId>/files.json`: the project's files by path, each stored once
* in the shared content store under its SHA-256 (see {@link contentStoreUri}).
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import type { S3Client } from "@aws-sdk/client-s3";
import { contentStoreUri } from "./contentStore.js";
import {
downloadS3ObjectToFile,
downloadS3ObjectToFileVerified,
formatS3Uri,
mapConcurrent,
parseS3Uri,
} from "./s3Transport.js";
const SITE_MANIFEST_NAME = "files.json";
export interface SiteFile {
/** `/`-separated, relative to the project root. */
readonly path: string;
readonly sha256: string;
readonly sizeBytes: number;
}
export function siteManifestUri(bucket: string, siteId: string): string {
return formatS3Uri({ bucket, key: `sites/${siteId}/${SITE_MANIFEST_NAME}` });
}
export function isSiteManifestUri(uri: string): boolean {
return uri.endsWith(`/${SITE_MANIFEST_NAME}`);
}
export function serializeSiteManifest(files: readonly SiteFile[]): string {
return JSON.stringify({ files });
}
export function parseSiteManifest(text: string): SiteFile[] {
const value = JSON.parse(text) as { files?: unknown };
if (!Array.isArray(value.files)) throw siteManifestError("files must be an array");
const seen = new Set<string>();
return value.files.map((entry: Partial<SiteFile>, index) => {
const { path, sha256, sizeBytes } = entry ?? {};
if (typeof path !== "string" || !isSafeRelativePath(path) || seen.has(path)) {
throw siteManifestError(`files[${index}].path is not a unique relative path`);
}
if (typeof sha256 !== "string" || !/^[a-f0-9]{64}$/.test(sha256)) {
throw siteManifestError(`files[${index}].sha256 is not a lowercase SHA-256 digest`);
}
if (!Number.isSafeInteger(sizeBytes) || (sizeBytes as number) < 0) {
throw siteManifestError(`files[${index}].sizeBytes is not a byte count`);
}
seen.add(path);
return { path, sha256, sizeBytes: sizeBytes as number };
});
}
/** Rebuild the project under `projectDir`, verifying every file's SHA-256 on the way. */
export async function downloadSiteFiles(
s3: S3Client,
manifestUri: string,
projectDir: string,
): Promise<void> {
const manifestPath = `${projectDir}.${SITE_MANIFEST_NAME}`;
await downloadS3ObjectToFile(s3, manifestUri, manifestPath);
const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));
const { bucket } = parseS3Uri(manifestUri);
await mapConcurrent(files, 16, (file) =>
downloadS3ObjectToFileVerified(
s3,
contentStoreUri(bucket, file.sha256),
join(projectDir, ...file.path.split("/")),
file.sha256,
),
);
}
function isSafeRelativePath(path: string): boolean {
return (
path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..") &&
!path.includes("\\")
);
}
function siteManifestError(detail: string): Error {
const error = new Error(`[siteManifest] SITE_MANIFEST_INVALID: ${detail}`);
error.name = "SITE_MANIFEST_INVALID";
return error;
}