mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-20 03:47:38 +00:00
perf(aws-lambda): upload project files once to a shared content-addressed area
This commit is contained in:
@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { App, Stack } from "aws-cdk-lib";
|
||||
import { Template } from "aws-cdk-lib/assertions";
|
||||
import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js";
|
||||
import { HyperframesRenderStack } from "./HyperframesRenderStack.js";
|
||||
|
||||
// CDK synth is slow on cold start and reached 32s under full-workspace CI
|
||||
@@ -61,7 +62,7 @@ describe("HyperframesRenderStack — contract", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("provisions exactly one S3 bucket with PublicAccessBlockConfiguration and a 7-day intermediates lifecycle", () => {
|
||||
it("provisions exactly one S3 bucket with PublicAccessBlockConfiguration and a 7-day intermediates and 30-day shared content lifecycle", () => {
|
||||
const t = DEFAULT_TEMPLATE;
|
||||
t.resourceCountIs("AWS::S3::Bucket", 1);
|
||||
t.hasResourceProperties("AWS::S3::Bucket", {
|
||||
@@ -79,6 +80,12 @@ describe("HyperframesRenderStack — contract", () => {
|
||||
Prefix: "renders/",
|
||||
ExpirationInDays: 7,
|
||||
},
|
||||
{
|
||||
Id: "ExpireSharedContent",
|
||||
Status: "Enabled",
|
||||
Prefix: `${CONTENT_STORE_PREFIX}/`,
|
||||
ExpirationInDays: CONTENT_STORE_RETENTION_DAYS,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
@@ -26,6 +26,11 @@ import { join } from "node:path";
|
||||
import { App, Stack } from "aws-cdk-lib";
|
||||
import { Template } from "aws-cdk-lib/assertions";
|
||||
import { parse as parseYaml } from "yaml";
|
||||
import {
|
||||
CONTENT_STORE_PREFIX,
|
||||
CONTENT_STORE_REFRESH_AFTER_DAYS,
|
||||
CONTENT_STORE_RETENTION_DAYS,
|
||||
} from "../contentStore.js";
|
||||
import { HyperframesRenderStack } from "./HyperframesRenderStack.js";
|
||||
|
||||
// CDK synth + Template.fromStack is slow on cold start in CI (~5-8s on
|
||||
@@ -264,6 +269,26 @@ describe("HyperframesRenderStack — snapshot", () => {
|
||||
expect(v2).not.toContain("PlanS3Uri");
|
||||
});
|
||||
|
||||
it("keeps the SAM bucket lifecycle identical to CDK's, shared content on the transport's terms", () => {
|
||||
const cdkBucket = Object.values(SYNTHED.template.findResources("AWS::S3::Bucket"))[0] as {
|
||||
Properties: { LifecycleConfiguration: unknown };
|
||||
};
|
||||
const samBucket = requireRecordProperty(readSamResources(), "RenderBucket", "SAM RenderBucket");
|
||||
const samProperties = requireRecordProperty(samBucket, "Properties", "SAM bucket properties");
|
||||
expect(samProperties.LifecycleConfiguration).toEqual(
|
||||
cdkBucket.Properties.LifecycleConfiguration,
|
||||
);
|
||||
expect(CONTENT_STORE_REFRESH_AFTER_DAYS * 2).toBeLessThan(CONTENT_STORE_RETENTION_DAYS);
|
||||
expect(cdkBucket.Properties.LifecycleConfiguration).toMatchObject({
|
||||
Rules: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
Prefix: `${CONTENT_STORE_PREFIX}/`,
|
||||
ExpirationInDays: CONTENT_STORE_RETENTION_DAYS,
|
||||
}),
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
it("sends the same payload from every Lambda task as the SAM template", () => {
|
||||
// CDK drops null-valued payload fields when it renders the definition (#5259), so compare
|
||||
// what each deployed task sends rather than the construct source.
|
||||
@@ -433,7 +458,7 @@ function getV1TaskStates(definition: {
|
||||
};
|
||||
}
|
||||
|
||||
function readSamDefinition(): { States: Record<string, unknown> } {
|
||||
function readSamResources(): Record<string, unknown> {
|
||||
const source = readFileSync(
|
||||
new URL("../../../../examples/aws-lambda/template.yaml", import.meta.url),
|
||||
"utf8",
|
||||
@@ -443,7 +468,11 @@ function readSamDefinition(): { States: Record<string, unknown> } {
|
||||
// warnings for the intentionally unresolved `!Ref`/`!GetAtt` tags.
|
||||
const parsed: unknown = parseYaml(source, { logLevel: "silent" });
|
||||
const root = requireRecord(parsed, "SAM template");
|
||||
const resources = requireRecordProperty(root, "Resources", "SAM resources");
|
||||
return requireRecordProperty(root, "Resources", "SAM resources");
|
||||
}
|
||||
|
||||
function readSamDefinition(): { States: Record<string, unknown> } {
|
||||
const resources = readSamResources();
|
||||
const stateMachine = requireRecordProperty(
|
||||
resources,
|
||||
"RenderStateMachine",
|
||||
|
||||
@@ -29,6 +29,7 @@ import * as s3 from "aws-cdk-lib/aws-s3";
|
||||
import * as sfn from "aws-cdk-lib/aws-stepfunctions";
|
||||
import * as tasks from "aws-cdk-lib/aws-stepfunctions-tasks";
|
||||
import { Construct } from "constructs";
|
||||
import { CONTENT_STORE_PREFIX, CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js";
|
||||
|
||||
/** Construction-time props for {@link HyperframesRenderStack}. */
|
||||
export interface HyperframesRenderStackProps {
|
||||
@@ -92,6 +93,12 @@ export class HyperframesRenderStack extends Construct {
|
||||
prefix: "renders/",
|
||||
expiration: Duration.days(7),
|
||||
},
|
||||
{
|
||||
id: "ExpireSharedContent",
|
||||
enabled: true,
|
||||
prefix: `${CONTENT_STORE_PREFIX}/`,
|
||||
expiration: Duration.days(CONTENT_STORE_RETENTION_DAYS),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
// Shared by every render: project files and plan-v2 artifacts by SHA-256. Dependency-free for CDK.
|
||||
|
||||
export const CONTENT_STORE_PREFIX = "cas/sha256";
|
||||
export const CONTENT_STORE_RETENTION_DAYS = 30;
|
||||
/** Reusing an object at least this old rewrites it, so a plan's objects outlive it by 23+ days. */
|
||||
export const CONTENT_STORE_REFRESH_AFTER_DAYS = 7;
|
||||
|
||||
export function contentStoreUri(bucket: string, sha256: string): string {
|
||||
if (!/^[a-f0-9]{64}$/.test(sha256)) {
|
||||
throw new Error(
|
||||
`[contentStore] expected lowercase SHA-256 digest, got ${JSON.stringify(sha256)}`,
|
||||
);
|
||||
}
|
||||
return `s3://${bucket}/${CONTENT_STORE_PREFIX}/${sha256.slice(0, 2)}/${sha256}`;
|
||||
}
|
||||
@@ -631,7 +631,7 @@ describe("handler dispatch", () => {
|
||||
expect(planned).toMatchObject({
|
||||
PlanProtocol: "v2",
|
||||
PlanV2ManifestS3Uri: "s3://bucket/renders/v2/v2/manifest.json",
|
||||
PlanV2ArtifactS3Prefix: "s3://bucket/renders/v2/v2/artifacts/sha256",
|
||||
PlanV2ArtifactS3Prefix: "s3://bucket/cas/sha256",
|
||||
});
|
||||
expect("PlanS3Uri" in planned).toBe(false);
|
||||
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2") {
|
||||
@@ -744,6 +744,111 @@ describe("handler dispatch", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handler — plan from a deployed site", () => {
|
||||
function storeSiteFile(s3: FakeS3Client, bytes: string): string {
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
const uri = `s3://bucket/cas/sha256/${sha256.slice(0, 2)}/${sha256}`;
|
||||
s3.objects.set(uri, Buffer.from(bytes));
|
||||
s3.metadata.set(uri, { sha256 });
|
||||
return sha256;
|
||||
}
|
||||
|
||||
function planEvent(): PlanEvent {
|
||||
return {
|
||||
Action: "plan",
|
||||
ProjectS3Uri: "s3://bucket/sites/site1/files.json",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/r1/",
|
||||
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
|
||||
};
|
||||
}
|
||||
|
||||
it("rebuilds the project from the content store and reuses its media without uploading it", async () => {
|
||||
const s3 = new FakeS3Client();
|
||||
const html = storeSiteFile(s3, "<html></html>");
|
||||
const media = storeSiteFile(s3, "VIDEO-BYTES");
|
||||
s3.objects.set(
|
||||
"s3://bucket/sites/site1/files.json",
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
files: [
|
||||
{ path: "assets/bg.mp4", sha256: media, sizeBytes: 11 },
|
||||
{ path: "index.html", sha256: html, sizeBytes: 13 },
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
const planV2WithPublisher = mock(
|
||||
async (projectDir: string, _config: unknown, publisher: PlanV2ArtifactPublisher) => {
|
||||
expect(readFileSync(join(projectDir, "index.html"), "utf-8")).toBe("<html></html>");
|
||||
const mediaPath = join(projectDir, "assets", "bg.mp4");
|
||||
await publisher.putBlob({ sourcePath: mediaPath, sha256: media, sizeBytes: 11 });
|
||||
const manifest = {
|
||||
planHash: "c".repeat(64),
|
||||
chunkCount: 1,
|
||||
totalFrames: 30,
|
||||
fps: 30,
|
||||
width: 640,
|
||||
height: 360,
|
||||
format: "mp4",
|
||||
ffmpegVersion: "test",
|
||||
producerVersion: "test",
|
||||
artifacts: [{ path: "compiled/assets/bg.mp4", sha256: media, sizeBytes: 11 }],
|
||||
};
|
||||
await publisher.commitManifest(JSON.stringify(manifest));
|
||||
return manifest as unknown as PlanV2Manifest;
|
||||
},
|
||||
);
|
||||
|
||||
const result = await handler(planEvent(), {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
planV2WithPublisher:
|
||||
planV2WithPublisher as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,
|
||||
renderChunk:
|
||||
mock() as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble: mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot: makeTmpRoot(),
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ PlanV2ArtifactS3Prefix: "s3://bucket/cas/sha256" });
|
||||
expect(planV2WithPublisher).toHaveBeenCalledTimes(1);
|
||||
expect(s3.ops.filter((op) => op.kind === "upload").map((op) => op.uri)).toEqual([
|
||||
"s3://bucket/renders/r1/v2/manifest.json",
|
||||
]);
|
||||
expect(s3.ops.some((op) => op.uri.endsWith(".tar.gz"))).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses a site manifest path that escapes the project", async () => {
|
||||
const s3 = new FakeS3Client();
|
||||
const sha256 = storeSiteFile(s3, "x");
|
||||
s3.objects.set(
|
||||
"s3://bucket/sites/site1/files.json",
|
||||
Buffer.from(JSON.stringify({ files: [{ path: "../escape", sha256, sizeBytes: 1 }] })),
|
||||
);
|
||||
const planV2WithPublisher = mock();
|
||||
await expect(
|
||||
handler(planEvent(), {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: mock() as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
planV2WithPublisher:
|
||||
planV2WithPublisher as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,
|
||||
renderChunk:
|
||||
mock() as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
mock() as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot: makeTmpRoot(),
|
||||
skipChromeResolution: true,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "SITE_MANIFEST_INVALID" });
|
||||
expect(planV2WithPublisher).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("handler — S3 URI allowlist (security: F-004)", () => {
|
||||
let prevBucket: string | undefined;
|
||||
|
||||
|
||||
@@ -54,12 +54,14 @@ import type {
|
||||
import {
|
||||
downloadS3ObjectToFile,
|
||||
downloadS3ObjectToFileVerified,
|
||||
mapConcurrent,
|
||||
parseS3Uri,
|
||||
tarDirectory,
|
||||
untarDirectory,
|
||||
uploadFileToS3,
|
||||
} from "./s3Transport.js";
|
||||
import { S3PlanV2ArtifactPublisher } from "./s3PlanV2Publisher.js";
|
||||
import { downloadSiteFiles, isSiteManifestUri } from "./siteManifest.js";
|
||||
|
||||
/**
|
||||
* Lazily-constructed S3 client. Cached at module scope so warm Lambda
|
||||
@@ -337,17 +339,10 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam
|
||||
}
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-"));
|
||||
// We use `.tar.gz` (not `.zip`) as the project archive's on-the-wire
|
||||
// format because Lambda's Amazon Linux base image ships GNU `tar` but
|
||||
// not `unzip` in `/usr/bin`. The smoke script + future CLI both
|
||||
// produce tar.gz uploads.
|
||||
const projectArchive = join(work, "project.tar.gz");
|
||||
const projectDir = join(work, "project");
|
||||
const planDir = join(work, "plan");
|
||||
|
||||
try {
|
||||
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
||||
await untarDirectory(projectArchive, projectDir);
|
||||
const projectDir = await fetchProject(s3, event.ProjectS3Uri, work);
|
||||
|
||||
const config: DistributedRenderConfig = {
|
||||
...event.Config,
|
||||
@@ -395,6 +390,22 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `ProjectS3Uri` is a site manifest from `deploySite`, or a `.tar.gz` of the project
|
||||
* (hand-built uploads such as `examples/aws-lambda/scripts/smoke.sh`).
|
||||
*/
|
||||
async function fetchProject(s3: S3Client, projectS3Uri: string, work: string): Promise<string> {
|
||||
const projectDir = join(work, "project");
|
||||
if (isSiteManifestUri(projectS3Uri)) {
|
||||
await downloadSiteFiles(s3, projectS3Uri, projectDir);
|
||||
} else {
|
||||
const projectArchive = join(work, "project.tar.gz");
|
||||
await downloadS3ObjectToFile(s3, projectS3Uri, projectArchive);
|
||||
await untarDirectory(projectArchive, projectDir);
|
||||
}
|
||||
return projectDir;
|
||||
}
|
||||
|
||||
// Plan v2 orchestration is kept as one transactional boundary: stage, CAS upload,
|
||||
// and manifest-last publication must stay ordered and fail together.
|
||||
// fallow-ignore-next-line complexity
|
||||
@@ -410,11 +421,8 @@ async function handlePlanV2(
|
||||
}
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-v2-"));
|
||||
const projectArchive = join(work, "project.tar.gz");
|
||||
const projectDir = join(work, "project");
|
||||
try {
|
||||
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
||||
await untarDirectory(projectArchive, projectDir);
|
||||
const projectDir = await fetchProject(s3, event.ProjectS3Uri, work);
|
||||
const publisher = new S3PlanV2ArtifactPublisher({
|
||||
s3,
|
||||
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
||||
@@ -749,30 +757,6 @@ function throwPlanHashMismatch(expected: string, actual: string): never {
|
||||
throw error;
|
||||
}
|
||||
|
||||
async function mapConcurrent<T>(
|
||||
values: readonly T[],
|
||||
concurrency: number,
|
||||
fn: (value: T) => Promise<void>,
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
async function worker(): Promise<void> {
|
||||
while (cursor < values.length) {
|
||||
const index = cursor++;
|
||||
await fn(values[index]!);
|
||||
}
|
||||
}
|
||||
const results = await Promise.allSettled(
|
||||
Array.from({ length: Math.min(concurrency, values.length) }, () => worker()),
|
||||
);
|
||||
const failure = results.find(
|
||||
(result): result is PromiseRejectedResult => result.status === "rejected",
|
||||
);
|
||||
// Do not reject while sibling workers may still be writing into invocation
|
||||
// scratch. The caller removes that directory in `finally`; draining the pool
|
||||
// first prevents late S3 streams from racing cleanup after another GET fails.
|
||||
if (failure) throw failure.reason;
|
||||
}
|
||||
|
||||
async function downloadChunkObjects(
|
||||
s3: S3Client,
|
||||
uris: string[],
|
||||
|
||||
@@ -110,14 +110,37 @@ describe("S3PlanV2ArtifactPublisher", () => {
|
||||
planOutputS3Prefix: `s3://bucket/render${"/".repeat(10_000)}`,
|
||||
});
|
||||
|
||||
expect(publisher.artifactPrefix).toBe("s3://bucket/render/v2/artifacts/sha256");
|
||||
expect(publisher.artifactPrefix).toBe("s3://bucket/cas/sha256");
|
||||
expect(publisher.manifestUri).toBe("s3://bucket/render/v2/manifest.json");
|
||||
});
|
||||
|
||||
it("reuses a blob the SDK or another render already stored instead of uploading it", async () => {
|
||||
const source = makeSource("shared media");
|
||||
const s3 = new FakeS3();
|
||||
const blobUri = `s3://bucket/cas/sha256/${source.digest.slice(0, 2)}/${source.digest}`;
|
||||
s3.objects.set(blobUri, { bytes: Buffer.from("shared media"), sha256: source.digest });
|
||||
const publisher = new S3PlanV2ArtifactPublisher({
|
||||
s3: s3.asClient(),
|
||||
planOutputS3Prefix: "s3://bucket/renders/second",
|
||||
temporaryRoot: source.root,
|
||||
});
|
||||
|
||||
await publisher.putBlob({
|
||||
sourcePath: source.path,
|
||||
sha256: source.digest,
|
||||
sizeBytes: source.sizeBytes,
|
||||
});
|
||||
await publisher.commitManifest(manifestFor(source.digest));
|
||||
|
||||
expect(s3.puts.map((operation) => operation.uri)).toEqual([
|
||||
"s3://bucket/renders/second/v2/manifest.json",
|
||||
]);
|
||||
});
|
||||
|
||||
it("publishes immutable blobs before the fixed-key manifest", async () => {
|
||||
const source = makeSource("hello");
|
||||
const s3 = new FakeS3();
|
||||
const artifactPrefix = "s3://bucket/render/v2/artifacts/sha256";
|
||||
const artifactPrefix = "s3://bucket/cas/sha256";
|
||||
const manifestUri = "s3://bucket/render/v2/manifest.json";
|
||||
const publisher = new S3PlanV2ArtifactPublisher({
|
||||
s3: s3.asClient(),
|
||||
|
||||
@@ -8,7 +8,13 @@ import {
|
||||
type PlanV2ArtifactPublisher,
|
||||
type PlanV2PublishBlob,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
import { parseS3Uri, uploadContentAddressedFileToS3 } from "./s3Transport.js";
|
||||
import { CONTENT_STORE_PREFIX } from "./contentStore.js";
|
||||
import {
|
||||
formatS3Uri,
|
||||
parseS3Uri,
|
||||
putContentStoreFile,
|
||||
uploadContentAddressedFileToS3,
|
||||
} from "./s3Transport.js";
|
||||
|
||||
export interface S3PlanV2ArtifactPublisherOptions {
|
||||
readonly s3: S3Client;
|
||||
@@ -66,15 +72,18 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
|
||||
readonly artifactPrefix: string;
|
||||
readonly manifestUri: string;
|
||||
readonly #s3: S3Client;
|
||||
readonly #bucket: string;
|
||||
readonly #temporaryRoot: string;
|
||||
readonly #publishedDigests = new Set<string>();
|
||||
#state: "open" | "committed" | "aborted" = "open";
|
||||
|
||||
constructor(options: Readonly<S3PlanV2ArtifactPublisherOptions>) {
|
||||
const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;
|
||||
parseS3Uri(outputPrefix);
|
||||
const { bucket } = parseS3Uri(outputPrefix);
|
||||
this.#s3 = options.s3;
|
||||
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
||||
this.#bucket = bucket;
|
||||
// Shared by every render, so a blob the SDK or an earlier render uploaded is reused, not re-sent.
|
||||
this.artifactPrefix = formatS3Uri({ bucket, key: CONTENT_STORE_PREFIX });
|
||||
this.manifestUri = `${outputPrefix}/manifest.json`;
|
||||
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
||||
mkdirSync(this.#temporaryRoot, { recursive: true });
|
||||
@@ -89,8 +98,7 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
|
||||
`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`,
|
||||
);
|
||||
}
|
||||
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
||||
await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);
|
||||
await putContentStoreFile(this.#s3, blob.sourcePath, this.#bucket, digest);
|
||||
this.#publishedDigests.add(digest);
|
||||
}
|
||||
|
||||
@@ -124,9 +132,8 @@ export class S3PlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
|
||||
|
||||
async abort(): Promise<void> {
|
||||
if (this.#state === "open") this.#state = "aborted";
|
||||
// Remote CAS blobs are immutable and may already be reused by a retry.
|
||||
// Without a committed manifest they are unreachable and expire under the
|
||||
// render bucket's intermediate-object lifecycle policy.
|
||||
// Remote CAS blobs are immutable and may already be reused by a retry or
|
||||
// another render. They expire under the shared content store's lifecycle.
|
||||
}
|
||||
|
||||
#assertOpen(operation: string): void {
|
||||
|
||||
@@ -9,6 +9,7 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js";
|
||||
import {
|
||||
downloadS3ObjectToFileVerified,
|
||||
formatS3Uri,
|
||||
@@ -16,6 +17,7 @@ import {
|
||||
sha256File,
|
||||
tarDirectory,
|
||||
untarDirectory,
|
||||
putContentStoreFile,
|
||||
uploadContentAddressedFileToS3,
|
||||
} from "./s3Transport.js";
|
||||
|
||||
@@ -162,6 +164,37 @@ describe("content-addressed v2 artifacts", () => {
|
||||
expect(s3.putCount).toBe(0);
|
||||
});
|
||||
|
||||
it("rewrites a reused object at the refresh age so its lifecycle expiry restarts", async () => {
|
||||
const source = join(scratchRoot, "artifact-aging.bin");
|
||||
writeFileSync(source, "aging bytes");
|
||||
const digest = await sha256File(source);
|
||||
const uri = contentStoreUri("bucket", digest);
|
||||
const s3 = new ContentAddressedFakeS3();
|
||||
const days = (n: number) => new Date(Date.now() - n * 86_400_000);
|
||||
s3.objects.set(uri, {
|
||||
bytes: Buffer.from("aging bytes"),
|
||||
sha256: digest,
|
||||
lastModified: days(CONTENT_STORE_REFRESH_AFTER_DAYS - 1),
|
||||
});
|
||||
|
||||
expect(await putContentStoreFile(s3.asClient(), source, "bucket", digest)).toBe("reused");
|
||||
expect(s3.copies).toEqual([]);
|
||||
|
||||
s3.objects.get(uri)!.lastModified = days(CONTENT_STORE_REFRESH_AFTER_DAYS);
|
||||
expect(await putContentStoreFile(s3.asClient(), source, "bucket", digest)).toBe("reused");
|
||||
expect(s3.copies).toEqual([
|
||||
{
|
||||
uri,
|
||||
CopySource: `bucket/cas/sha256/${digest.slice(0, 2)}/${digest}`,
|
||||
MetadataDirective: "REPLACE",
|
||||
Metadata: { sha256: digest },
|
||||
ChecksumAlgorithm: "SHA256",
|
||||
},
|
||||
]);
|
||||
expect(s3.objects.get(uri)).toMatchObject({ sha256: digest });
|
||||
expect(s3.putCount).toBe(0);
|
||||
});
|
||||
|
||||
it("deletes a downloaded artifact when digest verification fails", async () => {
|
||||
const expectedSource = join(scratchRoot, "artifact-expected.bin");
|
||||
const destination = join(scratchRoot, "artifact-download.bin");
|
||||
@@ -180,7 +213,8 @@ describe("content-addressed v2 artifacts", () => {
|
||||
});
|
||||
|
||||
class ContentAddressedFakeS3 {
|
||||
readonly objects = new Map<string, { bytes: Buffer; sha256: string }>();
|
||||
readonly objects = new Map<string, { bytes: Buffer; sha256: string; lastModified?: Date }>();
|
||||
readonly copies: Record<string, unknown>[] = [];
|
||||
putCount = 0;
|
||||
raceOnNextPut: { bytes: Buffer; sha256: string } | undefined;
|
||||
|
||||
@@ -199,9 +233,24 @@ class ContentAddressedFakeS3 {
|
||||
Key: string;
|
||||
Body?: NodeJS.ReadableStream;
|
||||
Metadata?: Record<string, string>;
|
||||
CopySource?: string;
|
||||
MetadataDirective?: string;
|
||||
ChecksumAlgorithm?: string;
|
||||
};
|
||||
};
|
||||
const uri = `s3://${value.input.Bucket}/${value.input.Key}`;
|
||||
if (value.constructor.name === "CopyObjectCommand") {
|
||||
const { CopySource, MetadataDirective, Metadata, ChecksumAlgorithm } = value.input;
|
||||
this.copies.push({ uri, CopySource, MetadataDirective, Metadata, ChecksumAlgorithm });
|
||||
const source = this.objects.get(`s3://${CopySource}`);
|
||||
if (!source) throw new Error("missing copy source");
|
||||
this.objects.set(uri, {
|
||||
bytes: source.bytes,
|
||||
sha256: Metadata?.sha256 ?? "",
|
||||
lastModified: new Date(),
|
||||
});
|
||||
return {};
|
||||
}
|
||||
if (value.constructor.name === "HeadObjectCommand") {
|
||||
const object = this.objects.get(uri);
|
||||
if (!object) {
|
||||
@@ -215,6 +264,7 @@ class ContentAddressedFakeS3 {
|
||||
return {
|
||||
ContentLength: object.bytes.length,
|
||||
Metadata: { sha256: object.sha256 },
|
||||
LastModified: object.lastModified,
|
||||
};
|
||||
}
|
||||
if (value.constructor.name === "GetObjectCommand") {
|
||||
|
||||
@@ -29,12 +29,14 @@ import { createHash } from "node:crypto";
|
||||
import { dirname } from "node:path";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import {
|
||||
CopyObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
type S3Client,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import * as tar from "tar";
|
||||
import { CONTENT_STORE_REFRESH_AFTER_DAYS, contentStoreUri } from "./contentStore.js";
|
||||
|
||||
/** Parsed `s3://bucket/key` URI. */
|
||||
export interface S3Location {
|
||||
@@ -144,6 +146,16 @@ export async function uploadContentAddressedFileToS3(
|
||||
expectedSha256: string,
|
||||
contentType?: string,
|
||||
): Promise<"uploaded" | "reused"> {
|
||||
return (await putImmutableFile(client, localPath, uri, expectedSha256, contentType)).outcome;
|
||||
}
|
||||
|
||||
async function putImmutableFile(
|
||||
client: S3Client,
|
||||
localPath: string,
|
||||
uri: string,
|
||||
expectedSha256: string,
|
||||
contentType?: string,
|
||||
): Promise<{ outcome: "uploaded" | "reused"; lastModified?: Date }> {
|
||||
assertSha256(expectedSha256);
|
||||
if (!existsSync(localPath)) {
|
||||
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
|
||||
@@ -160,8 +172,9 @@ export async function uploadContentAddressedFileToS3(
|
||||
const { bucket, key } = parseS3Uri(uri);
|
||||
const size = statSync(localPath).size;
|
||||
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
||||
if (existing === "matching") return "reused";
|
||||
if (existing === "conflict") throwImmutableObjectConflict(uri);
|
||||
if (existing.state === "matching")
|
||||
return { outcome: "reused", lastModified: existing.lastModified };
|
||||
if (existing.state === "conflict") throwImmutableObjectConflict(uri);
|
||||
|
||||
const body = createReadStream(localPath);
|
||||
try {
|
||||
@@ -180,12 +193,12 @@ export async function uploadContentAddressedFileToS3(
|
||||
IfNoneMatch: "*",
|
||||
}),
|
||||
);
|
||||
return "uploaded";
|
||||
return { outcome: "uploaded" };
|
||||
} catch (error) {
|
||||
if (!isS3PreconditionFailed(error)) throw error;
|
||||
const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
||||
if (raced === "matching") return "reused";
|
||||
if (raced === "conflict") throwImmutableObjectConflict(uri);
|
||||
if (raced.state === "matching") return { outcome: "reused" };
|
||||
if (raced.state === "conflict") throwImmutableObjectConflict(uri);
|
||||
// The winning object was deleted between the conditional failure and
|
||||
// verification. Preserve the service error so the orchestrator may retry.
|
||||
throw error;
|
||||
@@ -197,6 +210,30 @@ export async function uploadContentAddressedFileToS3(
|
||||
}
|
||||
}
|
||||
|
||||
export async function mapConcurrent<T>(
|
||||
values: readonly T[],
|
||||
concurrency: number,
|
||||
fn: (value: T) => Promise<void>,
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
async function worker(): Promise<void> {
|
||||
while (cursor < values.length) {
|
||||
const index = cursor++;
|
||||
await fn(values[index]!);
|
||||
}
|
||||
}
|
||||
const results = await Promise.allSettled(
|
||||
Array.from({ length: Math.min(concurrency, values.length) }, () => worker()),
|
||||
);
|
||||
const failure = results.find(
|
||||
(result): result is PromiseRejectedResult => result.status === "rejected",
|
||||
);
|
||||
// Do not reject while sibling workers may still be writing into invocation
|
||||
// scratch. The caller removes that directory in `finally`; draining the pool
|
||||
// first prevents late S3 streams from racing cleanup after another GET fails.
|
||||
if (failure) throw failure.reason;
|
||||
}
|
||||
|
||||
export async function sha256File(path: string): Promise<string> {
|
||||
const hash = createHash("sha256");
|
||||
for await (const chunk of createReadStream(path)) {
|
||||
@@ -213,7 +250,9 @@ function assertSha256(value: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
type ContentAddressedObjectState = "missing" | "matching" | "conflict";
|
||||
type ContentAddressedObjectState =
|
||||
| { state: "missing" | "conflict" }
|
||||
| { state: "matching"; lastModified?: Date };
|
||||
|
||||
async function inspectContentAddressedObject(
|
||||
client: S3Client,
|
||||
@@ -227,14 +266,49 @@ async function inspectContentAddressedObject(
|
||||
new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" }),
|
||||
);
|
||||
return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256
|
||||
? "matching"
|
||||
: "conflict";
|
||||
? { state: "matching", lastModified: existing.LastModified }
|
||||
: { state: "conflict" };
|
||||
} catch (error) {
|
||||
if (isS3NotFound(error)) return "missing";
|
||||
if (isS3NotFound(error)) return { state: "missing" };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Put a file into the shared content store once; reusing an aging object restarts its expiry. */
|
||||
export async function putContentStoreFile(
|
||||
client: S3Client,
|
||||
localPath: string,
|
||||
bucket: string,
|
||||
sha256: string,
|
||||
): Promise<"uploaded" | "reused"> {
|
||||
const uri = contentStoreUri(bucket, sha256);
|
||||
const { outcome, lastModified } = await putImmutableFile(client, localPath, uri, sha256);
|
||||
if (lastModified) await refreshAgingObject(client, uri, sha256, lastModified);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
// S3 lifecycle expiry counts from the last write, not the last read, so reuse has to write.
|
||||
async function refreshAgingObject(
|
||||
client: S3Client,
|
||||
uri: string,
|
||||
sha256: string,
|
||||
lastModified: Date,
|
||||
): Promise<void> {
|
||||
const { bucket, key } = parseS3Uri(uri);
|
||||
const ageMs = Date.now() - lastModified.getTime();
|
||||
if (ageMs < CONTENT_STORE_REFRESH_AFTER_DAYS * 86_400_000) return;
|
||||
await client.send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
CopySource: `${bucket}/${key}`,
|
||||
MetadataDirective: "REPLACE",
|
||||
Metadata: { sha256 },
|
||||
ChecksumAlgorithm: "SHA256",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function throwImmutableObjectConflict(uri: string): never {
|
||||
const error = new Error(
|
||||
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`,
|
||||
|
||||
@@ -21,28 +21,24 @@ export class FakeS3 {
|
||||
ops: FakeS3Op[] = [];
|
||||
existing = new Set<string>();
|
||||
bodies = new Map<string, Buffer>();
|
||||
metadata = new Map<string, Record<string, string> | undefined>();
|
||||
|
||||
async send(command: unknown): Promise<unknown> {
|
||||
const cmdName = (command as { constructor: { name: string } }).constructor.name;
|
||||
const input = (command as { input: { Bucket: string; Key: string } }).input;
|
||||
const input = (
|
||||
command as { input: { Bucket: string; Key: string; Metadata?: Record<string, string> } }
|
||||
).input;
|
||||
const id = `${input.Bucket}/${input.Key}`;
|
||||
if (cmdName === "HeadObjectCommand") {
|
||||
this.ops.push({ kind: "head", bucket: input.Bucket, key: input.Key });
|
||||
if (this.existing.has(`${input.Bucket}/${input.Key}`)) {
|
||||
return { ContentLength: 1, LastModified: new Date("2026-05-16T00:00:00Z") };
|
||||
}
|
||||
const err = new Error("Not Found") as Error & {
|
||||
$metadata: { httpStatusCode: number };
|
||||
name: string;
|
||||
};
|
||||
err.name = "NotFound";
|
||||
err.$metadata = { httpStatusCode: 404 };
|
||||
throw err;
|
||||
return this.head(id);
|
||||
}
|
||||
if (cmdName === "PutObjectCommand") {
|
||||
this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });
|
||||
this.existing.add(`${input.Bucket}/${input.Key}`);
|
||||
this.existing.add(id);
|
||||
this.metadata.set(id, input.Metadata);
|
||||
this.bodies.set(
|
||||
`${input.Bucket}/${input.Key}`,
|
||||
id,
|
||||
await drainBody(
|
||||
(command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body,
|
||||
),
|
||||
@@ -51,6 +47,19 @@ export class FakeS3 {
|
||||
}
|
||||
throw new Error(`FakeS3: unexpected command ${cmdName}`);
|
||||
}
|
||||
|
||||
private head(id: string): unknown {
|
||||
if (this.existing.has(id)) {
|
||||
return { ContentLength: this.bodies.get(id)?.length, Metadata: this.metadata.get(id) };
|
||||
}
|
||||
const err = new Error("Not Found") as Error & {
|
||||
$metadata: { httpStatusCode: number };
|
||||
name: string;
|
||||
};
|
||||
err.name = "NotFound";
|
||||
err.$metadata = { httpStatusCode: 404 };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Convenience cast so `deploySite({ s3: makeFakeS3() })` reads cleanly. */
|
||||
|
||||
@@ -2,8 +2,9 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import * as tar from "tar";
|
||||
import { parseSiteManifest } from "../siteManifest.js";
|
||||
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
|
||||
import { deploySite } from "./deploySite.js";
|
||||
|
||||
@@ -23,16 +24,25 @@ afterEach(() => {
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function uploadedEntries(s3: FakeS3, siteId: string): Promise<string[]> {
|
||||
const file = join(scratch, "uploaded.tar.gz");
|
||||
writeFileSync(file, s3.bodies.get(`test-bucket/sites/${siteId}/project.tar.gz`) ?? "");
|
||||
const entries: string[] = [];
|
||||
await tar.list({ file, onReadEntry: (entry) => entries.push(`${entry.type} ${entry.path}`) });
|
||||
return entries;
|
||||
function sha256(text: string): string {
|
||||
return createHash("sha256").update(text).digest("hex");
|
||||
}
|
||||
|
||||
function storeKey(text: string): string {
|
||||
const digest = sha256(text);
|
||||
return `cas/sha256/${digest.slice(0, 2)}/${digest}`;
|
||||
}
|
||||
|
||||
function sitePaths(s3: FakeS3, siteId: string): string[] {
|
||||
const manifest = s3.bodies.get(`test-bucket/sites/${siteId}/files.json`)?.toString() ?? "";
|
||||
return parseSiteManifest(manifest).map((file) => file.path);
|
||||
}
|
||||
|
||||
const HTML = "<html><body>hi</body></html>";
|
||||
const CSS = "body { color: red; }";
|
||||
|
||||
describe("deploySite", () => {
|
||||
it("uploads the tarball when no matching object exists", async () => {
|
||||
it("uploads each file once under its content hash, then the site manifest", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const result = await deploySite({
|
||||
projectDir,
|
||||
@@ -43,14 +53,33 @@ describe("deploySite", () => {
|
||||
expect(result.uploaded).toBe(true);
|
||||
expect(result.siteId).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(result.bucketName).toBe("test-bucket");
|
||||
expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/project.tar.gz`);
|
||||
expect(result.bytes).toBeGreaterThan(0);
|
||||
expect(s3.ops).toEqual([
|
||||
{ kind: "head", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },
|
||||
{ kind: "put", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },
|
||||
expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/files.json`);
|
||||
expect(result.bytes).toBe(HTML.length + CSS.length);
|
||||
const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);
|
||||
expect(puts.slice(0, 2).sort()).toEqual([storeKey(HTML), storeKey(CSS)].sort());
|
||||
expect(puts[2]).toBe(`sites/${result.siteId}/files.json`);
|
||||
expect(s3.bodies.get(`test-bucket/${storeKey(CSS)}`)?.toString()).toBe(CSS);
|
||||
expect(parseSiteManifest(s3.bodies.get(`test-bucket/${puts[2]}`)?.toString() ?? "")).toEqual([
|
||||
{ path: "assets/style.css", sha256: sha256(CSS), sizeBytes: CSS.length },
|
||||
{ path: "index.html", sha256: sha256(HTML), sizeBytes: HTML.length },
|
||||
]);
|
||||
});
|
||||
|
||||
it("skips a file another site already stored", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const cssId = `test-bucket/${storeKey(CSS)}`;
|
||||
s3.existing.add(cssId);
|
||||
s3.bodies.set(cssId, Buffer.from(CSS));
|
||||
s3.metadata.set(cssId, { sha256: sha256(CSS) });
|
||||
|
||||
await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) });
|
||||
|
||||
const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);
|
||||
expect(puts).toHaveLength(2);
|
||||
expect(puts).not.toContain(storeKey(CSS));
|
||||
expect(puts).toContain(storeKey(HTML));
|
||||
});
|
||||
|
||||
it("yields a stable siteId across re-runs of the same tree", async () => {
|
||||
const s3a = new FakeS3();
|
||||
const a = await deploySite({
|
||||
@@ -85,7 +114,7 @@ describe("deploySite", () => {
|
||||
expect(after.siteId).not.toBe(before.siteId);
|
||||
});
|
||||
|
||||
it("short-circuits on HEAD 200 (skips PUT)", async () => {
|
||||
it("uploads nothing when the same tree is deployed again", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const first = await deploySite({
|
||||
projectDir,
|
||||
@@ -101,9 +130,9 @@ describe("deploySite", () => {
|
||||
|
||||
expect(second.uploaded).toBe(false);
|
||||
expect(second.siteId).toBe(first.siteId);
|
||||
// Only one PUT total, plus two HEADs.
|
||||
expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(1);
|
||||
expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(2);
|
||||
// Two files plus the manifest, each put once and HEAD-checked on both deploys.
|
||||
expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(3);
|
||||
expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(6);
|
||||
});
|
||||
|
||||
it("honours a caller-supplied siteId", async () => {
|
||||
@@ -115,7 +144,7 @@ describe("deploySite", () => {
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(result.siteId).toBe("release-v1.2.3");
|
||||
expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/project.tar.gz");
|
||||
expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/files.json");
|
||||
});
|
||||
|
||||
it("propagates non-404 S3 errors", async () => {
|
||||
@@ -158,7 +187,7 @@ describe("deploySite", () => {
|
||||
expect(after.siteId).toBe(before.siteId);
|
||||
});
|
||||
|
||||
it("packs a symlinked project and its symlinked files and folders as content", async () => {
|
||||
it("ships a symlinked project and its symlinked files and folders as content", async () => {
|
||||
mkdirSync(join(scratch, "media"));
|
||||
writeFileSync(join(scratch, "media", "clip.txt"), "clip");
|
||||
writeFileSync(join(scratch, "logo.txt"), "logo");
|
||||
@@ -171,11 +200,11 @@ describe("deploySite", () => {
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(await uploadedEntries(s3, result.siteId)).toEqual([
|
||||
"File assets/style.css",
|
||||
"File index.html",
|
||||
"File logo.txt",
|
||||
"File media/clip.txt",
|
||||
expect(sitePaths(s3, result.siteId)).toEqual([
|
||||
"assets/style.css",
|
||||
"index.html",
|
||||
"logo.txt",
|
||||
"media/clip.txt",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,23 +1,28 @@
|
||||
/**
|
||||
* `deploySite` — upload a project directory to S3 once per content hash
|
||||
* and return a reusable handle.
|
||||
* `deploySite` — upload a project directory to S3 and return a reusable handle.
|
||||
*
|
||||
* `renderToLambda` calls this implicitly when no `siteHandle` is passed,
|
||||
* but exposing it as a standalone verb lets adopters bundle a project
|
||||
* ahead of time and reuse the handle across many renders without
|
||||
* re-tarring the project tree on every call.
|
||||
* but exposing it as a standalone verb lets adopters upload a project
|
||||
* ahead of time and reuse the handle across many renders.
|
||||
*
|
||||
* The handle is **content-addressed**: `siteId` is derived from a SHA-256
|
||||
* over the project files. Two `deploySite` calls on an unchanged tree
|
||||
* produce the same `siteId` and `HeadObject`-short-circuit the upload.
|
||||
* Every file goes into the bucket's shared content store under its SHA-256,
|
||||
* once: a file any earlier site or render already stored is skipped. The
|
||||
* site itself is a small `sites/<siteId>/files.json` listing path → hash,
|
||||
* where `siteId` is derived from a SHA-256 over the project files.
|
||||
*/
|
||||
|
||||
import { mkdtempSync, rmSync, statSync } from "node:fs";
|
||||
import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed";
|
||||
import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js";
|
||||
import {
|
||||
mapConcurrent,
|
||||
putContentStoreFile,
|
||||
sha256File,
|
||||
uploadContentAddressedFileToS3,
|
||||
} from "../s3Transport.js";
|
||||
import { serializeSiteManifest, siteManifestUri, type SiteFile } from "../siteManifest.js";
|
||||
|
||||
/** Options for {@link deploySite}. */
|
||||
export interface DeploySiteOptions {
|
||||
@@ -43,90 +48,65 @@ export interface SiteHandle {
|
||||
siteId: string;
|
||||
/** Bucket the site landed in. Surfaced separately so callers don't have to re-parse `projectS3Uri`. */
|
||||
bucketName: string;
|
||||
/** Full `s3://bucket/sites/<siteId>/project.tar.gz` URI; pass through to `renderToLambda`. */
|
||||
/** Full `s3://bucket/sites/<siteId>/files.json` URI; pass through to `renderToLambda`. */
|
||||
projectS3Uri: string;
|
||||
/** Tarball size in bytes; useful for "did we actually skip the upload?" assertions. */
|
||||
/** Total size of the project's files in bytes. */
|
||||
bytes: number;
|
||||
/** ISO timestamp of the most recent upload OR the existing object the short-circuit found. */
|
||||
/** ISO timestamp of this `deploySite` call. */
|
||||
uploadedAt: string;
|
||||
/** `false` if the object already existed and we skipped the PUT. */
|
||||
/** `false` if every file and the site manifest were already in the bucket. */
|
||||
uploaded: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload `projectDir` to `s3://bucketName/sites/<siteId>/project.tar.gz`.
|
||||
*
|
||||
* Short-circuits when an object with the same key already exists in the
|
||||
* bucket — `siteId` derives from the project's content hash, so the same
|
||||
* bytes produce the same key, and re-uploading would be redundant.
|
||||
*/
|
||||
/** Handle for a site an earlier {@link deploySite} uploaded; `bytes` and `uploadedAt` are unknown. */
|
||||
export function siteHandleFromId(bucketName: string, siteId: string): SiteHandle {
|
||||
const projectS3Uri = siteManifestUri(bucketName, siteId);
|
||||
return { siteId, bucketName, projectS3Uri, bytes: 0, uploadedAt: "", uploaded: false };
|
||||
}
|
||||
|
||||
/** Upload `projectDir`'s files to the content store and write the site manifest. */
|
||||
export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
|
||||
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
|
||||
if (!statSync(opts.projectDir).isDirectory()) {
|
||||
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
||||
}
|
||||
|
||||
const files = listSiteFiles(opts.projectDir);
|
||||
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir, files);
|
||||
const key = `sites/${siteId}/project.tar.gz`;
|
||||
const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key });
|
||||
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
|
||||
const projectS3Uri = siteManifestUri(opts.bucketName, siteId);
|
||||
|
||||
// HeadObject short-circuit. Adopters re-rendering the same project on
|
||||
// a tight inner loop (CI smoke, demo flows) save the tar+gzip+PUT pass
|
||||
// on every iteration.
|
||||
const existing = await headObject(s3, opts.bucketName, key);
|
||||
if (existing) {
|
||||
return {
|
||||
siteId,
|
||||
bucketName: opts.bucketName,
|
||||
projectS3Uri,
|
||||
bytes: existing.bytes,
|
||||
uploadedAt: existing.lastModified,
|
||||
uploaded: false,
|
||||
};
|
||||
}
|
||||
const stored: SiteFile[] = [];
|
||||
let uploaded = false;
|
||||
await mapConcurrent(files, 8, async (path) => {
|
||||
const localPath = join(opts.projectDir, path);
|
||||
const sha256 = await sha256File(localPath);
|
||||
stored.push({ path, sha256, sizeBytes: statSync(localPath).size });
|
||||
const outcome = await putContentStoreFile(s3, localPath, opts.bucketName, sha256);
|
||||
uploaded ||= outcome === "uploaded";
|
||||
});
|
||||
stored.sort((a, b) => (a.path < b.path ? -1 : 1));
|
||||
|
||||
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
|
||||
try {
|
||||
const tarball = join(workdir, "project.tar.gz");
|
||||
await tarDirectory(opts.projectDir, tarball, files);
|
||||
const size = statSync(tarball).size;
|
||||
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
|
||||
return {
|
||||
siteId,
|
||||
bucketName: opts.bucketName,
|
||||
const manifestPath = join(workdir, "files.json");
|
||||
const manifest = serializeSiteManifest(stored);
|
||||
writeFileSync(manifestPath, manifest);
|
||||
const outcome = await uploadContentAddressedFileToS3(
|
||||
s3,
|
||||
manifestPath,
|
||||
projectS3Uri,
|
||||
bytes: size,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
uploaded: true,
|
||||
};
|
||||
await sha256File(manifestPath),
|
||||
"application/json",
|
||||
);
|
||||
uploaded ||= outcome === "uploaded";
|
||||
} finally {
|
||||
rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function headObject(
|
||||
s3: S3Client,
|
||||
bucket: string,
|
||||
key: string,
|
||||
): Promise<{ bytes: number; lastModified: string } | null> {
|
||||
try {
|
||||
const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||
return {
|
||||
bytes: typeof res.ContentLength === "number" ? res.ContentLength : 0,
|
||||
lastModified:
|
||||
res.LastModified instanceof Date
|
||||
? res.LastModified.toISOString()
|
||||
: new Date().toISOString(),
|
||||
};
|
||||
} catch (err) {
|
||||
// The SDK throws different error shapes for 404 vs 403 vs network;
|
||||
// a 404 means "needs upload" and is the most common case. Anything
|
||||
// else propagates so callers see auth / network failures.
|
||||
const status = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
|
||||
if (status === 404) return null;
|
||||
const name = (err as { name?: string }).name;
|
||||
if (name === "NotFound" || name === "NoSuchKey") return null;
|
||||
throw err;
|
||||
}
|
||||
return {
|
||||
siteId,
|
||||
bucketName: opts.bucketName,
|
||||
projectS3Uri,
|
||||
bytes: stored.reduce((sum, file) => sum + file.sizeBytes, 0),
|
||||
uploadedAt: new Date().toISOString(),
|
||||
uploaded,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -8,7 +8,12 @@
|
||||
* AWS-SDK only — safe to load in any Node environment.
|
||||
*/
|
||||
|
||||
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./deploySite.js";
|
||||
export {
|
||||
deploySite,
|
||||
type DeploySiteOptions,
|
||||
type SiteHandle,
|
||||
siteHandleFromId,
|
||||
} from "./deploySite.js";
|
||||
export { renderToLambda, type RenderHandle, type RenderToLambdaOptions } from "./renderToLambda.js";
|
||||
export {
|
||||
getRenderProgress,
|
||||
|
||||
@@ -76,9 +76,7 @@ describe("renderToLambda", () => {
|
||||
expect(handle.executionArn).toContain("smoke-1");
|
||||
expect(handle.bucketName).toBe("test-bucket");
|
||||
expect(handle.outputS3Uri).toBe("s3://test-bucket/renders/smoke-1/output.mp4");
|
||||
expect(handle.projectS3Uri).toMatch(
|
||||
/^s3:\/\/test-bucket\/sites\/[0-9a-f]{16}\/project\.tar\.gz$/,
|
||||
);
|
||||
expect(handle.projectS3Uri).toMatch(/^s3:\/\/test-bucket\/sites\/[0-9a-f]{16}\/files\.json$/);
|
||||
expect(Object.keys(handle)).toEqual([
|
||||
"renderId",
|
||||
"executionArn",
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* A deployed site is `sites/<siteId>/files.json`: the project's files by path, each stored once
|
||||
* in the shared content store under its SHA-256 (see {@link contentStoreUri}).
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import { contentStoreUri } from "./contentStore.js";
|
||||
import {
|
||||
downloadS3ObjectToFile,
|
||||
downloadS3ObjectToFileVerified,
|
||||
formatS3Uri,
|
||||
mapConcurrent,
|
||||
parseS3Uri,
|
||||
} from "./s3Transport.js";
|
||||
|
||||
const SITE_MANIFEST_NAME = "files.json";
|
||||
|
||||
export interface SiteFile {
|
||||
/** `/`-separated, relative to the project root. */
|
||||
readonly path: string;
|
||||
readonly sha256: string;
|
||||
readonly sizeBytes: number;
|
||||
}
|
||||
|
||||
export function siteManifestUri(bucket: string, siteId: string): string {
|
||||
return formatS3Uri({ bucket, key: `sites/${siteId}/${SITE_MANIFEST_NAME}` });
|
||||
}
|
||||
|
||||
export function isSiteManifestUri(uri: string): boolean {
|
||||
return uri.endsWith(`/${SITE_MANIFEST_NAME}`);
|
||||
}
|
||||
|
||||
export function serializeSiteManifest(files: readonly SiteFile[]): string {
|
||||
return JSON.stringify({ files });
|
||||
}
|
||||
|
||||
export function parseSiteManifest(text: string): SiteFile[] {
|
||||
const value = JSON.parse(text) as { files?: unknown };
|
||||
if (!Array.isArray(value.files)) throw siteManifestError("files must be an array");
|
||||
const seen = new Set<string>();
|
||||
return value.files.map((entry: Partial<SiteFile>, index) => {
|
||||
const { path, sha256, sizeBytes } = entry ?? {};
|
||||
if (typeof path !== "string" || !isSafeRelativePath(path) || seen.has(path)) {
|
||||
throw siteManifestError(`files[${index}].path is not a unique relative path`);
|
||||
}
|
||||
if (typeof sha256 !== "string" || !/^[a-f0-9]{64}$/.test(sha256)) {
|
||||
throw siteManifestError(`files[${index}].sha256 is not a lowercase SHA-256 digest`);
|
||||
}
|
||||
if (!Number.isSafeInteger(sizeBytes) || (sizeBytes as number) < 0) {
|
||||
throw siteManifestError(`files[${index}].sizeBytes is not a byte count`);
|
||||
}
|
||||
seen.add(path);
|
||||
return { path, sha256, sizeBytes: sizeBytes as number };
|
||||
});
|
||||
}
|
||||
|
||||
/** Rebuild the project under `projectDir`, verifying every file's SHA-256 on the way. */
|
||||
export async function downloadSiteFiles(
|
||||
s3: S3Client,
|
||||
manifestUri: string,
|
||||
projectDir: string,
|
||||
): Promise<void> {
|
||||
const manifestPath = `${projectDir}.${SITE_MANIFEST_NAME}`;
|
||||
await downloadS3ObjectToFile(s3, manifestUri, manifestPath);
|
||||
const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));
|
||||
const { bucket } = parseS3Uri(manifestUri);
|
||||
await mapConcurrent(files, 16, (file) =>
|
||||
downloadS3ObjectToFileVerified(
|
||||
s3,
|
||||
contentStoreUri(bucket, file.sha256),
|
||||
join(projectDir, ...file.path.split("/")),
|
||||
file.sha256,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function isSafeRelativePath(path: string): boolean {
|
||||
return (
|
||||
path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..") &&
|
||||
!path.includes("\\")
|
||||
);
|
||||
}
|
||||
|
||||
function siteManifestError(detail: string): Error {
|
||||
const error = new Error(`[siteManifest] SITE_MANIFEST_INVALID: ${detail}`);
|
||||
error.name = "SITE_MANIFEST_INVALID";
|
||||
return error;
|
||||
}
|
||||
Reference in New Issue
Block a user