diff --git a/packages/aws-lambda/src/s3Transport.ts b/packages/aws-lambda/src/s3Transport.ts index 4a1e836016..78d33993ac 100644 --- a/packages/aws-lambda/src/s3Transport.ts +++ b/packages/aws-lambda/src/s3Transport.ts @@ -287,6 +287,28 @@ export async function putContentStoreFile( return outcome; } +/** Confirm an object the caller never uploaded is in the content store, restarting an aging one. */ +export async function ensureContentStoreObject( + client: S3Client, + bucket: string, + sha256: string, + sizeBytes: number, +): Promise { + const uri = contentStoreUri(bucket, sha256); + const { key } = parseS3Uri(uri); + const existing = await inspectContentAddressedObject(client, bucket, key, sizeBytes, sha256); + if (existing.state === "matching") { + if (existing.lastModified) await refreshAgingObject(client, uri, sha256, existing.lastModified); + return; + } + if (existing.state === "conflict") throwImmutableObjectConflict(uri); + const error = new Error( + `[s3Transport] PLAN_ARTIFACT_MISSING: ${uri} is not in the content store`, + ); + error.name = "PLAN_ARTIFACT_MISSING"; + throw error; +} + // S3 lifecycle expiry counts from the last write, not the last read, so reuse has to write. async function refreshAgingObject( client: S3Client, diff --git a/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts b/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts index 81283bfe35..7b6b05d0de 100644 --- a/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts +++ b/packages/aws-lambda/src/sdk/__fixtures__/fakeS3.ts @@ -8,6 +8,7 @@ * `deploySite`'s `finally` block has rmSync'd it. */ +import { Readable } from "node:stream"; import { buffer } from "node:stream/consumers"; import type { S3Client } from "@aws-sdk/client-s3"; @@ -26,26 +27,30 @@ export class FakeS3 { async send(command: unknown): Promise { const cmdName = (command as { constructor: { name: string } }).constructor.name; const input = ( - command as { input: { Bucket: string; Key: string; Metadata?: Record } } + command as { + input: { Bucket: string; Key: string; Metadata?: Record; Body?: unknown }; + } ).input; const id = `${input.Bucket}/${input.Key}`; - if (cmdName === "HeadObjectCommand") { - this.ops.push({ kind: "head", bucket: input.Bucket, key: input.Key }); - return this.head(id); - } - if (cmdName === "PutObjectCommand") { - this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key }); - this.existing.add(id); - this.metadata.set(id, input.Metadata); - this.bodies.set( - id, - await drainBody( - (command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body, - ), - ); - return {}; - } - throw new Error(`FakeS3: unexpected command ${cmdName}`); + const handlers: Record Promise> = { + HeadObjectCommand: async () => { + this.ops.push({ kind: "head", bucket: input.Bucket, key: input.Key }); + return this.head(id); + }, + GetObjectCommand: async () => ({ + Body: Readable.from([this.bodies.get(id) ?? Buffer.alloc(0)]), + }), + PutObjectCommand: async () => { + this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key }); + this.existing.add(id); + this.metadata.set(id, input.Metadata); + this.bodies.set(id, await drainBody(input.Body as NodeJS.ReadableStream | Buffer)); + return {}; + }, + }; + const handler = handlers[cmdName]; + if (!handler) throw new Error(`FakeS3: unexpected command ${cmdName}`); + return handler(); } private head(id: string): unknown { diff --git a/packages/aws-lambda/src/sdk/deploySite.ts b/packages/aws-lambda/src/sdk/deploySite.ts index 8cd98484f1..6feb22caff 100644 --- a/packages/aws-lambda/src/sdk/deploySite.ts +++ b/packages/aws-lambda/src/sdk/deploySite.ts @@ -11,18 +11,28 @@ * where `siteId` is derived from a SHA-256 over the project files. */ -import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { S3Client } from "@aws-sdk/client-s3"; +import { CONTENT_STORE_RETENTION_DAYS } from "../contentStore.js"; import { hashProjectDir, listSiteFiles } from "@hyperframes/producer/distributed"; import { + downloadS3ObjectToFile, + ensureContentStoreObject, mapConcurrent, + parseS3Uri, putContentStoreFile, sha256File, uploadContentAddressedFileToS3, } from "../s3Transport.js"; -import { serializeSiteManifest, siteManifestUri, type SiteFile } from "../siteManifest.js"; +import { + isSiteManifestUri, + parseSiteManifest, + serializeSiteManifest, + siteManifestUri, + type SiteFile, +} from "../siteManifest.js"; /** Options for {@link deploySite}. */ export interface DeploySiteOptions { @@ -64,6 +74,43 @@ export function siteHandleFromId(bucketName: string, siteId: string): SiteHandle return { siteId, bucketName, projectS3Uri, bytes: 0, uploadedAt: "", uploaded: false }; } +/** Before a render reuses `site`, re-store any file the content store expired, from `projectDir`. */ +export async function ensureSiteStored( + site: SiteHandle, + s3: S3Client, + projectDir?: string, +): Promise { + if (!isSiteManifestUri(site.projectS3Uri)) return; + const { bucket } = parseS3Uri(site.projectS3Uri); + const workdir = mkdtempSync(join(tmpdir(), "hf-site-check-")); + try { + const manifestPath = join(workdir, "files.json"); + await downloadS3ObjectToFile(s3, site.projectS3Uri, manifestPath); + const files = parseSiteManifest(readFileSync(manifestPath, "utf-8")); + const missing: string[] = []; + await mapConcurrent(files, 16, async ({ path, sha256, sizeBytes }) => { + try { + await ensureContentStoreObject(s3, bucket, sha256, sizeBytes); + } catch (error) { + if ((error as Error).name !== "PLAN_ARTIFACT_MISSING") throw error; + if (!projectDir) return void missing.push(path); + await putContentStoreFile(s3, join(projectDir, path), bucket, sha256); + } + }); + if (missing.length > 0) { + const error = new Error( + `[deploySite] SITE_FILES_MISSING: ${missing.length} of ${files.length} files of site ` + + `${site.siteId} are no longer stored (unused for ${CONTENT_STORE_RETENTION_DAYS} days). ` + + "Run `hyperframes lambda sites create`, or pass projectDir to re-upload them.", + ); + error.name = "SITE_FILES_MISSING"; + throw error; + } + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +} + /** Upload `projectDir`'s files to the content store and write the site manifest. */ export async function deploySite(opts: DeploySiteOptions): Promise { const s3 = opts.s3 ?? new S3Client({ region: opts.region }); diff --git a/packages/aws-lambda/src/sdk/renderToLambda.test.ts b/packages/aws-lambda/src/sdk/renderToLambda.test.ts index 01964b3631..f087341d91 100644 --- a/packages/aws-lambda/src/sdk/renderToLambda.test.ts +++ b/packages/aws-lambda/src/sdk/renderToLambda.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import type { SFNClient } from "@aws-sdk/client-sfn"; import type { SerializableDistributedRenderConfig } from "../events.js"; import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js"; -import type { SiteHandle } from "./deploySite.js"; +import { deploySite, type SiteHandle } from "./deploySite.js"; import { renderToLambda } from "./renderToLambda.js"; import { InvalidConfigError } from "./validateConfig.js"; @@ -163,6 +163,48 @@ describe("renderToLambda", () => { expect(s3.existing.size).toBe(0); }); + describe("with a site whose files expired from the content store", () => { + async function deployThenExpire(s3: FakeS3): Promise<{ site: SiteHandle; expiredKey: string }> { + const site = await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) }); + const expiredKey = [...s3.existing].find((id) => id.includes("/cas/sha256/"))!; + s3.existing.delete(expiredKey); + s3.ops.length = 0; + return { site, expiredKey }; + } + const renderWith = (s3: FakeS3, sfn: FakeSFN, site: SiteHandle, dir?: string) => + renderToLambda({ + siteHandle: site, + projectDir: dir, + bucketName: "test-bucket", + stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf", + config: baseConfig, + sfn: asSFNClient(sfn), + s3: asS3Client(s3), + }); + + it("re-uploads the expired file from the project dir, then renders", async () => { + const s3 = new FakeS3(); + const sfn = new FakeSFN(); + const { site, expiredKey } = await deployThenExpire(s3); + await renderWith(s3, sfn, site, projectDir); + expect( + s3.ops.filter((op) => op.kind === "put").map((op) => `${op.bucket}/${op.key}`), + ).toEqual([expiredKey]); + expect(sfn.starts).toHaveLength(1); + }); + + it("without a project dir, refuses with the missing-file count before starting", async () => { + const s3 = new FakeS3(); + const sfn = new FakeSFN(); + const { site } = await deployThenExpire(s3); + await expect(renderWith(s3, sfn, site)).rejects.toMatchObject({ + name: "SITE_FILES_MISSING", + message: expect.stringContaining("1 of 1 files"), + }); + expect(sfn.starts).toHaveLength(0); + }); + }); + it("rejects invalid configs synchronously before any AWS call", async () => { const sfn = new FakeSFN(); const s3 = new FakeS3(); diff --git a/packages/aws-lambda/src/sdk/renderToLambda.ts b/packages/aws-lambda/src/sdk/renderToLambda.ts index 1232e68761..054f240a1f 100644 --- a/packages/aws-lambda/src/sdk/renderToLambda.ts +++ b/packages/aws-lambda/src/sdk/renderToLambda.ts @@ -19,11 +19,11 @@ import { randomUUID } from "node:crypto"; import { SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn"; -import type { S3Client } from "@aws-sdk/client-s3"; +import { S3Client } from "@aws-sdk/client-s3"; import type { LambdaPlanProtocol, SerializableDistributedRenderConfig } from "../events.js"; import { formatExtension } from "../formatExtension.js"; import { formatS3Uri } from "../s3Transport.js"; -import { deploySite, type SiteHandle } from "./deploySite.js"; +import { deploySite, ensureSiteStored, type SiteHandle } from "./deploySite.js"; import { validateDistributedRenderConfig, validateStepFunctionsInputSize, @@ -31,7 +31,7 @@ import { /** Options for {@link renderToLambda}. */ export interface RenderToLambdaOptions { - /** Local project directory. Required when `siteHandle` is not supplied. */ + /** Local project directory. Required without `siteHandle`; with one, re-uploads expired files. */ projectDir?: string; /** Re-use an existing `deploySite` upload (skips tar+S3 PUT). */ siteHandle?: SiteHandle; @@ -101,6 +101,10 @@ export async function renderToLambda(opts: RenderToLambdaOptions): Promise { } try { const handle = await renderToLambda({ + projectDir, siteHandle, bucketName: stack.bucketName, stateMachineArn: stack.stateMachineArn, diff --git a/packages/cli/src/commands/lambda/render.ts b/packages/cli/src/commands/lambda/render.ts index c15a372fde..d8b2cae4c8 100644 --- a/packages/cli/src/commands/lambda/render.ts +++ b/packages/cli/src/commands/lambda/render.ts @@ -131,7 +131,7 @@ export async function runRender(args: RenderArgs): Promise { const siteHandle = args.siteId ? siteHandleFromId(stack.bucketName, args.siteId) : undefined; const handle = await renderToLambda({ - projectDir: siteHandle ? undefined : projectDir, + projectDir, siteHandle, bucketName: stack.bucketName, stateMachineArn: stack.stateMachineArn,