mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(aws-lambda): push a prebuilt image to ECR without Docker (#6063)
* feat(aws-lambda): push a prebuilt image to ECR without Docker Adds pushImageToEcr and ensureEcrRepository to the SDK subpath. The push reads a single-platform OCI image layout, verifies every blob against its descriptor before any ECR call, uploads only the blobs ECR lacks through the layer-upload API, and puts the manifest under a tag, returning the digest-pinned image URI. * fix(aws-lambda): keep the ECR client out of the SDK bundle Like the other AWS SDK clients, @aws-sdk/client-ecr is a dependency the consumer resolves. Bundled into the ESM SDK it hits a dynamic require of node:https and the packed package fails to import. * feat(aws-lambda): build the ECR client from a region like the other SDK calls * build(aws-lambda): add only the ECR client to the lockfile * fix(aws-lambda): adopt an existing ECR repository only when it carries the caller's tags * chore(aws-lambda): raise the size budget for pushing an image to ECR pushImageToEcr ships in dist (unminified chunk plus its .d.ts); no source maps, tests or release-only files are involved. Built dist, npm pack --dry-run, main versus main plus this branch: +2486 B packed, +8846 B unpacked. The merge-group run measured 73369 packed and 266805 unpacked, which are the new budgets (were 70874 and 258166).
This commit is contained in:
Miguel Ángel
@@ -29,6 +29,7 @@
|
||||
"name": "@hyperframes/aws-lambda",
|
||||
"version": "0.8.169",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-ecr": "^3.1146.0",
|
||||
"@aws-sdk/client-lambda": "^3.1146.0",
|
||||
"@aws-sdk/client-s3": "^3.1146.0",
|
||||
"@aws-sdk/client-sfn": "^3.1146.0",
|
||||
@@ -444,6 +445,8 @@
|
||||
|
||||
"@aws-sdk/checksums": ["@aws-sdk/checksums@3.1001.1", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-x12Q17KYlJAd3nKf8LV5LV0vt8sh8/6YfQLGPtrGnQf/tW4jqxPGq5GPpuVitpQYM3eUR4XB7CbxZf751NMbLw=="],
|
||||
|
||||
"@aws-sdk/client-ecr": ["@aws-sdk/client-ecr@3.1146.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/credential-provider-node": "^3.972.84", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-6SxQK33Jye0WHjOld95b9S12eHIYtPfpZhuAv9HYCMQYJJyVDhsV8BNWwsNjLEchTc/yjS8ksmzc8MhApDcD3g=="],
|
||||
|
||||
"@aws-sdk/client-lambda": ["@aws-sdk/client-lambda@3.1146.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/credential-provider-node": "^3.972.84", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-qCeVHGhkI5aw0Lvexhwuf1H41L2sCm+9C4K+IOUhErY/nx5ioCU0dFyXF4+P4m4Nk23uiTbON7lI7nmlPNncHw=="],
|
||||
|
||||
"@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.1146.0", "", { "dependencies": { "@aws-sdk/checksums": "^3.1001.1", "@aws-sdk/core": "^3.978.1", "@aws-sdk/credential-provider-node": "^3.972.84", "@aws-sdk/middleware-sdk-s3": "^3.972.77", "@aws-sdk/signature-v4-multi-region": "^3.996.47", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-WY0YCBzxc4muFfY6UbGpA+oib0nGT/Px2aEPGYX7pp1la2OmDLChnK81mu8H7LdYvwGU1pLAsLIa/qgad0Wd7g=="],
|
||||
|
||||
@@ -32,6 +32,7 @@ const sharedOpts = {
|
||||
format: "esm",
|
||||
minify: false,
|
||||
external: [
|
||||
"@aws-sdk/client-ecr",
|
||||
"@aws-sdk/client-lambda",
|
||||
"@aws-sdk/client-s3",
|
||||
"@aws-sdk/client-sfn",
|
||||
|
||||
@@ -72,6 +72,7 @@
|
||||
"verify:zip-size": "tsx scripts/verify-zip-size.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-ecr": "^3.1146.0",
|
||||
"@aws-sdk/client-lambda": "^3.1146.0",
|
||||
"@aws-sdk/client-s3": "^3.1146.0",
|
||||
"@aws-sdk/client-sfn": "^3.1146.0",
|
||||
|
||||
@@ -45,5 +45,12 @@ export {
|
||||
prebuiltVideoImage,
|
||||
type ReleaseAssetManifest,
|
||||
} from "./prebuiltHandler.js";
|
||||
export {
|
||||
ensureEcrRepository,
|
||||
type EnsureEcrRepositoryOptions,
|
||||
type PushedImage,
|
||||
pushImageToEcr,
|
||||
type PushImageToEcrOptions,
|
||||
} from "./pushImageToEcr.js";
|
||||
export type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
export type { DistributedFormat } from "../formatExtension.js";
|
||||
|
||||
@@ -0,0 +1,367 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ECRClient } from "@aws-sdk/client-ecr";
|
||||
import { ensureEcrRepository, pushImageToEcr } from "./pushImageToEcr.js";
|
||||
|
||||
const OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json";
|
||||
const OCI_INDEX = "application/vnd.oci.image.index.v1+json";
|
||||
const REGISTRY = "123456789012.dkr.ecr.us-east-1.amazonaws.com";
|
||||
const PART_SIZE = 7;
|
||||
// 21 bytes is exactly three parts; 17 bytes ends with a short part.
|
||||
const LAYER_A = Buffer.from("layer-a: 21 bytes!!!!");
|
||||
const LAYER_B = Buffer.from("layer-b: 17 bytes");
|
||||
|
||||
interface Descriptor {
|
||||
mediaType: string;
|
||||
digest: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
function sha256(bytes: Buffer | string): string {
|
||||
return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
||||
}
|
||||
|
||||
function awsError(name: string): Error {
|
||||
return Object.assign(new Error(name), { name });
|
||||
}
|
||||
|
||||
/** In-memory ECR that enforces the upload rules real ECR enforces. */
|
||||
class FakeEcr {
|
||||
calls: string[] = [];
|
||||
blobs = new Map<string, Buffer>();
|
||||
tags = new Map<string, string>();
|
||||
repositories = new Map<string, string>();
|
||||
repositoryTags = new Map<string, unknown>();
|
||||
/** Digests BatchCheckLayerAvailability reports missing even though stored (a concurrent push). */
|
||||
hiddenFromCheck = new Set<string>();
|
||||
private uploads = new Map<string, Buffer[]>();
|
||||
|
||||
async send(command: unknown): Promise<unknown> {
|
||||
const name = (command as { constructor: { name: string } }).constructor.name;
|
||||
const input = (command as { input: unknown }).input;
|
||||
this.calls.push(name);
|
||||
const handler = (this as unknown as Record<string, (input: unknown) => unknown>)[name];
|
||||
if (!handler) throw new Error(`FakeEcr: unexpected command ${name}`);
|
||||
return handler.call(this, input);
|
||||
}
|
||||
|
||||
CreateRepositoryCommand(input: { repositoryName: string; tags?: unknown }): unknown {
|
||||
if (this.repositories.has(input.repositoryName)) {
|
||||
throw awsError("RepositoryAlreadyExistsException");
|
||||
}
|
||||
const repositoryUri = `${REGISTRY}/${input.repositoryName}`;
|
||||
this.repositories.set(input.repositoryName, repositoryUri);
|
||||
this.repositoryTags.set(input.repositoryName, input.tags);
|
||||
return { repository: { repositoryUri } };
|
||||
}
|
||||
|
||||
DescribeRepositoriesCommand(input: { repositoryNames: string[] }): unknown {
|
||||
return {
|
||||
repositories: input.repositoryNames.map((name) => ({
|
||||
repositoryArn: `arn:aws:ecr:r:1:repository/${name}`,
|
||||
repositoryUri: this.repositories.get(name),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
ListTagsForResourceCommand(input: { resourceArn: string }): unknown {
|
||||
return { tags: this.repositoryTags.get(input.resourceArn.split("/").pop() ?? "") ?? [] };
|
||||
}
|
||||
|
||||
BatchCheckLayerAvailabilityCommand(input: { layerDigests: string[] }): unknown {
|
||||
if (input.layerDigests.length > 100) throw awsError("InvalidParameterException");
|
||||
const stored = (d: string) => this.blobs.has(d) && !this.hiddenFromCheck.has(d);
|
||||
return {
|
||||
layers: input.layerDigests
|
||||
.filter(stored)
|
||||
.map((layerDigest) => ({ layerDigest, layerAvailability: "AVAILABLE" })),
|
||||
failures: input.layerDigests
|
||||
.filter((d) => !stored(d))
|
||||
.map((layerDigest) => ({ layerDigest, failureCode: "MissingLayerDigest" })),
|
||||
};
|
||||
}
|
||||
|
||||
InitiateLayerUploadCommand(): unknown {
|
||||
const uploadId = `upload-${this.uploads.size}`;
|
||||
this.uploads.set(uploadId, []);
|
||||
return { uploadId, partSize: PART_SIZE };
|
||||
}
|
||||
|
||||
UploadLayerPartCommand(input: {
|
||||
uploadId: string;
|
||||
partFirstByte: number;
|
||||
partLastByte: number;
|
||||
layerPartBlob: Uint8Array;
|
||||
}): unknown {
|
||||
const parts = this.uploads.get(input.uploadId);
|
||||
if (!parts) throw awsError("UploadNotFoundException");
|
||||
const received = parts.reduce((sum, part) => sum + part.length, 0);
|
||||
const size = input.layerPartBlob.length;
|
||||
const contiguous = input.partFirstByte === received;
|
||||
const lastByteMatches = input.partLastByte === input.partFirstByte + size - 1;
|
||||
const previousWasFull = parts.every((part) => part.length === PART_SIZE);
|
||||
if (!contiguous || !lastByteMatches || size > PART_SIZE || !previousWasFull) {
|
||||
throw awsError("InvalidLayerPartException");
|
||||
}
|
||||
parts.push(Buffer.from(input.layerPartBlob));
|
||||
return { uploadId: input.uploadId, lastByteReceived: input.partLastByte };
|
||||
}
|
||||
|
||||
CompleteLayerUploadCommand(input: { uploadId: string; layerDigests: string[] }): unknown {
|
||||
const parts = this.uploads.get(input.uploadId);
|
||||
if (!parts) throw awsError("UploadNotFoundException");
|
||||
this.uploads.delete(input.uploadId);
|
||||
const bytes = Buffer.concat(parts);
|
||||
const [digest] = input.layerDigests;
|
||||
if (input.layerDigests.length !== 1 || sha256(bytes) !== digest) {
|
||||
throw awsError("InvalidLayerException");
|
||||
}
|
||||
if (this.blobs.has(digest)) throw awsError("LayerAlreadyExistsException");
|
||||
this.blobs.set(digest, bytes);
|
||||
return { layerDigest: digest };
|
||||
}
|
||||
|
||||
PutImageCommand(input: {
|
||||
imageManifest: string;
|
||||
imageTag: string;
|
||||
imageDigest: string;
|
||||
}): unknown {
|
||||
if (sha256(input.imageManifest) !== input.imageDigest)
|
||||
throw awsError("InvalidParameterException");
|
||||
const manifest = JSON.parse(input.imageManifest) as {
|
||||
config: Descriptor;
|
||||
layers: Descriptor[];
|
||||
};
|
||||
const referenced = [manifest.config, ...manifest.layers].map((blob) => blob.digest);
|
||||
if (referenced.some((digest) => !this.blobs.has(digest))) {
|
||||
throw awsError("LayersNotFoundException");
|
||||
}
|
||||
if (this.tags.get(input.imageTag) === input.imageDigest) {
|
||||
throw awsError("ImageAlreadyExistsException");
|
||||
}
|
||||
this.tags.set(input.imageTag, input.imageDigest);
|
||||
return { image: { imageId: { imageDigest: input.imageDigest, imageTag: input.imageTag } } };
|
||||
}
|
||||
}
|
||||
|
||||
function asEcr(fake: FakeEcr): ECRClient {
|
||||
return fake as unknown as ECRClient;
|
||||
}
|
||||
|
||||
let layoutDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
layoutDir = mkdtempSync(join(tmpdir(), "hf-ecr-layout-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(layoutDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function putBlob(mediaType: string, bytes: Buffer): Descriptor {
|
||||
mkdirSync(join(layoutDir, "blobs", "sha256"), { recursive: true });
|
||||
const digest = sha256(bytes);
|
||||
writeFileSync(join(layoutDir, "blobs", "sha256", digest.slice("sha256:".length)), bytes);
|
||||
return { mediaType, digest, size: bytes.length };
|
||||
}
|
||||
|
||||
function writeIndex(manifests: Descriptor[]): void {
|
||||
writeFileSync(
|
||||
join(layoutDir, "index.json"),
|
||||
JSON.stringify({ schemaVersion: 2, mediaType: OCI_INDEX, manifests }),
|
||||
);
|
||||
}
|
||||
|
||||
function writeLayout(
|
||||
layers: Buffer[] = [LAYER_A, LAYER_B],
|
||||
platform: Record<string, string> = { architecture: "arm64", os: "linux" },
|
||||
) {
|
||||
const config = putBlob(
|
||||
"application/vnd.oci.image.config.v1+json",
|
||||
Buffer.from(JSON.stringify({ ...platform, rootfs: { type: "layers" } })),
|
||||
);
|
||||
const layerDescriptors = layers.map((bytes) =>
|
||||
putBlob("application/vnd.oci.image.layer.v1.tar+gzip", bytes),
|
||||
);
|
||||
const manifest = putBlob(
|
||||
OCI_MANIFEST,
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
schemaVersion: 2,
|
||||
mediaType: OCI_MANIFEST,
|
||||
config,
|
||||
layers: layerDescriptors,
|
||||
}),
|
||||
),
|
||||
);
|
||||
writeIndex([manifest]);
|
||||
return { config, manifest, layers: layerDescriptors };
|
||||
}
|
||||
|
||||
function push(ecr: FakeEcr) {
|
||||
return pushImageToEcr({
|
||||
ecr: asEcr(ecr),
|
||||
layoutDir,
|
||||
repositoryName: "hyperframes-render",
|
||||
repositoryUri: `${REGISTRY}/hyperframes-render`,
|
||||
tag: "v1",
|
||||
});
|
||||
}
|
||||
|
||||
describe("pushImageToEcr", () => {
|
||||
it("uploads every blob in partSize parts and puts the manifest under the tag", async () => {
|
||||
const { config, manifest, layers } = writeLayout();
|
||||
const ecr = new FakeEcr();
|
||||
|
||||
const result = await push(ecr);
|
||||
|
||||
expect(result).toEqual({
|
||||
imageUri: `${REGISTRY}/hyperframes-render@${manifest.digest}`,
|
||||
digest: manifest.digest,
|
||||
architecture: "arm64",
|
||||
os: "linux",
|
||||
});
|
||||
expect(ecr.blobs.get(layers[0]!.digest)).toEqual(LAYER_A);
|
||||
expect(ecr.blobs.get(layers[1]!.digest)).toEqual(LAYER_B);
|
||||
expect(ecr.blobs.has(config.digest)).toBe(true);
|
||||
expect(ecr.tags.get("v1")).toBe(manifest.digest);
|
||||
const parts = ecr.calls.filter((call) => call === "UploadLayerPartCommand").length;
|
||||
expect(parts).toBe(Math.ceil(config.size / PART_SIZE) + 3 + 3);
|
||||
expect(ecr.calls[0]).toBe("BatchCheckLayerAvailabilityCommand");
|
||||
expect(ecr.calls.at(-1)).toBe("PutImageCommand");
|
||||
});
|
||||
|
||||
it("skips blobs ECR already has", async () => {
|
||||
const { config, layers } = writeLayout();
|
||||
const ecr = new FakeEcr();
|
||||
ecr.blobs.set(config.digest, Buffer.from("stored"));
|
||||
ecr.blobs.set(layers[0]!.digest, LAYER_A);
|
||||
|
||||
await push(ecr);
|
||||
|
||||
expect(ecr.calls.filter((call) => call === "InitiateLayerUploadCommand")).toHaveLength(1);
|
||||
expect(ecr.blobs.get(layers[1]!.digest)).toEqual(LAYER_B);
|
||||
});
|
||||
|
||||
it("checks availability in batches of at most 100 digests", async () => {
|
||||
writeLayout(Array.from({ length: 101 }, (_, i) => Buffer.from(`layer ${i}`)));
|
||||
const ecr = new FakeEcr();
|
||||
|
||||
await push(ecr);
|
||||
|
||||
expect(ecr.calls.filter((call) => call === "BatchCheckLayerAvailabilityCommand")).toHaveLength(
|
||||
2,
|
||||
);
|
||||
expect(ecr.blobs.size).toBe(102);
|
||||
});
|
||||
|
||||
it("refuses a tampered blob before making any ECR call", async () => {
|
||||
const { layers } = writeLayout();
|
||||
const tampered = Buffer.from(LAYER_B);
|
||||
tampered[0] = tampered[0]! ^ 1;
|
||||
writeFileSync(join(layoutDir, "blobs", "sha256", layers[1]!.digest.slice(7)), tampered);
|
||||
const ecr = new FakeEcr();
|
||||
|
||||
await expect(push(ecr)).rejects.toThrow(/does not match its descriptor/);
|
||||
expect(ecr.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses an index listing two manifests", async () => {
|
||||
const { manifest } = writeLayout();
|
||||
const attestation = putBlob(OCI_MANIFEST, Buffer.from("{}"));
|
||||
writeIndex([manifest, attestation]);
|
||||
const ecr = new FakeEcr();
|
||||
|
||||
await expect(push(ecr)).rejects.toThrow(/exactly one single-platform image manifest; found 2/);
|
||||
expect(ecr.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses a multi-platform image index", async () => {
|
||||
const { manifest } = writeLayout();
|
||||
writeIndex([{ ...manifest, mediaType: OCI_INDEX }]);
|
||||
const ecr = new FakeEcr();
|
||||
|
||||
await expect(push(ecr)).rejects.toThrow(`found 1 (${OCI_INDEX})`);
|
||||
expect(ecr.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses an image config with no architecture", async () => {
|
||||
writeLayout([LAYER_A], { os: "linux" });
|
||||
const ecr = new FakeEcr();
|
||||
|
||||
await expect(push(ecr)).rejects.toThrow(/names no architecture\/os/);
|
||||
expect(ecr.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("treats LayerAlreadyExists on complete as uploaded", async () => {
|
||||
const { layers, manifest } = writeLayout();
|
||||
const ecr = new FakeEcr();
|
||||
ecr.blobs.set(layers[0]!.digest, LAYER_A);
|
||||
ecr.hiddenFromCheck.add(layers[0]!.digest);
|
||||
|
||||
const result = await push(ecr);
|
||||
|
||||
expect(result.digest).toBe(manifest.digest);
|
||||
expect(ecr.tags.get("v1")).toBe(manifest.digest);
|
||||
});
|
||||
|
||||
it("redeploying the same image uploads nothing and succeeds", async () => {
|
||||
const { manifest } = writeLayout();
|
||||
const ecr = new FakeEcr();
|
||||
await push(ecr);
|
||||
ecr.calls = [];
|
||||
|
||||
const result = await push(ecr);
|
||||
|
||||
expect(result.digest).toBe(manifest.digest);
|
||||
expect(ecr.calls).toEqual(["BatchCheckLayerAvailabilityCommand", "PutImageCommand"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ensureEcrRepository", () => {
|
||||
it("creates the repository with its tags", async () => {
|
||||
const ecr = new FakeEcr();
|
||||
const tags = [{ Key: "app", Value: "hyperframes" }];
|
||||
|
||||
const { repositoryUri } = await ensureEcrRepository({
|
||||
ecr: asEcr(ecr),
|
||||
repositoryName: "hyperframes-render",
|
||||
tags,
|
||||
});
|
||||
|
||||
expect(repositoryUri).toBe(`${REGISTRY}/hyperframes-render`);
|
||||
expect(ecr.calls).toEqual(["CreateRepositoryCommand"]);
|
||||
expect(ecr.repositoryTags.get("hyperframes-render")).toEqual(tags);
|
||||
});
|
||||
|
||||
it("returns the existing repository's uri", async () => {
|
||||
const ecr = new FakeEcr();
|
||||
ecr.repositories.set("hyperframes-render", `${REGISTRY}/hyperframes-render`);
|
||||
|
||||
const { repositoryUri } = await ensureEcrRepository({
|
||||
ecr: asEcr(ecr),
|
||||
repositoryName: "hyperframes-render",
|
||||
});
|
||||
|
||||
expect(repositoryUri).toBe(`${REGISTRY}/hyperframes-render`);
|
||||
expect(ecr.calls).toEqual(["CreateRepositoryCommand", "DescribeRepositoriesCommand"]);
|
||||
});
|
||||
|
||||
it("adopts an existing repository only when it carries the tags", async () => {
|
||||
const ecr = new FakeEcr();
|
||||
const tags = [{ Key: "hyperframes:stack", Value: "s" }];
|
||||
await ensureEcrRepository({ ecr: asEcr(ecr), repositoryName: "s-video", tags });
|
||||
|
||||
const again = await ensureEcrRepository({ ecr: asEcr(ecr), repositoryName: "s-video", tags });
|
||||
expect(again.repositoryUri).toBe(`${REGISTRY}/s-video`);
|
||||
|
||||
ecr.repositories.set("other-video", `${REGISTRY}/other-video`);
|
||||
ecr.repositoryTags.set("other-video", [{ Key: "hyperframes:stack", Value: "other" }]);
|
||||
await expect(
|
||||
ensureEcrRepository({ ecr: asEcr(ecr), repositoryName: "other-video", tags }),
|
||||
).rejects.toThrow("other-video already exists without hyperframes:stack=s");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* Push a prebuilt single-platform OCI image layout into ECR through the ECR API alone,
|
||||
* so deploying the container-image function needs neither Docker nor registry HTTP calls.
|
||||
*/
|
||||
|
||||
import { open, readFile, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
BatchCheckLayerAvailabilityCommand,
|
||||
CompleteLayerUploadCommand,
|
||||
CreateRepositoryCommand,
|
||||
DescribeRepositoriesCommand,
|
||||
ECRClient,
|
||||
InitiateLayerUploadCommand,
|
||||
ListTagsForResourceCommand,
|
||||
PutImageCommand,
|
||||
type Repository,
|
||||
type Tag,
|
||||
UploadLayerPartCommand,
|
||||
} from "@aws-sdk/client-ecr";
|
||||
import { sha256File } from "../s3Transport.js";
|
||||
|
||||
const MANIFEST_MEDIA_TYPES = new Set([
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
]);
|
||||
// ECR's BatchCheckLayerAvailability accepts at most 100 digests per call.
|
||||
const CHECK_BATCH_SIZE = 100;
|
||||
|
||||
interface Descriptor {
|
||||
mediaType: string;
|
||||
digest: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
/** Options for {@link ensureEcrRepository}. */
|
||||
export interface EnsureEcrRepositoryOptions {
|
||||
ecr?: ECRClient;
|
||||
region?: string;
|
||||
repositoryName: string;
|
||||
/** Set on create. An existing repository is adopted only if it already carries every one. */
|
||||
tags?: Tag[];
|
||||
}
|
||||
|
||||
/** Create the repository, or adopt an existing one that carries `tags`. */
|
||||
export async function ensureEcrRepository(
|
||||
opts: EnsureEcrRepositoryOptions,
|
||||
): Promise<{ repositoryUri: string }> {
|
||||
const { repositoryName, tags } = opts;
|
||||
const ecr = opts.ecr ?? new ECRClient({ region: opts.region });
|
||||
try {
|
||||
const created = await ecr.send(new CreateRepositoryCommand({ repositoryName, tags }));
|
||||
return { repositoryUri: repositoryUriOf(created.repository, repositoryName) };
|
||||
} catch (error) {
|
||||
if ((error as Error).name !== "RepositoryAlreadyExistsException") throw error;
|
||||
}
|
||||
const found = await ecr.send(
|
||||
new DescribeRepositoriesCommand({ repositoryNames: [repositoryName] }),
|
||||
);
|
||||
const repository = found.repositories?.[0];
|
||||
if (tags?.length) {
|
||||
const listed = await ecr.send(
|
||||
new ListTagsForResourceCommand({ resourceArn: repository?.repositoryArn }),
|
||||
);
|
||||
const has = new Map((listed.tags ?? []).map((tag) => [tag.Key, tag.Value]));
|
||||
const missing = tags.filter((tag) => has.get(tag.Key) !== tag.Value);
|
||||
if (missing.length > 0) {
|
||||
const wanted = missing.map((tag) => `${tag.Key}=${tag.Value}`).join(", ");
|
||||
throw new Error(
|
||||
`[ensureEcrRepository] ${repositoryName} already exists without ${wanted}; refusing to adopt a repository this caller did not create`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { repositoryUri: repositoryUriOf(repository, repositoryName) };
|
||||
}
|
||||
|
||||
function repositoryUriOf(repository: Repository | undefined, repositoryName: string): string {
|
||||
if (!repository?.repositoryUri) {
|
||||
throw new Error(`[ensureEcrRepository] ECR returned no repositoryUri for ${repositoryName}`);
|
||||
}
|
||||
return repository.repositoryUri;
|
||||
}
|
||||
|
||||
/** Options for {@link pushImageToEcr}. */
|
||||
export interface PushImageToEcrOptions {
|
||||
ecr?: ECRClient;
|
||||
region?: string;
|
||||
/** OCI image layout directory: `index.json` plus `blobs/sha256/<hex>`. */
|
||||
layoutDir: string;
|
||||
repositoryName: string;
|
||||
/** From {@link ensureEcrRepository}. */
|
||||
repositoryUri: string;
|
||||
tag: string;
|
||||
log?: (message: string) => void;
|
||||
}
|
||||
|
||||
/** Result of {@link pushImageToEcr}. */
|
||||
export interface PushedImage {
|
||||
/** `<repositoryUri>@sha256:<manifest digest>`, pinned to exactly the pushed bytes. */
|
||||
imageUri: string;
|
||||
digest: string;
|
||||
/** From the image config, so callers can assert the platform they expect. */
|
||||
architecture: string;
|
||||
os: string;
|
||||
}
|
||||
|
||||
/** Upload the layout's missing blobs, then put its manifest under `tag`. Safe to re-run. */
|
||||
export async function pushImageToEcr(opts: PushImageToEcrOptions): Promise<PushedImage> {
|
||||
const { layoutDir, repositoryName, log = () => {} } = opts;
|
||||
const ecr = opts.ecr ?? new ECRClient({ region: opts.region });
|
||||
const manifestDescriptor = await readManifestDescriptor(layoutDir);
|
||||
await verifyBlob(layoutDir, manifestDescriptor);
|
||||
const manifestText = await readFile(blobPath(layoutDir, manifestDescriptor.digest), "utf-8");
|
||||
const manifest = JSON.parse(manifestText) as { config?: Descriptor; layers?: Descriptor[] };
|
||||
if (!manifest.config || !Array.isArray(manifest.layers)) {
|
||||
throw new Error(
|
||||
`[pushImageToEcr] manifest ${manifestDescriptor.digest} has no config or layers`,
|
||||
);
|
||||
}
|
||||
const blobs = [manifest.config, ...manifest.layers];
|
||||
for (const blob of blobs) await verifyBlob(layoutDir, blob);
|
||||
const config = JSON.parse(
|
||||
await readFile(blobPath(layoutDir, manifest.config.digest), "utf-8"),
|
||||
) as { architecture?: string; os?: string };
|
||||
if (!config.architecture || !config.os) {
|
||||
throw new Error(
|
||||
`[pushImageToEcr] image config ${manifest.config.digest} names no architecture/os; ` +
|
||||
"build the image for one platform",
|
||||
);
|
||||
}
|
||||
|
||||
const missing = await unavailableDigests(ecr, repositoryName, blobs);
|
||||
// ponytail: sequential uploads; upload time on big layers is the ceiling, bounded parallel layers is the upgrade.
|
||||
for (const blob of blobs) {
|
||||
if (!missing.delete(blob.digest)) continue;
|
||||
log(`uploading ${blob.digest} (${blob.size} bytes)`);
|
||||
await uploadBlob(ecr, repositoryName, blobPath(layoutDir, blob.digest), blob);
|
||||
}
|
||||
|
||||
try {
|
||||
await ecr.send(
|
||||
new PutImageCommand({
|
||||
repositoryName,
|
||||
imageManifest: manifestText,
|
||||
imageManifestMediaType: manifestDescriptor.mediaType,
|
||||
imageTag: opts.tag,
|
||||
imageDigest: manifestDescriptor.digest,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if ((error as Error).name !== "ImageAlreadyExistsException") throw error;
|
||||
}
|
||||
return {
|
||||
imageUri: `${opts.repositoryUri}@${manifestDescriptor.digest}`,
|
||||
digest: manifestDescriptor.digest,
|
||||
architecture: config.architecture,
|
||||
os: config.os,
|
||||
};
|
||||
}
|
||||
|
||||
async function readManifestDescriptor(layoutDir: string): Promise<Descriptor> {
|
||||
const index = JSON.parse(await readFile(join(layoutDir, "index.json"), "utf-8")) as {
|
||||
manifests?: Descriptor[];
|
||||
};
|
||||
const manifests = index.manifests ?? [];
|
||||
const [only] = manifests;
|
||||
if (manifests.length !== 1 || !only || !MANIFEST_MEDIA_TYPES.has(only.mediaType)) {
|
||||
const found = manifests.map((m) => m.mediaType).join(", ") || "none";
|
||||
throw new Error(
|
||||
`[pushImageToEcr] ${layoutDir}/index.json must list exactly one single-platform image ` +
|
||||
`manifest; found ${manifests.length} (${found}). Build one platform with ` +
|
||||
"--provenance=false --sbom=false.",
|
||||
);
|
||||
}
|
||||
return only;
|
||||
}
|
||||
|
||||
function blobPath(layoutDir: string, digest: string): string {
|
||||
const hex = /^sha256:([a-f0-9]{64})$/.exec(digest)?.[1];
|
||||
if (!hex) throw new Error(`[pushImageToEcr] unsupported blob digest ${JSON.stringify(digest)}`);
|
||||
return join(layoutDir, "blobs", "sha256", hex);
|
||||
}
|
||||
|
||||
async function verifyBlob(layoutDir: string, blob: Descriptor): Promise<void> {
|
||||
const path = blobPath(layoutDir, blob.digest);
|
||||
const size = (await stat(path)).size;
|
||||
if (size !== blob.size || `sha256:${await sha256File(path)}` !== blob.digest) {
|
||||
throw new Error(
|
||||
`[pushImageToEcr] blob ${blob.digest} (${size} bytes, expected ${blob.size}) does not ` +
|
||||
"match its descriptor; the image layout is corrupt",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function unavailableDigests(
|
||||
ecr: ECRClient,
|
||||
repositoryName: string,
|
||||
blobs: Descriptor[],
|
||||
): Promise<Set<string>> {
|
||||
const missing = new Set(blobs.map((blob) => blob.digest));
|
||||
const digests = [...missing];
|
||||
for (let i = 0; i < digests.length; i += CHECK_BATCH_SIZE) {
|
||||
const { layers = [] } = await ecr.send(
|
||||
new BatchCheckLayerAvailabilityCommand({
|
||||
repositoryName,
|
||||
layerDigests: digests.slice(i, i + CHECK_BATCH_SIZE),
|
||||
}),
|
||||
);
|
||||
for (const layer of layers) {
|
||||
if (layer.layerAvailability === "AVAILABLE" && layer.layerDigest) {
|
||||
missing.delete(layer.layerDigest);
|
||||
}
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
}
|
||||
|
||||
async function uploadBlob(
|
||||
ecr: ECRClient,
|
||||
repositoryName: string,
|
||||
path: string,
|
||||
blob: Descriptor,
|
||||
): Promise<void> {
|
||||
const { uploadId, partSize } = await ecr.send(new InitiateLayerUploadCommand({ repositoryName }));
|
||||
if (!uploadId || !partSize) {
|
||||
throw new Error(`[pushImageToEcr] InitiateLayerUpload returned no uploadId/partSize`);
|
||||
}
|
||||
const file = await open(path);
|
||||
try {
|
||||
for (let first = 0; first < blob.size; first += partSize) {
|
||||
const part = Buffer.alloc(Math.min(partSize, blob.size - first));
|
||||
const { bytesRead } = await file.read(part, 0, part.length, first);
|
||||
if (bytesRead !== part.length) throw new Error(`[pushImageToEcr] ${path} changed on disk`);
|
||||
await ecr.send(
|
||||
new UploadLayerPartCommand({
|
||||
repositoryName,
|
||||
uploadId,
|
||||
partFirstByte: first,
|
||||
partLastByte: first + part.length - 1,
|
||||
layerPartBlob: part,
|
||||
}),
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
try {
|
||||
await ecr.send(
|
||||
new CompleteLayerUploadCommand({ repositoryName, uploadId, layerDigests: [blob.digest] }),
|
||||
);
|
||||
} catch (error) {
|
||||
if ((error as Error).name !== "LayerAlreadyExistsException") throw error;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"@hyperframes/aws-lambda": {
|
||||
"packed": 70874,
|
||||
"unpacked": 258166
|
||||
"packed": 73369,
|
||||
"unpacked": 266805
|
||||
},
|
||||
"@hyperframes/cli": {
|
||||
"packed": 7902966,
|
||||
|
||||
Reference in New Issue
Block a user