mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-05 10:14:30 +00:00
James RussoandClaude Opus 4.8
4da567df22
* feat(gcp-cloud-run): add Google Cloud Run + Workflows distributed render adapter Adds @hyperframes/gcp-cloud-run, the GCP counterpart to @hyperframes/aws-lambda (issue #932). The OSS distributed primitives (plan, renderChunk x N, assemble) are unchanged; this package is the storage/compute/orchestration glue. Package: Cloud Run handler (one image, three actions), runs under bun; GCS transport; in-image chrome-headless-shell resolver; client SDK (renderToCloudRun, getRenderProgress, deploySite, computeRenderCost); Dockerfile; Cloud Workflows definition; Terraform module; CLI cloudrun deploy|sites|render|render-batch|progress|destroy with --output-resolution and --strict-variables; 62 unit tests + docs + live smoke script. Shared extraction (removes ~640 lines of adapter duplication): move the cloud-agnostic config validator + content-hash into producer/distributed; both adapters import them. Validated end-to-end on GCP at 37.4 dB PSNR vs baseline. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(cli): resolve @hyperframes/gcp-cloud-run in the CLI build + root build The CLI bundle (esbuild) couldn't resolve `@hyperframes/gcp-cloud-run/sdk`, failing Build/Typecheck/CLI-smoke (and the perf/windows/regression jobs that build first). Mirror the aws-lambda handling: mark the gcp adapter + its /sdk subpath external in tsup.config.ts with a source alias, and add gcp-cloud-run to the root `build` filter so its dist exists for publish + runtime. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(ci): copy gcp-cloud-run manifest in Dockerfile.test for frozen install The regression test image runs `bun install --frozen-lockfile` after copying each workspace package.json individually. The CLI now depends on @hyperframes/gcp-cloud-run (workspace:*), so the frozen install fails to resolve it unless its manifest is present. Add the COPY line. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(cli): add machine-sizing flags to `cloudrun deploy` Closes the parity gap with `lambda deploy` (which exposes --memory etc.). `cloudrun deploy` now threads --cpu, --memory, --max-instances, and --timeout into the Terraform apply; omitted flags keep the module defaults (4 vCPU / 16Gi / 100 instances / 3600s). For finer control, apply the module directly. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(gcp-cloud-run): address PR review (security, waste, limits, alerts) - server.ts: bucket-allowlist guard no longer fails open silently. Unset env logs a one-time WARNING; "*" is an explicit opt-out; otherwise it enforces. - server.ts: stop double-shipping audio.aac. It already rides in the plan tarball every consumer downloads, so drop the redundant standalone upload (plan) + re-download/overwrite (assemble); assemble reads it from the untar, falling back to a supplied AudioGcsUri for compat. - server.ts: chunk extension via path.extname() instead of slice(lastIndexOf). - workflow.yaml: clamp parallel concurrency_limit to math.min(chunkCount, 20) — Cloud Workflows hard-caps concurrent iterations at 20. - Dockerfile: pin bun (bun-v1.3.9) so an interop change can't silently break the image rebuild. - terraform: add min_instances var (default 0); add a workflow-failure alert (finished_execution_count status=FAILED) alongside the request-count one. - costAccounting: document that displayCost excludes GCS storage/egress. Verified against the actual APIs: @google-cloud/workflows@4.4.0 ICreateExecutionRequest has no executionId (so the idempotency-token suggestion isn't available in this client); Workflows concurrency cap is 20; failure metric is workflows.googleapis.com/finished_execution_count (status label). 174 adapter tests pass, fallow/oxlint/oxfmt/terraform clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(gcp-cloud-run): address round-2 review — error code + CFR forwarding - workflow.yaml: rename the zero-chunk failure code PLAN_TOO_LARGE → PLAN_PRODUCED_ZERO_CHUNKS. The old code implied a size-ceiling breach (the opposite cause), misleading anyone triaging the alert. - workflow.yaml: forward Config.cfr to the assemble step (`Cfr: ${("cfr" in config) and config.cfr}`). It was read by the handler but never sent, so exact-CFR was silently off for every Cloud Run render. Uses the same `in`-operator guard already proven in the retryable predicate. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(release): include gcp-cloud-run in set-version PACKAGES list set-version.ts (driven by release:prepare) bumps an explicit package list to the shared version on each release. gcp-cloud-run was wired into the build + publish.yml but missing here, so a release would leave it at a stale version and publish.yml would push the wrong version. Add it so the new package version-bumps + publishes in lockstep with the others. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
115 lines
3.7 KiB
TypeScript
115 lines
3.7 KiB
TypeScript
/**
|
|
* GCS transport unit tests — URI parsing, tar round-trip, and the
|
|
* download/upload bridge over the `FakeGcs` double.
|
|
*/
|
|
|
|
import { afterEach, describe, expect, it } from "bun:test";
|
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { asStorage, FakeGcs } from "./__fixtures__/fakeGcs.js";
|
|
import {
|
|
downloadGcsObjectToFile,
|
|
formatGcsUri,
|
|
parseGcsUri,
|
|
tarDirectory,
|
|
untarDirectory,
|
|
uploadFileToGcs,
|
|
} from "./gcsTransport.js";
|
|
|
|
const tmpDirs: string[] = [];
|
|
function mkTmp(prefix: string): string {
|
|
const dir = mkdtempSync(join(tmpdir(), prefix));
|
|
tmpDirs.push(dir);
|
|
return dir;
|
|
}
|
|
afterEach(() => {
|
|
for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
|
|
});
|
|
|
|
describe("parseGcsUri", () => {
|
|
it("splits bucket and key", () => {
|
|
expect(parseGcsUri("gs://my-bucket/path/to/object.tar.gz")).toEqual({
|
|
bucket: "my-bucket",
|
|
key: "path/to/object.tar.gz",
|
|
});
|
|
});
|
|
|
|
it("rejects non-gs URIs", () => {
|
|
expect(() => parseGcsUri("s3://b/k")).toThrow(/expected gs:\/\//);
|
|
});
|
|
|
|
it("rejects a bucket with no key", () => {
|
|
expect(() => parseGcsUri("gs://just-a-bucket")).toThrow(/missing key/);
|
|
});
|
|
|
|
it("rejects an empty bucket", () => {
|
|
expect(() => parseGcsUri("gs:///key")).toThrow(/empty bucket or key/);
|
|
});
|
|
|
|
it("round-trips through formatGcsUri", () => {
|
|
const uri = "gs://b/some/key";
|
|
expect(formatGcsUri(parseGcsUri(uri))).toBe(uri);
|
|
});
|
|
});
|
|
|
|
describe("tarDirectory / untarDirectory", () => {
|
|
it("round-trips a directory tree", async () => {
|
|
const src = mkTmp("hf-tar-src-");
|
|
mkdirSync(join(src, "nested"), { recursive: true });
|
|
writeFileSync(join(src, "index.html"), "<html>hi</html>");
|
|
writeFileSync(join(src, "nested", "data.json"), '{"a":1}');
|
|
|
|
const work = mkTmp("hf-tar-work-");
|
|
const tarball = join(work, "out.tar.gz");
|
|
await tarDirectory(src, tarball);
|
|
expect(existsSync(tarball)).toBe(true);
|
|
|
|
const dest = join(work, "extracted");
|
|
await untarDirectory(tarball, dest);
|
|
expect(readFileSync(join(dest, "index.html"), "utf8")).toBe("<html>hi</html>");
|
|
expect(readFileSync(join(dest, "nested", "data.json"), "utf8")).toBe('{"a":1}');
|
|
});
|
|
|
|
it("untar wipes a stale destination first", async () => {
|
|
const src = mkTmp("hf-tar-src2-");
|
|
writeFileSync(join(src, "keep.txt"), "new");
|
|
const work = mkTmp("hf-tar-work2-");
|
|
const tarball = join(work, "out.tar.gz");
|
|
await tarDirectory(src, tarball);
|
|
|
|
const dest = join(work, "extracted");
|
|
mkdirSync(dest, { recursive: true });
|
|
writeFileSync(join(dest, "stale.txt"), "should be gone");
|
|
|
|
await untarDirectory(tarball, dest);
|
|
expect(existsSync(join(dest, "stale.txt"))).toBe(false);
|
|
expect(readFileSync(join(dest, "keep.txt"), "utf8")).toBe("new");
|
|
});
|
|
});
|
|
|
|
describe("download/upload bridge", () => {
|
|
it("uploads a local file then downloads identical bytes", async () => {
|
|
const gcs = new FakeGcs();
|
|
const work = mkTmp("hf-dl-");
|
|
const srcFile = join(work, "src.bin");
|
|
writeFileSync(srcFile, Buffer.from("hello gcs"));
|
|
|
|
const uri = "gs://bucket/obj.bin";
|
|
await uploadFileToGcs(asStorage(gcs), srcFile, uri, "application/octet-stream");
|
|
|
|
const dest = join(work, "dl.bin");
|
|
await downloadGcsObjectToFile(asStorage(gcs), uri, dest);
|
|
expect(readFileSync(dest, "utf8")).toBe("hello gcs");
|
|
|
|
expect(gcs.ops.map((o) => o.kind)).toEqual(["upload", "download"]);
|
|
});
|
|
|
|
it("upload throws when the source file is missing", async () => {
|
|
const gcs = new FakeGcs();
|
|
await expect(uploadFileToGcs(asStorage(gcs), "/no/such/file", "gs://b/k")).rejects.toThrow(
|
|
/upload source missing/,
|
|
);
|
|
});
|
|
});
|