fix(gcp-cloud-run): normalize v2 integrity codes (#2790)

This commit is contained in:
James Russo
2026-07-25 23:59:36 -04:00
committed by GitHub
parent 5bf61d6df0
commit 0499a5cbcb
27 changed files with 2093 additions and 260 deletions
+1
View File
@@ -97,6 +97,7 @@ COPY packages/parsers/package.json packages/parsers/package.json
COPY packages/sdk/package.json packages/sdk/package.json
COPY packages/sdk-playground/package.json packages/sdk-playground/package.json
COPY packages/studio-server/package.json packages/studio-server/package.json
COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs
RUN bun install --frozen-lockfile
# Copy source for the packages the render path needs.
+2 -1
View File
@@ -81,7 +81,8 @@
"@types/tar": "^6.1.13",
"esbuild": "^0.25.12",
"tsx": "^4.21.0",
"typescript": "^5.7.2"
"typescript": "^5.7.2",
"yaml": "^2.9.0"
},
"engines": {
"node": ">=22"
@@ -24,6 +24,9 @@ export interface FakeGcsOp {
export class FakeGcs {
ops: FakeGcsOp[] = [];
objects = new Map<string, Buffer>();
metadata = new Map<string, Record<string, string>>();
/** Optional one-shot race hook used to model another writer before a create-only upload. */
beforeUpload?: (uri: string) => void;
// Accessed only through the `Storage` cast in tests, so fallow's static
// analysis can't see the reference.
@@ -57,13 +60,30 @@ class FakeBucket {
return new FakeFile(this.gcs, this.bucketName, key);
}
// The stateful command matrix intentionally stays inline so tests exercise
// generation preconditions and metadata updates as one fake GCS operation.
// fallow-ignore-next-line complexity
async upload(
localPath: string,
opts: { destination: string; contentType?: string },
opts: {
destination: string;
contentType?: string;
metadata?: { metadata?: Record<string, string> };
preconditionOpts?: { ifGenerationMatch?: number };
},
): Promise<unknown> {
const uri = `gs://${this.bucketName}/${opts.destination}`;
this.gcs.beforeUpload?.(uri);
if (opts.preconditionOpts?.ifGenerationMatch === 0 && this.gcs.objects.has(uri)) {
const error = new Error(`FakeGcs: precondition failed: ${uri}`) as Error & {
code: number;
};
error.code = 412;
throw error;
}
const bytes = readFileSync(localPath);
this.gcs.objects.set(uri, bytes);
this.gcs.metadata.set(uri, opts.metadata?.metadata ?? {});
this.gcs.ops.push({ kind: "upload", uri, bytes: bytes.length });
return [{}];
}
@@ -96,10 +116,18 @@ class FakeFile {
return [has];
}
async getMetadata(): Promise<[{ size?: string | number; updated?: string }]> {
async getMetadata(): Promise<
[{ size?: string | number; updated?: string; metadata?: Record<string, string> }]
> {
const bytes = this.gcs.objects.get(this.uri);
this.gcs.ops.push({ kind: "getMetadata", uri: this.uri });
return [{ size: bytes?.length ?? 0, updated: "2026-06-06T00:00:00.000Z" }];
return [
{
size: bytes?.length ?? 0,
updated: "2026-06-06T00:00:00.000Z",
metadata: this.gcs.metadata.get(this.uri) ?? {},
},
];
}
/** Helper for tests that want to materialize an object to disk. */
+84 -12
View File
@@ -31,6 +31,8 @@ export type { SerializableDistributedRenderConfig } from "@hyperframes/producer/
/** Discriminator for the three roles the one Cloud Run image fulfills. */
export type CloudRunAction = "plan" | "renderChunk" | "assemble";
/** Transport protocol selected for one complete distributed render. */
export type CloudRunPlanProtocol = "v1" | "v2";
/**
* Top-level shape of any request body the handler may receive.
@@ -48,7 +50,7 @@ export type CloudRunEvent =
| { Input: CloudRunEvent };
/** Activity A: produce a planDir, upload to GCS. */
export interface PlanEvent {
interface PlanEventBase {
Action: "plan";
/** GCS URI pointing at a `tar -czf`-archived project directory (`gs://bucket/key.tar.gz`). */
ProjectGcsUri: string;
@@ -58,11 +60,21 @@ export interface PlanEvent {
Config: SerializableDistributedRenderConfig;
}
/** Legacy/default plan transport. Absence is deliberately interpreted as v1. */
export interface PlanV1Event extends PlanEventBase {
PlanProtocol?: "v1";
}
/** Explicit opt-in to the content-addressed v2 plan transport. */
export interface PlanV2Event extends PlanEventBase {
PlanProtocol: "v2";
}
export type PlanEvent = PlanV1Event | PlanV2Event;
/** Activity B: fetch planDir, render one chunk, upload result. */
export interface RenderChunkEvent {
interface RenderChunkEventBase {
Action: "renderChunk";
/** GCS URI of the plan tar produced by a PlanEvent invocation. */
PlanGcsUri: string;
/**
* `PlanResult.planHash` from the Plan invocation. The handler verifies
* this against the untarred planDir's `plan.json` before invoking the
@@ -79,15 +91,33 @@ export interface RenderChunkEvent {
Format: DistributedFormat;
}
/** Activity C: fetch planDir + all chunks + audio, assemble, upload final. */
export interface AssembleEvent {
Action: "assemble";
/** GCS URI of the plan tar produced by a PlanEvent invocation. */
/** Legacy/default chunk event. */
export interface RenderChunkV1Event extends RenderChunkEventBase {
PlanProtocol?: "v1";
/** GCS URI of the v1 plan tar produced by a PlanEvent invocation. */
PlanGcsUri: string;
PlanV2ManifestGcsUri?: never;
PlanV2ArtifactGcsPrefix?: never;
}
/**
* V2 chunk event. It intentionally cannot carry `PlanGcsUri`: the manifest
* describes the exact content-addressed artifacts needed by this chunk.
*/
export interface RenderChunkV2Event extends RenderChunkEventBase {
PlanProtocol: "v2";
PlanV2ManifestGcsUri: string;
PlanV2ArtifactGcsPrefix: string;
PlanGcsUri?: never;
}
export type RenderChunkEvent = RenderChunkV1Event | RenderChunkV2Event;
/** Activity C: fetch planDir + all chunks + audio, assemble, upload final. */
interface AssembleEventBase {
Action: "assemble";
/** GCS URIs of every chunk, ordered by chunk index. Length must equal `chunkCount`. */
ChunkGcsUris: string[];
/** GCS URI of the planDir's `audio.aac` if the composition has audio; `null` otherwise. */
AudioGcsUri: string | null;
/** Final output GCS URI (`gs://bucket/key.mp4`). */
OutputGcsUri: string;
/** Output container format; drives file vs frame-dir handling. */
@@ -104,12 +134,35 @@ export interface AssembleEvent {
Cfr?: boolean;
}
/** Legacy/default assemble event. */
export interface AssembleV1Event extends AssembleEventBase {
PlanProtocol?: "v1";
/** GCS URI of the v1 plan tar produced by a PlanEvent invocation. */
PlanGcsUri: string;
/** Legacy standalone audio locator; `null` when audio is embedded in the v1 plan tar. */
AudioGcsUri: string | null;
PlanV2ManifestGcsUri?: never;
PlanV2ArtifactGcsPrefix?: never;
}
/** V2 assemble event, scoped to manifest-declared assembler artifacts. */
export interface AssembleV2Event extends AssembleEventBase {
PlanProtocol: "v2";
PlanV2ManifestGcsUri: string;
PlanV2ArtifactGcsPrefix: string;
PlanHash: string;
PlanGcsUri?: never;
/** V2 audio is a manifest artifact materialized only for the assembler. */
AudioGcsUri: null;
}
export type AssembleEvent = AssembleV1Event | AssembleV2Event;
// ── Result types — kept small to fit Cloud Workflows step budgets ────────────
/** Result of a `plan` invocation. Carries enough to size the Map(N) state. */
export interface PlanResultBody {
interface PlanResultBodyBase {
Action: "plan";
PlanGcsUri: string;
PlanHash: string;
ChunkCount: number;
TotalFrames: number;
@@ -124,6 +177,25 @@ export interface PlanResultBody {
DurationMs: number;
}
/** Existing v1 result. Kept unchanged for wire compatibility. */
export interface PlanV1ResultBody extends PlanResultBodyBase {
PlanGcsUri: string;
PlanProtocol?: never;
PlanV2ManifestGcsUri?: never;
PlanV2ArtifactGcsPrefix?: never;
}
/** V2 result. The two v2 locators are never aliases for `PlanGcsUri`. */
export interface PlanV2ResultBody extends PlanResultBodyBase {
PlanProtocol: "v2";
PlanV2ManifestGcsUri: string;
PlanV2ArtifactGcsPrefix: string;
PlanGcsUri?: never;
AudioGcsUri: null;
}
export type PlanResultBody = PlanV1ResultBody | PlanV2ResultBody;
/** Result of a `renderChunk` invocation. Sized ≤200 bytes. */
export interface RenderChunkResultBody {
Action: "renderChunk";
@@ -10,10 +10,13 @@ import { join } from "node:path";
import { asStorage, FakeGcs } from "./__fixtures__/fakeGcs.js";
import {
downloadGcsObjectToFile,
downloadGcsObjectToFileVerified,
formatGcsUri,
parseGcsUri,
sha256File,
tarDirectory,
untarDirectory,
uploadContentAddressedFileToGcs,
uploadFileToGcs,
} from "./gcsTransport.js";
@@ -112,3 +115,71 @@ describe("download/upload bridge", () => {
);
});
});
describe("content-addressed v2 artifacts", () => {
it("uploads once and reuses an object with matching digest metadata", async () => {
const gcs = new FakeGcs();
const source = join(mkTmp("hf-cas-upload-"), "artifact.bin");
writeFileSync(source, "immutable bytes");
const digest = await sha256File(source);
const uri = `gs://bucket/v2/artifacts/sha256/${digest.slice(0, 2)}/${digest}`;
expect(await uploadContentAddressedFileToGcs(asStorage(gcs), source, uri, digest)).toBe(
"uploaded",
);
expect(await uploadContentAddressedFileToGcs(asStorage(gcs), source, uri, digest)).toBe(
"reused",
);
expect(gcs.ops.filter((op) => op.kind === "upload")).toHaveLength(1);
expect(gcs.metadata.get(uri)?.sha256).toBe(digest);
});
it("refuses to overwrite an immutable key with conflicting metadata", async () => {
const gcs = new FakeGcs();
const source = join(mkTmp("hf-cas-conflict-"), "artifact.bin");
writeFileSync(source, "expected bytes");
const digest = await sha256File(source);
const uri = `gs://bucket/v2/artifacts/sha256/${digest.slice(0, 2)}/${digest}`;
gcs.seed(uri, Buffer.from("same length!!!"));
gcs.metadata.set(uri, { sha256: "0".repeat(64) });
await expect(
uploadContentAddressedFileToGcs(asStorage(gcs), source, uri, digest),
).rejects.toMatchObject({ name: "PLAN_ARTIFACT_DIGEST_MISMATCH" });
expect(gcs.ops.some((op) => op.kind === "upload")).toBe(false);
});
it("reuses an identical object that wins the create-only generation race", async () => {
const gcs = new FakeGcs();
const source = join(mkTmp("hf-cas-race-"), "artifact.bin");
writeFileSync(source, "racing bytes");
const digest = await sha256File(source);
const uri = `gs://bucket/v2/artifacts/sha256/${digest.slice(0, 2)}/${digest}`;
gcs.beforeUpload = (uploadUri) => {
gcs.beforeUpload = undefined;
gcs.seed(uploadUri, readFileSync(source));
gcs.metadata.set(uploadUri, { sha256: digest });
};
expect(await uploadContentAddressedFileToGcs(asStorage(gcs), source, uri, digest)).toBe(
"reused",
);
expect(gcs.ops.some((op) => op.kind === "upload")).toBe(false);
});
it("deletes a downloaded artifact when digest verification fails", async () => {
const gcs = new FakeGcs();
const work = mkTmp("hf-cas-download-");
const expectedSource = join(work, "expected.bin");
const destination = join(work, "download.bin");
writeFileSync(expectedSource, "expected");
const expected = await sha256File(expectedSource);
const uri = "gs://bucket/v2/artifacts/corrupt";
gcs.seed(uri, Buffer.from("corrupt"));
await expect(
downloadGcsObjectToFileVerified(asStorage(gcs), uri, destination, expected),
).rejects.toMatchObject({ name: "PLAN_ARTIFACT_DIGEST_MISMATCH" });
expect(existsSync(destination)).toBe(false);
});
});
+141 -1
View File
@@ -19,7 +19,15 @@
* is the same shape as `@hyperframes/aws-lambda`'s `s3Transport.ts`.
*/
import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
import {
createReadStream,
createWriteStream,
existsSync,
mkdirSync,
rmSync,
statSync,
} from "node:fs";
import { createHash } from "node:crypto";
import { dirname } from "node:path";
import { pipeline } from "node:stream/promises";
import type { Storage } from "@google-cloud/storage";
@@ -70,6 +78,26 @@ export async function downloadGcsObjectToFile(
await pipeline(file.createReadStream(), createWriteStream(destPath));
}
/** Download and verify an immutable plan-v2 artifact before materialization. */
export async function downloadGcsObjectToFileVerified(
storage: Storage,
uri: string,
destPath: string,
expectedSha256: string,
): Promise<void> {
assertSha256(expectedSha256);
await downloadGcsObjectToFile(storage, uri, destPath);
const actual = await sha256File(destPath);
if (actual !== expectedSha256) {
rmSync(destPath, { force: true });
const error = new Error(
`[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`,
);
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
throw error;
}
}
/**
* Upload a local file's contents to a GCS URI using a resumable upload.
* GCS objects have no practical size ceiling for the artifacts this adapter
@@ -96,6 +124,118 @@ export async function uploadFileToGcs(
});
}
/**
* Upload one content-addressed plan-v2 artifact exactly once.
*
* The zero-generation precondition makes creation atomic. Existing objects
* are reused only when their immutable digest metadata and byte length agree;
* a conflict is never overwritten because another render may already consume
* that object.
*/
export async function uploadContentAddressedFileToGcs(
storage: Storage,
localPath: string,
uri: string,
expectedSha256: string,
contentType?: string,
): Promise<"uploaded" | "reused"> {
assertSha256(expectedSha256);
if (!existsSync(localPath)) {
throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
}
const actualSha256 = await sha256File(localPath);
if (actualSha256 !== expectedSha256) {
throwDigestMismatch(
`local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`,
);
}
const { bucket, key } = parseGcsUri(uri);
const bucketHandle = storage.bucket(bucket);
const file = bucketHandle.file(key);
const size = statSync(localPath).size;
if (await isReusableContentAddressedObject(file, uri, size, expectedSha256)) {
return "reused";
}
try {
await bucketHandle.upload(localPath, {
destination: key,
contentType,
metadata: { metadata: { sha256: expectedSha256 } },
preconditionOpts: { ifGenerationMatch: 0 },
});
return "uploaded";
} catch (error) {
// A concurrent planner may win the create-only race. Reuse only after
// verifying that the winning object is exactly the immutable CAS value.
if (
isGcsPreconditionFailed(error) &&
(await isReusableContentAddressedObject(file, uri, size, expectedSha256))
) {
return "reused";
}
throw error;
}
}
interface GcsFileLike {
exists(): Promise<[boolean, ...unknown[]]>;
getMetadata(): Promise<
[
{
size?: string | number;
metadata?: Record<string, string | number | boolean | null>;
},
...unknown[],
]
>;
}
async function isReusableContentAddressedObject(
file: GcsFileLike,
uri: string,
expectedSize: number,
expectedSha256: string,
): Promise<boolean> {
const [exists] = await file.exists();
if (!exists) return false;
const [metadata] = await file.getMetadata();
if (Number(metadata.size) === expectedSize && metadata.metadata?.sha256 === expectedSha256) {
return true;
}
throwDigestMismatch(
`immutable object ${uri} already exists with different digest metadata or size`,
);
}
export async function sha256File(path: string): Promise<string> {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) {
hash.update(chunk as Buffer);
}
return hash.digest("hex");
}
function assertSha256(value: string): void {
if (!/^[a-f0-9]{64}$/.test(value)) {
throw new Error(
`[gcsTransport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`,
);
}
}
function throwDigestMismatch(detail: string): never {
const error = new Error(`[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${detail}`);
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
throw error;
}
function isGcsPreconditionFailed(error: unknown): boolean {
if (!error || typeof error !== "object") return false;
return (error as { code?: unknown }).code === 412;
}
/**
* Pack a directory into a `.tar.gz` at `destTarball`. Uses the `tar` npm
* package (pure JS over `node:zlib`) rather than spawning a system tar
+12
View File
@@ -21,24 +21,36 @@
export { createApp, dispatch, type HandlerDeps, startServer, unwrapEvent } from "./server.js";
export {
type AssembleEvent,
type AssembleV1Event,
type AssembleV2Event,
type AssembleResultBody,
type CloudRunAction,
type CloudRunEvent,
type CloudRunPlanProtocol,
type CloudRunResult,
type PlanEvent,
type PlanResultBody,
type PlanV1Event,
type PlanV1ResultBody,
type PlanV2Event,
type PlanV2ResultBody,
type RenderChunkEvent,
type RenderChunkResultBody,
type RenderChunkV1Event,
type RenderChunkV2Event,
type SerializableDistributedRenderConfig,
} from "./events.js";
export { ChromeBinaryUnavailableError, resolveChromeExecutablePath } from "./chromium.js";
export {
downloadGcsObjectToFile,
downloadGcsObjectToFileVerified,
formatGcsUri,
type GcsLocation,
parseGcsUri,
sha256File,
tarDirectory,
untarDirectory,
uploadContentAddressedFileToGcs,
uploadFileToGcs,
} from "./gcsTransport.js";
@@ -78,11 +78,19 @@ describe("renderToCloudRun", () => {
expect(arg.OutputGcsUri).toBe("gs://b/renders/hf-render-fixed/output.mp4");
expect(arg.ServiceUrl).toBe("https://render-abc.run.app");
expect(arg.Config.format).toBe("mp4");
expect(arg.PlanProtocol).toBe("v1");
expect(fake.lastParent).toBe(
"projects/proj/locations/us-central1/workflows/hyperframes-render",
);
});
it("forwards an explicit v2 whole-render opt-in", async () => {
const fake = new FakeExecutions();
await renderToCloudRun({ ...opts(fake), planProtocol: "v2" });
const arg = JSON.parse(fake.lastArgument ?? "{}");
expect(arg.PlanProtocol).toBe("v2");
});
it("derives the output extension from the format", async () => {
const fake = new FakeExecutions();
const handle = await renderToCloudRun({
@@ -26,7 +26,7 @@
import { randomUUID } from "node:crypto";
import type { Storage } from "@google-cloud/storage";
import type { SerializableDistributedRenderConfig } from "../events.js";
import type { CloudRunPlanProtocol, SerializableDistributedRenderConfig } from "../events.js";
import { formatExtension } from "../formatExtension.js";
import { formatGcsUri } from "../gcsTransport.js";
import { deploySite, type SiteHandle } from "./deploySite.js";
@@ -52,6 +52,11 @@ export interface RenderToCloudRunOptions {
siteHandle?: SiteHandle;
/** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */
config: SerializableDistributedRenderConfig;
/**
* Distributed plan transport. Defaults to `"v1"` for backwards
* compatibility; v2 is always an explicit whole-render opt-in.
*/
planProtocol?: CloudRunPlanProtocol;
/** GCS bucket from the Terraform output (`render_bucket_name`). */
bucketName: string;
/** GCP project id hosting the workflow. */
@@ -144,6 +149,7 @@ export async function renderToCloudRun(opts: RenderToCloudRunOptions): Promise<R
OutputGcsUri: outputGcsUri,
ServiceUrl: opts.serviceUrl,
Config: opts.config,
PlanProtocol: opts.planProtocol ?? "v1",
};
// Reject oversize input client-side. Cloud Workflows caps the execution
+224 -2
View File
@@ -15,16 +15,22 @@
*/
import { afterEach, describe, expect, it } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
CURRENT_PLAN_PROTOCOL,
createPlanV2FromV1,
PLAN_V2_INTEGRITY_UNRECOVERABLE,
PlanV2IntegrityError,
PlanProtocolUnsupportedError,
type AssembleResult,
type ChunkResult,
type PlanResult,
type PlanV2Result,
} from "@hyperframes/producer/distributed";
import { recomputePlanHashFromPlanDir } from "../../producer/src/services/render/stages/freezePlan.js";
import { asStorage, FakeGcs } from "./__fixtures__/fakeGcs.js";
import type { AssembleEvent, CloudRunEvent, PlanEvent, RenderChunkEvent } from "./events.js";
import { createApp, dispatch, type HandlerDeps, unwrapEvent } from "./server.js";
@@ -60,6 +66,30 @@ async function seedPlanTar(gcs: FakeGcs, uri: string, planHash: string): Promise
gcs.seedFromFile(uri, tarPath);
}
function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void {
mkdirSync(join(dir, "meta"), { recursive: true });
mkdirSync(join(dir, "compiled"), { recursive: true });
writeFileSync(join(dir, "compiled", "index.html"), "<html>gcp v2 fixture</html>");
const planJson = {
planHash: "a".repeat(64),
chunkCount: 1,
totalFrames: 30,
dimensions: { fpsNum: 30, fpsDen: 1, width: 640, height: 360, format: "mp4" },
ffmpegVersion: "6.0",
producerVersion: "test",
fontSnapshotSha: "font-snapshot-test",
};
writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));
writeFileSync(
join(dir, "meta", "chunks.json"),
JSON.stringify([{ index: 0, startFrame: 0, endFrame: 30 }]),
);
writeFileSync(join(dir, "meta", "encoder.json"), "{}");
if (withAudio) writeFileSync(join(dir, "audio.aac"), "AAC");
planJson.planHash = recomputePlanHashFromPlanDir(dir);
writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));
}
const planResult: PlanResult = {
planDir: "(set at call time)",
planProtocol: CURRENT_PLAN_PROTOCOL,
@@ -207,6 +237,139 @@ describe("dispatch", () => {
expect(gcs.objects.has("gs://b/renders/r1/output.mp4")).toBe(true);
});
// This end-to-end adapter contract is intentionally one narrative test: it
// verifies ordering and target isolation across all three handler roles.
// fallow-ignore-next-line complexity
it("runs v2 plan → target-scoped chunk → assemble with manifest-last CAS", async () => {
const gcs = new FakeGcs();
await seedProjectTar(gcs, "gs://b/sites/v2/project.tar.gz");
const root = mkTmp("hf-v2-e2e-");
const planV2 = async (
_projectDir: string,
_config: unknown,
planV2Dir: string,
): Promise<PlanV2Result> => {
const v1Dir = join(root, "v1");
makeMinimalV1PlanDir(v1Dir, true);
return createPlanV2FromV1(v1Dir, planV2Dir);
};
const renderChunk = async (
planDir: string,
chunkIndex: number,
outputBase: string,
): Promise<ChunkResult> => {
expect(existsSync(join(planDir, "audio.aac"))).toBe(false);
writeFileSync(outputBase, `chunk-${chunkIndex}`);
return {
outputPath: outputBase,
outputKind: "file",
framesEncoded: 30,
sha256: "b".repeat(64),
};
};
const assemble = async (
_planDir: string,
_chunks: string[],
audioPath: string | null,
finalOutput: string,
): Promise<AssembleResult> => {
expect(audioPath).not.toBeNull();
expect(readFileSync(audioPath as string, "utf8")).toBe("AAC");
writeFileSync(finalOutput, "v2-output");
return { framesEncoded: 30, fileSize: 9 };
};
const deps = depsWith(gcs, { planV2, renderChunk, assemble });
const planned = await dispatch(
{
Action: "plan",
PlanProtocol: "v2",
ProjectGcsUri: "gs://b/sites/v2/project.tar.gz",
PlanOutputGcsPrefix: "gs://b/renders/v2/",
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
},
deps,
);
expect(planned).toMatchObject({
PlanProtocol: "v2",
PlanV2ManifestGcsUri: "gs://b/renders/v2/v2/manifest.json",
PlanV2ArtifactGcsPrefix: "gs://b/renders/v2/v2/artifacts/sha256",
AudioGcsUri: null,
});
expect("PlanGcsUri" in planned).toBe(false);
const uploadUris = gcs.ops.filter((op) => op.kind === "upload").map((op) => op.uri);
expect(uploadUris.at(-1)).toBe("gs://b/renders/v2/v2/manifest.json");
expect(gcs.metadata.get(uploadUris.at(-1) ?? "")?.sha256).toMatch(/^[a-f0-9]{64}$/);
await dispatch(
{
Action: "plan",
PlanProtocol: "v2",
ProjectGcsUri: "gs://b/sites/v2/project.tar.gz",
PlanOutputGcsPrefix: "gs://b/renders/v2/",
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
},
deps,
);
expect(gcs.ops.filter((op) => op.kind === "upload").map((op) => op.uri)).toEqual(uploadUris);
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2") {
throw new Error("expected v2 plan result");
}
const beforeChunk = gcs.ops.length;
const chunk = await dispatch(
{
Action: "renderChunk",
PlanProtocol: "v2",
PlanV2ManifestGcsUri: planned.PlanV2ManifestGcsUri,
PlanV2ArtifactGcsPrefix: planned.PlanV2ArtifactGcsPrefix,
PlanHash: planned.PlanHash,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://b/renders/v2/",
Format: "mp4",
},
deps,
);
if (chunk.Action !== "renderChunk") throw new Error("expected chunk result");
const audioDigest = createHash("sha256").update("AAC").digest("hex");
const audioUri = `${planned.PlanV2ArtifactGcsPrefix}/${audioDigest.slice(0, 2)}/${audioDigest}`;
expect(gcs.ops.slice(beforeChunk).some((operation) => operation.uri === audioUri)).toBe(false);
await dispatch(
{
Action: "assemble",
PlanProtocol: "v2",
PlanV2ManifestGcsUri: planned.PlanV2ManifestGcsUri,
PlanV2ArtifactGcsPrefix: planned.PlanV2ArtifactGcsPrefix,
PlanHash: planned.PlanHash,
ChunkGcsUris: [chunk.ChunkGcsUri],
AudioGcsUri: null,
OutputGcsUri: "gs://b/renders/v2/output.mp4",
Format: "mp4",
},
deps,
);
expect(
gcs.ops.some((operation) => operation.kind === "download" && operation.uri === audioUri),
).toBe(true);
});
it("rejects mixed v1/v2 locators at runtime", async () => {
const event = {
Action: "renderChunk",
PlanProtocol: "v2",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanV2ManifestGcsUri: "gs://b/renders/r1/v2/manifest.json",
PlanV2ArtifactGcsPrefix: "gs://b/renders/r1/v2/artifacts/sha256",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
Format: "mp4",
} as unknown as RenderChunkEvent;
await expect(dispatch(event, depsWith(new FakeGcs()))).rejects.toMatchObject({
name: "PLAN_PROTOCOL_UNSUPPORTED",
});
});
it("rejects an unknown action", async () => {
const gcs = new FakeGcs();
await expect(
@@ -236,6 +399,27 @@ describe("bucket allowlist guard", () => {
}
});
it("checks both v2 manifest and artifact-prefix buckets", async () => {
const prev = process.env.HYPERFRAMES_RENDER_BUCKET;
process.env.HYPERFRAMES_RENDER_BUCKET = "allowed-bucket";
try {
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanProtocol: "v2",
PlanV2ManifestGcsUri: "gs://allowed-bucket/v2/manifest.json",
PlanV2ArtifactGcsPrefix: "gs://evil-bucket/v2/artifacts/sha256",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://allowed-bucket/renders/r1/",
Format: "mp4",
};
await expect(dispatch(event, depsWith(new FakeGcs()))).rejects.toThrow(/GCS_URI_NOT_ALLOWED/);
} finally {
if (prev === undefined) delete process.env.HYPERFRAMES_RENDER_BUCKET;
else process.env.HYPERFRAMES_RENDER_BUCKET = prev;
}
});
it('treats HYPERFRAMES_RENDER_BUCKET="*" as an explicit opt-out (off-bucket allowed)', async () => {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://any-bucket/renders/r1/plan.tar.gz", PLAN_HASH);
@@ -327,7 +511,45 @@ describe("createApp HTTP mapping", () => {
expect(res.status).toBe(400);
const body = (await res.json()) as { error: string };
expect(body.error).toBe("PlanProtocolUnsupportedError");
expect(body.error).toBe("PLAN_PROTOCOL_UNSUPPORTED");
});
it("returns 400 for plan v2 integrity error names and code aliases", async () => {
for (const error of [
new PlanV2IntegrityError("corrupt test artifact"),
Object.assign(new Error("corrupt test artifact"), {
name: PLAN_V2_INTEGRITY_UNRECOVERABLE,
}),
Object.assign(new Error("corrupt test artifact"), {
code: PLAN_V2_INTEGRITY_UNRECOVERABLE,
}),
]) {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
const app = createApp(
depsWith(gcs, {
renderChunk: async () => {
throw error;
},
}),
);
const res = await app.request("/", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
Format: "mp4",
}),
});
expect(res.status).toBe(400);
const body = (await res.json()) as { error: string };
expect(body.error).toBe(PLAN_V2_INTEGRITY_UNRECOVERABLE);
}
});
it("returns 500 for a retryable/unknown error", async () => {
+336 -5
View File
@@ -28,8 +28,15 @@ import {
type AssembleResult,
type ChunkResult,
type DistributedRenderConfig,
listPlanV2ArtifactsForTarget,
materializePlanV2Target,
plan,
planV2,
type PlanResult,
type PlanV2Artifact,
type PlanV2MaterializationTarget,
type PlanV2Result,
readPlanV2Manifest,
renderChunk,
} from "@hyperframes/producer/distributed";
import { resolveChromeExecutablePath } from "./chromium.js";
@@ -47,9 +54,12 @@ import type {
import { type DistributedFormat, formatExtension } from "./formatExtension.js";
import {
downloadGcsObjectToFile,
downloadGcsObjectToFileVerified,
parseGcsUri,
sha256File,
tarDirectory,
untarDirectory,
uploadContentAddressedFileToGcs,
uploadFileToGcs,
} from "./gcsTransport.js";
@@ -74,6 +84,7 @@ export interface HandlerDeps {
storage?: Storage;
primitives?: {
plan: typeof plan;
planV2?: typeof planV2;
renderChunk: typeof renderChunk;
assemble: typeof assemble;
};
@@ -91,6 +102,7 @@ export interface HandlerDeps {
// fallow-ignore-next-line complexity
export async function dispatch(event: CloudRunEvent, deps?: HandlerDeps): Promise<CloudRunResult> {
const unwrapped = unwrapEvent(event);
validatePlanProtocolShape(unwrapped);
validateEventGcsUris(unwrapped);
logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
try {
@@ -113,9 +125,11 @@ export async function dispatch(event: CloudRunEvent, deps?: HandlerDeps): Promis
}
}
} catch (err) {
normalizeTerminalErrorName(err);
logEvent({
event: "handler_error",
action: unwrapped.Action,
input: summarizeEvent(unwrapped),
message: err instanceof Error ? err.message : String(err),
name: err instanceof Error ? err.name : undefined,
});
@@ -123,6 +137,57 @@ export async function dispatch(event: CloudRunEvent, deps?: HandlerDeps): Promis
}
}
// This is the single fail-closed boundary for the wire union. Keeping all
// forbidden locator combinations together makes mixed-protocol input auditable.
// fallow-ignore-next-line complexity
function validatePlanProtocolShape(event: PlanEvent | RenderChunkEvent | AssembleEvent): void {
const raw = event as unknown as Record<string, unknown>;
const protocol = raw.PlanProtocol;
if (protocol !== undefined && protocol !== "v1" && protocol !== "v2") {
const error = new Error(
`[handler] unsupported PlanProtocol ${JSON.stringify(protocol)}; expected "v1", "v2", or absent`,
);
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
throw error;
}
if (event.Action === "plan") return;
const hasV1Locator = typeof raw.PlanGcsUri === "string";
const hasV2Manifest = typeof raw.PlanV2ManifestGcsUri === "string";
const hasV2Prefix = typeof raw.PlanV2ArtifactGcsPrefix === "string";
const valid =
protocol === "v2"
? !hasV1Locator && hasV2Manifest && hasV2Prefix
: hasV1Locator && !hasV2Manifest && !hasV2Prefix;
if (!valid) {
const error = new Error(
`[handler] ${protocol === "v2" ? "v2" : "v1"} ${event.Action} event has mixed or missing plan locators`,
);
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
throw error;
}
if (protocol === "v2" && event.Action === "assemble" && event.AudioGcsUri !== null) {
const error = new Error("[handler] v2 assemble audio must be materialized from the manifest");
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
throw error;
}
}
/** Normalize producer error codes to the stable HTTP/workflow discriminator. */
// The explicit mapping is the public Cloud Workflows retry contract.
// fallow-ignore-next-line complexity
function normalizeTerminalErrorName(error: unknown): void {
if (!error || typeof error !== "object") return;
const candidate = error as { code?: unknown; name?: string };
if (
candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" ||
candidate.code === "PLAN_TOO_LARGE" ||
candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE"
) {
candidate.name = candidate.code;
}
}
// At most `{Payload: {Input: ...}}` is expected; 4 levels is 2× headroom
// and prevents infinite loops on malformed input.
const MAX_ENVELOPE_DEPTH = 4;
@@ -171,6 +236,8 @@ function logEvent(payload: Record<string, unknown>): void {
* include the entire project config; we only emit the routable fields
* needed to triage a failure from Cloud Logging.
*/
// Keep event variants together so Cloud Logging has one redaction boundary.
// fallow-ignore-next-line complexity
function summarizeEvent(
event: PlanEvent | RenderChunkEvent | AssembleEvent,
): Record<string, unknown> {
@@ -179,18 +246,25 @@ function summarizeEvent(
return {
projectGcsUri: event.ProjectGcsUri,
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
planProtocol: event.PlanProtocol ?? "v1",
format: event.Config.format,
fps: event.Config.fps,
};
case "renderChunk":
return {
planGcsUri: event.PlanGcsUri,
planProtocol: event.PlanProtocol ?? "v1",
...(event.PlanProtocol === "v2"
? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri }
: { planGcsUri: event.PlanGcsUri }),
chunkIndex: event.ChunkIndex,
format: event.Format,
};
case "assemble":
return {
planGcsUri: event.PlanGcsUri,
planProtocol: event.PlanProtocol ?? "v1",
...(event.PlanProtocol === "v2"
? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri }
: { planGcsUri: event.PlanGcsUri }),
chunkCount: event.ChunkGcsUris.length,
hasAudio: event.AudioGcsUri !== null,
outputGcsUri: event.OutputGcsUri,
@@ -215,6 +289,9 @@ function primeChrome(deps?: HandlerDeps): void {
// fallow-ignore-next-line complexity
async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanResultBody> {
if (event.PlanProtocol === "v2") {
return handlePlanV2(event, deps);
}
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.plan ?? plan;
@@ -274,6 +351,79 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanRes
}
}
/**
* Stage immutable v2 artifacts, upload them to content-addressed keys, then
* publish the manifest as the final commit point.
*/
// fallow-ignore-next-line complexity
async function handlePlanV2(
event: Extract<PlanEvent, { PlanProtocol: "v2" }>,
deps?: HandlerDeps,
): Promise<Extract<PlanResultBody, { PlanProtocol: "v2" }>> {
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.planV2 ?? planV2;
primeChrome(deps);
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-plan-v2-"));
const projectArchive = join(work, "project.tar.gz");
const projectDir = join(work, "project");
const planV2Dir = join(work, "plan-v2");
try {
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
await untarDirectory(projectArchive, projectDir);
const result: PlanV2Result = await primitive(projectDir, { ...event.Config }, planV2Dir);
const manifest = readPlanV2Manifest(planV2Dir);
if (manifest.planHash !== result.planHash) {
throwPlanHashMismatch(result.planHash, manifest.planHash);
}
const outputPrefix = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/v2`;
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
const uniqueArtifacts = [
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values(),
];
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
await uploadContentAddressedFileToGcs(
storage,
planV2BlobPath(planV2Dir, artifact.sha256),
planV2BlobUri(artifactPrefix, artifact.sha256),
artifact.sha256,
);
});
const manifestUri = `${outputPrefix}/manifest.json`;
await uploadContentAddressedFileToGcs(
storage,
result.manifestPath,
manifestUri,
await sha256File(result.manifestPath),
"application/json",
);
return {
Action: "plan",
PlanProtocol: "v2",
PlanV2ManifestGcsUri: manifestUri,
PlanV2ArtifactGcsPrefix: artifactPrefix,
PlanHash: result.planHash,
ChunkCount: result.chunkCount,
TotalFrames: result.totalFrames,
Fps: result.fps,
Width: result.width,
Height: result.height,
Format: result.format,
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
AudioGcsUri: null,
FfmpegVersion: result.ffmpegVersion,
ProducerVersion: result.producerVersion,
DurationMs: Date.now() - started,
};
} finally {
cleanupDir(work);
}
}
// ── RenderChunk ─────────────────────────────────────────────────────────────
// fallow-ignore-next-line complexity
@@ -281,6 +431,9 @@ async function handleRenderChunk(
event: RenderChunkEvent,
deps?: HandlerDeps,
): Promise<RenderChunkResultBody> {
if (event.PlanProtocol === "v2") {
return handleRenderChunkV2(event, deps);
}
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
@@ -331,6 +484,51 @@ async function handleRenderChunk(
}
}
/** Materialize only this chunk's verified v2 dependencies before rendering. */
// fallow-ignore-next-line complexity
async function handleRenderChunkV2(
event: Extract<RenderChunkEvent, { PlanProtocol: "v2" }>,
deps?: HandlerDeps,
): Promise<RenderChunkResultBody> {
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
primeChrome(deps);
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-chunk-v2-"));
try {
const planDir = await downloadAndMaterializePlanV2(
storage,
event,
{ role: "chunk", chunkIndex: event.ChunkIndex },
work,
);
const chunkOutputBase = join(
work,
event.Format === "png-sequence"
? `chunk-${pad(event.ChunkIndex)}`
: `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`,
);
const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
const chunkUri = await uploadChunkOutput(
storage,
result,
event.ChunkOutputGcsPrefix,
event.ChunkIndex,
);
return {
Action: "renderChunk",
ChunkGcsUri: chunkUri,
ChunkIndex: event.ChunkIndex,
Sha256: result.sha256,
FramesEncoded: result.framesEncoded,
DurationMs: Date.now() - started,
};
} finally {
cleanupDir(work);
}
}
async function uploadChunkOutput(
storage: Storage,
result: ChunkResult,
@@ -361,6 +559,9 @@ async function handleAssemble(
event: AssembleEvent,
deps?: HandlerDeps,
): Promise<AssembleResultBody> {
if (event.PlanProtocol === "v2") {
return handleAssembleV2(event, deps);
}
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.assemble ?? assemble;
@@ -417,6 +618,127 @@ async function handleAssemble(
}
}
/**
* Materialize the assembler target. Audio is declared assembler-only by the
* v2 manifest and therefore is never downloaded by chunk workers.
*/
// fallow-ignore-next-line complexity
async function handleAssembleV2(
event: Extract<AssembleEvent, { PlanProtocol: "v2" }>,
deps?: HandlerDeps,
): Promise<AssembleResultBody> {
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.assemble ?? assemble;
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-assemble-v2-"));
try {
const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
const audioPath = existsSync(join(planDir, "audio.aac")) ? join(planDir, "audio.aac") : null;
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
const finalOutput =
event.Format === "png-sequence"
? join(work, "output-frames")
: join(work, `output${formatExtension(event.Format)}`);
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
cfr: event.Cfr === true,
});
if (event.Format === "png-sequence") {
const tarball = `${finalOutput}.tar.gz`;
await tarDirectory(finalOutput, tarball);
await uploadFileToGcs(storage, tarball, event.OutputGcsUri, "application/gzip");
} else {
await uploadFileToGcs(storage, finalOutput, event.OutputGcsUri);
}
return {
Action: "assemble",
OutputGcsUri: event.OutputGcsUri,
FramesEncoded: result.framesEncoded,
FileSize: result.fileSize,
DurationMs: Date.now() - started,
};
} finally {
cleanupDir(work);
}
}
async function downloadAndMaterializePlanV2(
storage: Storage,
event: {
PlanV2ManifestGcsUri: string;
PlanV2ArtifactGcsPrefix: string;
PlanHash: string;
},
target: PlanV2MaterializationTarget,
work: string,
): Promise<string> {
const transportDir = join(work, "plan-v2");
mkdirSync(transportDir, { recursive: true });
await downloadGcsObjectToFile(
storage,
event.PlanV2ManifestGcsUri,
join(transportDir, "plan.json"),
);
const manifest = readPlanV2Manifest(transportDir);
if (manifest.planHash !== event.PlanHash) {
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
}
const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
const uniqueArtifacts = [
...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values(),
];
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
await downloadPlanV2Artifact(storage, event.PlanV2ArtifactGcsPrefix, transportDir, artifact);
});
const planDir = join(work, "plan");
materializePlanV2Target(transportDir, target, planDir);
return planDir;
}
async function downloadPlanV2Artifact(
storage: Storage,
artifactPrefix: string,
planV2Dir: string,
artifact: Readonly<PlanV2Artifact>,
): Promise<void> {
await downloadGcsObjectToFileVerified(
storage,
planV2BlobUri(artifactPrefix, artifact.sha256),
planV2BlobPath(planV2Dir, artifact.sha256),
artifact.sha256,
);
}
function planV2BlobPath(planV2Dir: string, digest: string): string {
return join(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
}
function planV2BlobUri(prefix: string, digest: string): string {
return `${trimTrailingSlash(prefix)}/${digest.slice(0, 2)}/${digest}`;
}
function throwPlanHashMismatch(expected: string, actual: string): never {
const error = new Error(
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match v2 manifest planHash=${actual}`,
);
error.name = "PLAN_HASH_MISMATCH";
throw error;
}
async function mapConcurrent<T>(
values: readonly T[],
concurrency: number,
fn: (value: T) => Promise<void>,
): Promise<void> {
let cursor = 0;
async function worker(): Promise<void> {
while (cursor < values.length) {
const index = cursor++;
await fn(values[index]!);
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
}
async function downloadChunkObjects(
storage: Storage,
uris: string[],
@@ -454,15 +776,21 @@ async function downloadChunkObjects(
// ── Helpers ─────────────────────────────────────────────────────────────────
/** Collect every GCS URI that the handler will touch for a given event. */
// This exhaustive event projection is the bucket-allowlist security boundary.
// fallow-ignore-next-line complexity
function getEventGcsUris(event: PlanEvent | RenderChunkEvent | AssembleEvent): string[] {
switch (event.Action) {
case "plan":
return [event.ProjectGcsUri, event.PlanOutputGcsPrefix];
case "renderChunk":
return [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
return event.PlanProtocol === "v2"
? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix, event.ChunkOutputGcsPrefix]
: [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
case "assemble":
return [
event.PlanGcsUri,
...(event.PlanProtocol === "v2"
? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix]
: [event.PlanGcsUri]),
...event.ChunkGcsUris,
event.OutputGcsUri,
event.AudioGcsUri,
@@ -578,6 +906,9 @@ const NON_RETRYABLE_ERROR_NAMES = new Set([
// Handler-boundary guards.
"GCS_URI_NOT_ALLOWED",
"PLAN_HASH_MISMATCH",
"PLAN_ARTIFACT_DIGEST_MISMATCH",
"PLAN_PROTOCOL_UNSUPPORTED",
"PLAN_V2_INTEGRITY_UNRECOVERABLE",
// Producer error class names (`.name`) + their string code aliases — the
// class sets `.name` to the class name but wraps a `code`; cover both so a
// raw-code throw is caught too. Mirrors the AWS state machine's
@@ -585,11 +916,11 @@ const NON_RETRYABLE_ERROR_NAMES = new Set([
"FormatNotSupportedInDistributedError",
"PlanTooLargeError",
"PlanProtocolUnsupportedError",
"PlanV2IntegrityError",
"RenderChunkValidationError",
"FFMPEG_VERSION_MISMATCH",
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
"PLAN_TOO_LARGE",
"PLAN_PROTOCOL_UNSUPPORTED",
"BROWSER_GPU_NOT_SOFTWARE",
"FONT_FETCH_FAILED",
"ChromeBinaryUnavailableError",
+5
View File
@@ -133,6 +133,11 @@ resource "google_workflows_workflow" "render" {
region = var.region
service_account = google_service_account.workflow_sa.id
source_contents = file(local.workflow_source)
# Do not publish an executable workflow until its identity has permission to
# reach Cloud Run. IAM propagation remains eventually consistent, so the
# workflow also retries the edge's transient 403 response with bounded
# backoff.
depends_on = [google_cloud_run_v2_service_iam_member.workflow_invokes_run]
# Allow `terraform destroy` to remove the workflow without a manual step;
# the definition is reproducible from this module.
deletion_protection = false
@@ -3,6 +3,16 @@ output "render_bucket_name" {
value = google_storage_bucket.render.name
}
output "project_name" {
description = "Resource prefix used by this deployment."
value = var.project_name
}
output "render_service_name" {
description = "Cloud Run service name."
value = google_cloud_run_v2_service.render.name
}
output "service_url" {
description = "HTTPS URL of the Cloud Run render service. Pass as renderToCloudRun({ serviceUrl })."
value = google_cloud_run_v2_service.render.uri
@@ -0,0 +1,71 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "bun:test";
const smokePath = join(import.meta.dir, "../../../examples/gcp-cloud-run/scripts/smoke.sh");
const smoke = readFileSync(smokePath, "utf-8");
const dockerfile = readFileSync(join(import.meta.dir, "../Dockerfile"), "utf-8");
describe("GCP smoke ownership and protocol safety", () => {
it("defaults to v1 and requires an explicit v2 protocol argument", () => {
expect(smoke).toContain('PROTOCOLS="${PROTOCOLS:-v1}"');
expect(smoke).toContain("--protocols)");
expect(smoke).toContain("PlanProtocol: $protocol");
expect(smoke).toContain("decodedFramesEqual");
expect(smoke).toContain("decodedAudioEqual");
expect(smoke).toContain("normalizedMetadataEqual");
});
it("derives a length-safe owner prefix and isolates Terraform state", () => {
expect(smoke).toContain('OWNER_HASH="$(printf');
expect(smoke).toContain("RUN_NONCE=");
expect(smoke).toContain('STACK_NAME="hf-smoke-$OWNER_HASH"');
expect(smoke).toContain('TF_WORK_DIR="$ARTIFACT_DIR/terraform"');
expect(smoke).toContain('TF_DATA_DIR="$ARTIFACT_DIR/terraform-data"');
expect(smoke).toContain("export TF_DATA_DIR");
expect(smoke).toContain('-var "project_name=$STACK_NAME"');
expect(smoke).toContain('[ "$STACK_NAME" != "hyperframes" ]');
});
it("tracks owned registry resources and verifies stack deletion", () => {
expect(smoke).toContain("CREATED_IMAGE=0");
expect(smoke).toContain("CREATED_REPO=0");
expect(smoke).toContain('if [ "$CREATED_REPO" -eq 1 ]');
expect(smoke).toContain('if [ "$CREATED_IMAGE" -eq 1 ]');
for (const resource of [
"cloud-run-service",
"workflow",
"render-bucket",
"artifact-image",
"artifact-repository",
"cloud-build-staging-bucket",
]) {
expect(smoke).toContain(`verify_absent "${resource}"`);
}
expect(smoke).toContain('verify_service_account_absent "run-service-account"');
expect(smoke).toContain('verify_service_account_absent "workflow-service-account"');
expect(smoke).toContain('verify_absent "preflight-cloud-run-service"');
expect(smoke).toContain('verify_absent "preflight-artifact-image"');
expect(smoke).toContain("$STACK_NAME-render");
expect(smoke).toContain("--ignore-file");
expect(smoke).toContain("--gcs-source-staging-dir");
expect(smoke).toContain("!scripts/package-subpaths.mjs");
expect(dockerfile).toContain("COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs");
expect(smoke).not.toContain("gcloud services enable");
expect(smoke).toContain("gcloud services list");
expect(smoke).toContain("--enabled");
expect(smoke).not.toContain("gcloud services describe");
expect(smoke).toContain("cannot find");
expect(smoke).toContain("gcloud iam service-accounts list");
expect(smoke).toContain('--filter "email:$email"');
});
it("does not swallow Terraform cleanup failures", () => {
const cleanupStart = smoke.indexOf("cleanup() {");
const cleanupEnd = smoke.indexOf("\ntrap cleanup EXIT", cleanupStart);
const cleanup = smoke.slice(cleanupStart, cleanupEnd);
expect(cleanup).not.toContain("|| true");
expect(cleanup).toContain("exit 7");
});
});
@@ -13,6 +13,15 @@ variable "project_name" {
type = string
description = "Name prefix applied to the service / workflow / bucket / service accounts."
default = "hyperframes"
validation {
condition = (
length(var.project_name) >= 3 &&
length(var.project_name) <= 23 &&
can(regex("^[a-z][a-z0-9-]*[a-z0-9]$", var.project_name))
)
error_message = "project_name must be 3-23 lowercase letters, digits, or hyphens, begin with a letter, and end with a letter or digit so derived service-account IDs remain valid."
}
}
variable "image" {
@@ -0,0 +1,148 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "bun:test";
import { parse } from "yaml";
type Step = Record<string, unknown>;
const source = readFileSync(join(import.meta.dir, "workflow.yaml"), "utf-8");
// Cloud Workflows expressions are valid to Google's parser but `${...}`
// inside YAML flow collections is not valid generic YAML. Quote expressions
// for structural parsing while preserving their text for contract assertions.
const parseableSource = source.replace(/\$\{([^}]*)\}/g, (_match, expression: string) =>
JSON.stringify(`\${${expression}}`),
);
const workflow = parse(parseableSource) as {
main: {
steps: Step[];
};
retryable: {
steps: Step[];
};
};
function namedStep(name: string, steps = workflow.main.steps): Record<string, unknown> {
for (const step of steps) {
if (name in step) return step[name] as Record<string, unknown>;
}
throw new Error(`missing workflow step ${name}`);
}
function requestBody(stepName: string): Record<string, unknown> {
const step = namedStep(stepName);
const attempt = step.try as {
args: {
body: Record<string, unknown>;
};
};
return attempt.args.body;
}
function requestAuth(stepName: string): Record<string, unknown> {
const step = namedStep(stepName);
const attempt = step.try as {
args: {
auth: Record<string, unknown>;
};
};
return attempt.args.auth;
}
function chunkRequestBody(stepName: string): Record<string, unknown> {
const renderChunks = namedStep("renderChunks");
const parallel = renderChunks.parallel as {
for: {
steps: Step[];
};
};
const step = namedStep(stepName, parallel.for.steps);
const attempt = step.try as {
args: {
body: Record<string, unknown>;
};
};
return attempt.args.body;
}
describe("Cloud Workflows plan protocol routing", () => {
it("pins OIDC tokens to the Cloud Run root and retries IAM propagation", () => {
for (const stepName of ["planV1", "planV2", "assembleV1", "assembleV2"]) {
expect(requestAuth(stepName)).toEqual({
type: "OIDC",
audience: "${serviceUrl}",
});
}
expect(JSON.stringify(workflow.retryable)).toContain("e.code == 403");
expect(source.match(/max_retries: 6/g)).toHaveLength(2);
expect(source.match(/max_retries: 4/g)).toHaveLength(4);
});
it("keeps v1 as the default and rejects unknown protocols before plan", () => {
expect(source).toContain('default(map.get(args, "PlanProtocol"), "v1")');
expect(namedStep("selectPlanProtocol")).toMatchObject({
next: "unsupportedPlanProtocol",
});
expect(namedStep("unsupportedPlanProtocol")).toMatchObject({
raise: {
code: "PLAN_PROTOCOL_UNSUPPORTED",
},
});
});
it("uses disjoint v1 and v2 plan request/response contracts", () => {
expect(requestBody("planV1")).toMatchObject({
Action: "plan",
PlanProtocol: "v1",
});
expect(requestBody("planV2")).toMatchObject({
Action: "plan",
PlanProtocol: "v2",
});
const validation = JSON.stringify(namedStep("validatePlanResult"));
expect(validation).toContain("PlanGcsUri");
expect(validation).toContain("PlanV2ManifestGcsUri");
expect(validation).toContain("PlanV2ArtifactGcsPrefix");
expect(source).toContain('not("PlanGcsUri" in planResult)');
expect(source).toContain(
'(not("PlanProtocol" in planResult) or planResult.PlanProtocol == "v1")',
);
});
it("never mixes v1 and v2 chunk locators", () => {
const v1 = chunkRequestBody("renderOneChunkV1");
expect(v1).toMatchObject({
Action: "renderChunk",
PlanProtocol: "v1",
});
expect(v1).toHaveProperty("PlanGcsUri");
expect(v1).not.toHaveProperty("PlanV2ManifestGcsUri");
expect(v1).not.toHaveProperty("PlanV2ArtifactGcsPrefix");
const v2 = chunkRequestBody("renderOneChunkV2");
expect(v2).toMatchObject({
Action: "renderChunk",
PlanProtocol: "v2",
});
expect(v2).not.toHaveProperty("PlanGcsUri");
expect(v2).toHaveProperty("PlanV2ManifestGcsUri");
expect(v2).toHaveProperty("PlanV2ArtifactGcsPrefix");
expect(v2).toHaveProperty("PlanHash");
});
it("never mixes v1 and v2 assembler locators", () => {
const v1 = requestBody("assembleV1");
expect(v1).toHaveProperty("PlanGcsUri");
expect(v1).not.toHaveProperty("PlanV2ManifestGcsUri");
expect(v1).not.toHaveProperty("PlanV2ArtifactGcsPrefix");
const v2 = requestBody("assembleV2");
expect(v2).not.toHaveProperty("PlanGcsUri");
expect(v2).toHaveProperty("PlanV2ManifestGcsUri");
expect(v2).toHaveProperty("PlanV2ArtifactGcsPrefix");
expect(v2).toHaveProperty("PlanHash");
expect(v2).toMatchObject({
PlanProtocol: "v2",
AudioGcsUri: null,
});
});
});
+149 -10
View File
@@ -26,9 +26,23 @@ main:
- planOutputGcsPrefix: ${args.PlanOutputGcsPrefix}
- outputGcsUri: ${args.OutputGcsUri}
- config: ${args.Config}
# Backward-compatible default. v2 is accepted only through an
# explicit top-level PlanProtocol opt-in.
- planProtocol: ${default(map.get(args, "PlanProtocol"), "v1")}
# ── Plan (Activity A) ────────────────────────────────────────────────────
- plan:
- selectPlanProtocol:
switch:
- condition: ${planProtocol == "v1"}
next: planV1
- condition: ${planProtocol == "v2"}
next: planV2
next: unsupportedPlanProtocol
- unsupportedPlanProtocol:
raise:
code: PLAN_PROTOCOL_UNSUPPORTED
message: ${"PlanProtocol must be v1 or v2; got " + string(planProtocol)}
- planV1:
try:
call: http.post
args:
@@ -36,22 +50,69 @@ main:
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: plan
PlanProtocol: v1
ProjectGcsUri: ${projectGcsUri}
PlanOutputGcsPrefix: ${planOutputGcsPrefix}
Config: ${config}
result: planResp
result: planRespV1
retry:
predicate: ${retryable}
max_retries: 4
max_retries: 6
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
- capturePlan:
next: capturePlanV1
- capturePlanV1:
assign:
- planResult: ${planRespV1.body}
next: validatePlanResult
- planV2:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: plan
PlanProtocol: v2
ProjectGcsUri: ${projectGcsUri}
PlanOutputGcsPrefix: ${planOutputGcsPrefix}
Config: ${config}
result: planRespV2
retry:
predicate: ${retryable}
max_retries: 6
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
next: capturePlanV2
- capturePlanV2:
assign:
- planResult: ${planRespV2.body}
next: validatePlanResult
- validatePlanResult:
# Fail closed before fan-out. A v2 render may never fall back to a
# v1 PlanGcsUri, and a v1 render may never consume v2 locators.
switch:
- condition: ${planProtocol == "v1" and (not("PlanProtocol" in planResult) or planResult.PlanProtocol == "v1") and ("PlanGcsUri" in planResult) and not("PlanV2ManifestGcsUri" in planResult) and not("PlanV2ArtifactGcsPrefix" in planResult)}
next: captureChunkCount
- condition: ${planProtocol == "v2" and ("PlanProtocol" in planResult) and planResult.PlanProtocol == "v2" and ("PlanV2ManifestGcsUri" in planResult) and ("PlanV2ArtifactGcsPrefix" in planResult) and not("PlanGcsUri" in planResult)}
next: captureChunkCount
next: planProtocolLocatorMismatch
- planProtocolLocatorMismatch:
raise:
code: PLAN_PROTOCOL_LOCATOR_MISMATCH
message: "Plan response did not match the selected protocol's disjoint locator contract."
- captureChunkCount:
assign:
- planResult: ${planResp.body}
- chunkCount: ${planResult.ChunkCount}
# ── BuildChunkList + AssertChunkCount ──────────────────────────────────────
@@ -98,7 +159,12 @@ main:
value: idx
in: ${chunkIndexes}
steps:
- renderOneChunk:
- selectChunkProtocol:
switch:
- condition: ${planProtocol == "v2"}
next: renderOneChunkV2
next: renderOneChunkV1
- renderOneChunkV1:
try:
call: http.post
args:
@@ -106,8 +172,10 @@ main:
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: renderChunk
PlanProtocol: v1
ChunkIndex: ${idx}
PlanGcsUri: ${planResult.PlanGcsUri}
PlanHash: ${planResult.PlanHash}
@@ -121,13 +189,46 @@ main:
initial_delay: 2
max_delay: 60
multiplier: 2
next: storeChunk
- renderOneChunkV2:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: renderChunk
PlanProtocol: v2
ChunkIndex: ${idx}
PlanV2ManifestGcsUri: ${planResult.PlanV2ManifestGcsUri}
PlanV2ArtifactGcsPrefix: ${planResult.PlanV2ArtifactGcsPrefix}
PlanHash: ${planResult.PlanHash}
ChunkOutputGcsPrefix: ${planOutputGcsPrefix}
Format: ${planResult.Format}
result: chunkResp
retry:
predicate: ${retryable}
max_retries: 4
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
next: storeChunk
- storeChunk:
assign:
- chunkUris[idx]: ${chunkResp.body.ChunkGcsUri}
- chunkResults[idx]: ${chunkResp.body}
# ── Assemble (Activity C) ──────────────────────────────────────────────────
- assemble:
- selectAssembleProtocol:
switch:
- condition: ${planProtocol == "v2"}
next: assembleV2
next: assembleV1
- assembleV1:
try:
call: http.post
args:
@@ -135,8 +236,10 @@ main:
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: assemble
PlanProtocol: v1
PlanGcsUri: ${planResult.PlanGcsUri}
ChunkGcsUris: ${chunkUris}
AudioGcsUri: ${planResult.AudioGcsUri}
@@ -154,6 +257,38 @@ main:
initial_delay: 2
max_delay: 60
multiplier: 2
next: done
- assembleV2:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: assemble
PlanProtocol: v2
PlanV2ManifestGcsUri: ${planResult.PlanV2ManifestGcsUri}
PlanV2ArtifactGcsPrefix: ${planResult.PlanV2ArtifactGcsPrefix}
PlanHash: ${planResult.PlanHash}
ChunkGcsUris: ${chunkUris}
# Audio is an assembler-scoped v2 artifact and is materialized
# from the manifest, never carried through a v1 AudioGcsUri.
AudioGcsUri: null
OutputGcsUri: ${outputGcsUri}
Format: ${planResult.Format}
Cfr: ${("cfr" in config) and config.cfr}
result: assembleResp
retry:
predicate: ${retryable}
max_retries: 4
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
next: done
- done:
return:
@@ -161,9 +296,11 @@ main:
Chunks: ${chunkResults}
Assemble: ${assembleResp.body}
# Retry predicate: retry transient/server failures (429 + 5xx), never the
# handler's non-retryable 400s (bad input, plan-hash mismatch, unsupported
# format, …). Connection / timeout errors carry no `.code`; retry those too.
# Retry predicate: retry transient/server failures (403 from Cloud Run IAM
# propagation, 429, and 5xx), never the handler's non-retryable 400s (bad
# input, plan-hash mismatch, unsupported format, …). The handler does not emit
# 403, so that status is always from Cloud Run's authentication edge.
# Connection / timeout errors carry no `.code`; retry those too.
retryable:
params: [e]
steps:
@@ -173,6 +310,8 @@ retryable:
return: true
- condition: ${e.code == 429}
return: true
- condition: ${e.code == 403}
return: true
- condition: ${e.code >= 500 and e.code < 600}
return: true
- nonRetryable: