fix(gcp-cloud-run): normalize v2 integrity codes (#2790)

This commit is contained in:
James Russo
2026-07-25 23:59:36 -04:00
committed by GitHub
parent 5bf61d6df0
commit 0499a5cbcb
27 changed files with 2093 additions and 260 deletions
+5
View File
@@ -133,6 +133,11 @@ resource "google_workflows_workflow" "render" {
region = var.region
service_account = google_service_account.workflow_sa.id
source_contents = file(local.workflow_source)
# Do not publish an executable workflow until its identity has permission to
# reach Cloud Run. IAM propagation remains eventually consistent, so the
# workflow also retries the edge's transient 403 response with bounded
# backoff.
depends_on = [google_cloud_run_v2_service_iam_member.workflow_invokes_run]
# Allow `terraform destroy` to remove the workflow without a manual step;
# the definition is reproducible from this module.
deletion_protection = false
@@ -3,6 +3,16 @@ output "render_bucket_name" {
value = google_storage_bucket.render.name
}
output "project_name" {
description = "Resource prefix used by this deployment."
value = var.project_name
}
output "render_service_name" {
description = "Cloud Run service name."
value = google_cloud_run_v2_service.render.name
}
output "service_url" {
description = "HTTPS URL of the Cloud Run render service. Pass as renderToCloudRun({ serviceUrl })."
value = google_cloud_run_v2_service.render.uri
@@ -0,0 +1,71 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "bun:test";
const smokePath = join(import.meta.dir, "../../../examples/gcp-cloud-run/scripts/smoke.sh");
const smoke = readFileSync(smokePath, "utf-8");
const dockerfile = readFileSync(join(import.meta.dir, "../Dockerfile"), "utf-8");
describe("GCP smoke ownership and protocol safety", () => {
it("defaults to v1 and requires an explicit v2 protocol argument", () => {
expect(smoke).toContain('PROTOCOLS="${PROTOCOLS:-v1}"');
expect(smoke).toContain("--protocols)");
expect(smoke).toContain("PlanProtocol: $protocol");
expect(smoke).toContain("decodedFramesEqual");
expect(smoke).toContain("decodedAudioEqual");
expect(smoke).toContain("normalizedMetadataEqual");
});
it("derives a length-safe owner prefix and isolates Terraform state", () => {
expect(smoke).toContain('OWNER_HASH="$(printf');
expect(smoke).toContain("RUN_NONCE=");
expect(smoke).toContain('STACK_NAME="hf-smoke-$OWNER_HASH"');
expect(smoke).toContain('TF_WORK_DIR="$ARTIFACT_DIR/terraform"');
expect(smoke).toContain('TF_DATA_DIR="$ARTIFACT_DIR/terraform-data"');
expect(smoke).toContain("export TF_DATA_DIR");
expect(smoke).toContain('-var "project_name=$STACK_NAME"');
expect(smoke).toContain('[ "$STACK_NAME" != "hyperframes" ]');
});
it("tracks owned registry resources and verifies stack deletion", () => {
expect(smoke).toContain("CREATED_IMAGE=0");
expect(smoke).toContain("CREATED_REPO=0");
expect(smoke).toContain('if [ "$CREATED_REPO" -eq 1 ]');
expect(smoke).toContain('if [ "$CREATED_IMAGE" -eq 1 ]');
for (const resource of [
"cloud-run-service",
"workflow",
"render-bucket",
"artifact-image",
"artifact-repository",
"cloud-build-staging-bucket",
]) {
expect(smoke).toContain(`verify_absent "${resource}"`);
}
expect(smoke).toContain('verify_service_account_absent "run-service-account"');
expect(smoke).toContain('verify_service_account_absent "workflow-service-account"');
expect(smoke).toContain('verify_absent "preflight-cloud-run-service"');
expect(smoke).toContain('verify_absent "preflight-artifact-image"');
expect(smoke).toContain("$STACK_NAME-render");
expect(smoke).toContain("--ignore-file");
expect(smoke).toContain("--gcs-source-staging-dir");
expect(smoke).toContain("!scripts/package-subpaths.mjs");
expect(dockerfile).toContain("COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs");
expect(smoke).not.toContain("gcloud services enable");
expect(smoke).toContain("gcloud services list");
expect(smoke).toContain("--enabled");
expect(smoke).not.toContain("gcloud services describe");
expect(smoke).toContain("cannot find");
expect(smoke).toContain("gcloud iam service-accounts list");
expect(smoke).toContain('--filter "email:$email"');
});
it("does not swallow Terraform cleanup failures", () => {
const cleanupStart = smoke.indexOf("cleanup() {");
const cleanupEnd = smoke.indexOf("\ntrap cleanup EXIT", cleanupStart);
const cleanup = smoke.slice(cleanupStart, cleanupEnd);
expect(cleanup).not.toContain("|| true");
expect(cleanup).toContain("exit 7");
});
});
@@ -13,6 +13,15 @@ variable "project_name" {
type = string
description = "Name prefix applied to the service / workflow / bucket / service accounts."
default = "hyperframes"
validation {
condition = (
length(var.project_name) >= 3 &&
length(var.project_name) <= 23 &&
can(regex("^[a-z][a-z0-9-]*[a-z0-9]$", var.project_name))
)
error_message = "project_name must be 3-23 lowercase letters, digits, or hyphens, begin with a letter, and end with a letter or digit so derived service-account IDs remain valid."
}
}
variable "image" {
@@ -0,0 +1,148 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "bun:test";
import { parse } from "yaml";
type Step = Record<string, unknown>;
const source = readFileSync(join(import.meta.dir, "workflow.yaml"), "utf-8");
// Cloud Workflows expressions are valid to Google's parser but `${...}`
// inside YAML flow collections is not valid generic YAML. Quote expressions
// for structural parsing while preserving their text for contract assertions.
const parseableSource = source.replace(/\$\{([^}]*)\}/g, (_match, expression: string) =>
JSON.stringify(`\${${expression}}`),
);
const workflow = parse(parseableSource) as {
main: {
steps: Step[];
};
retryable: {
steps: Step[];
};
};
function namedStep(name: string, steps = workflow.main.steps): Record<string, unknown> {
for (const step of steps) {
if (name in step) return step[name] as Record<string, unknown>;
}
throw new Error(`missing workflow step ${name}`);
}
function requestBody(stepName: string): Record<string, unknown> {
const step = namedStep(stepName);
const attempt = step.try as {
args: {
body: Record<string, unknown>;
};
};
return attempt.args.body;
}
function requestAuth(stepName: string): Record<string, unknown> {
const step = namedStep(stepName);
const attempt = step.try as {
args: {
auth: Record<string, unknown>;
};
};
return attempt.args.auth;
}
function chunkRequestBody(stepName: string): Record<string, unknown> {
const renderChunks = namedStep("renderChunks");
const parallel = renderChunks.parallel as {
for: {
steps: Step[];
};
};
const step = namedStep(stepName, parallel.for.steps);
const attempt = step.try as {
args: {
body: Record<string, unknown>;
};
};
return attempt.args.body;
}
describe("Cloud Workflows plan protocol routing", () => {
it("pins OIDC tokens to the Cloud Run root and retries IAM propagation", () => {
for (const stepName of ["planV1", "planV2", "assembleV1", "assembleV2"]) {
expect(requestAuth(stepName)).toEqual({
type: "OIDC",
audience: "${serviceUrl}",
});
}
expect(JSON.stringify(workflow.retryable)).toContain("e.code == 403");
expect(source.match(/max_retries: 6/g)).toHaveLength(2);
expect(source.match(/max_retries: 4/g)).toHaveLength(4);
});
it("keeps v1 as the default and rejects unknown protocols before plan", () => {
expect(source).toContain('default(map.get(args, "PlanProtocol"), "v1")');
expect(namedStep("selectPlanProtocol")).toMatchObject({
next: "unsupportedPlanProtocol",
});
expect(namedStep("unsupportedPlanProtocol")).toMatchObject({
raise: {
code: "PLAN_PROTOCOL_UNSUPPORTED",
},
});
});
it("uses disjoint v1 and v2 plan request/response contracts", () => {
expect(requestBody("planV1")).toMatchObject({
Action: "plan",
PlanProtocol: "v1",
});
expect(requestBody("planV2")).toMatchObject({
Action: "plan",
PlanProtocol: "v2",
});
const validation = JSON.stringify(namedStep("validatePlanResult"));
expect(validation).toContain("PlanGcsUri");
expect(validation).toContain("PlanV2ManifestGcsUri");
expect(validation).toContain("PlanV2ArtifactGcsPrefix");
expect(source).toContain('not("PlanGcsUri" in planResult)');
expect(source).toContain(
'(not("PlanProtocol" in planResult) or planResult.PlanProtocol == "v1")',
);
});
it("never mixes v1 and v2 chunk locators", () => {
const v1 = chunkRequestBody("renderOneChunkV1");
expect(v1).toMatchObject({
Action: "renderChunk",
PlanProtocol: "v1",
});
expect(v1).toHaveProperty("PlanGcsUri");
expect(v1).not.toHaveProperty("PlanV2ManifestGcsUri");
expect(v1).not.toHaveProperty("PlanV2ArtifactGcsPrefix");
const v2 = chunkRequestBody("renderOneChunkV2");
expect(v2).toMatchObject({
Action: "renderChunk",
PlanProtocol: "v2",
});
expect(v2).not.toHaveProperty("PlanGcsUri");
expect(v2).toHaveProperty("PlanV2ManifestGcsUri");
expect(v2).toHaveProperty("PlanV2ArtifactGcsPrefix");
expect(v2).toHaveProperty("PlanHash");
});
it("never mixes v1 and v2 assembler locators", () => {
const v1 = requestBody("assembleV1");
expect(v1).toHaveProperty("PlanGcsUri");
expect(v1).not.toHaveProperty("PlanV2ManifestGcsUri");
expect(v1).not.toHaveProperty("PlanV2ArtifactGcsPrefix");
const v2 = requestBody("assembleV2");
expect(v2).not.toHaveProperty("PlanGcsUri");
expect(v2).toHaveProperty("PlanV2ManifestGcsUri");
expect(v2).toHaveProperty("PlanV2ArtifactGcsPrefix");
expect(v2).toHaveProperty("PlanHash");
expect(v2).toMatchObject({
PlanProtocol: "v2",
AudioGcsUri: null,
});
});
});
+149 -10
View File
@@ -26,9 +26,23 @@ main:
- planOutputGcsPrefix: ${args.PlanOutputGcsPrefix}
- outputGcsUri: ${args.OutputGcsUri}
- config: ${args.Config}
# Backward-compatible default. v2 is accepted only through an
# explicit top-level PlanProtocol opt-in.
- planProtocol: ${default(map.get(args, "PlanProtocol"), "v1")}
# ── Plan (Activity A) ────────────────────────────────────────────────────
- plan:
- selectPlanProtocol:
switch:
- condition: ${planProtocol == "v1"}
next: planV1
- condition: ${planProtocol == "v2"}
next: planV2
next: unsupportedPlanProtocol
- unsupportedPlanProtocol:
raise:
code: PLAN_PROTOCOL_UNSUPPORTED
message: ${"PlanProtocol must be v1 or v2; got " + string(planProtocol)}
- planV1:
try:
call: http.post
args:
@@ -36,22 +50,69 @@ main:
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: plan
PlanProtocol: v1
ProjectGcsUri: ${projectGcsUri}
PlanOutputGcsPrefix: ${planOutputGcsPrefix}
Config: ${config}
result: planResp
result: planRespV1
retry:
predicate: ${retryable}
max_retries: 4
max_retries: 6
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
- capturePlan:
next: capturePlanV1
- capturePlanV1:
assign:
- planResult: ${planRespV1.body}
next: validatePlanResult
- planV2:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: plan
PlanProtocol: v2
ProjectGcsUri: ${projectGcsUri}
PlanOutputGcsPrefix: ${planOutputGcsPrefix}
Config: ${config}
result: planRespV2
retry:
predicate: ${retryable}
max_retries: 6
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
next: capturePlanV2
- capturePlanV2:
assign:
- planResult: ${planRespV2.body}
next: validatePlanResult
- validatePlanResult:
# Fail closed before fan-out. A v2 render may never fall back to a
# v1 PlanGcsUri, and a v1 render may never consume v2 locators.
switch:
- condition: ${planProtocol == "v1" and (not("PlanProtocol" in planResult) or planResult.PlanProtocol == "v1") and ("PlanGcsUri" in planResult) and not("PlanV2ManifestGcsUri" in planResult) and not("PlanV2ArtifactGcsPrefix" in planResult)}
next: captureChunkCount
- condition: ${planProtocol == "v2" and ("PlanProtocol" in planResult) and planResult.PlanProtocol == "v2" and ("PlanV2ManifestGcsUri" in planResult) and ("PlanV2ArtifactGcsPrefix" in planResult) and not("PlanGcsUri" in planResult)}
next: captureChunkCount
next: planProtocolLocatorMismatch
- planProtocolLocatorMismatch:
raise:
code: PLAN_PROTOCOL_LOCATOR_MISMATCH
message: "Plan response did not match the selected protocol's disjoint locator contract."
- captureChunkCount:
assign:
- planResult: ${planResp.body}
- chunkCount: ${planResult.ChunkCount}
# ── BuildChunkList + AssertChunkCount ──────────────────────────────────────
@@ -98,7 +159,12 @@ main:
value: idx
in: ${chunkIndexes}
steps:
- renderOneChunk:
- selectChunkProtocol:
switch:
- condition: ${planProtocol == "v2"}
next: renderOneChunkV2
next: renderOneChunkV1
- renderOneChunkV1:
try:
call: http.post
args:
@@ -106,8 +172,10 @@ main:
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: renderChunk
PlanProtocol: v1
ChunkIndex: ${idx}
PlanGcsUri: ${planResult.PlanGcsUri}
PlanHash: ${planResult.PlanHash}
@@ -121,13 +189,46 @@ main:
initial_delay: 2
max_delay: 60
multiplier: 2
next: storeChunk
- renderOneChunkV2:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: renderChunk
PlanProtocol: v2
ChunkIndex: ${idx}
PlanV2ManifestGcsUri: ${planResult.PlanV2ManifestGcsUri}
PlanV2ArtifactGcsPrefix: ${planResult.PlanV2ArtifactGcsPrefix}
PlanHash: ${planResult.PlanHash}
ChunkOutputGcsPrefix: ${planOutputGcsPrefix}
Format: ${planResult.Format}
result: chunkResp
retry:
predicate: ${retryable}
max_retries: 4
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
next: storeChunk
- storeChunk:
assign:
- chunkUris[idx]: ${chunkResp.body.ChunkGcsUri}
- chunkResults[idx]: ${chunkResp.body}
# ── Assemble (Activity C) ──────────────────────────────────────────────────
- assemble:
- selectAssembleProtocol:
switch:
- condition: ${planProtocol == "v2"}
next: assembleV2
next: assembleV1
- assembleV1:
try:
call: http.post
args:
@@ -135,8 +236,10 @@ main:
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: assemble
PlanProtocol: v1
PlanGcsUri: ${planResult.PlanGcsUri}
ChunkGcsUris: ${chunkUris}
AudioGcsUri: ${planResult.AudioGcsUri}
@@ -154,6 +257,38 @@ main:
initial_delay: 2
max_delay: 60
multiplier: 2
next: done
- assembleV2:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
audience: ${serviceUrl}
body:
Action: assemble
PlanProtocol: v2
PlanV2ManifestGcsUri: ${planResult.PlanV2ManifestGcsUri}
PlanV2ArtifactGcsPrefix: ${planResult.PlanV2ArtifactGcsPrefix}
PlanHash: ${planResult.PlanHash}
ChunkGcsUris: ${chunkUris}
# Audio is an assembler-scoped v2 artifact and is materialized
# from the manifest, never carried through a v1 AudioGcsUri.
AudioGcsUri: null
OutputGcsUri: ${outputGcsUri}
Format: ${planResult.Format}
Cfr: ${("cfr" in config) and config.cfr}
result: assembleResp
retry:
predicate: ${retryable}
max_retries: 4
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
next: done
- done:
return:
@@ -161,9 +296,11 @@ main:
Chunks: ${chunkResults}
Assemble: ${assembleResp.body}
# Retry predicate: retry transient/server failures (429 + 5xx), never the
# handler's non-retryable 400s (bad input, plan-hash mismatch, unsupported
# format, …). Connection / timeout errors carry no `.code`; retry those too.
# Retry predicate: retry transient/server failures (403 from Cloud Run IAM
# propagation, 429, and 5xx), never the handler's non-retryable 400s (bad
# input, plan-hash mismatch, unsupported format, …). The handler does not emit
# 403, so that status is always from Cloud Run's authentication edge.
# Connection / timeout errors carry no `.code`; retry those too.
retryable:
params: [e]
steps:
@@ -173,6 +310,8 @@ retryable:
return: true
- condition: ${e.code == 429}
return: true
- condition: ${e.code == 403}
return: true
- condition: ${e.code >= 500 and e.code < 600}
return: true
- nonRetryable: