mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-04 16:42:27 +00:00
fix(gcp-cloud-run): normalize v2 integrity codes (#2790)
This commit is contained in:
@@ -133,6 +133,11 @@ resource "google_workflows_workflow" "render" {
|
||||
region = var.region
|
||||
service_account = google_service_account.workflow_sa.id
|
||||
source_contents = file(local.workflow_source)
|
||||
# Do not publish an executable workflow until its identity has permission to
|
||||
# reach Cloud Run. IAM propagation remains eventually consistent, so the
|
||||
# workflow also retries the edge's transient 403 response with bounded
|
||||
# backoff.
|
||||
depends_on = [google_cloud_run_v2_service_iam_member.workflow_invokes_run]
|
||||
# Allow `terraform destroy` to remove the workflow without a manual step;
|
||||
# the definition is reproducible from this module.
|
||||
deletion_protection = false
|
||||
|
||||
@@ -3,6 +3,16 @@ output "render_bucket_name" {
|
||||
value = google_storage_bucket.render.name
|
||||
}
|
||||
|
||||
output "project_name" {
|
||||
description = "Resource prefix used by this deployment."
|
||||
value = var.project_name
|
||||
}
|
||||
|
||||
output "render_service_name" {
|
||||
description = "Cloud Run service name."
|
||||
value = google_cloud_run_v2_service.render.name
|
||||
}
|
||||
|
||||
output "service_url" {
|
||||
description = "HTTPS URL of the Cloud Run render service. Pass as renderToCloudRun({ serviceUrl })."
|
||||
value = google_cloud_run_v2_service.render.uri
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "bun:test";
|
||||
|
||||
const smokePath = join(import.meta.dir, "../../../examples/gcp-cloud-run/scripts/smoke.sh");
|
||||
const smoke = readFileSync(smokePath, "utf-8");
|
||||
const dockerfile = readFileSync(join(import.meta.dir, "../Dockerfile"), "utf-8");
|
||||
|
||||
describe("GCP smoke ownership and protocol safety", () => {
|
||||
it("defaults to v1 and requires an explicit v2 protocol argument", () => {
|
||||
expect(smoke).toContain('PROTOCOLS="${PROTOCOLS:-v1}"');
|
||||
expect(smoke).toContain("--protocols)");
|
||||
expect(smoke).toContain("PlanProtocol: $protocol");
|
||||
expect(smoke).toContain("decodedFramesEqual");
|
||||
expect(smoke).toContain("decodedAudioEqual");
|
||||
expect(smoke).toContain("normalizedMetadataEqual");
|
||||
});
|
||||
|
||||
it("derives a length-safe owner prefix and isolates Terraform state", () => {
|
||||
expect(smoke).toContain('OWNER_HASH="$(printf');
|
||||
expect(smoke).toContain("RUN_NONCE=");
|
||||
expect(smoke).toContain('STACK_NAME="hf-smoke-$OWNER_HASH"');
|
||||
expect(smoke).toContain('TF_WORK_DIR="$ARTIFACT_DIR/terraform"');
|
||||
expect(smoke).toContain('TF_DATA_DIR="$ARTIFACT_DIR/terraform-data"');
|
||||
expect(smoke).toContain("export TF_DATA_DIR");
|
||||
expect(smoke).toContain('-var "project_name=$STACK_NAME"');
|
||||
expect(smoke).toContain('[ "$STACK_NAME" != "hyperframes" ]');
|
||||
});
|
||||
|
||||
it("tracks owned registry resources and verifies stack deletion", () => {
|
||||
expect(smoke).toContain("CREATED_IMAGE=0");
|
||||
expect(smoke).toContain("CREATED_REPO=0");
|
||||
expect(smoke).toContain('if [ "$CREATED_REPO" -eq 1 ]');
|
||||
expect(smoke).toContain('if [ "$CREATED_IMAGE" -eq 1 ]');
|
||||
|
||||
for (const resource of [
|
||||
"cloud-run-service",
|
||||
"workflow",
|
||||
"render-bucket",
|
||||
"artifact-image",
|
||||
"artifact-repository",
|
||||
"cloud-build-staging-bucket",
|
||||
]) {
|
||||
expect(smoke).toContain(`verify_absent "${resource}"`);
|
||||
}
|
||||
expect(smoke).toContain('verify_service_account_absent "run-service-account"');
|
||||
expect(smoke).toContain('verify_service_account_absent "workflow-service-account"');
|
||||
expect(smoke).toContain('verify_absent "preflight-cloud-run-service"');
|
||||
expect(smoke).toContain('verify_absent "preflight-artifact-image"');
|
||||
expect(smoke).toContain("$STACK_NAME-render");
|
||||
expect(smoke).toContain("--ignore-file");
|
||||
expect(smoke).toContain("--gcs-source-staging-dir");
|
||||
expect(smoke).toContain("!scripts/package-subpaths.mjs");
|
||||
expect(dockerfile).toContain("COPY scripts/package-subpaths.mjs scripts/package-subpaths.mjs");
|
||||
expect(smoke).not.toContain("gcloud services enable");
|
||||
expect(smoke).toContain("gcloud services list");
|
||||
expect(smoke).toContain("--enabled");
|
||||
expect(smoke).not.toContain("gcloud services describe");
|
||||
expect(smoke).toContain("cannot find");
|
||||
expect(smoke).toContain("gcloud iam service-accounts list");
|
||||
expect(smoke).toContain('--filter "email:$email"');
|
||||
});
|
||||
|
||||
it("does not swallow Terraform cleanup failures", () => {
|
||||
const cleanupStart = smoke.indexOf("cleanup() {");
|
||||
const cleanupEnd = smoke.indexOf("\ntrap cleanup EXIT", cleanupStart);
|
||||
const cleanup = smoke.slice(cleanupStart, cleanupEnd);
|
||||
expect(cleanup).not.toContain("|| true");
|
||||
expect(cleanup).toContain("exit 7");
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,15 @@ variable "project_name" {
|
||||
type = string
|
||||
description = "Name prefix applied to the service / workflow / bucket / service accounts."
|
||||
default = "hyperframes"
|
||||
|
||||
validation {
|
||||
condition = (
|
||||
length(var.project_name) >= 3 &&
|
||||
length(var.project_name) <= 23 &&
|
||||
can(regex("^[a-z][a-z0-9-]*[a-z0-9]$", var.project_name))
|
||||
)
|
||||
error_message = "project_name must be 3-23 lowercase letters, digits, or hyphens, begin with a letter, and end with a letter or digit so derived service-account IDs remain valid."
|
||||
}
|
||||
}
|
||||
|
||||
variable "image" {
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { parse } from "yaml";
|
||||
|
||||
type Step = Record<string, unknown>;
|
||||
|
||||
const source = readFileSync(join(import.meta.dir, "workflow.yaml"), "utf-8");
|
||||
// Cloud Workflows expressions are valid to Google's parser but `${...}`
|
||||
// inside YAML flow collections is not valid generic YAML. Quote expressions
|
||||
// for structural parsing while preserving their text for contract assertions.
|
||||
const parseableSource = source.replace(/\$\{([^}]*)\}/g, (_match, expression: string) =>
|
||||
JSON.stringify(`\${${expression}}`),
|
||||
);
|
||||
const workflow = parse(parseableSource) as {
|
||||
main: {
|
||||
steps: Step[];
|
||||
};
|
||||
retryable: {
|
||||
steps: Step[];
|
||||
};
|
||||
};
|
||||
|
||||
function namedStep(name: string, steps = workflow.main.steps): Record<string, unknown> {
|
||||
for (const step of steps) {
|
||||
if (name in step) return step[name] as Record<string, unknown>;
|
||||
}
|
||||
throw new Error(`missing workflow step ${name}`);
|
||||
}
|
||||
|
||||
function requestBody(stepName: string): Record<string, unknown> {
|
||||
const step = namedStep(stepName);
|
||||
const attempt = step.try as {
|
||||
args: {
|
||||
body: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
return attempt.args.body;
|
||||
}
|
||||
|
||||
function requestAuth(stepName: string): Record<string, unknown> {
|
||||
const step = namedStep(stepName);
|
||||
const attempt = step.try as {
|
||||
args: {
|
||||
auth: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
return attempt.args.auth;
|
||||
}
|
||||
|
||||
function chunkRequestBody(stepName: string): Record<string, unknown> {
|
||||
const renderChunks = namedStep("renderChunks");
|
||||
const parallel = renderChunks.parallel as {
|
||||
for: {
|
||||
steps: Step[];
|
||||
};
|
||||
};
|
||||
const step = namedStep(stepName, parallel.for.steps);
|
||||
const attempt = step.try as {
|
||||
args: {
|
||||
body: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
return attempt.args.body;
|
||||
}
|
||||
|
||||
describe("Cloud Workflows plan protocol routing", () => {
|
||||
it("pins OIDC tokens to the Cloud Run root and retries IAM propagation", () => {
|
||||
for (const stepName of ["planV1", "planV2", "assembleV1", "assembleV2"]) {
|
||||
expect(requestAuth(stepName)).toEqual({
|
||||
type: "OIDC",
|
||||
audience: "${serviceUrl}",
|
||||
});
|
||||
}
|
||||
expect(JSON.stringify(workflow.retryable)).toContain("e.code == 403");
|
||||
expect(source.match(/max_retries: 6/g)).toHaveLength(2);
|
||||
expect(source.match(/max_retries: 4/g)).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("keeps v1 as the default and rejects unknown protocols before plan", () => {
|
||||
expect(source).toContain('default(map.get(args, "PlanProtocol"), "v1")');
|
||||
expect(namedStep("selectPlanProtocol")).toMatchObject({
|
||||
next: "unsupportedPlanProtocol",
|
||||
});
|
||||
expect(namedStep("unsupportedPlanProtocol")).toMatchObject({
|
||||
raise: {
|
||||
code: "PLAN_PROTOCOL_UNSUPPORTED",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("uses disjoint v1 and v2 plan request/response contracts", () => {
|
||||
expect(requestBody("planV1")).toMatchObject({
|
||||
Action: "plan",
|
||||
PlanProtocol: "v1",
|
||||
});
|
||||
expect(requestBody("planV2")).toMatchObject({
|
||||
Action: "plan",
|
||||
PlanProtocol: "v2",
|
||||
});
|
||||
const validation = JSON.stringify(namedStep("validatePlanResult"));
|
||||
expect(validation).toContain("PlanGcsUri");
|
||||
expect(validation).toContain("PlanV2ManifestGcsUri");
|
||||
expect(validation).toContain("PlanV2ArtifactGcsPrefix");
|
||||
expect(source).toContain('not("PlanGcsUri" in planResult)');
|
||||
expect(source).toContain(
|
||||
'(not("PlanProtocol" in planResult) or planResult.PlanProtocol == "v1")',
|
||||
);
|
||||
});
|
||||
|
||||
it("never mixes v1 and v2 chunk locators", () => {
|
||||
const v1 = chunkRequestBody("renderOneChunkV1");
|
||||
expect(v1).toMatchObject({
|
||||
Action: "renderChunk",
|
||||
PlanProtocol: "v1",
|
||||
});
|
||||
expect(v1).toHaveProperty("PlanGcsUri");
|
||||
expect(v1).not.toHaveProperty("PlanV2ManifestGcsUri");
|
||||
expect(v1).not.toHaveProperty("PlanV2ArtifactGcsPrefix");
|
||||
|
||||
const v2 = chunkRequestBody("renderOneChunkV2");
|
||||
expect(v2).toMatchObject({
|
||||
Action: "renderChunk",
|
||||
PlanProtocol: "v2",
|
||||
});
|
||||
expect(v2).not.toHaveProperty("PlanGcsUri");
|
||||
expect(v2).toHaveProperty("PlanV2ManifestGcsUri");
|
||||
expect(v2).toHaveProperty("PlanV2ArtifactGcsPrefix");
|
||||
expect(v2).toHaveProperty("PlanHash");
|
||||
});
|
||||
|
||||
it("never mixes v1 and v2 assembler locators", () => {
|
||||
const v1 = requestBody("assembleV1");
|
||||
expect(v1).toHaveProperty("PlanGcsUri");
|
||||
expect(v1).not.toHaveProperty("PlanV2ManifestGcsUri");
|
||||
expect(v1).not.toHaveProperty("PlanV2ArtifactGcsPrefix");
|
||||
|
||||
const v2 = requestBody("assembleV2");
|
||||
expect(v2).not.toHaveProperty("PlanGcsUri");
|
||||
expect(v2).toHaveProperty("PlanV2ManifestGcsUri");
|
||||
expect(v2).toHaveProperty("PlanV2ArtifactGcsPrefix");
|
||||
expect(v2).toHaveProperty("PlanHash");
|
||||
expect(v2).toMatchObject({
|
||||
PlanProtocol: "v2",
|
||||
AudioGcsUri: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -26,9 +26,23 @@ main:
|
||||
- planOutputGcsPrefix: ${args.PlanOutputGcsPrefix}
|
||||
- outputGcsUri: ${args.OutputGcsUri}
|
||||
- config: ${args.Config}
|
||||
# Backward-compatible default. v2 is accepted only through an
|
||||
# explicit top-level PlanProtocol opt-in.
|
||||
- planProtocol: ${default(map.get(args, "PlanProtocol"), "v1")}
|
||||
|
||||
# ── Plan (Activity A) ────────────────────────────────────────────────────
|
||||
- plan:
|
||||
- selectPlanProtocol:
|
||||
switch:
|
||||
- condition: ${planProtocol == "v1"}
|
||||
next: planV1
|
||||
- condition: ${planProtocol == "v2"}
|
||||
next: planV2
|
||||
next: unsupportedPlanProtocol
|
||||
- unsupportedPlanProtocol:
|
||||
raise:
|
||||
code: PLAN_PROTOCOL_UNSUPPORTED
|
||||
message: ${"PlanProtocol must be v1 or v2; got " + string(planProtocol)}
|
||||
- planV1:
|
||||
try:
|
||||
call: http.post
|
||||
args:
|
||||
@@ -36,22 +50,69 @@ main:
|
||||
timeout: 1800
|
||||
auth:
|
||||
type: OIDC
|
||||
audience: ${serviceUrl}
|
||||
body:
|
||||
Action: plan
|
||||
PlanProtocol: v1
|
||||
ProjectGcsUri: ${projectGcsUri}
|
||||
PlanOutputGcsPrefix: ${planOutputGcsPrefix}
|
||||
Config: ${config}
|
||||
result: planResp
|
||||
result: planRespV1
|
||||
retry:
|
||||
predicate: ${retryable}
|
||||
max_retries: 4
|
||||
max_retries: 6
|
||||
backoff:
|
||||
initial_delay: 2
|
||||
max_delay: 60
|
||||
multiplier: 2
|
||||
- capturePlan:
|
||||
next: capturePlanV1
|
||||
- capturePlanV1:
|
||||
assign:
|
||||
- planResult: ${planRespV1.body}
|
||||
next: validatePlanResult
|
||||
- planV2:
|
||||
try:
|
||||
call: http.post
|
||||
args:
|
||||
url: ${serviceUrl}
|
||||
timeout: 1800
|
||||
auth:
|
||||
type: OIDC
|
||||
audience: ${serviceUrl}
|
||||
body:
|
||||
Action: plan
|
||||
PlanProtocol: v2
|
||||
ProjectGcsUri: ${projectGcsUri}
|
||||
PlanOutputGcsPrefix: ${planOutputGcsPrefix}
|
||||
Config: ${config}
|
||||
result: planRespV2
|
||||
retry:
|
||||
predicate: ${retryable}
|
||||
max_retries: 6
|
||||
backoff:
|
||||
initial_delay: 2
|
||||
max_delay: 60
|
||||
multiplier: 2
|
||||
next: capturePlanV2
|
||||
- capturePlanV2:
|
||||
assign:
|
||||
- planResult: ${planRespV2.body}
|
||||
next: validatePlanResult
|
||||
- validatePlanResult:
|
||||
# Fail closed before fan-out. A v2 render may never fall back to a
|
||||
# v1 PlanGcsUri, and a v1 render may never consume v2 locators.
|
||||
switch:
|
||||
- condition: ${planProtocol == "v1" and (not("PlanProtocol" in planResult) or planResult.PlanProtocol == "v1") and ("PlanGcsUri" in planResult) and not("PlanV2ManifestGcsUri" in planResult) and not("PlanV2ArtifactGcsPrefix" in planResult)}
|
||||
next: captureChunkCount
|
||||
- condition: ${planProtocol == "v2" and ("PlanProtocol" in planResult) and planResult.PlanProtocol == "v2" and ("PlanV2ManifestGcsUri" in planResult) and ("PlanV2ArtifactGcsPrefix" in planResult) and not("PlanGcsUri" in planResult)}
|
||||
next: captureChunkCount
|
||||
next: planProtocolLocatorMismatch
|
||||
- planProtocolLocatorMismatch:
|
||||
raise:
|
||||
code: PLAN_PROTOCOL_LOCATOR_MISMATCH
|
||||
message: "Plan response did not match the selected protocol's disjoint locator contract."
|
||||
- captureChunkCount:
|
||||
assign:
|
||||
- planResult: ${planResp.body}
|
||||
- chunkCount: ${planResult.ChunkCount}
|
||||
|
||||
# ── BuildChunkList + AssertChunkCount ──────────────────────────────────────
|
||||
@@ -98,7 +159,12 @@ main:
|
||||
value: idx
|
||||
in: ${chunkIndexes}
|
||||
steps:
|
||||
- renderOneChunk:
|
||||
- selectChunkProtocol:
|
||||
switch:
|
||||
- condition: ${planProtocol == "v2"}
|
||||
next: renderOneChunkV2
|
||||
next: renderOneChunkV1
|
||||
- renderOneChunkV1:
|
||||
try:
|
||||
call: http.post
|
||||
args:
|
||||
@@ -106,8 +172,10 @@ main:
|
||||
timeout: 1800
|
||||
auth:
|
||||
type: OIDC
|
||||
audience: ${serviceUrl}
|
||||
body:
|
||||
Action: renderChunk
|
||||
PlanProtocol: v1
|
||||
ChunkIndex: ${idx}
|
||||
PlanGcsUri: ${planResult.PlanGcsUri}
|
||||
PlanHash: ${planResult.PlanHash}
|
||||
@@ -121,13 +189,46 @@ main:
|
||||
initial_delay: 2
|
||||
max_delay: 60
|
||||
multiplier: 2
|
||||
next: storeChunk
|
||||
- renderOneChunkV2:
|
||||
try:
|
||||
call: http.post
|
||||
args:
|
||||
url: ${serviceUrl}
|
||||
timeout: 1800
|
||||
auth:
|
||||
type: OIDC
|
||||
audience: ${serviceUrl}
|
||||
body:
|
||||
Action: renderChunk
|
||||
PlanProtocol: v2
|
||||
ChunkIndex: ${idx}
|
||||
PlanV2ManifestGcsUri: ${planResult.PlanV2ManifestGcsUri}
|
||||
PlanV2ArtifactGcsPrefix: ${planResult.PlanV2ArtifactGcsPrefix}
|
||||
PlanHash: ${planResult.PlanHash}
|
||||
ChunkOutputGcsPrefix: ${planOutputGcsPrefix}
|
||||
Format: ${planResult.Format}
|
||||
result: chunkResp
|
||||
retry:
|
||||
predicate: ${retryable}
|
||||
max_retries: 4
|
||||
backoff:
|
||||
initial_delay: 2
|
||||
max_delay: 60
|
||||
multiplier: 2
|
||||
next: storeChunk
|
||||
- storeChunk:
|
||||
assign:
|
||||
- chunkUris[idx]: ${chunkResp.body.ChunkGcsUri}
|
||||
- chunkResults[idx]: ${chunkResp.body}
|
||||
|
||||
# ── Assemble (Activity C) ──────────────────────────────────────────────────
|
||||
- assemble:
|
||||
- selectAssembleProtocol:
|
||||
switch:
|
||||
- condition: ${planProtocol == "v2"}
|
||||
next: assembleV2
|
||||
next: assembleV1
|
||||
- assembleV1:
|
||||
try:
|
||||
call: http.post
|
||||
args:
|
||||
@@ -135,8 +236,10 @@ main:
|
||||
timeout: 1800
|
||||
auth:
|
||||
type: OIDC
|
||||
audience: ${serviceUrl}
|
||||
body:
|
||||
Action: assemble
|
||||
PlanProtocol: v1
|
||||
PlanGcsUri: ${planResult.PlanGcsUri}
|
||||
ChunkGcsUris: ${chunkUris}
|
||||
AudioGcsUri: ${planResult.AudioGcsUri}
|
||||
@@ -154,6 +257,38 @@ main:
|
||||
initial_delay: 2
|
||||
max_delay: 60
|
||||
multiplier: 2
|
||||
next: done
|
||||
- assembleV2:
|
||||
try:
|
||||
call: http.post
|
||||
args:
|
||||
url: ${serviceUrl}
|
||||
timeout: 1800
|
||||
auth:
|
||||
type: OIDC
|
||||
audience: ${serviceUrl}
|
||||
body:
|
||||
Action: assemble
|
||||
PlanProtocol: v2
|
||||
PlanV2ManifestGcsUri: ${planResult.PlanV2ManifestGcsUri}
|
||||
PlanV2ArtifactGcsPrefix: ${planResult.PlanV2ArtifactGcsPrefix}
|
||||
PlanHash: ${planResult.PlanHash}
|
||||
ChunkGcsUris: ${chunkUris}
|
||||
# Audio is an assembler-scoped v2 artifact and is materialized
|
||||
# from the manifest, never carried through a v1 AudioGcsUri.
|
||||
AudioGcsUri: null
|
||||
OutputGcsUri: ${outputGcsUri}
|
||||
Format: ${planResult.Format}
|
||||
Cfr: ${("cfr" in config) and config.cfr}
|
||||
result: assembleResp
|
||||
retry:
|
||||
predicate: ${retryable}
|
||||
max_retries: 4
|
||||
backoff:
|
||||
initial_delay: 2
|
||||
max_delay: 60
|
||||
multiplier: 2
|
||||
next: done
|
||||
|
||||
- done:
|
||||
return:
|
||||
@@ -161,9 +296,11 @@ main:
|
||||
Chunks: ${chunkResults}
|
||||
Assemble: ${assembleResp.body}
|
||||
|
||||
# Retry predicate: retry transient/server failures (429 + 5xx), never the
|
||||
# handler's non-retryable 400s (bad input, plan-hash mismatch, unsupported
|
||||
# format, …). Connection / timeout errors carry no `.code`; retry those too.
|
||||
# Retry predicate: retry transient/server failures (403 from Cloud Run IAM
|
||||
# propagation, 429, and 5xx), never the handler's non-retryable 400s (bad
|
||||
# input, plan-hash mismatch, unsupported format, …). The handler does not emit
|
||||
# 403, so that status is always from Cloud Run's authentication edge.
|
||||
# Connection / timeout errors carry no `.code`; retry those too.
|
||||
retryable:
|
||||
params: [e]
|
||||
steps:
|
||||
@@ -173,6 +310,8 @@ retryable:
|
||||
return: true
|
||||
- condition: ${e.code == 429}
|
||||
return: true
|
||||
- condition: ${e.code == 403}
|
||||
return: true
|
||||
- condition: ${e.code >= 500 and e.code < 600}
|
||||
return: true
|
||||
- nonRetryable:
|
||||
|
||||
Reference in New Issue
Block a user