feat(cloud): default distributed plans to v2 (#3311)

* feat(cloud): default distributed plans to v2

* fix(cloud): address plan v2 review feedback

* fix(examples): document explicit v2 samples
This commit is contained in:
James Russo
2026-08-17 17:24:31 -04:00
committed by GitHub
parent 6b17c24f98
commit 17a2a00ed5
36 changed files with 403 additions and 143 deletions
+10 -10
View File
@@ -61,17 +61,17 @@ interface PlanEventBase {
}
/**
* Legacy/default plan transport. Absence is deliberately interpreted as v1.
* Legacy plan transport. Callers must select it explicitly.
*
* @deprecated Use {@link PlanV2Event} for new integrations.
*/
export interface PlanV1Event extends PlanEventBase {
PlanProtocol?: "v1";
PlanProtocol: "v1";
}
/** Explicit opt-in to the content-addressed v2 plan transport. */
/** Default content-addressed v2 plan transport. */
export interface PlanV2Event extends PlanEventBase {
PlanProtocol: "v2";
PlanProtocol?: "v2";
}
export type PlanEvent = PlanV1Event | PlanV2Event;
@@ -96,12 +96,12 @@ interface RenderChunkEventBase {
}
/**
* Legacy/default chunk event.
* Legacy chunk event. Callers must select it explicitly.
*
* @deprecated Use {@link RenderChunkV2Event} for new integrations.
*/
export interface RenderChunkV1Event extends RenderChunkEventBase {
PlanProtocol?: "v1";
PlanProtocol: "v1";
/** GCS URI of the v1 plan tar produced by a PlanEvent invocation. */
PlanGcsUri: string;
PlanV2ManifestGcsUri?: never;
@@ -113,7 +113,7 @@ export interface RenderChunkV1Event extends RenderChunkEventBase {
* describes the exact content-addressed artifacts needed by this chunk.
*/
export interface RenderChunkV2Event extends RenderChunkEventBase {
PlanProtocol: "v2";
PlanProtocol?: "v2";
PlanV2ManifestGcsUri: string;
PlanV2ArtifactGcsPrefix: string;
PlanGcsUri?: never;
@@ -143,12 +143,12 @@ interface AssembleEventBase {
}
/**
* Legacy/default assemble event.
* Legacy assemble event. Callers must select it explicitly.
*
* @deprecated Use {@link AssembleV2Event} for new integrations.
*/
export interface AssembleV1Event extends AssembleEventBase {
PlanProtocol?: "v1";
PlanProtocol: "v1";
/** GCS URI of the v1 plan tar produced by a PlanEvent invocation. */
PlanGcsUri: string;
/** Legacy standalone audio locator; `null` when audio is embedded in the v1 plan tar. */
@@ -159,7 +159,7 @@ export interface AssembleV1Event extends AssembleEventBase {
/** V2 assemble event, scoped to manifest-declared assembler artifacts. */
export interface AssembleV2Event extends AssembleEventBase {
PlanProtocol: "v2";
PlanProtocol?: "v2";
PlanV2ManifestGcsUri: string;
PlanV2ArtifactGcsPrefix: string;
PlanHash: string;
@@ -88,16 +88,16 @@ describe("renderToCloudRun", () => {
OutputGcsUri: "gs://b/renders/hf-render-fixed/output.mp4",
ServiceUrl: "https://render-abc.run.app",
Config: config,
PlanProtocol: "v1",
PlanProtocol: "v2",
});
expect(fake.lastParent).toBe(
"projects/proj/locations/us-central1/workflows/hyperframes-render",
);
});
it("forwards an explicit v2 whole-render opt-in", async () => {
it("preserves explicit plan protocol v1 compatibility", async () => {
const fake = new FakeExecutions();
await renderToCloudRun({ ...opts(fake), planProtocol: "v2" });
await renderToCloudRun({ ...opts(fake), planProtocol: "v1" });
const arg = JSON.parse(fake.lastArgument ?? "{}");
expect(arg).toEqual({
RenderId: "hf-render-fixed",
@@ -106,7 +106,7 @@ describe("renderToCloudRun", () => {
OutputGcsUri: "gs://b/renders/hf-render-fixed/output.mp4",
ServiceUrl: "https://render-abc.run.app",
Config: config,
PlanProtocol: "v2",
PlanProtocol: "v1",
});
});
@@ -53,8 +53,8 @@ export interface RenderToCloudRunOptions {
/** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */
config: SerializableDistributedRenderConfig;
/**
* Distributed plan transport. Defaults to `"v1"` for backwards
* compatibility. New integrations should explicitly select `"v2"`.
* Distributed plan transport. Defaults to `"v2"`. Select `"v1"`
* explicitly only for deprecated compatibility with the monolithic plan.
*/
planProtocol?: CloudRunPlanProtocol;
/** GCS bucket from the Terraform output (`render_bucket_name`). */
@@ -149,7 +149,7 @@ export async function renderToCloudRun(opts: RenderToCloudRunOptions): Promise<R
OutputGcsUri: outputGcsUri,
ServiceUrl: opts.serviceUrl,
Config: opts.config,
PlanProtocol: opts.planProtocol ?? "v1",
PlanProtocol: opts.planProtocol ?? "v2",
};
// Reject oversize input client-side. Cloud Workflows caps the execution
+15 -6
View File
@@ -178,11 +178,12 @@ describe("unwrapEvent", () => {
});
describe("dispatch", () => {
it("routes plan, uploads the plan tarball", async () => {
it("preserves explicit v1 plan compatibility", async () => {
const gcs = new FakeGcs();
await seedProjectTar(gcs, "gs://b/sites/x/project.tar.gz");
const event: PlanEvent = {
Action: "plan",
PlanProtocol: "v1",
ProjectGcsUri: "gs://b/sites/x/project.tar.gz",
PlanOutputGcsPrefix: "gs://b/renders/r1/",
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" } as PlanEvent["Config"],
@@ -201,6 +202,7 @@ describe("dispatch", () => {
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 2,
@@ -220,6 +222,7 @@ describe("dispatch", () => {
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: "WRONG_HASH",
ChunkIndex: 0,
@@ -236,6 +239,7 @@ describe("dispatch", () => {
gcs.seed("gs://b/renders/r1/chunks/0001.mp4", Buffer.from("c1"));
const event: AssembleEvent = {
Action: "assemble",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
ChunkGcsUris: ["gs://b/renders/r1/chunks/0000.mp4", "gs://b/renders/r1/chunks/0001.mp4"],
AudioGcsUri: null,
@@ -251,7 +255,7 @@ describe("dispatch", () => {
// This end-to-end adapter contract is intentionally one narrative test: it
// verifies ordering and target isolation across all three handler roles.
// fallow-ignore-next-line complexity
it("runs v2 plan → target-scoped chunk → assemble with manifest-last CAS", async () => {
it("defaults omitted plan protocol to v2 across plan → chunk → assemble", async () => {
const gcs = new FakeGcs();
await seedProjectTar(gcs, "gs://b/sites/v2/project.tar.gz");
const root = mkTmp("hf-v2-e2e-");
@@ -306,7 +310,6 @@ describe("dispatch", () => {
const planned = await dispatch(
{
Action: "plan",
PlanProtocol: "v2",
ProjectGcsUri: "gs://b/sites/v2/project.tar.gz",
PlanOutputGcsPrefix: "gs://b/renders/v2/",
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
@@ -326,7 +329,6 @@ describe("dispatch", () => {
await dispatch(
{
Action: "plan",
PlanProtocol: "v2",
ProjectGcsUri: "gs://b/sites/v2/project.tar.gz",
PlanOutputGcsPrefix: "gs://b/renders/v2/",
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
@@ -342,7 +344,6 @@ describe("dispatch", () => {
const chunk = await dispatch(
{
Action: "renderChunk",
PlanProtocol: "v2",
PlanV2ManifestGcsUri: planned.PlanV2ManifestGcsUri,
PlanV2ArtifactGcsPrefix: planned.PlanV2ArtifactGcsPrefix,
PlanHash: planned.PlanHash,
@@ -360,7 +361,6 @@ describe("dispatch", () => {
await dispatch(
{
Action: "assemble",
PlanProtocol: "v2",
PlanV2ManifestGcsUri: planned.PlanV2ManifestGcsUri,
PlanV2ArtifactGcsPrefix: planned.PlanV2ArtifactGcsPrefix,
PlanHash: planned.PlanHash,
@@ -409,6 +409,7 @@ describe("bucket allowlist guard", () => {
try {
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://evil-bucket/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
@@ -451,6 +452,7 @@ describe("bucket allowlist guard", () => {
try {
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://any-bucket/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
@@ -476,6 +478,7 @@ describe("createApp HTTP mapping", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
@@ -497,6 +500,7 @@ describe("createApp HTTP mapping", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: "WRONG",
ChunkIndex: 0,
@@ -524,6 +528,7 @@ describe("createApp HTTP mapping", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
@@ -561,6 +566,7 @@ describe("createApp HTTP mapping", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
@@ -598,6 +604,7 @@ describe("createApp HTTP mapping", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
@@ -626,6 +633,7 @@ describe("createApp HTTP mapping", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "plan",
PlanProtocol: "v1",
ProjectGcsUri: "gs://b/sites/invalid-video-metadata/project.tar.gz",
PlanOutputGcsPrefix: "gs://b/renders/invalid-video-metadata/",
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
@@ -645,6 +653,7 @@ describe("createApp HTTP mapping", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanProtocol: "v1",
PlanGcsUri: "gs://b/renders/r1/missing.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
+20 -16
View File
@@ -46,13 +46,16 @@ import {
import { resolveChromeExecutablePath } from "./chromium.js";
import type {
AssembleEvent,
AssembleV2Event,
AssembleResultBody,
CloudRunAction,
CloudRunEvent,
CloudRunResult,
PlanEvent,
PlanV2Event,
PlanResultBody,
RenderChunkEvent,
RenderChunkV2Event,
RenderChunkResultBody,
} from "./events.js";
import { type DistributedFormat, formatExtension } from "./formatExtension.js";
@@ -155,21 +158,22 @@ function validatePlanProtocolShape(event: PlanEvent | RenderChunkEvent | Assembl
}
if (event.Action === "plan") return;
const effectiveProtocol = protocol ?? "v2";
const hasV1Locator = typeof raw.PlanGcsUri === "string";
const hasV2Manifest = typeof raw.PlanV2ManifestGcsUri === "string";
const hasV2Prefix = typeof raw.PlanV2ArtifactGcsPrefix === "string";
const valid =
protocol === "v2"
effectiveProtocol === "v2"
? !hasV1Locator && hasV2Manifest && hasV2Prefix
: hasV1Locator && !hasV2Manifest && !hasV2Prefix;
if (!valid) {
const error = new Error(
`[handler] ${protocol === "v2" ? "v2" : "v1"} ${event.Action} event has mixed or missing plan locators`,
`[handler] ${effectiveProtocol} ${event.Action} event has mixed or missing plan locators`,
);
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
throw error;
}
if (protocol === "v2" && event.Action === "assemble" && event.AudioGcsUri !== null) {
if (effectiveProtocol === "v2" && event.Action === "assemble" && event.AudioGcsUri !== null) {
const error = new Error("[handler] v2 assemble audio must be materialized from the manifest");
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
throw error;
@@ -254,14 +258,14 @@ function summarizeEvent(
return {
projectGcsUri: event.ProjectGcsUri,
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
planProtocol: event.PlanProtocol ?? "v1",
planProtocol: event.PlanProtocol ?? "v2",
format: event.Config.format,
fps: event.Config.fps,
};
case "renderChunk":
return {
planProtocol: event.PlanProtocol ?? "v1",
...(event.PlanProtocol === "v2"
planProtocol: event.PlanProtocol ?? "v2",
...(event.PlanProtocol !== "v1"
? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri }
: { planGcsUri: event.PlanGcsUri }),
chunkIndex: event.ChunkIndex,
@@ -269,8 +273,8 @@ function summarizeEvent(
};
case "assemble":
return {
planProtocol: event.PlanProtocol ?? "v1",
...(event.PlanProtocol === "v2"
planProtocol: event.PlanProtocol ?? "v2",
...(event.PlanProtocol !== "v1"
? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri }
: { planGcsUri: event.PlanGcsUri }),
chunkCount: event.ChunkGcsUris.length,
@@ -297,7 +301,7 @@ function primeChrome(deps?: HandlerDeps): void {
// fallow-ignore-next-line complexity
async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanResultBody> {
if (event.PlanProtocol === "v2") {
if (event.PlanProtocol !== "v1") {
return handlePlanV2(event, deps);
}
const started = Date.now();
@@ -365,7 +369,7 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanRes
*/
// fallow-ignore-next-line complexity
async function handlePlanV2(
event: Extract<PlanEvent, { PlanProtocol: "v2" }>,
event: PlanV2Event,
deps?: HandlerDeps,
): Promise<Extract<PlanResultBody, { PlanProtocol: "v2" }>> {
const started = Date.now();
@@ -418,7 +422,7 @@ async function handleRenderChunk(
event: RenderChunkEvent,
deps?: HandlerDeps,
): Promise<RenderChunkResultBody> {
if (event.PlanProtocol === "v2") {
if (event.PlanProtocol !== "v1") {
return handleRenderChunkV2(event, deps);
}
const started = Date.now();
@@ -475,7 +479,7 @@ async function handleRenderChunk(
/** Materialize only this chunk's verified v2 dependencies before rendering. */
// fallow-ignore-next-line complexity
async function handleRenderChunkV2(
event: Extract<RenderChunkEvent, { PlanProtocol: "v2" }>,
event: RenderChunkV2Event,
deps?: HandlerDeps,
): Promise<RenderChunkResultBody> {
const started = Date.now();
@@ -548,7 +552,7 @@ async function handleAssemble(
event: AssembleEvent,
deps?: HandlerDeps,
): Promise<AssembleResultBody> {
if (event.PlanProtocol === "v2") {
if (event.PlanProtocol !== "v1") {
return handleAssembleV2(event, deps);
}
const started = Date.now();
@@ -613,7 +617,7 @@ async function handleAssemble(
*/
// fallow-ignore-next-line complexity
async function handleAssembleV2(
event: Extract<AssembleEvent, { PlanProtocol: "v2" }>,
event: AssembleV2Event,
deps?: HandlerDeps,
): Promise<AssembleResultBody> {
const started = Date.now();
@@ -781,12 +785,12 @@ function getEventGcsUris(event: PlanEvent | RenderChunkEvent | AssembleEvent): s
case "plan":
return [event.ProjectGcsUri, event.PlanOutputGcsPrefix];
case "renderChunk":
return event.PlanProtocol === "v2"
return event.PlanProtocol !== "v1"
? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix, event.ChunkOutputGcsPrefix]
: [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
case "assemble":
return [
...(event.PlanProtocol === "v2"
...(event.PlanProtocol !== "v1"
? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix]
: [event.PlanGcsUri]),
...event.ChunkGcsUris,