mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
feat!: remove Plan v1 distributed transport (#6970)
* feat!: remove Plan v1 distributed transport BREAKING CHANGE: Cloud adapters accept only Plan v2; producer v1 exports are removed. * ci: authorize Plan v1 cleanup deletions * docs: avoid citing generated smoke result file * fix(gcp): reject unsupported plan protocols in workflow * ci: preserve gcp package size reduction * fix: reject stale Plan v1 SDK options * chore: remove retired Plan parity scripts
This commit is contained in:
@@ -19,24 +19,19 @@ import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:te
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { availableParallelism, tmpdir } from "node:os";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { LambdaClient } from "@aws-sdk/client-lambda";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
CURRENT_PLAN_PROTOCOL,
|
||||
PLAN_V2_MATERIALIZATION_MARKER,
|
||||
PlanVideosMetadataError,
|
||||
type AssembleResult,
|
||||
type ChunkResult,
|
||||
type DistributedRenderConfig,
|
||||
type PlanResult,
|
||||
type PlanV2ArtifactPublisher,
|
||||
type PlanV2Manifest,
|
||||
type PlanV2WithPublisherOptions,
|
||||
publishPlanV2FromExecutionPlan,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
import { resolveMaxParallelChunks } from "../../producer/src/services/distributed/plan.js";
|
||||
import { publishPlanV2FromExecutionPlan } from "../../producer/src/services/distributed/planV2.js";
|
||||
import { recomputePlanHashFromPlanDir } from "../../producer/src/services/render/stages/freezePlan.js";
|
||||
import type {
|
||||
AssembleEvent,
|
||||
@@ -228,8 +223,9 @@ describe("unwrapEvent", () => {
|
||||
it("unwraps a Step Functions { Payload } envelope", () => {
|
||||
const inner: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanProtocol: "v1",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanProtocol: "v2",
|
||||
PlanV2ManifestS3Uri: "s3://bucket/v2/manifest.json",
|
||||
PlanV2ArtifactS3Prefix: "s3://bucket/v2/artifacts/sha256",
|
||||
PlanHash: "deadbeef",
|
||||
ChunkIndex: 3,
|
||||
ChunkOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
@@ -242,8 +238,10 @@ describe("unwrapEvent", () => {
|
||||
it("unwraps multiple levels of envelopes", () => {
|
||||
const inner: AssembleEvent = {
|
||||
Action: "assemble",
|
||||
PlanProtocol: "v1",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanProtocol: "v2",
|
||||
PlanV2ManifestS3Uri: "s3://bucket/v2/manifest.json",
|
||||
PlanV2ArtifactS3Prefix: "s3://bucket/v2/artifacts/sha256",
|
||||
PlanHash: "deadbeef",
|
||||
ChunkS3Uris: ["s3://bucket/chunks/0001.mp4"],
|
||||
AudioS3Uri: null,
|
||||
OutputS3Uri: "s3://bucket/output.mp4",
|
||||
@@ -274,636 +272,23 @@ describe("handler dispatch", () => {
|
||||
expect(s3.ops).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves explicit v1 plan compatibility", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
it("rejects explicit Plan v1 before touching S3", async () => {
|
||||
const s3 = new FakeS3Client();
|
||||
// Seed a fake project tarball so the untar step has something to chew on.
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
|
||||
const planMock = mock(
|
||||
async (_projectDir: string, _config: unknown, planDir: string): Promise<PlanResult> => {
|
||||
// Simulate plan() writing a minimal planDir.
|
||||
mkdirSync(planDir, { recursive: true });
|
||||
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash: "fakehash" }));
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(join(planDir, "meta", "chunks.json"), "[]");
|
||||
return {
|
||||
planDir,
|
||||
planProtocol: CURRENT_PLAN_PROTOCOL,
|
||||
planHash: "fakehash",
|
||||
chunkCount: 4,
|
||||
totalFrames: 720,
|
||||
fps: 30 as const,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4" as const,
|
||||
ffmpegVersion: "6.0",
|
||||
producerVersion: "0.0.0-test",
|
||||
};
|
||||
},
|
||||
);
|
||||
const renderChunkMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: PlanEvent = {
|
||||
const event = {
|
||||
Action: "plan",
|
||||
PlanProtocol: "v1",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" },
|
||||
};
|
||||
} as unknown as PlanEvent;
|
||||
|
||||
const result = await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
|
||||
expect(result.Action).toBe("plan");
|
||||
if (result.Action !== "plan") throw new Error("unreachable");
|
||||
expect(result.PlanHash).toBe("fakehash");
|
||||
expect(result.ChunkCount).toBe(4);
|
||||
expect(planMock).toHaveBeenCalledTimes(1);
|
||||
expect((planMock.mock.calls[0] as unknown[])[3]).toEqual({ chunksCarryAudio: true });
|
||||
expect(renderChunkMock).not.toHaveBeenCalled();
|
||||
expect(assembleMock).not.toHaveBeenCalled();
|
||||
// Plan should have downloaded the project zip and uploaded the plan tar.
|
||||
expect(
|
||||
s3.ops.some((o) => o.kind === "download" && o.uri === "s3://bucket/project.tar.gz"),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("plans against the function's own timeout unless the config pins one", async () => {
|
||||
const seen: Array<number | undefined> = [];
|
||||
for (const pinned of [undefined, 300]) {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
await handler(
|
||||
{
|
||||
Action: "plan",
|
||||
PlanProtocol: "v1",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/timeout/",
|
||||
Config: { fps: 30, width: 640, height: 360, format: "mp4", workerTimeoutSeconds: pinned },
|
||||
},
|
||||
{
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: mock(async (_dir: string, config: { workerTimeoutSeconds?: number }) => {
|
||||
seen.push(config.workerTimeoutSeconds);
|
||||
throw new Error("stop after config");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk: mock(async () => {
|
||||
throw new Error("unused");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble: mock(async () => {
|
||||
throw new Error("unused");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
getRemainingTimeInMillis: () => 899_987,
|
||||
},
|
||||
).catch(() => undefined);
|
||||
}
|
||||
expect(seen).toEqual([900, 300]);
|
||||
});
|
||||
|
||||
describe("chunk concurrency when the render does not set one", () => {
|
||||
const THIRTY_MIN_FRAMES = 54_000;
|
||||
const MP_1080P = (1920 * 1080) / 1_000_000;
|
||||
|
||||
function fakeLambda(reply: () => Promise<unknown>) {
|
||||
const send = mock(reply);
|
||||
return { send, client: { send } as unknown as import("@aws-sdk/client-lambda").LambdaClient };
|
||||
}
|
||||
|
||||
async function planThirtyMinutes(lambda: { client: unknown }, workerConcurrency?: number) {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
const warnings: string[] = [];
|
||||
const log = { warn: (m: string) => warnings.push(m), error() {}, info() {}, debug() {} };
|
||||
let planned: DistributedRenderConfig | undefined;
|
||||
const result = await handler(
|
||||
{
|
||||
Action: "plan",
|
||||
PlanProtocol: "v1",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/conc/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4", workerConcurrency },
|
||||
},
|
||||
{
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
lambda: lambda.client as import("@aws-sdk/client-lambda").LambdaClient,
|
||||
primitives: {
|
||||
plan: mock(async (_dir: string, config: DistributedRenderConfig, planDir: string) => {
|
||||
planned = config;
|
||||
mkdirSync(planDir, { recursive: true });
|
||||
writeFileSync(join(planDir, "plan.json"), "{}");
|
||||
return {
|
||||
planDir,
|
||||
planProtocol: CURRENT_PLAN_PROTOCOL,
|
||||
planHash: "h",
|
||||
// The real sizing rule, so the cap and the chunk count are tested together.
|
||||
chunkCount: resolveMaxParallelChunks(config, THIRTY_MIN_FRAMES, MP_1080P, log),
|
||||
totalFrames: THIRTY_MIN_FRAMES,
|
||||
fps: 30 as const,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4" as const,
|
||||
ffmpegVersion: "6.0",
|
||||
producerVersion: "0.0.0-test",
|
||||
};
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk: mock(async () => {
|
||||
throw new Error("unused");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble: mock(async () => {
|
||||
throw new Error("unused");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
getRemainingTimeInMillis: () => 900_000,
|
||||
},
|
||||
);
|
||||
if (result.Action !== "plan") throw new Error("unreachable");
|
||||
return { result, planned, warnings };
|
||||
}
|
||||
|
||||
it("runs every chunk at once when the account has room", async () => {
|
||||
const lambda = fakeLambda(async () => ({
|
||||
AccountLimit: { UnreservedConcurrentExecutions: 1000 },
|
||||
}));
|
||||
const { result, planned, warnings } = await planThirtyMinutes(lambda);
|
||||
expect(planned?.workerConcurrency).toBe(900);
|
||||
expect(result.ChunkCount).toBe(204);
|
||||
expect(result.MaxChunkConcurrency).toBe(result.ChunkCount);
|
||||
expect(warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it("caps at 90% of a small account and says the render runs in waves", async () => {
|
||||
const lambda = fakeLambda(async () => ({
|
||||
AccountLimit: { UnreservedConcurrentExecutions: 10 },
|
||||
}));
|
||||
const { result, warnings } = await planThirtyMinutes(lambda);
|
||||
expect(result.MaxChunkConcurrency).toBe(9);
|
||||
expect(result.ChunkCount).toBe(21);
|
||||
expect(warnings).toHaveLength(1);
|
||||
expect(warnings[0]).toContain("in 3 waves");
|
||||
expect(warnings[0]).toContain("--concurrency 21");
|
||||
});
|
||||
|
||||
it("falls back to 16 at once and logs why when the account cannot be read", async () => {
|
||||
const lambda = fakeLambda(async () => {
|
||||
const denied = new Error("not authorized to perform: lambda:GetAccountSettings");
|
||||
denied.name = "AccessDeniedException";
|
||||
throw denied;
|
||||
});
|
||||
const logged = spyOn(console, "log").mockImplementation(() => undefined);
|
||||
try {
|
||||
const { result, planned } = await planThirtyMinutes(lambda);
|
||||
expect(planned?.workerConcurrency).toBe(16);
|
||||
expect(result.MaxChunkConcurrency).toBe(16);
|
||||
const lines = logged.mock.calls.map((call) => String(call[0]));
|
||||
const fallback = lines.find((line) => line.includes('"source":"fallback"'));
|
||||
expect(fallback).toContain('"level":"WARN"');
|
||||
expect(fallback).toContain("AccessDeniedException");
|
||||
expect(fallback).toContain("planning for 16 chunks at once");
|
||||
} finally {
|
||||
logged.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an explicit concurrency, then the reserved one, without asking the account", async () => {
|
||||
const lambda = fakeLambda(async () => ({
|
||||
AccountLimit: { UnreservedConcurrentExecutions: 1000 },
|
||||
}));
|
||||
const explicit = await planThirtyMinutes(lambda, 64);
|
||||
expect(explicit.planned?.workerConcurrency).toBe(64);
|
||||
expect(explicit.result.MaxChunkConcurrency).toBe(64);
|
||||
process.env.HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY = "32";
|
||||
try {
|
||||
const reserved = await planThirtyMinutes(lambda);
|
||||
expect(reserved.result.MaxChunkConcurrency).toBe(32);
|
||||
} finally {
|
||||
delete process.env.HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY;
|
||||
}
|
||||
expect(lambda.send).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes producer workflow codes to Step Functions error names", async () => {
|
||||
for (const code of [
|
||||
"PLAN_TOO_LARGE",
|
||||
"PLAN_PROTOCOL_UNSUPPORTED",
|
||||
"PLAN_V2_INTEGRITY_UNRECOVERABLE",
|
||||
"FONT_FETCH_FAILED",
|
||||
"FONT_FETCH_UNAVAILABLE",
|
||||
"VIDEO_SOURCE_UNRENDERABLE",
|
||||
"VIDEO_EXTRACTION_FAILED",
|
||||
"INVALID_VIDEO_METADATA",
|
||||
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
|
||||
"CHUNK_EXCEEDS_WORKER_TIMEOUT",
|
||||
] as const) {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
const terminal =
|
||||
code === "INVALID_VIDEO_METADATA"
|
||||
? new PlanVideosMetadataError("test invalid plan video metadata")
|
||||
: Object.assign(new Error(`terminal: ${code}`), {
|
||||
code,
|
||||
name: "ProducerError",
|
||||
});
|
||||
|
||||
await expect(
|
||||
handler(
|
||||
{
|
||||
Action: "plan",
|
||||
PlanProtocol: "v1",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/terminal/",
|
||||
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
|
||||
},
|
||||
{
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: mock(async () => {
|
||||
throw terminal;
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk: mock(async () => {
|
||||
throw new Error("unused");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble: mock(async () => {
|
||||
throw new Error("unused");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
},
|
||||
),
|
||||
).rejects.toMatchObject({ name: code });
|
||||
}
|
||||
});
|
||||
|
||||
it("plan honors a pre-set PRODUCER_HEADLESS_SHELL_PATH instead of re-resolving Chrome", async () => {
|
||||
// Mirrors the renderChunk env-var guard — when a caller (e.g. SAM-local
|
||||
// RIE smoke) seeds the path, handlePlan must not overwrite it.
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
|
||||
const planMock = mock(
|
||||
async (_projectDir: string, _config: unknown, planDir: string): Promise<PlanResult> => {
|
||||
mkdirSync(planDir, { recursive: true });
|
||||
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash: "fakehash" }));
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(join(planDir, "meta", "chunks.json"), "[]");
|
||||
return {
|
||||
planDir,
|
||||
planProtocol: CURRENT_PLAN_PROTOCOL,
|
||||
planHash: "fakehash",
|
||||
chunkCount: 1,
|
||||
totalFrames: 30,
|
||||
fps: 30 as const,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4" as const,
|
||||
ffmpegVersion: "6.0",
|
||||
producerVersion: "0.0.0-test",
|
||||
};
|
||||
},
|
||||
);
|
||||
const renderChunkMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: PlanEvent = {
|
||||
Action: "plan",
|
||||
PlanProtocol: "v1",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" },
|
||||
};
|
||||
|
||||
const sentinel = "/tmp/test-chrome-sentinel";
|
||||
const prev = process.env.PRODUCER_HEADLESS_SHELL_PATH;
|
||||
process.env.PRODUCER_HEADLESS_SHELL_PATH = sentinel;
|
||||
try {
|
||||
// Note: no skipChromeResolution flag — the guard must short-circuit
|
||||
// because PRODUCER_HEADLESS_SHELL_PATH is already set.
|
||||
await handler(event, {
|
||||
await expect(
|
||||
handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
});
|
||||
expect(process.env.PRODUCER_HEADLESS_SHELL_PATH).toBe(sentinel);
|
||||
expect(planMock).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
if (prev === undefined) {
|
||||
delete process.env.PRODUCER_HEADLESS_SHELL_PATH;
|
||||
} else {
|
||||
process.env.PRODUCER_HEADLESS_SHELL_PATH = prev;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("routes Action='renderChunk' to the renderChunk primitive", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
// Seed a planDir tarball with a minimal structure renderChunk would
|
||||
// observe; the test mock doesn't read it, but the handler untar step does.
|
||||
s3.objects.set("s3://bucket/plan.tar.gz", await makeMinimalPlanTar());
|
||||
|
||||
const renderChunkMock = mock(
|
||||
async (
|
||||
_planDir: string,
|
||||
_chunkIndex: number,
|
||||
outputChunkPath: string,
|
||||
): Promise<ChunkResult> => {
|
||||
// Write a fake chunk file so the upload step has bytes to send.
|
||||
writeFileSync(outputChunkPath, Buffer.from("FAKE-MP4-CHUNK"));
|
||||
return {
|
||||
outputPath: outputChunkPath,
|
||||
outputKind: "file",
|
||||
framesEncoded: 240,
|
||||
sha256: "0".repeat(64),
|
||||
durationMs: 12345,
|
||||
planHashMs: 1,
|
||||
sessionBootMs: 2,
|
||||
captureStageMs: 3,
|
||||
encodeStageMs: 4,
|
||||
workers: 1,
|
||||
captureMode: "beginframe",
|
||||
perfPath: outputChunkPath + ".perf.json",
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const planMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanProtocol: "v1",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanHash: "fakehash",
|
||||
ChunkIndex: 2,
|
||||
ChunkOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Format: "mp4",
|
||||
};
|
||||
|
||||
const result = await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
|
||||
expect(result.Action).toBe("renderChunk");
|
||||
if (result.Action !== "renderChunk") throw new Error("unreachable");
|
||||
expect(result.ChunkIndex).toBe(2);
|
||||
expect(result.Sha256).toBe("0".repeat(64));
|
||||
expect(result.FramesEncoded).toBe(240);
|
||||
expect(result.CaptureMode).toBe("beginframe");
|
||||
expect(renderChunkMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("gives a chunk as many capture browsers as the function's memory can host", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/plan.tar.gz", await makeMinimalPlanTar());
|
||||
const renderChunkMock = mock(
|
||||
async (
|
||||
_planDir: string,
|
||||
_chunkIndex: number,
|
||||
outputChunkPath: string,
|
||||
): Promise<ChunkResult> => {
|
||||
writeFileSync(outputChunkPath, Buffer.from("FAKE-MP4-CHUNK"));
|
||||
return {
|
||||
outputPath: outputChunkPath,
|
||||
outputKind: "file",
|
||||
framesEncoded: 121,
|
||||
sha256: "0".repeat(64),
|
||||
durationMs: 1,
|
||||
planHashMs: 1,
|
||||
sessionBootMs: 1,
|
||||
captureStageMs: 1,
|
||||
encodeStageMs: 1,
|
||||
workers: 5,
|
||||
captureMode: "beginframe",
|
||||
perfPath: outputChunkPath + ".perf.json",
|
||||
};
|
||||
},
|
||||
);
|
||||
const notCalled = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const renderOnce = () =>
|
||||
handler(
|
||||
{
|
||||
Action: "renderChunk",
|
||||
PlanProtocol: "v1",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanHash: "fakehash",
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Format: "mp4",
|
||||
},
|
||||
{
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: notCalled as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
notCalled as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
},
|
||||
);
|
||||
const saved = process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
|
||||
const logged: string[] = [];
|
||||
const logSpy = spyOn(console, "log").mockImplementation((line: string) => {
|
||||
logged.push(line);
|
||||
});
|
||||
try {
|
||||
delete process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
|
||||
await renderOnce();
|
||||
process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = "10240";
|
||||
await renderOnce();
|
||||
} finally {
|
||||
if (saved === undefined) delete process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
|
||||
else process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = saved;
|
||||
logSpy.mockRestore();
|
||||
}
|
||||
const options = renderChunkMock.mock.calls.map((call) => (call as unknown[])[3]);
|
||||
expect(options).toEqual([
|
||||
{ maxCaptureWorkers: undefined, interleaveCapture: true },
|
||||
{
|
||||
maxCaptureWorkers: Math.max(1, Math.min(5, availableParallelism() - 1)),
|
||||
interleaveCapture: true,
|
||||
},
|
||||
]);
|
||||
const captures = logged
|
||||
.filter((line) => line.includes('"chunk_capture"'))
|
||||
.map((line) => JSON.parse(line));
|
||||
const capture = {
|
||||
event: "chunk_capture",
|
||||
chunkIndex: 0,
|
||||
workers: 5,
|
||||
captureMode: "beginframe",
|
||||
setupMs: expect.any(Number),
|
||||
chromeMs: expect.any(Number),
|
||||
planHashMs: 1,
|
||||
sessionBootMs: 1,
|
||||
captureStageMs: 1,
|
||||
encodeStageMs: 1,
|
||||
durationMs: 1,
|
||||
};
|
||||
expect(captures).toEqual([capture, capture]);
|
||||
});
|
||||
|
||||
it("rejects renderChunk when event.PlanHash diverges from plan.json", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
// The fixture's plan.json has planHash="fakehash"; the event below
|
||||
// claims something else, so the handler must throw PLAN_HASH_MISMATCH
|
||||
// before invoking the primitive.
|
||||
s3.objects.set("s3://bucket/plan.tar.gz", await makeMinimalPlanTar());
|
||||
|
||||
const renderChunkMock = mock(async () => {
|
||||
throw new Error("primitive should not be called on a hash mismatch");
|
||||
});
|
||||
const planMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanProtocol: "v1",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanHash: "not-the-real-hash",
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Format: "mp4",
|
||||
};
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(Error);
|
||||
expect((caught as Error).name).toBe("PLAN_HASH_MISMATCH");
|
||||
expect((caught as Error).message).toMatch(/not-the-real-hash/);
|
||||
expect(renderChunkMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes Action='assemble' to the assemble primitive", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/plan.tar.gz", await makeMinimalPlanTar());
|
||||
s3.objects.set("s3://bucket/chunks/0001.mp4", Buffer.from("CHUNK-1"));
|
||||
s3.objects.set("s3://bucket/chunks/0002.mp4", Buffer.from("CHUNK-2"));
|
||||
|
||||
const assembleMock = mock(
|
||||
async (
|
||||
_planDir: string,
|
||||
_chunkPaths: readonly string[],
|
||||
_audioPath: string | null,
|
||||
outputPath: string,
|
||||
): Promise<AssembleResult> => {
|
||||
writeFileSync(outputPath, Buffer.from("FAKE-FINAL-MP4"));
|
||||
return {
|
||||
outputPath,
|
||||
durationMs: 7777,
|
||||
framesEncoded: 480,
|
||||
fileSize: 14,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const event: AssembleEvent = {
|
||||
Action: "assemble",
|
||||
PlanProtocol: "v1",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
ChunkS3Uris: ["s3://bucket/chunks/0001.mp4", "s3://bucket/chunks/0002.mp4"],
|
||||
AudioS3Uri: null,
|
||||
OutputS3Uri: "s3://bucket/renders/abc/output.mp4",
|
||||
Format: "mp4",
|
||||
};
|
||||
|
||||
const result = await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk: mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
|
||||
expect(result.Action).toBe("assemble");
|
||||
if (result.Action !== "assemble") throw new Error("unreachable");
|
||||
expect(result.OutputS3Uri).toBe("s3://bucket/renders/abc/output.mp4");
|
||||
expect(result.FramesEncoded).toBe(480);
|
||||
expect(assembleMock).toHaveBeenCalledTimes(1);
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "PLAN_PROTOCOL_UNSUPPORTED" });
|
||||
expect(s3.ops).toEqual([]);
|
||||
});
|
||||
|
||||
it("defaults omitted plan protocol to v2 across plan → chunk → assemble", async () => {
|
||||
@@ -1616,8 +1001,10 @@ describe("handler — S3 URI allowlist (security: F-004)", () => {
|
||||
|
||||
const event: AssembleEvent = {
|
||||
Action: "assemble",
|
||||
PlanProtocol: "v1",
|
||||
PlanS3Uri: "s3://good-bucket/plan.tar.gz",
|
||||
PlanProtocol: "v2",
|
||||
PlanV2ManifestS3Uri: "s3://good-bucket/plan/v2/manifest.json",
|
||||
PlanV2ArtifactS3Prefix: "s3://good-bucket/plan/v2/artifacts/sha256",
|
||||
PlanHash: "a".repeat(64),
|
||||
ChunkS3Uris: ["s3://good-bucket/chunks/0001.mp4", "s3://evil-bucket/chunks/0002.mp4"],
|
||||
AudioS3Uri: null,
|
||||
OutputS3Uri: "s3://good-bucket/renders/abc/output.mp4",
|
||||
@@ -1750,29 +1137,6 @@ async function makeMinimalProjectTar(): Promise<Buffer> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a minimal `.tar.gz` for a tiny planDir containing `plan.json` +
|
||||
* `meta/chunks.json`. Used by renderChunk/assemble tests where the handler
|
||||
* untars but the mock primitive doesn't inspect contents.
|
||||
*/
|
||||
async function makeMinimalPlanTar(): Promise<Buffer> {
|
||||
const tar = await import("tar");
|
||||
const {
|
||||
mkdtempSync: mk,
|
||||
mkdirSync: md,
|
||||
readFileSync: rf,
|
||||
writeFileSync: wf,
|
||||
} = await import("node:fs");
|
||||
const dir = mk(join(tmpdir(), "hf-lambda-test-plan-"));
|
||||
tmpDirs.push(dir);
|
||||
md(join(dir, "meta"), { recursive: true });
|
||||
wf(join(dir, "plan.json"), JSON.stringify({ planHash: "fakehash" }));
|
||||
wf(join(dir, "meta", "chunks.json"), "[]");
|
||||
const tarPath = join(dir, "out.tar.gz");
|
||||
await tar.create({ gzip: true, file: tarPath, cwd: dir }, ["plan.json", "meta"]);
|
||||
return rf(tarPath);
|
||||
}
|
||||
|
||||
function makeMinimalV1PlanDir(
|
||||
dir: string,
|
||||
withAudio: boolean,
|
||||
|
||||
Reference in New Issue
Block a user