mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
* perf(aws-lambda): small-chunk cloud renders stitch their output in parallel When every chunk's payload is under S3's 5 MiB part minimum (typical for short 4K chunks), the assemble step could not copy any part server-side and pulled the whole output through one Lambda's network link. Those range-only parts now go to parallel invocations of the warm function that just rendered the chunks, about 32 MiB each and never more at once than the stack's reserved concurrency allows. The output bytes are unchanged. * fix(aws-lambda): a throttled stitch helper builds its parts in the assemble instead Another render can hold the stack's reserved concurrency while this one assembles, so a helper invocation may be refused. The assemble now builds that batch itself, logs it, and counts it in assemble_stages; the helper's error type is kept as the error name. Comments trimmed back to each file's share at the base. * chore(aws-lambda): raise the package size budget to the measured size plus headroom The parallel stitch helpers add about 1.7 KB packed and 9.9 KB unpacked; the new budget is the measured size times the checker's own 1.05 headroom.
263 lines
8.6 KiB
TypeScript
263 lines
8.6 KiB
TypeScript
/**
|
|
* Lambda event + result types for the HyperFrames distributed render handler.
|
|
*
|
|
* The Step Functions state machine in `examples/aws-lambda/template.yaml`
|
|
* dispatches on the `Action` field. Each action maps 1:1 onto one of the
|
|
* three OSS distributed primitives:
|
|
*
|
|
* "plan" → `plan(projectDir, config, planDir)` (Activity A)
|
|
* "renderChunk" → `renderChunk(planDir, chunkIndex, output)` (Activity B)
|
|
* "assemble" → `assemble(planDir, chunkPaths, audio, out)` (Activity C)
|
|
*
|
|
* All file I/O is mediated by S3 — the handler downloads inputs into
|
|
* `/tmp` (Lambda's only writable filesystem path), invokes the primitive,
|
|
* uploads outputs back to S3, and returns a small JSON payload that fits
|
|
* inside Step Functions' history budget (under 200 bytes for chunk
|
|
* results per §2.4).
|
|
*/
|
|
|
|
import type {
|
|
DistributedFormat,
|
|
SerializableDistributedRenderConfig,
|
|
} from "@hyperframes/producer/distributed";
|
|
|
|
export type { SerializableDistributedRenderConfig } from "@hyperframes/producer/distributed";
|
|
|
|
/** Discriminator for the three roles the one Lambda image fulfills. */
|
|
export type LambdaAction = "plan" | "audio" | "renderChunk" | "assemble";
|
|
export interface WarmEvent {
|
|
Action: "warm";
|
|
}
|
|
/** Assemble hands a helper part of the final mp4's multipart upload: each part's S3 ranges, in order. */
|
|
export interface StitchPartsEvent {
|
|
Action: "stitchParts";
|
|
Bucket: string;
|
|
Key: string;
|
|
UploadId: string;
|
|
Parts: { partNumber: number; ranges: { uri: string; start: number; end: number }[] }[];
|
|
}
|
|
/** Transport protocol selected for one complete distributed render. */
|
|
export type LambdaPlanProtocol = "v1" | "v2";
|
|
|
|
/**
|
|
* Top-level shape of any event the handler may receive.
|
|
*
|
|
* Step Functions can also invoke with a wrapped payload (e.g. when a Map
|
|
* state's `ItemSelector` passes through `$$.Map.Item.Value`), so the
|
|
* handler unwraps both `event.Payload` and `event.Input` before
|
|
* dispatching.
|
|
*/
|
|
export type LambdaEvent =
|
|
| PlanEvent
|
|
| AudioEvent
|
|
| RenderChunkEvent
|
|
| AssembleEvent
|
|
| WarmEvent
|
|
| StitchPartsEvent
|
|
| { Payload: LambdaEvent }
|
|
| { Input: LambdaEvent };
|
|
|
|
/** Activity A: produce a planDir, upload to S3. */
|
|
interface PlanEventBase {
|
|
Action: "plan";
|
|
/** S3 URI pointing at a `tar -czf`-archived project directory (`s3://bucket/key.tar.gz`). */
|
|
ProjectS3Uri: string;
|
|
/** S3 URI prefix where the planDir tar should be uploaded (`s3://bucket/{prefix}/`). */
|
|
PlanOutputS3Prefix: string;
|
|
/** `DistributedRenderConfig` minus runtime-only fields (logger, abortSignal). */
|
|
Config: SerializableDistributedRenderConfig;
|
|
}
|
|
|
|
/**
|
|
* Legacy plan transport. Callers must select it explicitly.
|
|
*
|
|
* @deprecated Use {@link PlanV2Event} for new integrations.
|
|
*/
|
|
export interface PlanV1Event extends PlanEventBase {
|
|
PlanProtocol: "v1";
|
|
}
|
|
|
|
/** Default content-addressed v2 plan transport. */
|
|
export interface PlanV2Event extends PlanEventBase {
|
|
PlanProtocol?: "v2";
|
|
}
|
|
|
|
export type PlanEvent = PlanV1Event | PlanV2Event;
|
|
|
|
/** Makes a v2 plan's audio beside its chunks, from the same project and config as the plan. */
|
|
export interface AudioEvent extends Omit<PlanEventBase, "Action"> {
|
|
Action: "audio";
|
|
}
|
|
|
|
/** Activity B: fetch planDir, render one chunk, upload result. */
|
|
interface RenderChunkEventBase {
|
|
Action: "renderChunk";
|
|
/**
|
|
* `PlanResult.planHash` from the Plan invocation. For v1, the handler
|
|
* verifies it against the untarred planDir's `plan.json`; for v2, it
|
|
* verifies it against the content-addressed manifest before invoking the
|
|
* producer. Divergence throws a typed `PLAN_HASH_MISMATCH` so the state
|
|
* machine routes it as non-retryable.
|
|
*/
|
|
PlanHash: string;
|
|
/** 0-based chunk index this invocation should render. */
|
|
ChunkIndex: number;
|
|
/** S3 URI prefix where the chunk output should be uploaded (`s3://bucket/{prefix}/`). */
|
|
ChunkOutputS3Prefix: string;
|
|
/** Output container format from the plan's encoder.json; drives file vs frame-dir handling. */
|
|
Format: DistributedFormat;
|
|
}
|
|
|
|
/**
|
|
* Legacy chunk event. Callers must select it explicitly.
|
|
*
|
|
* @deprecated Use {@link RenderChunkV2Event} for new integrations.
|
|
*/
|
|
export interface RenderChunkV1Event extends RenderChunkEventBase {
|
|
PlanProtocol: "v1";
|
|
/** S3 URI of the v1 plan tar produced by a PlanEvent invocation. */
|
|
PlanS3Uri: string;
|
|
}
|
|
|
|
/**
|
|
* V2 chunk event. It intentionally cannot carry `PlanS3Uri`: the manifest
|
|
* describes the exact content-addressed artifacts needed by this chunk.
|
|
*/
|
|
export interface RenderChunkV2Event extends RenderChunkEventBase {
|
|
PlanProtocol?: "v2";
|
|
PlanV2ManifestS3Uri: string;
|
|
PlanV2ArtifactS3Prefix: string;
|
|
}
|
|
|
|
export type RenderChunkEvent = RenderChunkV1Event | RenderChunkV2Event;
|
|
|
|
/** Activity C: fetch planDir + all chunks + audio, assemble, upload final. */
|
|
interface AssembleEventBase {
|
|
Action: "assemble";
|
|
/** S3 URIs of every chunk, ordered by chunk index. Length must equal `chunkCount`. */
|
|
ChunkS3Uris: string[];
|
|
/** S3 URI of the planDir's audio artifact if the composition has audio; `null` otherwise. */
|
|
AudioS3Uri: string | null;
|
|
/** Final output S3 URI (`s3://bucket/key.mp4`). */
|
|
OutputS3Uri: string;
|
|
/** Output container format; drives file vs frame-dir handling. */
|
|
Format: DistributedFormat;
|
|
/**
|
|
* Optional exact-CFR re-encode at assemble time. When `true`, the final
|
|
* assembled video is re-encoded with `-fps_mode cfr -r <fps>` so the
|
|
* stream's `avg_frame_rate` matches the container's `r_frame_rate`
|
|
* exactly (and the file's duration is exact, not PTS-derived). Trade-off
|
|
* is ~2-5x the assemble wall-clock. mp4 only — webm / mov stream-copy
|
|
* paths already produce exact avg_frame_rate. Default `false` /
|
|
* unset preserves current `-c copy` behavior.
|
|
*/
|
|
Cfr?: boolean;
|
|
}
|
|
|
|
/**
|
|
* Legacy assemble event. Callers must select it explicitly.
|
|
*
|
|
* @deprecated Use {@link AssembleV2Event} for new integrations.
|
|
*/
|
|
export interface AssembleV1Event extends AssembleEventBase {
|
|
PlanProtocol: "v1";
|
|
/** S3 URI of the v1 plan tar produced by a PlanEvent invocation. */
|
|
PlanS3Uri: string;
|
|
}
|
|
|
|
/** V2 assemble event, scoped to manifest-declared assembler artifacts. */
|
|
export interface AssembleV2Event extends AssembleEventBase {
|
|
PlanProtocol?: "v2";
|
|
PlanV2ManifestS3Uri: string;
|
|
PlanV2ArtifactS3Prefix: string;
|
|
PlanHash: string;
|
|
}
|
|
|
|
export type AssembleEvent = AssembleV1Event | AssembleV2Event;
|
|
|
|
// ── Result types — kept small to fit Step Functions history budgets ─────────
|
|
|
|
/** Result of a `plan` invocation. Carries enough to size the Map(N) state. */
|
|
interface PlanLambdaResultBase {
|
|
Action: "plan";
|
|
PlanHash: string;
|
|
ChunkCount: number;
|
|
MaxChunkConcurrency: number;
|
|
TotalFrames: number;
|
|
Fps: SerializableDistributedRenderConfig["fps"];
|
|
Width: number;
|
|
Height: number;
|
|
Format: DistributedFormat;
|
|
HasAudio: boolean;
|
|
AudioS3Uri: string | null;
|
|
FfmpegVersion: string;
|
|
ProducerVersion: string;
|
|
DurationMs: number;
|
|
}
|
|
|
|
/**
|
|
* Existing v1 result. Kept unchanged for wire compatibility.
|
|
*
|
|
* @deprecated New integrations should consume {@link PlanV2LambdaResult}.
|
|
*/
|
|
export interface PlanV1LambdaResult extends PlanLambdaResultBase {
|
|
PlanS3Uri: string;
|
|
}
|
|
|
|
/** V2 result. The two v2 locators are never aliases for `PlanS3Uri`. */
|
|
export interface PlanV2LambdaResult extends PlanLambdaResultBase {
|
|
PlanProtocol: "v2";
|
|
PlanV2ManifestS3Uri: string;
|
|
PlanV2ArtifactS3Prefix: string;
|
|
/** The plan left its audio to the audio task, which has a worker kept free for it. */
|
|
SeparateAudio: boolean;
|
|
}
|
|
|
|
export type PlanLambdaResult = PlanV1LambdaResult | PlanV2LambdaResult;
|
|
|
|
export interface AudioLambdaResult {
|
|
Action: "audio";
|
|
HasAudio: boolean;
|
|
/** The track for the assembler to stitch in; `null` without audio or when the plan mixed it. */
|
|
AudioS3Uri: string | null;
|
|
DurationMs: number;
|
|
}
|
|
|
|
/** Result of a `renderChunk` invocation. Sized ≤200 bytes per §2.4. */
|
|
export interface RenderChunkLambdaResult {
|
|
Action: "renderChunk";
|
|
ChunkS3Uri: string;
|
|
ChunkIndex: number;
|
|
Sha256: string;
|
|
FramesEncoded: number;
|
|
/** Effective engine mode after browser probing. Emitted by current handlers. */
|
|
CaptureMode?: "beginframe" | "screenshot" | "drawelement";
|
|
DurationMs: number;
|
|
}
|
|
|
|
/** Result of an `assemble` invocation. */
|
|
export interface AssembleLambdaResult {
|
|
Action: "assemble";
|
|
OutputS3Uri: string;
|
|
FramesEncoded: number;
|
|
FileSize: number;
|
|
DurationMs: number;
|
|
}
|
|
|
|
export interface WarmLambdaResult {
|
|
Action: "warm";
|
|
}
|
|
|
|
export interface StitchPartsLambdaResult {
|
|
Action: "stitchParts";
|
|
ETags: string[];
|
|
}
|
|
|
|
export type LambdaResult =
|
|
| WarmLambdaResult
|
|
| StitchPartsLambdaResult
|
|
| PlanLambdaResult
|
|
| AudioLambdaResult
|
|
| RenderChunkLambdaResult
|
|
| AssembleLambdaResult;
|