mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-20 20:38:34 +00:00
perf(aws-lambda): build the final mp4 inside S3 with its index up front
Distributed mp4 chunks now carry their slice of the plan audio. Assemble writes one faststart header (ftyp, a moov indexing every chunk's samples, a 64-bit mdat header) and S3 copies each chunk's mdat payload after it with UploadPartCopy.
This commit is contained in:
@@ -279,6 +279,18 @@ describe("HyperframesRenderStack — snapshot", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("lets the render function abort the multipart uploads its S3 stitch starts", () => {
|
||||
const cdkPolicies = JSON.stringify(SYNTHED.template.findResources("AWS::IAM::Policy"));
|
||||
expect(cdkPolicies).toContain("s3:Abort*");
|
||||
const samFunction = requireRecordProperty(readSamResources(), "RenderFunction", "SAM function");
|
||||
const samProperties = requireRecordProperty(
|
||||
samFunction,
|
||||
"Properties",
|
||||
"SAM function properties",
|
||||
);
|
||||
expect(JSON.stringify(samProperties.Policies)).toContain("s3:AbortMultipartUpload");
|
||||
});
|
||||
|
||||
it("keeps same-named Lambda tasks in different branches apart", () => {
|
||||
const task = (format: string) => ({
|
||||
Type: "Task",
|
||||
@@ -433,7 +445,7 @@ function getV1TaskStates(definition: {
|
||||
};
|
||||
}
|
||||
|
||||
function readSamDefinition(): { States: Record<string, unknown> } {
|
||||
function readSamResources(): Record<string, unknown> {
|
||||
const source = readFileSync(
|
||||
new URL("../../../../examples/aws-lambda/template.yaml", import.meta.url),
|
||||
"utf8",
|
||||
@@ -443,7 +455,11 @@ function readSamDefinition(): { States: Record<string, unknown> } {
|
||||
// warnings for the intentionally unresolved `!Ref`/`!GetAtt` tags.
|
||||
const parsed: unknown = parseYaml(source, { logLevel: "silent" });
|
||||
const root = requireRecord(parsed, "SAM template");
|
||||
const resources = requireRecordProperty(root, "Resources", "SAM resources");
|
||||
return requireRecordProperty(root, "Resources", "SAM resources");
|
||||
}
|
||||
|
||||
function readSamDefinition(): { States: Record<string, unknown> } {
|
||||
const resources = readSamResources();
|
||||
const stateMachine = requireRecordProperty(
|
||||
resources,
|
||||
"RenderStateMachine",
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -35,7 +36,7 @@ import type { AssembleEvent, LambdaEvent, PlanEvent, RenderChunkEvent } from "./
|
||||
import { handler, unwrapEvent } from "./handler.js";
|
||||
|
||||
interface FakeS3Op {
|
||||
kind: "download" | "upload";
|
||||
kind: "download" | "upload" | "copy";
|
||||
uri: string;
|
||||
bytes?: number;
|
||||
}
|
||||
@@ -63,12 +64,19 @@ class FakeS3Client {
|
||||
const cmdName = op.constructor?.name ?? "";
|
||||
const uri = `s3://${op.input.Bucket}/${op.input.Key}`;
|
||||
if (cmdName === "GetObjectCommand") {
|
||||
const bytes = this.objects.get(uri) ?? Buffer.alloc(0);
|
||||
const bytes = this.sliceRange(
|
||||
this.objects.get(uri) ?? Buffer.alloc(0),
|
||||
(command as { input: { Range?: string } }).input.Range,
|
||||
);
|
||||
this.ops.push({ kind: "download", uri, bytes: bytes.length });
|
||||
// Mock the AWS SDK stream contract just enough for pipeline() to
|
||||
// pump bytes into a write stream.
|
||||
const { Readable } = await import("node:stream");
|
||||
return { Body: Readable.from([bytes]) };
|
||||
return {
|
||||
Body: Object.assign(Readable.from([bytes]), {
|
||||
transformToByteArray: async () => new Uint8Array(bytes),
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (cmdName === "HeadObjectCommand") {
|
||||
const bytes = this.objects.get(uri);
|
||||
@@ -104,8 +112,37 @@ class FakeS3Client {
|
||||
if (metadata) this.metadata.set(uri, metadata);
|
||||
return {};
|
||||
}
|
||||
if (cmdName === "CreateMultipartUploadCommand") return { UploadId: "upload-1" };
|
||||
if (cmdName === "UploadPartCommand") {
|
||||
const input = (command as { input: { Body: Buffer; PartNumber: number } }).input;
|
||||
this.parts.set(input.PartNumber, input.Body);
|
||||
return { ETag: `"part-${input.PartNumber}"` };
|
||||
}
|
||||
if (cmdName === "UploadPartCopyCommand") {
|
||||
const input = (
|
||||
command as { input: { CopySource: string; CopySourceRange?: string; PartNumber: number } }
|
||||
).input;
|
||||
const source = `s3://${decodeURI(input.CopySource)}`;
|
||||
this.ops.push({ kind: "copy", uri: source });
|
||||
const bytes = this.objects.get(source) ?? Buffer.alloc(0);
|
||||
this.parts.set(input.PartNumber, this.sliceRange(bytes, input.CopySourceRange));
|
||||
return { CopyPartResult: { ETag: `"part-${input.PartNumber}"` } };
|
||||
}
|
||||
if (cmdName === "CompleteMultipartUploadCommand") {
|
||||
const parts = (command as { input: { MultipartUpload: { Parts: { PartNumber: number }[] } } })
|
||||
.input.MultipartUpload.Parts;
|
||||
this.objects.set(uri, Buffer.concat(parts.map((part) => this.parts.get(part.PartNumber)!)));
|
||||
return {};
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
parts = new Map<number, Buffer>();
|
||||
|
||||
private sliceRange(bytes: Buffer, range?: string): Buffer {
|
||||
const match = /^bytes=(\d+)-(\d+)$/.exec(range ?? "");
|
||||
return match ? bytes.subarray(Number(match[1]), Number(match[2]) + 1) : bytes;
|
||||
}
|
||||
}
|
||||
|
||||
const tmpDirs: string[] = [];
|
||||
@@ -730,6 +767,90 @@ describe("handler dispatch", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
async function planStitched(chunkCount: number) {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
const assembleMock = mock(
|
||||
async (_p: string, _c: readonly string[], _a: string | null, outputPath: string) => {
|
||||
writeFileSync(outputPath, "LOCAL-JOIN");
|
||||
return { outputPath, durationMs: 1, framesEncoded: 30 * chunkCount, fileSize: 10 };
|
||||
},
|
||||
);
|
||||
const planV2WithPublisherMock = mock(
|
||||
async (_dir: string, _config: unknown, publisher: PlanV2ArtifactPublisher) => {
|
||||
const v1Dir = join(tmpRoot, "v1");
|
||||
makeMinimalV1PlanDir(v1Dir, true, { chunkCount, stitchMp4: true });
|
||||
return publishPlanV2FromExecutionPlan(v1Dir, publisher);
|
||||
},
|
||||
);
|
||||
const deps = {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
planV2WithPublisher:
|
||||
planV2WithPublisherMock as unknown as typeof import("@hyperframes/producer/distributed").planV2WithPublisher,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
};
|
||||
const planned = await handler(
|
||||
{
|
||||
Action: "plan",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/f/",
|
||||
Config: { fps: 30, width: 640, height: 360, format: "mp4" },
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2")
|
||||
throw new Error("expected v2");
|
||||
const assemble = (chunkS3Uris: string[]) =>
|
||||
handler(
|
||||
{
|
||||
Action: "assemble",
|
||||
PlanV2ManifestS3Uri: planned.PlanV2ManifestS3Uri,
|
||||
PlanV2ArtifactS3Prefix: planned.PlanV2ArtifactS3Prefix,
|
||||
PlanHash: planned.PlanHash,
|
||||
ChunkS3Uris: chunkS3Uris,
|
||||
AudioS3Uri: null,
|
||||
OutputS3Uri: "s3://bucket/renders/f/output.mp4",
|
||||
Format: "mp4",
|
||||
},
|
||||
deps,
|
||||
);
|
||||
return { s3, assembleMock, assemble };
|
||||
}
|
||||
|
||||
it.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(
|
||||
"stitches mp4 chunks inside S3 instead of joining them locally",
|
||||
async () => {
|
||||
const { s3, assembleMock, assemble } = await planStitched(2);
|
||||
const chunks = [0, 1].map((index) => {
|
||||
const path = join(makeTmpRoot(), "chunk.mp4");
|
||||
const lavfi = ["testsrc2=size=64x48:rate=30:duration=1", "sine=duration=1"];
|
||||
const muxed = spawnSync("ffmpeg", [
|
||||
...["-nostdin", "-loglevel", "error", "-f", "lavfi", "-i", lavfi[0]!],
|
||||
...["-f", "lavfi", "-i", lavfi[1]!, "-c:v", "libx264", "-bf", "0", "-c:a", "aac"],
|
||||
...["-shortest", "-y", path],
|
||||
]);
|
||||
if (muxed.status !== 0) throw new Error(String(muxed.stderr));
|
||||
const uri = `s3://bucket/renders/f/chunks/000${index}.mp4`;
|
||||
s3.objects.set(uri, readFileSync(path));
|
||||
return uri;
|
||||
});
|
||||
const result = await assemble(chunks);
|
||||
const output = s3.objects.get("s3://bucket/renders/f/output.mp4")!;
|
||||
expect(output.toString("latin1", 4, 8)).toBe("ftyp");
|
||||
expect(
|
||||
output.toString("latin1", output.readUInt32BE(0) + 4, output.readUInt32BE(0) + 8),
|
||||
).toBe("moov");
|
||||
expect(result).toMatchObject({ FramesEncoded: 60, FileSize: output.length });
|
||||
expect(assembleMock).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects unknown plan protocol values", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
@@ -903,14 +1024,19 @@ async function makeMinimalPlanTar(): Promise<Buffer> {
|
||||
return rf(tarPath);
|
||||
}
|
||||
|
||||
function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void {
|
||||
function makeMinimalV1PlanDir(
|
||||
dir: string,
|
||||
withAudio: boolean,
|
||||
options: { chunkCount?: number; stitchMp4?: boolean } = {},
|
||||
): void {
|
||||
const chunkCount = options.chunkCount ?? 1;
|
||||
mkdirSync(join(dir, "meta"), { recursive: true });
|
||||
mkdirSync(join(dir, "compiled"), { recursive: true });
|
||||
writeFileSync(join(dir, "compiled", "index.html"), "<html>aws v2 fixture</html>");
|
||||
const planJson = {
|
||||
planHash: "a".repeat(64),
|
||||
chunkCount: 1,
|
||||
totalFrames: 30,
|
||||
chunkCount,
|
||||
totalFrames: 30 * chunkCount,
|
||||
dimensions: { fpsNum: 30, fpsDen: 1, width: 640, height: 360, format: "mp4" },
|
||||
ffmpegVersion: "6.0",
|
||||
producerVersion: "test",
|
||||
@@ -919,9 +1045,18 @@ function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void {
|
||||
writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));
|
||||
writeFileSync(
|
||||
join(dir, "meta", "chunks.json"),
|
||||
JSON.stringify([{ index: 0, startFrame: 0, endFrame: 30 }]),
|
||||
JSON.stringify(
|
||||
Array.from({ length: chunkCount }, (_, index) => ({
|
||||
index,
|
||||
startFrame: index * 30,
|
||||
endFrame: (index + 1) * 30,
|
||||
})),
|
||||
),
|
||||
);
|
||||
writeFileSync(
|
||||
join(dir, "meta", "encoder.json"),
|
||||
options.stitchMp4 ? JSON.stringify({ stitchMp4: true }) : "{}",
|
||||
);
|
||||
writeFileSync(join(dir, "meta", "encoder.json"), "{}");
|
||||
if (withAudio) writeFileSync(join(dir, "audio.m4a"), "AAC");
|
||||
planJson.planHash = recomputePlanHashFromPlanDir(dir);
|
||||
writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));
|
||||
|
||||
@@ -17,6 +17,7 @@ import { basename, join } from "node:path";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
assemble,
|
||||
chunksAreStitchedMp4,
|
||||
type AssembleResult,
|
||||
type ChunkRenderer,
|
||||
type ChunkResult,
|
||||
@@ -36,6 +37,7 @@ import {
|
||||
renderChunk,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
import { resolveChromeExecutablePath } from "./chromium.js";
|
||||
import { stitchMp4InS3 } from "./s3Stitch.js";
|
||||
import { type DistributedFormat, formatExtension } from "./formatExtension.js";
|
||||
import type {
|
||||
AssembleEvent,
|
||||
@@ -440,6 +442,7 @@ async function handlePlanV2(
|
||||
});
|
||||
const manifest: PlanV2Manifest = await primitive(projectDir, config, publisher, {
|
||||
stagingParentDir: work,
|
||||
stitchMp4: true,
|
||||
});
|
||||
|
||||
return {
|
||||
@@ -551,7 +554,7 @@ async function handleRenderChunkV2(
|
||||
}
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-chunk-v2-"));
|
||||
try {
|
||||
const planDir = await downloadAndMaterializePlanV2(
|
||||
const { planDir } = await downloadAndMaterializePlanV2(
|
||||
s3,
|
||||
event,
|
||||
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
||||
@@ -677,7 +680,15 @@ async function handleAssembleV2(
|
||||
const primitive = deps?.primitives?.assemble ?? assemble;
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-assemble-v2-"));
|
||||
try {
|
||||
const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
|
||||
const { planDir, manifest } = await downloadAndMaterializePlanV2(
|
||||
s3,
|
||||
event,
|
||||
{ role: "assembler" },
|
||||
work,
|
||||
);
|
||||
if (chunksAreStitchedMp4(planDir, event.Cfr === true)) {
|
||||
return await stitchChunksInS3(s3, event, manifest, started);
|
||||
}
|
||||
// `downloadAndMaterializePlanV2` materializes atomically. Audio is
|
||||
// assembler-only and lives at the familiar v1-compatible location.
|
||||
const audioPath = resolvePlanAudioPath(planDir);
|
||||
@@ -718,6 +729,36 @@ async function handleAssembleV2(
|
||||
}
|
||||
}
|
||||
|
||||
async function stitchChunksInS3(
|
||||
s3: S3Client,
|
||||
event: AssembleV2Event,
|
||||
plan: PlanV2Manifest,
|
||||
started: number,
|
||||
): Promise<AssembleLambdaResult> {
|
||||
if (event.ChunkS3Uris.length !== plan.chunkCount) {
|
||||
throw new Error(
|
||||
`[handler] ${event.ChunkS3Uris.length} chunk URIs for a plan of ${plan.chunkCount} chunks`,
|
||||
);
|
||||
}
|
||||
const stitchStarted = Date.now();
|
||||
const stitched = await stitchMp4InS3(s3, event.ChunkS3Uris, event.OutputS3Uri);
|
||||
logEvent({
|
||||
event: "assemble_stages",
|
||||
mode: "s3-stitch",
|
||||
planMs: stitchStarted - started,
|
||||
stitchMs: Date.now() - stitchStarted,
|
||||
parts: stitched.parts,
|
||||
downloadedBytes: stitched.downloadedBytes,
|
||||
});
|
||||
return {
|
||||
Action: "assemble",
|
||||
OutputS3Uri: event.OutputS3Uri,
|
||||
FramesEncoded: plan.totalFrames,
|
||||
FileSize: stitched.fileSize,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadAndMaterializePlanV2(
|
||||
s3: S3Client,
|
||||
event: {
|
||||
@@ -727,7 +768,7 @@ async function downloadAndMaterializePlanV2(
|
||||
},
|
||||
target: PlanV2MaterializationTarget,
|
||||
work: string,
|
||||
): Promise<string> {
|
||||
): Promise<{ planDir: string; manifest: PlanV2Manifest }> {
|
||||
const transportDir = join(work, "plan-v2");
|
||||
mkdirSync(transportDir, { recursive: true });
|
||||
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join(transportDir, "plan.json"));
|
||||
@@ -744,7 +785,7 @@ async function downloadAndMaterializePlanV2(
|
||||
});
|
||||
const planDir = join(work, "plan");
|
||||
materializePlanV2Target(transportDir, target, planDir);
|
||||
return planDir;
|
||||
return { planDir, manifest };
|
||||
}
|
||||
|
||||
async function downloadPlanV2Artifact(
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import { assemble, muxChunkWithAudio } from "@hyperframes/producer/distributed";
|
||||
import { type Part, planParts, stitchMp4InS3, uploadParts } from "./s3Stitch.js";
|
||||
|
||||
const MiB = 1024 * 1024;
|
||||
const range = (uri: string, start: number, end: number) => ({ uri, start, end });
|
||||
|
||||
describe("planParts", () => {
|
||||
it("copies big ranges and merges the header into a part that borrows from chunk 0", () => {
|
||||
const header = { bytes: Buffer.alloc(MiB) };
|
||||
const parts = planParts([
|
||||
header,
|
||||
range("s3://b/0", 40, 40 + 10 * MiB),
|
||||
range("s3://b/1", 40, 40 + 6 * MiB),
|
||||
range("s3://b/2", 40, 1040),
|
||||
]);
|
||||
expect(parts).toEqual([
|
||||
{ upload: [header, range("s3://b/0", 40, 40 + 4 * MiB)] },
|
||||
{ copy: range("s3://b/0", 40 + 4 * MiB, 40 + 10 * MiB) },
|
||||
{ copy: range("s3://b/1", 40, 40 + 6 * MiB) },
|
||||
{ upload: [range("s3://b/2", 40, 1040)] },
|
||||
]);
|
||||
});
|
||||
|
||||
it("downloads a chunk whole when borrowing would leave its rest under the minimum", () => {
|
||||
const header = { bytes: Buffer.alloc(MiB) };
|
||||
const small = range("s3://b/1", 0, 2 * MiB);
|
||||
const parts = planParts([
|
||||
header,
|
||||
range("s3://b/0", 0, 8 * MiB),
|
||||
small,
|
||||
range("s3://b/2", 0, 9 * MiB),
|
||||
]);
|
||||
expect(parts).toEqual([
|
||||
{ upload: [header, range("s3://b/0", 0, 8 * MiB)] },
|
||||
{ upload: [small, range("s3://b/2", 0, 3 * MiB)] },
|
||||
{ copy: range("s3://b/2", 3 * MiB, 9 * MiB) },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
/** S3 stand-in: ranged reads, and multipart parts concatenated on complete under S3's size rule. */
|
||||
class FakeS3 {
|
||||
readonly objects = new Map<string, Buffer>();
|
||||
readonly sent: { name: string; input: Record<string, unknown> }[] = [];
|
||||
private readonly parts = new Map<number, Buffer>();
|
||||
constructor(
|
||||
private readonly minPartBytes: number,
|
||||
private readonly failPart?: number,
|
||||
) {}
|
||||
|
||||
private object(bucket: unknown, key: unknown, byteRange?: unknown): Buffer {
|
||||
const bytes = this.objects.get(`s3://${bucket}/${key}`);
|
||||
if (!bytes) throw new Error(`NoSuchKey ${bucket}/${key}`);
|
||||
const match = /^bytes=(\d+)-(\d+)$/.exec(String(byteRange ?? ""));
|
||||
return match ? bytes.subarray(Number(match[1]), Number(match[2]) + 1) : bytes;
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
async send(command: { constructor: { name: string }; input: Record<string, unknown> }) {
|
||||
const { name } = command.constructor;
|
||||
const input = command.input;
|
||||
this.sent.push({ name, input });
|
||||
if (this.failPart && input.PartNumber === this.failPart) throw new Error("part failed");
|
||||
if (name === "GetObjectCommand") {
|
||||
const bytes = this.object(input.Bucket, input.Key, input.Range);
|
||||
return { Body: { transformToByteArray: async () => new Uint8Array(bytes) } };
|
||||
}
|
||||
if (name === "CreateMultipartUploadCommand") return { UploadId: "up-1" };
|
||||
if (name === "UploadPartCommand") {
|
||||
this.parts.set(input.PartNumber as number, input.Body as Buffer);
|
||||
return { ETag: `"p${input.PartNumber}"` };
|
||||
}
|
||||
if (name === "UploadPartCopyCommand") {
|
||||
const [bucket, ...key] = decodeURI(String(input.CopySource)).split("/");
|
||||
const bytes = this.object(bucket, key.join("/"), input.CopySourceRange);
|
||||
this.parts.set(input.PartNumber as number, bytes);
|
||||
return { CopyPartResult: { ETag: `"p${input.PartNumber}"` } };
|
||||
}
|
||||
if (name === "CompleteMultipartUploadCommand") {
|
||||
const listed = (input.MultipartUpload as { Parts: { PartNumber: number }[] }).Parts;
|
||||
const parts = listed.map((part) => this.parts.get(part.PartNumber)!);
|
||||
if (parts.slice(0, -1).some((part) => part.length < this.minPartBytes)) {
|
||||
throw new Error("EntityTooSmall");
|
||||
}
|
||||
this.objects.set(`s3://${input.Bucket}/${input.Key}`, Buffer.concat(parts));
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
asClient(): S3Client {
|
||||
return this as unknown as S3Client;
|
||||
}
|
||||
}
|
||||
|
||||
describe("uploadParts", () => {
|
||||
const parts: Part[] = [
|
||||
{ upload: [{ bytes: Buffer.from("header") }] },
|
||||
{ copy: range("s3://bucket/r/chunks/0001.mp4", 0, 4) },
|
||||
];
|
||||
|
||||
it("aborts the upload when a part fails", async () => {
|
||||
const s3 = new FakeS3(0, 2);
|
||||
s3.objects.set("s3://bucket/r/chunks/0001.mp4", Buffer.from("abcd"));
|
||||
await expect(
|
||||
uploadParts(s3.asClient(), parts, "s3://bucket/r/out.mp4", "video/mp4"),
|
||||
).rejects.toThrow("part failed");
|
||||
expect(s3.sent.at(-1)).toEqual({
|
||||
name: "AbortMultipartUploadCommand",
|
||||
input: { Bucket: "bucket", Key: "r/out.mp4", UploadId: "up-1" },
|
||||
});
|
||||
expect(s3.sent.some((c) => c.name === "CompleteMultipartUploadCommand")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
const hasFfmpeg = spawnSync("ffmpeg", ["-version"]).status === 0;
|
||||
|
||||
describe.skipIf(!hasFfmpeg)("stitchMp4InS3 on real chunks", () => {
|
||||
const FPS = { num: 30, den: 1 };
|
||||
// Uneven chunks and a 1-frame tail.
|
||||
const CHUNKS = [
|
||||
{ index: 0, startFrame: 0, endFrame: 47 },
|
||||
{ index: 1, startFrame: 47, endFrame: 90 },
|
||||
{ index: 2, startFrame: 90, endFrame: 91 },
|
||||
];
|
||||
const TOTAL_FRAMES = 91;
|
||||
let root: string;
|
||||
|
||||
beforeAll(() => {
|
||||
root = mkdtempSync(join(tmpdir(), "hf-s3-stitch-"));
|
||||
});
|
||||
afterAll(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function run(cmd: string, args: string[]): string {
|
||||
const result = spawnSync(cmd, args, { maxBuffer: 1 << 28, encoding: "latin1" });
|
||||
if (result.status !== 0) throw new Error(`${cmd} ${args.join(" ")}: ${result.stderr}`);
|
||||
return result.stdout;
|
||||
}
|
||||
const ffmpeg = (args: string[]) => run("ffmpeg", ["-nostdin", "-loglevel", "error", ...args]);
|
||||
|
||||
/** Closed-GOP chunk shaped like renderChunk's: IDR at frame 0, no B-frames. */
|
||||
function encodeChunk(path: string, slice: (typeof CHUNKS)[number]): void {
|
||||
const frames = String(slice.endFrame - slice.startFrame);
|
||||
ffmpeg(
|
||||
`-f lavfi -i testsrc2=size=160x120:rate=30,trim=start_frame=${slice.startFrame}:end_frame=${slice.endFrame} -c:v libx264 -preset veryfast -g ${frames} -keyint_min ${frames} -sc_threshold 0 -bf 0 -x264-params scenecut=0:open-gop=0:repeat-headers=1 -pix_fmt yuv420p -avoid_negative_ts make_zero -y`
|
||||
.split(" ")
|
||||
.concat(path),
|
||||
);
|
||||
}
|
||||
|
||||
const frames = (path: string) =>
|
||||
ffmpeg(["-i", path, "-map", "0:v", "-f", "framemd5", "-"])
|
||||
.split("\n")
|
||||
.filter((line) => line && !line.startsWith("#"));
|
||||
const pcm = (path: string) => ffmpeg(["-i", path, "-map", "0:a", "-f", "s16le", "-"]);
|
||||
const probe = (path: string) =>
|
||||
JSON.parse(
|
||||
run("ffprobe", [
|
||||
"-v",
|
||||
"error",
|
||||
"-count_packets",
|
||||
"-show_entries",
|
||||
"stream=codec_type,nb_read_packets,duration",
|
||||
"-of",
|
||||
"json",
|
||||
"--",
|
||||
path,
|
||||
]),
|
||||
).streams as { codec_type: string; nb_read_packets: string; duration: string }[];
|
||||
|
||||
it("assembles a faststart mp4 with the frames and audio of today's join", async () => {
|
||||
const audioPath = join(root, "audio.m4a");
|
||||
ffmpeg(
|
||||
`-f lavfi -i aevalsrc=0.4*sin(2*PI*(220+40*t)*t):s=48000:d=${TOTAL_FRAMES / 30} -c:a aac -b:a 192k -y`
|
||||
.split(" ")
|
||||
.concat(audioPath),
|
||||
);
|
||||
const planDir = join(root, "plan");
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(
|
||||
join(planDir, "plan.json"),
|
||||
JSON.stringify({
|
||||
planHash: "fake",
|
||||
totalFrames: TOTAL_FRAMES,
|
||||
hasAudio: true,
|
||||
dimensions: { fpsNum: 30, fpsDen: 1, width: 160, height: 120, format: "mp4" },
|
||||
}),
|
||||
);
|
||||
writeFileSync(join(planDir, "meta", "chunks.json"), JSON.stringify(CHUNKS));
|
||||
|
||||
const s3 = new FakeS3(16 * 1024);
|
||||
const videoPaths: string[] = [];
|
||||
const chunkPaths: string[] = [];
|
||||
for (const slice of CHUNKS) {
|
||||
videoPaths.push(join(root, `video-${slice.index}.mp4`));
|
||||
chunkPaths.push(join(root, `chunk-${slice.index}.mp4`));
|
||||
encodeChunk(videoPaths[slice.index]!, slice);
|
||||
await muxChunkWithAudio({
|
||||
videoPath: videoPaths[slice.index]!,
|
||||
audioPath,
|
||||
outputPath: chunkPaths[slice.index]!,
|
||||
chunks: CHUNKS,
|
||||
chunkIndex: slice.index,
|
||||
fps: FPS,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
s3.objects.set(
|
||||
`s3://bucket/chunks/${slice.index}.mp4`,
|
||||
readFileSync(chunkPaths[slice.index]!),
|
||||
);
|
||||
}
|
||||
const uris = CHUNKS.map((slice) => `s3://bucket/chunks/${slice.index}.mp4`);
|
||||
// A 16 KiB part minimum makes these small chunks take every path S3 takes at 5 MiB.
|
||||
const result = await stitchMp4InS3(s3.asClient(), uris, "s3://bucket/out.mp4", 16 * 1024);
|
||||
const outputPath = join(root, "out.mp4");
|
||||
const output = s3.objects.get("s3://bucket/out.mp4")!;
|
||||
writeFileSync(outputPath, output);
|
||||
expect(result.fileSize).toBe(output.length);
|
||||
const sent = s3.sent.map((c) => c.name);
|
||||
expect(sent).toContain("UploadPartCopyCommand");
|
||||
expect(sent).toContain("UploadPartCommand");
|
||||
|
||||
const check = spawnSync("ffprobe", ["-v", "error", "--", outputPath], { encoding: "utf-8" });
|
||||
expect([check.status, check.stderr]).toEqual([0, ""]);
|
||||
const topLevel: string[] = [];
|
||||
for (let at = 0; at < output.length;) {
|
||||
topLevel.push(output.toString("latin1", at + 4, at + 8));
|
||||
const size = output.readUInt32BE(at);
|
||||
at += size === 1 ? Number(output.readBigUInt64BE(at + 8)) : size;
|
||||
}
|
||||
expect(topLevel).toEqual(["ftyp", "moov", "mdat"]);
|
||||
|
||||
// The same chunks joined on disk give the same bytes.
|
||||
writeFileSync(join(planDir, "meta", "encoder.json"), JSON.stringify({ stitchMp4: true }));
|
||||
const localPath = join(root, "local.mp4");
|
||||
await assemble(planDir, chunkPaths, null, localPath);
|
||||
expect(readFileSync(localPath).equals(output)).toBe(true);
|
||||
|
||||
// Today's join: the video-only chunks plus the plan audio.
|
||||
writeFileSync(join(planDir, "meta", "encoder.json"), "{}");
|
||||
const referencePath = join(root, "reference.mp4");
|
||||
await assemble(planDir, videoPaths, audioPath, referencePath);
|
||||
expect(frames(outputPath)).toEqual(frames(referencePath));
|
||||
expect(pcm(outputPath) === pcm(referencePath)).toBe(true);
|
||||
|
||||
const [video, sound] = probe(outputPath);
|
||||
const audioPackets = probe(audioPath)[0]!.nb_read_packets;
|
||||
expect([video!.nb_read_packets, sound!.nb_read_packets]).toEqual([
|
||||
String(TOTAL_FRAMES),
|
||||
audioPackets,
|
||||
]);
|
||||
// Same lengths as today's mux: the edit lists trim only the AAC priming (both round
|
||||
// through the 1000-tick movie timescale).
|
||||
const reference = probe(referencePath);
|
||||
expect([video!.duration, sound!.duration]).toEqual([
|
||||
reference[0]!.duration,
|
||||
reference[1]!.duration,
|
||||
]);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,172 @@
|
||||
// Builds the final mp4 inside S3: part 1 carries the stitched header, the other parts copy each
|
||||
// chunk's mdat payload with UploadPartCopy, so almost no video byte passes through the function.
|
||||
|
||||
import {
|
||||
AbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
GetObjectCommand,
|
||||
type S3Client,
|
||||
UploadPartCommand,
|
||||
UploadPartCopyCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { buildStitchedHeader, readMp4Layout } from "@hyperframes/producer/distributed";
|
||||
import { parseS3Uri } from "./s3Transport.js";
|
||||
|
||||
const MIN_PART_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
/** Bytes `[start, end)` of an S3 object. */
|
||||
interface RangeSegment {
|
||||
uri: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
type Segment = { bytes: Buffer } | RangeSegment;
|
||||
export type Part = { copy: RangeSegment } | { upload: Segment[] };
|
||||
|
||||
const segmentSize = (s: Segment): number => ("bytes" in s ? s.bytes.length : s.end - s.start);
|
||||
|
||||
// Every part but the last must be >= `min` and a copy reads one object: big ranges are copied,
|
||||
// smaller pieces are downloaded into one upload part that borrows the next range's head if its rest
|
||||
// stays copyable.
|
||||
// fallow-ignore-next-line complexity
|
||||
export function planParts(segments: readonly Segment[], min = MIN_PART_BYTES): Part[] {
|
||||
const parts: Part[] = [];
|
||||
let pending: Segment[] = [];
|
||||
let pendingSize = 0;
|
||||
const flush = () => {
|
||||
if (pending.length) parts.push({ upload: pending });
|
||||
pending = [];
|
||||
pendingSize = 0;
|
||||
};
|
||||
for (const segment of segments) {
|
||||
const borrow = pendingSize ? min - pendingSize : 0;
|
||||
if ("uri" in segment && segmentSize(segment) - borrow >= min) {
|
||||
if (borrow) pending.push({ ...segment, end: segment.start + borrow });
|
||||
flush();
|
||||
parts.push({ copy: { ...segment, start: segment.start + borrow } });
|
||||
continue;
|
||||
}
|
||||
pending.push(segment);
|
||||
pendingSize += segmentSize(segment);
|
||||
if (pendingSize >= min) flush();
|
||||
}
|
||||
flush();
|
||||
return parts;
|
||||
}
|
||||
|
||||
async function getRange(s3: S3Client, uri: string, start: number, end: number): Promise<Buffer> {
|
||||
const { bucket, key } = parseS3Uri(uri);
|
||||
const response = await s3.send(
|
||||
new GetObjectCommand({ Bucket: bucket, Key: key, Range: `bytes=${start}-${end - 1}` }),
|
||||
);
|
||||
if (!response.Body) throw new Error(`[s3Stitch] empty body for ${uri}`);
|
||||
return Buffer.from(await response.Body.transformToByteArray());
|
||||
}
|
||||
|
||||
type UploadTarget = { Bucket: string; Key: string; UploadId: string };
|
||||
|
||||
async function sendPart(s3: S3Client, part: Part, partNumber: number, target: UploadTarget) {
|
||||
if ("copy" in part) {
|
||||
const { uri, start, end } = part.copy;
|
||||
const source = parseS3Uri(uri);
|
||||
const result = await s3.send(
|
||||
new UploadPartCopyCommand({
|
||||
...target,
|
||||
PartNumber: partNumber,
|
||||
CopySource: encodeURI(`${source.bucket}/${source.key}`),
|
||||
CopySourceRange: `bytes=${start}-${end - 1}`,
|
||||
}),
|
||||
);
|
||||
return result.CopyPartResult?.ETag;
|
||||
}
|
||||
const body = await Promise.all(
|
||||
part.upload.map((s) => ("bytes" in s ? s.bytes : getRange(s3, s.uri, s.start, s.end))),
|
||||
);
|
||||
const result = await s3.send(
|
||||
new UploadPartCommand({ ...target, PartNumber: partNumber, Body: Buffer.concat(body) }),
|
||||
);
|
||||
return result.ETag;
|
||||
}
|
||||
|
||||
/** Settles every part before failing, so an abort never races a part still in flight. */
|
||||
async function sendParts(s3: S3Client, parts: readonly Part[], target: UploadTarget) {
|
||||
const settled = await Promise.allSettled(
|
||||
parts.map(async (part, index) => {
|
||||
const etag = await sendPart(s3, part, index + 1, target);
|
||||
if (!etag) throw new Error(`[s3Stitch] part ${index + 1} returned no ETag`);
|
||||
return etag;
|
||||
}),
|
||||
);
|
||||
const failure = settled.find(
|
||||
(result): result is PromiseRejectedResult => result.status === "rejected",
|
||||
);
|
||||
if (failure) throw failure.reason;
|
||||
return settled.map((result) => (result as PromiseFulfilledResult<string>).value);
|
||||
}
|
||||
|
||||
/** Writes `parts` in order to `destUri`; a failure aborts the upload so no parts linger. */
|
||||
export async function uploadParts(
|
||||
s3: S3Client,
|
||||
parts: readonly Part[],
|
||||
destUri: string,
|
||||
contentType: string,
|
||||
): Promise<void> {
|
||||
const { bucket, key } = parseS3Uri(destUri);
|
||||
const created = await s3.send(
|
||||
new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ContentType: contentType }),
|
||||
);
|
||||
if (!created.UploadId) throw new Error(`[s3Stitch] no upload id for ${destUri}`);
|
||||
const target = { Bucket: bucket, Key: key, UploadId: created.UploadId };
|
||||
try {
|
||||
const etags = await sendParts(s3, parts, target);
|
||||
await s3.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
...target,
|
||||
MultipartUpload: { Parts: etags.map((ETag, index) => ({ ETag, PartNumber: index + 1 })) },
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
await s3.send(new AbortMultipartUploadCommand(target)).catch((abortError: unknown) => {
|
||||
console.error(
|
||||
`[s3Stitch] abort failed for ${destUri}; parts remain until a lifecycle rule removes them`,
|
||||
abortError,
|
||||
);
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export interface StitchResult {
|
||||
fileSize: number;
|
||||
parts: number;
|
||||
downloadedBytes: number;
|
||||
}
|
||||
|
||||
/** One faststart mp4 at `destUri` from the stitched mp4 chunks at `chunkUris`, in order. */
|
||||
export async function stitchMp4InS3(
|
||||
s3: S3Client,
|
||||
chunkUris: readonly string[],
|
||||
destUri: string,
|
||||
minPartBytes = MIN_PART_BYTES,
|
||||
): Promise<StitchResult> {
|
||||
const layouts = await Promise.all(
|
||||
chunkUris.map((uri) => readMp4Layout((start, end) => getRange(s3, uri, start, end))),
|
||||
);
|
||||
const segments: Segment[] = [
|
||||
{ bytes: buildStitchedHeader(layouts) },
|
||||
...layouts.map((layout, i) => ({
|
||||
uri: chunkUris[i]!,
|
||||
start: layout.payloadStart,
|
||||
end: layout.payloadEnd,
|
||||
})),
|
||||
];
|
||||
const parts = planParts(segments, minPartBytes);
|
||||
await uploadParts(s3, parts, destUri, "video/mp4");
|
||||
const uploaded = parts.flatMap((part) => ("upload" in part ? part.upload : []));
|
||||
return {
|
||||
fileSize: segments.reduce((total, s) => total + segmentSize(s), 0),
|
||||
parts: parts.length,
|
||||
downloadedBytes: uploaded.reduce((total, s) => total + ("uri" in s ? segmentSize(s) : 0), 0),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user