mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
* perf(engine): a remote source's tail decodes run many at a time, not one per half-core * perf(producer): the plan logs where its extract stage spends its time * perf(engine): a window that ends inside its source takes its frame count without a decode * test(engine): pin the tail-decode slot test to the windows that decode * style(engine): format the tail-probe window check * test(engine): deferred counts match extraction on a B-frame NTSC source; VFR is not deferred * style(engine): format the tail-probe tests * perf(aws-lambda): a site stores each video's ffprobe JSON, and the plan reads it instead of probing * test(aws-lambda): a site stores and hands out each video's probe * style(aws-lambda): format the probe-facts upload and tests * perf(engine): a site's stored probe also holds the final-frame timestamp the plan would probe for * test(engine): drop an unused import * test(aws-lambda): the stored probe holds the ffprobe JSON and the final frame * test(aws-lambda): the stored probe holds the ffprobe JSON and the final frame * perf(producer): a distributed chunk encodes while it captures * perf(producer): the audio task logs where its mix spends its time One line per audio stage: prepare, groups, mix, master, encode and the peak check, in ms. * perf(engine): locked warmup ticks stop sleeping once the page is ready * perf(engine): a window reaching a stored video's end takes its frame count from stored facts * fix(producer): the cloud audio task logs its phases * perf(engine): stored facts keep the last packet length so late-start windows skip the tail decode * fix(engine): a probe spawns without waiting when no stored facts are declared * chore(aws-lambda): shorten comments over the line limit * fix(engine): warmup keeps pacing after a failed tick; stored final frames check ffprobe major * test(engine): the recorded ffprobe major matches the build's own banner * test(engine): the version-banner probes go through a named binary like the other spawns
330 lines
12 KiB
TypeScript
330 lines
12 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
import { spawnSync } from "node:child_process";
|
|
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { createHash } from "node:crypto";
|
|
import type { S3Client } from "@aws-sdk/client-s3";
|
|
import { AUDIO_SIDECAR_MIN_BYTES } from "@hyperframes/producer/distributed";
|
|
import { audioSidecarUri, probeFactsUri } from "../contentStore.js";
|
|
import { parseSiteManifest } from "../siteManifest.js";
|
|
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
|
|
import { deploySite } from "./deploySite.js";
|
|
|
|
let projectDir: string;
|
|
let scratch: string;
|
|
|
|
beforeEach(() => {
|
|
projectDir = mkdtempSync(join(tmpdir(), "hf-deploy-site-test-"));
|
|
scratch = mkdtempSync(join(tmpdir(), "hf-deploy-site-scratch-"));
|
|
mkdirSync(join(projectDir, "assets"));
|
|
writeFileSync(join(projectDir, "index.html"), "<html><body>hi</body></html>");
|
|
writeFileSync(join(projectDir, "assets", "style.css"), "body { color: red; }");
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(projectDir, { recursive: true, force: true });
|
|
rmSync(scratch, { recursive: true, force: true });
|
|
});
|
|
|
|
function sha256(text: string): string {
|
|
return createHash("sha256").update(text).digest("hex");
|
|
}
|
|
|
|
function storeKey(text: string): string {
|
|
const digest = sha256(text);
|
|
return `cas/sha256/${digest.slice(0, 2)}/${digest}`;
|
|
}
|
|
|
|
function sitePaths(s3: FakeS3, siteId: string): string[] {
|
|
const manifest = s3.bodies.get(`test-bucket/sites/${siteId}/files.json`)?.toString() ?? "";
|
|
return parseSiteManifest(manifest).map((file) => file.path);
|
|
}
|
|
|
|
const HTML = "<html><body>hi</body></html>";
|
|
const CSS = "body { color: red; }";
|
|
|
|
describe("deploySite", () => {
|
|
it("uploads each file once under its content hash, then the site manifest", async () => {
|
|
const s3 = new FakeS3();
|
|
const result = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
});
|
|
|
|
expect(result.uploaded).toBe(true);
|
|
expect(result.siteId).toMatch(/^[0-9a-f]{16}$/);
|
|
expect(result.bucketName).toBe("test-bucket");
|
|
expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/files.json`);
|
|
expect(result.bytes).toBe(HTML.length + CSS.length);
|
|
const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);
|
|
expect(puts.slice(0, 2).sort()).toEqual([storeKey(HTML), storeKey(CSS)].sort());
|
|
expect(puts[2]).toBe(`sites/${result.siteId}/files.json`);
|
|
expect(s3.bodies.get(`test-bucket/${storeKey(CSS)}`)?.toString()).toBe(CSS);
|
|
expect(parseSiteManifest(s3.bodies.get(`test-bucket/${puts[2]}`)?.toString() ?? "")).toEqual([
|
|
{ path: "assets/style.css", sha256: sha256(CSS), sizeBytes: CSS.length },
|
|
{ path: "index.html", sha256: sha256(HTML), sizeBytes: HTML.length },
|
|
]);
|
|
});
|
|
|
|
it("skips a file another site already stored", async () => {
|
|
const s3 = new FakeS3();
|
|
const cssId = `test-bucket/${storeKey(CSS)}`;
|
|
s3.existing.add(cssId);
|
|
s3.bodies.set(cssId, Buffer.from(CSS));
|
|
s3.metadata.set(cssId, { sha256: sha256(CSS) });
|
|
|
|
await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) });
|
|
|
|
const puts = s3.ops.filter((op) => op.kind === "put").map((op) => op.key);
|
|
expect(puts).toHaveLength(2);
|
|
expect(puts).not.toContain(storeKey(CSS));
|
|
expect(puts).toContain(storeKey(HTML));
|
|
});
|
|
|
|
it("yields a stable siteId across re-runs of the same tree", async () => {
|
|
const s3a = new FakeS3();
|
|
const a = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3a),
|
|
});
|
|
const s3b = new FakeS3();
|
|
const b = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3b),
|
|
});
|
|
expect(a.siteId).toBe(b.siteId);
|
|
});
|
|
|
|
it("changes siteId when a file's content changes", async () => {
|
|
const s3 = new FakeS3();
|
|
const before = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
});
|
|
|
|
writeFileSync(join(projectDir, "index.html"), "<html><body>changed</body></html>");
|
|
const s3b = new FakeS3();
|
|
const after = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3b),
|
|
});
|
|
expect(after.siteId).not.toBe(before.siteId);
|
|
});
|
|
|
|
it("uploads nothing when the same tree is deployed again", async () => {
|
|
const s3 = new FakeS3();
|
|
const first = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
});
|
|
|
|
const second = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
});
|
|
|
|
expect(second.uploaded).toBe(false);
|
|
expect(second.siteId).toBe(first.siteId);
|
|
// Two files plus the manifest, each put once and HEAD-checked on both deploys.
|
|
expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(3);
|
|
expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(6);
|
|
});
|
|
|
|
it("honours a caller-supplied siteId", async () => {
|
|
const s3 = new FakeS3();
|
|
const result = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
siteId: "release-v1.2.3",
|
|
s3: asS3Client(s3),
|
|
});
|
|
expect(result.siteId).toBe("release-v1.2.3");
|
|
expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/files.json");
|
|
});
|
|
|
|
it("propagates non-404 S3 errors", async () => {
|
|
const errS3 = {
|
|
async send(_cmd: unknown): Promise<unknown> {
|
|
const err = new Error("Access Denied") as Error & {
|
|
$metadata: { httpStatusCode: number };
|
|
};
|
|
err.$metadata = { httpStatusCode: 403 };
|
|
throw err;
|
|
},
|
|
};
|
|
await expect(
|
|
deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: errS3 as unknown as S3Client,
|
|
}),
|
|
).rejects.toThrow(/Access Denied/);
|
|
});
|
|
|
|
it("ignores SKIP_TOP_LEVEL dirs when hashing", async () => {
|
|
const s3 = new FakeS3();
|
|
const before = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
});
|
|
|
|
mkdirSync(join(projectDir, "node_modules"));
|
|
writeFileSync(join(projectDir, "node_modules", "junk.bin"), "x".repeat(100));
|
|
const s3b = new FakeS3();
|
|
const after = await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3b),
|
|
});
|
|
|
|
// node_modules contents shouldn't move the hash.
|
|
expect(after.siteId).toBe(before.siteId);
|
|
});
|
|
|
|
it("ships a symlinked project and its symlinked files and folders as content", async () => {
|
|
mkdirSync(join(scratch, "media"));
|
|
writeFileSync(join(scratch, "media", "clip.txt"), "clip");
|
|
writeFileSync(join(scratch, "logo.txt"), "logo");
|
|
symlinkSync(join(scratch, "media"), join(projectDir, "media"));
|
|
symlinkSync(join(scratch, "logo.txt"), join(projectDir, "logo.txt"));
|
|
symlinkSync(projectDir, join(scratch, "fx"));
|
|
const s3 = new FakeS3();
|
|
const result = await deploySite({
|
|
projectDir: join(scratch, "fx"),
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
});
|
|
expect(sitePaths(s3, result.siteId)).toEqual([
|
|
"assets/style.css",
|
|
"index.html",
|
|
"logo.txt",
|
|
"media/clip.txt",
|
|
]);
|
|
});
|
|
});
|
|
|
|
const FFMPEG = process.env.HYPERFRAMES_FFMPEG_PATH?.trim() || "ffmpeg";
|
|
const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;
|
|
|
|
describe.skipIf(!HAS_FFMPEG)("deploySite audio sidecars", { timeout: 60_000 }, () => {
|
|
function addLargeVideo(): void {
|
|
const made = spawnSync(FFMPEG, [
|
|
...[
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=gray:size=640x360:rate=25:duration=30,noise=alls=100:allf=t+u",
|
|
],
|
|
...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=30"],
|
|
...["-c:v", "mpeg4", "-b:v", "16M", "-c:a", "aac", join(projectDir, "assets", "take.mp4")],
|
|
]);
|
|
expect(made.status).toBe(0);
|
|
}
|
|
|
|
it("stores a large video's audio beside it, and leaves the site's file list as it was", async () => {
|
|
addLargeVideo();
|
|
const s3 = new FakeS3();
|
|
const site = await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) });
|
|
|
|
const take = parseSiteManifest(
|
|
s3.bodies.get(`test-bucket/sites/${site.siteId}/files.json`)?.toString() ?? "",
|
|
).find((file) => file.path === "assets/take.mp4");
|
|
expect(take?.sizeBytes).toBeGreaterThan(AUDIO_SIDECAR_MIN_BYTES);
|
|
const sidecar = audioSidecarUri("test-bucket", take?.sha256 ?? "").replace("s3://", "");
|
|
expect(s3.existing.has(sidecar)).toBe(true);
|
|
expect(s3.bodies.get(sidecar)?.length).toBeLessThan((take?.sizeBytes ?? 0) / 4);
|
|
expect(sitePaths(s3, site.siteId)).toEqual([
|
|
"assets/style.css",
|
|
"assets/take.mp4",
|
|
"index.html",
|
|
]);
|
|
});
|
|
|
|
it("deploys the same site with no sidecar when there is no ffmpeg, and says so", async () => {
|
|
addLargeVideo();
|
|
const notices: string[] = [];
|
|
const before = process.env.HYPERFRAMES_FFMPEG_PATH;
|
|
process.env.HYPERFRAMES_FFMPEG_PATH = join(scratch, "no-such-ffmpeg");
|
|
const s3 = new FakeS3();
|
|
try {
|
|
await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
onNotice: (message) => notices.push(message),
|
|
});
|
|
} finally {
|
|
if (before === undefined) delete process.env.HYPERFRAMES_FFMPEG_PATH;
|
|
else process.env.HYPERFRAMES_FFMPEG_PATH = before;
|
|
}
|
|
|
|
expect([...s3.existing].filter((id) => id.endsWith(".audio.m4a"))).toEqual([]);
|
|
expect(notices).toHaveLength(1);
|
|
expect(notices[0]).toContain("ffmpeg");
|
|
});
|
|
});
|
|
|
|
describe.skipIf(!HAS_FFMPEG)("deploySite probe facts", { timeout: 60_000 }, () => {
|
|
function addVideo(): void {
|
|
const made = spawnSync(FFMPEG, [
|
|
...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=64x64:rate=25:duration=2"],
|
|
...["-c:v", "mpeg4", join(projectDir, "assets", "take.mp4")],
|
|
]);
|
|
expect(made.status).toBe(0);
|
|
}
|
|
|
|
it("stores each video's ffprobe JSON beside it, and leaves the site's file list as it was", async () => {
|
|
addVideo();
|
|
const s3 = new FakeS3();
|
|
const site = await deploySite({ projectDir, bucketName: "test-bucket", s3: asS3Client(s3) });
|
|
|
|
const take = parseSiteManifest(
|
|
s3.bodies.get(`test-bucket/sites/${site.siteId}/files.json`)?.toString() ?? "",
|
|
).find((file) => file.path === "assets/take.mp4");
|
|
const facts = probeFactsUri("test-bucket", take?.sha256 ?? "").replace("s3://", "");
|
|
const stored = JSON.parse(s3.bodies.get(facts)?.toString() ?? "{}");
|
|
expect(stored.ffprobe.streams[0].codec_type).toBe("video");
|
|
expect(stored.finalFrame.seconds).toBeGreaterThan(1.5);
|
|
expect(stored.finalFrame.lastPacketSeconds).toBeGreaterThan(0);
|
|
expect(sitePaths(s3, site.siteId)).toEqual([
|
|
"assets/style.css",
|
|
"assets/take.mp4",
|
|
"index.html",
|
|
]);
|
|
});
|
|
|
|
it("deploys the same site with no probe stored when there is no ffprobe, and says so", async () => {
|
|
addVideo();
|
|
const notices: string[] = [];
|
|
const before = process.env.HYPERFRAMES_FFPROBE_PATH;
|
|
process.env.HYPERFRAMES_FFPROBE_PATH = join(scratch, "no-such-ffprobe");
|
|
const s3 = new FakeS3();
|
|
try {
|
|
await deploySite({
|
|
projectDir,
|
|
bucketName: "test-bucket",
|
|
s3: asS3Client(s3),
|
|
onNotice: (message) => notices.push(message),
|
|
});
|
|
} finally {
|
|
if (before === undefined) delete process.env.HYPERFRAMES_FFPROBE_PATH;
|
|
else process.env.HYPERFRAMES_FFPROBE_PATH = before;
|
|
}
|
|
|
|
expect([...s3.existing].filter((id) => id.endsWith(".probe.v2.json"))).toEqual([]);
|
|
expect(notices.some((message) => message.includes("ffprobe"))).toBe(true);
|
|
});
|
|
});
|