Files
hyperframes/packages/aws-lambda/src/siteManifest.ts
T
Miguel Ángel da4386f0d0 perf(aws-lambda): cloud renders plan without reading video bytes and start chunks sooner (#6194)
* perf(engine): a remote source's tail decodes run many at a time, not one per half-core

* perf(producer): the plan logs where its extract stage spends its time

* perf(engine): a window that ends inside its source takes its frame count without a decode

* test(engine): pin the tail-decode slot test to the windows that decode

* style(engine): format the tail-probe window check

* test(engine): deferred counts match extraction on a B-frame NTSC source; VFR is not deferred

* style(engine): format the tail-probe tests

* perf(aws-lambda): a site stores each video's ffprobe JSON, and the plan reads it instead of probing

* test(aws-lambda): a site stores and hands out each video's probe

* style(aws-lambda): format the probe-facts upload and tests

* perf(engine): a site's stored probe also holds the final-frame timestamp the plan would probe for

* test(engine): drop an unused import

* test(aws-lambda): the stored probe holds the ffprobe JSON and the final frame

* test(aws-lambda): the stored probe holds the ffprobe JSON and the final frame

* perf(producer): a distributed chunk encodes while it captures

* perf(producer): the audio task logs where its mix spends its time

One line per audio stage: prepare, groups, mix, master, encode and the peak check, in ms.

* perf(engine): locked warmup ticks stop sleeping once the page is ready

* perf(engine): a window reaching a stored video's end takes its frame count from stored facts

* fix(producer): the cloud audio task logs its phases

* perf(engine): stored facts keep the last packet length so late-start windows skip the tail decode

* fix(engine): a probe spawns without waiting when no stored facts are declared

* chore(aws-lambda): shorten comments over the line limit

* fix(engine): warmup keeps pacing after a failed tick; stored final frames check ffprobe major

* test(engine): the recorded ffprobe major matches the build's own banner

* test(engine): the version-banner probes go through a named binary like the other spawns
2026-10-24 14:30:16 +00:00

120 lines
4.6 KiB
TypeScript

// A deployed site is `sites/<siteId>/files.json`: the project's files by path, each stored once
// in the shared content store under its SHA-256 (see `contentStoreUri`).
import { readFileSync } from "node:fs";
import { extname, join } from "node:path";
import type { S3Client } from "@aws-sdk/client-s3";
import type { RemoteProjectFile } from "@hyperframes/producer/distributed";
import { audioSidecarUri, contentStoreUri, probeFactsUri } from "./contentStore.js";
import {
downloadS3ObjectToFile,
downloadS3ObjectToFileVerified,
formatS3Uri,
mapConcurrent,
parseS3Uri,
presignS3GetUrl,
s3ObjectExists,
} from "./s3Transport.js";
const SITE_MANIFEST_NAME = "files.json";
/** Stacks that read site manifests list this as a terminal plan error; the SDK checks for it. */
export const SITE_MANIFEST_INVALID = "SITE_MANIFEST_INVALID";
const VIDEO_EXTENSIONS_PLANNED_IN_PLACE = new Set([".mp4", ".mov", ".m4v", ".webm", ".mkv"]);
const REMOTE_URL_TTL_SECONDS = 3600;
export interface SiteFile {
/** `/`-separated, relative to the project root. */
readonly path: string;
readonly sha256: string;
readonly sizeBytes: number;
}
export function siteManifestUri(bucket: string, siteId: string): string {
return formatS3Uri({ bucket, key: `sites/${siteId}/${SITE_MANIFEST_NAME}` });
}
/** Sites uploaded before site manifests are one tarball; the plan step still reads them. */
export function siteTarballUri(bucket: string, siteId: string): string {
return formatS3Uri({ bucket, key: `sites/${siteId}/project.tar.gz` });
}
export function isSiteManifestUri(uri: string): boolean {
return uri.endsWith(`/${SITE_MANIFEST_NAME}`);
}
export function serializeSiteManifest(files: readonly SiteFile[]): string {
return JSON.stringify({ files });
}
export function parseSiteManifest(text: string): SiteFile[] {
const value = JSON.parse(text) as { files?: unknown };
if (!Array.isArray(value.files)) throw siteManifestError("files must be an array");
const seen = new Set<string>();
return value.files.map((entry: Partial<SiteFile>, index) => {
const { path, sha256, sizeBytes } = entry ?? {};
if (typeof path !== "string" || !isSafeRelativePath(path) || seen.has(path)) {
throw siteManifestError(`files[${index}].path is not a unique relative path`);
}
if (typeof sha256 !== "string" || !/^[a-f0-9]{64}$/.test(sha256)) {
throw siteManifestError(`files[${index}].sha256 is not a lowercase SHA-256 digest`);
}
if (!Number.isSafeInteger(sizeBytes) || (sizeBytes as number) < 0) {
throw siteManifestError(`files[${index}].sizeBytes is not a byte count`);
}
seen.add(path);
return { path, sha256, sizeBytes: sizeBytes as number };
});
}
/** Rebuild the project under `projectDir`, SHA-256 verified; videos can stay remote, as URLs. */
export async function downloadSiteFiles(
s3: S3Client,
manifestUri: string,
projectDir: string,
leaveVideosRemote: boolean,
): Promise<Record<string, RemoteProjectFile>> {
const manifestPath = `${projectDir}.${SITE_MANIFEST_NAME}`;
await downloadS3ObjectToFile(s3, manifestUri, manifestPath);
const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));
const { bucket } = parseS3Uri(manifestUri);
const remoteFiles: Record<string, RemoteProjectFile> = {};
await mapConcurrent(files, 16, async ({ path, sha256, sizeBytes }) => {
const uri = contentStoreUri(bucket, sha256);
if (leaveVideosRemote && VIDEO_EXTENSIONS_PLANNED_IN_PLACE.has(extname(path).toLowerCase())) {
const sidecar = audioSidecarUri(bucket, sha256);
const facts = probeFactsUri(bucket, sha256);
const [hasSidecar, hasFacts] = await Promise.all([
s3ObjectExists(s3, sidecar),
s3ObjectExists(s3, facts),
]);
remoteFiles[path] = {
url: await presignS3GetUrl(s3, uri, REMOTE_URL_TTL_SECONDS),
...(hasSidecar
? { audioSidecarUrl: await presignS3GetUrl(s3, sidecar, REMOTE_URL_TTL_SECONDS) }
: {}),
...(hasFacts
? { probeFactsUrl: await presignS3GetUrl(s3, facts, REMOTE_URL_TTL_SECONDS) }
: {}),
sha256,
sizeBytes,
};
return;
}
await downloadS3ObjectToFileVerified(s3, uri, join(projectDir, ...path.split("/")), sha256);
});
return remoteFiles;
}
function isSafeRelativePath(path: string): boolean {
return (
path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..") &&
!path.includes("\\")
);
}
function siteManifestError(detail: string): Error {
const error = new Error(`[siteManifest] ${SITE_MANIFEST_INVALID}: ${detail}`);
error.name = SITE_MANIFEST_INVALID;
return error;
}