mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
* perf(engine): a remote source's tail decodes run many at a time, not one per half-core * perf(producer): the plan logs where its extract stage spends its time * perf(engine): a window that ends inside its source takes its frame count without a decode * test(engine): pin the tail-decode slot test to the windows that decode * style(engine): format the tail-probe window check * test(engine): deferred counts match extraction on a B-frame NTSC source; VFR is not deferred * style(engine): format the tail-probe tests * perf(aws-lambda): a site stores each video's ffprobe JSON, and the plan reads it instead of probing * test(aws-lambda): a site stores and hands out each video's probe * style(aws-lambda): format the probe-facts upload and tests * perf(engine): a site's stored probe also holds the final-frame timestamp the plan would probe for * test(engine): drop an unused import * test(aws-lambda): the stored probe holds the ffprobe JSON and the final frame * test(aws-lambda): the stored probe holds the ffprobe JSON and the final frame * perf(producer): a distributed chunk encodes while it captures * perf(producer): the audio task logs where its mix spends its time One line per audio stage: prepare, groups, mix, master, encode and the peak check, in ms. * perf(engine): locked warmup ticks stop sleeping once the page is ready * perf(engine): a window reaching a stored video's end takes its frame count from stored facts * fix(producer): the cloud audio task logs its phases * perf(engine): stored facts keep the last packet length so late-start windows skip the tail decode * fix(engine): a probe spawns without waiting when no stored facts are declared * chore(aws-lambda): shorten comments over the line limit * fix(engine): warmup keeps pacing after a failed tick; stored final frames check ffprobe major * test(engine): the recorded ffprobe major matches the build's own banner * test(engine): the version-banner probes go through a named binary like the other spawns
120 lines
4.6 KiB
TypeScript
120 lines
4.6 KiB
TypeScript
// A deployed site is `sites/<siteId>/files.json`: the project's files by path, each stored once
|
|
// in the shared content store under its SHA-256 (see `contentStoreUri`).
|
|
|
|
import { readFileSync } from "node:fs";
|
|
import { extname, join } from "node:path";
|
|
import type { S3Client } from "@aws-sdk/client-s3";
|
|
import type { RemoteProjectFile } from "@hyperframes/producer/distributed";
|
|
import { audioSidecarUri, contentStoreUri, probeFactsUri } from "./contentStore.js";
|
|
import {
|
|
downloadS3ObjectToFile,
|
|
downloadS3ObjectToFileVerified,
|
|
formatS3Uri,
|
|
mapConcurrent,
|
|
parseS3Uri,
|
|
presignS3GetUrl,
|
|
s3ObjectExists,
|
|
} from "./s3Transport.js";
|
|
|
|
const SITE_MANIFEST_NAME = "files.json";
|
|
/** Stacks that read site manifests list this as a terminal plan error; the SDK checks for it. */
|
|
export const SITE_MANIFEST_INVALID = "SITE_MANIFEST_INVALID";
|
|
const VIDEO_EXTENSIONS_PLANNED_IN_PLACE = new Set([".mp4", ".mov", ".m4v", ".webm", ".mkv"]);
|
|
const REMOTE_URL_TTL_SECONDS = 3600;
|
|
|
|
export interface SiteFile {
|
|
/** `/`-separated, relative to the project root. */
|
|
readonly path: string;
|
|
readonly sha256: string;
|
|
readonly sizeBytes: number;
|
|
}
|
|
|
|
export function siteManifestUri(bucket: string, siteId: string): string {
|
|
return formatS3Uri({ bucket, key: `sites/${siteId}/${SITE_MANIFEST_NAME}` });
|
|
}
|
|
|
|
/** Sites uploaded before site manifests are one tarball; the plan step still reads them. */
|
|
export function siteTarballUri(bucket: string, siteId: string): string {
|
|
return formatS3Uri({ bucket, key: `sites/${siteId}/project.tar.gz` });
|
|
}
|
|
|
|
export function isSiteManifestUri(uri: string): boolean {
|
|
return uri.endsWith(`/${SITE_MANIFEST_NAME}`);
|
|
}
|
|
|
|
export function serializeSiteManifest(files: readonly SiteFile[]): string {
|
|
return JSON.stringify({ files });
|
|
}
|
|
|
|
export function parseSiteManifest(text: string): SiteFile[] {
|
|
const value = JSON.parse(text) as { files?: unknown };
|
|
if (!Array.isArray(value.files)) throw siteManifestError("files must be an array");
|
|
const seen = new Set<string>();
|
|
return value.files.map((entry: Partial<SiteFile>, index) => {
|
|
const { path, sha256, sizeBytes } = entry ?? {};
|
|
if (typeof path !== "string" || !isSafeRelativePath(path) || seen.has(path)) {
|
|
throw siteManifestError(`files[${index}].path is not a unique relative path`);
|
|
}
|
|
if (typeof sha256 !== "string" || !/^[a-f0-9]{64}$/.test(sha256)) {
|
|
throw siteManifestError(`files[${index}].sha256 is not a lowercase SHA-256 digest`);
|
|
}
|
|
if (!Number.isSafeInteger(sizeBytes) || (sizeBytes as number) < 0) {
|
|
throw siteManifestError(`files[${index}].sizeBytes is not a byte count`);
|
|
}
|
|
seen.add(path);
|
|
return { path, sha256, sizeBytes: sizeBytes as number };
|
|
});
|
|
}
|
|
|
|
/** Rebuild the project under `projectDir`, SHA-256 verified; videos can stay remote, as URLs. */
|
|
export async function downloadSiteFiles(
|
|
s3: S3Client,
|
|
manifestUri: string,
|
|
projectDir: string,
|
|
leaveVideosRemote: boolean,
|
|
): Promise<Record<string, RemoteProjectFile>> {
|
|
const manifestPath = `${projectDir}.${SITE_MANIFEST_NAME}`;
|
|
await downloadS3ObjectToFile(s3, manifestUri, manifestPath);
|
|
const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));
|
|
const { bucket } = parseS3Uri(manifestUri);
|
|
const remoteFiles: Record<string, RemoteProjectFile> = {};
|
|
await mapConcurrent(files, 16, async ({ path, sha256, sizeBytes }) => {
|
|
const uri = contentStoreUri(bucket, sha256);
|
|
if (leaveVideosRemote && VIDEO_EXTENSIONS_PLANNED_IN_PLACE.has(extname(path).toLowerCase())) {
|
|
const sidecar = audioSidecarUri(bucket, sha256);
|
|
const facts = probeFactsUri(bucket, sha256);
|
|
const [hasSidecar, hasFacts] = await Promise.all([
|
|
s3ObjectExists(s3, sidecar),
|
|
s3ObjectExists(s3, facts),
|
|
]);
|
|
remoteFiles[path] = {
|
|
url: await presignS3GetUrl(s3, uri, REMOTE_URL_TTL_SECONDS),
|
|
...(hasSidecar
|
|
? { audioSidecarUrl: await presignS3GetUrl(s3, sidecar, REMOTE_URL_TTL_SECONDS) }
|
|
: {}),
|
|
...(hasFacts
|
|
? { probeFactsUrl: await presignS3GetUrl(s3, facts, REMOTE_URL_TTL_SECONDS) }
|
|
: {}),
|
|
sha256,
|
|
sizeBytes,
|
|
};
|
|
return;
|
|
}
|
|
await downloadS3ObjectToFileVerified(s3, uri, join(projectDir, ...path.split("/")), sha256);
|
|
});
|
|
return remoteFiles;
|
|
}
|
|
|
|
function isSafeRelativePath(path: string): boolean {
|
|
return (
|
|
path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..") &&
|
|
!path.includes("\\")
|
|
);
|
|
}
|
|
|
|
function siteManifestError(detail: string): Error {
|
|
const error = new Error(`[siteManifest] ${SITE_MANIFEST_INVALID}: ${detail}`);
|
|
error.name = SITE_MANIFEST_INVALID;
|
|
return error;
|
|
}
|