// fallow-ignore-file unused-class-member code-duplication complexity
/**
* Video Frame Extractor Service
*
* Pre-extracts video frames using FFmpeg for frame-accurate rendering.
* Videos are replaced with
elements during capture.
*/
import { spawn } from "child_process";
import { copyFileSync, existsSync, linkSync, mkdirSync, readdirSync, rmSync } from "fs";
import { isAbsolute, join, posix, resolve, sep } from "path";
import { parseHTML } from "linkedom";
import { decodeUrlPathVariants, MEDIA_DURATION_CLAMP_EPSILON_SECONDS } from "@hyperframes/core";
import { trackChildProcess } from "../utils/processTracker.js";
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
import {
analyzeCompositionHdr,
isHdrColorSpace as isHdrColorSpaceUtil,
type HdrTransfer,
} from "../utils/hdr.js";
import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
import {
FRAME_FILENAME_PREFIX,
gcExtractionCache,
gcSweepDue,
lookupCacheEntry,
partialCacheEntryDir,
publishCacheEntry,
readKeyStat,
rehydrateCacheEntry,
touchCacheEntry,
type CacheEntry,
type CacheFrameFormat,
} from "./extractionCache.js";
export interface VideoElement {
id: string;
src: string;
start: number;
end: number;
mediaStart: number;
loop: boolean;
hasAudio: boolean;
}
export interface ExtractedFrames {
videoId: string;
srcPath: string;
outputDir: string;
framePattern: string;
fps: number;
totalFrames: number;
metadata: VideoMetadata;
framePaths: Map;
/**
* True when the extractor owns `outputDir` and cleanup should rm it when
* the render ends. Cache hits set this to false so the shared entry isn't
* deleted by a single render's cleanup — the cache dir is owned by the
* caller's gc policy, not any one render.
*/
ownedByLookup?: boolean;
}
/**
* The single source of truth for the source-video frame-extraction allow-list.
* The CLI flag parser, the producer HTTP server, and the distributed-config
* validator all validate against this same set via {@link isVideoFrameFormat}
* so the boundaries can't drift when a new format is added.
*/
export const VIDEO_FRAME_FORMATS = ["auto", "jpg", "png"] as const;
export type VideoFrameFormat = (typeof VIDEO_FRAME_FORMATS)[number];
/** Runtime guard for {@link VideoFrameFormat} over an untrusted value. */
export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat {
return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value);
}
export interface ExtractionOptions {
fps: number;
outputDir: string;
quality?: number;
format?: VideoFrameFormat;
sdrToHdrTransfer?: HdrTransfer;
}
const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
function sdrToHdrTransformKey(transfer: HdrTransfer): string {
return `sdr2hdr-${transfer}`;
}
/**
* Per-phase timings and counters emitted by `extractAllVideoFrames`.
*
* Used by the producer to surface `perfSummary.videoExtractBreakdown` — without
* this breakdown, a single `videoExtractMs` stage timing hides where cost lives
* (HDR preflight, VFR classification, per-video ffmpeg extract) when tuning renders.
*
* Field semantics:
* - *Ms fields are wall-clock durations inside each phase.
* - *Count fields report how many sources triggered that phase.
* - extractMs wraps the parallel `extractVideoFramesRange` calls; it
* reflects max-across-parallel-workers, not sum.
* - hdrPreflightMs includes its probe-time sibling (hdrProbeMs); the
* probe-only field is a finer decomposition, not a separate carve-out.
* - vfrPreflightCount reports sources classified as VFR and routed through
* the one-pass `-fps_mode cfr -r` extraction path. DEFINITION CHANGE:
* before the one-pass refactor, vfrPreflightMs timed a per-source
* VFR-to-CFR re-encode and could reach seconds; it now times only the
* (promise-cached) classification probe and is expected to be ~0.
* Dashboards alerting on vfrPreflightMs thresholds should key on
* vfrPreflightCount or extractMs instead.
*/
export interface ExtractionPhaseBreakdown {
resolveMs: number;
/** Publishes that could not land atomically — the render still succeeded
* from the partial dir, but future renders re-extract. A rising rate is
* the first signal that warm renders are silently going cold. */
cachePublishFailures: number;
/** Entries evicted by the post-extraction LRU sweep. */
cacheGcEvictions: number;
/** Bytes reclaimed by the LRU sweep. */
cacheGcBytesFreed: number;
/** Aged .partial-* dirs (crashed writers) removed by the sweep. */
cacheAgedPartialsCleared: number;
hdrProbeMs: number;
hdrPreflightMs: number;
hdrPreflightCount: number;
vfrProbeMs: number;
vfrPreflightMs: number;
vfrPreflightCount: number;
extractMs: number;
cacheHits: number;
cacheMisses: number;
}
export interface ExtractionResult {
success: boolean;
extracted: ExtractedFrames[];
errors: Array<{ videoId: string; error: string }>;
totalFramesExtracted: number;
durationMs: number;
phaseBreakdown: ExtractionPhaseBreakdown;
}
export function parseVideoElements(html: string): VideoElement[] {
const videos: VideoElement[] = [];
const { document } = parseHTML(unwrapTemplate(html));
const startCache = new Map();
const visiting = new Set();
const videoEls = document.querySelectorAll("video[src]");
let autoIdCounter = 0;
for (const el of videoEls) {
const src = el.getAttribute("src");
if (!src) continue;
// Generate a stable ID for videos without one — the producer needs IDs
// to track extracted frames and composite them during encoding.
const id = el.getAttribute("id") || `hf-video-${autoIdCounter++}`;
if (!el.getAttribute("id")) {
el.setAttribute("id", id);
}
const startAttr = el.getAttribute("data-start");
const endAttr = el.getAttribute("data-end");
const durationAttr = el.getAttribute("data-duration");
const mediaStartAttr = el.getAttribute("data-media-start");
const hasAudioAttr = el.getAttribute("data-has-audio");
// Resolve data-start, including relative references ("intro", "intro + 2")
// to another clip's end — the browser runtime resolves these but a raw
// parseFloat here would yield NaN, placing the clip at NaN so it composites
// blank in the final render. `startAttr` may be a plain number or a
// reference; the resolver handles both.
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
// Derive end from data-end → data-start+data-duration → Infinity (natural duration).
// The caller (htmlCompiler) clamps Infinity to the composition's absoluteEnd.
let end = 0;
if (endAttr) {
end = parseFloat(endAttr);
} else if (durationAttr) {
end = start + parseFloat(durationAttr);
} else {
end = Infinity; // no explicit bounds — play for the full natural video duration
}
videos.push({
id,
src,
start,
end,
mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
loop: el.hasAttribute("loop"),
hasAudio: hasAudioAttr === "true",
});
}
return videos;
}
export interface ImageElement {
id: string;
src: string;
start: number;
end: number;
}
export function parseImageElements(html: string): ImageElement[] {
const images: ImageElement[] = [];
const { document } = parseHTML(unwrapTemplate(html));
const startCache = new Map();
const visiting = new Set();
const imgEls = document.querySelectorAll("img[src]");
let autoIdCounter = 0;
for (const el of imgEls) {
const src = el.getAttribute("src");
if (!src) continue;
const id = el.getAttribute("id") || `hf-img-${autoIdCounter++}`;
if (!el.getAttribute("id")) {
el.setAttribute("id", id);
}
const startAttr = el.getAttribute("data-start");
const endAttr = el.getAttribute("data-end");
const durationAttr = el.getAttribute("data-duration");
// Resolve relative data-start references (see parseVideoElements) so a
// referenced image start doesn't become NaN and drop the image from the render.
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
let end = 0;
if (endAttr) {
end = parseFloat(endAttr);
} else if (durationAttr) {
end = start + parseFloat(durationAttr);
} else {
end = Infinity;
}
images.push({ id, src, start, end });
}
return images;
}
export async function extractVideoFramesRange(
videoPath: string,
videoId: string,
startTime: number,
duration: number,
options: ExtractionOptions,
signal?: AbortSignal,
config?: Partial>,
/**
* Override the output directory for this extraction. When provided, frames
* are written directly into `outputDirOverride` (no per-videoId subdir).
* Used by the cache layer to materialize frames straight into the keyed
* cache entry directory.
*/
outputDirOverride?: string,
): Promise {
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
const { fps, outputDir, quality = 95 } = options;
const videoOutputDir = outputDirOverride ?? join(outputDir, videoId);
if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });
const metadata = await extractMediaMetadata(videoPath);
const format = resolveFrameFormat(metadata, options.format);
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
const outputPattern = join(videoOutputDir, framePattern);
// When extracting from HDR source, tone-map to SDR in FFmpeg rather than
// letting Chrome's uncontrollable tone-mapper handle it (which washes out).
// macOS: VideoToolbox hardware decoder does HDR→SDR natively on Apple Silicon.
// Linux: zscale filter (when available) or colorspace filter as fallback.
const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
const isMacOS = process.platform === "darwin";
const args: string[] = [];
if (isHdr && isMacOS) {
args.push("-hwaccel", "videotoolbox");
}
// Always force the alpha-aware decoder on codecs that can carry alpha. The
// alternative — gating on `metadata.hasAlpha` — relies on tag detection that
// has at least three known failure modes: case-sensitivity across ffmpeg
// versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
// and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
// silently strips alpha during decode and the bug doesn't surface until
// the rendered video is missing layers. Codec-based default has no such
// ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
// normally when it isn't.
if (codecMayHaveAlpha(metadata.videoCodec)) {
args.push("-c:v", decoderForCodec(metadata.videoCodec));
}
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
const vfFilters: string[] = [];
if (isHdr && isMacOS) {
// VideoToolbox tone-maps during decode; force output to bt709 SDR format
vfFilters.push("format=nv12");
}
if (!metadata.isVFR) {
vfFilters.push(`fps=${fps}`);
}
if (options.sdrToHdrTransfer) {
// Ordering intent: fps sampling runs BEFORE the colorspace remap so only
// kept frames are converted. The remap is pointwise per-frame, so the
// output is identical either way for the SDR (BT.709, 8-bit) inputs this
// flag is set for. If format=nv12 (macOS HDR-source decode) ever combines
// with this flag, revisit: nv12 subsampling before a BT.2020 remap is an
// untested interaction (today the flags are mutually exclusive — the
// remap only applies to SDR sources, nv12 only to HDR sources).
vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
}
if (vfFilters.length > 0) args.push("-vf", vfFilters.join(","));
if (metadata.isVFR) args.push("-fps_mode", "cfr", "-r", String(fps));
args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
// Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.
if (format === "png") args.push("-compression_level", "1");
args.push("-y", outputPattern);
return new Promise((resolve, reject) => {
const ffmpeg = spawn(getFfmpegBinary(), args);
trackChildProcess(ffmpeg);
let stderr = "";
const onAbort = () => {
ffmpeg.kill("SIGTERM");
};
if (signal) {
if (signal.aborted) {
ffmpeg.kill("SIGTERM");
} else {
signal.addEventListener("abort", onAbort, { once: true });
}
}
const timer = setTimeout(() => {
ffmpeg.kill("SIGTERM");
}, ffmpegProcessTimeout);
ffmpeg.stderr.on("data", (data) => {
stderr += data.toString();
});
ffmpeg.on("close", (code) => {
clearTimeout(timer);
if (signal) signal.removeEventListener("abort", onAbort);
if (signal?.aborted) {
reject(new Error("Video frame extraction cancelled"));
return;
}
if (code !== 0) {
// With the SDR-to-HDR remap folded into this pass, a filter failure
// (e.g. an ffmpeg built without the colorspace filter) would otherwise
// surface as a generic extract error and the operator has to grep the
// filter chain to learn it was the HDR conversion. Attribute it.
const hdrPrefix = options.sdrToHdrTransfer
? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): `
: "";
reject(new Error(`${hdrPrefix}FFmpeg exited with code ${code}: ${stderr.slice(-500)}`));
return;
}
const framePaths = new Map();
const files = readdirSync(videoOutputDir)
.filter((f) => f.startsWith(FRAME_FILENAME_PREFIX) && f.endsWith(`.${format}`))
.sort();
files.forEach((file, index) => {
framePaths.set(index, join(videoOutputDir, file));
});
resolve({
videoId,
srcPath: videoPath,
outputDir: videoOutputDir,
framePattern,
fps,
totalFrames: framePaths.size,
metadata,
framePaths,
});
});
ffmpeg.on("error", (err) => {
clearTimeout(timer);
if (signal) signal.removeEventListener("abort", onAbort);
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
reject(new Error("[FFmpeg] ffmpeg not found"));
} else {
reject(err);
}
});
});
}
/**
* Resolve the used-segment duration for a video, falling back to the source's
* natural duration when the caller hasn't specified bounds (end=Infinity) or
* the bounds are nonsensical (end<=start).
*/
function resolveSegmentDuration(
requested: number,
mediaStart: number,
metadata: VideoMetadata,
): number {
if (Number.isFinite(requested) && requested > 0) return requested;
const sourceRemaining = metadata.durationSeconds - mediaStart;
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
}
/**
* Codecs whose bitstream is allowed to carry an alpha channel. Default the
* extraction path to PNG output for these regardless of `metadata.hasAlpha`
* so a missed sidecar tag doesn't silently strip transparency. Opaque content
* encoded in one of these codecs pays a small file-size cost on the cached
* frames but stays correct on the rare case where alpha IS present and the
* tag was missed.
*/
const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
export function codecMayHaveAlpha(codec: string | undefined): boolean {
return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
}
export function decoderForCodec(codec: string | undefined): string {
const c = (codec ?? "").toLowerCase();
if (c === "vp9") return "libvpx-vp9";
if (c === "vp8") return "libvpx";
return c;
}
export function resolveFrameFormat(
metadata: VideoMetadata,
requested?: VideoFrameFormat,
): CacheFrameFormat {
if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
if (requested === "png" || requested === "jpg") return requested;
return "jpg";
}
type PreparedExtraction = {
video: VideoElement;
videoPath: string;
index: number;
metadata: VideoMetadata;
videoDuration: number;
format: CacheFrameFormat;
sdrToHdrTransfer?: HdrTransfer;
dedupeKey: string;
};
type CacheMissTarget = {
entry: CacheEntry;
srcPath: string;
};
type UniqueExtractionMiss = {
work: PreparedExtraction;
cacheTarget?: CacheMissTarget;
};
type SupersetMemberPlan = {
miss: UniqueExtractionMiss;
offsetFrames: number;
};
type SupersetGroupPlan = {
groupId: string;
baseStart: number;
unionDuration: number;
members: SupersetMemberPlan[];
};
function extractedFrameFileNames(outputDir: string, format: CacheFrameFormat): string[] {
const suffix = `.${format}`;
return readdirSync(outputDir)
.filter((file) => file.startsWith(FRAME_FILENAME_PREFIX) && file.endsWith(suffix))
.sort();
}
function extractedFramesFromDirectory(
work: PreparedExtraction,
outputDir: string,
srcPath: string,
fps: number,
): ExtractedFrames {
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${work.format}`;
const framePaths = new Map();
extractedFrameFileNames(outputDir, work.format).forEach((file, index) => {
framePaths.set(index, join(outputDir, file));
});
return {
videoId: work.video.id,
srcPath,
outputDir,
framePattern,
fps,
totalFrames: framePaths.size,
metadata: work.metadata,
framePaths,
};
}
function frameFileName(frameNumber: number, format: CacheFrameFormat): string {
return `${FRAME_FILENAME_PREFIX}${String(frameNumber).padStart(5, "0")}.${format}`;
}
function linkOrCopyFrame(src: string, dest: string): void {
try {
linkSync(src, dest);
} catch {
copyFileSync(src, dest);
}
}
function supersetGroupingKey(work: PreparedExtraction, fps: number): string {
return [work.videoPath, String(fps), work.format, work.sdrToHdrTransfer ?? ""].join("\0");
}
function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean {
const frames = offsetSeconds * fps;
return Math.abs(frames - Math.round(frames)) <= 1e-4;
}
function windowsOverlapOrTouch(misses: UniqueExtractionMiss[], baseStart: number): boolean {
const unionEnd = Math.max(
...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
);
const unionDuration = unionEnd - baseStart;
const summedDuration = misses.reduce((sum, { work }) => sum + work.videoDuration, 0);
return unionDuration > 0 && unionDuration <= summedDuration + 1e-9;
}
function buildSupersetGroup(
groupId: string,
misses: UniqueExtractionMiss[],
fps: number,
): SupersetGroupPlan | null {
if (misses.length < 2) return null;
const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart));
if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) {
return null;
}
if (!windowsOverlapOrTouch(misses, baseStart)) return null;
const unionEnd = Math.max(
...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
);
return {
groupId,
baseStart,
unionDuration: unionEnd - baseStart,
members: misses.map((miss) => ({
miss,
offsetFrames: Math.round((miss.work.video.mediaStart - baseStart) * fps),
})),
};
}
/**
* Partition one source's misses into overlap-connected components: sort by
* window start and cut wherever the next window starts past the running end.
* Without this, one disjoint outlier trim (e.g. [100..105] next to three
* overlapping trims at [0..11]) fails the union<=sum check for the whole
* bucket and every trim falls back to direct extraction.
*/
function overlapClusters(misses: UniqueExtractionMiss[]): UniqueExtractionMiss[][] {
const sorted = [...misses].sort((a, b) => a.work.video.mediaStart - b.work.video.mediaStart);
const clusters: UniqueExtractionMiss[][] = [];
let current: UniqueExtractionMiss[] = [];
let currentEnd = -Infinity;
for (const miss of sorted) {
const start = miss.work.video.mediaStart;
const end = start + miss.work.videoDuration;
if (current.length > 0 && start > currentEnd + 1e-9) {
clusters.push(current);
current = [];
currentEnd = -Infinity;
}
current.push(miss);
currentEnd = Math.max(currentEnd, end);
}
if (current.length > 0) clusters.push(current);
return clusters;
}
function planSupersetGroups(
misses: UniqueExtractionMiss[],
fps: number,
): { groups: SupersetGroupPlan[]; direct: UniqueExtractionMiss[] } {
const bySource = new Map();
for (const miss of misses) {
const key = supersetGroupingKey(miss.work, fps);
bySource.set(key, [...(bySource.get(key) ?? []), miss]);
}
const groups: SupersetGroupPlan[] = [];
const direct: UniqueExtractionMiss[] = [];
let groupIndex = 0;
for (const groupMisses of bySource.values()) {
for (const cluster of overlapClusters(groupMisses)) {
const group = buildSupersetGroup(`__superset-${groupIndex}`, cluster, fps);
if (group) {
groups.push(group);
groupIndex += 1;
} else {
direct.push(...cluster);
}
}
}
return { groups, direct };
}
function sliceSupersetMember(
member: SupersetMemberPlan,
superset: ExtractedFrames,
outputDir: string,
fps: number,
): ExtractedFrames {
const { work } = member.miss;
rmSync(outputDir, { recursive: true, force: true });
mkdirSync(outputDir, { recursive: true });
// Sample-time correctness: member frame k uses superset frame
// offset_i + k, so its source time is
// baseStart + (offset_i + k) / fps = mediaStart_i + k / fps.
// The frame-alignment precondition is what makes offset_i integral.
const requestedFrames = Math.round(work.videoDuration * fps);
const availableFrames = Math.max(0, superset.totalFrames - member.offsetFrames);
const frameCount = Math.min(requestedFrames, availableFrames);
for (let i = 0; i < frameCount; i += 1) {
const sourceFrame = superset.framePaths.get(member.offsetFrames + i);
if (!sourceFrame) throw new Error(`superset frame ${member.offsetFrames + i} missing`);
linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format)));
}
return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps);
}
/**
* Resolve a relative `