// fallow-ignore-file unused-class-member code-duplication complexity /** * Video Frame Extractor Service * * Pre-extracts video frames using FFmpeg for frame-accurate rendering. * Videos are replaced with elements during capture. */ import { spawn } from "child_process"; import { copyFileSync, existsSync, linkSync, mkdirSync, readdirSync, rmSync } from "fs"; import { isAbsolute, join, posix, resolve, sep } from "path"; import { parseHTML } from "linkedom"; import { decodeUrlPathVariants, MEDIA_DURATION_CLAMP_EPSILON_SECONDS } from "@hyperframes/core"; import { trackChildProcess } from "../utils/processTracker.js"; import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js"; import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, type HdrTransfer, } from "../utils/hdr.js"; import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js"; import { getFfmpegBinary } from "../utils/ffmpegBinaries.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { FRAME_FILENAME_PREFIX, gcExtractionCache, gcSweepDue, lookupCacheEntry, partialCacheEntryDir, publishCacheEntry, readKeyStat, rehydrateCacheEntry, touchCacheEntry, type CacheEntry, type CacheFrameFormat, } from "./extractionCache.js"; export interface VideoElement { id: string; src: string; start: number; end: number; mediaStart: number; loop: boolean; hasAudio: boolean; } export interface ExtractedFrames { videoId: string; srcPath: string; outputDir: string; framePattern: string; fps: number; totalFrames: number; metadata: VideoMetadata; framePaths: Map; /** * True when the extractor owns `outputDir` and cleanup should rm it when * the render ends. Cache hits set this to false so the shared entry isn't * deleted by a single render's cleanup — the cache dir is owned by the * caller's gc policy, not any one render. */ ownedByLookup?: boolean; } /** * The single source of truth for the source-video frame-extraction allow-list. * The CLI flag parser, the producer HTTP server, and the distributed-config * validator all validate against this same set via {@link isVideoFrameFormat} * so the boundaries can't drift when a new format is added. */ export const VIDEO_FRAME_FORMATS = ["auto", "jpg", "png"] as const; export type VideoFrameFormat = (typeof VIDEO_FRAME_FORMATS)[number]; /** Runtime guard for {@link VideoFrameFormat} over an untrusted value. */ export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat { return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value); } export interface ExtractionOptions { fps: number; outputDir: string; quality?: number; format?: VideoFrameFormat; sdrToHdrTransfer?: HdrTransfer; } const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000; const GC_STALENESS_MS = 24 * 60 * 60 * 1000; const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"; function sdrToHdrTransformKey(transfer: HdrTransfer): string { return `sdr2hdr-${transfer}`; } /** * Per-phase timings and counters emitted by `extractAllVideoFrames`. * * Used by the producer to surface `perfSummary.videoExtractBreakdown` — without * this breakdown, a single `videoExtractMs` stage timing hides where cost lives * (HDR preflight, VFR classification, per-video ffmpeg extract) when tuning renders. * * Field semantics: * - *Ms fields are wall-clock durations inside each phase. * - *Count fields report how many sources triggered that phase. * - extractMs wraps the parallel `extractVideoFramesRange` calls; it * reflects max-across-parallel-workers, not sum. * - hdrPreflightMs includes its probe-time sibling (hdrProbeMs); the * probe-only field is a finer decomposition, not a separate carve-out. * - vfrPreflightCount reports sources classified as VFR and routed through * the one-pass `-fps_mode cfr -r` extraction path. DEFINITION CHANGE: * before the one-pass refactor, vfrPreflightMs timed a per-source * VFR-to-CFR re-encode and could reach seconds; it now times only the * (promise-cached) classification probe and is expected to be ~0. * Dashboards alerting on vfrPreflightMs thresholds should key on * vfrPreflightCount or extractMs instead. */ export interface ExtractionPhaseBreakdown { resolveMs: number; /** Publishes that could not land atomically — the render still succeeded * from the partial dir, but future renders re-extract. A rising rate is * the first signal that warm renders are silently going cold. */ cachePublishFailures: number; /** Entries evicted by the post-extraction LRU sweep. */ cacheGcEvictions: number; /** Bytes reclaimed by the LRU sweep. */ cacheGcBytesFreed: number; /** Aged .partial-* dirs (crashed writers) removed by the sweep. */ cacheAgedPartialsCleared: number; hdrProbeMs: number; hdrPreflightMs: number; hdrPreflightCount: number; vfrProbeMs: number; vfrPreflightMs: number; vfrPreflightCount: number; extractMs: number; cacheHits: number; cacheMisses: number; } export interface ExtractionResult { success: boolean; extracted: ExtractedFrames[]; errors: Array<{ videoId: string; error: string }>; totalFramesExtracted: number; durationMs: number; phaseBreakdown: ExtractionPhaseBreakdown; } export function parseVideoElements(html: string): VideoElement[] { const videos: VideoElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); const startCache = new Map(); const visiting = new Set(); const videoEls = document.querySelectorAll("video[src]"); let autoIdCounter = 0; for (const el of videoEls) { const src = el.getAttribute("src"); if (!src) continue; // Generate a stable ID for videos without one — the producer needs IDs // to track extracted frames and composite them during encoding. const id = el.getAttribute("id") || `hf-video-${autoIdCounter++}`; if (!el.getAttribute("id")) { el.setAttribute("id", id); } const startAttr = el.getAttribute("data-start"); const endAttr = el.getAttribute("data-end"); const durationAttr = el.getAttribute("data-duration"); const mediaStartAttr = el.getAttribute("data-media-start"); const hasAudioAttr = el.getAttribute("data-has-audio"); // Resolve data-start, including relative references ("intro", "intro + 2") // to another clip's end — the browser runtime resolves these but a raw // parseFloat here would yield NaN, placing the clip at NaN so it composites // blank in the final render. `startAttr` may be a plain number or a // reference; the resolver handles both. const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0; // Derive end from data-end → data-start+data-duration → Infinity (natural duration). // The caller (htmlCompiler) clamps Infinity to the composition's absoluteEnd. let end = 0; if (endAttr) { end = parseFloat(endAttr); } else if (durationAttr) { end = start + parseFloat(durationAttr); } else { end = Infinity; // no explicit bounds — play for the full natural video duration } videos.push({ id, src, start, end, mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0, loop: el.hasAttribute("loop"), hasAudio: hasAudioAttr === "true", }); } return videos; } export interface ImageElement { id: string; src: string; start: number; end: number; } export function parseImageElements(html: string): ImageElement[] { const images: ImageElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); const startCache = new Map(); const visiting = new Set(); const imgEls = document.querySelectorAll("img[src]"); let autoIdCounter = 0; for (const el of imgEls) { const src = el.getAttribute("src"); if (!src) continue; const id = el.getAttribute("id") || `hf-img-${autoIdCounter++}`; if (!el.getAttribute("id")) { el.setAttribute("id", id); } const startAttr = el.getAttribute("data-start"); const endAttr = el.getAttribute("data-end"); const durationAttr = el.getAttribute("data-duration"); // Resolve relative data-start references (see parseVideoElements) so a // referenced image start doesn't become NaN and drop the image from the render. const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0; let end = 0; if (endAttr) { end = parseFloat(endAttr); } else if (durationAttr) { end = start + parseFloat(durationAttr); } else { end = Infinity; } images.push({ id, src, start, end }); } return images; } export async function extractVideoFramesRange( videoPath: string, videoId: string, startTime: number, duration: number, options: ExtractionOptions, signal?: AbortSignal, config?: Partial>, /** * Override the output directory for this extraction. When provided, frames * are written directly into `outputDirOverride` (no per-videoId subdir). * Used by the cache layer to materialize frames straight into the keyed * cache entry directory. */ outputDirOverride?: string, ): Promise { const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const { fps, outputDir, quality = 95 } = options; const videoOutputDir = outputDirOverride ?? join(outputDir, videoId); if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true }); const metadata = await extractMediaMetadata(videoPath); const format = resolveFrameFormat(metadata, options.format); const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`; const outputPattern = join(videoOutputDir, framePattern); // When extracting from HDR source, tone-map to SDR in FFmpeg rather than // letting Chrome's uncontrollable tone-mapper handle it (which washes out). // macOS: VideoToolbox hardware decoder does HDR→SDR natively on Apple Silicon. // Linux: zscale filter (when available) or colorspace filter as fallback. const isHdr = isHdrColorSpaceUtil(metadata.colorSpace); const isMacOS = process.platform === "darwin"; const args: string[] = []; if (isHdr && isMacOS) { args.push("-hwaccel", "videotoolbox"); } // Always force the alpha-aware decoder on codecs that can carry alpha. The // alternative — gating on `metadata.hasAlpha` — relies on tag detection that // has at least three known failure modes: case-sensitivity across ffmpeg // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers, // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there // silently strips alpha during decode and the bug doesn't surface until // the rendered video is missing layers. Codec-based default has no such // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes // normally when it isn't. if (codecMayHaveAlpha(metadata.videoCodec)) { args.push("-c:v", decoderForCodec(metadata.videoCodec)); } args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration)); const vfFilters: string[] = []; if (isHdr && isMacOS) { // VideoToolbox tone-maps during decode; force output to bt709 SDR format vfFilters.push("format=nv12"); } if (!metadata.isVFR) { vfFilters.push(`fps=${fps}`); } if (options.sdrToHdrTransfer) { // Ordering intent: fps sampling runs BEFORE the colorspace remap so only // kept frames are converted. The remap is pointwise per-frame, so the // output is identical either way for the SDR (BT.709, 8-bit) inputs this // flag is set for. If format=nv12 (macOS HDR-source decode) ever combines // with this flag, revisit: nv12 subsampling before a BT.2020 remap is an // untested interaction (today the flags are mutually exclusive — the // remap only applies to SDR sources, nv12 only to HDR sources). vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER); } if (vfFilters.length > 0) args.push("-vf", vfFilters.join(",")); if (metadata.isVFR) args.push("-fps_mode", "cfr", "-r", String(fps)); args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"); // Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files. if (format === "png") args.push("-compression_level", "1"); args.push("-y", outputPattern); return new Promise((resolve, reject) => { const ffmpeg = spawn(getFfmpegBinary(), args); trackChildProcess(ffmpeg); let stderr = ""; const onAbort = () => { ffmpeg.kill("SIGTERM"); }; if (signal) { if (signal.aborted) { ffmpeg.kill("SIGTERM"); } else { signal.addEventListener("abort", onAbort, { once: true }); } } const timer = setTimeout(() => { ffmpeg.kill("SIGTERM"); }, ffmpegProcessTimeout); ffmpeg.stderr.on("data", (data) => { stderr += data.toString(); }); ffmpeg.on("close", (code) => { clearTimeout(timer); if (signal) signal.removeEventListener("abort", onAbort); if (signal?.aborted) { reject(new Error("Video frame extraction cancelled")); return; } if (code !== 0) { // With the SDR-to-HDR remap folded into this pass, a filter failure // (e.g. an ffmpeg built without the colorspace filter) would otherwise // surface as a generic extract error and the operator has to grep the // filter chain to learn it was the HDR conversion. Attribute it. const hdrPrefix = options.sdrToHdrTransfer ? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): ` : ""; reject(new Error(`${hdrPrefix}FFmpeg exited with code ${code}: ${stderr.slice(-500)}`)); return; } const framePaths = new Map(); const files = readdirSync(videoOutputDir) .filter((f) => f.startsWith(FRAME_FILENAME_PREFIX) && f.endsWith(`.${format}`)) .sort(); files.forEach((file, index) => { framePaths.set(index, join(videoOutputDir, file)); }); resolve({ videoId, srcPath: videoPath, outputDir: videoOutputDir, framePattern, fps, totalFrames: framePaths.size, metadata, framePaths, }); }); ffmpeg.on("error", (err) => { clearTimeout(timer); if (signal) signal.removeEventListener("abort", onAbort); if ((err as NodeJS.ErrnoException).code === "ENOENT") { reject(new Error("[FFmpeg] ffmpeg not found")); } else { reject(err); } }); }); } /** * Resolve the used-segment duration for a video, falling back to the source's * natural duration when the caller hasn't specified bounds (end=Infinity) or * the bounds are nonsensical (end<=start). */ function resolveSegmentDuration( requested: number, mediaStart: number, metadata: VideoMetadata, ): number { if (Number.isFinite(requested) && requested > 0) return requested; const sourceRemaining = metadata.durationSeconds - mediaStart; return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds; } /** * Codecs whose bitstream is allowed to carry an alpha channel. Default the * extraction path to PNG output for these regardless of `metadata.hasAlpha` * so a missed sidecar tag doesn't silently strip transparency. Opaque content * encoded in one of these codecs pays a small file-size cost on the cached * frames but stays correct on the rare case where alpha IS present and the * tag was missed. */ const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]); export function codecMayHaveAlpha(codec: string | undefined): boolean { return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase()); } export function decoderForCodec(codec: string | undefined): string { const c = (codec ?? "").toLowerCase(); if (c === "vp9") return "libvpx-vp9"; if (c === "vp8") return "libvpx"; return c; } export function resolveFrameFormat( metadata: VideoMetadata, requested?: VideoFrameFormat, ): CacheFrameFormat { if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png"; if (requested === "png" || requested === "jpg") return requested; return "jpg"; } type PreparedExtraction = { video: VideoElement; videoPath: string; index: number; metadata: VideoMetadata; videoDuration: number; format: CacheFrameFormat; sdrToHdrTransfer?: HdrTransfer; dedupeKey: string; }; type CacheMissTarget = { entry: CacheEntry; srcPath: string; }; type UniqueExtractionMiss = { work: PreparedExtraction; cacheTarget?: CacheMissTarget; }; type SupersetMemberPlan = { miss: UniqueExtractionMiss; offsetFrames: number; }; type SupersetGroupPlan = { groupId: string; baseStart: number; unionDuration: number; members: SupersetMemberPlan[]; }; function extractedFrameFileNames(outputDir: string, format: CacheFrameFormat): string[] { const suffix = `.${format}`; return readdirSync(outputDir) .filter((file) => file.startsWith(FRAME_FILENAME_PREFIX) && file.endsWith(suffix)) .sort(); } function extractedFramesFromDirectory( work: PreparedExtraction, outputDir: string, srcPath: string, fps: number, ): ExtractedFrames { const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${work.format}`; const framePaths = new Map(); extractedFrameFileNames(outputDir, work.format).forEach((file, index) => { framePaths.set(index, join(outputDir, file)); }); return { videoId: work.video.id, srcPath, outputDir, framePattern, fps, totalFrames: framePaths.size, metadata: work.metadata, framePaths, }; } function frameFileName(frameNumber: number, format: CacheFrameFormat): string { return `${FRAME_FILENAME_PREFIX}${String(frameNumber).padStart(5, "0")}.${format}`; } function linkOrCopyFrame(src: string, dest: string): void { try { linkSync(src, dest); } catch { copyFileSync(src, dest); } } function supersetGroupingKey(work: PreparedExtraction, fps: number): string { return [work.videoPath, String(fps), work.format, work.sdrToHdrTransfer ?? ""].join("\0"); } function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean { const frames = offsetSeconds * fps; return Math.abs(frames - Math.round(frames)) <= 1e-4; } function windowsOverlapOrTouch(misses: UniqueExtractionMiss[], baseStart: number): boolean { const unionEnd = Math.max( ...misses.map(({ work }) => work.video.mediaStart + work.videoDuration), ); const unionDuration = unionEnd - baseStart; const summedDuration = misses.reduce((sum, { work }) => sum + work.videoDuration, 0); return unionDuration > 0 && unionDuration <= summedDuration + 1e-9; } function buildSupersetGroup( groupId: string, misses: UniqueExtractionMiss[], fps: number, ): SupersetGroupPlan | null { if (misses.length < 2) return null; const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart)); if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) { return null; } if (!windowsOverlapOrTouch(misses, baseStart)) return null; const unionEnd = Math.max( ...misses.map(({ work }) => work.video.mediaStart + work.videoDuration), ); return { groupId, baseStart, unionDuration: unionEnd - baseStart, members: misses.map((miss) => ({ miss, offsetFrames: Math.round((miss.work.video.mediaStart - baseStart) * fps), })), }; } /** * Partition one source's misses into overlap-connected components: sort by * window start and cut wherever the next window starts past the running end. * Without this, one disjoint outlier trim (e.g. [100..105] next to three * overlapping trims at [0..11]) fails the union<=sum check for the whole * bucket and every trim falls back to direct extraction. */ function overlapClusters(misses: UniqueExtractionMiss[]): UniqueExtractionMiss[][] { const sorted = [...misses].sort((a, b) => a.work.video.mediaStart - b.work.video.mediaStart); const clusters: UniqueExtractionMiss[][] = []; let current: UniqueExtractionMiss[] = []; let currentEnd = -Infinity; for (const miss of sorted) { const start = miss.work.video.mediaStart; const end = start + miss.work.videoDuration; if (current.length > 0 && start > currentEnd + 1e-9) { clusters.push(current); current = []; currentEnd = -Infinity; } current.push(miss); currentEnd = Math.max(currentEnd, end); } if (current.length > 0) clusters.push(current); return clusters; } function planSupersetGroups( misses: UniqueExtractionMiss[], fps: number, ): { groups: SupersetGroupPlan[]; direct: UniqueExtractionMiss[] } { const bySource = new Map(); for (const miss of misses) { const key = supersetGroupingKey(miss.work, fps); bySource.set(key, [...(bySource.get(key) ?? []), miss]); } const groups: SupersetGroupPlan[] = []; const direct: UniqueExtractionMiss[] = []; let groupIndex = 0; for (const groupMisses of bySource.values()) { for (const cluster of overlapClusters(groupMisses)) { const group = buildSupersetGroup(`__superset-${groupIndex}`, cluster, fps); if (group) { groups.push(group); groupIndex += 1; } else { direct.push(...cluster); } } } return { groups, direct }; } function sliceSupersetMember( member: SupersetMemberPlan, superset: ExtractedFrames, outputDir: string, fps: number, ): ExtractedFrames { const { work } = member.miss; rmSync(outputDir, { recursive: true, force: true }); mkdirSync(outputDir, { recursive: true }); // Sample-time correctness: member frame k uses superset frame // offset_i + k, so its source time is // baseStart + (offset_i + k) / fps = mediaStart_i + k / fps. // The frame-alignment precondition is what makes offset_i integral. const requestedFrames = Math.round(work.videoDuration * fps); const availableFrames = Math.max(0, superset.totalFrames - member.offsetFrames); const frameCount = Math.min(requestedFrames, availableFrames); for (let i = 0; i < frameCount; i += 1) { const sourceFrame = superset.framePaths.get(member.offsetFrames + i); if (!sourceFrame) throw new Error(`superset frame ${member.offsetFrames + i} missing`); linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format))); } return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps); } /** * Resolve a relative `