/** * Video Frame Extractor Service * * Pre-extracts video frames using FFmpeg for frame-accurate rendering. * Videos are replaced with elements during capture. */ import { spawn } from "child_process"; import { existsSync, mkdirSync, readdirSync, rmSync } from "fs"; import { isAbsolute, join, posix, resolve, sep } from "path"; import { parseHTML } from "linkedom"; import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, type HdrTransfer, } from "../utils/hdr.js"; import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { FRAME_FILENAME_PREFIX, ensureCacheEntryDir, lookupCacheEntry, markCacheEntryComplete, readKeyStat, rehydrateCacheEntry, type CacheFrameFormat, } from "./extractionCache.js"; export interface VideoElement { id: string; src: string; start: number; end: number; mediaStart: number; loop: boolean; hasAudio: boolean; } export interface ExtractedFrames { videoId: string; srcPath: string; outputDir: string; framePattern: string; fps: number; totalFrames: number; metadata: VideoMetadata; framePaths: Map; /** * True when the extractor owns `outputDir` and cleanup should rm it when * the render ends. Cache hits set this to false so the shared entry isn't * deleted by a single render's cleanup — the cache dir is owned by the * caller's gc policy, not any one render. */ ownedByLookup?: boolean; } export interface ExtractionOptions { fps: number; outputDir: string; quality?: number; format?: "jpg" | "png"; } /** * Per-phase timings and counters emitted by `extractAllVideoFrames`. * * Used by the producer to surface `perfSummary.videoExtractBreakdown` — without * this breakdown, a single `videoExtractMs` stage timing hides where cost lives * (HDR preflight, VFR preflight, per-video ffmpeg extract) when tuning renders. * * Field semantics: * - *Ms fields are wall-clock durations inside each phase. * - *Count fields report how many sources triggered that phase. * - extractMs wraps the parallel `extractVideoFramesRange` calls; it * reflects max-across-parallel-workers, not sum. * - hdrPreflightMs / vfrPreflightMs both include their probe-time sibling * (hdrProbeMs / vfrProbeMs) for symmetric semantics. The probe-only fields * are a finer decomposition, not a separate carve-out. */ export interface ExtractionPhaseBreakdown { resolveMs: number; hdrProbeMs: number; hdrPreflightMs: number; hdrPreflightCount: number; vfrProbeMs: number; vfrPreflightMs: number; vfrPreflightCount: number; extractMs: number; cacheHits: number; cacheMisses: number; } export interface ExtractionResult { success: boolean; extracted: ExtractedFrames[]; errors: Array<{ videoId: string; error: string }>; totalFramesExtracted: number; durationMs: number; phaseBreakdown: ExtractionPhaseBreakdown; } export function parseVideoElements(html: string): VideoElement[] { const videos: VideoElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); const videoEls = document.querySelectorAll("video[src]"); let autoIdCounter = 0; for (const el of videoEls) { const src = el.getAttribute("src"); if (!src) continue; // Generate a stable ID for videos without one — the producer needs IDs // to track extracted frames and composite them during encoding. const id = el.getAttribute("id") || `hf-video-${autoIdCounter++}`; if (!el.getAttribute("id")) { el.setAttribute("id", id); } const startAttr = el.getAttribute("data-start"); const endAttr = el.getAttribute("data-end"); const durationAttr = el.getAttribute("data-duration"); const mediaStartAttr = el.getAttribute("data-media-start"); const hasAudioAttr = el.getAttribute("data-has-audio"); const start = startAttr ? parseFloat(startAttr) : 0; // Derive end from data-end → data-start+data-duration → Infinity (natural duration). // The caller (htmlCompiler) clamps Infinity to the composition's absoluteEnd. let end = 0; if (endAttr) { end = parseFloat(endAttr); } else if (durationAttr) { end = start + parseFloat(durationAttr); } else { end = Infinity; // no explicit bounds — play for the full natural video duration } videos.push({ id, src, start, end, mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0, loop: el.hasAttribute("loop"), hasAudio: hasAudioAttr === "true", }); } return videos; } export interface ImageElement { id: string; src: string; start: number; end: number; } export function parseImageElements(html: string): ImageElement[] { const images: ImageElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); const imgEls = document.querySelectorAll("img[src]"); let autoIdCounter = 0; for (const el of imgEls) { const src = el.getAttribute("src"); if (!src) continue; const id = el.getAttribute("id") || `hf-img-${autoIdCounter++}`; if (!el.getAttribute("id")) { el.setAttribute("id", id); } const startAttr = el.getAttribute("data-start"); const endAttr = el.getAttribute("data-end"); const durationAttr = el.getAttribute("data-duration"); const start = startAttr ? parseFloat(startAttr) : 0; let end = 0; if (endAttr) { end = parseFloat(endAttr); } else if (durationAttr) { end = start + parseFloat(durationAttr); } else { end = Infinity; } images.push({ id, src, start, end }); } return images; } export async function extractVideoFramesRange( videoPath: string, videoId: string, startTime: number, duration: number, options: ExtractionOptions, signal?: AbortSignal, config?: Partial>, /** * Override the output directory for this extraction. When provided, frames * are written directly into `outputDirOverride` (no per-videoId subdir). * Used by the cache layer to materialize frames straight into the keyed * cache entry directory. */ outputDirOverride?: string, ): Promise { const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const { fps, outputDir, quality = 95 } = options; const videoOutputDir = outputDirOverride ?? join(outputDir, videoId); if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true }); const metadata = await extractMediaMetadata(videoPath); const format = resolveFrameFormat(metadata, options.format); const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`; const outputPattern = join(videoOutputDir, framePattern); // When extracting from HDR source, tone-map to SDR in FFmpeg rather than // letting Chrome's uncontrollable tone-mapper handle it (which washes out). // macOS: VideoToolbox hardware decoder does HDR→SDR natively on Apple Silicon. // Linux: zscale filter (when available) or colorspace filter as fallback. const isHdr = isHdrColorSpaceUtil(metadata.colorSpace); const isMacOS = process.platform === "darwin"; const args: string[] = []; if (isHdr && isMacOS) { args.push("-hwaccel", "videotoolbox"); } // Always force the alpha-aware decoder on codecs that can carry alpha. The // alternative — gating on `metadata.hasAlpha` — relies on tag detection that // has at least three known failure modes: case-sensitivity across ffmpeg // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers, // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there // silently strips alpha during decode and the bug doesn't surface until // the rendered video is missing layers. Codec-based default has no such // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes // normally when it isn't. if (codecMayHaveAlpha(metadata.videoCodec)) { args.push("-c:v", decoderForCodec(metadata.videoCodec)); } args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration)); const vfFilters: string[] = []; if (isHdr && isMacOS) { // VideoToolbox tone-maps during decode; force output to bt709 SDR format vfFilters.push("format=nv12"); } vfFilters.push(`fps=${fps}`); args.push("-vf", vfFilters.join(",")); args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"); if (format === "png") args.push("-compression_level", "6"); args.push("-y", outputPattern); return new Promise((resolve, reject) => { const ffmpeg = spawn("ffmpeg", args); let stderr = ""; const onAbort = () => { ffmpeg.kill("SIGTERM"); }; if (signal) { if (signal.aborted) { ffmpeg.kill("SIGTERM"); } else { signal.addEventListener("abort", onAbort, { once: true }); } } const timer = setTimeout(() => { ffmpeg.kill("SIGTERM"); }, ffmpegProcessTimeout); ffmpeg.stderr.on("data", (data) => { stderr += data.toString(); }); ffmpeg.on("close", (code) => { clearTimeout(timer); if (signal) signal.removeEventListener("abort", onAbort); if (signal?.aborted) { reject(new Error("Video frame extraction cancelled")); return; } if (code !== 0) { reject(new Error(`FFmpeg exited with code ${code}: ${stderr.slice(-500)}`)); return; } const framePaths = new Map(); const files = readdirSync(videoOutputDir) .filter((f) => f.startsWith(FRAME_FILENAME_PREFIX) && f.endsWith(`.${format}`)) .sort(); files.forEach((file, index) => { framePaths.set(index, join(videoOutputDir, file)); }); resolve({ videoId, srcPath: videoPath, outputDir: videoOutputDir, framePattern, fps, totalFrames: framePaths.size, metadata, framePaths, }); }); ffmpeg.on("error", (err) => { clearTimeout(timer); if (signal) signal.removeEventListener("abort", onAbort); if ((err as NodeJS.ErrnoException).code === "ENOENT") { reject(new Error("[FFmpeg] ffmpeg not found")); } else { reject(err); } }); }); } /** * Convert an SDR (BT.709) video to BT.2020 wide-gamut so it can be composited * alongside HDR content without looking washed out. * * Uses FFmpeg's `colorspace` filter to remap BT.709 → BT.2020 (no real tone * mapping — just a primaries swap so the input fits inside the wider HDR * gamut), then re-tags the stream with the caller's target HDR transfer * function (PQ for HDR10, HLG for broadcast HDR). The output transfer must * match the dominant transfer of the surrounding HDR content; otherwise the * downstream encoder will tag the final video with the wrong curve. * * `startTime` and `duration` bound the re-encode to the segment the composition * actually uses. Without them a 30-minute screen recording that contributes a * 2-second clip was transcoded in full — a >100× waste for long sources. * Mirrors the segment-scope fix already applied to the VFR→CFR preflight. */ async function convertSdrToHdr( inputPath: string, outputPath: string, startTime: number, duration: number, targetTransfer: HdrTransfer, signal?: AbortSignal, config?: Partial>, ): Promise { // Positive duration is required — FFmpeg's `-t 0` silently produces a 0-byte // output that the downstream extractor then treats as a valid (empty) file. if (duration <= 0) { throw new Error(`convertSdrToHdr: duration must be positive (got ${duration})`); } const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; // smpte2084 = PQ (HDR10), arib-std-b67 = HLG. const colorTrc = targetTransfer === "pq" ? "smpte2084" : "arib-std-b67"; const args = [ "-ss", String(startTime), "-i", inputPath, "-t", String(duration), "-vf", "colorspace=all=bt2020:iall=bt709:range=tv", "-color_primaries", "bt2020", "-color_trc", colorTrc, "-colorspace", "bt2020nc", "-c:v", "libx264", "-preset", "fast", "-crf", "16", "-c:a", "copy", "-y", outputPath, ]; const result = await runFfmpeg(args, { signal, timeout }); if (!result.success) { throw new Error( `SDR→HDR conversion failed (exit ${result.exitCode}): ${result.stderr.slice(-300)}`, ); } } /** * Resolve the used-segment duration for a video, falling back to the source's * natural duration when the caller hasn't specified bounds (end=Infinity) or * the bounds are nonsensical (end<=start). */ function resolveSegmentDuration( requested: number, mediaStart: number, metadata: VideoMetadata, ): number { if (Number.isFinite(requested) && requested > 0) return requested; const sourceRemaining = metadata.durationSeconds - mediaStart; return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds; } /** * Codecs whose bitstream is allowed to carry an alpha channel. Default the * extraction path to PNG output for these regardless of `metadata.hasAlpha` * so a missed sidecar tag doesn't silently strip transparency. Opaque content * encoded in one of these codecs pays a small file-size cost on the cached * frames but stays correct on the rare case where alpha IS present and the * tag was missed. */ const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]); export function codecMayHaveAlpha(codec: string | undefined): boolean { return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase()); } export function decoderForCodec(codec: string | undefined): string { const c = (codec ?? "").toLowerCase(); if (c === "vp9") return "libvpx-vp9"; if (c === "vp8") return "libvpx"; return c; } function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat { if (requested) return requested; if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png"; return "jpg"; } /** * Re-encode a VFR (variable frame rate) video segment to CFR so the downstream * fps filter can extract frames reliably. Screen recordings, phone videos, and * some webcams emit irregular timestamps that cause two failure modes: * 1. Output has fewer frames than expected (e.g. -ss 3 -t 4 produces 90 * frames instead of 120 @ 30fps). FrameLookupTable.getFrameAtTime then * returns null for late timestamps and the caller freezes on the last * valid frame. * 2. Large duplicate-frame runs where source PTS don't land on target * timestamps. * * Only the [startTime, startTime+duration] window is re-encoded, so long * recordings aren't fully transcoded when only a short clip is used. */ async function convertVfrToCfr( inputPath: string, outputPath: string, targetFps: number, startTime: number, duration: number, signal?: AbortSignal, config?: Partial>, ): Promise { const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const args = [ "-ss", String(startTime), "-i", inputPath, "-t", String(duration), "-fps_mode", "cfr", "-r", String(targetFps), "-c:v", "libx264", "-preset", "fast", "-crf", "18", "-c:a", "copy", "-y", outputPath, ]; const result = await runFfmpeg(args, { signal, timeout }); if (!result.success) { throw new Error( `VFR→CFR conversion failed (exit ${result.exitCode}): ${result.stderr.slice(-300)}`, ); } } /** * Resolve a relative `