// fallow-ignore-file unused-class-member code-duplication complexity /** * Video Frame Extractor Service * * Pre-extracts video frames using FFmpeg for frame-accurate rendering. * Videos are replaced with elements during capture. */ import { usableCpuCount } from "./systemMemory.js"; import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./mediaHidden.js"; import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs"; import { basename, isAbsolute, join, normalize, relative, win32 } from "path"; import { createConcurrencyLimit } from "../utils/concurrencyLimit.js"; import { parseHTML } from "linkedom"; import { HF_COLOR_GRADING_ATTR } from "@hyperframes/core"; import { isPathInside } from "@hyperframes/parsers/asset-paths"; import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution"; import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha"; import { MEDIA_RENDER_ID_ATTR, isAudibleVideoElement, fpsToFfmpegArg, fpsToNumber, MEDIA_DURATION_CLAMP_EPSILON_SECONDS, normalizeRateSpec, readElementRateSpec, shiftRateLane, sourceTimeAt, timeAtSourceTime, type RateSpec, exportClipWindow, hasClipStarted, isInClipWindow, mediaElementTime, parseStrictFiniteTimingNumber, readMediaStart, toFps, type Fps, type FpsInput, firstFrameColourArgs, keyframeTransferArgs, parseKeyframeTransfers, type ToneMapFrameColour, hdrToSdrToneMapFilter, parseFirstFrameColour, readsKeyframeTransfers, } from "@hyperframes/core"; import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js"; import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js"; import { extractFinalVideoFrameTimestamp, extractMediaMetadata, ffmpegSeekSeconds, formatFfmpegSeconds, keyframeGapSeconds, mediaFileIdentity, readMetadataCache, readStoredFinalFrame, resolvePlayableVideoDuration, videoStartOnMediaTimeline, writeMetadataCache, type MediaProbeCacheEntry, type VideoMetadata, } from "../utils/ffprobe.js"; import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js"; import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js"; import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, type HdrTransfer, } from "../utils/hdr.js"; import { downloadToTemp, isHttpUrl, safeDownloadUrlIdentity, UrlDownloadError, writeUrlDownloadTelemetry, } from "../utils/urlDownloader.js"; import { ffmpegStatsReader, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js"; import { isFfmpegFilterAvailable, isFfmpegHwaccelAvailable, } from "../utils/psnrFilterAvailability.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { mediaIdPathSegment } from "../utils/mediaPathSegment.js"; import { FRAME_FILENAME_PREFIX, gcExtractionCache, gcSweepDue, lookupCacheEntry, partialCacheEntryDir, publishCacheEntry, readContentSha256, readKeyStat, rehydrateCacheEntry, touchCacheEntry, type CacheEntry, type CacheFrameFormat, } from "./extractionCache.js"; import { framePathsFromDirectory, frameSequencePath } from "./extractedFrameIndex.js"; export { resolveProjectRelativeSrc }; export interface VideoElement { id: string; src: string; start: number; end: number; mediaStart: number; playbackRate?: RateSpec; loop: boolean; hasAudio: boolean; hidden?: boolean; graded?: boolean; } export interface ExtractedFrames { videoId: string; srcPath: string; outputDir: string; framePattern: string; fps: number; totalFrames: number; metadata: VideoMetadata; framePaths: Map; deferredRange?: DeferredFrameRange; /** * True when the extractor owns `outputDir` and cleanup should rm it when * the render ends. Cache hits set this to false so the shared entry isn't * deleted by a single render's cleanup — the cache dir is owned by the * caller's gc policy, not any one render. */ ownedByLookup?: boolean; } /** * The single source of truth for the source-video frame-extraction allow-list. * The CLI flag parser, the producer HTTP server, and the distributed-config * validator all validate against this same set via {@link isVideoFrameFormat} * so the boundaries can't drift when a new format is added. */ export const VIDEO_FRAME_FORMATS = ["auto", "jpg", "png"] as const; export type VideoFrameFormat = (typeof VIDEO_FRAME_FORMATS)[number]; /** Runtime guard for {@link VideoFrameFormat} over an untrusted value. */ export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat { return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value); } const SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES = 1e-4; /** * Frames a window gets: one per output slot it covers (CFR), or rounded up (VFR). * Superset slicing and the coverage gate share it, so they never disagree by a frame. */ export function extractionFrameCountForDuration( durationSeconds: number, fps: FpsInput, isVFR: boolean, ): number { if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) return 0; // FFmpeg receives `String(durationSeconds)` and parses at microsecond // precision. Derive the integer microseconds from that same decimal text: // multiplying the binary float first is not equivalent (`2.05 * 1e6` is // 2049999.9999999998 in JS and would incorrectly truncate one microsecond). const serialized = String(durationSeconds).toLowerCase(); const decimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serialized); if (!decimal) return 0; const whole = decimal[1] ?? "0"; const fraction = decimal[2] ?? ""; const exponent = Number.parseInt(decimal[3] ?? "0", 10); const digits = BigInt(`${whole}${fraction}`); const microsecondScale = exponent + 6 - fraction.length; const microseconds = microsecondScale >= 0 ? digits * 10n ** BigInt(microsecondScale) : digits / 10n ** BigInt(-microsecondScale); // Keep the frame-boundary calculation rational too. Converting the exact // microseconds back to a binary float recreates the same problem at .5-frame // boundaries (`2.05 * 30` is 61.49999999999999 in JS). let fpsNumerator: bigint; let fpsDenominator: bigint; if (typeof fps === "object") { if ( !Number.isSafeInteger(fps.num) || !Number.isSafeInteger(fps.den) || fps.num <= 0 || fps.den <= 0 ) { return 0; } fpsNumerator = BigInt(fps.num); fpsDenominator = BigInt(fps.den); } else { if (!Number.isFinite(fps) || fps <= 0) return 0; // Number-only callers retain their decimal FFmpeg argument exactly. The // production render path supplies Fps, so NTSC rates never round-trip // through `String(30000 / 1001)` here. const serializedFps = String(fps).toLowerCase(); const fpsDecimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serializedFps); if (!fpsDecimal) return 0; const fpsWhole = fpsDecimal[1] ?? "0"; const fpsFraction = fpsDecimal[2] ?? ""; const fpsExponent = Number.parseInt(fpsDecimal[3] ?? "0", 10); const fpsDigits = BigInt(`${fpsWhole}${fpsFraction}`); const fpsScale = fpsExponent - fpsFraction.length; fpsNumerator = fpsScale >= 0 ? fpsDigits * 10n ** BigInt(fpsScale) : fpsDigits; fpsDenominator = fpsScale >= 0 ? 1n : 10n ** BigInt(-fpsScale); } const frameNumerator = microseconds * fpsNumerator; const frameDenominator = 1_000_000n * fpsDenominator; const slotsPerTolerance = BigInt(Math.round(1 / SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES)); const frameCount = isVFR ? (frameNumerator + frameDenominator - 1n) / frameDenominator : (slotsPerTolerance * (frameNumerator + frameDenominator) - frameDenominator - 1n) / (slotsPerTolerance * frameDenominator); const frames = Number(frameCount); return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER); } /** The exact rate behind a probed one (rounded to 0.01): a whole number or its 1000/1001 NTSC form. */ function exactSourceRate(probed: number): number | null { const nominal = Math.round(probed * 1.001); return ( [nominal, (nominal * 1000) / 1001].find((rate) => Math.abs(rate - probed) <= 0.005 + 1e-9) ?? null ); } /** Slots of a window reaching the source's end that hold a frame, from the stored final frame; null if unsure. */ export function tailFramesFromFinalFrame(input: { startTime: number; firstFrame: number; tailFrames: number; outputFps: number; sourceFps: number; finalFrameSeconds: number; streamLeadSeconds: number; lastPacketSeconds?: number; }): number | null { const sourceRate = exactSourceRate(input.sourceFps); if (!sourceRate || !(input.outputFps > 0)) return null; // A late-starting stream ends where its last packet does, and ffmpeg shortens the last frame // to a short packet in ways the stored facts cannot predict: trust only a full-frame packet. const lead = input.streamLeadSeconds; if (lead !== 0 && !(Math.abs((input.lastPacketSeconds ?? NaN) - 1 / sourceRate) <= 1e-4)) { return null; } const seek = input.startTime + input.firstFrame / input.outputFps; const slots = (lead + input.finalFrameSeconds + 1 / sourceRate - seek) * input.outputFps; const distanceToSlot = Math.abs(slots - Math.round(slots)); // ffprobe prints the final time to 1 us; a slot closer than 50 us/fps to the end is a tie, up to 500 us/fps unknown. if (distanceToSlot >= 5e-5 && distanceToSlot < 5e-4) return null; const count = Math.min(input.tailFrames, Math.ceil(slots - 5e-5)); return count > 0 ? count : null; } export interface FrameRange { firstFrame: number; frames: number; } /** The window a deferred extraction samples; frame k is the source frame on screen at `startTime + k / fps`. */ export interface DeferredFrameRange { startTime: number; durationSeconds: number; format: "jpg" | "png"; } export interface ExtractionOptions { /** Exact configured rate. Rational rates are passed to FFmpeg verbatim. */ fps: FpsInput; outputDir: string; quality?: number; format?: VideoFrameFormat; sdrToHdrTransfer?: HdrTransfer; toneMapHdrToSdr?: boolean; /** Extract exactly one frame at `startTime`. Used only after ffprobe has * resolved the actual final decoded-frame timestamp for a held tail. */ finalFrameOnly?: boolean; /** * Absolute composition/timeline end in seconds. Applied only after source * metadata resolves open-ended/natural-duration media. Invisible negative * preroll is trimmed while advancing mediaStart to preserve source alignment. */ timelineEnd?: number; /** * Bounded per-source FFmpeg retries. Default 0 preserves stable behavior; * the producer may canary at most one retry after observing typed failures. */ maxTransientRetries?: number; /** * Collect metadata-probe failures into `ExtractionResult.errors` instead * of preserving the legacy Promise rejection. Default false; only the * candidate enforce lane may opt into typed aggregation. */ collectProbeFailures?: boolean; /** Extract only these runs of output frames, named as a full extraction names them. CFR only. */ frameRanges?: readonly FrameRange[]; onProgress?: (progress: { planned?: number; done?: number }) => void; /** A short decode of a remote source, only to count its frames: it waits on the network, so it runs wider. */ tailProbe?: boolean; /** * Leave CFR windows longer than a second unextracted, returning `deferredRange` and an exact * `totalFrames` so another process can extract just the frames it needs with `frameRanges`. */ deferRangeExtraction?: boolean; /** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */ contentKeyedDirs?: readonly string[]; urlByAbsentLocalPath?: ReadonlyMap; shownBox?: { width: number; height: number }; shownBoxes?: ReadonlyMap; } const DOWNLOAD_SUBDIR = "_downloads"; export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000; const GC_STALENESS_MS = 24 * 60 * 60 * 1000; const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"; const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb"; const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr"; const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1"; function shownBoxKey(box: { width: number; height: number } | undefined): string | undefined { return box && `shown-${Math.ceil(box.width)}x${Math.ceil(box.height)}`; } function halveToCoverScaleFilter(box: { width: number; height: number }): string { const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`; const halvings = `floor(log(1/${fit})/log(2))`; const side = (dim: "iw" | "ih") => `if(lte(${fit}\\,0.5)\\,2*ceil(${dim}/pow(2\\,${halvings})/2)\\,${dim})`; return `scale=w=${side("iw")}:h=${side("ih")}:flags=area+accurate_rnd`; } const SDR_JPEG_AS_BT601_FULL_RANGE_FILTER = "scale=flags=neighbor+accurate_rnd,format=gbrp,scale=out_color_matrix=bt601:out_range=pc:flags=neighbor,format=yuv420p"; let warnedMissingZscale = false; async function toneMapsWithZscale(): Promise { return process.platform !== "darwin" || isFfmpegFilterAvailable("zscale"); } async function hdrToSdrTransformKey(): Promise { if (await toneMapsWithZscale()) return HDR_TO_SDR_ZSCALE_TRANSFORM_KEY; if (!warnedMissingZscale) { warnedMissingZscale = true; process.stderr.write( "[hyperframes:render] WARNING: this ffmpeg has no zscale filter, so forced-SDR HDR footage " + "is tone-mapped by Chrome instead of the hable curve used on Linux and in Studio, and " + "renders darker. Install an ffmpeg built with libzimg to match.\n", ); } return HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY; } function sdrToHdrTransformKey(transfer: HdrTransfer): string { return `sdr2hdr-${transfer}`; } /** * Per-phase timings and counters emitted by `extractAllVideoFrames`. * * Used by the producer to surface `perfSummary.videoExtractBreakdown` — without * this breakdown, a single `videoExtractMs` stage timing hides where cost lives * (HDR preflight, VFR classification, per-video ffmpeg extract) when tuning renders. * * Field semantics: * - *Ms fields are wall-clock durations inside each phase. * - *Count fields report how many sources triggered that phase. * - extractMs wraps the parallel `extractVideoFramesRange` calls; it * reflects max-across-parallel-workers, not sum. * - hdrPreflightMs includes its probe-time sibling (hdrProbeMs); the * probe-only field is a finer decomposition, not a separate carve-out. * - vfrPreflightCount reports sources classified as VFR and extracted with * a rounded-up frame count. DEFINITION CHANGE: * before the one-pass refactor, vfrPreflightMs timed a per-source * VFR-to-CFR re-encode and could reach seconds; it now times only the * (promise-cached) classification probe and is expected to be ~0. * Dashboards alerting on vfrPreflightMs thresholds should key on * vfrPreflightCount or extractMs instead. */ export interface ExtractionPhaseBreakdown { resolveMs: number; /** Publishes that could not land atomically — the render still succeeded * from the partial dir, but future renders re-extract. A rising rate is * the first signal that warm renders are silently going cold. */ cachePublishFailures: number; /** Entries evicted by the post-extraction LRU sweep. */ cacheGcEvictions: number; /** Bytes reclaimed by the LRU sweep. */ cacheGcBytesFreed: number; /** Aged .partial-* dirs (crashed writers) removed by the sweep. */ cacheAgedPartialsCleared: number; hdrProbeMs: number; hdrPreflightMs: number; hdrPreflightCount: number; vfrProbeMs: number; vfrPreflightMs: number; vfrPreflightCount: number; extractMs: number; cacheHits: number; cacheMisses: number; /** Number of per-source transient failures retried inside this extraction. */ transientRetries?: number; } export type VideoExtractionFailureKind = | "cancelled" | "external_interruption" | "source_missing" | "source_rejected" | "download_not_found" | "download_transient" | "invalid_media" | "media_start_out_of_range" | "ffmpeg_unavailable" | "ffmpeg_timeout" | "ffmpeg_transient" | "ffmpeg_failed" | "disk_full" | "zero_output" | "internal"; export interface VideoExtractionFailure { videoId: string; /** Always populated by this engine version; optional for source compatibility with older consumers. */ kind?: VideoExtractionFailureKind; /** Always populated by this engine version; absent legacy values fail closed. */ retryable?: boolean; /** Bounded, path-free grouping data safe for producer-owned error metadata. */ group?: VideoExtractionFailureGroupDetails; /** * Operator diagnostic retained inside the engine result. Producer-facing * errors must summarize `kind`/counts and must not forward this field: it * can contain a local path or a signed source URL. */ error: string; /** ffmpeg's last stderr lines, without paths or URLs: what a producer-facing error may say. */ ffmpegReason?: string; /** The clip's file, project-relative; a remote source shows only its file name. Safe for producer-facing errors. */ source?: string; } const URL_SCHEME = /^[a-z][a-z0-9+.-]+:/i; const WINDOWS_ABSOLUTE = /^([a-zA-Z]:[\\/]|\\\\)/; /** A clip's src as a producer-facing error may print it: no scheme payload, host, query, fragment or absolute path. */ export function videoExtractionSourceLabel(src: string, baseDir: string): string { if (/^(data|blob):/i.test(src)) return "inline video"; if (URL_SCHEME.test(src)) { try { return basename(decodeURIComponent(new URL(src).pathname)) || "remote video"; } catch { return "remote video"; } } const path = src.split(/[?#]/)[0]!; const lib = WINDOWS_ABSOLUTE.test(path) ? win32 : { isAbsolute, relative, basename, normalize }; const relativePath = lib.isAbsolute(path) ? lib.relative(baseDir, path) : lib.normalize(path); const label = relativePath.startsWith("..") || lib.isAbsolute(relativePath) ? lib.basename(path) : relativePath; return label.replace(/\\/g, "/"); } export type VideoExtractionFailureStatusClass = | "http_4xx" | "http_5xx" | "timeout" | "network" | "other"; export interface VideoExtractionFailureRetry { phase: "download"; used: 0 | 1; budget: 1; } export interface VideoExtractionFailureGroupDetails { sourceFingerprint?: string; host?: string; statusClass?: VideoExtractionFailureStatusClass; retry?: VideoExtractionFailureRetry; } export interface SafeVideoExtractionSourceIdentity { sourceFingerprint: string; host: string; } /** Query/fragment-free remote identity safe for extraction logs and wire metadata. */ export function safeVideoExtractionSourceIdentity( source: string, ): SafeVideoExtractionSourceIdentity | null { if (!isHttpUrl(source)) return null; const identity = safeDownloadUrlIdentity(source); return { sourceFingerprint: `sha256:${identity.urlFingerprint}`, host: identity.host ?? "other", }; } function downloadStatusClass(error: UrlDownloadError): VideoExtractionFailureStatusClass { const status = error.status ?? error.telemetry?.status; if (typeof status === "number" && status >= 400 && status < 500) return "http_4xx"; if (typeof status === "number" && status >= 500 && status < 600) return "http_5xx"; if (error.kind === "timeout") return "timeout"; if (error.kind === "network") return "network"; return "other"; } function downloadFailureGroup( source: string, error: UrlDownloadError, ): VideoExtractionFailureGroupDetails { const sourceIdentity = safeVideoExtractionSourceIdentity(source); const failureHost = error.telemetry?.finalHost ?? error.telemetry?.initialHost; const attempt = error.telemetry?.attempt; return { ...(sourceIdentity ? { sourceFingerprint: sourceIdentity.sourceFingerprint } : {}), ...((failureHost ?? sourceIdentity?.host) ? { host: failureHost ?? sourceIdentity?.host } : {}), statusClass: downloadStatusClass(error), retry: { phase: "download", used: typeof attempt === "number" && attempt >= 2 ? 1 : 0, budget: 1, }, }; } export class VideoSourceExtractionError extends Error { readonly hyperframesVideoSourceExtractionError = true as const; constructor( readonly kind: VideoExtractionFailureKind, readonly retryable: boolean, message: string, readonly diagnostic: string = message, readonly ffmpegReason?: string, ) { super(message); this.name = "VideoSourceExtractionError"; } } const FFMPEG_REASON_LINES = 3; const FFMPEG_REASON_CHARS = 300; /** ffmpeg's last stderr lines for an error that leaves this host: no local path, file name or URL. */ function ffmpegReasonFrom(stderr: string, paths: readonly string[]): string { let text = stderr; for (const known of paths.filter(Boolean).sort((a, b) => b.length - a.length)) text = text.split(known).join(""); return text .replace(/[a-z][a-z\d+.-]*:\/\/[^\s'"]+/gi, "") .replace(/(?:[a-z]:)?(?:[\\/][^\s\\/:'"]+){2,}/gi, "") .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean) .slice(-FFMPEG_REASON_LINES) .join(" | ") .slice(-FFMPEG_REASON_CHARS); } // ffmpeg ends a full disk with its last frames' summary, so the strerror text is the only sign of it const DISK_FULL = /no space left on device|disk quota exceeded/i; export function isVideoSourceExtractionError(error: unknown): error is VideoSourceExtractionError { return ( typeof error === "object" && error !== null && "hyperframesVideoSourceExtractionError" in error && error.hyperframesVideoSourceExtractionError === true ); } function boundedTransientRetryBudget(value: number | undefined): 0 | 1 { return Number.isFinite(value) && (value ?? 0) >= 1 ? 1 : 0; } function errorText(error: unknown): string { return error instanceof Error ? error.message : String(error); } /** * Convert legacy/raw downloader and filesystem errors into the bounded * extraction taxonomy. New extraction code should throw * `VideoSourceExtractionError` directly; this classifier keeps older utility * boundaries safe while they migrate. */ export function classifyVideoExtractionError(error: unknown): VideoSourceExtractionError { if (isVideoSourceExtractionError(error)) return error; const diagnostic = errorText(error); const lowered = diagnostic.toLowerCase(); if (error instanceof UrlDownloadError) { if (error.kind === "cancelled") { return new VideoSourceExtractionError( "cancelled", false, "Video extraction cancelled", diagnostic, ); } if (error.kind === "http_not_found") { return new VideoSourceExtractionError( "download_not_found", false, "Video source was not found", diagnostic, ); } if (error.kind === "http_rejected") { return new VideoSourceExtractionError( "source_rejected", false, "Video source download was rejected", diagnostic, ); } if (error.kind === "invalid_payload") { return new VideoSourceExtractionError( "invalid_media", false, "Video source download returned a non-media payload", diagnostic, ); } if (error.retryable) { return new VideoSourceExtractionError( "download_transient", true, "Video source download failed transiently", diagnostic, ); } return new VideoSourceExtractionError( "internal", false, "Video source download failed internally", diagnostic, ); } if (lowered.includes("cancelled") || lowered.includes("aborted")) { return new VideoSourceExtractionError( "cancelled", false, "Video extraction cancelled", diagnostic, ); } if (lowered.includes("video file not found")) { return new VideoSourceExtractionError( "source_missing", false, "Video source is missing", diagnostic, ); } if ( lowered.includes("only https urls are permitted") || lowered.includes("private/reserved address") || lowered.includes("invalid url") ) { return new VideoSourceExtractionError( "source_rejected", false, "Video source URL is not permitted", diagnostic, ); } const httpStatus = diagnostic.match(/\bHTTP\s+(\d{3})\b/i)?.[1]; if (httpStatus) { const status = Number(httpStatus); if (status === 404 || status === 410) { return new VideoSourceExtractionError( "download_not_found", false, "Video source was not found", diagnostic, ); } if (status === 408 || status === 429 || status >= 500) { return new VideoSourceExtractionError( "download_transient", true, "Video source download failed transiently", diagnostic, ); } return new VideoSourceExtractionError( "source_rejected", false, "Video source download was rejected", diagnostic, ); } if ( lowered.includes("[urldownloader] download timeout") || lowered.includes("[urldownloader] download failed") || lowered.includes("fetch failed") || lowered.includes("network") ) { return new VideoSourceExtractionError( "download_transient", true, "Video source download failed transiently", diagnostic, ); } if (lowered.includes("ffprobe not found")) { return new VideoSourceExtractionError( "ffmpeg_unavailable", false, "FFprobe is unavailable", diagnostic, ); } if (lowered.includes("ffprobe deadline")) { return new VideoSourceExtractionError( "ffmpeg_timeout", true, "Video inspection timed out", diagnostic, ); } if ( lowered.includes("ffprobe") || lowered.includes("failed to parse ffprobe output") || lowered.includes("no video stream found") ) { return new VideoSourceExtractionError( "invalid_media", false, "Video source could not be inspected", diagnostic, ); } return new VideoSourceExtractionError( "internal", false, "Video extraction failed internally", diagnostic, ); } export async function runVideoExtractionWithRetry( operation: () => Promise, options: { signal?: AbortSignal; onRetry?: () => Promise | void; maxTransientRetries?: number; } = {}, ): Promise<{ result: T; retries: number }> { const maxTransientRetries = boundedTransientRetryBudget(options.maxTransientRetries); let retries = 0; for (;;) { if (options.signal?.aborted) { throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled"); } try { return { result: await operation(), retries }; } catch (error) { const classified = classifyVideoExtractionError(error); if (options.signal?.aborted) { throw new VideoSourceExtractionError( "cancelled", false, "Video extraction cancelled", classified.diagnostic, ); } if ( classified.kind === "cancelled" || !classified.retryable || retries >= maxTransientRetries ) { throw classified; } retries += 1; await options.onRetry?.(); } } } export interface ExtractionResult { success: boolean; extracted: ExtractedFrames[]; errors: VideoExtractionFailure[]; totalFramesExtracted: number; durationMs: number; phaseBreakdown: ExtractionPhaseBreakdown; } /** Minimal structural shape for resolving parent/`` media `src`. */ interface MediaSrcEl { getAttribute(name: string): string | null; querySelectorAll(selectors: string): Iterable<{ getAttribute(name: string): string | null }>; } /** * Parent `src`, else a ``. Prefer local paths over http(s) so a * localized sibling wins when another `` failed to download. */ export function resolveMediaElementSrc(el: MediaSrcEl): string | null { const direct = el.getAttribute("src"); if (direct) return direct; let remote: string | null = null; for (const source of el.querySelectorAll("source")) { const src = source.getAttribute("src"); if (!src) continue; if (!/^https?:\/\//i.test(src)) return src; remote ??= src; } return remote; } export function parseVideoElements(html: string): VideoElement[] { const videos: VideoElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); const startCache = new Map(); const visiting = new Set(); const groupsById = audioGroupsById(document); const videoEls = document.querySelectorAll("video"); let autoIdCounter = 0; for (const el of videoEls) { const src = resolveMediaElementSrc(el); if (!src) continue; // Generate a stable ID for videos without one — the producer needs IDs // to track extracted frames and composite them during encoding. // A compiled render document stamps a document-unique render id; prefer it, // because element ids are only unique within one composition file and the // render document inlines many. const id = el.getAttribute(MEDIA_RENDER_ID_ATTR) || el.getAttribute("id") || `hf-video-${autoIdCounter++}`; if (!el.getAttribute("id")) { el.setAttribute("id", id); } const startAttr = el.getAttribute("data-start"); const endAttr = el.getAttribute("data-end"); const durationAttr = el.getAttribute("data-duration"); const hasAudioAttr = el.getAttribute("data-has-audio"); // Resolve data-start, including relative references ("intro", "intro + 2") // to another clip's end — the browser runtime resolves these but a raw // parseFloat here would yield NaN, placing the clip at NaN so it composites // blank in the final render. `startAttr` may be a plain number or a // reference; the resolver handles both. const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0; if (isKnownInactiveTimelineWindow(el, start)) continue; // Derive end from data-end → data-start+data-duration → Infinity (natural duration). // Static compilation cannot always clamp root media because GSAP may supply // the root duration at runtime. The producer passes the resolved timeline // end into frame extraction, which caps the source duration only after the // natural duration is known without rewriting authored timing metadata. let end = 0; const authoredEnd = parseStrictFiniteTimingNumber(endAttr); const authoredDuration = parseStrictFiniteTimingNumber(durationAttr); if (authoredEnd != null) { end = authoredEnd; } else if (authoredDuration != null) { end = start + authoredDuration; } else { end = Infinity; // no explicit bounds — play for the full natural video duration } videos.push({ id, src, start, end, mediaStart: readMediaStart(el), playbackRate: readElementRateSpec(el), loop: el.hasAttribute("loop"), hasAudio: hasAudioAttr === "true" && isAudibleVideoElement(el), ...(isSelfOrAncestorHidden(el) || (isAudibleVideoElement(el) && isMemberGroupHidden(groupsById, el)) ? { hidden: true } : {}), ...(el.hasAttribute(HF_COLOR_GRADING_ATTR) ? { graded: true } : {}), }); } return videos; } export interface ImageElement { id: string; src: string; start: number; end: number; graded?: boolean; } export function hasGradedCanvas(html: string): boolean { return ( parseHTML(unwrapTemplate(html)).document.querySelector(`canvas[${HF_COLOR_GRADING_ATTR}]`) !== null ); } export function parseImageElements(html: string): ImageElement[] { const images: ImageElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); const startCache = new Map(); const visiting = new Set(); const imgEls = document.querySelectorAll("img[src]"); let autoIdCounter = 0; for (const el of imgEls) { const src = el.getAttribute("src"); if (!src) continue; // See parseVideoElements: the stamped render id wins over the authored id. const id = el.getAttribute(MEDIA_RENDER_ID_ATTR) || el.getAttribute("id") || `hf-img-${autoIdCounter++}`; if (!el.getAttribute("id")) { el.setAttribute("id", id); } const startAttr = el.getAttribute("data-start"); const endAttr = el.getAttribute("data-end"); const durationAttr = el.getAttribute("data-duration"); // Resolve relative data-start references (see parseVideoElements) so a // referenced image start doesn't become NaN and drop the image from the render. const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0; let end = 0; const authoredEnd = parseStrictFiniteTimingNumber(endAttr); const authoredDuration = parseStrictFiniteTimingNumber(durationAttr); if (authoredEnd != null) { end = authoredEnd; } else if (authoredDuration != null) { end = start + authoredDuration; } else { end = Infinity; } images.push({ id, src, start, end, ...(el.hasAttribute(HF_COLOR_GRADING_ATTR) ? { graded: true } : {}), }); } return images; } const firstFrameColourCache = new Map>(); /** A source's SPS transfers, read as the trace streams (a run keeps only stderr's tail); null when the read failed. */ function readKeyframeTransfers(videoPath: string, codec: string) { let traced = ""; let rest = ""; const onStderr = (chunk: string) => { const lines = (rest + chunk).split("\n"); rest = lines.pop() ?? ""; for (const line of lines) if (line.includes("transfer_characteristics")) traced += `${line}\n`; }; return inExtractionSlot(() => runFfmpeg([...urlInputArgs(videoPath), ...keyframeTransferArgs(videoPath, codec)], { onStderr, }), ).then((result) => (result.success ? parseKeyframeTransfers(traced + rest) : null)); } /** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */ function readFirstFrameColour(videoPath: string, codec: string) { const identity = mediaFileIdentity(videoPath); const cached = readMetadataCache(firstFrameColourCache, videoPath, identity); if (cached) return cached; // In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others. const probe = inExtractionSlot(() => runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]), ).then(async (result): Promise => { const forget = () => { if (firstFrameColourCache.get(videoPath)?.promise === probe) firstFrameColourCache.delete(videoPath); }; // A failed probe answers this cut only; the next one probes again. if (!result.success) forget(); const first = parseFirstFrameColour(result.stderr); if (!result.success || !readsKeyframeTransfers(first)) return first; const keyframeTransfers = await readKeyframeTransfers(videoPath, codec); if (!keyframeTransfers) forget(); return { ...first, keyframeTransfers: keyframeTransfers ?? [] }; }); writeMetadataCache(firstFrameColourCache, videoPath, identity, probe); return probe; } /** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */ const CHROME_BT601_UNTAGGED_CODECS = new Set(["vp9", "av1"]); function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] { const matrix = metadata.colorSpace?.colorSpace; if (matrix && matrix !== "unknown") return []; const hd = metadata.height >= 720 && !CHROME_BT601_UNTAGGED_CODECS.has(metadata.videoCodec); return [`setparams=colorspace=${hd ? "bt709" : "smpte170m"}`]; } export function sdrCanvasColourFilters(metadata: VideoMetadata, toneMappedToSdr = false): string[] { if (!isHdrColorSpaceUtil(metadata.colorSpace)) { return [...chromeGuessForUntaggedMatrix(metadata), SDR_CANVAS_PASSTHROUGH_FILTER]; } return toneMappedToSdr ? [SDR_CANVAS_PASSTHROUGH_FILTER] : []; } /** * Longest stretch of output one ffmpeg process extracts; longer ranges split into segments * that run side by side. Segments sample the same absolute times as a single pass, so their * frames are byte-identical to it. */ const EXTRACTION_SEGMENT_SECONDS = 120; /** Slowest decode rate the stall clock allows while ffmpeg discards frames from a keyframe up to the seek point. */ const SKIP_DECODE_FPS_FLOOR = 30; async function leadInSecondsUnlessAborted( videoPath: string, seek: number, signal?: AbortSignal, ): Promise { try { return await keyframeGapSeconds(videoPath, seek, signal); } catch (error) { // Aborted: let runFfmpeg report the abort, so a segment stopped by another's failure reads as stopped. if (signal?.aborted) return 0; throw error; } } function stallTimeoutMs(baseMs: number, skippedSeconds: number, sourceFps: number): number { return baseMs + Math.ceil((skippedSeconds * sourceFps * 1000) / SKIP_DECODE_FPS_FLOOR); } const KEEP_END_PAST_A_ONE_TICK_FINAL_FRAME_THROUGH_THE_SNAP = "tpad=stop_mode=clone:stop=1"; /** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */ function undoTickRoundingOnTheSourceFrameGrid( seek: number, lead: number, sourceFps: number, ): string[] { const sinceFileStart = `(PTS*TB+${seek}-${lead})`; return [ ...(sourceFps > 0 ? [KEEP_END_PAST_A_ONE_TICK_FINAL_FRAME_THROUGH_THE_SNAP] : []), "settb=AVTB", `setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`, ]; } // Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts. const EXTRACTION_SLOTS = Math.max(1, Math.floor(usableCpuCount() / 2)); const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS); const TAIL_PROBE_SLOTS = Math.max(1, Number(process.env.HF_TAIL_PROBE_SLOTS) || 16); const inTailProbeSlot = createConcurrencyLimit(TAIL_PROBE_SLOTS); interface SegmentedExtraction { decodeArgs: string[]; filterAndEncodeArgs: (seek: number) => string[]; ffmpegSeek: (mediaTime: number) => number; videoPath: string; startTime: number; fps: number; sourceFps: number; segments: readonly FrameRange[]; gate: typeof inExtractionSlot; runOptions: { signal?: AbortSignal; timeout: number }; onProgress?: ExtractionOptions["onProgress"]; } function framesWrittenReader(onProgress: ExtractionOptions["onProgress"]) { if (!onProgress) return undefined; let seen = 0; const read = ffmpegStatsReader(({ frames }) => { if (frames === undefined || frames <= seen) return; onProgress({ done: frames - seen }); seen = frames; }); return (chunk: string) => read(chunk); } /** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */ function splitFrameRanges( ranges: readonly FrameRange[], segmentFrames: number, frameLimit: number, ): FrameRange[] { const segments: FrameRange[] = []; for (const range of ranges) { const frames = Math.min(range.frames, frameLimit - range.firstFrame); for (let offset = 0; offset < frames; offset += segmentFrames) { segments.push({ firstFrame: range.firstFrame + offset, frames: Math.min(segmentFrames, frames - offset), }); } } return segments; } async function runSegmentedExtraction(job: SegmentedExtraction): Promise { const { decodeArgs, filterAndEncodeArgs, videoPath, startTime, fps, sourceFps, segments, gate } = job; const segmentCount = segments.length; // One failed segment fails the range, so stop the others instead of finishing them. const failed = new AbortController(); const signal = job.runOptions.signal ? AbortSignal.any([job.runOptions.signal, failed.signal]) : failed.signal; const results: RunFfmpegResult[] = []; let next = 0; const worker = async () => { while (next < segmentCount && !signal.aborted) { const index = next++; const { firstFrame, frames } = segments[index]!; const seek = startTime + firstFrame / fps; const result = await gate( async () => runFfmpeg( [ ...decodeArgs, "-noaccurate_seek", "-ss", formatFfmpegSeconds(job.ffmpegSeek(seek)), "-i", videoPath, // Read one frame past the segment and keep exactly its own frames. "-t", formatFfmpegSeconds((frames + 1) / fps), "-frames:v", String(frames), "-start_number", String(firstFrame + 1), ...filterAndEncodeArgs(seek), ], { ...job.runOptions, signal, onStderr: framesWrittenReader(job.onProgress), timeout: stallTimeoutMs( job.runOptions.timeout, await leadInSecondsUnlessAborted(videoPath, seek, signal), sourceFps, ), }, ), signal, ); results[index] = result; if (!result.success) failed.abort(); } }; const workers = Math.min(segmentCount, EXTRACTION_SLOTS); await Promise.all(Array.from({ length: workers }, worker)).catch((error) => { failed.abort(); throw error; }); const ran = results.filter(Boolean); // Report the segment that failed, not the ones stopped because of it. const failure = ran.find((r) => !r.success && r.terminationReason !== "abort") ?? ran.find((r) => !r.success); if (failure) return failure; // A cancel between segments leaves later ones unrun; their frames are missing, not done. if (ran.length < segmentCount) { return { success: false, exitCode: null, stderr: "", durationMs: 0, terminationReason: "abort", }; } return ran[ran.length - 1]!; } export async function extractVideoFramesRange( ...args: Parameters ): Promise { try { return await extractVideoFramesRangeUnredacted(...args); } catch (error) { throw redactUrlQueriesInError(error); } } interface VideoFilterChainInput { metadata: VideoMetadata; format: CacheFrameFormat; ffmpegFps: string; sampleAtOutputFps: boolean; keepsHdrForChrome: boolean; /** The first frame's colour tags; set exactly when HDR is tone-mapped to SDR. */ toneMapFrom?: ToneMapFrameColour; isHdr: boolean; sdrToHdrTransfer?: ExtractionOptions["sdrToHdrTransfer"]; shownBox?: ExtractionOptions["shownBox"]; } /** The one place that decides which source frame a slot shows and how its colour is converted. */ function videoFilterChain(input: VideoFilterChainInput): (seek: number) => string[] { const { metadata, format, ffmpegFps, sampleAtOutputFps, keepsHdrForChrome, toneMapFrom } = input; const toneMappedToSdr = toneMapFrom !== undefined; const head: string[] = []; if (keepsHdrForChrome) head.push("format=nv12"); const tail: string[] = []; if (metadata.sampleAspectRatio) { tail.push("scale=trunc(iw*sar/2)*2:ih", "setsar=1"); } // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms). const sampling = (seek: number): string[] => sampleAtOutputFps ? [ ...(metadata.isVFR ? [] : undoTickRoundingOnTheSourceFrameGrid( seek, videoStartOnMediaTimeline(metadata), metadata.fps, )), "settb=intb/2", "setpts=PTS-lte(TB\\,0.001)", `fps=${ffmpegFps}:start_time=0:round=up`, ] : []; if (input.sdrToHdrTransfer) { // Fps sampling runs before the colorspace remap so only kept frames are converted. The // remap is pointwise, so the result is identical either way for the SDR inputs it applies // to (it never meets nv12, which is HDR-only); revisit if the two ever combine. tail.push(SDR_TO_HDR_COLORSPACE_FILTER); } if (toneMapFrom) { tail.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, toneMapFrom)); } if (toneMappedToSdr || (!input.isHdr && !input.sdrToHdrTransfer)) { tail.push(...sdrCanvasColourFilters(metadata, toneMappedToSdr)); if (format === "jpg") { if (input.shownBox) tail.push(halveToCoverScaleFilter(input.shownBox)); tail.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER); } } return (seek) => [...head, ...sampling(seek), ...tail]; } export interface VideoLayerInput { inputArgs: string[]; filters: string[]; } const VIDEO_LAYER_TAIL_SLACK_SECONDS = 0.5; /** ffmpeg input and filters showing the same frames and colour as extraction; undefined if unsupported. */ export async function buildVideoLayerInput( videoPath: string, startTime: number, duration: number, fps: Fps, canvas: { width: number; height: number }, fit: "fill" | "cover" = "fill", ): Promise { const metadata = await extractMediaMetadata(videoPath); if ( startTime + duration > resolvePlayableVideoDuration(metadata) + VIDEO_LAYER_TAIL_SLACK_SECONDS || isHdrColorSpaceUtil(metadata.colorSpace) || metadata.isVFR || metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec) ) { return undefined; } const chain = videoFilterChain({ metadata, format: "png", ffmpegFps: fpsToFfmpegArg(fps), sampleAtOutputFps: true, keepsHdrForChrome: false, isHdr: false, }); const filters = chain(startTime); if (metadata.width !== canvas.width || metadata.height !== canvas.height) { filters.push( fit === "cover" ? `scale=${canvas.width}:${canvas.height}:force_original_aspect_ratio=increase:flags=bicubic,crop=${canvas.width}:${canvas.height}` : `scale=${canvas.width}:${canvas.height}:flags=bicubic`, ); } filters.push(`tpad=stop_mode=clone:stop_duration=${VIDEO_LAYER_TAIL_SLACK_SECONDS * 2}`); return { inputArgs: [ "-noaccurate_seek", "-ss", formatFfmpegSeconds(ffmpegSeekSeconds(metadata, startTime)), ...urlInputArgs(videoPath), "-i", videoPath, ], filters, }; } async function extractVideoFramesRangeUnredacted( videoPath: string, videoId: string, startTime: number, duration: number, options: ExtractionOptions, signal?: AbortSignal, config?: Partial>, /** Frames go straight here, with no per-videoId subdir (the cache layer's keyed entry). */ outputDirOverride?: string, ): Promise { const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const { outputDir, quality = 95 } = options; const normalizedFps = toFps(options.fps); const fps = fpsToNumber(normalizedFps); const ffmpegFps = fpsToFfmpegArg(normalizedFps); const videoOutputDir = outputDirOverride ?? join(outputDir, mediaIdPathSegment(videoId)); if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true }); let metadata: VideoMetadata; try { metadata = await extractMediaMetadata(videoPath); } catch (error) { throw classifyVideoExtractionError(error); } const playableDuration = resolvePlayableVideoDuration(metadata); if (!(playableDuration > 0)) { throw new VideoSourceExtractionError( "invalid_media", false, "Video source has no positive duration", `Playable video stream duration is ${playableDuration}s`, ); } if (startTime >= playableDuration) { throw new VideoSourceExtractionError( "media_start_out_of_range", false, "Video media start is outside the source duration", `Video media start ${startTime}s is outside playable video duration ${playableDuration}s`, ); } const format = resolveFrameFormat(metadata, options.format); const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`; const outputPattern = frameSequencePath(videoOutputDir, framePattern); // Forced-SDR extraction tone-maps HDR with zscale, the Studio proxy policy. A macOS ffmpeg // without zscale falls back to VideoToolbox, whose frames stay HDR and are tone-mapped by // Chrome instead (darker), and the render warns once. const isHdr = isHdrColorSpaceUtil(metadata.colorSpace); const isMacOS = process.platform === "darwin"; const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale()); const keepsHdrForChrome = isHdr && isMacOS && !toneMappedToSdr; const decodeWithVideoToolbox = keepsHdrForChrome || (isMacOS && !metadata.hasAlpha && !codecMayHaveAlpha(metadata.videoCodec) && (await isFfmpegHwaccelAvailable("videotoolbox"))); const sampleAtOutputFps = !options.finalFrameOnly; if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) { throw new Error("frameRanges extraction needs a constant-frame-rate source"); } const args: string[] = urlInputArgs(videoPath); // Always force the alpha-aware decoder on codecs that can carry alpha. The // alternative — gating on `metadata.hasAlpha` — relies on tag detection that // has at least three known failure modes: case-sensitivity across ffmpeg // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers, // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there // silently strips alpha during decode and the bug doesn't surface until // the rendered video is missing layers. Codec-based default has no such // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes // normally when it isn't. if (codecMayHaveAlpha(metadata.videoCodec)) { args.push("-c:v", decoderForCodec(metadata.videoCodec)); } const decodeArgs = [...args]; const ffmpegSeek = (mediaTime: number) => ffmpegSeekSeconds(metadata, mediaTime); if (options.finalFrameOnly) { // Output-side seek decodes from the start before selecting the final // sample. This is intentionally reserved for the one-frame path: input // seeking is faster, but valid unindexed transports (notably MPEG-TS with // a negative timestamp base) can seek to EOF and emit zero frames. args.push( "-i", videoPath, "-ss", formatFfmpegSeconds(Math.max(0, ffmpegSeek(startTime))), "-frames:v", "1", ); } else { // An accurate seek drops the frame on screen at startTime; the fps sampling below needs it. if (sampleAtOutputFps) args.push("-noaccurate_seek"); args.push( "-ss", formatFfmpegSeconds(ffmpegSeek(startTime)), "-i", videoPath, "-t", formatFfmpegSeconds(duration), ); } const frameColour = toneMappedToSdr ? await readFirstFrameColour(videoPath, metadata.videoCodec) : undefined; const filtersFor = videoFilterChain({ metadata, format, ffmpegFps, sampleAtOutputFps, keepsHdrForChrome, toneMapFrom: frameColour, isHdr, sdrToHdrTransfer: options.sdrToHdrTransfer, shownBox: options.shownBox, }); const encodeArgs = ["-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"]; // Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files. if (format === "png") encodeArgs.push("-compression_level", "1"); encodeArgs.push("-y", outputPattern); const runOptions = { signal, timeout: ffmpegProcessTimeout }; const filterArgsFor = (seek: number): string[] => { const filters = filtersFor(seek); return filters.length > 0 ? ["-vf", filters.join(",")] : []; }; const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps); const windowFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR); const framesBeforeSourceEnd = extractionFrameCountForDuration( playableDuration - startTime, normalizedFps, false, ); const totalFrames = metadata.isVFR ? windowFrames : Math.min(windowFrames, framesBeforeSourceEnd); // A `-t` window ends at the nearest slot; segments read a frame past and keep exactly their own. // A decimal rate counts zero frames here and keeps the `-t` window. const extract = (hwaccel: string[]) => sampleAtOutputFps && (options.frameRanges || totalFrames > 0) ? runSegmentedExtraction({ decodeArgs: [...hwaccel, ...decodeArgs], filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs], ffmpegSeek, videoPath, startTime, fps, sourceFps: metadata.fps, gate: options.tailProbe ? inTailProbeSlot : inExtractionSlot, segments: splitFrameRanges( options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }], segmentFrames, totalFrames > 0 ? totalFrames : Number.POSITIVE_INFINITY, ), runOptions, onProgress: options.onProgress, }) : inExtractionSlot( async () => runFfmpeg([...hwaccel, ...args, ...filterArgsFor(startTime), ...encodeArgs], { ...runOptions, onStderr: framesWrittenReader(options.onProgress), timeout: stallTimeoutMs( ffmpegProcessTimeout, options.finalFrameOnly ? startTime : await leadInSecondsUnlessAborted(videoPath, startTime, signal), metadata.fps, ), }), signal, ); options.onProgress?.({ planned: options.frameRanges ? options.frameRanges.reduce((sum, range) => sum + range.frames, 0) : totalFrames, }); let processResult = await extract(decodeWithVideoToolbox ? ["-hwaccel", "videotoolbox"] : []); const ffmpegExitedFailing = processResult.terminationReason === "exit" && !processResult.success && !processResult.failureReason; const videoToolboxWasOptional = decodeWithVideoToolbox && !keepsHdrForChrome; if (ffmpegExitedFailing && videoToolboxWasOptional && !DISK_FULL.test(processResult.stderr)) processResult = await extract([]); if (processResult.failureReason === "external_interruption") { throw new VideoSourceExtractionError( "external_interruption", true, "Video frame extraction interrupted by host lifecycle", `FFmpeg exited with code ${processResult.exitCode}: ${processResult.stderr.slice(-500)}`, ); } if (processResult.terminationReason === "abort") { throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled"); } if (processResult.terminationReason === "spawn_error") { throw classifyFfmpegSpawnError(processResult.error, processResult.stderr); } if (!processResult.success) { // With the SDR-to-HDR remap folded into this pass, a filter failure // (e.g. an ffmpeg built without the colorspace filter) would otherwise // surface as a generic extract error and the operator has to grep the // filter chain to learn it was the HDR conversion. Attribute it. const hdrPrefix = options.sdrToHdrTransfer ? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): ` : ""; const timedOut = processResult.terminationReason === "inactivity"; const timeoutSuffix = timedOut ? ` (no new frame for ${ffmpegProcessTimeout} ms)` : ""; const diagnostic = `${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ` + processResult.stderr.slice(-500); const ffmpegReason = ffmpegReasonFrom(processResult.stderr, [ videoPath, basename(videoPath), videoOutputDir, ]); if (timedOut) { throw new VideoSourceExtractionError( "ffmpeg_timeout", true, "Video frame extraction timed out", diagnostic, ffmpegReason, ); } if (DISK_FULL.test(processResult.stderr)) { throw new VideoSourceExtractionError( "disk_full", false, "The disk filled up while video frames were extracted", diagnostic, ffmpegReason, ); } const transientIo = /resource temporarily unavailable|device or resource busy|input\/output error/i.test( processResult.stderr, ); throw new VideoSourceExtractionError( transientIo ? "ffmpeg_transient" : "ffmpeg_failed", transientIo, transientIo ? "Video frame extraction hit a transient I/O failure" : "Video source could not be decoded", diagnostic, ffmpegReason, ); } const framePaths = framePathsFromDirectory(videoOutputDir, format, !!options.frameRanges); if (framePaths.size === 0 && duration > 0) { throw new VideoSourceExtractionError( "zero_output", false, "Video source produced no decodable frames", `FFmpeg exited successfully but produced no frames (start=${startTime}, duration=${duration})`, ); } // Post-extraction reconciliation: when FFmpeg delivers <=60% of the // frames predicted by the probed duration, the duration is likely // inflated (container duration includes a longer audio track). Correct // it so downstream coverage accounting uses the actual video extent. if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) { const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR); if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) { const correctedStreamDuration = startTime + framePaths.size / fps - (metadata.videoStreamLeadSeconds ?? 0); if (correctedStreamDuration < metadata.videoStreamDurationSeconds) { metadata.videoStreamDurationSeconds = correctedStreamDuration; } } } return { videoId, srcPath: videoPath, outputDir: videoOutputDir, framePattern, fps, totalFrames: framePaths.size, metadata, framePaths, }; } const TRANSIENT_FFMPEG_SPAWN_CODES = new Set(["EAGAIN", "EMFILE", "ENFILE"]); export function classifyFfmpegSpawnError(error: unknown, stderr = ""): VideoSourceExtractionError { const code = typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" ? error.code : ""; if (code === "ENOENT") { return new VideoSourceExtractionError( "ffmpeg_unavailable", false, "FFmpeg is unavailable", "[FFmpeg] ffmpeg not found", ); } const diagnostic = error instanceof Error ? error.message : stderr; const retryable = TRANSIENT_FFMPEG_SPAWN_CODES.has(code); return new VideoSourceExtractionError( retryable ? "ffmpeg_transient" : "ffmpeg_failed", retryable, retryable ? "FFmpeg could not be started due to transient resource pressure" : "FFmpeg could not be started", diagnostic, ); } /** * Resolve the used-segment duration for a video, falling back to the source's * natural duration when the caller hasn't specified bounds (end=Infinity) or * the bounds are nonsensical (end<=start). */ function resolveSegmentDuration( requested: number, mediaStart: number, sourceDuration: number, ): number { if (Number.isFinite(requested) && requested > 0) return requested; const sourceRemaining = sourceDuration - mediaStart; return sourceRemaining > 0 ? sourceRemaining : sourceDuration; } /** * Return the range that can actually produce video frames. * * Container duration may include a longer audio stream or mux padding. Using * it for video extraction planning can reserve raw-frame scratch for seconds * where no video frames exist. `extractMediaMetadata` already falls back to * the container duration when ffprobe omits the stream duration; keep the * explicit fallback here for callers supplying older/manual metadata. */ export { ffmpegSeekSeconds, resolvePlayableVideoDuration, videoStartOnMediaTimeline }; export interface TimelineExtractionWindow { compositionStart: number; mediaStart: number; durationSeconds: number; /** Visible duration on the authored composition timeline after retiming. */ timelineDurationSeconds?: number; /** * Preserve the authored timeline origin and mediaStart for lookup. This is * required when a looped visible interval crosses a source boundary and * still needs modulo phase against the complete extracted source cycle. */ preserveTimelinePhase?: boolean; /** * Keep the authored end while rebasing start/mediaStart to the extracted * source suffix. Non-looping lookup then holds the suffix's final frame * through the remainder of the authored slot. */ preserveTimelineEnd?: boolean; /** This window reaches a held tail and must be checked against the actual * final decoded-frame timestamp before extraction. */ ensureFinalFrame?: boolean; /** Source timestamp used by FFmpeg when it differs from the logical lookup * mediaStart (the one-frame held-tail representation). */ extractionMediaStart?: number; /** FFmpeg emits one decoded frame; lookup then holds that frame. */ finalFrameOnly?: boolean; } type TimelineWindowVideo = Pick & Partial> & Partial>; function canHoldFinalFramePastEof(video: TimelineWindowVideo): boolean { const timelineDuration = video.end - video.start; return !video.loop && Number.isFinite(timelineDuration) && timelineDuration > 0; } // Logical duration assigned to a one-frame held-tail representation. This is // deliberately below any supported output frame interval: coverage expects // one frame, while FFmpeg seeks to the separately probed real frame timestamp. const FINAL_FRAME_LOGICAL_DURATION_SECONDS = 1e-6; export function isVideoPastTimelineEnd(video: Pick, timelineEnd: number) { return video.start >= timelineEnd; } export function alignClipStartToFirstOutputFrame(video: VideoElement, fps: number): void { if (video.loop || !Number.isFinite(video.start) || video.start < 0 || !(fps > 0)) return; const position = exportClipWindow(video.start, Infinity, fps).start * fps; const firstFrame = Math.abs(position - Math.round(position)) < 1e-6 ? Math.round(position) : Math.ceil(position); const shift = firstFrame / fps - video.start; // Float dust (a sum landing a hair off the grid) snaps the start; it never reaches mediaStart. if (Math.abs(shift) < 1e-9) { video.start = firstFrame / fps; return; } const playbackRate = normalizeRateSpec(video.playbackRate); video.mediaStart += sourceTimeAt(playbackRate, shift); video.playbackRate = shiftRateLane(playbackRate, shift); video.start += shift; } /** Move a clip to its extraction window; a rate lane is shifted so it keeps integrating from the new origin. */ export function rebaseVideoToWindow(video: VideoElement, window: TimelineExtractionWindow): void { if (window.preserveTimelinePhase) return; // A trim landing on the timeline origin can come out as 1e-16 and drop the first frame. const start = Math.abs(window.compositionStart) < 1e-9 ? 0 : window.compositionStart; video.playbackRate = shiftRateLane(normalizeRateSpec(video.playbackRate), start - video.start); video.start = start; if (!window.preserveTimelineEnd) { video.end = start + (window.timelineDurationSeconds ?? window.durationSeconds); } video.mediaStart = window.mediaStart; } /** * Intersect an authored slot with the render timeline, then select the * smallest playable source range that preserves timeline lookup semantics. * * A finite authored slot can outlive the source. In that case FFmpeg should * still extract at most one source range: lookup either wraps that range for * loops or holds its final frame for non-looping video. Keeping the authored * timeline origin separate from the extracted range is what makes both * behaviours survive the source-duration cap. */ export function resolveTimelineExtractionWindow( video: TimelineWindowVideo, resolvedDuration: number, timelineEnd?: number, sourceDuration?: number, ): TimelineExtractionWindow { const playbackRate = normalizeRateSpec(video.playbackRate); const withTimelineDuration = ( window: TimelineExtractionWindow, timelineDurationSeconds: number, ): TimelineExtractionWindow => playbackRate === 1 ? window : { ...window, timelineDurationSeconds }; if (timelineEnd === undefined) { return withTimelineDuration( { compositionStart: video.start, mediaStart: video.mediaStart, durationSeconds: sourceTimeAt(playbackRate, resolvedDuration), }, resolvedDuration, ); } if (!Number.isFinite(timelineEnd)) { throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`); } const compositionStart = Math.max(0, video.start); const trimmedPreroll = compositionStart - video.start; const trimmedSourcePreroll = sourceTimeAt(playbackRate, trimmedPreroll); const timelineDuration = Math.max(0, timelineEnd - compositionStart); // Infinity means "natural source duration", not an authored infinite slot. // Explicit finite slots may outlive the source (loop or held tail), while an // omitted duration remains source-bounded exactly like the browser runtime. const resolvedVisibleDuration = resolvedDuration - trimmedPreroll; const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration)); const visibleSourceDuration = typeof playbackRate === "number" ? visibleDuration * playbackRate : sourceTimeAt(playbackRate, trimmedPreroll + visibleDuration) - trimmedSourcePreroll; let mediaStart = video.mediaStart + trimmedSourcePreroll; if (visibleDuration > 0 && sourceDuration !== undefined) { const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart); if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) { const phaseOffset = trimmedSourcePreroll % sourceRemaining; const phaseRemaining = sourceRemaining - phaseOffset; // Keep a complete cycle on equality too, otherwise a rebased suffix would // wrap to its own first frame instead of the source cycle's first frame. if (visibleSourceDuration >= phaseRemaining) { return withTimelineDuration( { compositionStart: video.start, mediaStart: video.mediaStart, durationSeconds: sourceRemaining, preserveTimelinePhase: true, }, visibleDuration, ); } mediaStart = video.mediaStart + phaseOffset; } else if (sourceRemaining > 0) { const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedSourcePreroll); if (visibleSourceDuration <= sourceVisibleAfterPreroll) { return withTimelineDuration( { compositionStart, mediaStart, durationSeconds: visibleSourceDuration, }, visibleDuration, ); } // The visible interval enters (or is entirely inside) the held tail. // Extract the visible source suffix. If preroll is already at/past the // final decoded timestamp, the async resolver below replaces this tiny // provisional suffix with one exact final frame. const extractionDuration = Math.min( sourceRemaining, Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS), ); const extractionOffset = sourceRemaining - extractionDuration; return withTimelineDuration( { compositionStart: video.start + timeAtSourceTime(playbackRate, extractionOffset), mediaStart: video.mediaStart + extractionOffset, durationSeconds: extractionDuration, preserveTimelineEnd: true, ensureFinalFrame: true, }, visibleDuration, ); } else if (canHoldFinalFramePastEof(video)) { const logicalDuration = Math.min(sourceDuration, FINAL_FRAME_LOGICAL_DURATION_SECONDS); return withTimelineDuration( { compositionStart: video.start, mediaStart: sourceDuration - logicalDuration, durationSeconds: logicalDuration, preserveTimelineEnd: true, ensureFinalFrame: true, }, visibleDuration, ); } } return withTimelineDuration( { compositionStart, mediaStart, durationSeconds: visibleSourceDuration, }, visibleDuration, ); } /** * Replace a held-tail suffix that starts at/after the final decoded timestamp * with one exact frame. This keeps raw HDR scratch O(one frame) without * assuming a one-second seek window contains a CFR/VFR timestamp. */ export async function resolveFinalFrameExtractionWindow( videoPath: string, video: TimelineWindowVideo, metadata: VideoMetadata, window: TimelineExtractionWindow, signal?: AbortSignal, ): Promise { if (!window.ensureFinalFrame) return window; const playableDuration = resolvePlayableVideoDuration(metadata); const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0; const lastInstantTheBrowserPlays = playableDuration - FINAL_FRAME_LOGICAL_DURATION_SECONDS; const finalFrameTimestamp = Math.min( lastInstantTheBrowserPlays, videoStartOnMediaTimeline(metadata) + (await extractFinalVideoFrameTimestamp( videoPath, { videoStreamDurationSeconds: playableDuration - videoStreamLead, videoStreamStartSeconds: metadata.videoStreamStartSeconds, }, signal, )), ); if (window.mediaStart < finalFrameTimestamp - 1e-9) return window; const sourceRemaining = playableDuration - video.mediaStart; const logicalDuration = Math.min( Math.max(sourceRemaining, window.durationSeconds), FINAL_FRAME_LOGICAL_DURATION_SECONDS, ); return { compositionStart: Math.max(0, video.start), mediaStart: playableDuration - logicalDuration, extractionMediaStart: finalFrameTimestamp, durationSeconds: logicalDuration, ...(window.timelineDurationSeconds !== undefined ? { timelineDurationSeconds: window.timelineDurationSeconds } : {}), preserveTimelineEnd: true, finalFrameOnly: true, }; } /** Resolve source duration first, then intersect it with the render timeline. */ export function resolveVideoExtractionWindow( video: TimelineWindowVideo, metadata: VideoMetadata, timelineEnd?: number, ): TimelineExtractionWindow { const playableDuration = resolvePlayableVideoDuration(metadata); if (!(playableDuration > 0)) { throw new VideoSourceExtractionError( "invalid_media", false, "Video source has no positive duration", `Playable video stream duration is ${playableDuration}s`, ); } const requestedTimelineDuration = video.end - video.start; const heldPastEof = video.mediaStart >= playableDuration && canHoldFinalFramePastEof(video); if (video.mediaStart >= playableDuration && !heldPastEof) { throw new VideoSourceExtractionError( "media_start_out_of_range", false, "Video media start is outside the source duration", `Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`, ); } const playbackRate = normalizeRateSpec(video.playbackRate); const resolvedDuration = Number.isFinite(requestedTimelineDuration) && requestedTimelineDuration > 0 ? requestedTimelineDuration : timeAtSourceTime( playbackRate, resolveSegmentDuration(requestedTimelineDuration, video.mediaStart, playableDuration), ); return resolveTimelineExtractionWindow( video, resolvedDuration, timelineEnd ?? (heldPastEof ? video.end : undefined), playableDuration, ); } export function resolveVideoExtractionDuration( video: TimelineWindowVideo, metadata: VideoMetadata, timelineEnd?: number, ): number { return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds; } /** Codecs whose bitstream can carry alpha: decoded with the alpha-aware decoder whether or not a tag says so. */ const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]); /** Alpha can hide from ffprobe: PNG unless a decoded frame had no alpha; ProRes alpha shows in pix_fmt. */ const SIDECAR_ALPHA_CODECS = new Set(["vp9", "vp8"]); export function codecMayHaveAlpha(codec: string | undefined): boolean { return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase()); } export function decoderForCodec(codec: string | undefined): string { return alphaDecoderFor(codec) ?? (codec ?? "").toLowerCase(); } export function resolveFrameFormat( metadata: VideoMetadata, requested?: VideoFrameFormat, ): CacheFrameFormat { const sidecar = SIDECAR_ALPHA_CODECS.has((metadata.videoCodec ?? "").toLowerCase()); if (metadata.hasAlpha || (sidecar && metadata.alphaPlaneDecoded !== false)) return "png"; if (requested === "png" || requested === "jpg") return requested; return "jpg"; } type PreparedExtraction = { video: VideoElement; videoPath: string; index: number; metadata: VideoMetadata; videoDuration: number; extractionMediaStart: number; finalFrameOnly: boolean; format: CacheFrameFormat; sdrToHdrTransfer?: HdrTransfer; hdrToSdrTransformKey?: string; dedupeKey: string; }; type CacheMissTarget = { entry: CacheEntry; srcPath: string; }; type UniqueExtractionMiss = { work: PreparedExtraction; cacheTarget?: CacheMissTarget; }; type SupersetMemberPlan = { miss: UniqueExtractionMiss; offsetFrames: number; }; type SupersetGroupPlan = { groupId: string; baseStart: number; unionDuration: number; members: SupersetMemberPlan[]; }; function extractedFramesFromDirectory( work: PreparedExtraction, outputDir: string, srcPath: string, fps: number, ): ExtractedFrames { const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${work.format}`; const framePaths = framePathsFromDirectory(outputDir, work.format); return { videoId: work.video.id, srcPath, outputDir, framePattern, fps, totalFrames: framePaths.size, metadata: work.metadata, framePaths, }; } function frameFileName(frameNumber: number, format: CacheFrameFormat): string { return `${FRAME_FILENAME_PREFIX}${String(frameNumber).padStart(5, "0")}.${format}`; } function linkOrCopyFrame(src: string, dest: string): void { try { linkSync(src, dest); } catch { copyFileSync(src, dest); } } function supersetGroupingKey(work: PreparedExtraction, fps: number): string { return [ work.videoPath, String(fps), work.format, work.sdrToHdrTransfer ?? "", work.hdrToSdrTransformKey ?? "", work.finalFrameOnly ? "final" : "range", ].join("\0"); } function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean { const frames = offsetSeconds * fps; return Math.abs(frames - Math.round(frames)) <= 1e-4; } function windowsOverlapOrTouch(misses: UniqueExtractionMiss[], baseStart: number): boolean { const unionEnd = Math.max( ...misses.map(({ work }) => work.video.mediaStart + work.videoDuration), ); const unionDuration = unionEnd - baseStart; const summedDuration = misses.reduce((sum, { work }) => sum + work.videoDuration, 0); return unionDuration > 0 && unionDuration <= summedDuration + 1e-9; } function buildSupersetGroup( groupId: string, misses: UniqueExtractionMiss[], fps: number, ): SupersetGroupPlan | null { if (misses.length < 2) return null; if (misses.some(({ work }) => work.finalFrameOnly)) return null; // VFR ranges round their frame counts up, so a union extraction's slices // are not proven to match independently sought member ranges. Keep them direct. if (misses.some(({ work }) => work.metadata.isVFR)) return null; const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart)); if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) { return null; } if (!windowsOverlapOrTouch(misses, baseStart)) return null; const unionEnd = Math.max( ...misses.map(({ work }) => work.video.mediaStart + work.videoDuration), ); return { groupId, baseStart, unionDuration: unionEnd - baseStart, members: misses.map((miss) => ({ miss, offsetFrames: Math.round((miss.work.video.mediaStart - baseStart) * fps), })), }; } /** * Partition one source's misses into overlap-connected components: sort by * window start and cut wherever the next window starts past the running end. * Without this, one disjoint outlier trim (e.g. [100..105] next to three * overlapping trims at [0..11]) fails the union<=sum check for the whole * bucket and every trim falls back to direct extraction. */ function overlapClusters(misses: UniqueExtractionMiss[]): UniqueExtractionMiss[][] { const sorted = [...misses].sort((a, b) => a.work.video.mediaStart - b.work.video.mediaStart); const clusters: UniqueExtractionMiss[][] = []; let current: UniqueExtractionMiss[] = []; let currentEnd = -Infinity; for (const miss of sorted) { const start = miss.work.video.mediaStart; const end = start + miss.work.videoDuration; if (current.length > 0 && start > currentEnd + 1e-9) { clusters.push(current); current = []; currentEnd = -Infinity; } current.push(miss); currentEnd = Math.max(currentEnd, end); } if (current.length > 0) clusters.push(current); return clusters; } function planSupersetGroups( misses: UniqueExtractionMiss[], fps: number, ): { groups: SupersetGroupPlan[]; direct: UniqueExtractionMiss[] } { const bySource = new Map(); for (const miss of misses) { const key = supersetGroupingKey(miss.work, fps); bySource.set(key, [...(bySource.get(key) ?? []), miss]); } const groups: SupersetGroupPlan[] = []; const direct: UniqueExtractionMiss[] = []; let groupIndex = 0; for (const groupMisses of bySource.values()) { for (const cluster of overlapClusters(groupMisses)) { const group = buildSupersetGroup(`__superset-${groupIndex}`, cluster, fps); if (group) { groups.push(group); groupIndex += 1; } else { direct.push(...cluster); } } } return { groups, direct }; } function sliceSupersetMember( member: SupersetMemberPlan, superset: ExtractedFrames, outputDir: string, fps: number, configuredFps: FpsInput, ): ExtractedFrames { const { work } = member.miss; rmSync(outputDir, { recursive: true, force: true }); mkdirSync(outputDir, { recursive: true }); // Sample-time correctness: member frame k uses superset frame // offset_i + k, so its source time is // baseStart + (offset_i + k) / fps = mediaStart_i + k / fps. // The frame-alignment precondition is what makes offset_i integral. const requestedFrames = extractionFrameCountForDuration( work.videoDuration, configuredFps, work.metadata.isVFR, ); const availableFrames = Math.max(0, superset.totalFrames - member.offsetFrames); const frameCount = Math.min(requestedFrames, availableFrames); for (let i = 0; i < frameCount; i += 1) { const sourceFrame = superset.framePaths.get(member.offsetFrames + i); if (!sourceFrame) throw new Error(`superset frame ${member.offsetFrames + i} missing`); linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format))); } return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps); } export async function extractAllVideoFrames( videos: VideoElement[], baseDir: string, options: ExtractionOptions, signal?: AbortSignal, config?: Partial< Pick >, compiledDir?: string, ): Promise { if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) { throw new Error( `Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`, ); } const configuredFps = toFps(options.fps); const fps = fpsToNumber(configuredFps); const fpsKey = fpsToFfmpegArg(configuredFps); const startTime = Date.now(); const extracted: ExtractedFrames[] = []; const errors: VideoExtractionFailure[] = []; let totalFramesExtracted = 0; const breakdown: ExtractionPhaseBreakdown = { resolveMs: 0, cachePublishFailures: 0, cacheGcEvictions: 0, cacheGcBytesFreed: 0, cacheAgedPartialsCleared: 0, hdrProbeMs: 0, hdrPreflightMs: 0, hdrPreflightCount: 0, vfrProbeMs: 0, vfrPreflightMs: 0, vfrPreflightCount: 0, extractMs: 0, cacheHits: 0, cacheMisses: 0, transientRetries: 0, }; const recordTransientRetries = (count: number): void => { breakdown.transientRetries = (breakdown.transientRetries ?? 0) + count; }; // Phase 1: Resolve paths and download remote videos const phase1Start = Date.now(); const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = []; // Dedupe missing-src warnings: a composition with N