// fallow-ignore-file unused-class-member code-duplication complexity
/**
* Video Frame Extractor Service
*
* Pre-extracts video frames using FFmpeg for frame-accurate rendering.
* Videos are replaced with
elements during capture.
*/
import { usableCpuCount } from "./systemMemory.js";
import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./mediaHidden.js";
import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs";
import { basename, isAbsolute, join, normalize, relative, win32 } from "path";
import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";
import { parseHTML } from "linkedom";
import { HF_COLOR_GRADING_ATTR } from "@hyperframes/core";
import { isPathInside } from "@hyperframes/parsers/asset-paths";
import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";
import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha";
import {
MEDIA_RENDER_ID_ATTR,
isAudibleVideoElement,
fpsToFfmpegArg,
fpsToNumber,
MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
normalizeRateSpec,
readElementRateSpec,
shiftRateLane,
sourceTimeAt,
timeAtSourceTime,
type RateSpec,
exportClipWindow,
hasClipStarted,
isInClipWindow,
mediaElementTime,
parseStrictFiniteTimingNumber,
readMediaStart,
toFps,
type Fps,
type FpsInput,
firstFrameColourArgs,
keyframeTransferArgs,
parseKeyframeTransfers,
type ToneMapFrameColour,
hdrToSdrToneMapFilter,
parseFirstFrameColour,
readsKeyframeTransfers,
} from "@hyperframes/core";
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
import {
extractFinalVideoFrameTimestamp,
extractMediaMetadata,
ffmpegSeekSeconds,
formatFfmpegSeconds,
keyframeGapSeconds,
mediaFileIdentity,
readMetadataCache,
readStoredFinalFrame,
resolvePlayableVideoDuration,
videoStartOnMediaTimeline,
writeMetadataCache,
type MediaProbeCacheEntry,
type VideoMetadata,
} from "../utils/ffprobe.js";
import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js";
import {
analyzeCompositionHdr,
isHdrColorSpace as isHdrColorSpaceUtil,
type HdrTransfer,
} from "../utils/hdr.js";
import {
downloadToTemp,
isHttpUrl,
safeDownloadUrlIdentity,
UrlDownloadError,
writeUrlDownloadTelemetry,
} from "../utils/urlDownloader.js";
import { ffmpegStatsReader, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import {
isFfmpegFilterAvailable,
isFfmpegHwaccelAvailable,
} from "../utils/psnrFilterAvailability.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
import { mediaIdPathSegment } from "../utils/mediaPathSegment.js";
import {
FRAME_FILENAME_PREFIX,
gcExtractionCache,
gcSweepDue,
lookupCacheEntry,
partialCacheEntryDir,
publishCacheEntry,
readContentSha256,
readKeyStat,
rehydrateCacheEntry,
touchCacheEntry,
type CacheEntry,
type CacheFrameFormat,
} from "./extractionCache.js";
import { framePathsFromDirectory, frameSequencePath } from "./extractedFrameIndex.js";
export { resolveProjectRelativeSrc };
export interface VideoElement {
id: string;
src: string;
start: number;
end: number;
mediaStart: number;
playbackRate?: RateSpec;
loop: boolean;
hasAudio: boolean;
hidden?: boolean;
graded?: boolean;
}
export interface ExtractedFrames {
videoId: string;
srcPath: string;
outputDir: string;
framePattern: string;
fps: number;
totalFrames: number;
metadata: VideoMetadata;
framePaths: Map;
deferredRange?: DeferredFrameRange;
/**
* True when the extractor owns `outputDir` and cleanup should rm it when
* the render ends. Cache hits set this to false so the shared entry isn't
* deleted by a single render's cleanup — the cache dir is owned by the
* caller's gc policy, not any one render.
*/
ownedByLookup?: boolean;
}
/**
* The single source of truth for the source-video frame-extraction allow-list.
* The CLI flag parser, the producer HTTP server, and the distributed-config
* validator all validate against this same set via {@link isVideoFrameFormat}
* so the boundaries can't drift when a new format is added.
*/
export const VIDEO_FRAME_FORMATS = ["auto", "jpg", "png"] as const;
export type VideoFrameFormat = (typeof VIDEO_FRAME_FORMATS)[number];
/** Runtime guard for {@link VideoFrameFormat} over an untrusted value. */
export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat {
return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value);
}
const SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES = 1e-4;
/**
* Frames a window gets: one per output slot it covers (CFR), or rounded up (VFR).
* Superset slicing and the coverage gate share it, so they never disagree by a frame.
*/
export function extractionFrameCountForDuration(
durationSeconds: number,
fps: FpsInput,
isVFR: boolean,
): number {
if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) return 0;
// FFmpeg receives `String(durationSeconds)` and parses at microsecond
// precision. Derive the integer microseconds from that same decimal text:
// multiplying the binary float first is not equivalent (`2.05 * 1e6` is
// 2049999.9999999998 in JS and would incorrectly truncate one microsecond).
const serialized = String(durationSeconds).toLowerCase();
const decimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serialized);
if (!decimal) return 0;
const whole = decimal[1] ?? "0";
const fraction = decimal[2] ?? "";
const exponent = Number.parseInt(decimal[3] ?? "0", 10);
const digits = BigInt(`${whole}${fraction}`);
const microsecondScale = exponent + 6 - fraction.length;
const microseconds =
microsecondScale >= 0
? digits * 10n ** BigInt(microsecondScale)
: digits / 10n ** BigInt(-microsecondScale);
// Keep the frame-boundary calculation rational too. Converting the exact
// microseconds back to a binary float recreates the same problem at .5-frame
// boundaries (`2.05 * 30` is 61.49999999999999 in JS).
let fpsNumerator: bigint;
let fpsDenominator: bigint;
if (typeof fps === "object") {
if (
!Number.isSafeInteger(fps.num) ||
!Number.isSafeInteger(fps.den) ||
fps.num <= 0 ||
fps.den <= 0
) {
return 0;
}
fpsNumerator = BigInt(fps.num);
fpsDenominator = BigInt(fps.den);
} else {
if (!Number.isFinite(fps) || fps <= 0) return 0;
// Number-only callers retain their decimal FFmpeg argument exactly. The
// production render path supplies Fps, so NTSC rates never round-trip
// through `String(30000 / 1001)` here.
const serializedFps = String(fps).toLowerCase();
const fpsDecimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serializedFps);
if (!fpsDecimal) return 0;
const fpsWhole = fpsDecimal[1] ?? "0";
const fpsFraction = fpsDecimal[2] ?? "";
const fpsExponent = Number.parseInt(fpsDecimal[3] ?? "0", 10);
const fpsDigits = BigInt(`${fpsWhole}${fpsFraction}`);
const fpsScale = fpsExponent - fpsFraction.length;
fpsNumerator = fpsScale >= 0 ? fpsDigits * 10n ** BigInt(fpsScale) : fpsDigits;
fpsDenominator = fpsScale >= 0 ? 1n : 10n ** BigInt(-fpsScale);
}
const frameNumerator = microseconds * fpsNumerator;
const frameDenominator = 1_000_000n * fpsDenominator;
const slotsPerTolerance = BigInt(Math.round(1 / SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES));
const frameCount = isVFR
? (frameNumerator + frameDenominator - 1n) / frameDenominator
: (slotsPerTolerance * (frameNumerator + frameDenominator) - frameDenominator - 1n) /
(slotsPerTolerance * frameDenominator);
const frames = Number(frameCount);
return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER);
}
/** The exact rate behind a probed one (rounded to 0.01): a whole number or its 1000/1001 NTSC form. */
function exactSourceRate(probed: number): number | null {
const nominal = Math.round(probed * 1.001);
return (
[nominal, (nominal * 1000) / 1001].find((rate) => Math.abs(rate - probed) <= 0.005 + 1e-9) ??
null
);
}
/** Slots of a window reaching the source's end that hold a frame, from the stored final frame; null if unsure. */
export function tailFramesFromFinalFrame(input: {
startTime: number;
firstFrame: number;
tailFrames: number;
outputFps: number;
sourceFps: number;
finalFrameSeconds: number;
streamLeadSeconds: number;
lastPacketSeconds?: number;
}): number | null {
const sourceRate = exactSourceRate(input.sourceFps);
if (!sourceRate || !(input.outputFps > 0)) return null;
// A late-starting stream ends where its last packet does, and ffmpeg shortens the last frame
// to a short packet in ways the stored facts cannot predict: trust only a full-frame packet.
const lead = input.streamLeadSeconds;
if (lead !== 0 && !(Math.abs((input.lastPacketSeconds ?? NaN) - 1 / sourceRate) <= 1e-4)) {
return null;
}
const seek = input.startTime + input.firstFrame / input.outputFps;
const slots = (lead + input.finalFrameSeconds + 1 / sourceRate - seek) * input.outputFps;
const distanceToSlot = Math.abs(slots - Math.round(slots));
// ffprobe prints the final time to 1 us; a slot closer than 50 us/fps to the end is a tie, up to 500 us/fps unknown.
if (distanceToSlot >= 5e-5 && distanceToSlot < 5e-4) return null;
const count = Math.min(input.tailFrames, Math.ceil(slots - 5e-5));
return count > 0 ? count : null;
}
export interface FrameRange {
firstFrame: number;
frames: number;
}
/** The window a deferred extraction samples; frame k is the source frame on screen at `startTime + k / fps`. */
export interface DeferredFrameRange {
startTime: number;
durationSeconds: number;
format: "jpg" | "png";
}
export interface ExtractionOptions {
/** Exact configured rate. Rational rates are passed to FFmpeg verbatim. */
fps: FpsInput;
outputDir: string;
quality?: number;
format?: VideoFrameFormat;
sdrToHdrTransfer?: HdrTransfer;
toneMapHdrToSdr?: boolean;
/** Extract exactly one frame at `startTime`. Used only after ffprobe has
* resolved the actual final decoded-frame timestamp for a held tail. */
finalFrameOnly?: boolean;
/**
* Absolute composition/timeline end in seconds. Applied only after source
* metadata resolves open-ended/natural-duration media. Invisible negative
* preroll is trimmed while advancing mediaStart to preserve source alignment.
*/
timelineEnd?: number;
/**
* Bounded per-source FFmpeg retries. Default 0 preserves stable behavior;
* the producer may canary at most one retry after observing typed failures.
*/
maxTransientRetries?: number;
/**
* Collect metadata-probe failures into `ExtractionResult.errors` instead
* of preserving the legacy Promise rejection. Default false; only the
* candidate enforce lane may opt into typed aggregation.
*/
collectProbeFailures?: boolean;
/** Extract only these runs of output frames, named as a full extraction names them. CFR only. */
frameRanges?: readonly FrameRange[];
onProgress?: (progress: { planned?: number; done?: number }) => void;
/** A short decode of a remote source, only to count its frames: it waits on the network, so it runs wider. */
tailProbe?: boolean;
/**
* Leave CFR windows longer than a second unextracted, returning `deferredRange` and an exact
* `totalFrames` so another process can extract just the frames it needs with `frameRanges`.
*/
deferRangeExtraction?: boolean;
/** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
contentKeyedDirs?: readonly string[];
urlByAbsentLocalPath?: ReadonlyMap;
shownBox?: { width: number; height: number };
shownBoxes?: ReadonlyMap;
}
const DOWNLOAD_SUBDIR = "_downloads";
export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
function shownBoxKey(box: { width: number; height: number } | undefined): string | undefined {
return box && `shown-${Math.ceil(box.width)}x${Math.ceil(box.height)}`;
}
function halveToCoverScaleFilter(box: { width: number; height: number }): string {
const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`;
const halvings = `floor(log(1/${fit})/log(2))`;
const side = (dim: "iw" | "ih") =>
`if(lte(${fit}\\,0.5)\\,2*ceil(${dim}/pow(2\\,${halvings})/2)\\,${dim})`;
return `scale=w=${side("iw")}:h=${side("ih")}:flags=area+accurate_rnd`;
}
const SDR_JPEG_AS_BT601_FULL_RANGE_FILTER =
"scale=flags=neighbor+accurate_rnd,format=gbrp,scale=out_color_matrix=bt601:out_range=pc:flags=neighbor,format=yuv420p";
let warnedMissingZscale = false;
async function toneMapsWithZscale(): Promise {
return process.platform !== "darwin" || isFfmpegFilterAvailable("zscale");
}
async function hdrToSdrTransformKey(): Promise {
if (await toneMapsWithZscale()) return HDR_TO_SDR_ZSCALE_TRANSFORM_KEY;
if (!warnedMissingZscale) {
warnedMissingZscale = true;
process.stderr.write(
"[hyperframes:render] WARNING: this ffmpeg has no zscale filter, so forced-SDR HDR footage " +
"is tone-mapped by Chrome instead of the hable curve used on Linux and in Studio, and " +
"renders darker. Install an ffmpeg built with libzimg to match.\n",
);
}
return HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY;
}
function sdrToHdrTransformKey(transfer: HdrTransfer): string {
return `sdr2hdr-${transfer}`;
}
/**
* Per-phase timings and counters emitted by `extractAllVideoFrames`.
*
* Used by the producer to surface `perfSummary.videoExtractBreakdown` — without
* this breakdown, a single `videoExtractMs` stage timing hides where cost lives
* (HDR preflight, VFR classification, per-video ffmpeg extract) when tuning renders.
*
* Field semantics:
* - *Ms fields are wall-clock durations inside each phase.
* - *Count fields report how many sources triggered that phase.
* - extractMs wraps the parallel `extractVideoFramesRange` calls; it
* reflects max-across-parallel-workers, not sum.
* - hdrPreflightMs includes its probe-time sibling (hdrProbeMs); the
* probe-only field is a finer decomposition, not a separate carve-out.
* - vfrPreflightCount reports sources classified as VFR and extracted with
* a rounded-up frame count. DEFINITION CHANGE:
* before the one-pass refactor, vfrPreflightMs timed a per-source
* VFR-to-CFR re-encode and could reach seconds; it now times only the
* (promise-cached) classification probe and is expected to be ~0.
* Dashboards alerting on vfrPreflightMs thresholds should key on
* vfrPreflightCount or extractMs instead.
*/
export interface ExtractionPhaseBreakdown {
resolveMs: number;
/** Publishes that could not land atomically — the render still succeeded
* from the partial dir, but future renders re-extract. A rising rate is
* the first signal that warm renders are silently going cold. */
cachePublishFailures: number;
/** Entries evicted by the post-extraction LRU sweep. */
cacheGcEvictions: number;
/** Bytes reclaimed by the LRU sweep. */
cacheGcBytesFreed: number;
/** Aged .partial-* dirs (crashed writers) removed by the sweep. */
cacheAgedPartialsCleared: number;
hdrProbeMs: number;
hdrPreflightMs: number;
hdrPreflightCount: number;
vfrProbeMs: number;
vfrPreflightMs: number;
vfrPreflightCount: number;
extractMs: number;
cacheHits: number;
cacheMisses: number;
/** Number of per-source transient failures retried inside this extraction. */
transientRetries?: number;
}
export type VideoExtractionFailureKind =
| "cancelled"
| "external_interruption"
| "source_missing"
| "source_rejected"
| "download_not_found"
| "download_transient"
| "invalid_media"
| "media_start_out_of_range"
| "ffmpeg_unavailable"
| "ffmpeg_timeout"
| "ffmpeg_transient"
| "ffmpeg_failed"
| "disk_full"
| "zero_output"
| "internal";
export interface VideoExtractionFailure {
videoId: string;
/** Always populated by this engine version; optional for source compatibility with older consumers. */
kind?: VideoExtractionFailureKind;
/** Always populated by this engine version; absent legacy values fail closed. */
retryable?: boolean;
/** Bounded, path-free grouping data safe for producer-owned error metadata. */
group?: VideoExtractionFailureGroupDetails;
/**
* Operator diagnostic retained inside the engine result. Producer-facing
* errors must summarize `kind`/counts and must not forward this field: it
* can contain a local path or a signed source URL.
*/
error: string;
/** ffmpeg's last stderr lines, without paths or URLs: what a producer-facing error may say. */
ffmpegReason?: string;
/** The clip's file, project-relative; a remote source shows only its file name. Safe for producer-facing errors. */
source?: string;
}
const URL_SCHEME = /^[a-z][a-z0-9+.-]+:/i;
const WINDOWS_ABSOLUTE = /^([a-zA-Z]:[\\/]|\\\\)/;
/** A clip's src as a producer-facing error may print it: no scheme payload, host, query, fragment or absolute path. */
export function videoExtractionSourceLabel(src: string, baseDir: string): string {
if (/^(data|blob):/i.test(src)) return "inline video";
if (URL_SCHEME.test(src)) {
try {
return basename(decodeURIComponent(new URL(src).pathname)) || "remote video";
} catch {
return "remote video";
}
}
const path = src.split(/[?#]/)[0]!;
const lib = WINDOWS_ABSOLUTE.test(path) ? win32 : { isAbsolute, relative, basename, normalize };
const relativePath = lib.isAbsolute(path) ? lib.relative(baseDir, path) : lib.normalize(path);
const label =
relativePath.startsWith("..") || lib.isAbsolute(relativePath)
? lib.basename(path)
: relativePath;
return label.replace(/\\/g, "/");
}
export type VideoExtractionFailureStatusClass =
| "http_4xx"
| "http_5xx"
| "timeout"
| "network"
| "other";
export interface VideoExtractionFailureRetry {
phase: "download";
used: 0 | 1;
budget: 1;
}
export interface VideoExtractionFailureGroupDetails {
sourceFingerprint?: string;
host?: string;
statusClass?: VideoExtractionFailureStatusClass;
retry?: VideoExtractionFailureRetry;
}
export interface SafeVideoExtractionSourceIdentity {
sourceFingerprint: string;
host: string;
}
/** Query/fragment-free remote identity safe for extraction logs and wire metadata. */
export function safeVideoExtractionSourceIdentity(
source: string,
): SafeVideoExtractionSourceIdentity | null {
if (!isHttpUrl(source)) return null;
const identity = safeDownloadUrlIdentity(source);
return {
sourceFingerprint: `sha256:${identity.urlFingerprint}`,
host: identity.host ?? "other",
};
}
function downloadStatusClass(error: UrlDownloadError): VideoExtractionFailureStatusClass {
const status = error.status ?? error.telemetry?.status;
if (typeof status === "number" && status >= 400 && status < 500) return "http_4xx";
if (typeof status === "number" && status >= 500 && status < 600) return "http_5xx";
if (error.kind === "timeout") return "timeout";
if (error.kind === "network") return "network";
return "other";
}
function downloadFailureGroup(
source: string,
error: UrlDownloadError,
): VideoExtractionFailureGroupDetails {
const sourceIdentity = safeVideoExtractionSourceIdentity(source);
const failureHost = error.telemetry?.finalHost ?? error.telemetry?.initialHost;
const attempt = error.telemetry?.attempt;
return {
...(sourceIdentity ? { sourceFingerprint: sourceIdentity.sourceFingerprint } : {}),
...((failureHost ?? sourceIdentity?.host) ? { host: failureHost ?? sourceIdentity?.host } : {}),
statusClass: downloadStatusClass(error),
retry: {
phase: "download",
used: typeof attempt === "number" && attempt >= 2 ? 1 : 0,
budget: 1,
},
};
}
export class VideoSourceExtractionError extends Error {
readonly hyperframesVideoSourceExtractionError = true as const;
constructor(
readonly kind: VideoExtractionFailureKind,
readonly retryable: boolean,
message: string,
readonly diagnostic: string = message,
readonly ffmpegReason?: string,
) {
super(message);
this.name = "VideoSourceExtractionError";
}
}
const FFMPEG_REASON_LINES = 3;
const FFMPEG_REASON_CHARS = 300;
/** ffmpeg's last stderr lines for an error that leaves this host: no local path, file name or URL. */
function ffmpegReasonFrom(stderr: string, paths: readonly string[]): string {
let text = stderr;
for (const known of paths.filter(Boolean).sort((a, b) => b.length - a.length))
text = text.split(known).join("");
return text
.replace(/[a-z][a-z\d+.-]*:\/\/[^\s'"]+/gi, "")
.replace(/(?:[a-z]:)?(?:[\\/][^\s\\/:'"]+){2,}/gi, "")
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean)
.slice(-FFMPEG_REASON_LINES)
.join(" | ")
.slice(-FFMPEG_REASON_CHARS);
}
// ffmpeg ends a full disk with its last frames' summary, so the strerror text is the only sign of it
const DISK_FULL = /no space left on device|disk quota exceeded/i;
export function isVideoSourceExtractionError(error: unknown): error is VideoSourceExtractionError {
return (
typeof error === "object" &&
error !== null &&
"hyperframesVideoSourceExtractionError" in error &&
error.hyperframesVideoSourceExtractionError === true
);
}
function boundedTransientRetryBudget(value: number | undefined): 0 | 1 {
return Number.isFinite(value) && (value ?? 0) >= 1 ? 1 : 0;
}
function errorText(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
/**
* Convert legacy/raw downloader and filesystem errors into the bounded
* extraction taxonomy. New extraction code should throw
* `VideoSourceExtractionError` directly; this classifier keeps older utility
* boundaries safe while they migrate.
*/
export function classifyVideoExtractionError(error: unknown): VideoSourceExtractionError {
if (isVideoSourceExtractionError(error)) return error;
const diagnostic = errorText(error);
const lowered = diagnostic.toLowerCase();
if (error instanceof UrlDownloadError) {
if (error.kind === "cancelled") {
return new VideoSourceExtractionError(
"cancelled",
false,
"Video extraction cancelled",
diagnostic,
);
}
if (error.kind === "http_not_found") {
return new VideoSourceExtractionError(
"download_not_found",
false,
"Video source was not found",
diagnostic,
);
}
if (error.kind === "http_rejected") {
return new VideoSourceExtractionError(
"source_rejected",
false,
"Video source download was rejected",
diagnostic,
);
}
if (error.kind === "invalid_payload") {
return new VideoSourceExtractionError(
"invalid_media",
false,
"Video source download returned a non-media payload",
diagnostic,
);
}
if (error.retryable) {
return new VideoSourceExtractionError(
"download_transient",
true,
"Video source download failed transiently",
diagnostic,
);
}
return new VideoSourceExtractionError(
"internal",
false,
"Video source download failed internally",
diagnostic,
);
}
if (lowered.includes("cancelled") || lowered.includes("aborted")) {
return new VideoSourceExtractionError(
"cancelled",
false,
"Video extraction cancelled",
diagnostic,
);
}
if (lowered.includes("video file not found")) {
return new VideoSourceExtractionError(
"source_missing",
false,
"Video source is missing",
diagnostic,
);
}
if (
lowered.includes("only https urls are permitted") ||
lowered.includes("private/reserved address") ||
lowered.includes("invalid url")
) {
return new VideoSourceExtractionError(
"source_rejected",
false,
"Video source URL is not permitted",
diagnostic,
);
}
const httpStatus = diagnostic.match(/\bHTTP\s+(\d{3})\b/i)?.[1];
if (httpStatus) {
const status = Number(httpStatus);
if (status === 404 || status === 410) {
return new VideoSourceExtractionError(
"download_not_found",
false,
"Video source was not found",
diagnostic,
);
}
if (status === 408 || status === 429 || status >= 500) {
return new VideoSourceExtractionError(
"download_transient",
true,
"Video source download failed transiently",
diagnostic,
);
}
return new VideoSourceExtractionError(
"source_rejected",
false,
"Video source download was rejected",
diagnostic,
);
}
if (
lowered.includes("[urldownloader] download timeout") ||
lowered.includes("[urldownloader] download failed") ||
lowered.includes("fetch failed") ||
lowered.includes("network")
) {
return new VideoSourceExtractionError(
"download_transient",
true,
"Video source download failed transiently",
diagnostic,
);
}
if (lowered.includes("ffprobe not found")) {
return new VideoSourceExtractionError(
"ffmpeg_unavailable",
false,
"FFprobe is unavailable",
diagnostic,
);
}
if (lowered.includes("ffprobe deadline")) {
return new VideoSourceExtractionError(
"ffmpeg_timeout",
true,
"Video inspection timed out",
diagnostic,
);
}
if (
lowered.includes("ffprobe") ||
lowered.includes("failed to parse ffprobe output") ||
lowered.includes("no video stream found")
) {
return new VideoSourceExtractionError(
"invalid_media",
false,
"Video source could not be inspected",
diagnostic,
);
}
return new VideoSourceExtractionError(
"internal",
false,
"Video extraction failed internally",
diagnostic,
);
}
export async function runVideoExtractionWithRetry(
operation: () => Promise,
options: {
signal?: AbortSignal;
onRetry?: () => Promise | void;
maxTransientRetries?: number;
} = {},
): Promise<{ result: T; retries: number }> {
const maxTransientRetries = boundedTransientRetryBudget(options.maxTransientRetries);
let retries = 0;
for (;;) {
if (options.signal?.aborted) {
throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
}
try {
return { result: await operation(), retries };
} catch (error) {
const classified = classifyVideoExtractionError(error);
if (options.signal?.aborted) {
throw new VideoSourceExtractionError(
"cancelled",
false,
"Video extraction cancelled",
classified.diagnostic,
);
}
if (
classified.kind === "cancelled" ||
!classified.retryable ||
retries >= maxTransientRetries
) {
throw classified;
}
retries += 1;
await options.onRetry?.();
}
}
}
export interface ExtractionResult {
success: boolean;
extracted: ExtractedFrames[];
errors: VideoExtractionFailure[];
totalFramesExtracted: number;
durationMs: number;
phaseBreakdown: ExtractionPhaseBreakdown;
}
/** Minimal structural shape for resolving parent/`` media `src`. */
interface MediaSrcEl {
getAttribute(name: string): string | null;
querySelectorAll(selectors: string): Iterable<{ getAttribute(name: string): string | null }>;
}
/**
* Parent `src`, else a ``. Prefer local paths over http(s) so a
* localized sibling wins when another `` failed to download.
*/
export function resolveMediaElementSrc(el: MediaSrcEl): string | null {
const direct = el.getAttribute("src");
if (direct) return direct;
let remote: string | null = null;
for (const source of el.querySelectorAll("source")) {
const src = source.getAttribute("src");
if (!src) continue;
if (!/^https?:\/\//i.test(src)) return src;
remote ??= src;
}
return remote;
}
export function parseVideoElements(html: string): VideoElement[] {
const videos: VideoElement[] = [];
const { document } = parseHTML(unwrapTemplate(html));
const startCache = new Map();
const visiting = new Set();
const groupsById = audioGroupsById(document);
const videoEls = document.querySelectorAll("video");
let autoIdCounter = 0;
for (const el of videoEls) {
const src = resolveMediaElementSrc(el);
if (!src) continue;
// Generate a stable ID for videos without one — the producer needs IDs
// to track extracted frames and composite them during encoding.
// A compiled render document stamps a document-unique render id; prefer it,
// because element ids are only unique within one composition file and the
// render document inlines many.
const id =
el.getAttribute(MEDIA_RENDER_ID_ATTR) ||
el.getAttribute("id") ||
`hf-video-${autoIdCounter++}`;
if (!el.getAttribute("id")) {
el.setAttribute("id", id);
}
const startAttr = el.getAttribute("data-start");
const endAttr = el.getAttribute("data-end");
const durationAttr = el.getAttribute("data-duration");
const hasAudioAttr = el.getAttribute("data-has-audio");
// Resolve data-start, including relative references ("intro", "intro + 2")
// to another clip's end — the browser runtime resolves these but a raw
// parseFloat here would yield NaN, placing the clip at NaN so it composites
// blank in the final render. `startAttr` may be a plain number or a
// reference; the resolver handles both.
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
if (isKnownInactiveTimelineWindow(el, start)) continue;
// Derive end from data-end → data-start+data-duration → Infinity (natural duration).
// Static compilation cannot always clamp root media because GSAP may supply
// the root duration at runtime. The producer passes the resolved timeline
// end into frame extraction, which caps the source duration only after the
// natural duration is known without rewriting authored timing metadata.
let end = 0;
const authoredEnd = parseStrictFiniteTimingNumber(endAttr);
const authoredDuration = parseStrictFiniteTimingNumber(durationAttr);
if (authoredEnd != null) {
end = authoredEnd;
} else if (authoredDuration != null) {
end = start + authoredDuration;
} else {
end = Infinity; // no explicit bounds — play for the full natural video duration
}
videos.push({
id,
src,
start,
end,
mediaStart: readMediaStart(el),
playbackRate: readElementRateSpec(el),
loop: el.hasAttribute("loop"),
hasAudio: hasAudioAttr === "true" && isAudibleVideoElement(el),
...(isSelfOrAncestorHidden(el) ||
(isAudibleVideoElement(el) && isMemberGroupHidden(groupsById, el))
? { hidden: true }
: {}),
...(el.hasAttribute(HF_COLOR_GRADING_ATTR) ? { graded: true } : {}),
});
}
return videos;
}
export interface ImageElement {
id: string;
src: string;
start: number;
end: number;
graded?: boolean;
}
export function hasGradedCanvas(html: string): boolean {
return (
parseHTML(unwrapTemplate(html)).document.querySelector(`canvas[${HF_COLOR_GRADING_ATTR}]`) !==
null
);
}
export function parseImageElements(html: string): ImageElement[] {
const images: ImageElement[] = [];
const { document } = parseHTML(unwrapTemplate(html));
const startCache = new Map();
const visiting = new Set();
const imgEls = document.querySelectorAll("img[src]");
let autoIdCounter = 0;
for (const el of imgEls) {
const src = el.getAttribute("src");
if (!src) continue;
// See parseVideoElements: the stamped render id wins over the authored id.
const id =
el.getAttribute(MEDIA_RENDER_ID_ATTR) || el.getAttribute("id") || `hf-img-${autoIdCounter++}`;
if (!el.getAttribute("id")) {
el.setAttribute("id", id);
}
const startAttr = el.getAttribute("data-start");
const endAttr = el.getAttribute("data-end");
const durationAttr = el.getAttribute("data-duration");
// Resolve relative data-start references (see parseVideoElements) so a
// referenced image start doesn't become NaN and drop the image from the render.
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
let end = 0;
const authoredEnd = parseStrictFiniteTimingNumber(endAttr);
const authoredDuration = parseStrictFiniteTimingNumber(durationAttr);
if (authoredEnd != null) {
end = authoredEnd;
} else if (authoredDuration != null) {
end = start + authoredDuration;
} else {
end = Infinity;
}
images.push({
id,
src,
start,
end,
...(el.hasAttribute(HF_COLOR_GRADING_ATTR) ? { graded: true } : {}),
});
}
return images;
}
const firstFrameColourCache = new Map>();
/** A source's SPS transfers, read as the trace streams (a run keeps only stderr's tail); null when the read failed. */
function readKeyframeTransfers(videoPath: string, codec: string) {
let traced = "";
let rest = "";
const onStderr = (chunk: string) => {
const lines = (rest + chunk).split("\n");
rest = lines.pop() ?? "";
for (const line of lines) if (line.includes("transfer_characteristics")) traced += `${line}\n`;
};
return inExtractionSlot(() =>
runFfmpeg([...urlInputArgs(videoPath), ...keyframeTransferArgs(videoPath, codec)], {
onStderr,
}),
).then((result) => (result.success ? parseKeyframeTransfers(traced + rest) : null));
}
/** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */
function readFirstFrameColour(videoPath: string, codec: string) {
const identity = mediaFileIdentity(videoPath);
const cached = readMetadataCache(firstFrameColourCache, videoPath, identity);
if (cached) return cached;
// In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others.
const probe = inExtractionSlot(() =>
runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]),
).then(async (result): Promise => {
const forget = () => {
if (firstFrameColourCache.get(videoPath)?.promise === probe)
firstFrameColourCache.delete(videoPath);
};
// A failed probe answers this cut only; the next one probes again.
if (!result.success) forget();
const first = parseFirstFrameColour(result.stderr);
if (!result.success || !readsKeyframeTransfers(first)) return first;
const keyframeTransfers = await readKeyframeTransfers(videoPath, codec);
if (!keyframeTransfers) forget();
return { ...first, keyframeTransfers: keyframeTransfers ?? [] };
});
writeMetadataCache(firstFrameColourCache, videoPath, identity, probe);
return probe;
}
/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */
const CHROME_BT601_UNTAGGED_CODECS = new Set(["vp9", "av1"]);
function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
const matrix = metadata.colorSpace?.colorSpace;
if (matrix && matrix !== "unknown") return [];
const hd = metadata.height >= 720 && !CHROME_BT601_UNTAGGED_CODECS.has(metadata.videoCodec);
return [`setparams=colorspace=${hd ? "bt709" : "smpte170m"}`];
}
export function sdrCanvasColourFilters(metadata: VideoMetadata, toneMappedToSdr = false): string[] {
if (!isHdrColorSpaceUtil(metadata.colorSpace)) {
return [...chromeGuessForUntaggedMatrix(metadata), SDR_CANVAS_PASSTHROUGH_FILTER];
}
return toneMappedToSdr ? [SDR_CANVAS_PASSTHROUGH_FILTER] : [];
}
/**
* Longest stretch of output one ffmpeg process extracts; longer ranges split into segments
* that run side by side. Segments sample the same absolute times as a single pass, so their
* frames are byte-identical to it.
*/
const EXTRACTION_SEGMENT_SECONDS = 120;
/** Slowest decode rate the stall clock allows while ffmpeg discards frames from a keyframe up to the seek point. */
const SKIP_DECODE_FPS_FLOOR = 30;
async function leadInSecondsUnlessAborted(
videoPath: string,
seek: number,
signal?: AbortSignal,
): Promise {
try {
return await keyframeGapSeconds(videoPath, seek, signal);
} catch (error) {
// Aborted: let runFfmpeg report the abort, so a segment stopped by another's failure reads as stopped.
if (signal?.aborted) return 0;
throw error;
}
}
function stallTimeoutMs(baseMs: number, skippedSeconds: number, sourceFps: number): number {
return baseMs + Math.ceil((skippedSeconds * sourceFps * 1000) / SKIP_DECODE_FPS_FLOOR);
}
const KEEP_END_PAST_A_ONE_TICK_FINAL_FRAME_THROUGH_THE_SNAP = "tpad=stop_mode=clone:stop=1";
/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
function undoTickRoundingOnTheSourceFrameGrid(
seek: number,
lead: number,
sourceFps: number,
): string[] {
const sinceFileStart = `(PTS*TB+${seek}-${lead})`;
return [
...(sourceFps > 0 ? [KEEP_END_PAST_A_ONE_TICK_FINAL_FRAME_THROUGH_THE_SNAP] : []),
"settb=AVTB",
`setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,
];
}
// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.
const EXTRACTION_SLOTS = Math.max(1, Math.floor(usableCpuCount() / 2));
const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);
const TAIL_PROBE_SLOTS = Math.max(1, Number(process.env.HF_TAIL_PROBE_SLOTS) || 16);
const inTailProbeSlot = createConcurrencyLimit(TAIL_PROBE_SLOTS);
interface SegmentedExtraction {
decodeArgs: string[];
filterAndEncodeArgs: (seek: number) => string[];
ffmpegSeek: (mediaTime: number) => number;
videoPath: string;
startTime: number;
fps: number;
sourceFps: number;
segments: readonly FrameRange[];
gate: typeof inExtractionSlot;
runOptions: { signal?: AbortSignal; timeout: number };
onProgress?: ExtractionOptions["onProgress"];
}
function framesWrittenReader(onProgress: ExtractionOptions["onProgress"]) {
if (!onProgress) return undefined;
let seen = 0;
const read = ffmpegStatsReader(({ frames }) => {
if (frames === undefined || frames <= seen) return;
onProgress({ done: frames - seen });
seen = frames;
});
return (chunk: string) => read(chunk);
}
/** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */
function splitFrameRanges(
ranges: readonly FrameRange[],
segmentFrames: number,
frameLimit: number,
): FrameRange[] {
const segments: FrameRange[] = [];
for (const range of ranges) {
const frames = Math.min(range.frames, frameLimit - range.firstFrame);
for (let offset = 0; offset < frames; offset += segmentFrames) {
segments.push({
firstFrame: range.firstFrame + offset,
frames: Math.min(segmentFrames, frames - offset),
});
}
}
return segments;
}
async function runSegmentedExtraction(job: SegmentedExtraction): Promise {
const { decodeArgs, filterAndEncodeArgs, videoPath, startTime, fps, sourceFps, segments, gate } =
job;
const segmentCount = segments.length;
// One failed segment fails the range, so stop the others instead of finishing them.
const failed = new AbortController();
const signal = job.runOptions.signal
? AbortSignal.any([job.runOptions.signal, failed.signal])
: failed.signal;
const results: RunFfmpegResult[] = [];
let next = 0;
const worker = async () => {
while (next < segmentCount && !signal.aborted) {
const index = next++;
const { firstFrame, frames } = segments[index]!;
const seek = startTime + firstFrame / fps;
const result = await gate(
async () =>
runFfmpeg(
[
...decodeArgs,
"-noaccurate_seek",
"-ss",
formatFfmpegSeconds(job.ffmpegSeek(seek)),
"-i",
videoPath,
// Read one frame past the segment and keep exactly its own frames.
"-t",
formatFfmpegSeconds((frames + 1) / fps),
"-frames:v",
String(frames),
"-start_number",
String(firstFrame + 1),
...filterAndEncodeArgs(seek),
],
{
...job.runOptions,
signal,
onStderr: framesWrittenReader(job.onProgress),
timeout: stallTimeoutMs(
job.runOptions.timeout,
await leadInSecondsUnlessAborted(videoPath, seek, signal),
sourceFps,
),
},
),
signal,
);
results[index] = result;
if (!result.success) failed.abort();
}
};
const workers = Math.min(segmentCount, EXTRACTION_SLOTS);
await Promise.all(Array.from({ length: workers }, worker)).catch((error) => {
failed.abort();
throw error;
});
const ran = results.filter(Boolean);
// Report the segment that failed, not the ones stopped because of it.
const failure =
ran.find((r) => !r.success && r.terminationReason !== "abort") ?? ran.find((r) => !r.success);
if (failure) return failure;
// A cancel between segments leaves later ones unrun; their frames are missing, not done.
if (ran.length < segmentCount) {
return {
success: false,
exitCode: null,
stderr: "",
durationMs: 0,
terminationReason: "abort",
};
}
return ran[ran.length - 1]!;
}
export async function extractVideoFramesRange(
...args: Parameters
): Promise {
try {
return await extractVideoFramesRangeUnredacted(...args);
} catch (error) {
throw redactUrlQueriesInError(error);
}
}
interface VideoFilterChainInput {
metadata: VideoMetadata;
format: CacheFrameFormat;
ffmpegFps: string;
sampleAtOutputFps: boolean;
keepsHdrForChrome: boolean;
/** The first frame's colour tags; set exactly when HDR is tone-mapped to SDR. */
toneMapFrom?: ToneMapFrameColour;
isHdr: boolean;
sdrToHdrTransfer?: ExtractionOptions["sdrToHdrTransfer"];
shownBox?: ExtractionOptions["shownBox"];
}
/** The one place that decides which source frame a slot shows and how its colour is converted. */
function videoFilterChain(input: VideoFilterChainInput): (seek: number) => string[] {
const { metadata, format, ffmpegFps, sampleAtOutputFps, keepsHdrForChrome, toneMapFrom } = input;
const toneMappedToSdr = toneMapFrom !== undefined;
const head: string[] = [];
if (keepsHdrForChrome) head.push("format=nv12");
const tail: string[] = [];
if (metadata.sampleAspectRatio) {
tail.push("scale=trunc(iw*sar/2)*2:ih", "setsar=1");
}
// Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
// first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
const sampling = (seek: number): string[] =>
sampleAtOutputFps
? [
...(metadata.isVFR
? []
: undoTickRoundingOnTheSourceFrameGrid(
seek,
videoStartOnMediaTimeline(metadata),
metadata.fps,
)),
"settb=intb/2",
"setpts=PTS-lte(TB\\,0.001)",
`fps=${ffmpegFps}:start_time=0:round=up`,
]
: [];
if (input.sdrToHdrTransfer) {
// Fps sampling runs before the colorspace remap so only kept frames are converted. The
// remap is pointwise, so the result is identical either way for the SDR inputs it applies
// to (it never meets nv12, which is HDR-only); revisit if the two ever combine.
tail.push(SDR_TO_HDR_COLORSPACE_FILTER);
}
if (toneMapFrom) {
tail.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, toneMapFrom));
}
if (toneMappedToSdr || (!input.isHdr && !input.sdrToHdrTransfer)) {
tail.push(...sdrCanvasColourFilters(metadata, toneMappedToSdr));
if (format === "jpg") {
if (input.shownBox) tail.push(halveToCoverScaleFilter(input.shownBox));
tail.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER);
}
}
return (seek) => [...head, ...sampling(seek), ...tail];
}
export interface VideoLayerInput {
inputArgs: string[];
filters: string[];
}
const VIDEO_LAYER_TAIL_SLACK_SECONDS = 0.5;
/** ffmpeg input and filters showing the same frames and colour as extraction; undefined if unsupported. */
export async function buildVideoLayerInput(
videoPath: string,
startTime: number,
duration: number,
fps: Fps,
canvas: { width: number; height: number },
fit: "fill" | "cover" = "fill",
): Promise {
const metadata = await extractMediaMetadata(videoPath);
if (
startTime + duration >
resolvePlayableVideoDuration(metadata) + VIDEO_LAYER_TAIL_SLACK_SECONDS ||
isHdrColorSpaceUtil(metadata.colorSpace) ||
metadata.isVFR ||
metadata.hasAlpha ||
codecMayHaveAlpha(metadata.videoCodec)
) {
return undefined;
}
const chain = videoFilterChain({
metadata,
format: "png",
ffmpegFps: fpsToFfmpegArg(fps),
sampleAtOutputFps: true,
keepsHdrForChrome: false,
isHdr: false,
});
const filters = chain(startTime);
if (metadata.width !== canvas.width || metadata.height !== canvas.height) {
filters.push(
fit === "cover"
? `scale=${canvas.width}:${canvas.height}:force_original_aspect_ratio=increase:flags=bicubic,crop=${canvas.width}:${canvas.height}`
: `scale=${canvas.width}:${canvas.height}:flags=bicubic`,
);
}
filters.push(`tpad=stop_mode=clone:stop_duration=${VIDEO_LAYER_TAIL_SLACK_SECONDS * 2}`);
return {
inputArgs: [
"-noaccurate_seek",
"-ss",
formatFfmpegSeconds(ffmpegSeekSeconds(metadata, startTime)),
...urlInputArgs(videoPath),
"-i",
videoPath,
],
filters,
};
}
async function extractVideoFramesRangeUnredacted(
videoPath: string,
videoId: string,
startTime: number,
duration: number,
options: ExtractionOptions,
signal?: AbortSignal,
config?: Partial>,
/** Frames go straight here, with no per-videoId subdir (the cache layer's keyed entry). */
outputDirOverride?: string,
): Promise {
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
const { outputDir, quality = 95 } = options;
const normalizedFps = toFps(options.fps);
const fps = fpsToNumber(normalizedFps);
const ffmpegFps = fpsToFfmpegArg(normalizedFps);
const videoOutputDir = outputDirOverride ?? join(outputDir, mediaIdPathSegment(videoId));
if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });
let metadata: VideoMetadata;
try {
metadata = await extractMediaMetadata(videoPath);
} catch (error) {
throw classifyVideoExtractionError(error);
}
const playableDuration = resolvePlayableVideoDuration(metadata);
if (!(playableDuration > 0)) {
throw new VideoSourceExtractionError(
"invalid_media",
false,
"Video source has no positive duration",
`Playable video stream duration is ${playableDuration}s`,
);
}
if (startTime >= playableDuration) {
throw new VideoSourceExtractionError(
"media_start_out_of_range",
false,
"Video media start is outside the source duration",
`Video media start ${startTime}s is outside playable video duration ${playableDuration}s`,
);
}
const format = resolveFrameFormat(metadata, options.format);
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
const outputPattern = frameSequencePath(videoOutputDir, framePattern);
// Forced-SDR extraction tone-maps HDR with zscale, the Studio proxy policy. A macOS ffmpeg
// without zscale falls back to VideoToolbox, whose frames stay HDR and are tone-mapped by
// Chrome instead (darker), and the render warns once.
const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
const isMacOS = process.platform === "darwin";
const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale());
const keepsHdrForChrome = isHdr && isMacOS && !toneMappedToSdr;
const decodeWithVideoToolbox =
keepsHdrForChrome ||
(isMacOS &&
!metadata.hasAlpha &&
!codecMayHaveAlpha(metadata.videoCodec) &&
(await isFfmpegHwaccelAvailable("videotoolbox")));
const sampleAtOutputFps = !options.finalFrameOnly;
if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
throw new Error("frameRanges extraction needs a constant-frame-rate source");
}
const args: string[] = urlInputArgs(videoPath);
// Always force the alpha-aware decoder on codecs that can carry alpha. The
// alternative — gating on `metadata.hasAlpha` — relies on tag detection that
// has at least three known failure modes: case-sensitivity across ffmpeg
// versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
// and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
// silently strips alpha during decode and the bug doesn't surface until
// the rendered video is missing layers. Codec-based default has no such
// ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
// normally when it isn't.
if (codecMayHaveAlpha(metadata.videoCodec)) {
args.push("-c:v", decoderForCodec(metadata.videoCodec));
}
const decodeArgs = [...args];
const ffmpegSeek = (mediaTime: number) => ffmpegSeekSeconds(metadata, mediaTime);
if (options.finalFrameOnly) {
// Output-side seek decodes from the start before selecting the final
// sample. This is intentionally reserved for the one-frame path: input
// seeking is faster, but valid unindexed transports (notably MPEG-TS with
// a negative timestamp base) can seek to EOF and emit zero frames.
args.push(
"-i",
videoPath,
"-ss",
formatFfmpegSeconds(Math.max(0, ffmpegSeek(startTime))),
"-frames:v",
"1",
);
} else {
// An accurate seek drops the frame on screen at startTime; the fps sampling below needs it.
if (sampleAtOutputFps) args.push("-noaccurate_seek");
args.push(
"-ss",
formatFfmpegSeconds(ffmpegSeek(startTime)),
"-i",
videoPath,
"-t",
formatFfmpegSeconds(duration),
);
}
const frameColour = toneMappedToSdr
? await readFirstFrameColour(videoPath, metadata.videoCodec)
: undefined;
const filtersFor = videoFilterChain({
metadata,
format,
ffmpegFps,
sampleAtOutputFps,
keepsHdrForChrome,
toneMapFrom: frameColour,
isHdr,
sdrToHdrTransfer: options.sdrToHdrTransfer,
shownBox: options.shownBox,
});
const encodeArgs = ["-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"];
// Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.
if (format === "png") encodeArgs.push("-compression_level", "1");
encodeArgs.push("-y", outputPattern);
const runOptions = { signal, timeout: ffmpegProcessTimeout };
const filterArgsFor = (seek: number): string[] => {
const filters = filtersFor(seek);
return filters.length > 0 ? ["-vf", filters.join(",")] : [];
};
const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
const windowFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
const framesBeforeSourceEnd = extractionFrameCountForDuration(
playableDuration - startTime,
normalizedFps,
false,
);
const totalFrames = metadata.isVFR ? windowFrames : Math.min(windowFrames, framesBeforeSourceEnd);
// A `-t` window ends at the nearest slot; segments read a frame past and keep exactly their own.
// A decimal rate counts zero frames here and keeps the `-t` window.
const extract = (hwaccel: string[]) =>
sampleAtOutputFps && (options.frameRanges || totalFrames > 0)
? runSegmentedExtraction({
decodeArgs: [...hwaccel, ...decodeArgs],
filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
ffmpegSeek,
videoPath,
startTime,
fps,
sourceFps: metadata.fps,
gate: options.tailProbe ? inTailProbeSlot : inExtractionSlot,
segments: splitFrameRanges(
options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }],
segmentFrames,
totalFrames > 0 ? totalFrames : Number.POSITIVE_INFINITY,
),
runOptions,
onProgress: options.onProgress,
})
: inExtractionSlot(
async () =>
runFfmpeg([...hwaccel, ...args, ...filterArgsFor(startTime), ...encodeArgs], {
...runOptions,
onStderr: framesWrittenReader(options.onProgress),
timeout: stallTimeoutMs(
ffmpegProcessTimeout,
options.finalFrameOnly
? startTime
: await leadInSecondsUnlessAborted(videoPath, startTime, signal),
metadata.fps,
),
}),
signal,
);
options.onProgress?.({
planned: options.frameRanges
? options.frameRanges.reduce((sum, range) => sum + range.frames, 0)
: totalFrames,
});
let processResult = await extract(decodeWithVideoToolbox ? ["-hwaccel", "videotoolbox"] : []);
const ffmpegExitedFailing =
processResult.terminationReason === "exit" &&
!processResult.success &&
!processResult.failureReason;
const videoToolboxWasOptional = decodeWithVideoToolbox && !keepsHdrForChrome;
if (ffmpegExitedFailing && videoToolboxWasOptional && !DISK_FULL.test(processResult.stderr))
processResult = await extract([]);
if (processResult.failureReason === "external_interruption") {
throw new VideoSourceExtractionError(
"external_interruption",
true,
"Video frame extraction interrupted by host lifecycle",
`FFmpeg exited with code ${processResult.exitCode}: ${processResult.stderr.slice(-500)}`,
);
}
if (processResult.terminationReason === "abort") {
throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
}
if (processResult.terminationReason === "spawn_error") {
throw classifyFfmpegSpawnError(processResult.error, processResult.stderr);
}
if (!processResult.success) {
// With the SDR-to-HDR remap folded into this pass, a filter failure
// (e.g. an ffmpeg built without the colorspace filter) would otherwise
// surface as a generic extract error and the operator has to grep the
// filter chain to learn it was the HDR conversion. Attribute it.
const hdrPrefix = options.sdrToHdrTransfer
? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): `
: "";
const timedOut = processResult.terminationReason === "inactivity";
const timeoutSuffix = timedOut ? ` (no new frame for ${ffmpegProcessTimeout} ms)` : "";
const diagnostic =
`${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ` +
processResult.stderr.slice(-500);
const ffmpegReason = ffmpegReasonFrom(processResult.stderr, [
videoPath,
basename(videoPath),
videoOutputDir,
]);
if (timedOut) {
throw new VideoSourceExtractionError(
"ffmpeg_timeout",
true,
"Video frame extraction timed out",
diagnostic,
ffmpegReason,
);
}
if (DISK_FULL.test(processResult.stderr)) {
throw new VideoSourceExtractionError(
"disk_full",
false,
"The disk filled up while video frames were extracted",
diagnostic,
ffmpegReason,
);
}
const transientIo =
/resource temporarily unavailable|device or resource busy|input\/output error/i.test(
processResult.stderr,
);
throw new VideoSourceExtractionError(
transientIo ? "ffmpeg_transient" : "ffmpeg_failed",
transientIo,
transientIo
? "Video frame extraction hit a transient I/O failure"
: "Video source could not be decoded",
diagnostic,
ffmpegReason,
);
}
const framePaths = framePathsFromDirectory(videoOutputDir, format, !!options.frameRanges);
if (framePaths.size === 0 && duration > 0) {
throw new VideoSourceExtractionError(
"zero_output",
false,
"Video source produced no decodable frames",
`FFmpeg exited successfully but produced no frames (start=${startTime}, duration=${duration})`,
);
}
// Post-extraction reconciliation: when FFmpeg delivers <=60% of the
// frames predicted by the probed duration, the duration is likely
// inflated (container duration includes a longer audio track). Correct
// it so downstream coverage accounting uses the actual video extent.
if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) {
const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) {
const correctedStreamDuration =
startTime + framePaths.size / fps - (metadata.videoStreamLeadSeconds ?? 0);
if (correctedStreamDuration < metadata.videoStreamDurationSeconds) {
metadata.videoStreamDurationSeconds = correctedStreamDuration;
}
}
}
return {
videoId,
srcPath: videoPath,
outputDir: videoOutputDir,
framePattern,
fps,
totalFrames: framePaths.size,
metadata,
framePaths,
};
}
const TRANSIENT_FFMPEG_SPAWN_CODES = new Set(["EAGAIN", "EMFILE", "ENFILE"]);
export function classifyFfmpegSpawnError(error: unknown, stderr = ""): VideoSourceExtractionError {
const code =
typeof error === "object" && error !== null && "code" in error && typeof error.code === "string"
? error.code
: "";
if (code === "ENOENT") {
return new VideoSourceExtractionError(
"ffmpeg_unavailable",
false,
"FFmpeg is unavailable",
"[FFmpeg] ffmpeg not found",
);
}
const diagnostic = error instanceof Error ? error.message : stderr;
const retryable = TRANSIENT_FFMPEG_SPAWN_CODES.has(code);
return new VideoSourceExtractionError(
retryable ? "ffmpeg_transient" : "ffmpeg_failed",
retryable,
retryable
? "FFmpeg could not be started due to transient resource pressure"
: "FFmpeg could not be started",
diagnostic,
);
}
/**
* Resolve the used-segment duration for a video, falling back to the source's
* natural duration when the caller hasn't specified bounds (end=Infinity) or
* the bounds are nonsensical (end<=start).
*/
function resolveSegmentDuration(
requested: number,
mediaStart: number,
sourceDuration: number,
): number {
if (Number.isFinite(requested) && requested > 0) return requested;
const sourceRemaining = sourceDuration - mediaStart;
return sourceRemaining > 0 ? sourceRemaining : sourceDuration;
}
/**
* Return the range that can actually produce video frames.
*
* Container duration may include a longer audio stream or mux padding. Using
* it for video extraction planning can reserve raw-frame scratch for seconds
* where no video frames exist. `extractMediaMetadata` already falls back to
* the container duration when ffprobe omits the stream duration; keep the
* explicit fallback here for callers supplying older/manual metadata.
*/
export { ffmpegSeekSeconds, resolvePlayableVideoDuration, videoStartOnMediaTimeline };
export interface TimelineExtractionWindow {
compositionStart: number;
mediaStart: number;
durationSeconds: number;
/** Visible duration on the authored composition timeline after retiming. */
timelineDurationSeconds?: number;
/**
* Preserve the authored timeline origin and mediaStart for lookup. This is
* required when a looped visible interval crosses a source boundary and
* still needs modulo phase against the complete extracted source cycle.
*/
preserveTimelinePhase?: boolean;
/**
* Keep the authored end while rebasing start/mediaStart to the extracted
* source suffix. Non-looping lookup then holds the suffix's final frame
* through the remainder of the authored slot.
*/
preserveTimelineEnd?: boolean;
/** This window reaches a held tail and must be checked against the actual
* final decoded-frame timestamp before extraction. */
ensureFinalFrame?: boolean;
/** Source timestamp used by FFmpeg when it differs from the logical lookup
* mediaStart (the one-frame held-tail representation). */
extractionMediaStart?: number;
/** FFmpeg emits one decoded frame; lookup then holds that frame. */
finalFrameOnly?: boolean;
}
type TimelineWindowVideo = Pick &
Partial> &
Partial>;
function canHoldFinalFramePastEof(video: TimelineWindowVideo): boolean {
const timelineDuration = video.end - video.start;
return !video.loop && Number.isFinite(timelineDuration) && timelineDuration > 0;
}
// Logical duration assigned to a one-frame held-tail representation. This is
// deliberately below any supported output frame interval: coverage expects
// one frame, while FFmpeg seeks to the separately probed real frame timestamp.
const FINAL_FRAME_LOGICAL_DURATION_SECONDS = 1e-6;
export function isVideoPastTimelineEnd(video: Pick, timelineEnd: number) {
return video.start >= timelineEnd;
}
export function alignClipStartToFirstOutputFrame(video: VideoElement, fps: number): void {
if (video.loop || !Number.isFinite(video.start) || video.start < 0 || !(fps > 0)) return;
const position = exportClipWindow(video.start, Infinity, fps).start * fps;
const firstFrame =
Math.abs(position - Math.round(position)) < 1e-6 ? Math.round(position) : Math.ceil(position);
const shift = firstFrame / fps - video.start;
// Float dust (a sum landing a hair off the grid) snaps the start; it never reaches mediaStart.
if (Math.abs(shift) < 1e-9) {
video.start = firstFrame / fps;
return;
}
const playbackRate = normalizeRateSpec(video.playbackRate);
video.mediaStart += sourceTimeAt(playbackRate, shift);
video.playbackRate = shiftRateLane(playbackRate, shift);
video.start += shift;
}
/** Move a clip to its extraction window; a rate lane is shifted so it keeps integrating from the new origin. */
export function rebaseVideoToWindow(video: VideoElement, window: TimelineExtractionWindow): void {
if (window.preserveTimelinePhase) return;
// A trim landing on the timeline origin can come out as 1e-16 and drop the first frame.
const start = Math.abs(window.compositionStart) < 1e-9 ? 0 : window.compositionStart;
video.playbackRate = shiftRateLane(normalizeRateSpec(video.playbackRate), start - video.start);
video.start = start;
if (!window.preserveTimelineEnd) {
video.end = start + (window.timelineDurationSeconds ?? window.durationSeconds);
}
video.mediaStart = window.mediaStart;
}
/**
* Intersect an authored slot with the render timeline, then select the
* smallest playable source range that preserves timeline lookup semantics.
*
* A finite authored slot can outlive the source. In that case FFmpeg should
* still extract at most one source range: lookup either wraps that range for
* loops or holds its final frame for non-looping video. Keeping the authored
* timeline origin separate from the extracted range is what makes both
* behaviours survive the source-duration cap.
*/
export function resolveTimelineExtractionWindow(
video: TimelineWindowVideo,
resolvedDuration: number,
timelineEnd?: number,
sourceDuration?: number,
): TimelineExtractionWindow {
const playbackRate = normalizeRateSpec(video.playbackRate);
const withTimelineDuration = (
window: TimelineExtractionWindow,
timelineDurationSeconds: number,
): TimelineExtractionWindow =>
playbackRate === 1 ? window : { ...window, timelineDurationSeconds };
if (timelineEnd === undefined) {
return withTimelineDuration(
{
compositionStart: video.start,
mediaStart: video.mediaStart,
durationSeconds: sourceTimeAt(playbackRate, resolvedDuration),
},
resolvedDuration,
);
}
if (!Number.isFinite(timelineEnd)) {
throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`);
}
const compositionStart = Math.max(0, video.start);
const trimmedPreroll = compositionStart - video.start;
const trimmedSourcePreroll = sourceTimeAt(playbackRate, trimmedPreroll);
const timelineDuration = Math.max(0, timelineEnd - compositionStart);
// Infinity means "natural source duration", not an authored infinite slot.
// Explicit finite slots may outlive the source (loop or held tail), while an
// omitted duration remains source-bounded exactly like the browser runtime.
const resolvedVisibleDuration = resolvedDuration - trimmedPreroll;
const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration));
const visibleSourceDuration =
typeof playbackRate === "number"
? visibleDuration * playbackRate
: sourceTimeAt(playbackRate, trimmedPreroll + visibleDuration) - trimmedSourcePreroll;
let mediaStart = video.mediaStart + trimmedSourcePreroll;
if (visibleDuration > 0 && sourceDuration !== undefined) {
const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart);
if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) {
const phaseOffset = trimmedSourcePreroll % sourceRemaining;
const phaseRemaining = sourceRemaining - phaseOffset;
// Keep a complete cycle on equality too, otherwise a rebased suffix would
// wrap to its own first frame instead of the source cycle's first frame.
if (visibleSourceDuration >= phaseRemaining) {
return withTimelineDuration(
{
compositionStart: video.start,
mediaStart: video.mediaStart,
durationSeconds: sourceRemaining,
preserveTimelinePhase: true,
},
visibleDuration,
);
}
mediaStart = video.mediaStart + phaseOffset;
} else if (sourceRemaining > 0) {
const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedSourcePreroll);
if (visibleSourceDuration <= sourceVisibleAfterPreroll) {
return withTimelineDuration(
{
compositionStart,
mediaStart,
durationSeconds: visibleSourceDuration,
},
visibleDuration,
);
}
// The visible interval enters (or is entirely inside) the held tail.
// Extract the visible source suffix. If preroll is already at/past the
// final decoded timestamp, the async resolver below replaces this tiny
// provisional suffix with one exact final frame.
const extractionDuration = Math.min(
sourceRemaining,
Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS),
);
const extractionOffset = sourceRemaining - extractionDuration;
return withTimelineDuration(
{
compositionStart: video.start + timeAtSourceTime(playbackRate, extractionOffset),
mediaStart: video.mediaStart + extractionOffset,
durationSeconds: extractionDuration,
preserveTimelineEnd: true,
ensureFinalFrame: true,
},
visibleDuration,
);
} else if (canHoldFinalFramePastEof(video)) {
const logicalDuration = Math.min(sourceDuration, FINAL_FRAME_LOGICAL_DURATION_SECONDS);
return withTimelineDuration(
{
compositionStart: video.start,
mediaStart: sourceDuration - logicalDuration,
durationSeconds: logicalDuration,
preserveTimelineEnd: true,
ensureFinalFrame: true,
},
visibleDuration,
);
}
}
return withTimelineDuration(
{
compositionStart,
mediaStart,
durationSeconds: visibleSourceDuration,
},
visibleDuration,
);
}
/**
* Replace a held-tail suffix that starts at/after the final decoded timestamp
* with one exact frame. This keeps raw HDR scratch O(one frame) without
* assuming a one-second seek window contains a CFR/VFR timestamp.
*/
export async function resolveFinalFrameExtractionWindow(
videoPath: string,
video: TimelineWindowVideo,
metadata: VideoMetadata,
window: TimelineExtractionWindow,
signal?: AbortSignal,
): Promise {
if (!window.ensureFinalFrame) return window;
const playableDuration = resolvePlayableVideoDuration(metadata);
const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;
const lastInstantTheBrowserPlays = playableDuration - FINAL_FRAME_LOGICAL_DURATION_SECONDS;
const finalFrameTimestamp = Math.min(
lastInstantTheBrowserPlays,
videoStartOnMediaTimeline(metadata) +
(await extractFinalVideoFrameTimestamp(
videoPath,
{
videoStreamDurationSeconds: playableDuration - videoStreamLead,
videoStreamStartSeconds: metadata.videoStreamStartSeconds,
},
signal,
)),
);
if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
const sourceRemaining = playableDuration - video.mediaStart;
const logicalDuration = Math.min(
Math.max(sourceRemaining, window.durationSeconds),
FINAL_FRAME_LOGICAL_DURATION_SECONDS,
);
return {
compositionStart: Math.max(0, video.start),
mediaStart: playableDuration - logicalDuration,
extractionMediaStart: finalFrameTimestamp,
durationSeconds: logicalDuration,
...(window.timelineDurationSeconds !== undefined
? { timelineDurationSeconds: window.timelineDurationSeconds }
: {}),
preserveTimelineEnd: true,
finalFrameOnly: true,
};
}
/** Resolve source duration first, then intersect it with the render timeline. */
export function resolveVideoExtractionWindow(
video: TimelineWindowVideo,
metadata: VideoMetadata,
timelineEnd?: number,
): TimelineExtractionWindow {
const playableDuration = resolvePlayableVideoDuration(metadata);
if (!(playableDuration > 0)) {
throw new VideoSourceExtractionError(
"invalid_media",
false,
"Video source has no positive duration",
`Playable video stream duration is ${playableDuration}s`,
);
}
const requestedTimelineDuration = video.end - video.start;
const heldPastEof = video.mediaStart >= playableDuration && canHoldFinalFramePastEof(video);
if (video.mediaStart >= playableDuration && !heldPastEof) {
throw new VideoSourceExtractionError(
"media_start_out_of_range",
false,
"Video media start is outside the source duration",
`Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`,
);
}
const playbackRate = normalizeRateSpec(video.playbackRate);
const resolvedDuration =
Number.isFinite(requestedTimelineDuration) && requestedTimelineDuration > 0
? requestedTimelineDuration
: timeAtSourceTime(
playbackRate,
resolveSegmentDuration(requestedTimelineDuration, video.mediaStart, playableDuration),
);
return resolveTimelineExtractionWindow(
video,
resolvedDuration,
timelineEnd ?? (heldPastEof ? video.end : undefined),
playableDuration,
);
}
export function resolveVideoExtractionDuration(
video: TimelineWindowVideo,
metadata: VideoMetadata,
timelineEnd?: number,
): number {
return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds;
}
/** Codecs whose bitstream can carry alpha: decoded with the alpha-aware decoder whether or not a tag says so. */
const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
/** Alpha can hide from ffprobe: PNG unless a decoded frame had no alpha; ProRes alpha shows in pix_fmt. */
const SIDECAR_ALPHA_CODECS = new Set(["vp9", "vp8"]);
export function codecMayHaveAlpha(codec: string | undefined): boolean {
return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
}
export function decoderForCodec(codec: string | undefined): string {
return alphaDecoderFor(codec) ?? (codec ?? "").toLowerCase();
}
export function resolveFrameFormat(
metadata: VideoMetadata,
requested?: VideoFrameFormat,
): CacheFrameFormat {
const sidecar = SIDECAR_ALPHA_CODECS.has((metadata.videoCodec ?? "").toLowerCase());
if (metadata.hasAlpha || (sidecar && metadata.alphaPlaneDecoded !== false)) return "png";
if (requested === "png" || requested === "jpg") return requested;
return "jpg";
}
type PreparedExtraction = {
video: VideoElement;
videoPath: string;
index: number;
metadata: VideoMetadata;
videoDuration: number;
extractionMediaStart: number;
finalFrameOnly: boolean;
format: CacheFrameFormat;
sdrToHdrTransfer?: HdrTransfer;
hdrToSdrTransformKey?: string;
dedupeKey: string;
};
type CacheMissTarget = {
entry: CacheEntry;
srcPath: string;
};
type UniqueExtractionMiss = {
work: PreparedExtraction;
cacheTarget?: CacheMissTarget;
};
type SupersetMemberPlan = {
miss: UniqueExtractionMiss;
offsetFrames: number;
};
type SupersetGroupPlan = {
groupId: string;
baseStart: number;
unionDuration: number;
members: SupersetMemberPlan[];
};
function extractedFramesFromDirectory(
work: PreparedExtraction,
outputDir: string,
srcPath: string,
fps: number,
): ExtractedFrames {
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${work.format}`;
const framePaths = framePathsFromDirectory(outputDir, work.format);
return {
videoId: work.video.id,
srcPath,
outputDir,
framePattern,
fps,
totalFrames: framePaths.size,
metadata: work.metadata,
framePaths,
};
}
function frameFileName(frameNumber: number, format: CacheFrameFormat): string {
return `${FRAME_FILENAME_PREFIX}${String(frameNumber).padStart(5, "0")}.${format}`;
}
function linkOrCopyFrame(src: string, dest: string): void {
try {
linkSync(src, dest);
} catch {
copyFileSync(src, dest);
}
}
function supersetGroupingKey(work: PreparedExtraction, fps: number): string {
return [
work.videoPath,
String(fps),
work.format,
work.sdrToHdrTransfer ?? "",
work.hdrToSdrTransformKey ?? "",
work.finalFrameOnly ? "final" : "range",
].join("\0");
}
function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean {
const frames = offsetSeconds * fps;
return Math.abs(frames - Math.round(frames)) <= 1e-4;
}
function windowsOverlapOrTouch(misses: UniqueExtractionMiss[], baseStart: number): boolean {
const unionEnd = Math.max(
...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
);
const unionDuration = unionEnd - baseStart;
const summedDuration = misses.reduce((sum, { work }) => sum + work.videoDuration, 0);
return unionDuration > 0 && unionDuration <= summedDuration + 1e-9;
}
function buildSupersetGroup(
groupId: string,
misses: UniqueExtractionMiss[],
fps: number,
): SupersetGroupPlan | null {
if (misses.length < 2) return null;
if (misses.some(({ work }) => work.finalFrameOnly)) return null;
// VFR ranges round their frame counts up, so a union extraction's slices
// are not proven to match independently sought member ranges. Keep them direct.
if (misses.some(({ work }) => work.metadata.isVFR)) return null;
const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart));
if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) {
return null;
}
if (!windowsOverlapOrTouch(misses, baseStart)) return null;
const unionEnd = Math.max(
...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
);
return {
groupId,
baseStart,
unionDuration: unionEnd - baseStart,
members: misses.map((miss) => ({
miss,
offsetFrames: Math.round((miss.work.video.mediaStart - baseStart) * fps),
})),
};
}
/**
* Partition one source's misses into overlap-connected components: sort by
* window start and cut wherever the next window starts past the running end.
* Without this, one disjoint outlier trim (e.g. [100..105] next to three
* overlapping trims at [0..11]) fails the union<=sum check for the whole
* bucket and every trim falls back to direct extraction.
*/
function overlapClusters(misses: UniqueExtractionMiss[]): UniqueExtractionMiss[][] {
const sorted = [...misses].sort((a, b) => a.work.video.mediaStart - b.work.video.mediaStart);
const clusters: UniqueExtractionMiss[][] = [];
let current: UniqueExtractionMiss[] = [];
let currentEnd = -Infinity;
for (const miss of sorted) {
const start = miss.work.video.mediaStart;
const end = start + miss.work.videoDuration;
if (current.length > 0 && start > currentEnd + 1e-9) {
clusters.push(current);
current = [];
currentEnd = -Infinity;
}
current.push(miss);
currentEnd = Math.max(currentEnd, end);
}
if (current.length > 0) clusters.push(current);
return clusters;
}
function planSupersetGroups(
misses: UniqueExtractionMiss[],
fps: number,
): { groups: SupersetGroupPlan[]; direct: UniqueExtractionMiss[] } {
const bySource = new Map();
for (const miss of misses) {
const key = supersetGroupingKey(miss.work, fps);
bySource.set(key, [...(bySource.get(key) ?? []), miss]);
}
const groups: SupersetGroupPlan[] = [];
const direct: UniqueExtractionMiss[] = [];
let groupIndex = 0;
for (const groupMisses of bySource.values()) {
for (const cluster of overlapClusters(groupMisses)) {
const group = buildSupersetGroup(`__superset-${groupIndex}`, cluster, fps);
if (group) {
groups.push(group);
groupIndex += 1;
} else {
direct.push(...cluster);
}
}
}
return { groups, direct };
}
function sliceSupersetMember(
member: SupersetMemberPlan,
superset: ExtractedFrames,
outputDir: string,
fps: number,
configuredFps: FpsInput,
): ExtractedFrames {
const { work } = member.miss;
rmSync(outputDir, { recursive: true, force: true });
mkdirSync(outputDir, { recursive: true });
// Sample-time correctness: member frame k uses superset frame
// offset_i + k, so its source time is
// baseStart + (offset_i + k) / fps = mediaStart_i + k / fps.
// The frame-alignment precondition is what makes offset_i integral.
const requestedFrames = extractionFrameCountForDuration(
work.videoDuration,
configuredFps,
work.metadata.isVFR,
);
const availableFrames = Math.max(0, superset.totalFrames - member.offsetFrames);
const frameCount = Math.min(requestedFrames, availableFrames);
for (let i = 0; i < frameCount; i += 1) {
const sourceFrame = superset.framePaths.get(member.offsetFrames + i);
if (!sourceFrame) throw new Error(`superset frame ${member.offsetFrames + i} missing`);
linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format)));
}
return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps);
}
export async function extractAllVideoFrames(
videos: VideoElement[],
baseDir: string,
options: ExtractionOptions,
signal?: AbortSignal,
config?: Partial<
Pick
>,
compiledDir?: string,
): Promise {
if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) {
throw new Error(
`Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`,
);
}
const configuredFps = toFps(options.fps);
const fps = fpsToNumber(configuredFps);
const fpsKey = fpsToFfmpegArg(configuredFps);
const startTime = Date.now();
const extracted: ExtractedFrames[] = [];
const errors: VideoExtractionFailure[] = [];
let totalFramesExtracted = 0;
const breakdown: ExtractionPhaseBreakdown = {
resolveMs: 0,
cachePublishFailures: 0,
cacheGcEvictions: 0,
cacheGcBytesFreed: 0,
cacheAgedPartialsCleared: 0,
hdrProbeMs: 0,
hdrPreflightMs: 0,
hdrPreflightCount: 0,
vfrProbeMs: 0,
vfrPreflightMs: 0,
vfrPreflightCount: 0,
extractMs: 0,
cacheHits: 0,
cacheMisses: 0,
transientRetries: 0,
};
const recordTransientRetries = (count: number): void => {
breakdown.transientRetries = (breakdown.transientRetries ?? 0) + count;
};
// Phase 1: Resolve paths and download remote videos
const phase1Start = Date.now();
const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
// Dedupe missing-src warnings: a composition with N