mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-06 14:47:29 +00:00
Miguel Ángel
472ee43c3a
* fix(engine): long 4K video exports no longer time out while frames are still coming * test(engine): the paced extraction test keeps a wide margin on a loaded runner * fix(engine): long phone-video exports probe each source once instead of once per cut * fix(engine): a failed source probe is retried, and probes share the extraction slots * test(engine): synthesize the tone-mapped concurrency fixture with known input colour
2866 lines
102 KiB
TypeScript
2866 lines
102 KiB
TypeScript
// fallow-ignore-file unused-class-member code-duplication complexity
|
|
/**
|
|
* Video Frame Extractor Service
|
|
*
|
|
* Pre-extracts video frames using FFmpeg for frame-accurate rendering.
|
|
* Videos are replaced with <img> elements during capture.
|
|
*/
|
|
|
|
import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./mediaHidden.js";
|
|
import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs";
|
|
import { join } from "path";
|
|
import { cpus } from "os";
|
|
import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";
|
|
import { parseHTML } from "linkedom";
|
|
import { isPathInside } from "@hyperframes/parsers/asset-paths";
|
|
import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";
|
|
import {
|
|
MEDIA_RENDER_ID_ATTR,
|
|
isAudibleVideoElement,
|
|
fpsToFfmpegArg,
|
|
fpsToNumber,
|
|
MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
|
|
normalizeRateSpec,
|
|
readElementRateSpec,
|
|
shiftRateLane,
|
|
sourceTimeAt,
|
|
timeAtSourceTime,
|
|
type RateSpec,
|
|
exportClipWindow,
|
|
hasClipStarted,
|
|
isInClipWindow,
|
|
parseStrictFiniteTimingNumber,
|
|
readMediaStart,
|
|
toFps,
|
|
type FpsInput,
|
|
firstFrameColourArgs,
|
|
hdrToSdrToneMapFilter,
|
|
parseFirstFrameColour,
|
|
} from "@hyperframes/core";
|
|
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
|
|
import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
|
|
import {
|
|
extractFinalVideoFrameTimestamp,
|
|
extractMediaMetadata,
|
|
mediaFileIdentity,
|
|
readMetadataCache,
|
|
writeMetadataCache,
|
|
type MediaProbeCacheEntry,
|
|
type VideoMetadata,
|
|
} from "../utils/ffprobe.js";
|
|
import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
|
|
import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js";
|
|
import {
|
|
analyzeCompositionHdr,
|
|
isHdrColorSpace as isHdrColorSpaceUtil,
|
|
type HdrTransfer,
|
|
} from "../utils/hdr.js";
|
|
import {
|
|
downloadToTemp,
|
|
isHttpUrl,
|
|
safeDownloadUrlIdentity,
|
|
UrlDownloadError,
|
|
writeUrlDownloadTelemetry,
|
|
} from "../utils/urlDownloader.js";
|
|
import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
|
|
import { isFfmpegFilterAvailable } from "../utils/psnrFilterAvailability.js";
|
|
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
|
import { unwrapTemplate } from "../utils/htmlTemplate.js";
|
|
import {
|
|
FRAME_FILENAME_PREFIX,
|
|
gcExtractionCache,
|
|
gcSweepDue,
|
|
lookupCacheEntry,
|
|
partialCacheEntryDir,
|
|
publishCacheEntry,
|
|
readContentSha256,
|
|
readKeyStat,
|
|
rehydrateCacheEntry,
|
|
touchCacheEntry,
|
|
type CacheEntry,
|
|
type CacheFrameFormat,
|
|
} from "./extractionCache.js";
|
|
import { framePathsFromDirectory } from "./extractedFrameIndex.js";
|
|
|
|
export { resolveProjectRelativeSrc };
|
|
|
|
export interface VideoElement {
|
|
id: string;
|
|
src: string;
|
|
start: number;
|
|
end: number;
|
|
mediaStart: number;
|
|
playbackRate?: RateSpec;
|
|
loop: boolean;
|
|
hasAudio: boolean;
|
|
hidden?: boolean;
|
|
}
|
|
|
|
export interface ExtractedFrames {
|
|
videoId: string;
|
|
srcPath: string;
|
|
outputDir: string;
|
|
framePattern: string;
|
|
fps: number;
|
|
totalFrames: number;
|
|
metadata: VideoMetadata;
|
|
framePaths: Map<number, string>;
|
|
deferredRange?: DeferredFrameRange;
|
|
/**
|
|
* True when the extractor owns `outputDir` and cleanup should rm it when
|
|
* the render ends. Cache hits set this to false so the shared entry isn't
|
|
* deleted by a single render's cleanup — the cache dir is owned by the
|
|
* caller's gc policy, not any one render.
|
|
*/
|
|
ownedByLookup?: boolean;
|
|
}
|
|
|
|
/**
|
|
* The single source of truth for the source-video frame-extraction allow-list.
|
|
* The CLI flag parser, the producer HTTP server, and the distributed-config
|
|
* validator all validate against this same set via {@link isVideoFrameFormat}
|
|
* so the boundaries can't drift when a new format is added.
|
|
*/
|
|
export const VIDEO_FRAME_FORMATS = ["auto", "jpg", "png"] as const;
|
|
export type VideoFrameFormat = (typeof VIDEO_FRAME_FORMATS)[number];
|
|
|
|
/** Runtime guard for {@link VideoFrameFormat} over an untrusted value. */
|
|
export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat {
|
|
return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value);
|
|
}
|
|
|
|
/**
|
|
* Resolve the frame count produced for a requested extraction duration.
|
|
*
|
|
* Both paths sample with FFmpeg's fps filter. A CFR window ends at the nearest
|
|
* frame boundary; a VFR window rounds up, keeping the slot a trailing still
|
|
* covers. Keep this calculation shared by superset slicing and
|
|
* producer coverage accounting so a complete VFR extraction cannot be
|
|
* rejected because the two paths disagree by one frame.
|
|
*/
|
|
export function extractionFrameCountForDuration(
|
|
durationSeconds: number,
|
|
fps: FpsInput,
|
|
isVFR: boolean,
|
|
): number {
|
|
if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) return 0;
|
|
// FFmpeg receives `String(durationSeconds)` and parses at microsecond
|
|
// precision. Derive the integer microseconds from that same decimal text:
|
|
// multiplying the binary float first is not equivalent (`2.05 * 1e6` is
|
|
// 2049999.9999999998 in JS and would incorrectly truncate one microsecond).
|
|
const serialized = String(durationSeconds).toLowerCase();
|
|
const decimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serialized);
|
|
if (!decimal) return 0;
|
|
const whole = decimal[1] ?? "0";
|
|
const fraction = decimal[2] ?? "";
|
|
const exponent = Number.parseInt(decimal[3] ?? "0", 10);
|
|
const digits = BigInt(`${whole}${fraction}`);
|
|
const microsecondScale = exponent + 6 - fraction.length;
|
|
const microseconds =
|
|
microsecondScale >= 0
|
|
? digits * 10n ** BigInt(microsecondScale)
|
|
: digits / 10n ** BigInt(-microsecondScale);
|
|
|
|
// Keep the frame-boundary calculation rational too. Converting the exact
|
|
// microseconds back to a binary float recreates the same problem at .5-frame
|
|
// boundaries (`2.05 * 30` is 61.49999999999999 in JS).
|
|
let fpsNumerator: bigint;
|
|
let fpsDenominator: bigint;
|
|
if (typeof fps === "object") {
|
|
if (
|
|
!Number.isSafeInteger(fps.num) ||
|
|
!Number.isSafeInteger(fps.den) ||
|
|
fps.num <= 0 ||
|
|
fps.den <= 0
|
|
) {
|
|
return 0;
|
|
}
|
|
fpsNumerator = BigInt(fps.num);
|
|
fpsDenominator = BigInt(fps.den);
|
|
} else {
|
|
if (!Number.isFinite(fps) || fps <= 0) return 0;
|
|
// Number-only callers retain their decimal FFmpeg argument exactly. The
|
|
// production render path supplies Fps, so NTSC rates never round-trip
|
|
// through `String(30000 / 1001)` here.
|
|
const serializedFps = String(fps).toLowerCase();
|
|
const fpsDecimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serializedFps);
|
|
if (!fpsDecimal) return 0;
|
|
const fpsWhole = fpsDecimal[1] ?? "0";
|
|
const fpsFraction = fpsDecimal[2] ?? "";
|
|
const fpsExponent = Number.parseInt(fpsDecimal[3] ?? "0", 10);
|
|
const fpsDigits = BigInt(`${fpsWhole}${fpsFraction}`);
|
|
const fpsScale = fpsExponent - fpsFraction.length;
|
|
fpsNumerator = fpsScale >= 0 ? fpsDigits * 10n ** BigInt(fpsScale) : fpsDigits;
|
|
fpsDenominator = fpsScale >= 0 ? 1n : 10n ** BigInt(-fpsScale);
|
|
}
|
|
|
|
const frameNumerator = microseconds * fpsNumerator;
|
|
const frameDenominator = 1_000_000n * fpsDenominator;
|
|
const frameCount = isVFR
|
|
? (frameNumerator + frameDenominator - 1n) / frameDenominator
|
|
: (2n * frameNumerator + frameDenominator) / (2n * frameDenominator);
|
|
const frames = Number(frameCount);
|
|
return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER);
|
|
}
|
|
|
|
export interface FrameRange {
|
|
firstFrame: number;
|
|
frames: number;
|
|
}
|
|
|
|
/** The window a deferred extraction samples; frame k is the source frame on screen at `startTime + k / fps`. */
|
|
export interface DeferredFrameRange {
|
|
startTime: number;
|
|
durationSeconds: number;
|
|
format: "jpg" | "png";
|
|
}
|
|
|
|
export interface ExtractionOptions {
|
|
/** Exact configured rate. Rational rates are passed to FFmpeg verbatim. */
|
|
fps: FpsInput;
|
|
outputDir: string;
|
|
quality?: number;
|
|
format?: VideoFrameFormat;
|
|
sdrToHdrTransfer?: HdrTransfer;
|
|
toneMapHdrToSdr?: boolean;
|
|
/** Extract exactly one frame at `startTime`. Used only after ffprobe has
|
|
* resolved the actual final decoded-frame timestamp for a held tail. */
|
|
finalFrameOnly?: boolean;
|
|
/**
|
|
* Absolute composition/timeline end in seconds. Applied only after source
|
|
* metadata resolves open-ended/natural-duration media. Invisible negative
|
|
* preroll is trimmed while advancing mediaStart to preserve source alignment.
|
|
*/
|
|
timelineEnd?: number;
|
|
/**
|
|
* Bounded per-source FFmpeg retries. Default 0 preserves stable behavior;
|
|
* the producer may canary at most one retry after observing typed failures.
|
|
*/
|
|
maxTransientRetries?: number;
|
|
/**
|
|
* Collect metadata-probe failures into `ExtractionResult.errors` instead
|
|
* of preserving the legacy Promise rejection. Default false; only the
|
|
* candidate enforce lane may opt into typed aggregation.
|
|
*/
|
|
collectProbeFailures?: boolean;
|
|
/** Extract only these runs of output frames, named as a full extraction names them. CFR only. */
|
|
frameRanges?: readonly FrameRange[];
|
|
/**
|
|
* Leave CFR windows longer than a second unextracted, returning `deferredRange` and an exact
|
|
* `totalFrames` so another process can extract just the frames it needs with `frameRanges`.
|
|
*/
|
|
deferRangeExtraction?: boolean;
|
|
/** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
|
|
contentKeyedDirs?: readonly string[];
|
|
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
|
|
}
|
|
|
|
const DOWNLOAD_SUBDIR = "_downloads";
|
|
|
|
export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
|
|
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
|
|
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
|
|
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
|
|
const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
|
|
const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
|
|
const SDR_JPEG_AS_BT601_FULL_RANGE_FILTER =
|
|
"scale=flags=neighbor+accurate_rnd,format=gbrp,scale=out_color_matrix=bt601:out_range=pc:flags=neighbor,format=yuv420p";
|
|
|
|
let warnedMissingZscale = false;
|
|
|
|
async function toneMapsWithZscale(): Promise<boolean> {
|
|
return process.platform !== "darwin" || isFfmpegFilterAvailable("zscale");
|
|
}
|
|
|
|
async function hdrToSdrTransformKey(): Promise<string> {
|
|
if (await toneMapsWithZscale()) return HDR_TO_SDR_ZSCALE_TRANSFORM_KEY;
|
|
if (!warnedMissingZscale) {
|
|
warnedMissingZscale = true;
|
|
process.stderr.write(
|
|
"[hyperframes:render] WARNING: this ffmpeg has no zscale filter, so forced-SDR HDR footage " +
|
|
"is tone-mapped by Chrome instead of the hable curve used on Linux and in Studio, and " +
|
|
"renders darker. Install an ffmpeg built with libzimg to match.\n",
|
|
);
|
|
}
|
|
return HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY;
|
|
}
|
|
|
|
function sdrToHdrTransformKey(transfer: HdrTransfer): string {
|
|
return `sdr2hdr-${transfer}`;
|
|
}
|
|
|
|
/**
|
|
* Per-phase timings and counters emitted by `extractAllVideoFrames`.
|
|
*
|
|
* Used by the producer to surface `perfSummary.videoExtractBreakdown` — without
|
|
* this breakdown, a single `videoExtractMs` stage timing hides where cost lives
|
|
* (HDR preflight, VFR classification, per-video ffmpeg extract) when tuning renders.
|
|
*
|
|
* Field semantics:
|
|
* - *Ms fields are wall-clock durations inside each phase.
|
|
* - *Count fields report how many sources triggered that phase.
|
|
* - extractMs wraps the parallel `extractVideoFramesRange` calls; it
|
|
* reflects max-across-parallel-workers, not sum.
|
|
* - hdrPreflightMs includes its probe-time sibling (hdrProbeMs); the
|
|
* probe-only field is a finer decomposition, not a separate carve-out.
|
|
* - vfrPreflightCount reports sources classified as VFR and extracted with
|
|
* a rounded-up frame count. DEFINITION CHANGE:
|
|
* before the one-pass refactor, vfrPreflightMs timed a per-source
|
|
* VFR-to-CFR re-encode and could reach seconds; it now times only the
|
|
* (promise-cached) classification probe and is expected to be ~0.
|
|
* Dashboards alerting on vfrPreflightMs thresholds should key on
|
|
* vfrPreflightCount or extractMs instead.
|
|
*/
|
|
export interface ExtractionPhaseBreakdown {
|
|
resolveMs: number;
|
|
/** Publishes that could not land atomically — the render still succeeded
|
|
* from the partial dir, but future renders re-extract. A rising rate is
|
|
* the first signal that warm renders are silently going cold. */
|
|
cachePublishFailures: number;
|
|
/** Entries evicted by the post-extraction LRU sweep. */
|
|
cacheGcEvictions: number;
|
|
/** Bytes reclaimed by the LRU sweep. */
|
|
cacheGcBytesFreed: number;
|
|
/** Aged .partial-* dirs (crashed writers) removed by the sweep. */
|
|
cacheAgedPartialsCleared: number;
|
|
hdrProbeMs: number;
|
|
hdrPreflightMs: number;
|
|
hdrPreflightCount: number;
|
|
vfrProbeMs: number;
|
|
vfrPreflightMs: number;
|
|
vfrPreflightCount: number;
|
|
extractMs: number;
|
|
cacheHits: number;
|
|
cacheMisses: number;
|
|
/** Number of per-source transient failures retried inside this extraction. */
|
|
transientRetries?: number;
|
|
}
|
|
|
|
export type VideoExtractionFailureKind =
|
|
| "cancelled"
|
|
| "external_interruption"
|
|
| "source_missing"
|
|
| "source_rejected"
|
|
| "download_not_found"
|
|
| "download_transient"
|
|
| "invalid_media"
|
|
| "media_start_out_of_range"
|
|
| "ffmpeg_unavailable"
|
|
| "ffmpeg_timeout"
|
|
| "ffmpeg_transient"
|
|
| "ffmpeg_failed"
|
|
| "zero_output"
|
|
| "internal";
|
|
|
|
export interface VideoExtractionFailure {
|
|
videoId: string;
|
|
/** Always populated by this engine version; optional for source compatibility with older consumers. */
|
|
kind?: VideoExtractionFailureKind;
|
|
/** Always populated by this engine version; absent legacy values fail closed. */
|
|
retryable?: boolean;
|
|
/** Bounded, path-free grouping data safe for producer-owned error metadata. */
|
|
group?: VideoExtractionFailureGroupDetails;
|
|
/**
|
|
* Operator diagnostic retained inside the engine result. Producer-facing
|
|
* errors must summarize `kind`/counts and must not forward this field: it
|
|
* can contain a local path or a signed source URL.
|
|
*/
|
|
error: string;
|
|
}
|
|
|
|
export type VideoExtractionFailureStatusClass =
|
|
| "http_4xx"
|
|
| "http_5xx"
|
|
| "timeout"
|
|
| "network"
|
|
| "other";
|
|
|
|
export interface VideoExtractionFailureRetry {
|
|
phase: "download";
|
|
used: 0 | 1;
|
|
budget: 1;
|
|
}
|
|
|
|
export interface VideoExtractionFailureGroupDetails {
|
|
sourceFingerprint?: string;
|
|
host?: string;
|
|
statusClass?: VideoExtractionFailureStatusClass;
|
|
retry?: VideoExtractionFailureRetry;
|
|
}
|
|
|
|
export interface SafeVideoExtractionSourceIdentity {
|
|
sourceFingerprint: string;
|
|
host: string;
|
|
}
|
|
|
|
/** Query/fragment-free remote identity safe for extraction logs and wire metadata. */
|
|
export function safeVideoExtractionSourceIdentity(
|
|
source: string,
|
|
): SafeVideoExtractionSourceIdentity | null {
|
|
if (!isHttpUrl(source)) return null;
|
|
const identity = safeDownloadUrlIdentity(source);
|
|
return {
|
|
sourceFingerprint: `sha256:${identity.urlFingerprint}`,
|
|
host: identity.host ?? "other",
|
|
};
|
|
}
|
|
|
|
function downloadStatusClass(error: UrlDownloadError): VideoExtractionFailureStatusClass {
|
|
const status = error.status ?? error.telemetry?.status;
|
|
if (typeof status === "number" && status >= 400 && status < 500) return "http_4xx";
|
|
if (typeof status === "number" && status >= 500 && status < 600) return "http_5xx";
|
|
if (error.kind === "timeout") return "timeout";
|
|
if (error.kind === "network") return "network";
|
|
return "other";
|
|
}
|
|
|
|
function downloadFailureGroup(
|
|
source: string,
|
|
error: UrlDownloadError,
|
|
): VideoExtractionFailureGroupDetails {
|
|
const sourceIdentity = safeVideoExtractionSourceIdentity(source);
|
|
const failureHost = error.telemetry?.finalHost ?? error.telemetry?.initialHost;
|
|
const attempt = error.telemetry?.attempt;
|
|
return {
|
|
...(sourceIdentity ? { sourceFingerprint: sourceIdentity.sourceFingerprint } : {}),
|
|
...((failureHost ?? sourceIdentity?.host) ? { host: failureHost ?? sourceIdentity?.host } : {}),
|
|
statusClass: downloadStatusClass(error),
|
|
retry: {
|
|
phase: "download",
|
|
used: typeof attempt === "number" && attempt >= 2 ? 1 : 0,
|
|
budget: 1,
|
|
},
|
|
};
|
|
}
|
|
|
|
export class VideoSourceExtractionError extends Error {
|
|
readonly hyperframesVideoSourceExtractionError = true as const;
|
|
|
|
constructor(
|
|
readonly kind: VideoExtractionFailureKind,
|
|
readonly retryable: boolean,
|
|
message: string,
|
|
readonly diagnostic: string = message,
|
|
) {
|
|
super(message);
|
|
this.name = "VideoSourceExtractionError";
|
|
}
|
|
}
|
|
|
|
export function isVideoSourceExtractionError(error: unknown): error is VideoSourceExtractionError {
|
|
return (
|
|
typeof error === "object" &&
|
|
error !== null &&
|
|
"hyperframesVideoSourceExtractionError" in error &&
|
|
error.hyperframesVideoSourceExtractionError === true
|
|
);
|
|
}
|
|
|
|
function boundedTransientRetryBudget(value: number | undefined): 0 | 1 {
|
|
return Number.isFinite(value) && (value ?? 0) >= 1 ? 1 : 0;
|
|
}
|
|
|
|
function errorText(error: unknown): string {
|
|
return error instanceof Error ? error.message : String(error);
|
|
}
|
|
|
|
/**
|
|
* Convert legacy/raw downloader and filesystem errors into the bounded
|
|
* extraction taxonomy. New extraction code should throw
|
|
* `VideoSourceExtractionError` directly; this classifier keeps older utility
|
|
* boundaries safe while they migrate.
|
|
*/
|
|
export function classifyVideoExtractionError(error: unknown): VideoSourceExtractionError {
|
|
if (isVideoSourceExtractionError(error)) return error;
|
|
const diagnostic = errorText(error);
|
|
const lowered = diagnostic.toLowerCase();
|
|
|
|
if (error instanceof UrlDownloadError) {
|
|
if (error.kind === "cancelled") {
|
|
return new VideoSourceExtractionError(
|
|
"cancelled",
|
|
false,
|
|
"Video extraction cancelled",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (error.kind === "http_not_found") {
|
|
return new VideoSourceExtractionError(
|
|
"download_not_found",
|
|
false,
|
|
"Video source was not found",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (error.kind === "http_rejected") {
|
|
return new VideoSourceExtractionError(
|
|
"source_rejected",
|
|
false,
|
|
"Video source download was rejected",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (error.kind === "invalid_payload") {
|
|
return new VideoSourceExtractionError(
|
|
"invalid_media",
|
|
false,
|
|
"Video source download returned a non-media payload",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (error.retryable) {
|
|
return new VideoSourceExtractionError(
|
|
"download_transient",
|
|
true,
|
|
"Video source download failed transiently",
|
|
diagnostic,
|
|
);
|
|
}
|
|
return new VideoSourceExtractionError(
|
|
"internal",
|
|
false,
|
|
"Video source download failed internally",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (lowered.includes("cancelled") || lowered.includes("aborted")) {
|
|
return new VideoSourceExtractionError(
|
|
"cancelled",
|
|
false,
|
|
"Video extraction cancelled",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (lowered.includes("video file not found")) {
|
|
return new VideoSourceExtractionError(
|
|
"source_missing",
|
|
false,
|
|
"Video source is missing",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (
|
|
lowered.includes("only https urls are permitted") ||
|
|
lowered.includes("private/reserved address") ||
|
|
lowered.includes("invalid url")
|
|
) {
|
|
return new VideoSourceExtractionError(
|
|
"source_rejected",
|
|
false,
|
|
"Video source URL is not permitted",
|
|
diagnostic,
|
|
);
|
|
}
|
|
const httpStatus = diagnostic.match(/\bHTTP\s+(\d{3})\b/i)?.[1];
|
|
if (httpStatus) {
|
|
const status = Number(httpStatus);
|
|
if (status === 404 || status === 410) {
|
|
return new VideoSourceExtractionError(
|
|
"download_not_found",
|
|
false,
|
|
"Video source was not found",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (status === 408 || status === 429 || status >= 500) {
|
|
return new VideoSourceExtractionError(
|
|
"download_transient",
|
|
true,
|
|
"Video source download failed transiently",
|
|
diagnostic,
|
|
);
|
|
}
|
|
return new VideoSourceExtractionError(
|
|
"source_rejected",
|
|
false,
|
|
"Video source download was rejected",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (
|
|
lowered.includes("[urldownloader] download timeout") ||
|
|
lowered.includes("[urldownloader] download failed") ||
|
|
lowered.includes("fetch failed") ||
|
|
lowered.includes("network")
|
|
) {
|
|
return new VideoSourceExtractionError(
|
|
"download_transient",
|
|
true,
|
|
"Video source download failed transiently",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (lowered.includes("ffprobe not found")) {
|
|
return new VideoSourceExtractionError(
|
|
"ffmpeg_unavailable",
|
|
false,
|
|
"FFprobe is unavailable",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (lowered.includes("ffprobe deadline")) {
|
|
return new VideoSourceExtractionError(
|
|
"ffmpeg_timeout",
|
|
true,
|
|
"Video inspection timed out",
|
|
diagnostic,
|
|
);
|
|
}
|
|
if (
|
|
lowered.includes("ffprobe") ||
|
|
lowered.includes("failed to parse ffprobe output") ||
|
|
lowered.includes("no video stream found")
|
|
) {
|
|
return new VideoSourceExtractionError(
|
|
"invalid_media",
|
|
false,
|
|
"Video source could not be inspected",
|
|
diagnostic,
|
|
);
|
|
}
|
|
return new VideoSourceExtractionError(
|
|
"internal",
|
|
false,
|
|
"Video extraction failed internally",
|
|
diagnostic,
|
|
);
|
|
}
|
|
|
|
export async function runVideoExtractionWithRetry<T>(
|
|
operation: () => Promise<T>,
|
|
options: {
|
|
signal?: AbortSignal;
|
|
onRetry?: () => Promise<void> | void;
|
|
maxTransientRetries?: number;
|
|
} = {},
|
|
): Promise<{ result: T; retries: number }> {
|
|
const maxTransientRetries = boundedTransientRetryBudget(options.maxTransientRetries);
|
|
let retries = 0;
|
|
for (;;) {
|
|
if (options.signal?.aborted) {
|
|
throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
|
|
}
|
|
try {
|
|
return { result: await operation(), retries };
|
|
} catch (error) {
|
|
const classified = classifyVideoExtractionError(error);
|
|
if (options.signal?.aborted) {
|
|
throw new VideoSourceExtractionError(
|
|
"cancelled",
|
|
false,
|
|
"Video extraction cancelled",
|
|
classified.diagnostic,
|
|
);
|
|
}
|
|
if (
|
|
classified.kind === "cancelled" ||
|
|
!classified.retryable ||
|
|
retries >= maxTransientRetries
|
|
) {
|
|
throw classified;
|
|
}
|
|
retries += 1;
|
|
await options.onRetry?.();
|
|
}
|
|
}
|
|
}
|
|
|
|
export interface ExtractionResult {
|
|
success: boolean;
|
|
extracted: ExtractedFrames[];
|
|
errors: VideoExtractionFailure[];
|
|
totalFramesExtracted: number;
|
|
durationMs: number;
|
|
phaseBreakdown: ExtractionPhaseBreakdown;
|
|
}
|
|
|
|
/** Minimal structural shape for resolving parent/`<source>` media `src`. */
|
|
interface MediaSrcEl {
|
|
getAttribute(name: string): string | null;
|
|
querySelectorAll(selectors: string): Iterable<{ getAttribute(name: string): string | null }>;
|
|
}
|
|
|
|
/**
|
|
* Parent `src`, else a `<source src>`. Prefer local paths over http(s) so a
|
|
* localized sibling wins when another `<source>` failed to download.
|
|
*/
|
|
export function resolveMediaElementSrc(el: MediaSrcEl): string | null {
|
|
const direct = el.getAttribute("src");
|
|
if (direct) return direct;
|
|
let remote: string | null = null;
|
|
for (const source of el.querySelectorAll("source")) {
|
|
const src = source.getAttribute("src");
|
|
if (!src) continue;
|
|
if (!/^https?:\/\//i.test(src)) return src;
|
|
remote ??= src;
|
|
}
|
|
return remote;
|
|
}
|
|
|
|
export function parseVideoElements(html: string): VideoElement[] {
|
|
const videos: VideoElement[] = [];
|
|
const { document } = parseHTML(unwrapTemplate(html));
|
|
const startCache = new Map<RefResolverEl, number>();
|
|
const visiting = new Set<RefResolverEl>();
|
|
|
|
const groupsById = audioGroupsById(document);
|
|
const videoEls = document.querySelectorAll("video");
|
|
let autoIdCounter = 0;
|
|
for (const el of videoEls) {
|
|
const src = resolveMediaElementSrc(el);
|
|
if (!src) continue;
|
|
// Generate a stable ID for videos without one — the producer needs IDs
|
|
// to track extracted frames and composite them during encoding.
|
|
// A compiled render document stamps a document-unique render id; prefer it,
|
|
// because element ids are only unique within one composition file and the
|
|
// render document inlines many.
|
|
const id =
|
|
el.getAttribute(MEDIA_RENDER_ID_ATTR) ||
|
|
el.getAttribute("id") ||
|
|
`hf-video-${autoIdCounter++}`;
|
|
if (!el.getAttribute("id")) {
|
|
el.setAttribute("id", id);
|
|
}
|
|
|
|
const startAttr = el.getAttribute("data-start");
|
|
const endAttr = el.getAttribute("data-end");
|
|
const durationAttr = el.getAttribute("data-duration");
|
|
const hasAudioAttr = el.getAttribute("data-has-audio");
|
|
|
|
// Resolve data-start, including relative references ("intro", "intro + 2")
|
|
// to another clip's end — the browser runtime resolves these but a raw
|
|
// parseFloat here would yield NaN, placing the clip at NaN so it composites
|
|
// blank in the final render. `startAttr` may be a plain number or a
|
|
// reference; the resolver handles both.
|
|
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
|
|
if (isKnownInactiveTimelineWindow(el, start)) continue;
|
|
// Derive end from data-end → data-start+data-duration → Infinity (natural duration).
|
|
// Static compilation cannot always clamp root media because GSAP may supply
|
|
// the root duration at runtime. The producer passes the resolved timeline
|
|
// end into frame extraction, which caps the source duration only after the
|
|
// natural duration is known without rewriting authored timing metadata.
|
|
let end = 0;
|
|
const authoredEnd = parseStrictFiniteTimingNumber(endAttr);
|
|
const authoredDuration = parseStrictFiniteTimingNumber(durationAttr);
|
|
if (authoredEnd != null) {
|
|
end = authoredEnd;
|
|
} else if (authoredDuration != null) {
|
|
end = start + authoredDuration;
|
|
} else {
|
|
end = Infinity; // no explicit bounds — play for the full natural video duration
|
|
}
|
|
|
|
videos.push({
|
|
id,
|
|
src,
|
|
start,
|
|
end,
|
|
mediaStart: readMediaStart(el),
|
|
playbackRate: readElementRateSpec(el),
|
|
loop: el.hasAttribute("loop"),
|
|
hasAudio: hasAudioAttr === "true",
|
|
...(isSelfOrAncestorHidden(el) ||
|
|
(isAudibleVideoElement(el) && isMemberGroupHidden(groupsById, el))
|
|
? { hidden: true }
|
|
: {}),
|
|
});
|
|
}
|
|
|
|
return videos;
|
|
}
|
|
|
|
export interface ImageElement {
|
|
id: string;
|
|
src: string;
|
|
start: number;
|
|
end: number;
|
|
}
|
|
|
|
export function parseImageElements(html: string): ImageElement[] {
|
|
const images: ImageElement[] = [];
|
|
const { document } = parseHTML(unwrapTemplate(html));
|
|
const startCache = new Map<RefResolverEl, number>();
|
|
const visiting = new Set<RefResolverEl>();
|
|
|
|
const imgEls = document.querySelectorAll("img[src]");
|
|
let autoIdCounter = 0;
|
|
for (const el of imgEls) {
|
|
const src = el.getAttribute("src");
|
|
if (!src) continue;
|
|
|
|
// See parseVideoElements: the stamped render id wins over the authored id.
|
|
const id =
|
|
el.getAttribute(MEDIA_RENDER_ID_ATTR) || el.getAttribute("id") || `hf-img-${autoIdCounter++}`;
|
|
if (!el.getAttribute("id")) {
|
|
el.setAttribute("id", id);
|
|
}
|
|
|
|
const startAttr = el.getAttribute("data-start");
|
|
const endAttr = el.getAttribute("data-end");
|
|
const durationAttr = el.getAttribute("data-duration");
|
|
|
|
// Resolve relative data-start references (see parseVideoElements) so a
|
|
// referenced image start doesn't become NaN and drop the image from the render.
|
|
const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
|
|
let end = 0;
|
|
const authoredEnd = parseStrictFiniteTimingNumber(endAttr);
|
|
const authoredDuration = parseStrictFiniteTimingNumber(durationAttr);
|
|
if (authoredEnd != null) {
|
|
end = authoredEnd;
|
|
} else if (authoredDuration != null) {
|
|
end = start + authoredDuration;
|
|
} else {
|
|
end = Infinity;
|
|
}
|
|
|
|
images.push({ id, src, start, end });
|
|
}
|
|
|
|
return images;
|
|
}
|
|
|
|
const firstFrameColourCache = new Map<
|
|
string,
|
|
MediaProbeCacheEntry<ReturnType<typeof parseFirstFrameColour>>
|
|
>();
|
|
|
|
/** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */
|
|
function readFirstFrameColour(videoPath: string) {
|
|
const identity = mediaFileIdentity(videoPath);
|
|
const cached = readMetadataCache(firstFrameColourCache, videoPath, identity);
|
|
if (cached) return cached;
|
|
// In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others.
|
|
const probe = inExtractionSlot(() =>
|
|
runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]),
|
|
).then((result) => {
|
|
// A failed probe answers this cut only; the next one probes again.
|
|
if (!result.success && firstFrameColourCache.get(videoPath)?.promise === probe) {
|
|
firstFrameColourCache.delete(videoPath);
|
|
}
|
|
return parseFirstFrameColour(result.stderr);
|
|
});
|
|
writeMetadataCache(firstFrameColourCache, videoPath, identity, probe);
|
|
return probe;
|
|
}
|
|
|
|
/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */
|
|
const CHROME_BT601_UNTAGGED_CODECS = new Set(["vp9", "av1"]);
|
|
|
|
function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
|
|
const matrix = metadata.colorSpace?.colorSpace;
|
|
if (matrix && matrix !== "unknown") return [];
|
|
const hd = metadata.height >= 720 && !CHROME_BT601_UNTAGGED_CODECS.has(metadata.videoCodec);
|
|
return [`setparams=colorspace=${hd ? "bt709" : "smpte170m"}`];
|
|
}
|
|
|
|
/**
|
|
* Longest stretch of output one ffmpeg process extracts; longer ranges split into segments
|
|
* that run side by side. Segments sample the same absolute times as a single pass, so their
|
|
* frames are byte-identical to it.
|
|
*/
|
|
const EXTRACTION_SEGMENT_SECONDS = 120;
|
|
/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
|
|
function undoTickRoundingOnTheSourceFrameGrid(seek: number, lead: number): string[] {
|
|
const sinceFileStart = `(PTS*TB+${seek}-${lead})`;
|
|
return [
|
|
"settb=AVTB",
|
|
`setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,
|
|
];
|
|
}
|
|
|
|
// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.
|
|
const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));
|
|
const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);
|
|
|
|
interface SegmentedExtraction {
|
|
decodeArgs: string[];
|
|
filterAndEncodeArgs: (seek: number) => string[];
|
|
videoPath: string;
|
|
startTime: number;
|
|
fps: number;
|
|
segments: readonly FrameRange[];
|
|
runOptions: { signal?: AbortSignal; stallTimeout: number };
|
|
}
|
|
|
|
/** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */
|
|
function splitFrameRanges(ranges: readonly FrameRange[], segmentFrames: number): FrameRange[] {
|
|
const segments: FrameRange[] = [];
|
|
for (const range of ranges) {
|
|
for (let offset = 0; offset < range.frames; offset += segmentFrames) {
|
|
segments.push({
|
|
firstFrame: range.firstFrame + offset,
|
|
frames: Math.min(segmentFrames, range.frames - offset),
|
|
});
|
|
}
|
|
}
|
|
return segments;
|
|
}
|
|
|
|
async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmpegResult> {
|
|
const { decodeArgs, filterAndEncodeArgs, videoPath, startTime, fps, segments } = job;
|
|
const segmentCount = segments.length;
|
|
// One failed segment fails the range, so stop the others instead of finishing them.
|
|
const failed = new AbortController();
|
|
const signal = job.runOptions.signal
|
|
? AbortSignal.any([job.runOptions.signal, failed.signal])
|
|
: failed.signal;
|
|
const results: RunFfmpegResult[] = [];
|
|
let next = 0;
|
|
const worker = async () => {
|
|
while (next < segmentCount && !signal.aborted) {
|
|
const index = next++;
|
|
const { firstFrame, frames } = segments[index]!;
|
|
const seek = startTime + firstFrame / fps;
|
|
const result = await inExtractionSlot(
|
|
() =>
|
|
runFfmpeg(
|
|
[
|
|
...decodeArgs,
|
|
"-noaccurate_seek",
|
|
"-ss",
|
|
String(seek),
|
|
"-i",
|
|
videoPath,
|
|
// Read one frame past the segment and keep exactly its own frames.
|
|
"-t",
|
|
String((frames + 1) / fps),
|
|
"-frames:v",
|
|
String(frames),
|
|
"-start_number",
|
|
String(firstFrame + 1),
|
|
...filterAndEncodeArgs(seek),
|
|
],
|
|
{ ...job.runOptions, signal },
|
|
),
|
|
signal,
|
|
);
|
|
results[index] = result;
|
|
if (!result.success) failed.abort();
|
|
}
|
|
};
|
|
const workers = Math.min(segmentCount, EXTRACTION_SLOTS);
|
|
await Promise.all(Array.from({ length: workers }, worker));
|
|
const ran = results.filter(Boolean);
|
|
// Report the segment that failed, not the ones stopped because of it.
|
|
const failure =
|
|
ran.find((r) => !r.success && r.terminationReason !== "abort") ?? ran.find((r) => !r.success);
|
|
if (failure) return failure;
|
|
// A cancel between segments leaves later ones unrun; their frames are missing, not done.
|
|
if (ran.length < segmentCount) {
|
|
return {
|
|
success: false,
|
|
exitCode: null,
|
|
stderr: "",
|
|
durationMs: 0,
|
|
terminationReason: "abort",
|
|
};
|
|
}
|
|
return ran[ran.length - 1]!;
|
|
}
|
|
|
|
export async function extractVideoFramesRange(
|
|
...args: Parameters<typeof extractVideoFramesRangeUnredacted>
|
|
): Promise<ExtractedFrames> {
|
|
try {
|
|
return await extractVideoFramesRangeUnredacted(...args);
|
|
} catch (error) {
|
|
throw redactUrlQueriesInError(error);
|
|
}
|
|
}
|
|
|
|
async function extractVideoFramesRangeUnredacted(
|
|
videoPath: string,
|
|
videoId: string,
|
|
startTime: number,
|
|
duration: number,
|
|
options: ExtractionOptions,
|
|
signal?: AbortSignal,
|
|
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
|
|
/** Frames go straight here, with no per-videoId subdir (the cache layer's keyed entry). */
|
|
outputDirOverride?: string,
|
|
): Promise<ExtractedFrames> {
|
|
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
|
const { outputDir, quality = 95 } = options;
|
|
const normalizedFps = toFps(options.fps);
|
|
const fps = fpsToNumber(normalizedFps);
|
|
const ffmpegFps = fpsToFfmpegArg(normalizedFps);
|
|
|
|
const videoOutputDir = outputDirOverride ?? join(outputDir, videoId);
|
|
if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });
|
|
|
|
let metadata: VideoMetadata;
|
|
try {
|
|
metadata = await extractMediaMetadata(videoPath);
|
|
} catch (error) {
|
|
throw classifyVideoExtractionError(error);
|
|
}
|
|
const playableDuration = resolvePlayableVideoDuration(metadata);
|
|
if (!(playableDuration > 0)) {
|
|
throw new VideoSourceExtractionError(
|
|
"invalid_media",
|
|
false,
|
|
"Video source has no positive duration",
|
|
`Playable video stream duration is ${playableDuration}s`,
|
|
);
|
|
}
|
|
if (startTime >= playableDuration) {
|
|
throw new VideoSourceExtractionError(
|
|
"media_start_out_of_range",
|
|
false,
|
|
"Video media start is outside the source duration",
|
|
`Video media start ${startTime}s is outside playable video duration ${playableDuration}s`,
|
|
);
|
|
}
|
|
const format = resolveFrameFormat(metadata, options.format);
|
|
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
|
|
const outputPattern = join(videoOutputDir, framePattern);
|
|
|
|
// Forced-SDR extraction tone-maps HDR with zscale, the Studio proxy policy. A macOS ffmpeg
|
|
// without zscale falls back to VideoToolbox, whose frames stay HDR and are tone-mapped by
|
|
// Chrome instead (darker), and the render warns once.
|
|
const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
|
|
const isMacOS = process.platform === "darwin";
|
|
const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale());
|
|
const decodeWithVideoToolbox = isHdr && isMacOS && !toneMappedToSdr;
|
|
|
|
const sampleAtOutputFps = !options.finalFrameOnly;
|
|
if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
|
|
throw new Error("frameRanges extraction needs a constant-frame-rate source");
|
|
}
|
|
|
|
const args: string[] = urlInputArgs(videoPath);
|
|
if (decodeWithVideoToolbox) {
|
|
args.push("-hwaccel", "videotoolbox");
|
|
}
|
|
// Always force the alpha-aware decoder on codecs that can carry alpha. The
|
|
// alternative — gating on `metadata.hasAlpha` — relies on tag detection that
|
|
// has at least three known failure modes: case-sensitivity across ffmpeg
|
|
// versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
|
|
// and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
|
|
// silently strips alpha during decode and the bug doesn't surface until
|
|
// the rendered video is missing layers. Codec-based default has no such
|
|
// ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
|
|
// normally when it isn't.
|
|
if (codecMayHaveAlpha(metadata.videoCodec)) {
|
|
args.push("-c:v", decoderForCodec(metadata.videoCodec));
|
|
}
|
|
const decodeArgs = [...args];
|
|
if (options.finalFrameOnly) {
|
|
// Output-side seek decodes from the start before selecting the final
|
|
// sample. This is intentionally reserved for the one-frame path: input
|
|
// seeking is faster, but valid unindexed transports (notably MPEG-TS with
|
|
// a negative timestamp base) can seek to EOF and emit zero frames.
|
|
args.push("-i", videoPath, "-ss", String(startTime), "-frames:v", "1");
|
|
} else {
|
|
// An accurate seek drops the frame on screen at startTime; the fps sampling below needs it.
|
|
if (sampleAtOutputFps) args.push("-noaccurate_seek");
|
|
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
|
}
|
|
|
|
const vfFilters: string[] = [];
|
|
if (decodeWithVideoToolbox) {
|
|
vfFilters.push("format=nv12");
|
|
}
|
|
const samplingAt = vfFilters.length;
|
|
// Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
|
|
// first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
|
|
const samplingFilters = (seek: number): string[] =>
|
|
sampleAtOutputFps
|
|
? [
|
|
...(metadata.isVFR
|
|
? []
|
|
: undoTickRoundingOnTheSourceFrameGrid(seek, metadata.videoStreamLeadSeconds ?? 0)),
|
|
"settb=intb/2",
|
|
"setpts=PTS-lte(TB\\,0.001)",
|
|
`fps=${ffmpegFps}:start_time=0:round=up`,
|
|
]
|
|
: [];
|
|
if (options.sdrToHdrTransfer) {
|
|
// Ordering intent: fps sampling runs BEFORE the colorspace remap so only
|
|
// kept frames are converted. The remap is pointwise per-frame, so the
|
|
// output is identical either way for the SDR (BT.709, 8-bit) inputs this
|
|
// flag is set for. If format=nv12 (macOS HDR-source decode) ever combines
|
|
// with this flag, revisit: nv12 subsampling before a BT.2020 remap is an
|
|
// untested interaction (today the flags are mutually exclusive — the
|
|
// remap only applies to SDR sources, nv12 only to HDR sources).
|
|
vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
|
|
}
|
|
if (toneMappedToSdr) {
|
|
const frameColour = await readFirstFrameColour(videoPath);
|
|
vfFilters.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, frameColour));
|
|
}
|
|
if (toneMappedToSdr || (!isHdr && !options.sdrToHdrTransfer)) {
|
|
vfFilters.push(
|
|
...(isHdr ? [] : chromeGuessForUntaggedMatrix(metadata)),
|
|
SDR_CANVAS_PASSTHROUGH_FILTER,
|
|
);
|
|
if (format === "jpg") vfFilters.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER);
|
|
}
|
|
const encodeArgs = ["-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"];
|
|
// Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.
|
|
if (format === "png") encodeArgs.push("-compression_level", "1");
|
|
encodeArgs.push("-y", outputPattern);
|
|
|
|
// A long 4K run can take far longer than the timeout and still be healthy; only a stall is a hang.
|
|
const runOptions = { signal, stallTimeout: ffmpegProcessTimeout };
|
|
const filterArgsFor = (seek: number): string[] => {
|
|
const filters = [...vfFilters];
|
|
filters.splice(samplingAt, 0, ...samplingFilters(seek));
|
|
return filters.length > 0 ? ["-vf", filters.join(",")] : [];
|
|
};
|
|
const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
|
|
const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
|
|
// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
|
|
// emits that last slot: the fps filter ends a `-t` window at the nearest slot. A
|
|
// decimal rate counts zero frames here and keeps the `-t` window.
|
|
const boundVfrFrameCount = metadata.isVFR && totalFrames > 0;
|
|
const processResult =
|
|
sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
|
|
? await runSegmentedExtraction({
|
|
decodeArgs,
|
|
filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
|
|
videoPath,
|
|
startTime,
|
|
fps,
|
|
segments: splitFrameRanges(
|
|
options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }],
|
|
segmentFrames,
|
|
),
|
|
runOptions,
|
|
})
|
|
: await inExtractionSlot(
|
|
() => runFfmpeg([...args, ...filterArgsFor(startTime), ...encodeArgs], runOptions),
|
|
signal,
|
|
);
|
|
if (processResult.failureReason === "external_interruption") {
|
|
throw new VideoSourceExtractionError(
|
|
"external_interruption",
|
|
true,
|
|
"Video frame extraction interrupted by host lifecycle",
|
|
`FFmpeg exited with code ${processResult.exitCode}: ${processResult.stderr.slice(-500)}`,
|
|
);
|
|
}
|
|
if (processResult.terminationReason === "abort") {
|
|
throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
|
|
}
|
|
if (processResult.terminationReason === "spawn_error") {
|
|
throw classifyFfmpegSpawnError(processResult.error, processResult.stderr);
|
|
}
|
|
if (!processResult.success) {
|
|
// With the SDR-to-HDR remap folded into this pass, a filter failure
|
|
// (e.g. an ffmpeg built without the colorspace filter) would otherwise
|
|
// surface as a generic extract error and the operator has to grep the
|
|
// filter chain to learn it was the HDR conversion. Attribute it.
|
|
const hdrPrefix = options.sdrToHdrTransfer
|
|
? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): `
|
|
: "";
|
|
const timedOut = processResult.terminationReason === "inactivity";
|
|
const timeoutSuffix = timedOut ? ` (no new frame for ${ffmpegProcessTimeout} ms)` : "";
|
|
const diagnostic =
|
|
`${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ` +
|
|
processResult.stderr.slice(-500);
|
|
if (timedOut) {
|
|
throw new VideoSourceExtractionError(
|
|
"ffmpeg_timeout",
|
|
true,
|
|
"Video frame extraction timed out",
|
|
diagnostic,
|
|
);
|
|
}
|
|
const transientIo =
|
|
/resource temporarily unavailable|device or resource busy|input\/output error/i.test(
|
|
processResult.stderr,
|
|
);
|
|
throw new VideoSourceExtractionError(
|
|
transientIo ? "ffmpeg_transient" : "ffmpeg_failed",
|
|
transientIo,
|
|
transientIo
|
|
? "Video frame extraction hit a transient I/O failure"
|
|
: "Video source could not be decoded",
|
|
diagnostic,
|
|
);
|
|
}
|
|
|
|
const framePaths = framePathsFromDirectory(videoOutputDir, format, !!options.frameRanges);
|
|
if (framePaths.size === 0 && duration > 0) {
|
|
throw new VideoSourceExtractionError(
|
|
"zero_output",
|
|
false,
|
|
"Video source produced no decodable frames",
|
|
`FFmpeg exited successfully but produced no frames (start=${startTime}, duration=${duration})`,
|
|
);
|
|
}
|
|
|
|
// Post-extraction reconciliation: when FFmpeg delivers <=60% of the
|
|
// frames predicted by the probed duration, the duration is likely
|
|
// inflated (container duration includes a longer audio track). Correct
|
|
// it so downstream coverage accounting uses the actual video extent.
|
|
if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) {
|
|
const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
|
|
if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) {
|
|
const correctedStreamDuration =
|
|
startTime + framePaths.size / fps - (metadata.videoStreamLeadSeconds ?? 0);
|
|
if (correctedStreamDuration < metadata.videoStreamDurationSeconds) {
|
|
metadata.videoStreamDurationSeconds = correctedStreamDuration;
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
videoId,
|
|
srcPath: videoPath,
|
|
outputDir: videoOutputDir,
|
|
framePattern,
|
|
fps,
|
|
totalFrames: framePaths.size,
|
|
metadata,
|
|
framePaths,
|
|
};
|
|
}
|
|
|
|
const TRANSIENT_FFMPEG_SPAWN_CODES = new Set(["EAGAIN", "EMFILE", "ENFILE"]);
|
|
|
|
export function classifyFfmpegSpawnError(error: unknown, stderr = ""): VideoSourceExtractionError {
|
|
const code =
|
|
typeof error === "object" && error !== null && "code" in error && typeof error.code === "string"
|
|
? error.code
|
|
: "";
|
|
if (code === "ENOENT") {
|
|
return new VideoSourceExtractionError(
|
|
"ffmpeg_unavailable",
|
|
false,
|
|
"FFmpeg is unavailable",
|
|
"[FFmpeg] ffmpeg not found",
|
|
);
|
|
}
|
|
const diagnostic = error instanceof Error ? error.message : stderr;
|
|
const retryable = TRANSIENT_FFMPEG_SPAWN_CODES.has(code);
|
|
return new VideoSourceExtractionError(
|
|
retryable ? "ffmpeg_transient" : "ffmpeg_failed",
|
|
retryable,
|
|
retryable
|
|
? "FFmpeg could not be started due to transient resource pressure"
|
|
: "FFmpeg could not be started",
|
|
diagnostic,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Resolve the used-segment duration for a video, falling back to the source's
|
|
* natural duration when the caller hasn't specified bounds (end=Infinity) or
|
|
* the bounds are nonsensical (end<=start).
|
|
*/
|
|
function resolveSegmentDuration(
|
|
requested: number,
|
|
mediaStart: number,
|
|
sourceDuration: number,
|
|
): number {
|
|
if (Number.isFinite(requested) && requested > 0) return requested;
|
|
const sourceRemaining = sourceDuration - mediaStart;
|
|
return sourceRemaining > 0 ? sourceRemaining : sourceDuration;
|
|
}
|
|
|
|
/**
|
|
* Return the range that can actually produce video frames.
|
|
*
|
|
* Container duration may include a longer audio stream or mux padding. Using
|
|
* it for video extraction planning can reserve raw-frame scratch for seconds
|
|
* where no video frames exist. `extractMediaMetadata` already falls back to
|
|
* the container duration when ffprobe omits the stream duration; keep the
|
|
* explicit fallback here for callers supplying older/manual metadata.
|
|
*/
|
|
export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {
|
|
return Number.isFinite(metadata.videoStreamDurationSeconds) &&
|
|
metadata.videoStreamDurationSeconds > 0
|
|
? (metadata.videoStreamLeadSeconds ?? 0) + metadata.videoStreamDurationSeconds
|
|
: metadata.durationSeconds;
|
|
}
|
|
|
|
export interface TimelineExtractionWindow {
|
|
compositionStart: number;
|
|
mediaStart: number;
|
|
durationSeconds: number;
|
|
/** Visible duration on the authored composition timeline after retiming. */
|
|
timelineDurationSeconds?: number;
|
|
/**
|
|
* Preserve the authored timeline origin and mediaStart for lookup. This is
|
|
* required when a looped visible interval crosses a source boundary and
|
|
* still needs modulo phase against the complete extracted source cycle.
|
|
*/
|
|
preserveTimelinePhase?: boolean;
|
|
/**
|
|
* Keep the authored end while rebasing start/mediaStart to the extracted
|
|
* source suffix. Non-looping lookup then holds the suffix's final frame
|
|
* through the remainder of the authored slot.
|
|
*/
|
|
preserveTimelineEnd?: boolean;
|
|
/** This window reaches a held tail and must be checked against the actual
|
|
* final decoded-frame timestamp before extraction. */
|
|
ensureFinalFrame?: boolean;
|
|
/** Source timestamp used by FFmpeg when it differs from the logical lookup
|
|
* mediaStart (the one-frame held-tail representation). */
|
|
extractionMediaStart?: number;
|
|
/** FFmpeg emits one decoded frame; lookup then holds that frame. */
|
|
finalFrameOnly?: boolean;
|
|
}
|
|
|
|
type TimelineWindowVideo = Pick<VideoElement, "start" | "end" | "mediaStart"> &
|
|
Partial<Pick<VideoElement, "playbackRate">> &
|
|
Partial<Pick<VideoElement, "loop">>;
|
|
|
|
function canHoldFinalFramePastEof(video: TimelineWindowVideo): boolean {
|
|
const timelineDuration = video.end - video.start;
|
|
return !video.loop && Number.isFinite(timelineDuration) && timelineDuration > 0;
|
|
}
|
|
|
|
// Logical duration assigned to a one-frame held-tail representation. This is
|
|
// deliberately below any supported output frame interval: coverage expects
|
|
// one frame, while FFmpeg seeks to the separately probed real frame timestamp.
|
|
const FINAL_FRAME_LOGICAL_DURATION_SECONDS = 1e-6;
|
|
|
|
export function isVideoPastTimelineEnd(video: Pick<VideoElement, "start">, timelineEnd: number) {
|
|
return video.start >= timelineEnd;
|
|
}
|
|
|
|
/** Move a clip to its extraction window; a rate lane is shifted so it keeps integrating from the new origin. */
|
|
export function rebaseVideoToWindow(video: VideoElement, window: TimelineExtractionWindow): void {
|
|
if (window.preserveTimelinePhase) return;
|
|
// A trim landing on the timeline origin can come out as 1e-16 and drop the first frame.
|
|
const start = Math.abs(window.compositionStart) < 1e-9 ? 0 : window.compositionStart;
|
|
video.playbackRate = shiftRateLane(normalizeRateSpec(video.playbackRate), start - video.start);
|
|
video.start = start;
|
|
if (!window.preserveTimelineEnd) {
|
|
video.end = start + (window.timelineDurationSeconds ?? window.durationSeconds);
|
|
}
|
|
video.mediaStart = window.mediaStart;
|
|
}
|
|
|
|
/**
|
|
* Intersect an authored slot with the render timeline, then select the
|
|
* smallest playable source range that preserves timeline lookup semantics.
|
|
*
|
|
* A finite authored slot can outlive the source. In that case FFmpeg should
|
|
* still extract at most one source range: lookup either wraps that range for
|
|
* loops or holds its final frame for non-looping video. Keeping the authored
|
|
* timeline origin separate from the extracted range is what makes both
|
|
* behaviours survive the source-duration cap.
|
|
*/
|
|
export function resolveTimelineExtractionWindow(
|
|
video: TimelineWindowVideo,
|
|
resolvedDuration: number,
|
|
timelineEnd?: number,
|
|
sourceDuration?: number,
|
|
): TimelineExtractionWindow {
|
|
const playbackRate = normalizeRateSpec(video.playbackRate);
|
|
const withTimelineDuration = (
|
|
window: TimelineExtractionWindow,
|
|
timelineDurationSeconds: number,
|
|
): TimelineExtractionWindow =>
|
|
playbackRate === 1 ? window : { ...window, timelineDurationSeconds };
|
|
if (timelineEnd === undefined) {
|
|
return withTimelineDuration(
|
|
{
|
|
compositionStart: video.start,
|
|
mediaStart: video.mediaStart,
|
|
durationSeconds: sourceTimeAt(playbackRate, resolvedDuration),
|
|
},
|
|
resolvedDuration,
|
|
);
|
|
}
|
|
if (!Number.isFinite(timelineEnd)) {
|
|
throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`);
|
|
}
|
|
const compositionStart = Math.max(0, video.start);
|
|
const trimmedPreroll = compositionStart - video.start;
|
|
const trimmedSourcePreroll = sourceTimeAt(playbackRate, trimmedPreroll);
|
|
const timelineDuration = Math.max(0, timelineEnd - compositionStart);
|
|
// Infinity means "natural source duration", not an authored infinite slot.
|
|
// Explicit finite slots may outlive the source (loop or held tail), while an
|
|
// omitted duration remains source-bounded exactly like the browser runtime.
|
|
const resolvedVisibleDuration = resolvedDuration - trimmedPreroll;
|
|
const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration));
|
|
const visibleSourceDuration =
|
|
typeof playbackRate === "number"
|
|
? visibleDuration * playbackRate
|
|
: sourceTimeAt(playbackRate, trimmedPreroll + visibleDuration) - trimmedSourcePreroll;
|
|
let mediaStart = video.mediaStart + trimmedSourcePreroll;
|
|
if (visibleDuration > 0 && sourceDuration !== undefined) {
|
|
const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart);
|
|
if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) {
|
|
const phaseOffset = trimmedSourcePreroll % sourceRemaining;
|
|
const phaseRemaining = sourceRemaining - phaseOffset;
|
|
// Keep a complete cycle on equality too, otherwise a rebased suffix would
|
|
// wrap to its own first frame instead of the source cycle's first frame.
|
|
if (visibleSourceDuration >= phaseRemaining) {
|
|
return withTimelineDuration(
|
|
{
|
|
compositionStart: video.start,
|
|
mediaStart: video.mediaStart,
|
|
durationSeconds: sourceRemaining,
|
|
preserveTimelinePhase: true,
|
|
},
|
|
visibleDuration,
|
|
);
|
|
}
|
|
mediaStart = video.mediaStart + phaseOffset;
|
|
} else if (sourceRemaining > 0) {
|
|
const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedSourcePreroll);
|
|
if (visibleSourceDuration <= sourceVisibleAfterPreroll) {
|
|
return withTimelineDuration(
|
|
{
|
|
compositionStart,
|
|
mediaStart,
|
|
durationSeconds: visibleSourceDuration,
|
|
},
|
|
visibleDuration,
|
|
);
|
|
}
|
|
|
|
// The visible interval enters (or is entirely inside) the held tail.
|
|
// Extract the visible source suffix. If preroll is already at/past the
|
|
// final decoded timestamp, the async resolver below replaces this tiny
|
|
// provisional suffix with one exact final frame.
|
|
const extractionDuration = Math.min(
|
|
sourceRemaining,
|
|
Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS),
|
|
);
|
|
const extractionOffset = sourceRemaining - extractionDuration;
|
|
return withTimelineDuration(
|
|
{
|
|
compositionStart: video.start + timeAtSourceTime(playbackRate, extractionOffset),
|
|
mediaStart: video.mediaStart + extractionOffset,
|
|
durationSeconds: extractionDuration,
|
|
preserveTimelineEnd: true,
|
|
ensureFinalFrame: true,
|
|
},
|
|
visibleDuration,
|
|
);
|
|
} else if (canHoldFinalFramePastEof(video)) {
|
|
const logicalDuration = Math.min(sourceDuration, FINAL_FRAME_LOGICAL_DURATION_SECONDS);
|
|
return withTimelineDuration(
|
|
{
|
|
compositionStart: video.start,
|
|
mediaStart: sourceDuration - logicalDuration,
|
|
durationSeconds: logicalDuration,
|
|
preserveTimelineEnd: true,
|
|
ensureFinalFrame: true,
|
|
},
|
|
visibleDuration,
|
|
);
|
|
}
|
|
}
|
|
return withTimelineDuration(
|
|
{
|
|
compositionStart,
|
|
mediaStart,
|
|
durationSeconds: visibleSourceDuration,
|
|
},
|
|
visibleDuration,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Replace a held-tail suffix that starts at/after the final decoded timestamp
|
|
* with one exact frame. This keeps raw HDR scratch O(one frame) without
|
|
* assuming a one-second seek window contains a CFR/VFR timestamp.
|
|
*/
|
|
export async function resolveFinalFrameExtractionWindow(
|
|
videoPath: string,
|
|
video: TimelineWindowVideo,
|
|
metadata: VideoMetadata,
|
|
window: TimelineExtractionWindow,
|
|
signal?: AbortSignal,
|
|
): Promise<TimelineExtractionWindow> {
|
|
if (!window.ensureFinalFrame) return window;
|
|
const playableDuration = resolvePlayableVideoDuration(metadata);
|
|
const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;
|
|
const finalFrameTimestamp =
|
|
videoStreamLead +
|
|
(await extractFinalVideoFrameTimestamp(
|
|
videoPath,
|
|
{
|
|
videoStreamDurationSeconds: playableDuration - videoStreamLead,
|
|
videoStreamStartSeconds: metadata.videoStreamStartSeconds,
|
|
},
|
|
signal,
|
|
));
|
|
if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
|
|
|
|
const sourceRemaining = playableDuration - video.mediaStart;
|
|
const logicalDuration = Math.min(
|
|
Math.max(sourceRemaining, window.durationSeconds),
|
|
FINAL_FRAME_LOGICAL_DURATION_SECONDS,
|
|
);
|
|
return {
|
|
compositionStart: Math.max(0, video.start),
|
|
mediaStart: playableDuration - logicalDuration,
|
|
extractionMediaStart: finalFrameTimestamp,
|
|
durationSeconds: logicalDuration,
|
|
...(window.timelineDurationSeconds !== undefined
|
|
? { timelineDurationSeconds: window.timelineDurationSeconds }
|
|
: {}),
|
|
preserveTimelineEnd: true,
|
|
finalFrameOnly: true,
|
|
};
|
|
}
|
|
|
|
/** Resolve source duration first, then intersect it with the render timeline. */
|
|
export function resolveVideoExtractionWindow(
|
|
video: TimelineWindowVideo,
|
|
metadata: VideoMetadata,
|
|
timelineEnd?: number,
|
|
): TimelineExtractionWindow {
|
|
const playableDuration = resolvePlayableVideoDuration(metadata);
|
|
if (!(playableDuration > 0)) {
|
|
throw new VideoSourceExtractionError(
|
|
"invalid_media",
|
|
false,
|
|
"Video source has no positive duration",
|
|
`Playable video stream duration is ${playableDuration}s`,
|
|
);
|
|
}
|
|
const requestedTimelineDuration = video.end - video.start;
|
|
const heldPastEof = video.mediaStart >= playableDuration && canHoldFinalFramePastEof(video);
|
|
if (video.mediaStart >= playableDuration && !heldPastEof) {
|
|
throw new VideoSourceExtractionError(
|
|
"media_start_out_of_range",
|
|
false,
|
|
"Video media start is outside the source duration",
|
|
`Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`,
|
|
);
|
|
}
|
|
const playbackRate = normalizeRateSpec(video.playbackRate);
|
|
const resolvedDuration =
|
|
Number.isFinite(requestedTimelineDuration) && requestedTimelineDuration > 0
|
|
? requestedTimelineDuration
|
|
: timeAtSourceTime(
|
|
playbackRate,
|
|
resolveSegmentDuration(requestedTimelineDuration, video.mediaStart, playableDuration),
|
|
);
|
|
return resolveTimelineExtractionWindow(
|
|
video,
|
|
resolvedDuration,
|
|
timelineEnd ?? (heldPastEof ? video.end : undefined),
|
|
playableDuration,
|
|
);
|
|
}
|
|
|
|
export function resolveVideoExtractionDuration(
|
|
video: TimelineWindowVideo,
|
|
metadata: VideoMetadata,
|
|
timelineEnd?: number,
|
|
): number {
|
|
return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds;
|
|
}
|
|
|
|
/**
|
|
* Codecs whose bitstream is allowed to carry an alpha channel. Default the
|
|
* extraction path to PNG output for these regardless of `metadata.hasAlpha`
|
|
* so a missed sidecar tag doesn't silently strip transparency. Opaque content
|
|
* encoded in one of these codecs pays a small file-size cost on the cached
|
|
* frames but stays correct on the rare case where alpha IS present and the
|
|
* tag was missed.
|
|
*/
|
|
const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
|
|
|
|
export function codecMayHaveAlpha(codec: string | undefined): boolean {
|
|
return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
|
|
}
|
|
|
|
export function decoderForCodec(codec: string | undefined): string {
|
|
const c = (codec ?? "").toLowerCase();
|
|
if (c === "vp9") return "libvpx-vp9";
|
|
if (c === "vp8") return "libvpx";
|
|
return c;
|
|
}
|
|
|
|
export function resolveFrameFormat(
|
|
metadata: VideoMetadata,
|
|
requested?: VideoFrameFormat,
|
|
): CacheFrameFormat {
|
|
if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
|
|
if (requested === "png" || requested === "jpg") return requested;
|
|
return "jpg";
|
|
}
|
|
|
|
type PreparedExtraction = {
|
|
video: VideoElement;
|
|
videoPath: string;
|
|
index: number;
|
|
metadata: VideoMetadata;
|
|
videoDuration: number;
|
|
extractionMediaStart: number;
|
|
finalFrameOnly: boolean;
|
|
format: CacheFrameFormat;
|
|
sdrToHdrTransfer?: HdrTransfer;
|
|
hdrToSdrTransformKey?: string;
|
|
dedupeKey: string;
|
|
};
|
|
|
|
type CacheMissTarget = {
|
|
entry: CacheEntry;
|
|
srcPath: string;
|
|
};
|
|
|
|
type UniqueExtractionMiss = {
|
|
work: PreparedExtraction;
|
|
cacheTarget?: CacheMissTarget;
|
|
};
|
|
|
|
type SupersetMemberPlan = {
|
|
miss: UniqueExtractionMiss;
|
|
offsetFrames: number;
|
|
};
|
|
|
|
type SupersetGroupPlan = {
|
|
groupId: string;
|
|
baseStart: number;
|
|
unionDuration: number;
|
|
members: SupersetMemberPlan[];
|
|
};
|
|
|
|
function extractedFramesFromDirectory(
|
|
work: PreparedExtraction,
|
|
outputDir: string,
|
|
srcPath: string,
|
|
fps: number,
|
|
): ExtractedFrames {
|
|
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${work.format}`;
|
|
const framePaths = framePathsFromDirectory(outputDir, work.format);
|
|
return {
|
|
videoId: work.video.id,
|
|
srcPath,
|
|
outputDir,
|
|
framePattern,
|
|
fps,
|
|
totalFrames: framePaths.size,
|
|
metadata: work.metadata,
|
|
framePaths,
|
|
};
|
|
}
|
|
|
|
function frameFileName(frameNumber: number, format: CacheFrameFormat): string {
|
|
return `${FRAME_FILENAME_PREFIX}${String(frameNumber).padStart(5, "0")}.${format}`;
|
|
}
|
|
|
|
function linkOrCopyFrame(src: string, dest: string): void {
|
|
try {
|
|
linkSync(src, dest);
|
|
} catch {
|
|
copyFileSync(src, dest);
|
|
}
|
|
}
|
|
|
|
function supersetGroupingKey(work: PreparedExtraction, fps: number): string {
|
|
return [
|
|
work.videoPath,
|
|
String(fps),
|
|
work.format,
|
|
work.sdrToHdrTransfer ?? "",
|
|
work.hdrToSdrTransformKey ?? "",
|
|
work.finalFrameOnly ? "final" : "range",
|
|
].join("\0");
|
|
}
|
|
|
|
function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean {
|
|
const frames = offsetSeconds * fps;
|
|
return Math.abs(frames - Math.round(frames)) <= 1e-4;
|
|
}
|
|
|
|
function windowsOverlapOrTouch(misses: UniqueExtractionMiss[], baseStart: number): boolean {
|
|
const unionEnd = Math.max(
|
|
...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
|
|
);
|
|
const unionDuration = unionEnd - baseStart;
|
|
const summedDuration = misses.reduce((sum, { work }) => sum + work.videoDuration, 0);
|
|
return unionDuration > 0 && unionDuration <= summedDuration + 1e-9;
|
|
}
|
|
|
|
function buildSupersetGroup(
|
|
groupId: string,
|
|
misses: UniqueExtractionMiss[],
|
|
fps: number,
|
|
): SupersetGroupPlan | null {
|
|
if (misses.length < 2) return null;
|
|
if (misses.some(({ work }) => work.finalFrameOnly)) return null;
|
|
// VFR ranges round their frame counts up, so a union extraction's slices
|
|
// are not proven to match independently sought member ranges. Keep them direct.
|
|
if (misses.some(({ work }) => work.metadata.isVFR)) return null;
|
|
const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart));
|
|
if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) {
|
|
return null;
|
|
}
|
|
if (!windowsOverlapOrTouch(misses, baseStart)) return null;
|
|
|
|
const unionEnd = Math.max(
|
|
...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
|
|
);
|
|
return {
|
|
groupId,
|
|
baseStart,
|
|
unionDuration: unionEnd - baseStart,
|
|
members: misses.map((miss) => ({
|
|
miss,
|
|
offsetFrames: Math.round((miss.work.video.mediaStart - baseStart) * fps),
|
|
})),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Partition one source's misses into overlap-connected components: sort by
|
|
* window start and cut wherever the next window starts past the running end.
|
|
* Without this, one disjoint outlier trim (e.g. [100..105] next to three
|
|
* overlapping trims at [0..11]) fails the union<=sum check for the whole
|
|
* bucket and every trim falls back to direct extraction.
|
|
*/
|
|
function overlapClusters(misses: UniqueExtractionMiss[]): UniqueExtractionMiss[][] {
|
|
const sorted = [...misses].sort((a, b) => a.work.video.mediaStart - b.work.video.mediaStart);
|
|
const clusters: UniqueExtractionMiss[][] = [];
|
|
let current: UniqueExtractionMiss[] = [];
|
|
let currentEnd = -Infinity;
|
|
for (const miss of sorted) {
|
|
const start = miss.work.video.mediaStart;
|
|
const end = start + miss.work.videoDuration;
|
|
if (current.length > 0 && start > currentEnd + 1e-9) {
|
|
clusters.push(current);
|
|
current = [];
|
|
currentEnd = -Infinity;
|
|
}
|
|
current.push(miss);
|
|
currentEnd = Math.max(currentEnd, end);
|
|
}
|
|
if (current.length > 0) clusters.push(current);
|
|
return clusters;
|
|
}
|
|
|
|
function planSupersetGroups(
|
|
misses: UniqueExtractionMiss[],
|
|
fps: number,
|
|
): { groups: SupersetGroupPlan[]; direct: UniqueExtractionMiss[] } {
|
|
const bySource = new Map<string, UniqueExtractionMiss[]>();
|
|
for (const miss of misses) {
|
|
const key = supersetGroupingKey(miss.work, fps);
|
|
bySource.set(key, [...(bySource.get(key) ?? []), miss]);
|
|
}
|
|
|
|
const groups: SupersetGroupPlan[] = [];
|
|
const direct: UniqueExtractionMiss[] = [];
|
|
let groupIndex = 0;
|
|
for (const groupMisses of bySource.values()) {
|
|
for (const cluster of overlapClusters(groupMisses)) {
|
|
const group = buildSupersetGroup(`__superset-${groupIndex}`, cluster, fps);
|
|
if (group) {
|
|
groups.push(group);
|
|
groupIndex += 1;
|
|
} else {
|
|
direct.push(...cluster);
|
|
}
|
|
}
|
|
}
|
|
return { groups, direct };
|
|
}
|
|
|
|
function sliceSupersetMember(
|
|
member: SupersetMemberPlan,
|
|
superset: ExtractedFrames,
|
|
outputDir: string,
|
|
fps: number,
|
|
configuredFps: FpsInput,
|
|
): ExtractedFrames {
|
|
const { work } = member.miss;
|
|
rmSync(outputDir, { recursive: true, force: true });
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
// Sample-time correctness: member frame k uses superset frame
|
|
// offset_i + k, so its source time is
|
|
// baseStart + (offset_i + k) / fps = mediaStart_i + k / fps.
|
|
// The frame-alignment precondition is what makes offset_i integral.
|
|
const requestedFrames = extractionFrameCountForDuration(
|
|
work.videoDuration,
|
|
configuredFps,
|
|
work.metadata.isVFR,
|
|
);
|
|
const availableFrames = Math.max(0, superset.totalFrames - member.offsetFrames);
|
|
const frameCount = Math.min(requestedFrames, availableFrames);
|
|
for (let i = 0; i < frameCount; i += 1) {
|
|
const sourceFrame = superset.framePaths.get(member.offsetFrames + i);
|
|
if (!sourceFrame) throw new Error(`superset frame ${member.offsetFrames + i} missing`);
|
|
linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format)));
|
|
}
|
|
|
|
return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps);
|
|
}
|
|
|
|
export async function extractAllVideoFrames(
|
|
videos: VideoElement[],
|
|
baseDir: string,
|
|
options: ExtractionOptions,
|
|
signal?: AbortSignal,
|
|
config?: Partial<
|
|
Pick<EngineConfig, "ffmpegProcessTimeout" | "extractCacheDir" | "extractCacheMaxBytes">
|
|
>,
|
|
compiledDir?: string,
|
|
): Promise<ExtractionResult> {
|
|
if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) {
|
|
throw new Error(
|
|
`Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`,
|
|
);
|
|
}
|
|
const configuredFps = toFps(options.fps);
|
|
const fps = fpsToNumber(configuredFps);
|
|
const fpsKey = fpsToFfmpegArg(configuredFps);
|
|
const startTime = Date.now();
|
|
const extracted: ExtractedFrames[] = [];
|
|
const errors: VideoExtractionFailure[] = [];
|
|
let totalFramesExtracted = 0;
|
|
const breakdown: ExtractionPhaseBreakdown = {
|
|
resolveMs: 0,
|
|
cachePublishFailures: 0,
|
|
cacheGcEvictions: 0,
|
|
cacheGcBytesFreed: 0,
|
|
cacheAgedPartialsCleared: 0,
|
|
hdrProbeMs: 0,
|
|
hdrPreflightMs: 0,
|
|
hdrPreflightCount: 0,
|
|
vfrProbeMs: 0,
|
|
vfrPreflightMs: 0,
|
|
vfrPreflightCount: 0,
|
|
extractMs: 0,
|
|
cacheHits: 0,
|
|
cacheMisses: 0,
|
|
transientRetries: 0,
|
|
};
|
|
const recordTransientRetries = (count: number): void => {
|
|
breakdown.transientRetries = (breakdown.transientRetries ?? 0) + count;
|
|
};
|
|
|
|
// Phase 1: Resolve paths and download remote videos
|
|
const phase1Start = Date.now();
|
|
const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
|
|
// Dedupe missing-src warnings: a composition with N <video> elements all
|
|
// pointing at the same broken src should only print one warning, not N.
|
|
const warnedSrcs = new Set<string>();
|
|
for (const video of videos) {
|
|
if (signal?.aborted) break;
|
|
if (options.timelineEnd !== undefined && isVideoPastTimelineEnd(video, options.timelineEnd)) {
|
|
continue;
|
|
}
|
|
try {
|
|
let videoPath = video.src;
|
|
if (!isHttpUrl(videoPath)) {
|
|
videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
|
|
}
|
|
const remoteUrl = options.urlByAbsentLocalPath?.get(videoPath);
|
|
if (remoteUrl) {
|
|
resolvedVideos.push({ video, videoPath: remoteUrl });
|
|
continue;
|
|
}
|
|
|
|
if (isHttpUrl(videoPath)) {
|
|
const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);
|
|
mkdirSync(downloadDir, { recursive: true });
|
|
videoPath = await downloadToTemp(
|
|
videoPath,
|
|
downloadDir,
|
|
undefined,
|
|
signal,
|
|
() => recordTransientRetries(1),
|
|
{ onTelemetry: writeUrlDownloadTelemetry },
|
|
);
|
|
}
|
|
|
|
if (!existsSync(videoPath)) {
|
|
// Loud: silent miss leaves the rendered video frozen at frame 0 with
|
|
// no error in stdout — extremely confusing for authors. Dedupe by
|
|
// src so 50 broken videos pointing at the same path don't spam.
|
|
if (!warnedSrcs.has(video.src)) {
|
|
warnedSrcs.add(video.src);
|
|
process.stderr.write(
|
|
`[hyperframes:render] WARNING: video src="${video.src}" ` +
|
|
`could not be resolved on disk (looked for ${videoPath}). ` +
|
|
`The rendered output will show this video's first frame for the entire clip duration. ` +
|
|
`If your <video> lives inside a sub-composition, prefer project-root-relative paths ` +
|
|
`(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`,
|
|
);
|
|
}
|
|
errors.push({
|
|
videoId: video.id,
|
|
kind: "source_missing",
|
|
retryable: false,
|
|
error: `Video file not found: ${videoPath}`,
|
|
});
|
|
continue;
|
|
}
|
|
resolvedVideos.push({ video, videoPath });
|
|
} catch (err) {
|
|
const classified = classifyVideoExtractionError(err);
|
|
errors.push({
|
|
videoId: video.id,
|
|
kind: classified.kind,
|
|
retryable: classified.retryable,
|
|
...(err instanceof UrlDownloadError ? { group: downloadFailureGroup(video.src, err) } : {}),
|
|
error: classified.diagnostic,
|
|
});
|
|
}
|
|
}
|
|
|
|
breakdown.resolveMs = Date.now() - phase1Start;
|
|
|
|
// Snapshot the pre-preflight key inputs so the extraction cache keys on the
|
|
// user-visible source path rather than the
|
|
// workDir-local normalized file produced by the
|
|
// HDR preflight. Without this, every render would write a new
|
|
// normalized file with a fresh mtime → fresh cache key → perpetual misses.
|
|
// Phase 3 updates mediaStart after trimming any invisible negative preroll.
|
|
// Downloads land at a new path and mtime every render, so they are keyed by content instead.
|
|
const contentKeyedDirs = [
|
|
join(options.outputDir, DOWNLOAD_SUBDIR),
|
|
...(options.contentKeyedDirs ?? []),
|
|
];
|
|
const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => {
|
|
const stat = readKeyStat(videoPath);
|
|
// Missing files return null — skip the cache path for that entry. The
|
|
// extractor will surface the real file-not-found error downstream, and we
|
|
// avoid polluting the cache with a `(mtimeMs: 0, size: 0)` tuple that two
|
|
// unrelated missing paths would otherwise share.
|
|
if (!stat) return null;
|
|
return {
|
|
videoPath,
|
|
mtimeMs: stat.mtimeMs,
|
|
size: stat.size,
|
|
mediaStart: video.mediaStart,
|
|
contentKeyed: contentKeyedDirs.some((dir) => isPathInside(videoPath, dir)),
|
|
};
|
|
});
|
|
|
|
// Phase 2: Probe color spaces and normalize if mixed HDR/SDR
|
|
const phase2ProbeStart = Date.now();
|
|
const metadataResults = await Promise.all(
|
|
resolvedVideos.map(async ({ video, videoPath }, index) => {
|
|
try {
|
|
// Keep the default/off path byte-for-byte compatible with the legacy
|
|
// Promise.all rejection. Classification is introduced only when a
|
|
// bounded retry or explicit typed aggregation is enabled.
|
|
const attempted =
|
|
!options.collectProbeFailures &&
|
|
boundedTransientRetryBudget(options.maxTransientRetries) === 0
|
|
? { result: await extractMediaMetadata(videoPath), retries: 0 }
|
|
: await runVideoExtractionWithRetry(() => extractMediaMetadata(videoPath), {
|
|
signal,
|
|
maxTransientRetries: options.maxTransientRetries,
|
|
onRetry: () => recordTransientRetries(1),
|
|
});
|
|
return {
|
|
video,
|
|
videoPath,
|
|
metadata: attempted.result,
|
|
cacheKeyInput: cacheKeyInputs[index] ?? null,
|
|
};
|
|
} catch (error) {
|
|
if (!options.collectProbeFailures) throw error;
|
|
errors.push(extractionError(video.id, error));
|
|
return null;
|
|
}
|
|
}),
|
|
);
|
|
const probedVideos = metadataResults.filter((entry) => entry !== null);
|
|
resolvedVideos.splice(
|
|
0,
|
|
resolvedVideos.length,
|
|
...probedVideos.map(({ video, videoPath }) => ({ video, videoPath })),
|
|
);
|
|
cacheKeyInputs.splice(
|
|
0,
|
|
cacheKeyInputs.length,
|
|
...probedVideos.map(({ cacheKeyInput }) => cacheKeyInput),
|
|
);
|
|
const videoMetadata = probedVideos.map(({ metadata }) => metadata);
|
|
const videoColorSpaces = videoMetadata.map((m) => m.colorSpace);
|
|
// Canonical per-index record of the SDR-to-HDR transform decision. BOTH the
|
|
// cache key (transform discriminator) and the extraction options read from
|
|
// this array via the prepared work items — never set one side independently
|
|
// or cache lookups and written frames drift apart (the poisoning bug this
|
|
// field exists to fix).
|
|
const sdrToHdrTransfers: Array<HdrTransfer | undefined> = resolvedVideos.map(() => undefined);
|
|
breakdown.hdrProbeMs = Date.now() - phase2ProbeStart;
|
|
|
|
// Warning only: an opaque video used as a full-frame background is legitimate.
|
|
const alphaWarnedSrcs = new Set<string>();
|
|
if (resolvedVideos.length > 0) {
|
|
await Promise.all(
|
|
resolvedVideos.map(async ({ video, videoPath }, index) => {
|
|
if (signal?.aborted) return;
|
|
const metadata = videoMetadata[index];
|
|
if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return;
|
|
if (alphaWarnedSrcs.has(video.src)) return;
|
|
alphaWarnedSrcs.add(video.src);
|
|
const decoder = decoderForCodec(metadata.videoCodec);
|
|
if ((await probeInputAlphaPlane(videoPath, decoder, signal)) !== true) return;
|
|
process.stderr.write(inputAlphaOpaqueWarning(video.src));
|
|
}),
|
|
);
|
|
}
|
|
|
|
const hdrPreflightStart = Date.now();
|
|
const hdrInfo = analyzeCompositionHdr(videoColorSpaces);
|
|
// Track entries the HDR preflight validated as non-extractable so they can
|
|
// be removed from every parallel array before Phase 2b and Phase 3 see them.
|
|
// Without this, `errors.push({...}); continue;` only short-circuits the
|
|
// normalization step — the invalid entry stays in `resolvedVideos` and
|
|
// Phase 3 still calls `extractVideoFramesRange` on the same past-EOF
|
|
// mediaStart, surfacing a second raw FFmpeg error for the same clip.
|
|
const hdrSkippedIndices = new Set<number>();
|
|
if (hdrInfo.hasHdr && hdrInfo.dominantTransfer) {
|
|
// dominantTransfer is "majority wins" — if a composition mixes PQ and HLG
|
|
// sources (rare but legal), the minority transfer's videos get converted
|
|
// with the wrong curve. We treat this as caller-error: a single composition
|
|
// should not mix PQ and HLG sources, the orchestrator picks one transfer
|
|
// for the whole render, and any source not on that curve is normalized to
|
|
// it. If you need both transfers, render two separate compositions.
|
|
const targetTransfer = hdrInfo.dominantTransfer;
|
|
|
|
for (let i = 0; i < resolvedVideos.length; i++) {
|
|
if (signal?.aborted) break;
|
|
const cs = videoColorSpaces[i] ?? null;
|
|
if (!isHdrColorSpaceUtil(cs)) {
|
|
// SDR video in a mixed timeline — extract through a BT.709→BT.2020
|
|
// colorspace filter so the encoder tags the final video correctly
|
|
// (PQ vs HLG) without a separate normalized intermediate.
|
|
const entry = resolvedVideos[i];
|
|
const metadata = videoMetadata[i];
|
|
if (!entry || !metadata) continue;
|
|
|
|
// Guard past-EOF windows that cannot use the non-looping held-tail plan.
|
|
// FFmpeg's `-ss` otherwise silently produces a 0-byte intermediate.
|
|
const playableDuration = resolvePlayableVideoDuration(metadata);
|
|
if (entry.video.mediaStart >= playableDuration && !canHoldFinalFramePastEof(entry.video)) {
|
|
errors.push({
|
|
videoId: entry.video.id,
|
|
kind: "media_start_out_of_range",
|
|
retryable: false,
|
|
error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ playable video duration (${playableDuration}s)`,
|
|
});
|
|
hdrSkippedIndices.add(i);
|
|
continue;
|
|
}
|
|
|
|
sdrToHdrTransfers[i] = targetTransfer;
|
|
breakdown.hdrPreflightCount += 1;
|
|
}
|
|
}
|
|
}
|
|
breakdown.hdrPreflightMs = Date.now() - hdrPreflightStart;
|
|
|
|
// Remove HDR-preflight-skipped entries from every parallel array so Phase 2b
|
|
// (VFR classification) and Phase 3 (extract) don't re-process them. Iterate
|
|
// backwards to keep indices stable while splicing.
|
|
if (hdrSkippedIndices.size > 0) {
|
|
for (let i = resolvedVideos.length - 1; i >= 0; i--) {
|
|
if (hdrSkippedIndices.has(i)) {
|
|
resolvedVideos.splice(i, 1);
|
|
videoMetadata.splice(i, 1);
|
|
videoColorSpaces.splice(i, 1);
|
|
// Added by the extraction-cache commit: keep cacheKeyInputs aligned
|
|
// with the other parallel arrays so Phase 3's `cacheKeyInputs[i]`
|
|
// lookup doesn't point at a stale slot after the splice.
|
|
cacheKeyInputs.splice(i, 1);
|
|
sdrToHdrTransfers.splice(i, 1);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Phase 2b: Keep VFR observability while routing VFR inputs through the
|
|
// one-pass CFR extraction path in Phase 3.
|
|
const vfrPreflightStart = Date.now();
|
|
for (let i = 0; i < resolvedVideos.length; i++) {
|
|
if (signal?.aborted) break;
|
|
const vfrProbeStart = Date.now();
|
|
const metadata = videoMetadata[i];
|
|
breakdown.vfrProbeMs += Date.now() - vfrProbeStart;
|
|
if (metadata?.isVFR) breakdown.vfrPreflightCount += 1;
|
|
}
|
|
breakdown.vfrPreflightMs = Date.now() - vfrPreflightStart;
|
|
|
|
const phase3Start = Date.now();
|
|
const configuredCacheRootDir = config?.extractCacheDir;
|
|
let cacheRootDir: string | undefined;
|
|
if (configuredCacheRootDir) {
|
|
try {
|
|
mkdirSync(configuredCacheRootDir, { recursive: true });
|
|
cacheRootDir = configuredCacheRootDir;
|
|
} catch {
|
|
process.stderr.write(
|
|
`[hyperframes:render] WARNING: extraction cache dir ${configuredCacheRootDir} is not writable; caching disabled for this render\n`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function extractionError(videoId: string, err: unknown): VideoExtractionFailure {
|
|
const classified = classifyVideoExtractionError(err);
|
|
return {
|
|
videoId,
|
|
kind: classified.kind,
|
|
retryable: classified.retryable,
|
|
error: classified.diagnostic,
|
|
};
|
|
}
|
|
|
|
type PreparedExtractionResult =
|
|
| { work: PreparedExtraction }
|
|
| { error: VideoExtractionFailure }
|
|
| { skipped: true };
|
|
|
|
type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure };
|
|
|
|
function scopedExtractionOptions(work: PreparedExtraction): ExtractionOptions {
|
|
return {
|
|
...options,
|
|
format: work.format,
|
|
sdrToHdrTransfer: work.sdrToHdrTransfer,
|
|
toneMapHdrToSdr: work.hdrToSdrTransformKey !== undefined,
|
|
finalFrameOnly: work.finalFrameOnly,
|
|
};
|
|
}
|
|
|
|
function rehydratePublishedCache(work: PreparedExtraction, target: CacheMissTarget) {
|
|
const rehydrated = rehydrateCacheEntry(target.entry, {
|
|
videoId: work.video.id,
|
|
srcPath: target.srcPath,
|
|
fps,
|
|
format: work.format,
|
|
metadata: work.metadata,
|
|
});
|
|
return { ...rehydrated, ownedByLookup: true };
|
|
}
|
|
|
|
// Hashed only for works that reach a lookup (deferred ranges never do), once per file.
|
|
const contentSha256ByPath = new Map<string, string | null>();
|
|
async function hashContentKeyedSources(works: PreparedExtraction[]): Promise<void> {
|
|
if (!cacheRootDir) return;
|
|
const paths = new Set<string>();
|
|
for (const work of works) {
|
|
const keyInput = cacheKeyInputs[work.index];
|
|
if (keyInput?.contentKeyed) paths.add(keyInput.videoPath);
|
|
}
|
|
await Promise.all(
|
|
[...paths].map(async (path) =>
|
|
contentSha256ByPath.set(path, await readContentSha256(path, signal)),
|
|
),
|
|
);
|
|
}
|
|
|
|
function lookupCacheFor(work: PreparedExtraction): ExtractionOutcome | UniqueExtractionMiss {
|
|
if (!cacheRootDir) return { work };
|
|
const keyInput = cacheKeyInputs[work.index];
|
|
if (!keyInput) return { work };
|
|
const contentSha256 = keyInput.contentKeyed
|
|
? (contentSha256ByPath.get(keyInput.videoPath) ?? null)
|
|
: undefined;
|
|
if (contentSha256 === null) return { work };
|
|
const transformParts = [
|
|
work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
|
|
work.hdrToSdrTransformKey,
|
|
work.finalFrameOnly ? "final-frame" : undefined,
|
|
].filter((part): part is string => part !== undefined);
|
|
const transform = transformParts.length > 0 ? transformParts.join("+") : undefined;
|
|
|
|
const lookup = lookupCacheEntry(cacheRootDir, {
|
|
videoPath: keyInput.videoPath,
|
|
mtimeMs: keyInput.mtimeMs,
|
|
size: keyInput.size,
|
|
mediaStart: keyInput.mediaStart,
|
|
contentSha256,
|
|
duration: work.videoDuration,
|
|
fps: fpsKey,
|
|
format: work.format,
|
|
transform,
|
|
});
|
|
|
|
if (!lookup.hit) {
|
|
breakdown.cacheMisses += 1;
|
|
return { work, cacheTarget: { entry: lookup.entry, srcPath: keyInput.videoPath } };
|
|
}
|
|
|
|
breakdown.cacheHits += 1;
|
|
touchCacheEntry(lookup.entry);
|
|
return {
|
|
result: rehydratePublishedCache(work, { entry: lookup.entry, srcPath: keyInput.videoPath }),
|
|
};
|
|
}
|
|
|
|
async function extractDirectMiss(
|
|
miss: UniqueExtractionMiss,
|
|
maxTransientRetries = options.maxTransientRetries ?? 0,
|
|
): Promise<ExtractedFrames> {
|
|
const { work, cacheTarget } = miss;
|
|
if (!cacheTarget) {
|
|
const outputDir = join(options.outputDir, work.video.id);
|
|
const attempted = await runVideoExtractionWithRetry(
|
|
() =>
|
|
extractVideoFramesRange(
|
|
work.videoPath,
|
|
work.video.id,
|
|
work.extractionMediaStart,
|
|
work.videoDuration,
|
|
scopedExtractionOptions(work),
|
|
signal,
|
|
config,
|
|
),
|
|
{
|
|
signal,
|
|
maxTransientRetries,
|
|
onRetry: () => {
|
|
recordTransientRetries(1);
|
|
rmSync(outputDir, { recursive: true, force: true });
|
|
},
|
|
},
|
|
);
|
|
return attempted.result;
|
|
}
|
|
|
|
const partialDir = partialCacheEntryDir(cacheTarget.entry);
|
|
rmSync(partialDir, { recursive: true, force: true });
|
|
mkdirSync(partialDir, { recursive: true });
|
|
const attempted = await runVideoExtractionWithRetry(
|
|
() =>
|
|
extractVideoFramesRange(
|
|
work.videoPath,
|
|
work.video.id,
|
|
work.extractionMediaStart,
|
|
work.videoDuration,
|
|
scopedExtractionOptions(work),
|
|
signal,
|
|
config,
|
|
partialDir,
|
|
),
|
|
{
|
|
signal,
|
|
maxTransientRetries,
|
|
onRetry: () => {
|
|
recordTransientRetries(1);
|
|
rmSync(partialDir, { recursive: true, force: true });
|
|
mkdirSync(partialDir, { recursive: true });
|
|
},
|
|
},
|
|
);
|
|
const result = attempted.result;
|
|
const published = publishCacheEntry(cacheTarget.entry, partialDir);
|
|
if (!published.published) {
|
|
breakdown.cachePublishFailures += 1;
|
|
return { ...result, ownedByLookup: false };
|
|
}
|
|
return rehydratePublishedCache(work, cacheTarget);
|
|
}
|
|
|
|
/**
|
|
* Every output slot before the source's end holds a frame, so decoding the last second gives
|
|
* the exact count. Anything unusual returns null and takes the normal path.
|
|
*/
|
|
async function deferExtraction(work: PreparedExtraction): Promise<ExtractedFrames | null> {
|
|
if (
|
|
work.finalFrameOnly ||
|
|
work.metadata.isVFR ||
|
|
work.sdrToHdrTransfer !== undefined ||
|
|
work.hdrToSdrTransformKey !== undefined
|
|
) {
|
|
return null;
|
|
}
|
|
const expectedFrames = extractionFrameCountForDuration(
|
|
work.videoDuration,
|
|
configuredFps,
|
|
false,
|
|
);
|
|
const tailFrames = Math.min(expectedFrames, Math.ceil(fps));
|
|
if (expectedFrames <= tailFrames) return null;
|
|
const probeDir = join(options.outputDir, `${work.video.id}.tail-probe`);
|
|
try {
|
|
const tail = await extractVideoFramesRange(
|
|
work.videoPath,
|
|
work.video.id,
|
|
work.extractionMediaStart,
|
|
work.videoDuration,
|
|
{
|
|
...scopedExtractionOptions(work),
|
|
frameRanges: [{ firstFrame: expectedFrames - tailFrames, frames: tailFrames }],
|
|
},
|
|
signal,
|
|
config,
|
|
probeDir,
|
|
);
|
|
const totalFrames = expectedFrames - tailFrames + tail.totalFrames;
|
|
// A full extraction this short rewrites the probed duration; let it.
|
|
if (totalFrames <= expectedFrames * 0.6) return null;
|
|
const outputDir = join(options.outputDir, work.video.id);
|
|
mkdirSync(outputDir, { recursive: true });
|
|
return {
|
|
videoId: work.video.id,
|
|
srcPath: work.videoPath,
|
|
outputDir,
|
|
framePattern: `${FRAME_FILENAME_PREFIX}%05d.${work.format}`,
|
|
fps,
|
|
totalFrames,
|
|
metadata: work.metadata,
|
|
framePaths: new Map(),
|
|
deferredRange: {
|
|
startTime: work.extractionMediaStart,
|
|
durationSeconds: work.videoDuration,
|
|
format: work.format,
|
|
},
|
|
};
|
|
} catch {
|
|
return null;
|
|
} finally {
|
|
rmSync(probeDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function executeDirectMiss(
|
|
miss: UniqueExtractionMiss,
|
|
maxTransientRetries = options.maxTransientRetries ?? 0,
|
|
): Promise<ExtractionOutcome> {
|
|
try {
|
|
return { result: await extractDirectMiss(miss, maxTransientRetries) };
|
|
} catch (err) {
|
|
return { error: extractionError(miss.work.video.id, err) };
|
|
}
|
|
}
|
|
|
|
function materializeSupersetMember(
|
|
member: SupersetMemberPlan,
|
|
superset: ExtractedFrames,
|
|
): ExtractedFrames {
|
|
const { miss } = member;
|
|
const { work, cacheTarget } = miss;
|
|
if (!cacheTarget) {
|
|
return sliceSupersetMember(
|
|
member,
|
|
superset,
|
|
join(options.outputDir, work.video.id),
|
|
fps,
|
|
configuredFps,
|
|
);
|
|
}
|
|
|
|
const partialDir = partialCacheEntryDir(cacheTarget.entry);
|
|
const sliced = sliceSupersetMember(member, superset, partialDir, fps, configuredFps);
|
|
const published = publishCacheEntry(cacheTarget.entry, partialDir);
|
|
if (!published.published) {
|
|
breakdown.cachePublishFailures += 1;
|
|
return { ...sliced, ownedByLookup: false };
|
|
}
|
|
return rehydratePublishedCache(work, cacheTarget);
|
|
}
|
|
|
|
async function executeSupersetGroup(
|
|
group: SupersetGroupPlan,
|
|
): Promise<Array<[string, ExtractionOutcome]>> {
|
|
const first = group.members[0]?.miss.work;
|
|
if (!first) return [];
|
|
// Hardlinks require source and destination on ONE filesystem. Cache-bound
|
|
// members link into partial dirs under cacheRootDir, which is commonly a
|
|
// different mount than the render's outputDir — extracting the superset
|
|
// next to the cache keeps linkSync viable there (the EXDEV copyFileSync
|
|
// fallback would silently multiply disk usage per member). The
|
|
// `.partial-` name puts crashed leftovers under the GC's aged-partial
|
|
// sweep.
|
|
const tempDir = cacheRootDir
|
|
? join(cacheRootDir, `${group.groupId}.partial-${process.pid}`)
|
|
: join(options.outputDir, group.groupId);
|
|
|
|
try {
|
|
rmSync(tempDir, { recursive: true, force: true });
|
|
// A long union can fail even when each shorter
|
|
// member range succeeds. Do not retry the optimization itself; preserve
|
|
// the established grouped→direct fallback and apply bounded retries only
|
|
// to the individual source ranges below.
|
|
const superset = await extractVideoFramesRange(
|
|
first.videoPath,
|
|
group.groupId,
|
|
group.baseStart,
|
|
group.unionDuration,
|
|
scopedExtractionOptions(first),
|
|
signal,
|
|
config,
|
|
tempDir,
|
|
);
|
|
const outcomes: Array<[string, ExtractionOutcome]> = [];
|
|
for (const member of group.members) {
|
|
outcomes.push([
|
|
member.miss.work.dedupeKey,
|
|
{ result: materializeSupersetMember(member, superset) },
|
|
]);
|
|
}
|
|
return outcomes;
|
|
} catch (err) {
|
|
// On abort, the union failure is the cancellation itself — re-running
|
|
// every member through direct extraction would spawn N doomed ffmpeg
|
|
// processes. Surface the cancellation per member instead.
|
|
if (signal?.aborted) {
|
|
return group.members.map((member) => [
|
|
member.miss.work.dedupeKey,
|
|
{ error: extractionError(member.miss.work.video.id, err) },
|
|
]);
|
|
}
|
|
const fallback = await Promise.all(
|
|
group.members.map(
|
|
async (member) =>
|
|
[member.miss.work.dedupeKey, await executeDirectMiss(member.miss)] as [
|
|
string,
|
|
ExtractionOutcome,
|
|
],
|
|
),
|
|
);
|
|
return fallback;
|
|
} finally {
|
|
rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
const preparedExtractions: PreparedExtractionResult[] = await Promise.all(
|
|
resolvedVideos.map(async ({ video, videoPath }, index) => {
|
|
if (signal?.aborted) {
|
|
throw new Error("Video frame extraction cancelled");
|
|
}
|
|
try {
|
|
const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath));
|
|
const initialWindow = resolveVideoExtractionWindow(video, metadata, options.timelineEnd);
|
|
const window = await resolveFinalFrameExtractionWindow(
|
|
videoPath,
|
|
video,
|
|
metadata,
|
|
initialWindow,
|
|
signal,
|
|
);
|
|
const videoDuration = window.durationSeconds;
|
|
if (videoDuration <= 0) {
|
|
return { skipped: true };
|
|
}
|
|
rebaseVideoToWindow(video, window);
|
|
const keyInput = cacheKeyInputs[index];
|
|
const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;
|
|
if (keyInput) keyInput.mediaStart = extractionMediaStart;
|
|
|
|
const format = resolveFrameFormat(metadata, options.format);
|
|
const sdrToHdrTransfer = sdrToHdrTransfers[index];
|
|
const hdrToSdrTransform =
|
|
options.toneMapHdrToSdr === true && isHdrColorSpaceUtil(metadata.colorSpace)
|
|
? await hdrToSdrTransformKey()
|
|
: undefined;
|
|
const finalFrameOnly = window.finalFrameOnly === true;
|
|
const dedupeKey = `${videoPath}\0${extractionMediaStart}\0${videoDuration}\0${fpsKey}\0${format}\0${sdrToHdrTransfer ?? ""}\0${hdrToSdrTransform ?? ""}\0${finalFrameOnly ? "final" : "range"}`;
|
|
|
|
return {
|
|
work: {
|
|
video,
|
|
videoPath,
|
|
index,
|
|
metadata,
|
|
videoDuration,
|
|
extractionMediaStart,
|
|
finalFrameOnly,
|
|
format,
|
|
sdrToHdrTransfer,
|
|
hdrToSdrTransformKey: hdrToSdrTransform,
|
|
dedupeKey,
|
|
},
|
|
};
|
|
} catch (err) {
|
|
return { error: extractionError(video.id, err) };
|
|
}
|
|
}),
|
|
);
|
|
|
|
const uniqueWorks = new Map<string, PreparedExtraction>();
|
|
for (const prepared of preparedExtractions) {
|
|
if ("work" in prepared && !uniqueWorks.has(prepared.work.dedupeKey)) {
|
|
uniqueWorks.set(prepared.work.dedupeKey, prepared.work);
|
|
}
|
|
}
|
|
|
|
const uniqueOutcomes = new Map<string, ExtractionOutcome>();
|
|
if (options.deferRangeExtraction) {
|
|
const works = [...uniqueWorks.values()];
|
|
const deferred = await Promise.all(works.map(deferExtraction));
|
|
works.forEach((work, i) => {
|
|
const result = deferred[i];
|
|
if (result) uniqueOutcomes.set(work.dedupeKey, { result });
|
|
});
|
|
}
|
|
const cacheMisses: UniqueExtractionMiss[] = [];
|
|
const worksToLookUp = [...uniqueWorks.values()].filter(
|
|
(work) => !uniqueOutcomes.has(work.dedupeKey),
|
|
);
|
|
await hashContentKeyedSources(worksToLookUp);
|
|
for (const work of worksToLookUp) {
|
|
const lookup = lookupCacheFor(work);
|
|
if ("work" in lookup) {
|
|
cacheMisses.push(lookup);
|
|
} else {
|
|
uniqueOutcomes.set(work.dedupeKey, lookup);
|
|
}
|
|
}
|
|
|
|
const supersetPlan = planSupersetGroups(cacheMisses, fps);
|
|
const directOutcomes = await Promise.all(
|
|
supersetPlan.direct.map(
|
|
async (miss) =>
|
|
[miss.work.dedupeKey, await executeDirectMiss(miss)] as [string, ExtractionOutcome],
|
|
),
|
|
);
|
|
for (const [key, outcome] of directOutcomes) uniqueOutcomes.set(key, outcome);
|
|
|
|
const supersetOutcomes = await Promise.all(
|
|
supersetPlan.groups.map((group) => executeSupersetGroup(group)),
|
|
);
|
|
for (const groupOutcomes of supersetOutcomes) {
|
|
for (const [key, outcome] of groupOutcomes) uniqueOutcomes.set(key, outcome);
|
|
}
|
|
|
|
const results: ExtractionOutcome[] = [];
|
|
for (const prepared of preparedExtractions) {
|
|
if ("skipped" in prepared) continue;
|
|
if ("error" in prepared) {
|
|
results.push(prepared);
|
|
continue;
|
|
}
|
|
const outcome = uniqueOutcomes.get(prepared.work.dedupeKey);
|
|
if (!outcome) {
|
|
results.push({
|
|
error: extractionError(prepared.work.video.id, "missing extraction result"),
|
|
});
|
|
continue;
|
|
}
|
|
if ("error" in outcome) {
|
|
// A shared (deduped/superset) failure fans out to every element with the
|
|
// same key; annotate followers with the leader's videoId so N copies of
|
|
// one root failure are traceable to a single extraction in traces.
|
|
const isFollower = outcome.error.videoId !== prepared.work.video.id;
|
|
const message = isFollower
|
|
? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}`
|
|
: outcome.error.error;
|
|
results.push({
|
|
error: {
|
|
videoId: prepared.work.video.id,
|
|
kind: outcome.error.kind,
|
|
retryable: outcome.error.retryable,
|
|
error: message,
|
|
},
|
|
});
|
|
continue;
|
|
}
|
|
results.push({ result: { ...outcome.result, videoId: prepared.work.video.id } });
|
|
}
|
|
|
|
breakdown.extractMs = Date.now() - phase3Start;
|
|
|
|
// Collect results and errors
|
|
for (const item of results) {
|
|
if ("error" in item && item.error) {
|
|
errors.push(item.error);
|
|
} else if ("result" in item) {
|
|
extracted.push(item.result);
|
|
if (!item.result.deferredRange) totalFramesExtracted += item.result.totalFrames;
|
|
}
|
|
}
|
|
|
|
// Sweep when this render wrote something, plus a staleness fallback so a
|
|
// 100%-warm workload (misses never > 0) still reclaims space once a day.
|
|
const sweepDue =
|
|
breakdown.cacheMisses > 0 ||
|
|
(cacheRootDir !== undefined && gcSweepDue(cacheRootDir, GC_STALENESS_MS));
|
|
if (cacheRootDir && sweepDue) {
|
|
const gcStats = gcExtractionCache(cacheRootDir, {
|
|
maxBytes: config?.extractCacheMaxBytes ?? DEFAULT_CONFIG.extractCacheMaxBytes,
|
|
minAgeMs: EXTRACT_CACHE_MIN_AGE_MS,
|
|
});
|
|
breakdown.cacheGcEvictions = gcStats.evictedEntries;
|
|
breakdown.cacheGcBytesFreed = gcStats.evictedBytes;
|
|
breakdown.cacheAgedPartialsCleared = gcStats.agedPartialsRemoved;
|
|
}
|
|
|
|
return {
|
|
success: errors.length === 0,
|
|
extracted,
|
|
errors,
|
|
totalFramesExtracted,
|
|
durationMs: Date.now() - startTime,
|
|
phaseBreakdown: breakdown,
|
|
};
|
|
}
|
|
|
|
const BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS = 1;
|
|
|
|
export function isVideoHiddenBeforeStreamStart(
|
|
videoStreamLeadSeconds: number | undefined,
|
|
mediaTime: number,
|
|
): boolean {
|
|
return (
|
|
(videoStreamLeadSeconds ?? 0) - mediaTime >=
|
|
BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6
|
|
);
|
|
}
|
|
|
|
function getFrameIndexAtTime(
|
|
extracted: ExtractedFrames,
|
|
globalTime: number,
|
|
videoStart: number,
|
|
loop = false,
|
|
mediaStart = 0,
|
|
holdLastFrame = false,
|
|
playbackRate: RateSpec = 1,
|
|
): number | null {
|
|
let localTime = Math.max(0, globalTime - videoStart);
|
|
const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
|
|
const loopDuration = timeAtSourceTime(
|
|
normalizedPlaybackRate,
|
|
Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart),
|
|
);
|
|
if (
|
|
typeof normalizedPlaybackRate === "number" &&
|
|
loop &&
|
|
loopDuration > 0 &&
|
|
localTime >= loopDuration
|
|
) {
|
|
localTime %= loopDuration;
|
|
}
|
|
let sourceTime = sourceTimeAt(normalizedPlaybackRate, localTime);
|
|
if (typeof normalizedPlaybackRate === "object" && loop) {
|
|
// A ramped loop wraps in source space, so the lane keeps running across cycles as in the preview.
|
|
const cycle = Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart);
|
|
if (cycle > 0 && sourceTime >= cycle) sourceTime %= cycle;
|
|
}
|
|
// Add epsilon before flooring to avoid IEEE 754 boundary errors where
|
|
// e.g. 0.28 * 25 === 6.999999999999999 instead of 7.
|
|
const frameIndex = Math.floor(sourceTime * extracted.fps + 1e-9);
|
|
if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
|
|
if (
|
|
isVideoHiddenBeforeStreamStart(
|
|
extracted.metadata.videoStreamLeadSeconds,
|
|
mediaStart + sourceTime,
|
|
)
|
|
) {
|
|
return null;
|
|
}
|
|
if (frameIndex >= extracted.totalFrames) {
|
|
return loop || holdLastFrame ? extracted.totalFrames - 1 : null;
|
|
}
|
|
return frameIndex;
|
|
}
|
|
|
|
export function getFrameAtTime(
|
|
extracted: ExtractedFrames,
|
|
globalTime: number,
|
|
videoStart: number,
|
|
loop = false,
|
|
mediaStart = 0,
|
|
): string | null {
|
|
if (!hasClipStarted(globalTime, videoStart)) return null;
|
|
const frameIndex = getFrameIndexAtTime(extracted, globalTime, videoStart, loop, mediaStart);
|
|
return frameIndex == null ? null : extracted.framePaths.get(frameIndex) || null;
|
|
}
|
|
|
|
/**
|
|
* Whether a media source is shorter than its `data-duration` slot by more than
|
|
* the compiler tolerance. The calculation stays tag-agnostic; current in-repo
|
|
* warnings call it for audio only because video slots may intentionally outlive
|
|
* their source and hold the final frame.
|
|
*/
|
|
export function analyzeClipMediaFit(params: {
|
|
/** Timeline slot length in seconds — `end - start` (a.k.a. data-duration). */
|
|
slotSeconds: number;
|
|
/** Playable source media after the trim offset — `duration - mediaStart`. */
|
|
mediaSeconds: number;
|
|
/** Looping clips repeat to fill the slot, so they never fall short. */
|
|
loop?: boolean;
|
|
}): { shortfallSeconds: number; toleranceSeconds: number } | null {
|
|
const { slotSeconds, mediaSeconds, loop } = params;
|
|
if (loop) return null;
|
|
if (!(slotSeconds > 0) || !Number.isFinite(mediaSeconds) || mediaSeconds < 0) return null;
|
|
const toleranceSeconds = MEDIA_DURATION_CLAMP_EPSILON_SECONDS;
|
|
const shortfallSeconds = slotSeconds - mediaSeconds;
|
|
if (shortfallSeconds <= toleranceSeconds) return null;
|
|
return { shortfallSeconds, toleranceSeconds };
|
|
}
|
|
|
|
export class FrameLookupTable {
|
|
private videos: Map<
|
|
string,
|
|
{
|
|
extracted: ExtractedFrames;
|
|
start: number;
|
|
clipWindow: { start: number; end: number };
|
|
mediaStart: number;
|
|
loop: boolean;
|
|
playbackRate: RateSpec;
|
|
}
|
|
> = new Map();
|
|
private orderedVideos: Array<{
|
|
videoId: string;
|
|
extracted: ExtractedFrames;
|
|
start: number;
|
|
clipWindow: { start: number; end: number };
|
|
mediaStart: number;
|
|
loop: boolean;
|
|
playbackRate: RateSpec;
|
|
}> = [];
|
|
private activeVideoIds: Set<string> = new Set();
|
|
private startCursor = 0;
|
|
private lastTime: number | null = null;
|
|
|
|
/** The render's `fps`, so a clip shows in exactly export visibility's frames; omit for authored times. */
|
|
constructor(private readonly fps?: number) {}
|
|
|
|
addVideo(
|
|
extracted: ExtractedFrames,
|
|
start: number,
|
|
end: number,
|
|
mediaStart: number,
|
|
loop = false,
|
|
playbackRate: RateSpec = 1,
|
|
): void {
|
|
this.videos.set(extracted.videoId, {
|
|
extracted,
|
|
start,
|
|
clipWindow: this.fps ? exportClipWindow(start, end, this.fps) : { start, end },
|
|
mediaStart,
|
|
loop,
|
|
playbackRate: normalizeRateSpec(playbackRate),
|
|
});
|
|
this.orderedVideos = Array.from(this.videos.entries())
|
|
.map(([videoId, video]) => ({ videoId, ...video }))
|
|
.sort((a, b) => a.clipWindow.start - b.clipWindow.start);
|
|
this.resetActiveState();
|
|
}
|
|
|
|
getFrame(videoId: string, globalTime: number): string | null {
|
|
const video = this.videos.get(videoId);
|
|
if (!video) return null;
|
|
if (!isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) return null;
|
|
const frameIndex = getFrameIndexAtTime(
|
|
video.extracted,
|
|
globalTime,
|
|
video.start,
|
|
video.loop,
|
|
video.mediaStart,
|
|
true,
|
|
video.playbackRate,
|
|
);
|
|
return frameIndex == null ? null : video.extracted.framePaths.get(frameIndex) || null;
|
|
}
|
|
|
|
frameDirs(): string[] {
|
|
return [...new Set(Array.from(this.videos.values(), (video) => video.extracted.outputDir))];
|
|
}
|
|
|
|
private resetActiveState(): void {
|
|
this.activeVideoIds.clear();
|
|
this.startCursor = 0;
|
|
this.lastTime = null;
|
|
}
|
|
|
|
private refreshActiveSet(globalTime: number): void {
|
|
// Half-open exportClipWindow; rendered times stay below the composition end, so no terminal hold here.
|
|
if (this.lastTime == null || globalTime < this.lastTime) {
|
|
this.activeVideoIds.clear();
|
|
this.startCursor = 0;
|
|
for (const entry of this.orderedVideos) {
|
|
if (!hasClipStarted(globalTime, entry.clipWindow.start)) break;
|
|
if (isInClipWindow(globalTime, entry.clipWindow.start, entry.clipWindow.end)) {
|
|
this.activeVideoIds.add(entry.videoId);
|
|
}
|
|
this.startCursor += 1;
|
|
}
|
|
this.lastTime = globalTime;
|
|
return;
|
|
}
|
|
|
|
while (this.startCursor < this.orderedVideos.length) {
|
|
const candidate = this.orderedVideos[this.startCursor];
|
|
if (!candidate) break;
|
|
if (!hasClipStarted(globalTime, candidate.clipWindow.start)) {
|
|
break;
|
|
}
|
|
if (isInClipWindow(globalTime, candidate.clipWindow.start, candidate.clipWindow.end)) {
|
|
this.activeVideoIds.add(candidate.videoId);
|
|
}
|
|
this.startCursor += 1;
|
|
}
|
|
|
|
for (const videoId of Array.from(this.activeVideoIds)) {
|
|
const video = this.videos.get(videoId);
|
|
if (!video || !isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) {
|
|
this.activeVideoIds.delete(videoId);
|
|
}
|
|
}
|
|
this.lastTime = globalTime;
|
|
}
|
|
|
|
getActiveFramePayloads(
|
|
globalTime: number,
|
|
): Map<string, { framePath: string; frameIndex: number }> {
|
|
const frames = new Map<string, { framePath: string; frameIndex: number }>();
|
|
this.refreshActiveSet(globalTime);
|
|
for (const videoId of this.activeVideoIds) {
|
|
const video = this.videos.get(videoId);
|
|
if (!video) continue;
|
|
const frameIndex = getFrameIndexAtTime(
|
|
video.extracted,
|
|
globalTime,
|
|
video.start,
|
|
video.loop,
|
|
video.mediaStart,
|
|
true,
|
|
video.playbackRate,
|
|
);
|
|
if (frameIndex == null) continue;
|
|
const framePath = video.extracted.framePaths.get(frameIndex);
|
|
if (!framePath) continue;
|
|
frames.set(videoId, { framePath, frameIndex });
|
|
}
|
|
return frames;
|
|
}
|
|
|
|
getActiveFrames(globalTime: number): Map<string, string> {
|
|
const payloads = this.getActiveFramePayloads(globalTime);
|
|
const frames = new Map<string, string>();
|
|
for (const [videoId, payload] of payloads) {
|
|
frames.set(videoId, payload.framePath);
|
|
}
|
|
return frames;
|
|
}
|
|
|
|
cleanup(): void {
|
|
for (const video of this.videos.values()) {
|
|
// Cache-hit / cache-write entries are owned by the extraction cache —
|
|
// a single render must not delete them, or the next render's lookup
|
|
// would miss and re-extract unnecessarily.
|
|
if (video.extracted.ownedByLookup) continue;
|
|
if (existsSync(video.extracted.outputDir)) {
|
|
rmSync(video.extracted.outputDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
this.videos.clear();
|
|
this.orderedVideos = [];
|
|
this.resetActiveState();
|
|
}
|
|
}
|
|
|
|
export function createFrameLookupTable(
|
|
videos: VideoElement[],
|
|
extracted: ExtractedFrames[],
|
|
fps?: number,
|
|
): FrameLookupTable {
|
|
const table = new FrameLookupTable(fps);
|
|
const extractedMap = new Map<string, ExtractedFrames>();
|
|
for (const ext of extracted) extractedMap.set(ext.videoId, ext);
|
|
|
|
for (const video of videos) {
|
|
const ext = extractedMap.get(video.id);
|
|
if (ext) {
|
|
table.addVideo(ext, video.start, video.end, video.mediaStart, video.loop, video.playbackRate);
|
|
}
|
|
}
|
|
|
|
return table;
|
|
}
|