mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-05 17:30:50 +00:00
Part of Phase 2 of the distributed rendering plan (determinism hardening).
See DISTRIBUTED-RENDERING-PLAN.md §17.2 (PR 2.7 row).
Distributed renders mix audio once at `plan()` time against the
composition's declared duration; the actual assembled video duration is
`Σ(chunkFrames) / fps`. Even with closed-GOP concat-copy the absolute
result is deterministic, but downstream muxers (especially ffmpeg's
`-shortest` plus Apple's mov demuxer) are sensitive to ±1ms audio/video
drift and produce silent "audio cuts off early" or "video freezes on the
last frame" bugs.
Adds packages/producer/src/services/render/audioPadTrim.ts:
- buildPadTrimAudioArgs(audio, out, sourceSec, targetSec) — pure helper
that decides the operation (pad/trim/copy) and emits the matching
ffmpeg argv. Uses `apad=pad_dur=Δ` (re-encode to AAC because filters
can't combine with `-c:a copy`), `-t target -c:a copy` (trim is a
lossless AAC packet boundary snap), or a plain `-c:a copy` when the
delta is below ~1ms.
- padOrTrimAudioToVideoFrameCount(input) — probes the assembled video
for exact frame count (`-count_packets` + `nb_read_packets`, which
equals frame count when chunks were encoded with `-bf 0` as Phase 2's
PR 2.1 already enforces), probes the audio for current duration,
computes target = `frameCount * fpsDen / fpsNum`, runs ffmpeg with the
args from the pure helper. Probes and ffmpeg runner are injectable so
unit tests don't shell out.
Six-decimal-place seconds formatting avoids ffmpeg's inconsistent handling
of scientific notation in time args across versions.
No caller invokes either function yet — Phase 3's `assemble()` will run
this after the chunk concat-copy step, before muxing audio onto the final
mp4/mov output.
15 unit tests at packages/producer/src/services/render/
audioPadTrim.test.ts pin both layers: the pure arg builder for all three
operations (incl. NTSC fps), and the wrapper for normal flow, probe
failures, invalid video info, and ffmpeg failures.
In-process behavior is unchanged. The producer's existing
`muxVideoWithAudio` path in chunkEncoder is untouched.
This is part of a stack of 10 PRs; this is PR 7 of 10.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
242 lines
8.2 KiB
TypeScript
242 lines
8.2 KiB
TypeScript
/**
|
|
* @hyperframes/engine
|
|
*
|
|
* Seekable web page to video rendering engine.
|
|
* Framework-agnostic: works with GSAP, Lottie, Three.js, CSS animations,
|
|
* or any web content that implements the window.__hf seek protocol.
|
|
*
|
|
* ## Error Convention
|
|
*
|
|
* Engine services use three error strategies depending on the operation type:
|
|
*
|
|
* - **Orchestration services throw on failure.** Browser launch, session init,
|
|
* frame capture, and CDP operations propagate errors as thrown exceptions.
|
|
* Callers are expected to catch and handle (e.g. frameCapture, browserManager,
|
|
* screenshotService, videoFrameExtractor.extractVideoFramesRange).
|
|
*
|
|
* - **FFmpeg process wrappers return `{ success, error? }` result objects.**
|
|
* Encoding, muxing, audio mixing, and streaming encode operations never reject.
|
|
* They resolve with a result that includes `success: boolean` and an optional
|
|
* `error` string (e.g. chunkEncoder, audioMixer, streamingEncoder).
|
|
*
|
|
* - **Cleanup and teardown functions never throw.** Browser close, session close,
|
|
* temp directory removal, and resource release swallow errors via `.catch(() => {})`
|
|
* to avoid masking the original failure (e.g. releaseBrowser, closeCaptureSession,
|
|
* FrameLookupTable.cleanup).
|
|
*
|
|
* - **Optional lookups return `T | undefined` or `T | null`.**
|
|
* Functions that may legitimately find nothing (resolveHeadlessShellPath,
|
|
* getFrameAtTime, detectGpuEncoder) return a nullable value instead of throwing.
|
|
*
|
|
*/
|
|
|
|
// ── Protocol types ─────────────────────────────────────────────────────────────
|
|
export type {
|
|
HfProtocol,
|
|
HfMediaElement,
|
|
HfTransitionMeta,
|
|
CaptureOptions,
|
|
CaptureVideoMetadataHint,
|
|
CaptureResult,
|
|
CaptureBufferResult,
|
|
CapturePerfSummary,
|
|
} from "./types.js";
|
|
|
|
// ── Configuration ──────────────────────────────────────────────────────────────
|
|
export { resolveConfig, DEFAULT_CONFIG, type EngineConfig } from "./config.js";
|
|
|
|
// ── Browser management ─────────────────────────────────────────────────────────
|
|
export {
|
|
acquireBrowser,
|
|
releaseBrowser,
|
|
resolveHeadlessShellPath,
|
|
resolveBrowserGpuMode,
|
|
buildChromeArgs,
|
|
ENABLE_BROWSER_POOL,
|
|
type BuildChromeArgsOptions,
|
|
type CaptureMode,
|
|
type AcquiredBrowser,
|
|
} from "./services/browserManager.js";
|
|
|
|
// ── Frame capture pipeline ──────────────────────────────────────────────────────
|
|
export {
|
|
createCaptureSession,
|
|
initializeSession,
|
|
closeCaptureSession,
|
|
captureFrame,
|
|
captureFrameToBuffer,
|
|
discardWarmupCapture,
|
|
getCompositionDuration,
|
|
getCapturePerfSummary,
|
|
prepareCaptureSessionForReuse,
|
|
type CaptureSession,
|
|
type BeforeCaptureHook,
|
|
type DiscardWarmupInnerCapture,
|
|
} from "./services/frameCapture.js";
|
|
|
|
// ── Screenshot (BeginFrame) ─────────────────────────────────────────────────────
|
|
export {
|
|
beginFrameCapture,
|
|
pageScreenshotCapture,
|
|
getCdpSession,
|
|
injectVideoFramesBatch,
|
|
syncVideoFrameVisibility,
|
|
cdpSessionCache,
|
|
initTransparentBackground,
|
|
captureAlphaPng,
|
|
applyDomLayerMask,
|
|
removeDomLayerMask,
|
|
DOM_LAYER_MASK_STYLE_ID,
|
|
type BeginFrameResult,
|
|
} from "./services/screenshotService.js";
|
|
|
|
// ── Encoding ───────────────────────────────────────────────────────────────────
|
|
export {
|
|
encodeFramesFromDir,
|
|
encodeFramesChunkedConcat,
|
|
muxVideoWithAudio,
|
|
applyFaststart,
|
|
detectGpuEncoder,
|
|
ENCODER_PRESETS,
|
|
getEncoderPreset,
|
|
type GpuEncoder,
|
|
} from "./services/chunkEncoder.js";
|
|
export type { EncoderOptions, EncodeResult, MuxResult } from "./services/chunkEncoder.types.js";
|
|
|
|
export {
|
|
spawnStreamingEncoder,
|
|
createFrameReorderBuffer,
|
|
type StreamingEncoder,
|
|
type StreamingEncoderOptions,
|
|
type StreamingEncoderResult,
|
|
type FrameReorderBuffer,
|
|
} from "./services/streamingEncoder.js";
|
|
|
|
// ── Media processing ───────────────────────────────────────────────────────────
|
|
export {
|
|
parseVideoElements,
|
|
parseImageElements,
|
|
extractVideoFramesRange,
|
|
extractAllVideoFrames,
|
|
resolveProjectRelativeSrc,
|
|
getFrameAtTime,
|
|
createFrameLookupTable,
|
|
FrameLookupTable,
|
|
type VideoElement,
|
|
type ImageElement,
|
|
type ExtractedFrames,
|
|
type ExtractionOptions,
|
|
type ExtractionResult,
|
|
type ExtractionPhaseBreakdown,
|
|
} from "./services/videoFrameExtractor.js";
|
|
|
|
export { createVideoFrameInjector } from "./services/videoFrameInjector.js";
|
|
|
|
export { parseAudioElements, processCompositionAudio } from "./services/audioMixer.js";
|
|
export type { AudioElement, AudioTrack, MixResult } from "./services/audioMixer.types.js";
|
|
|
|
// ── Parallel rendering ─────────────────────────────────────────────────────────
|
|
export {
|
|
calculateOptimalWorkers,
|
|
distributeFrames,
|
|
executeParallelCapture,
|
|
mergeWorkerFrames,
|
|
getSystemResources,
|
|
type WorkerTask,
|
|
type WorkerResult,
|
|
type ParallelProgress,
|
|
} from "./services/parallelCoordinator.js";
|
|
|
|
// ── File server ────────────────────────────────────────────────────────────────
|
|
export {
|
|
createFileServer,
|
|
type FileServerOptions,
|
|
type FileServerHandle,
|
|
} from "./services/fileServer.js";
|
|
|
|
// ── Utilities ──────────────────────────────────────────────────────────────────
|
|
export { quantizeTimeToFrame, MEDIA_VISUAL_STYLE_PROPERTIES } from "@hyperframes/core";
|
|
|
|
export {
|
|
assertSwiftShader,
|
|
readWebGlVendorInfo,
|
|
SwiftShaderAssertionError,
|
|
BROWSER_GPU_NOT_SOFTWARE,
|
|
} from "./utils/assertSwiftShader.js";
|
|
|
|
export {
|
|
extractMediaMetadata,
|
|
extractVideoMetadata,
|
|
extractAudioMetadata,
|
|
analyzeKeyframeIntervals,
|
|
type VideoMetadata,
|
|
type AudioMetadata,
|
|
type KeyframeAnalysis,
|
|
} from "./utils/ffprobe.js";
|
|
|
|
export { downloadToTemp, isHttpUrl } from "./utils/urlDownloader.js";
|
|
export {
|
|
runFfmpeg,
|
|
formatFfmpegError,
|
|
type RunFfmpegOptions,
|
|
type RunFfmpegResult,
|
|
} from "./utils/runFfmpeg.js";
|
|
|
|
export {
|
|
decodePng,
|
|
decodePngToRgb48le,
|
|
blitRgba8OverRgb48le,
|
|
blitRgb48leRegion,
|
|
blitRgb48leAffine,
|
|
parseTransformMatrix,
|
|
roundedRectAlpha,
|
|
resampleRgb48leObjectFit,
|
|
normalizeObjectFit,
|
|
type ObjectFit,
|
|
} from "./utils/alphaBlit.js";
|
|
|
|
export { groupIntoLayers, type CompositeLayer } from "./utils/layerCompositor.js";
|
|
|
|
// ── Shader transitions ────────────────────────────────────────────────────────
|
|
export {
|
|
type TransitionFn,
|
|
TRANSITIONS,
|
|
crossfade,
|
|
sampleRgb48le,
|
|
hdrToLinear,
|
|
linearToHdr,
|
|
convertTransfer,
|
|
} from "./utils/shaderTransitions.js";
|
|
|
|
export {
|
|
initHdrReadback,
|
|
uploadAndReadbackHdrFrame,
|
|
float16ToPqRgb,
|
|
buildHdrChromeArgs,
|
|
launchHdrBrowser,
|
|
} from "./services/hdrCapture.js";
|
|
|
|
export { captureScreenshotWithAlpha } from "./services/screenshotService.js";
|
|
|
|
export {
|
|
hideVideoElements,
|
|
showVideoElements,
|
|
queryVideoElementBounds,
|
|
queryElementStacking,
|
|
type VideoElementBounds,
|
|
type ElementStackingInfo,
|
|
} from "./services/videoFrameInjector.js";
|
|
|
|
export {
|
|
isHdrColorSpace,
|
|
detectTransfer,
|
|
getHdrEncoderColorParams,
|
|
analyzeCompositionHdr,
|
|
DEFAULT_HDR10_MASTERING,
|
|
type HdrTransfer,
|
|
type HdrEncoderColorParams,
|
|
type CompositionHdrInfo,
|
|
type HdrMasteringMetadata,
|
|
} from "./utils/hdr.js";
|
|
export type { VideoColorSpace } from "./utils/ffprobe.js";
|