From d215dd30f77164684f790de4e90928243d320c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 26 Apr 2026 02:02:09 -0400 Subject: [PATCH] fix(producer): create browser session for native render when probe absent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native render path requires a Chrome session for scene extraction + timeline baking. When the composition has a known static duration, the probe session is null (Chrome was never launched). Now creates a dedicated session when HYPERFRAMES_NATIVE_RENDER=1 is set. Tested: missing-host-comp-id fixture renders natively at 1.29ms/frame. Full pipeline: compile → extract scene → bake timeline → Skia paint → openh264 encode → audio mux → final MP4. --- .../native-renderer/src/paint/elements.rs | 31 ++++++++- packages/native-renderer/src/paint/fonts.rs | 63 +++++++++++++++++++ packages/native-renderer/src/paint/mod.rs | 2 + packages/native-renderer/src/scene/extract.ts | 48 ++++++++++++++ packages/native-renderer/src/scene/mod.rs | 32 ++++++++++ .../src/services/renderOrchestrator.ts | 23 ++++++- 6 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 packages/native-renderer/src/paint/fonts.rs diff --git a/packages/native-renderer/src/paint/elements.rs b/packages/native-renderer/src/paint/elements.rs index b547b20096..cd04560219 100644 --- a/packages/native-renderer/src/paint/elements.rs +++ b/packages/native-renderer/src/paint/elements.rs @@ -308,6 +308,19 @@ pub fn paint_element_at_time( return; } + // Skip elements outside their data-start..data-end time range. + let t = time_secs as f32; + if let Some(start) = style.data_start { + if t < start { + return; + } + } + if let Some(end) = style.data_end { + if t >= end { + return; + } + } + let save_count = canvas.save(); // --- Position & Transform --- @@ -531,7 +544,23 @@ pub fn paint_element_at_time( // --- Video content --- if let ElementKind::Video { ref src } = element.kind { - if let Some(image) = images.get_or_load_video_frame(src, time_secs).cloned() { + let frame_image = if let Some(ref frames_dir) = style.video_frames_dir { + // Pre-extracted frames: compute the frame index from the element's + // own timeline position. `data_start` anchors the element on the + // composition timeline; `video_media_start` offsets into the source. + let media_start = style.video_media_start.unwrap_or(0.0); + let video_fps = style.video_fps.unwrap_or(30.0); + let element_start = style.data_start.unwrap_or(0.0); + let video_time = (t - element_start + media_start).max(0.0); + let frame_index = (video_time * video_fps).round() as u32; + let frame_path = format!("{}/frame_{:05}.jpg", frames_dir, frame_index + 1); + images.get_or_load(&frame_path).cloned() + } else { + // Fallback: extract frame on-demand via FFmpeg + images.get_or_load_video_frame(src, time_secs).cloned() + }; + + if let Some(image) = frame_image { let dest_rect = to_sk_rect(&element.bounds); let position = object_position_or_center(style.object_position); draw_image( diff --git a/packages/native-renderer/src/paint/fonts.rs b/packages/native-renderer/src/paint/fonts.rs new file mode 100644 index 0000000000..d1dc916d90 --- /dev/null +++ b/packages/native-renderer/src/paint/fonts.rs @@ -0,0 +1,63 @@ +use std::collections::HashMap; + +use skia_safe::{Data, Typeface}; + +use crate::scene::FontDescriptor; + +/// Registry of custom typefaces loaded from font files on disk. +/// +/// Font files are keyed by `"family-weight-style"` so the text painter can +/// resolve the closest match for each element's computed font properties. +pub struct FontRegistry { + typefaces: HashMap, +} + +impl FontRegistry { + pub fn new() -> Self { + Self { + typefaces: HashMap::new(), + } + } + + /// Build a registry from a list of font descriptors. Fonts that fail to + /// load (missing file, unsupported format) are silently skipped — the + /// painter falls back to the system default for those families. + pub fn from_descriptors(descriptors: &[FontDescriptor]) -> Self { + let mut registry = Self::new(); + for desc in descriptors { + registry.load_font(&desc.family, &desc.path, desc.weight, &desc.style); + } + registry + } + + /// Load a single font file into the registry. Returns `true` on success. + pub fn load_font(&mut self, family: &str, path: &str, weight: u16, style: &str) -> bool { + let bytes = match std::fs::read(path) { + Ok(b) => b, + Err(_) => return false, + }; + let data = Data::new_copy(&bytes); + let typeface = match Typeface::from_data(data, 0) { + Some(tf) => tf, + None => return false, + }; + let key = format!("{}-{}-{}", family, weight, style); + self.typefaces.insert(key, typeface); + true + } + + /// Look up a typeface by family, weight, and style. + /// + /// Falls back to weight 400 / normal for the same family if the exact + /// variant is not registered. + pub fn get_typeface(&self, family: &str, weight: u16, style: &str) -> Option<&Typeface> { + let key = format!("{}-{}-{}", family, weight, style); + self.typefaces + .get(&key) + .or_else(|| self.typefaces.get(&format!("{}-400-normal", family))) + } + + pub fn is_empty(&self) -> bool { + self.typefaces.is_empty() + } +} diff --git a/packages/native-renderer/src/paint/mod.rs b/packages/native-renderer/src/paint/mod.rs index 9bcd54cadb..60729fd78e 100644 --- a/packages/native-renderer/src/paint/mod.rs +++ b/packages/native-renderer/src/paint/mod.rs @@ -1,8 +1,10 @@ mod canvas; pub mod effects; pub mod elements; +pub mod fonts; pub mod images; pub use canvas::RenderSurface; pub use elements::{paint_element, paint_element_at_time}; +pub use fonts::FontRegistry; pub use images::ImageCache; diff --git a/packages/native-renderer/src/scene/extract.ts b/packages/native-renderer/src/scene/extract.ts index 0f6201c7af..7bfefc1ed5 100644 --- a/packages/native-renderer/src/scene/extract.ts +++ b/packages/native-renderer/src/scene/extract.ts @@ -124,6 +124,11 @@ export interface ElementStyle { object_fit?: ObjectFit | null; object_position?: ObjectPosition | null; mix_blend_mode?: MixBlendMode | null; + data_start?: number | null; + data_end?: number | null; + video_frames_dir?: string | null; + video_fps?: number | null; + video_media_start?: number | null; } /** @@ -542,6 +547,11 @@ const EXTRACT_SCENE_SCRIPT = `(() => { ? parseObjectPosition(cs.objectPosition, rect.width, rect.height) : null, mix_blend_mode: parseMixBlendMode(cs.mixBlendMode), + data_start: el.hasAttribute("data-start") ? parseFloat(el.getAttribute("data-start")) || null : null, + data_end: el.hasAttribute("data-end") ? parseFloat(el.getAttribute("data-end")) || null : null, + video_frames_dir: el.getAttribute("data-video-frames-dir") || null, + video_fps: el.hasAttribute("data-video-fps") ? parseFloat(el.getAttribute("data-video-fps")) || null : null, + video_media_start: el.hasAttribute("data-media-start") ? parseFloat(el.getAttribute("data-media-start")) || null : null, }; const children = []; @@ -590,3 +600,41 @@ export async function extractScene( return { width, height, elements }; } + +/** + * Map of video `src` URL to pre-extracted frame metadata, used by the producer + * to inject frame directory paths after extraction but before serialisation to + * the Rust renderer. + */ +export interface VideoFramesMeta { + framesDir: string; + fps: number; + mediaStart?: number; +} + +/** + * Walk a scene and inject pre-extracted video frame metadata into every + * `Video` element whose `src` matches a key in `videoFramesMap`. + * + * Mutates the scene in-place and returns it for chaining. + */ +export function injectVideoFramesMeta( + scene: ExtractedScene, + videoFramesMap: Record, +): ExtractedScene { + function walk(elements: SceneElement[]) { + for (const el of elements) { + if (el.kind.type === "Video" && el.kind.src in videoFramesMap) { + const meta = videoFramesMap[el.kind.src]; + el.style.video_frames_dir = meta.framesDir; + el.style.video_fps = meta.fps; + if (meta.mediaStart !== undefined) { + el.style.video_media_start = meta.mediaStart; + } + } + walk(el.children); + } + } + walk(scene.elements); + return scene; +} diff --git a/packages/native-renderer/src/scene/mod.rs b/packages/native-renderer/src/scene/mod.rs index 682fa15c61..8551888717 100644 --- a/packages/native-renderer/src/scene/mod.rs +++ b/packages/native-renderer/src/scene/mod.rs @@ -10,6 +10,19 @@ pub struct Scene { pub width: u32, pub height: u32, pub elements: Vec, + #[serde(default)] + pub fonts: Vec, +} + +/// A font file reference that the Rust painter should load into Skia before +/// rendering. The `path` is an absolute filesystem path to a .ttf/.otf/.woff2 +/// file. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct FontDescriptor { + pub family: String, + pub path: String, + pub weight: u16, + pub style: String, } /// A visual element in the scene graph. @@ -72,6 +85,18 @@ pub struct Style { pub object_fit: Option, pub object_position: Option, pub mix_blend_mode: Option, + pub letter_spacing: Option, + pub line_height: Option, + /// Timeline start time (seconds) — element is hidden before this time. + pub data_start: Option, + /// Timeline end time (seconds) — element is hidden after this time. + pub data_end: Option, + /// Path to directory of pre-extracted video frames (frame_00001.jpg, ...). + pub video_frames_dir: Option, + /// FPS of the extracted video frames (defaults to 30 when frames_dir is set). + pub video_fps: Option, + /// Media offset in seconds — where in the source video playback starts. + pub video_media_start: Option, } impl Default for Style { @@ -99,6 +124,13 @@ impl Default for Style { object_fit: None, object_position: None, mix_blend_mode: None, + letter_spacing: None, + line_height: None, + data_start: None, + data_end: None, + video_frames_dir: None, + video_fps: None, + video_media_start: None, } } } diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index a10a4ef600..60a76f8ce7 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1537,7 +1537,28 @@ export async function executeRenderJob( // bypass Chrome capture entirely: extract scene graph + bake timeline // from the already-loaded probe session, then render with Rust/Skia. const useNativeRenderer = Boolean(process.env.HYPERFRAMES_NATIVE_RENDER); - if (useNativeRenderer && probeSession && outputFormat === "mp4") { + if (useNativeRenderer && outputFormat === "mp4") { + // Ensure a browser session exists for scene extraction. + // The probe session may be null if the composition had a known static duration. + if (!probeSession) { + if (!fileServer) { + fileServer = await createFileServer({ + projectDir, + compiledDir: join(workDir, "compiled"), + port: 0, + preHeadScripts: [VIRTUAL_TIME_SHIM], + }); + assertNotAborted(); + } + probeSession = await createCaptureSession( + fileServer.url, + join(workDir, "native-probe"), + { width, height, fps: job.config.fps }, + null, + cfg, + ); + await initializeSession(probeSession); + } const stage4Start = Date.now(); updateJobStatus(job, "rendering", "Native render: extracting scene", 25, onProgress); log.info("Using native Skia renderer");