fix(producer): create browser session for native render when probe absent

The native render path requires a Chrome session for scene extraction +
timeline baking. When the composition has a known static duration, the
probe session is null (Chrome was never launched). Now creates a
dedicated session when HYPERFRAMES_NATIVE_RENDER=1 is set.

Tested: missing-host-comp-id fixture renders natively at 1.29ms/frame.
Full pipeline: compile → extract scene → bake timeline → Skia paint →
openh264 encode → audio mux → final MP4.
This commit is contained in:
Miguel Ángel
2026-04-26 02:02:09 -04:00
parent 38b7b798bd
commit d215dd30f7
6 changed files with 197 additions and 2 deletions
+30 -1
View File
@@ -308,6 +308,19 @@ pub fn paint_element_at_time(
return;
}
// Skip elements outside their data-start..data-end time range.
let t = time_secs as f32;
if let Some(start) = style.data_start {
if t < start {
return;
}
}
if let Some(end) = style.data_end {
if t >= end {
return;
}
}
let save_count = canvas.save();
// --- Position & Transform ---
@@ -531,7 +544,23 @@ pub fn paint_element_at_time(
// --- Video content ---
if let ElementKind::Video { ref src } = element.kind {
if let Some(image) = images.get_or_load_video_frame(src, time_secs).cloned() {
let frame_image = if let Some(ref frames_dir) = style.video_frames_dir {
// Pre-extracted frames: compute the frame index from the element's
// own timeline position. `data_start` anchors the element on the
// composition timeline; `video_media_start` offsets into the source.
let media_start = style.video_media_start.unwrap_or(0.0);
let video_fps = style.video_fps.unwrap_or(30.0);
let element_start = style.data_start.unwrap_or(0.0);
let video_time = (t - element_start + media_start).max(0.0);
let frame_index = (video_time * video_fps).round() as u32;
let frame_path = format!("{}/frame_{:05}.jpg", frames_dir, frame_index + 1);
images.get_or_load(&frame_path).cloned()
} else {
// Fallback: extract frame on-demand via FFmpeg
images.get_or_load_video_frame(src, time_secs).cloned()
};
if let Some(image) = frame_image {
let dest_rect = to_sk_rect(&element.bounds);
let position = object_position_or_center(style.object_position);
draw_image(
@@ -0,0 +1,63 @@
use std::collections::HashMap;
use skia_safe::{Data, Typeface};
use crate::scene::FontDescriptor;
/// Registry of custom typefaces loaded from font files on disk.
///
/// Font files are keyed by `"family-weight-style"` so the text painter can
/// resolve the closest match for each element's computed font properties.
pub struct FontRegistry {
typefaces: HashMap<String, Typeface>,
}
impl FontRegistry {
pub fn new() -> Self {
Self {
typefaces: HashMap::new(),
}
}
/// Build a registry from a list of font descriptors. Fonts that fail to
/// load (missing file, unsupported format) are silently skipped — the
/// painter falls back to the system default for those families.
pub fn from_descriptors(descriptors: &[FontDescriptor]) -> Self {
let mut registry = Self::new();
for desc in descriptors {
registry.load_font(&desc.family, &desc.path, desc.weight, &desc.style);
}
registry
}
/// Load a single font file into the registry. Returns `true` on success.
pub fn load_font(&mut self, family: &str, path: &str, weight: u16, style: &str) -> bool {
let bytes = match std::fs::read(path) {
Ok(b) => b,
Err(_) => return false,
};
let data = Data::new_copy(&bytes);
let typeface = match Typeface::from_data(data, 0) {
Some(tf) => tf,
None => return false,
};
let key = format!("{}-{}-{}", family, weight, style);
self.typefaces.insert(key, typeface);
true
}
/// Look up a typeface by family, weight, and style.
///
/// Falls back to weight 400 / normal for the same family if the exact
/// variant is not registered.
pub fn get_typeface(&self, family: &str, weight: u16, style: &str) -> Option<&Typeface> {
let key = format!("{}-{}-{}", family, weight, style);
self.typefaces
.get(&key)
.or_else(|| self.typefaces.get(&format!("{}-400-normal", family)))
}
pub fn is_empty(&self) -> bool {
self.typefaces.is_empty()
}
}
@@ -1,8 +1,10 @@
mod canvas;
pub mod effects;
pub mod elements;
pub mod fonts;
pub mod images;
pub use canvas::RenderSurface;
pub use elements::{paint_element, paint_element_at_time};
pub use fonts::FontRegistry;
pub use images::ImageCache;
@@ -124,6 +124,11 @@ export interface ElementStyle {
object_fit?: ObjectFit | null;
object_position?: ObjectPosition | null;
mix_blend_mode?: MixBlendMode | null;
data_start?: number | null;
data_end?: number | null;
video_frames_dir?: string | null;
video_fps?: number | null;
video_media_start?: number | null;
}
/**
@@ -542,6 +547,11 @@ const EXTRACT_SCENE_SCRIPT = `(() => {
? parseObjectPosition(cs.objectPosition, rect.width, rect.height)
: null,
mix_blend_mode: parseMixBlendMode(cs.mixBlendMode),
data_start: el.hasAttribute("data-start") ? parseFloat(el.getAttribute("data-start")) || null : null,
data_end: el.hasAttribute("data-end") ? parseFloat(el.getAttribute("data-end")) || null : null,
video_frames_dir: el.getAttribute("data-video-frames-dir") || null,
video_fps: el.hasAttribute("data-video-fps") ? parseFloat(el.getAttribute("data-video-fps")) || null : null,
video_media_start: el.hasAttribute("data-media-start") ? parseFloat(el.getAttribute("data-media-start")) || null : null,
};
const children = [];
@@ -590,3 +600,41 @@ export async function extractScene(
return { width, height, elements };
}
/**
* Map of video `src` URL to pre-extracted frame metadata, used by the producer
* to inject frame directory paths after extraction but before serialisation to
* the Rust renderer.
*/
export interface VideoFramesMeta {
framesDir: string;
fps: number;
mediaStart?: number;
}
/**
* Walk a scene and inject pre-extracted video frame metadata into every
* `Video` element whose `src` matches a key in `videoFramesMap`.
*
* Mutates the scene in-place and returns it for chaining.
*/
export function injectVideoFramesMeta(
scene: ExtractedScene,
videoFramesMap: Record<string, VideoFramesMeta>,
): ExtractedScene {
function walk(elements: SceneElement[]) {
for (const el of elements) {
if (el.kind.type === "Video" && el.kind.src in videoFramesMap) {
const meta = videoFramesMap[el.kind.src];
el.style.video_frames_dir = meta.framesDir;
el.style.video_fps = meta.fps;
if (meta.mediaStart !== undefined) {
el.style.video_media_start = meta.mediaStart;
}
}
walk(el.children);
}
}
walk(scene.elements);
return scene;
}
+32
View File
@@ -10,6 +10,19 @@ pub struct Scene {
pub width: u32,
pub height: u32,
pub elements: Vec<Element>,
#[serde(default)]
pub fonts: Vec<FontDescriptor>,
}
/// A font file reference that the Rust painter should load into Skia before
/// rendering. The `path` is an absolute filesystem path to a .ttf/.otf/.woff2
/// file.
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct FontDescriptor {
pub family: String,
pub path: String,
pub weight: u16,
pub style: String,
}
/// A visual element in the scene graph.
@@ -72,6 +85,18 @@ pub struct Style {
pub object_fit: Option<ObjectFit>,
pub object_position: Option<ObjectPosition>,
pub mix_blend_mode: Option<MixBlendMode>,
pub letter_spacing: Option<f32>,
pub line_height: Option<f32>,
/// Timeline start time (seconds) — element is hidden before this time.
pub data_start: Option<f32>,
/// Timeline end time (seconds) — element is hidden after this time.
pub data_end: Option<f32>,
/// Path to directory of pre-extracted video frames (frame_00001.jpg, ...).
pub video_frames_dir: Option<String>,
/// FPS of the extracted video frames (defaults to 30 when frames_dir is set).
pub video_fps: Option<f32>,
/// Media offset in seconds — where in the source video playback starts.
pub video_media_start: Option<f32>,
}
impl Default for Style {
@@ -99,6 +124,13 @@ impl Default for Style {
object_fit: None,
object_position: None,
mix_blend_mode: None,
letter_spacing: None,
line_height: None,
data_start: None,
data_end: None,
video_frames_dir: None,
video_fps: None,
video_media_start: None,
}
}
}
@@ -1537,7 +1537,28 @@ export async function executeRenderJob(
// bypass Chrome capture entirely: extract scene graph + bake timeline
// from the already-loaded probe session, then render with Rust/Skia.
const useNativeRenderer = Boolean(process.env.HYPERFRAMES_NATIVE_RENDER);
if (useNativeRenderer && probeSession && outputFormat === "mp4") {
if (useNativeRenderer && outputFormat === "mp4") {
// Ensure a browser session exists for scene extraction.
// The probe session may be null if the composition had a known static duration.
if (!probeSession) {
if (!fileServer) {
fileServer = await createFileServer({
projectDir,
compiledDir: join(workDir, "compiled"),
port: 0,
preHeadScripts: [VIRTUAL_TIME_SHIM],
});
assertNotAborted();
}
probeSession = await createCaptureSession(
fileServer.url,
join(workDir, "native-probe"),
{ width, height, fps: job.config.fps },
null,
cfg,
);
await initializeSession(probeSession);
}
const stage4Start = Date.now();
updateJobStatus(job, "rendering", "Native render: extracting scene", 25, onProgress);
log.info("Using native Skia renderer");