mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-12 20:47:30 +00:00
fix(producer): create browser session for native render when probe absent
The native render path requires a Chrome session for scene extraction + timeline baking. When the composition has a known static duration, the probe session is null (Chrome was never launched). Now creates a dedicated session when HYPERFRAMES_NATIVE_RENDER=1 is set. Tested: missing-host-comp-id fixture renders natively at 1.29ms/frame. Full pipeline: compile → extract scene → bake timeline → Skia paint → openh264 encode → audio mux → final MP4.
This commit is contained in:
@@ -308,6 +308,19 @@ pub fn paint_element_at_time(
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip elements outside their data-start..data-end time range.
|
||||
let t = time_secs as f32;
|
||||
if let Some(start) = style.data_start {
|
||||
if t < start {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if let Some(end) = style.data_end {
|
||||
if t >= end {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let save_count = canvas.save();
|
||||
|
||||
// --- Position & Transform ---
|
||||
@@ -531,7 +544,23 @@ pub fn paint_element_at_time(
|
||||
|
||||
// --- Video content ---
|
||||
if let ElementKind::Video { ref src } = element.kind {
|
||||
if let Some(image) = images.get_or_load_video_frame(src, time_secs).cloned() {
|
||||
let frame_image = if let Some(ref frames_dir) = style.video_frames_dir {
|
||||
// Pre-extracted frames: compute the frame index from the element's
|
||||
// own timeline position. `data_start` anchors the element on the
|
||||
// composition timeline; `video_media_start` offsets into the source.
|
||||
let media_start = style.video_media_start.unwrap_or(0.0);
|
||||
let video_fps = style.video_fps.unwrap_or(30.0);
|
||||
let element_start = style.data_start.unwrap_or(0.0);
|
||||
let video_time = (t - element_start + media_start).max(0.0);
|
||||
let frame_index = (video_time * video_fps).round() as u32;
|
||||
let frame_path = format!("{}/frame_{:05}.jpg", frames_dir, frame_index + 1);
|
||||
images.get_or_load(&frame_path).cloned()
|
||||
} else {
|
||||
// Fallback: extract frame on-demand via FFmpeg
|
||||
images.get_or_load_video_frame(src, time_secs).cloned()
|
||||
};
|
||||
|
||||
if let Some(image) = frame_image {
|
||||
let dest_rect = to_sk_rect(&element.bounds);
|
||||
let position = object_position_or_center(style.object_position);
|
||||
draw_image(
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use skia_safe::{Data, Typeface};
|
||||
|
||||
use crate::scene::FontDescriptor;
|
||||
|
||||
/// Registry of custom typefaces loaded from font files on disk.
|
||||
///
|
||||
/// Font files are keyed by `"family-weight-style"` so the text painter can
|
||||
/// resolve the closest match for each element's computed font properties.
|
||||
pub struct FontRegistry {
|
||||
typefaces: HashMap<String, Typeface>,
|
||||
}
|
||||
|
||||
impl FontRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
typefaces: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a registry from a list of font descriptors. Fonts that fail to
|
||||
/// load (missing file, unsupported format) are silently skipped — the
|
||||
/// painter falls back to the system default for those families.
|
||||
pub fn from_descriptors(descriptors: &[FontDescriptor]) -> Self {
|
||||
let mut registry = Self::new();
|
||||
for desc in descriptors {
|
||||
registry.load_font(&desc.family, &desc.path, desc.weight, &desc.style);
|
||||
}
|
||||
registry
|
||||
}
|
||||
|
||||
/// Load a single font file into the registry. Returns `true` on success.
|
||||
pub fn load_font(&mut self, family: &str, path: &str, weight: u16, style: &str) -> bool {
|
||||
let bytes = match std::fs::read(path) {
|
||||
Ok(b) => b,
|
||||
Err(_) => return false,
|
||||
};
|
||||
let data = Data::new_copy(&bytes);
|
||||
let typeface = match Typeface::from_data(data, 0) {
|
||||
Some(tf) => tf,
|
||||
None => return false,
|
||||
};
|
||||
let key = format!("{}-{}-{}", family, weight, style);
|
||||
self.typefaces.insert(key, typeface);
|
||||
true
|
||||
}
|
||||
|
||||
/// Look up a typeface by family, weight, and style.
|
||||
///
|
||||
/// Falls back to weight 400 / normal for the same family if the exact
|
||||
/// variant is not registered.
|
||||
pub fn get_typeface(&self, family: &str, weight: u16, style: &str) -> Option<&Typeface> {
|
||||
let key = format!("{}-{}-{}", family, weight, style);
|
||||
self.typefaces
|
||||
.get(&key)
|
||||
.or_else(|| self.typefaces.get(&format!("{}-400-normal", family)))
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.typefaces.is_empty()
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
mod canvas;
|
||||
pub mod effects;
|
||||
pub mod elements;
|
||||
pub mod fonts;
|
||||
pub mod images;
|
||||
|
||||
pub use canvas::RenderSurface;
|
||||
pub use elements::{paint_element, paint_element_at_time};
|
||||
pub use fonts::FontRegistry;
|
||||
pub use images::ImageCache;
|
||||
|
||||
@@ -124,6 +124,11 @@ export interface ElementStyle {
|
||||
object_fit?: ObjectFit | null;
|
||||
object_position?: ObjectPosition | null;
|
||||
mix_blend_mode?: MixBlendMode | null;
|
||||
data_start?: number | null;
|
||||
data_end?: number | null;
|
||||
video_frames_dir?: string | null;
|
||||
video_fps?: number | null;
|
||||
video_media_start?: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -542,6 +547,11 @@ const EXTRACT_SCENE_SCRIPT = `(() => {
|
||||
? parseObjectPosition(cs.objectPosition, rect.width, rect.height)
|
||||
: null,
|
||||
mix_blend_mode: parseMixBlendMode(cs.mixBlendMode),
|
||||
data_start: el.hasAttribute("data-start") ? parseFloat(el.getAttribute("data-start")) || null : null,
|
||||
data_end: el.hasAttribute("data-end") ? parseFloat(el.getAttribute("data-end")) || null : null,
|
||||
video_frames_dir: el.getAttribute("data-video-frames-dir") || null,
|
||||
video_fps: el.hasAttribute("data-video-fps") ? parseFloat(el.getAttribute("data-video-fps")) || null : null,
|
||||
video_media_start: el.hasAttribute("data-media-start") ? parseFloat(el.getAttribute("data-media-start")) || null : null,
|
||||
};
|
||||
|
||||
const children = [];
|
||||
@@ -590,3 +600,41 @@ export async function extractScene(
|
||||
|
||||
return { width, height, elements };
|
||||
}
|
||||
|
||||
/**
|
||||
* Map of video `src` URL to pre-extracted frame metadata, used by the producer
|
||||
* to inject frame directory paths after extraction but before serialisation to
|
||||
* the Rust renderer.
|
||||
*/
|
||||
export interface VideoFramesMeta {
|
||||
framesDir: string;
|
||||
fps: number;
|
||||
mediaStart?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk a scene and inject pre-extracted video frame metadata into every
|
||||
* `Video` element whose `src` matches a key in `videoFramesMap`.
|
||||
*
|
||||
* Mutates the scene in-place and returns it for chaining.
|
||||
*/
|
||||
export function injectVideoFramesMeta(
|
||||
scene: ExtractedScene,
|
||||
videoFramesMap: Record<string, VideoFramesMeta>,
|
||||
): ExtractedScene {
|
||||
function walk(elements: SceneElement[]) {
|
||||
for (const el of elements) {
|
||||
if (el.kind.type === "Video" && el.kind.src in videoFramesMap) {
|
||||
const meta = videoFramesMap[el.kind.src];
|
||||
el.style.video_frames_dir = meta.framesDir;
|
||||
el.style.video_fps = meta.fps;
|
||||
if (meta.mediaStart !== undefined) {
|
||||
el.style.video_media_start = meta.mediaStart;
|
||||
}
|
||||
}
|
||||
walk(el.children);
|
||||
}
|
||||
}
|
||||
walk(scene.elements);
|
||||
return scene;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,19 @@ pub struct Scene {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub elements: Vec<Element>,
|
||||
#[serde(default)]
|
||||
pub fonts: Vec<FontDescriptor>,
|
||||
}
|
||||
|
||||
/// A font file reference that the Rust painter should load into Skia before
|
||||
/// rendering. The `path` is an absolute filesystem path to a .ttf/.otf/.woff2
|
||||
/// file.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct FontDescriptor {
|
||||
pub family: String,
|
||||
pub path: String,
|
||||
pub weight: u16,
|
||||
pub style: String,
|
||||
}
|
||||
|
||||
/// A visual element in the scene graph.
|
||||
@@ -72,6 +85,18 @@ pub struct Style {
|
||||
pub object_fit: Option<ObjectFit>,
|
||||
pub object_position: Option<ObjectPosition>,
|
||||
pub mix_blend_mode: Option<MixBlendMode>,
|
||||
pub letter_spacing: Option<f32>,
|
||||
pub line_height: Option<f32>,
|
||||
/// Timeline start time (seconds) — element is hidden before this time.
|
||||
pub data_start: Option<f32>,
|
||||
/// Timeline end time (seconds) — element is hidden after this time.
|
||||
pub data_end: Option<f32>,
|
||||
/// Path to directory of pre-extracted video frames (frame_00001.jpg, ...).
|
||||
pub video_frames_dir: Option<String>,
|
||||
/// FPS of the extracted video frames (defaults to 30 when frames_dir is set).
|
||||
pub video_fps: Option<f32>,
|
||||
/// Media offset in seconds — where in the source video playback starts.
|
||||
pub video_media_start: Option<f32>,
|
||||
}
|
||||
|
||||
impl Default for Style {
|
||||
@@ -99,6 +124,13 @@ impl Default for Style {
|
||||
object_fit: None,
|
||||
object_position: None,
|
||||
mix_blend_mode: None,
|
||||
letter_spacing: None,
|
||||
line_height: None,
|
||||
data_start: None,
|
||||
data_end: None,
|
||||
video_frames_dir: None,
|
||||
video_fps: None,
|
||||
video_media_start: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1537,7 +1537,28 @@ export async function executeRenderJob(
|
||||
// bypass Chrome capture entirely: extract scene graph + bake timeline
|
||||
// from the already-loaded probe session, then render with Rust/Skia.
|
||||
const useNativeRenderer = Boolean(process.env.HYPERFRAMES_NATIVE_RENDER);
|
||||
if (useNativeRenderer && probeSession && outputFormat === "mp4") {
|
||||
if (useNativeRenderer && outputFormat === "mp4") {
|
||||
// Ensure a browser session exists for scene extraction.
|
||||
// The probe session may be null if the composition had a known static duration.
|
||||
if (!probeSession) {
|
||||
if (!fileServer) {
|
||||
fileServer = await createFileServer({
|
||||
projectDir,
|
||||
compiledDir: join(workDir, "compiled"),
|
||||
port: 0,
|
||||
preHeadScripts: [VIRTUAL_TIME_SHIM],
|
||||
});
|
||||
assertNotAborted();
|
||||
}
|
||||
probeSession = await createCaptureSession(
|
||||
fileServer.url,
|
||||
join(workDir, "native-probe"),
|
||||
{ width, height, fps: job.config.fps },
|
||||
null,
|
||||
cfg,
|
||||
);
|
||||
await initializeSession(probeSession);
|
||||
}
|
||||
const stage4Start = Date.now();
|
||||
updateJobStatus(job, "rendering", "Native render: extracting scene", 25, onProgress);
|
||||
log.info("Using native Skia renderer");
|
||||
|
||||
Reference in New Issue
Block a user