Merge pull request #508 from heygen-com/skill/r2hf-corpus-t1-t2

feat(skills): remotion-to-hyperframes corpus T1+T2 (3/7)
This commit is contained in:
James Russo
2026-04-27 22:17:37 -07:00
committed by GitHub
21 changed files with 644 additions and 0 deletions
@@ -0,0 +1,10 @@
# Render output
remotion-src/out/
hf-src/out/
hf.mp4
diff/
strip/
# Remotion / HF dependencies
node_modules/
package-lock.json
@@ -0,0 +1,41 @@
# Tier 1 — title-card-fade
## What it tests
The simplest non-trivial Remotion → HyperFrames translation. A single text
element fades in over the first 0.5 s, holds for 2.0 s, and fades out over
the last 0.5 s. No audio, no media, no custom components.
If a translation can't pass T1, it's broken on table-stakes basics:
`AbsoluteFill`, `useCurrentFrame`, `interpolate` with multi-segment input,
and the timing offset between Remotion's frame-based driver and HF's
paused-GSAP driver.
## Translation walk-through
| Remotion | HyperFrames |
| ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- |
| `<AbsoluteFill style={{ backgroundColor: "#0a0a0a" }}>` | `<body style="background: #0a0a0a">` + a positioned root div |
| `useCurrentFrame()` | dropped — HF seeks the timeline |
| `interpolate(frame, [0, 15, 75, 90], [0, 1, 1, 0])` at fps=30 | `gsap.timeline({ paused: true })` with three `.to()` calls at offsets 0s/0.5s/2.5s, each `ease: "none"` |
| `<div style={{ opacity }}>HELLO</div>` | static markup; opacity is animated by the timeline |
The Remotion→HF time conversion is `time = frame / fps`. So
`[0, 15, 75, 90]` at 30 fps becomes `[0, 0.5, 2.5, 3.0]` seconds.
## How to render and evaluate
```bash
# Render Remotion baseline
cd remotion-src && npm install && npm run render
# Renders to remotion-src/out/baseline.mp4
# Render HyperFrames translation
cd ../hf-src && npx hyperframes render --output ../hf.mp4
# Compare with the eval harness (from skill scripts/)
../../../scripts/render_diff.sh ./remotion-src/out/baseline.mp4 ./hf.mp4 ./diff
```
`expected.json` documents the SSIM threshold (0.95) for this fixture; the
calibrated mean against Remotion @ 4.0 with PNG/BT.709 output is 0.974.
@@ -0,0 +1,26 @@
{
"tier": 1,
"name": "title-card-fade",
"composition_id": "TitleCard",
"description": "Solid black background, single 'HELLO' element fades in 0-0.5s, holds 0.5-2.5s, fades out 2.5-3.0s. Tests the most basic Remotion → HyperFrames translation: a single AbsoluteFill, a single useCurrentFrame-driven interpolate, no audio, no media, no custom React components.",
"duration_seconds": 3,
"fps": 30,
"width": 1280,
"height": 720,
"ssim_threshold": 0.95,
"validation": {
"measured_mean_ssim": 0.974,
"measured_min_ssim": 0.972,
"measured_p05_ssim": 0.972,
"measured_p95_ssim": 0.983,
"measured_at": "2026-04-27",
"measured_against": "remotion@4.0 (PNG output, BT.709) vs hyperframes@0.4.15-alpha.1"
},
"translation_notes": [
"Remotion: AbsoluteFill → HF: position:absolute;inset:0 div",
"Remotion: interpolate(frame, [0,15,75,90], [0,1,1,0]) at fps=30 → HF: paused GSAP timeline with three keyframed tweens at 0s, 0.5s, 2.5s with ease:'none' (linear matches Remotion's default linear interpolation)",
"No fonts loaded; both renderers use system Helvetica/Arial fallback. The Linux fallback diverges between Remotion's bundled Chromium and HyperFrames' chrome-headless-shell — same fontWeight:800 renders perceptibly bolder in HF. This costs ~0.025 mean SSIM and is the dominant non-translation noise floor.",
"Remotion config must use setVideoImageFormat('png') + setColorSpace('bt709'); the JPEG default writes yuvj420p (full-range) which costs ~0.05 SSIM vs HF's yuv420p (limited-range)."
],
"rationale": "Threshold 0.95 sits ~0.02 below measured p05. A real translation regression (wrong easing, wrong durations) drops mean SSIM by 0.05+. Encoder/font drift between CI runs is bounded at ~0.01."
}
@@ -0,0 +1,61 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>tier-1-title-card</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/gsap/3.12.5/gsap.min.js"></script>
<style>
html,
body {
margin: 0;
padding: 0;
width: 1280px;
height: 720px;
overflow: hidden;
background: #0a0a0a;
font-family: Helvetica, Arial, sans-serif;
}
.title {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
color: #ffffff;
font-size: 160px;
font-weight: 800;
letter-spacing: 0.05em;
opacity: 0;
}
</style>
</head>
<body>
<div
id="stage"
data-composition-id="tier-1-title-card"
data-start="0"
data-width="1280"
data-height="720"
data-duration="3"
data-fps="30"
>
<div id="title" class="clip" data-start="0" data-duration="3" data-track-index="0">
<div class="title">HELLO</div>
</div>
<script>
// Translation of Remotion's
// interpolate(frame, [0, 15, 75, 90], [0, 1, 1, 0]) at fps=30
// into a paused GSAP timeline keyed in seconds.
// Frame ranges → time ranges: 0/30=0, 15/30=0.5, 75/30=2.5, 90/30=3.0
const tl = gsap.timeline({ paused: true });
const target = document.querySelector("#title .title");
tl.to(target, { opacity: 1, duration: 0.5, ease: "none" }, 0);
tl.to(target, { opacity: 1, duration: 2.0, ease: "none" }, 0.5);
tl.to(target, { opacity: 0, duration: 0.5, ease: "none" }, 2.5);
window.__timelines = window.__timelines || {};
window.__timelines["tier-1-title-card"] = tl;
</script>
</div>
</body>
</html>
@@ -0,0 +1,14 @@
{
"name": "tier-1-title-card-remotion",
"version": "0.0.0",
"private": true,
"scripts": {
"render": "remotion render TitleCard out/baseline.mp4"
},
"dependencies": {
"@remotion/cli": "^4.0.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"remotion": "^4.0.0"
}
}
@@ -0,0 +1,13 @@
import { Config } from "@remotion/cli/config";
// Match HyperFrames' default render so SSIM diffs measure translation
// fidelity, not encoder differences.
//
// setVideoImageFormat("png") avoids the JPEG limited-range/full-range
// colorspace flag (yuvj420p vs yuv420p) that otherwise costs ~0.05 SSIM.
//
// setColorSpace("bt709") matches HF's BT.709 SDR output.
Config.setVideoImageFormat("png");
Config.setColorSpace("bt709");
Config.setOverwriteOutput(true);
Config.setConcurrency(1);
@@ -0,0 +1,13 @@
import { Composition } from "remotion";
import { TitleCard } from "./TitleCard";
export const RemotionRoot = () => (
<Composition
id="TitleCard"
component={TitleCard}
durationInFrames={90}
fps={30}
width={1280}
height={720}
/>
);
@@ -0,0 +1,34 @@
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
export const TitleCard = () => {
const frame = useCurrentFrame();
// Fade in 0-15, hold 15-75, fade out 75-90.
const opacity = interpolate(frame, [0, 15, 75, 90], [0, 1, 1, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<AbsoluteFill
style={{
backgroundColor: "#0a0a0a",
justifyContent: "center",
alignItems: "center",
fontFamily: "Helvetica, Arial, sans-serif",
}}
>
<div
style={{
fontSize: 160,
fontWeight: 800,
color: "#ffffff",
opacity,
letterSpacing: "0.05em",
}}
>
HELLO
</div>
</AbsoluteFill>
);
};
@@ -0,0 +1,4 @@
import { registerRoot } from "remotion";
import { RemotionRoot } from "./Root";
registerRoot(RemotionRoot);
@@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "ES2018",
"module": "ESNext",
"jsx": "react-jsx",
"strict": true,
"esModuleInterop": true,
"moduleResolution": "node",
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"isolatedModules": true
},
"include": ["src"]
}
@@ -0,0 +1,14 @@
# Generated by setup.sh
remotion-src/public/
hf-src/assets/
# Remotion / HF dependencies
remotion-src/node_modules/
remotion-src/package-lock.json
# Render output
remotion-src/out/
hf-src/out/
hf.mp4
diff/
strip/
@@ -0,0 +1,54 @@
# Tier 2 — title-image-outro
## What it tests
Three-scene composition. Each scene exercises a different Remotion idiom:
1. **Scene 1 (02 s)** — TitleScene with `spring({damping:12, stiffness:100, mass:1})`
driving a `transform: scale()` on text. Tests the lossy `spring → GSAP ease` translation.
2. **Scene 2 (24 s)** — ImageScene that fades in a `staticFile`-loaded image and
linearly scales it from 0.8 → 1.0. Tests asset paths + linear `interpolate`.
3. **Scene 3 (46 s)** — OutroScene with a 1-s linear fade-in. Sanity check after
the harder scenes.
A silent 6-second WAV plays throughout at `volume={0.5}`. Tests `<Audio>` translation.
If a translation passes T2, the skill correctly handles `<Sequence>` boundaries,
`<Audio>` / `<Img>` / `staticFile`, and the Remotion `spring → GSAP ease` heuristic.
## Translation walk-through
| Remotion | HyperFrames |
| ------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- |
| `<Sequence from={0} durationInFrames={60}>` | `<div data-start="0" data-duration="2" data-track-index="0">` |
| `spring({frame, fps, config: {damping:12, stiffness:100, mass:1}})` | `gsap.to(target, { scale: 1, duration: 0.7, ease: "back.out(1.4)" })` |
| `<Audio src={staticFile("music.wav")} volume={0.5} />` | `<audio src="assets/music.wav" data-start="0" data-duration="6" data-volume="0.5" data-track-index="1">` |
| `<Img src={staticFile("square.png")} />` | `<img src="assets/square.png">` (with setup.sh copying into both trees) |
| `interpolate(frame, [0, 15], [0, 1])` at 30 fps | `gsap.to(target, { opacity: 1, duration: 0.5, ease: "none" })` |
The scene crossfading is a HyperFrames idiom, not a Remotion one: at scene boundaries
we `gsap.set(scene, { opacity: 0 })` so the previous scene disappears at the
right time. Remotion does this implicitly by virtue of `<Sequence>`'s durationInFrames.
## How to render and evaluate
```bash
# 1. Generate the binary assets (PNG + WAV) via ffmpeg
./setup.sh
# 2. Render Remotion baseline
cd remotion-src && npm install && npm run render
# 3. Render HyperFrames translation
cd ../hf-src && npx hyperframes render --output ../hf.mp4
# 4. Compare
../../../scripts/render_diff.sh ./remotion-src/out/baseline.mp4 ./hf.mp4 ./diff
```
## Why threshold 0.95?
Same threshold as T1 (`expected.json` codifies it for the orchestrator). Spring → `back.out(1.4)`
came in cleaner than predicted during calibration — the validated mean is 0.985 against the
0.95 gate. If the translation breaks anything else (spring overshoot wrong, stagger off,
asset path drift), mean SSIM will fall well below 0.95 — that's the failure signal.
@@ -0,0 +1,29 @@
{
"tier": 2,
"name": "title-image-outro",
"composition_id": "MultiScene",
"description": "Three-scene composition exercising Sequence, spring, interpolate, Audio, Img, and staticFile. Title scene uses Remotion's spring (translated to GSAP back.out as an approximation). Image scene scales an Img (from staticFile) with linear interpolate. Outro scene fades text in linearly. A silent WAV plays throughout at volume 0.5.",
"duration_seconds": 6,
"fps": 30,
"width": 1280,
"height": 720,
"ssim_threshold": 0.95,
"validation": {
"measured_mean_ssim": 0.985,
"measured_min_ssim": 0.963,
"measured_p05_ssim": 0.966,
"measured_p95_ssim": 0.999,
"measured_at": "2026-04-27",
"measured_against": "remotion@4.0 (PNG output, BT.709) vs hyperframes@0.4.15-alpha.1",
"notes": "Spring → back.out(1.4) translation came out cleaner than expected; mean 0.985 leaves substantial headroom over 0.95."
},
"translation_notes": [
"spring({damping:12, stiffness:100, mass:1}) → back.out(1.4) over 0.7s. Spring overshoot+settle and back.out's overshoot+settle have similar shape; budget ~0.02 SSIM for the late-tail curvature mismatch (validated lower than predicted in spec; original notes overestimated drift).",
"<Sequence from durationInFrames> → wrapping div with data-start/data-duration in seconds and explicit gsap.set(opacity, 0/1) at scene boundaries to crossfade in/out cleanly",
"<Audio src volume> → <audio data-start data-duration data-volume>",
"<Img src={staticFile('x')}> → <img src='assets/x'>; setup.sh copies the asset into both fixture trees",
"interpolate with default linear easing → ease:'none' in GSAP",
"Fonts again rely on system Helvetica/Arial; ~0.015 SSIM cost from AA differences"
],
"rationale": "Threshold 0.95 sits ~0.015 below measured p05 (0.966). T2 actually validated cleaner than T1 because the lower title fontWeight (140px vs T1's 160px) shows less of the system-font fallback divergence."
}
@@ -0,0 +1,118 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>tier-2-multi-scene</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/gsap/3.12.5/gsap.min.js"></script>
<style>
html,
body {
margin: 0;
padding: 0;
width: 1280px;
height: 720px;
overflow: hidden;
background: #0a0a0a;
font-family: Helvetica, Arial, sans-serif;
}
.scene {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
opacity: 0;
}
.title {
font-size: 140px;
font-weight: 800;
color: #ffffff;
letter-spacing: 0.05em;
transform: scale(0);
}
.square {
width: 200px;
height: 200px;
opacity: 0;
transform: scale(0.8);
}
.outro {
font-size: 100px;
font-weight: 600;
color: #ffffff;
}
</style>
</head>
<body>
<div
id="stage"
data-composition-id="tier-2-multi-scene"
data-start="0"
data-width="1280"
data-height="720"
data-duration="6"
data-fps="30"
>
<div id="scene-1" class="scene clip" data-start="0" data-duration="2" data-track-index="0">
<div class="title">Welcome</div>
</div>
<div id="scene-2" class="scene clip" data-start="2" data-duration="2" data-track-index="0">
<img class="square" src="assets/square.png" alt="" />
</div>
<div id="scene-3" class="scene clip" data-start="4" data-duration="2" data-track-index="0">
<div class="outro">Goodbye</div>
</div>
<audio
id="bg-music"
data-start="0"
data-duration="6"
data-track-index="1"
data-volume="0.5"
src="assets/music.wav"
></audio>
<script>
// Translation of Remotion's 3-Sequence MultiScene at fps=30, total 6s.
//
// Scene 1 (0-2s): TitleScene — spring scale on "Welcome".
// Remotion: spring({frame, fps, config: {damping:12, stiffness:100, mass:1}})
// GSAP equivalent: back.out(1.4) over ~0.7s. Spring → ease translation
// is approximate; expect minor differences in the tail of the curve.
//
// Scene 2 (2-4s): ImageScene — opacity 0→1 over 0-0.5s + scale 0.8→1.0 over 0-2.0s.
// Both Remotion interpolates use linear (default), match with ease:"none".
//
// Scene 3 (4-6s): OutroScene — opacity 0→1 over 4-5s, linear.
const tl = gsap.timeline({ paused: true });
// Scene 1 setup: opacity flick on so scene becomes visible at start
const scene1 = document.querySelector("#scene-1");
const title = scene1.querySelector(".title");
tl.set(scene1, { opacity: 1 }, 0);
tl.to(title, { scale: 1, duration: 0.7, ease: "back.out(1.4)" }, 0);
tl.set(scene1, { opacity: 0 }, 2);
// Scene 2
const scene2 = document.querySelector("#scene-2");
const square = scene2.querySelector(".square");
tl.set(scene2, { opacity: 1 }, 2);
tl.to(square, { opacity: 1, duration: 0.5, ease: "none" }, 2);
tl.to(square, { scale: 1.0, duration: 2.0, ease: "none" }, 2);
tl.set(scene2, { opacity: 0 }, 4);
// Scene 3
const scene3 = document.querySelector("#scene-3");
const outro = scene3.querySelector(".outro");
tl.set(scene3, { opacity: 1 }, 4);
tl.fromTo(outro, { opacity: 0 }, { opacity: 1, duration: 1.0, ease: "none" }, 4);
window.__timelines = window.__timelines || {};
window.__timelines["tier-2-multi-scene"] = tl;
</script>
</div>
</body>
</html>
@@ -0,0 +1,14 @@
{
"name": "tier-2-multi-scene-remotion",
"version": "0.0.0",
"private": true,
"scripts": {
"render": "remotion render MultiScene out/baseline.mp4"
},
"dependencies": {
"@remotion/cli": "^4.0.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"remotion": "^4.0.0"
}
}
@@ -0,0 +1,13 @@
import { Config } from "@remotion/cli/config";
// Match HyperFrames' default render so SSIM diffs measure translation
// fidelity, not encoder differences.
//
// setVideoImageFormat("png") avoids the JPEG limited-range/full-range
// colorspace flag (yuvj420p vs yuv420p) that otherwise costs ~0.05 SSIM.
//
// setColorSpace("bt709") matches HF's BT.709 SDR output.
Config.setVideoImageFormat("png");
Config.setColorSpace("bt709");
Config.setOverwriteOutput(true);
Config.setConcurrency(1);
@@ -0,0 +1,105 @@
import {
AbsoluteFill,
Audio,
Img,
Sequence,
interpolate,
spring,
staticFile,
useCurrentFrame,
useVideoConfig,
} from "remotion";
const TitleScene = () => {
const frame = useCurrentFrame();
const { fps } = useVideoConfig();
const scale = spring({ frame, fps, config: { damping: 12, stiffness: 100, mass: 1 } });
return (
<AbsoluteFill
style={{
backgroundColor: "#0a0a0a",
justifyContent: "center",
alignItems: "center",
fontFamily: "Helvetica, Arial, sans-serif",
}}
>
<div
style={{
fontSize: 140,
fontWeight: 800,
color: "#ffffff",
transform: `scale(${scale})`,
letterSpacing: "0.05em",
}}
>
Welcome
</div>
</AbsoluteFill>
);
};
const ImageScene = () => {
const frame = useCurrentFrame();
const opacity = interpolate(frame, [0, 15], [0, 1], { extrapolateRight: "clamp" });
const scale = interpolate(frame, [0, 60], [0.8, 1.0], { extrapolateRight: "clamp" });
return (
<AbsoluteFill
style={{
backgroundColor: "#0a0a0a",
justifyContent: "center",
alignItems: "center",
}}
>
<Img
src={staticFile("square.png")}
style={{
width: 200,
height: 200,
opacity,
transform: `scale(${scale})`,
}}
/>
</AbsoluteFill>
);
};
const OutroScene = () => {
const frame = useCurrentFrame();
const opacity = interpolate(frame, [0, 30], [0, 1], { extrapolateRight: "clamp" });
return (
<AbsoluteFill
style={{
backgroundColor: "#0a0a0a",
justifyContent: "center",
alignItems: "center",
fontFamily: "Helvetica, Arial, sans-serif",
}}
>
<div
style={{
fontSize: 100,
fontWeight: 600,
color: "#ffffff",
opacity,
}}
>
Goodbye
</div>
</AbsoluteFill>
);
};
export const MultiScene = () => (
<AbsoluteFill>
<Sequence from={0} durationInFrames={60}>
<TitleScene />
</Sequence>
<Sequence from={60} durationInFrames={60}>
<ImageScene />
</Sequence>
<Sequence from={120} durationInFrames={60}>
<OutroScene />
</Sequence>
<Audio src={staticFile("music.wav")} volume={0.5} />
</AbsoluteFill>
);
@@ -0,0 +1,13 @@
import { Composition } from "remotion";
import { MultiScene } from "./MultiScene";
export const RemotionRoot = () => (
<Composition
id="MultiScene"
component={MultiScene}
durationInFrames={180}
fps={30}
width={1280}
height={720}
/>
);
@@ -0,0 +1,4 @@
import { registerRoot } from "remotion";
import { RemotionRoot } from "./Root";
registerRoot(RemotionRoot);
@@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "ES2018",
"module": "ESNext",
"jsx": "react-jsx",
"strict": true,
"esModuleInterop": true,
"moduleResolution": "node",
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"isolatedModules": true
},
"include": ["src"]
}
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
# setup.sh — generate the binary assets this fixture needs.
#
# Both Remotion and HyperFrames variants need a 200x200 blue PNG and a
# 6-second silent WAV. Generating them via ffmpeg keeps binaries out of
# the repo while still letting the fixture render reproducibly.
#
# Run from the fixture root: ./setup.sh
set -euo pipefail
THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if ! command -v ffmpeg >/dev/null 2>&1; then
echo "error: ffmpeg not on PATH" >&2
exit 2
fi
mkdir -p "$THIS_DIR/remotion-src/public" "$THIS_DIR/hf-src/assets"
# 200x200 solid blue PNG, ~200 bytes.
ffmpeg -y -hide_banner -loglevel error \
-f lavfi -i "color=color=#3066be:size=200x200" -frames:v 1 \
"$THIS_DIR/remotion-src/public/square.png"
cp "$THIS_DIR/remotion-src/public/square.png" "$THIS_DIR/hf-src/assets/square.png"
# 6-second silent WAV at 8 kHz mono. ~96 KB if checked in, but it is generated.
ffmpeg -y -hide_banner -loglevel error \
-f lavfi -i "anullsrc=cl=mono:r=8000" -t 6 -acodec pcm_s16le \
"$THIS_DIR/remotion-src/public/music.wav"
cp "$THIS_DIR/remotion-src/public/music.wav" "$THIS_DIR/hf-src/assets/music.wav"
echo "generated:"
ls -la "$THIS_DIR/remotion-src/public/" "$THIS_DIR/hf-src/assets/"