mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
fix: address PR #299 review — lint correctness, docs, Gemini benchmark
- lintMultipleRootCompositions: scan filesystem for HTML files with data-composition-id (was filtering results array — always 1 entry) - lintDuplicateAudioTracks: order-independent attribute extraction, dedup by (src,start,duration,trackIndex), Infinity fallback for missing data-duration (matches runtime behavior) - 10 new tests for both lint rules - docs: explicit skill invocation, remove gsap-skills, fix indentation - Gemini: env override (HYPERFRAMES_GEMINI_MODEL), benchmark data in code comment (49 imgs: 3.1-lite ~507ms/img, 2.5-lite ~230ms/img) - cli.mdx: version-agnostic "Gemini vision" reference Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
de5b53c08b
commit
274db7a5ef
+1
-1
@@ -69,7 +69,7 @@
|
||||
{
|
||||
"group": "Guides",
|
||||
"pages": [
|
||||
"guides/website-to-video",
|
||||
"guides/website-to-video",
|
||||
"guides/prompting",
|
||||
"guides/gsap-animation",
|
||||
"guides/rendering",
|
||||
|
||||
@@ -18,7 +18,6 @@ Give your AI agent a URL and a creative direction. It captures the site, extract
|
||||
|
||||
```bash
|
||||
npx skills add heygen-com/hyperframes
|
||||
npx skills add greensock/gsap-skills
|
||||
```
|
||||
|
||||
Works with [Claude Code](https://claude.ai/claude-code), [Cursor](https://cursor.sh), [Gemini CLI](https://github.com/google-gemini/gemini-cli), and [Codex CLI](https://github.com/openai/codex).
|
||||
@@ -27,11 +26,14 @@ Give your AI agent a URL and a creative direction. It captures the site, extract
|
||||
Open your agent in any directory and describe the video you want:
|
||||
|
||||
```
|
||||
Create a 25-second product launch video from https://stripe.com.
|
||||
Bold, cinematic, financial infrastructure energy.
|
||||
Create a 25-second product launch video from https://example.com. Bold, cinematic, dark theme energy.
|
||||
```
|
||||
|
||||
The agent discovers the `/website-to-hyperframes` skill and runs the full pipeline automatically — capture, design, script, storyboard, voiceover, build, validate.
|
||||
The agent loads the skill when they see a URL and a video request, and runs the full pipeline — capture, design, script, storyboard, voiceover, build, validate.
|
||||
|
||||
<Note>
|
||||
Agents also trigger this skill automatically when they see a URL and a video request.
|
||||
</Note>
|
||||
</Step>
|
||||
<Step title="Preview">
|
||||
```bash
|
||||
@@ -203,7 +205,7 @@ You don't need to re-run the full pipeline to make changes:
|
||||
npx skills add heygen-com/hyperframes
|
||||
```
|
||||
|
||||
The skill triggers automatically when the agent sees a URL and a video request. To invoke explicitly: _"Use the /website-to-hyperframes skill."_
|
||||
Lead your prompt with _"Use the /website-to-hyperframes skill"_ for the most reliable results. Agents also discover it automatically when they see a URL and a video request.
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
|
||||
@@ -354,7 +354,7 @@ This is suppressed in CI environments, non-TTY shells, and when `HYPERFRAMES_NO_
|
||||
|
||||
Output is a self-contained directory with a `CLAUDE.md` file that any AI agent can read to understand the captured site. Used by the `/website-to-hyperframes` skill as step 1 of the video production pipeline.
|
||||
|
||||
Set `GEMINI_API_KEY` in a `.env` file for AI-powered image descriptions via Gemini 2.5 Flash vision (~$0.001/image). See the [Website to Video](/guides/website-to-video#enriching-captures-with-gemini-vision) guide for details.
|
||||
Set `GEMINI_API_KEY` in a `.env` file for AI-powered image descriptions via Gemini vision (~$0.001/image). See the [Website to Video](/guides/website-to-video#enriching-captures-with-gemini-vision) guide for details.
|
||||
|
||||
</Tab>
|
||||
<Tab title="Preview">
|
||||
|
||||
@@ -174,7 +174,11 @@ export async function captionImagesWithGemini(
|
||||
// Free tier: 5 RPM → batch 5, 12s pause (~$0 but slow)
|
||||
// Paid tier: 2000 RPM → batch 20, 1s pause (~$0.001/image, fast)
|
||||
// We try a larger batch first; if rate-limited, fall back to smaller batches.
|
||||
const model = "gemini-3.1-flash-lite-preview";
|
||||
// Default is a preview model — update when GA ships.
|
||||
// Benchmark (49 images, paid tier): 3.1-flash-lite-preview ~507ms/img 131ch avg,
|
||||
// 2.5-flash-lite ~230ms/img 117ch avg. Preview has richer captions but higher variance.
|
||||
// Override: HYPERFRAMES_GEMINI_MODEL=gemini-2.5-flash-lite
|
||||
const model = process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview";
|
||||
const BATCH_SIZE = 20;
|
||||
for (let i = 0; i < imageFiles.length; i += BATCH_SIZE) {
|
||||
const batch = imageFiles.slice(i, i + BATCH_SIZE);
|
||||
|
||||
@@ -326,6 +326,146 @@ describe("audio_src_not_found", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("multiple_root_compositions", () => {
|
||||
it("fires when two HTML files have data-composition-id", () => {
|
||||
const project = makeProject(validHtml());
|
||||
writeFileSync(
|
||||
join(project.dir, "scaffold.html"),
|
||||
'<div data-composition-id="scaffold" data-width="1920" data-height="1080" data-duration="10"></div>',
|
||||
);
|
||||
const { totalErrors, results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find(
|
||||
(f) => f.code === "multiple_root_compositions",
|
||||
);
|
||||
expect(finding).toBeDefined();
|
||||
expect(finding?.severity).toBe("error");
|
||||
expect(finding?.message).toContain("scaffold.html");
|
||||
expect(totalErrors).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("does NOT fire with a single root composition", () => {
|
||||
const project = makeProject(validHtml());
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find(
|
||||
(f) => f.code === "multiple_root_compositions",
|
||||
);
|
||||
expect(finding).toBeUndefined();
|
||||
});
|
||||
|
||||
it("ignores HTML files without data-composition-id", () => {
|
||||
const project = makeProject(validHtml());
|
||||
writeFileSync(join(project.dir, "readme.html"), "<html><body>Not a composition</body></html>");
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find(
|
||||
(f) => f.code === "multiple_root_compositions",
|
||||
);
|
||||
expect(finding).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("duplicate_audio_track", () => {
|
||||
it("detects overlapping audio with attributes in any order", () => {
|
||||
// The original scaffold bug: data-start BEFORE data-track-index
|
||||
const html = `<html><body>
|
||||
<div data-composition-id="main" data-width="1920" data-height="1080" data-duration="30">
|
||||
<audio id="narration" data-start="0" data-duration="28" data-track-index="0" src="narration.wav">
|
||||
<audio id="bg" src="bg.wav" data-track-index="0" data-start="5" data-duration="20">
|
||||
</div>
|
||||
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
|
||||
</body></html>`;
|
||||
const project = makeProject(html);
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find((f) => f.code === "duplicate_audio_track");
|
||||
expect(finding).toBeDefined();
|
||||
expect(finding?.severity).toBe("warning");
|
||||
});
|
||||
|
||||
it("does NOT fire for non-overlapping audio on the same track", () => {
|
||||
const html = `<html><body>
|
||||
<div data-composition-id="main" data-width="1920" data-height="1080" data-duration="20">
|
||||
<audio id="a" src="a.wav" data-track-index="0" data-start="0" data-duration="10">
|
||||
<audio id="b" src="b.wav" data-track-index="0" data-start="10" data-duration="10">
|
||||
</div>
|
||||
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
|
||||
</body></html>`;
|
||||
const project = makeProject(html);
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find((f) => f.code === "duplicate_audio_track");
|
||||
expect(finding).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does NOT fire for audio on different tracks", () => {
|
||||
const html = `<html><body>
|
||||
<div data-composition-id="main" data-width="1920" data-height="1080" data-duration="20">
|
||||
<audio id="a" src="a.wav" data-track-index="0" data-start="0" data-duration="20">
|
||||
<audio id="b" src="b.wav" data-track-index="1" data-start="5" data-duration="10">
|
||||
</div>
|
||||
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
|
||||
</body></html>`;
|
||||
const project = makeProject(html);
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find((f) => f.code === "duplicate_audio_track");
|
||||
expect(finding).toBeUndefined();
|
||||
});
|
||||
|
||||
it("deduplicates same audio found in root + sub-composition", () => {
|
||||
const project = makeProject(validHtmlWithAudio(), {
|
||||
"scene.html": validHtmlWithAudio("scene"),
|
||||
});
|
||||
writeFileSync(join(project.dir, "song.mp3"), "fake");
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find((f) => f.code === "duplicate_audio_track");
|
||||
expect(finding).toBeUndefined();
|
||||
});
|
||||
|
||||
it("detects overlap when data-duration is missing (Infinity fallback)", () => {
|
||||
const html = `<html><body>
|
||||
<div data-composition-id="main" data-width="1920" data-height="1080" data-duration="30">
|
||||
<audio id="a" src="a.wav" data-track-index="0" data-start="0" data-duration="20">
|
||||
<audio id="b" src="b.wav" data-track-index="0" data-start="15">
|
||||
</div>
|
||||
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
|
||||
</body></html>`;
|
||||
const project = makeProject(html);
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find((f) => f.code === "duplicate_audio_track");
|
||||
expect(finding).toBeDefined();
|
||||
});
|
||||
|
||||
it("formats Infinity end times as 'end' without crashing", () => {
|
||||
const html = `<html><body>
|
||||
<div data-composition-id="main" data-width="1920" data-height="1080" data-duration="30">
|
||||
<audio id="a" src="a.wav" data-track-index="0" data-start="0">
|
||||
<audio id="b" src="b.wav" data-track-index="0" data-start="5">
|
||||
</div>
|
||||
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
|
||||
</body></html>`;
|
||||
const project = makeProject(html);
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find((f) => f.code === "duplicate_audio_track");
|
||||
expect(finding).toBeDefined();
|
||||
expect(finding?.message).toContain("end");
|
||||
expect(finding?.message).not.toContain("Infinity");
|
||||
});
|
||||
|
||||
it("finds audio across multiple HTML sources (g-flag regression)", () => {
|
||||
const project = makeProject(validHtmlWithAudio(), {
|
||||
"scene.html": `<html><body>
|
||||
<div data-composition-id="scene" data-width="1920" data-height="1080">
|
||||
<audio id="overlap" src="music.wav" data-track-index="0" data-start="5" data-duration="20">
|
||||
</div>
|
||||
<script>window.__timelines = window.__timelines || {}; window.__timelines["scene"] = gsap.timeline({ paused: true });</script>
|
||||
</body></html>`,
|
||||
});
|
||||
writeFileSync(join(project.dir, "song.mp3"), "fake");
|
||||
writeFileSync(join(project.dir, "music.wav"), "fake");
|
||||
const { results } = lintProject(project);
|
||||
const finding = results[0]?.result.findings.find((f) => f.code === "duplicate_audio_track");
|
||||
// song.mp3@0 (from validHtmlWithAudio, no data-duration → Infinity) and music.wav@5-25 overlap
|
||||
expect(finding).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldBlockRender", () => {
|
||||
it("default: does not block on errors", () => {
|
||||
expect(shouldBlockRender(false, false, 5, 0)).toBe(false);
|
||||
|
||||
@@ -53,7 +53,7 @@ export function lintProject(project: ProjectDir): ProjectLintResult {
|
||||
const projectFindings = [
|
||||
...lintProjectAudioFiles(project.dir, allHtmlSources),
|
||||
...lintAudioSrcNotFound(project.dir, allHtmlSources),
|
||||
...lintMultipleRootCompositions(results),
|
||||
...lintMultipleRootCompositions(project.dir),
|
||||
...lintDuplicateAudioTracks(allHtmlSources),
|
||||
];
|
||||
if (projectFindings.length > 0) {
|
||||
@@ -109,7 +109,7 @@ function lintProjectAudioFiles(projectDir: string, htmlSources: string[]): Hyper
|
||||
fixHint:
|
||||
'Add an <audio id="my-audio" src="' +
|
||||
audioFiles[0] +
|
||||
'" data-start="0" data-track-index="0" data-volume="1"></audio> element inside the composition root.',
|
||||
'" data-start="0" data-duration="__DURATION__" data-track-index="0" data-volume="1"></audio> element inside the composition root. Replace __DURATION__ with the audio length in seconds.',
|
||||
});
|
||||
}
|
||||
|
||||
@@ -157,46 +157,76 @@ function lintAudioSrcNotFound(projectDir: string, htmlSources: string[]): Hyperf
|
||||
}
|
||||
|
||||
/**
|
||||
* Error if multiple root-level HTML files exist (not in compositions/).
|
||||
* Catches the double-audio bug where a scaffold and the real index.html
|
||||
* both register as root compositions.
|
||||
* Error if multiple root-level HTML files with data-composition-id exist.
|
||||
* Scans the project directory filesystem (not just what lintProject chose to read)
|
||||
* to catch stray scaffold files, duplicates, or backup copies.
|
||||
*/
|
||||
function lintMultipleRootCompositions(
|
||||
results: Array<{ file: string; result: HyperframeLintResult }>,
|
||||
): HyperframeLintFinding[] {
|
||||
function lintMultipleRootCompositions(projectDir: string): HyperframeLintFinding[] {
|
||||
const findings: HyperframeLintFinding[] = [];
|
||||
const rootFiles = results.map((r) => r.file).filter((f) => !f.startsWith("compositions/"));
|
||||
|
||||
if (rootFiles.length > 1) {
|
||||
findings.push({
|
||||
code: "multiple_root_compositions",
|
||||
severity: "error",
|
||||
message: `Multiple root-level HTML files found: ${rootFiles.join(", ")}. The runtime may discover both as composition entry points, causing duplicate audio playback.`,
|
||||
fixHint:
|
||||
"A project should have exactly one root index.html. Remove or rename extra root-level HTML files.",
|
||||
});
|
||||
try {
|
||||
const rootHtmlFiles = readdirSync(projectDir).filter((f) => f.endsWith(".html"));
|
||||
const rootCompositions: string[] = [];
|
||||
for (const file of rootHtmlFiles) {
|
||||
const content = readFileSync(join(projectDir, file), "utf-8");
|
||||
if (/data-composition-id/i.test(content)) {
|
||||
rootCompositions.push(file);
|
||||
}
|
||||
}
|
||||
if (rootCompositions.length > 1) {
|
||||
findings.push({
|
||||
code: "multiple_root_compositions",
|
||||
severity: "error",
|
||||
message: `Multiple root-level HTML files with data-composition-id: ${rootCompositions.join(", ")}. The runtime may discover both as entry points, causing duplicate audio playback.`,
|
||||
fixHint:
|
||||
"A project should have exactly one root index.html with data-composition-id. Remove or rename extra files.",
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
/* directory read failed — skip */
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Warn if multiple <audio> elements on the same data-track-index overlap in time.
|
||||
* This causes layered audio playback.
|
||||
* Extracts each attribute independently (order-insensitive) to handle any HTML attribute order.
|
||||
* Deduplicates by (src, start, duration) to avoid flagging the same audio reached via sub-compositions.
|
||||
*/
|
||||
function lintDuplicateAudioTracks(htmlSources: string[]): HyperframeLintFinding[] {
|
||||
const findings: HyperframeLintFinding[] = [];
|
||||
const audioRe =
|
||||
/<audio\b[^>]*\bdata-track-index\s*=\s*["'](\d+)["'][^>]*\bdata-start\s*=\s*["']([^"']+)["'][^>]*\bdata-duration\s*=\s*["']([^"']+)["'][^>]*>/gi;
|
||||
function extractAttr(tag: string, name: string): string | null {
|
||||
const re = new RegExp(`\\b${name}\\s*=\\s*["']([^"']+)["']`, "i");
|
||||
const m = tag.match(re);
|
||||
return m?.[1] ?? null;
|
||||
}
|
||||
|
||||
const tracks: Array<{ trackIndex: number; start: number; end: number; src: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const html of htmlSources) {
|
||||
// Regex with g flag must be created inside the loop — a shared g-regex
|
||||
// carries lastIndex across strings, silently skipping matches.
|
||||
const audioTagRe = /<audio\b[^>]*>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = audioRe.exec(html)) !== null) {
|
||||
const trackIndex = parseInt(match[1]!, 10);
|
||||
const start = parseFloat(match[2]!);
|
||||
const duration = parseFloat(match[3]!);
|
||||
const srcMatch = match[0].match(/\bsrc\s*=\s*["']([^"']+)["']/);
|
||||
tracks.push({ trackIndex, start, end: start + duration, src: srcMatch?.[1] ?? "unknown" });
|
||||
while ((match = audioTagRe.exec(html)) !== null) {
|
||||
const tag = match[0];
|
||||
const trackStr = extractAttr(tag, "data-track-index");
|
||||
const startStr = extractAttr(tag, "data-start");
|
||||
const durStr = extractAttr(tag, "data-duration");
|
||||
const src = extractAttr(tag, "src") ?? "unknown";
|
||||
if (!trackStr || !startStr) continue;
|
||||
|
||||
const trackIndex = parseInt(trackStr, 10);
|
||||
const start = parseFloat(startStr);
|
||||
// Runtime falls back to Infinity when data-duration is absent (plays full track).
|
||||
// Mirror that here so audio without explicit duration still participates in overlap checks.
|
||||
const duration = durStr ? parseFloat(durStr) : Infinity;
|
||||
// Deduplicate: same audio reached from multiple HTML sources
|
||||
const key = `${src}:${start}:${duration}:${trackIndex}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
tracks.push({ trackIndex, start, end: start + duration, src });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,7 +239,7 @@ function lintDuplicateAudioTracks(htmlSources: string[]): HyperframeLintFinding[
|
||||
findings.push({
|
||||
code: "duplicate_audio_track",
|
||||
severity: "warning",
|
||||
message: `Multiple <audio> elements on track ${a.trackIndex} overlap (${a.src} at ${a.start}-${a.end.toFixed(1)}s, ${b.src} at ${b.start}-${b.end.toFixed(1)}s). This causes layered audio playback.`,
|
||||
message: `Multiple <audio> elements on track ${a.trackIndex} overlap (${a.src} at ${a.start}-${Number.isFinite(a.end) ? a.end.toFixed(1) : "end"}s, ${b.src} at ${b.start}-${Number.isFinite(b.end) ? b.end.toFixed(1) : "end"}s). This causes layered audio playback.`,
|
||||
fixHint: "Use non-overlapping time windows or different track indices.",
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user