Merge pull request #1447 from heygen-com/feat/cli-capture-video

feat(cli): capture-video on-demand fetcher + capture pipeline robustness
This commit is contained in:
Ular Kimsanov
2026-06-15 01:54:01 -07:00
committed by GitHub
10 changed files with 913 additions and 30 deletions
+33 -1
View File
@@ -25,6 +25,12 @@ export interface CatalogedAsset {
sectionClasses?: string;
/** Whether the image is above the fold (visible without scrolling) */
aboveFold?: boolean;
/** Element sits inside <header>, <nav>, or [role="banner"] — logo signal */
inBanner?: boolean;
/** Element sits inside <a> with site-root href ("/", "#", origin-only) — brand-home link */
inHomeLink?: boolean;
/** alt/aria-label/title contains the brand segment of document.title */
matchesTitleBrand?: boolean;
}
/**
@@ -62,6 +68,26 @@ export async function catalogAssets(page: Page): Promise<CatalogedAsset[]> {
var rect = el.getBoundingClientRect();
ctx.aboveFold = rect.top < window.innerHeight;
} catch(e) {}
// Structural logo-candidate signals: class-substring alone caught 0/32 SVGs on heygen.com.
ctx.inBanner = el.closest('header, nav, [role="banner"]') !== null;
var homeAnchor = el.closest('a[href]');
if (homeAnchor) {
var aHref = homeAnchor.getAttribute('href') || '';
ctx.inHomeLink = aHref === '/' || aHref === '#' || aHref === './' ||
/^https?:\\/\\/[^/]+\\/?$/.test(aHref);
}
// Brand can be first ("HeyGen - Ideas"), last ("Ideas - HeyGen"), or colon-separated ("Vercel: Build").
var titleParts = (document.title || '').split(/[-|—:]/);
if (desc) {
for (var ti = 0; ti < titleParts.length; ti++) {
var part = titleParts[ti].trim();
if (part.length > 1 && part.length < 30 &&
desc.toLowerCase().indexOf(part.toLowerCase()) !== -1) {
ctx.matchesTitleBrand = true;
break;
}
}
}
return ctx;
}
@@ -92,12 +118,15 @@ export async function catalogAssets(page: Page): Promise<CatalogedAsset[]> {
if (notes && !entry.notes) {
entry.notes = notes;
}
// Merge rich context (first one wins)
// Text fields: first-occurrence wins. Boolean signals: any positive sample wins.
if (richCtx) {
if (richCtx.description && !entry.description) entry.description = richCtx.description;
if (richCtx.nearestHeading && !entry.nearestHeading) entry.nearestHeading = richCtx.nearestHeading;
if (richCtx.sectionClasses && !entry.sectionClasses) entry.sectionClasses = richCtx.sectionClasses;
if (richCtx.aboveFold !== undefined && entry.aboveFold === undefined) entry.aboveFold = richCtx.aboveFold;
if (richCtx.inBanner) entry.inBanner = true;
if (richCtx.inHomeLink) entry.inHomeLink = true;
if (richCtx.matchesTitleBrand) entry.matchesTitleBrand = true;
}
}
@@ -324,6 +353,9 @@ function deduplicateSrcsetVariants(assets: CatalogedAsset[]): CatalogedAsset[] {
if (a.notes && !existing.notes) {
existing.notes = a.notes;
}
if (a.inBanner) existing.inBanner = true;
if (a.inHomeLink) existing.inHomeLink = true;
if (a.matchesTitleBrand) existing.matchesTitleBrand = true;
// Keep the URL with highest w= value (largest image)
const existingW = getWidthParam(existing.url);
const newW = getWidthParam(a.url);
+25 -6
View File
@@ -7,9 +7,16 @@
import { writeFileSync, mkdirSync } from "node:fs";
import { join, extname } from "node:path";
import { createHash } from "node:crypto";
import type { DesignTokens, DownloadedAsset } from "./types.js";
import type { CatalogedAsset } from "./assetCataloger.js";
// SVGs: hash-of-bytes filename so it can't drift from content; label-derived names mis-assigned brands.
function svgContentHashSlug(svgSource: string | Buffer, isLogo: boolean): string {
const hash = createHash("sha1").update(svgSource).digest("hex").slice(0, 8);
return isLogo ? `logo-${hash}` : `svg-${hash}`;
}
export async function downloadAssets(
tokens: DesignTokens,
outputDir: string,
@@ -22,15 +29,12 @@ export async function downloadAssets(
const assets: DownloadedAsset[] = [];
const downloadedUrls = new Set<string>();
// 1. ALL inline SVGs — save as files (logos get priority naming)
mkdirSync(join(outputDir, "assets", "svgs"), { recursive: true });
const usedSvgNames = new Set<string>();
for (let i = 0; i < tokens.svgs.length && i < 30; i++) {
const svg = tokens.svgs[i]!;
if (!svg.outerHTML || svg.outerHTML.length < 50) continue;
const label = svg.label?.replace(/[^a-zA-Z0-9-_ ]/g, "").trim();
let slug = label ? slugify(label) : svg.isLogo ? `logo-${i}` : `icon-${i}`;
// Deduplicate — two SVGs with same aria-label get suffixed
const slug = svgContentHashSlug(svg.outerHTML, !!svg.isLogo);
let finalSlug = slug;
let suffix = 2;
while (usedSvgNames.has(finalSlug)) {
@@ -135,8 +139,23 @@ export async function downloadAssets(
if (result.status !== "fulfilled" || !result.value) continue;
const { url, isPoster, parsedUrl, ext, buffer, catalog } = result.value;
try {
// Generate human-readable name from catalog context
const slug = deriveAssetName(parsedUrl, catalog, isPoster, imgIdx, usedNames);
let slug: string;
if (ext === ".svg") {
const c = catalog;
const brandRe = /logo|brand|wordmark/i;
const isLogo = !!(
c?.inBanner ||
c?.inHomeLink ||
c?.matchesTitleBrand ||
c?.contexts?.some((s) => brandRe.test(s)) ||
(c?.description && brandRe.test(c.description)) ||
(c?.nearestHeading && brandRe.test(c.nearestHeading)) ||
(c?.sectionClasses && brandRe.test(c.sectionClasses))
);
slug = svgContentHashSlug(buffer, isLogo);
} else {
slug = deriveAssetName(parsedUrl, catalog, isPoster, imgIdx, usedNames);
}
const name = `${slug}${ext}`;
usedNames.add(slug);
const localPath = `assets/${name}`;
+52 -17
View File
@@ -11,6 +11,7 @@
import type { Page } from "puppeteer-core";
import { existsSync, readdirSync, statSync, readFileSync } from "node:fs";
import { join } from "node:path";
import sharp from "sharp";
import type { CatalogedAsset } from "./assetCataloger.js";
import type { DesignTokens } from "./types.js";
@@ -232,7 +233,7 @@ export async function captionImagesWithGemini(
}
progress("design", `${Object.keys(geminiCaptions).length} images captioned with Gemini`);
// Caption SVGs by sending source code as text (vision API rejects image/svg+xml).
// Rasterize SVGs to PNG before captioning — Vision hallucinates wordmarks when reading SVG path text.
const svgFiles: Array<{ file: string; relPath: string }> = [];
const assetsDir = join(outputDir, "assets");
for (const f of readdirSync(assetsDir)) {
@@ -246,17 +247,45 @@ export async function captionImagesWithGemini(
}
if (svgFiles.length > 0) {
progress("design", `Captioning ${svgFiles.length} SVGs via code analysis...`);
progress("design", `Rasterizing + captioning ${svgFiles.length} SVGs via vision API...`);
const SVG_BATCH = 20;
const MAX_SVG_CHARS = 10_000;
const SVG_RENDER_SIZE = 256; // px — enough resolution for Gemini to read wordmarks, small enough to keep payload sub-MB
let svgsSkipped = 0;
for (let i = 0; i < svgFiles.length; i += SVG_BATCH) {
const batch = svgFiles.slice(i, i + SVG_BATCH);
const results = await Promise.allSettled(
batch.map(async ({ relPath }) => {
const filePath = join(assetsDir, relPath);
let svgText = readFileSync(filePath, "utf-8");
if (svgText.length > MAX_SVG_CHARS) {
svgText = svgText.slice(0, MAX_SVG_CHARS) + "\n<!-- truncated -->";
let pngBase64: string;
try {
// Flatten against a contrasting background — white-on-white SVGs render invisible to Vision.
const svgSource = readFileSync(filePath, "utf-8");
const lightFillHits = (
svgSource.match(/fill\s*=\s*["'](#fff(fff)?|white|#[ef][ef][ef]|#[ef]{6})["']/gi) ||
[]
).length;
const darkFillHits = (
svgSource.match(/fill\s*=\s*["'](#000(000)?|black|#[0-3]{6}|#[0-3]{3})["']/gi) || []
).length;
const bg =
lightFillHits > darkFillHits
? { r: 32, g: 32, b: 32 } // dark slate behind light glyphs
: { r: 255, g: 255, b: 255 }; // white behind dark glyphs (default)
const pngBuffer = await sharp(filePath)
.resize({
width: SVG_RENDER_SIZE,
height: SVG_RENDER_SIZE,
fit: "inside",
withoutEnlargement: false,
})
.flatten({ background: bg })
.png()
.toBuffer();
pngBase64 = pngBuffer.toString("base64");
} catch {
// exotic SVG features may break sharp; skip caption rather than block
svgsSkipped++;
return { file: relPath, caption: "" };
}
const response = await ai.models.generateContent({
model,
@@ -264,12 +293,13 @@ export async function captionImagesWithGemini(
{
role: "user",
parts: [
{ inlineData: { mimeType: "image/png", data: pngBase64 } },
{
text:
"This SVG code is from a website. Describe what it renders in ONE short sentence " +
"for a video storyboard. Focus on: what shape/icon/illustration it is, its colors. " +
"Be factual.\n\n" +
svgText,
"Describe this SVG asset rendered from a website in ONE short sentence for a video storyboard. " +
"Focus on: what shape/icon/illustration/wordmark it is, its colors, any text it contains. " +
"If you see a wordmark, READ THE LETTERS LITERALLY — do not guess a brand from context. " +
"Be factual.",
},
],
},
@@ -293,6 +323,12 @@ export async function captionImagesWithGemini(
);
}
progress("design", `${Object.keys(geminiCaptions).length} total assets captioned`);
if (svgsSkipped > 0) {
progress(
"design",
`skipped rasterizing ${svgsSkipped} SVG(s) — fell back to label-derived`,
);
}
}
} catch (err) {
warnings.push(`Gemini captioning failed: ${err}`);
@@ -358,11 +394,6 @@ export function generateAssetDescriptions(
const svgsPath = join(assetsPath, "svgs");
for (const file of readdirSync(svgsPath)) {
if (!file.endsWith(".svg")) continue;
const geminiCaption = geminiCaptions[`svgs/${file}`];
if (geminiCaption) {
svgLines.push(`svgs/${file}${geminiCaption}`);
continue;
}
const svgMatch = tokens.svgs.find(
(s) =>
s.label &&
@@ -373,9 +404,13 @@ export function generateAssetDescriptions(
.slice(0, 15),
),
);
const geminiCaption = geminiCaptions[`svgs/${file}`];
if (geminiCaption) {
svgLines.push(`svgs/${file}${geminiCaption}`);
continue;
}
const label = svgMatch?.label || file.replace(".svg", "").replace(/-/g, " ");
const isLogo = svgMatch?.isLogo || file.includes("logo");
svgLines.push(`svgs/${file}${isLogo ? "logo: " : "icon: "}${label}`);
svgLines.push(`svgs/${file}${label}`);
}
} catch {
/* no svgs dir */
+9 -4
View File
@@ -579,14 +579,19 @@ export async function captureWebsite(
const lines = generateAssetDescriptions(outputDir, tokens, catalogedAssets, geminiCaptions);
if (lines.length > 0) {
const hasGeminiKey = !!(process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY);
const header = hasGeminiKey
? "# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually.\n\nTo find a specific brand or icon, **grep this file for the brand name in the description text** (e.g. `grep -i 'autodesk' asset-descriptions.md`). The Gemini Vision captions identify what's actually in each file — that's the agent's selector.\n\nThe `logo-<hash>.svg` filename prefix is a cheap structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). It is NOT a content claim — many `logo-*` files are nav icons or decorative shapes. Trust the captions, not the filename prefix.\n\n"
: "# Asset Descriptions\n\n⚠️ GEMINI_API_KEY not set — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY) and re-run.\n\nThe `logo-<hash>.svg` filename prefix is a structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). To pick the actual brand logo without Vision, open the `logo-*` candidates in a previewer or rasterize them with `sharp` before referencing — composing a fake logo ships off-brand in the final video.\n\n";
writeFileSync(
join(outputDir, "extracted", "asset-descriptions.md"),
"# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually.\n\n" +
lines.map((l) => "- " + l).join("\n") +
"\n",
header + lines.map((l) => "- " + l).join("\n") + "\n",
"utf-8",
);
progress("design", `${lines.length} asset descriptions written`);
progress(
"design",
`${lines.length} asset descriptions written${hasGeminiKey ? "" : " (no Gemini key — catalog-fallback mode)"}`,
);
}
} catch {
/* non-critical */
+28 -1
View File
@@ -353,6 +353,33 @@ const EXTRACT_SCRIPT = `(() => {
// Keep SVGs that have a label OR are at least 16px wide OR are inside a logo/brand context
var inLogoContext = svg.closest('[class*="logo"], [class*="brand"], [class*="partner"], [class*="customer"], [class*="marquee"]') !== null;
if (!label && !inLogoContext && (!w || parseInt(w) < 16)) return null;
var isLogo = (label && label.toLowerCase().indexOf("logo") !== -1) ||
svg.closest('[class*="logo"], [class*="brand"], [class*="home"], [class*="marquee"], [class*="partner"], [class*="customer"]') !== null;
if (!isLogo) {
var bannerEl = svg.closest('header, nav, [role="banner"]');
if (bannerEl) {
var firstSvg = bannerEl.querySelector('svg');
if (firstSvg === svg) isLogo = true;
}
}
if (!isLogo) {
var anchor = svg.closest('a[href]');
if (anchor) {
var href = anchor.getAttribute('href') || '';
if (href === '/' || href === '#' || href === './' ||
/^https?:\\/\\/[^/]+\\/?$/.test(href)) {
isLogo = true;
}
}
}
if (!isLogo) {
var ariaLabel = svg.getAttribute('aria-label') || svg.getAttribute('title') || '';
var titleBrand = (document.title || '').split(/[-|—]/)[0].trim();
if (titleBrand.length > 1 && titleBrand.length < 30 &&
ariaLabel.toLowerCase().indexOf(titleBrand.toLowerCase()) !== -1) {
isLogo = true;
}
}
var rect = svg.getBoundingClientRect();
return {
label: label || undefined,
@@ -360,7 +387,7 @@ const EXTRACT_SCRIPT = `(() => {
width: Math.round(rect.width),
height: Math.round(rect.height),
outerHTML: svg.outerHTML.slice(0, 10000),
isLogo: (label && label.toLowerCase().indexOf("logo") !== -1) || svg.closest('[class*="logo"], [class*="brand"], [class*="home"], [class*="marquee"], [class*="partner"], [class*="customer"]') !== null
isLogo: isLogo
};
}).filter(Boolean).slice(0, 50);
+10 -1
View File
@@ -6,6 +6,10 @@ export const examples: Example[] = [
["Capture a website", "hyperframes capture https://stripe.com"],
["Capture to a specific directory", "hyperframes capture https://linear.app -o linear-video"],
["JSON output for AI agents", "hyperframes capture https://example.com --json"],
[
"Pull a video from the captured manifest by index",
"hyperframes capture video ./linear-video --index 0",
],
];
export default defineCommand({
@@ -13,6 +17,9 @@ export default defineCommand({
name: "capture",
description: "Capture a website as editable HyperFrames components",
},
subCommands: {
video: () => import("./capture/video.js").then((m) => m.default),
},
args: {
url: {
type: "positional",
@@ -46,7 +53,9 @@ export default defineCommand({
async run({ args }) {
const url = args.url as string;
// Validate URL
// citty fires parent's run AFTER routing to a subcommand; skip when args.url is a subcommand name.
if (url === "video") return;
try {
new URL(url);
} catch {
@@ -0,0 +1,162 @@
import { describe, expect, it } from "vitest";
import {
MAX_VIDEO_BYTES,
VIDEO_CONTENT_TYPE_RE,
findFilenameCollision,
pickManifestEntry,
safeFilename,
type ManifestEntry,
} from "./video.js";
const ENTRY = (index: number, partial: Partial<ManifestEntry> = {}): ManifestEntry => ({
index,
url: `https://cdn.example.com/video-${index}.mp4`,
filename: `video-${index}.mp4`,
width: 1920,
height: 1080,
heading: "",
caption: "",
ariaLabel: "",
preview: `assets/videos/previews/video-${index}-preview.png`,
...partial,
});
describe("safeFilename", () => {
it("decodes percent-encoded chars", () => {
expect(safeFilename("Frame-2147227325%20(1).mp4")).toBe("Frame-2147227325_1_.mp4");
});
it("strips characters outside [A-Za-z0-9._-]", () => {
expect(safeFilename("video with spaces & symbols!.mp4")).toBe("video_with_spaces_symbols_.mp4");
});
it("preserves the extension and version markers", () => {
expect(safeFilename("hero.mp4")).toBe("hero.mp4");
expect(safeFilename("hero-v2.webm")).toBe("hero-v2.webm");
});
it("falls back when decodeURIComponent throws on a malformed sequence", () => {
// `%E0%A4%A` is a truncated UTF-8 multibyte sequence and throws URIError.
// We should keep the raw input rather than crashing.
expect(safeFilename("Bad%E0%A4%A.mp4")).toBe("Bad_E0_A4_A.mp4");
});
it("collapses runs of disallowed characters into a single underscore", () => {
expect(safeFilename("a b___c")).toBe("a_b___c");
});
});
describe("VIDEO_CONTENT_TYPE_RE", () => {
it("matches common video content-types", () => {
expect(VIDEO_CONTENT_TYPE_RE.test("video/mp4")).toBe(true);
expect(VIDEO_CONTENT_TYPE_RE.test("video/webm")).toBe(true);
expect(VIDEO_CONTENT_TYPE_RE.test("video/quicktime")).toBe(true);
});
it("matches application/* containers that CDNs commonly use", () => {
expect(VIDEO_CONTENT_TYPE_RE.test("application/mp4")).toBe(true);
expect(VIDEO_CONTENT_TYPE_RE.test("application/octet-stream")).toBe(true);
expect(VIDEO_CONTENT_TYPE_RE.test("application/x-mpegURL")).toBe(true);
});
it("rejects HTML / JSON error pages that pretend to be videos", () => {
expect(VIDEO_CONTENT_TYPE_RE.test("text/html")).toBe(false);
expect(VIDEO_CONTENT_TYPE_RE.test("application/json")).toBe(false);
expect(VIDEO_CONTENT_TYPE_RE.test("image/png")).toBe(false);
});
});
describe("MAX_VIDEO_BYTES", () => {
it("is 250 MB", () => {
expect(MAX_VIDEO_BYTES).toBe(250 * 1024 * 1024);
});
});
describe("pickManifestEntry", () => {
it("returns no-selector when neither --index nor --url is given", () => {
const r = pickManifestEntry([ENTRY(0)], {});
expect(r.ok).toBe(false);
if (!r.ok) expect(r.code).toBe("no-selector");
});
it("looks up by the entry's `index` field, NOT array offset (manifest gaps)", () => {
const manifest = [ENTRY(0), ENTRY(2), ENTRY(3)]; // index 1 missing
const r = pickManifestEntry(manifest, { index: 3 });
expect(r.ok).toBe(true);
if (r.ok) expect(r.entry.url).toBe("https://cdn.example.com/video-3.mp4");
});
it("rejects a request for an index that's not in the manifest", () => {
const manifest = [ENTRY(0), ENTRY(2)]; // index 1 missing
const r = pickManifestEntry(manifest, { index: 1 });
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.code).toBe("no-match-index");
expect(r.message).toContain("index=1");
expect(r.message).toContain("available: 0, 2");
}
});
it("rejects a negative or non-integer index up front", () => {
expect(pickManifestEntry([ENTRY(0)], { index: -1 }).ok).toBe(false);
expect(pickManifestEntry([ENTRY(0)], { index: 1.5 }).ok).toBe(false);
expect(pickManifestEntry([ENTRY(0)], { index: "abc" }).ok).toBe(false);
});
it("accepts numeric-string indices (citty parses positional args as strings)", () => {
const r = pickManifestEntry([ENTRY(0), ENTRY(1)], { index: "1" });
expect(r.ok).toBe(true);
if (r.ok) expect(r.entry.index).toBe(1);
});
it("looks up by exact URL match", () => {
const manifest = [ENTRY(0), ENTRY(1)];
const r = pickManifestEntry(manifest, { url: "https://cdn.example.com/video-1.mp4" });
expect(r.ok).toBe(true);
if (r.ok) expect(r.entry.index).toBe(1);
});
it("rejects a URL that doesn't appear in the manifest", () => {
const manifest = [ENTRY(0)];
const r = pickManifestEntry(manifest, { url: "https://other.com/missing.mp4" });
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.code).toBe("no-match-url");
expect(r.message).toContain("missing.mp4");
}
});
it("when both --index and --url are passed, --index wins (CLI's declared priority)", () => {
const manifest = [ENTRY(0), ENTRY(1)];
const r = pickManifestEntry(manifest, {
index: 1,
url: "https://cdn.example.com/video-0.mp4",
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.entry.index).toBe(1);
});
});
describe("findFilenameCollision", () => {
it("returns [] when no other manifest entry produces the same safeFilename", () => {
const manifest = [ENTRY(0), ENTRY(1), ENTRY(2)];
expect(findFilenameCollision(manifest, manifest[1]!)).toEqual([]);
});
it("flags collisions when two URL forms collapse to the same safeFilename", () => {
// "hero%20clip.mp4" and "hero clip.mp4" both → "hero_clip.mp4"
const manifest = [
ENTRY(0, { url: "https://cdn.example.com/hero%20clip.mp4", filename: "hero%20clip.mp4" }),
ENTRY(1, { url: "https://cdn.example.com/hero clip.mp4", filename: "hero clip.mp4" }),
];
const collisions = findFilenameCollision(manifest, manifest[0]!);
expect(collisions).toHaveLength(1);
expect(collisions[0]?.index).toBe(1);
});
it("does not return the selected entry itself as a collision", () => {
const manifest = [ENTRY(0), ENTRY(1)];
const collisions = findFilenameCollision(manifest, manifest[0]!);
expect(collisions.every((c) => c.index !== 0)).toBe(true);
});
});
+311
View File
@@ -0,0 +1,311 @@
import { defineCommand } from "citty";
import { createWriteStream, existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs";
import { resolve, join, basename } from "node:path";
import { c } from "../../ui/colors.js";
import { safeFetch } from "../../capture/assetDownloader.js";
import type { Example } from "../_examples.js";
export const examples: Example[] = [
[
"Download the hero video (index 0) from a captured project's manifest",
"capture video ./my-project --index 0",
],
[
"Download a specific video by exact URL",
"capture video ./my-project --url https://cdn.example.com/hero.mp4",
],
["List entries in the manifest without downloading", "capture video ./my-project --list"],
];
const MAX_VIDEO_BYTES = 250 * 1024 * 1024;
const VIDEO_CONTENT_TYPE_RE = /^(video\/|application\/(mp4|octet-stream|x-mpegurl))/i;
// fallow-ignore-next-line complexity
async function streamToFile(url: string, destPath: string): Promise<number> {
// safeFetch re-validates redirect hops; bare redirect:"follow" leaks to private hosts.
const r = await safeFetch(url, {
signal: AbortSignal.timeout(120_000),
headers: { "User-Agent": "HyperFrames/1.0" },
});
if (!r) {
throw new Error(
`fetch blocked or failed (private/metadata host, redirect chain, or network error): ${url}`,
);
}
if (!r.ok) throw new Error(`HTTP ${r.status} ${r.statusText} for ${url}`);
const ct = r.headers.get("content-type") || "";
if (!VIDEO_CONTENT_TYPE_RE.test(ct)) {
throw new Error(
`unexpected content-type "${ct}" for ${url} — expected video/*. The URL probably doesn't point at a real video file.`,
);
}
const cl = r.headers.get("content-length");
if (cl && Number(cl) > MAX_VIDEO_BYTES) {
throw new Error(
`video too large (${Math.round(Number(cl) / 1024 / 1024)}MB > ${Math.round(MAX_VIDEO_BYTES / 1024 / 1024)}MB cap) for ${url}`,
);
}
if (!r.body) throw new Error(`empty response body for ${url}`);
// `flags: "wx"` = exclusive-create; throws EEXIST if destPath exists. Stream chunks
// and abort mid-transfer if cumulative bytes exceed the cap so a hostile CDN can't
// OOM the process by lying about content-length.
const file = createWriteStream(destPath, { flags: "wx" });
// Single shared error promise: avoids re-attaching `error` listeners per chunk (MaxListeners warning).
let streamError: Error | null = null;
const streamErrored = new Promise<never>((_, reject) => {
file.once("error", (e) => {
streamError = e;
reject(e);
});
});
let bytes = 0;
try {
await Promise.race([
streamErrored,
new Promise<void>((resolveOpen) => file.once("open", () => resolveOpen())),
]);
for await (const chunk of r.body as unknown as AsyncIterable<Uint8Array>) {
if (streamError) throw streamError;
bytes += chunk.byteLength;
if (bytes > MAX_VIDEO_BYTES) {
throw new Error(
`video exceeded ${Math.round(MAX_VIDEO_BYTES / 1024 / 1024)}MB cap mid-stream for ${url}`,
);
}
// lgtm[js/http-to-file-access] — manifest-vetted URL, content-type whitelist, 250MB cap with mid-stream abort, SSRF-safe fetch
if (!file.write(chunk)) {
await Promise.race([
streamErrored,
new Promise<void>((resolveDrain) => file.once("drain", () => resolveDrain())),
]);
}
}
await new Promise<void>((resolveEnd, rejectEnd) => {
file.end((err?: Error | null) => (err ? rejectEnd(err) : resolveEnd()));
});
return bytes;
} catch (e) {
file.destroy();
// EEXIST means destPath ALREADY existed before we wrote anything — leave it alone.
// Any other error means we created a partial file that the caller should not see.
if ((e as NodeJS.ErrnoException).code !== "EEXIST") {
try {
unlinkSync(destPath);
} catch {
/* partial file may not exist */
}
}
throw e;
}
}
export function safeFilename(name: string): string {
let decoded = name;
try {
decoded = decodeURIComponent(name);
} catch {
/* malformed percent-encoding */
}
return decoded.replace(/[^A-Za-z0-9._-]+/g, "_");
}
export { VIDEO_CONTENT_TYPE_RE, MAX_VIDEO_BYTES };
export interface ManifestEntry {
index: number;
url: string;
filename: string;
width: number;
height: number;
heading: string;
caption: string;
ariaLabel: string;
preview: string;
}
export type PickResult =
| { ok: true; entry: ManifestEntry }
| {
ok: false;
code: "no-selector" | "bad-index" | "no-match-index" | "no-match-url";
message: string;
};
// Two manifest entries can produce the same safeFilename (URL-encoded variants of the same name
// collapse after decode). The wx exclusive-create + EEXIST handler would silently misreport the
// second as "already downloaded" while serving the first's bytes. Fail loudly instead.
export function findFilenameCollision(
manifest: ManifestEntry[],
selected: ManifestEntry,
): ManifestEntry[] {
const selectedName = safeFilename(selected.filename || basename(selected.url));
return manifest.filter(
(e) =>
e.index !== selected.index && safeFilename(e.filename || basename(e.url)) === selectedName,
);
}
// Looks up by `entry.index`, not array offset — captureVideoManifest leaves gaps when previews fail.
export function pickManifestEntry(
manifest: ManifestEntry[],
args: { index?: string | number | null; url?: string | null },
): PickResult {
if (args.index != null) {
const i = Number(args.index);
if (!Number.isInteger(i) || i < 0) {
return {
ok: false,
code: "bad-index",
message: `--index ${args.index} must be a non-negative integer`,
};
}
const found = manifest.find((e) => e.index === i);
if (!found) {
const available = manifest.map((e) => e.index).join(", ");
return {
ok: false,
code: "no-match-index",
message: `no manifest entry with index=${i} (available: ${available || "none"})`,
};
}
return { ok: true, entry: found };
}
if (args.url != null) {
const found = manifest.find((e) => e.url === args.url);
if (!found) {
return { ok: false, code: "no-match-url", message: `no manifest entry with url=${args.url}` };
}
return { ok: true, entry: found };
}
return {
ok: false,
code: "no-selector",
message: "specify --index <N> or --url <URL> (or --list to see what's in the manifest)",
};
}
export default defineCommand({
meta: {
name: "video",
description:
"Download a video referenced in capture/extracted/video-manifest.json (on-demand; the capture pipeline only writes the manifest + preview PNGs)",
},
args: {
project: {
type: "positional",
description: "Path to the captured project directory",
required: true,
},
index: {
type: "string",
description: "Manifest entry index to download (0-based)",
},
url: {
type: "string",
description: "Exact video URL to download (must match a manifest entry)",
},
list: {
type: "boolean",
description: "List manifest entries (index, dimensions, heading) and exit",
},
},
// fallow-ignore-next-line complexity
async run({ args }) {
const projectDir = resolve(String(args.project));
// standalone capture writes `<dir>/extracted/…`; W2H project nests under `<dir>/capture/extracted/…`.
const directPath = join(projectDir, "extracted", "video-manifest.json");
const w2hPath = join(projectDir, "capture", "extracted", "video-manifest.json");
const manifestPath = existsSync(directPath) ? directPath : w2hPath;
const isW2hLayout = manifestPath === w2hPath;
if (!existsSync(manifestPath)) {
console.error(
`${c.error("✗")} no video-manifest.json at ${directPath} or ${w2hPath}\n` +
` Was this directory produced by \`hyperframes capture\`?`,
);
process.exitCode = 1;
return;
}
let manifest: ManifestEntry[];
try {
manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
} catch (e) {
console.error(`${c.error("✗")} video-manifest.json is malformed: ${(e as Error).message}`);
process.exitCode = 1;
return;
}
if (args.list) {
if (manifest.length === 0) {
console.log(c.dim("(manifest is empty — no <video> elements on the captured page)"));
return;
}
console.log(
`${manifest.length} video entr${manifest.length === 1 ? "y" : "ies"} in ${manifestPath}:`,
);
for (const e of manifest) {
console.log(
` ${c.bold(`[${e.index}]`)} ${e.filename}${e.width}×${e.height}` +
(e.heading ? `\n heading: "${e.heading}"` : "") +
`\n url: ${e.url}`,
);
}
return;
}
const pick = pickManifestEntry(manifest, args);
if (!pick.ok) {
console.error(
`${c.error("✗")} ${pick.message}` +
(pick.code === "no-match-url" ? `\n Run with --list to see what's available.` : ""),
);
process.exitCode = 1;
return;
}
const entry = pick.entry;
const collisions = findFilenameCollision(manifest, entry);
if (collisions.length > 0) {
console.error(
`${c.error("✗")} filename "${safeFilename(entry.filename || basename(entry.url))}" ` +
`collides with manifest entr${collisions.length === 1 ? "y" : "ies"} ` +
`${collisions.map((co) => `[${co.index}]`).join(", ")}. ` +
`Refusing to download — the on-disk file's bytes would not match the requested entry.`,
);
process.exitCode = 1;
return;
}
const outDir = isW2hLayout
? join(projectDir, "capture", "assets", "videos")
: join(projectDir, "assets", "videos");
mkdirSync(outDir, { recursive: true });
const fname = safeFilename(entry.filename || basename(entry.url));
const outPath = join(outDir, fname);
const relPath = isW2hLayout ? `capture/assets/videos/${fname}` : `assets/videos/${fname}`;
console.log(
`${c.accent("▸")} downloading [${entry.index}] ${entry.filename} (${entry.width}×${entry.height})`,
);
console.log(` from: ${entry.url}`);
try {
const bytes = await streamToFile(entry.url, outPath);
const sizeKb = Math.round(bytes / 1024);
const sizeStr = sizeKb > 1024 ? `${(sizeKb / 1024).toFixed(1)}MB` : `${sizeKb}KB`;
console.log(`${c.success("◇")} wrote ${relPath} (${sizeStr})`);
const snippetId = `video-${entry.index}`;
console.log(
` Reference it from a beat composition as:\n` +
` <video id="${snippetId}" src="${relPath}" data-start="0" data-duration="${entry.width === entry.height ? 5 : 4}" data-track-index="0" autoplay muted loop></video>`,
);
} catch (e) {
if ((e as NodeJS.ErrnoException).code === "EEXIST") {
console.log(`${c.warn("⚠")} already downloaded: ${relPath} (skipping)`);
console.log(` Delete the file and re-run to refetch.`);
return;
}
console.error(`${c.error("✗")} download failed: ${(e as Error).message}`);
process.exitCode = 1;
}
},
});
+209
View File
@@ -475,6 +475,215 @@ describe("audio_src_not_found", () => {
});
});
describe("missing_local_asset", () => {
it("errors when <img> src references a file that does not exist", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<img src="capture/assets/hero.png" />
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { totalErrors, results } = await lintProject(project);
expect(totalErrors).toBeGreaterThan(0);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeDefined();
expect(finding?.severity).toBe("error");
expect(finding?.message).toContain("hero.png");
expect(finding?.message).toContain("<img>");
});
it("errors when <video> src references a missing file", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<video id="hero" src="capture/assets/videos/clip.mp4" muted playsinline></video>
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { totalErrors, results } = await lintProject(project);
expect(totalErrors).toBeGreaterThan(0);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeDefined();
expect(finding?.message).toContain("clip.mp4");
expect(finding?.message).toContain("<video>");
});
it("errors when <source> src inside <video> references a missing file", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<video muted playsinline><source src="capture/assets/videos/clip.webm" /></video>
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { totalErrors, results } = await lintProject(project);
expect(totalErrors).toBeGreaterThan(0);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeDefined();
expect(finding?.message).toContain("clip.webm");
expect(finding?.message).toContain("<source>");
});
it("does NOT report <audio> srcs (handled by audio_src_not_found)", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<audio id="vo" src="missing.mp3" data-start="0" data-duration="3" data-track-index="0" data-volume="1"></audio>
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { results } = await lintProject(project);
const localAsset = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
const audio = results[0]?.result.findings.find((f) => f.code === "audio_src_not_found");
expect(localAsset).toBeUndefined();
expect(audio).toBeDefined();
});
it("does NOT report remote URLs (https:, data:, blob:)", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<img src="https://example.com/x.png" />
<img src="data:image/png;base64,iVBOR" />
<img src="blob:foo" />
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { results } = await lintProject(project);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeUndefined();
});
it("does NOT report template placeholders (__VIDEO_SRC__)", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<video src="__VIDEO_SRC__"></video>
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { results } = await lintProject(project);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeUndefined();
});
it("does not error when referenced files exist on disk", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<img src="hero.png" />
<video src="clip.mp4"></video>
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
writeFileSync(join(project.dir, "hero.png"), "fake");
writeFileSync(join(project.dir, "clip.mp4"), "fake");
const { results } = await lintProject(project);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeUndefined();
});
it("resolves sub-composition relative paths (../assets/foo.png)", async () => {
const subComp = `<html><body>
<div data-composition-id="scene" data-width="1920" data-height="1080">
<img src="../assets/foo.png" />
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["scene"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(validHtml(), { "scene.html": subComp });
mkdirSync(join(project.dir, "assets"), { recursive: true });
writeFileSync(join(project.dir, "assets", "foo.png"), "fake");
const { results } = await lintProject(project);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeUndefined();
});
it("deduplicates the same missing src across multiple compositions", async () => {
const project = makeProject(
`<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<img src="capture/assets/x.png" />
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`,
{
"scene-a.html": `<html><body>
<div data-composition-id="a" data-width="1920" data-height="1080">
<img src="../capture/assets/x.png" />
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["a"] = gsap.timeline({ paused: true });</script>
</body></html>`,
"scene-b.html": `<html><body>
<div data-composition-id="b" data-width="1920" data-height="1080">
<img src="../capture/assets/x.png" />
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["b"] = gsap.timeline({ paused: true });</script>
</body></html>`,
},
);
const { results } = await lintProject(project);
const finding = results[0]?.result.findings.find((f) => f.code === "missing_local_asset");
expect(finding).toBeDefined();
// x.png mentioned only once despite three references
const occurrences = (finding?.message.match(/x\.png/g) ?? []).length;
expect(occurrences).toBe(1);
});
it("emits separate findings per tag type (img + video) for clear messaging", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<img src="missing.png" />
<video src="missing.mp4"></video>
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { results } = await lintProject(project);
const findings = results[0]?.result.findings.filter((f) => f.code === "missing_local_asset");
expect(findings).toHaveLength(2);
expect(findings?.some((f) => f.message.includes("<img>"))).toBe(true);
expect(findings?.some((f) => f.message.includes("<video>"))).toBe(true);
});
it("does not flag <img>/<video> tokens inside <!-- -->, <style>, or <script>", async () => {
const html = `<html><body>
<div data-composition-id="main" data-width="1920" data-height="1080">
<!-- example: <img src="commented.png"> -->
<style>/* card uses <video src="styled.mp4"> as the surface */ .card { background: black; }</style>
<script>const example = '<source src="scripted.webm">';</script>
</div>
<script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
</body></html>`;
const project = makeProject(html);
const { results } = await lintProject(project);
const findings = results[0]?.result.findings.filter((f) => f.code === "missing_local_asset");
expect(findings).toEqual([]);
});
});
describe("texture_mask_asset_not_found", () => {
it("errors when CSS mask-image references a missing local texture", async () => {
const html = `<html><body>
+74
View File
@@ -223,6 +223,7 @@ export async function lintProject(project: ProjectDir): Promise<ProjectLintResul
const projectFindings = [
...lintProjectAudioFiles(project.dir, allHtmlSources),
...lintAudioSrcNotFound(project.dir, allHtmlSources),
...lintMissingLocalAsset(project.dir, allHtmlSources),
...lintTextureMaskAssetNotFound(project.dir, allHtmlSources),
...lintMultipleRootCompositions(project.dir),
...lintDuplicateAudioTracks(allHtmlSources),
@@ -337,6 +338,79 @@ function lintAudioSrcNotFound(
return findings;
}
// Same-length whitespace preserves offsets.
function maskRange(src: string, pattern: RegExp): string {
return src.replace(pattern, (m) => " ".repeat(m.length));
}
// Closing tags allow junk before `>` (`</script foo>` is valid HTML); use `[^>]*` to mask permissively.
function maskNonScannableRanges(html: string): string {
let out = maskRange(html, /<!--[\s\S]*?-->/g);
out = maskRange(out, /<style\b[^>]*>[\s\S]*?<\/style\b[^>]*>/gi);
out = maskRange(out, /<script\b[^>]*>[\s\S]*?<\/script\b[^>]*>/gi);
return out;
}
// <audio> is handled by lintAudioSrcNotFound — its "silent video" message is tailored.
// fallow-ignore-next-line complexity
function lintMissingLocalAsset(
projectDir: string,
htmlSources: HtmlSource[],
): HyperframeLintFinding[] {
const findings: HyperframeLintFinding[] = [];
const localAssetSrcRe = /<(video|img|source)\b[^>]*\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi;
// Dedup by resolved path: same missing file from root + sub-comp → ONE finding.
const missingByTag = new Map<string, Map<string, string>>();
for (const { html, compSrcPath } of htmlSources) {
const scannable = maskNonScannableRanges(html);
const re = new RegExp(localAssetSrcRe.source, localAssetSrcRe.flags);
let match: RegExpExecArray | null;
while ((match = re.exec(scannable)) !== null) {
const tagName = (match[1] ?? "").toLowerCase();
const rawSrc = match[2] ?? "";
const src = cleanAssetUrl(rawSrc);
if (!src) continue;
if (isRemoteOrInlineUrl(src)) continue;
if (/^__[A-Z_]+__$/.test(src)) continue; // template placeholder
const rootRelative = compSrcPath ? rewriteAssetPath(compSrcPath, src) : src;
// resolveExistingLocalAsset matches the bundler's notion of "resolves" (handles root-absolute, rejects escapes).
const resolvedAsset = resolveExistingLocalAsset(projectDir, rootRelative);
if (resolvedAsset) continue;
const resolvedKey = resolve(projectDir, rootRelative);
let bucket = missingByTag.get(tagName);
if (!bucket) {
bucket = new Map<string, string>();
missingByTag.set(tagName, bucket);
}
if (!bucket.has(resolvedKey)) bucket.set(resolvedKey, src);
}
}
for (const [tagName, byResolved] of missingByTag) {
const unique = [...byResolved.values()];
findings.push({
code: "missing_local_asset",
severity: "error",
message:
`<${tagName}> element references local file(s) not found in the project: ${unique.join(", ")}. ` +
"The renderer will silently skip these and produce a video with missing visuals.",
fixHint:
unique.length === 1
? `Add "${unique[0]}" to the project directory, or update the src attribute to point to an existing file. ` +
"Common cause: captured asset filenames are unreliable (heygen-logo.svg often contains Google, nvidia-logo.svg may contain Autodesk, etc.). " +
"Open the contact sheets and verify the file actually exists at this path before referencing it."
: "Add the missing files to the project directory, or update the src attributes to point to existing files. " +
"Captured asset filenames are unreliable — verify against capture/contact-sheets/ and capture/extracted/asset-descriptions.md.",
});
}
return findings;
}
function lintTextureMaskAssetNotFound(
projectDir: string,
htmlSources: HtmlSource[],