mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 23:47:33 +00:00
feat(cli): caption SVGs via Gemini text analysis in capture pipeline
The vision API rejects image/svg+xml, so SVGs were silently skipped during Gemini captioning — leaving 97% of icons/logos undescribed. This caused asset utilization as low as 3.8% on SVG-heavy sites (Huly, Raycast). Now sends SVG source code as text to Gemini, which reads the XML structure and describes what it renders. Truncates SVGs >10KB to keep token costs low. Tested on Huly: 2/52 → 37/52 assets described (3.8% → 71%).
This commit is contained in:
@@ -9,7 +9,7 @@
|
||||
*/
|
||||
|
||||
import type { Page } from "puppeteer-core";
|
||||
import { readdirSync, statSync, readFileSync } from "node:fs";
|
||||
import { existsSync, readdirSync, statSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { CatalogedAsset } from "./assetCataloger.js";
|
||||
import type { DesignTokens } from "./types.js";
|
||||
@@ -231,6 +231,69 @@ export async function captionImagesWithGemini(
|
||||
);
|
||||
}
|
||||
progress("design", `${Object.keys(geminiCaptions).length} images captioned with Gemini`);
|
||||
|
||||
// Caption SVGs by sending source code as text (vision API rejects image/svg+xml).
|
||||
const svgFiles: Array<{ file: string; relPath: string }> = [];
|
||||
const assetsDir = join(outputDir, "assets");
|
||||
for (const f of readdirSync(assetsDir)) {
|
||||
if (/\.svg$/i.test(f)) svgFiles.push({ file: f, relPath: f });
|
||||
}
|
||||
const svgsSubdir = join(assetsDir, "svgs");
|
||||
if (existsSync(svgsSubdir)) {
|
||||
for (const f of readdirSync(svgsSubdir)) {
|
||||
if (/\.svg$/i.test(f)) svgFiles.push({ file: f, relPath: `svgs/${f}` });
|
||||
}
|
||||
}
|
||||
|
||||
if (svgFiles.length > 0) {
|
||||
progress("design", `Captioning ${svgFiles.length} SVGs via code analysis...`);
|
||||
const SVG_BATCH = 20;
|
||||
const MAX_SVG_CHARS = 10_000;
|
||||
for (let i = 0; i < svgFiles.length; i += SVG_BATCH) {
|
||||
const batch = svgFiles.slice(i, i + SVG_BATCH);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(async ({ relPath }) => {
|
||||
const filePath = join(assetsDir, relPath);
|
||||
let svgText = readFileSync(filePath, "utf-8");
|
||||
if (svgText.length > MAX_SVG_CHARS) {
|
||||
svgText = svgText.slice(0, MAX_SVG_CHARS) + "\n<!-- truncated -->";
|
||||
}
|
||||
const response = await ai.models.generateContent({
|
||||
model,
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
text:
|
||||
"This SVG code is from a website. Describe what it renders in ONE short sentence " +
|
||||
"for a video storyboard. Focus on: what shape/icon/illustration it is, its colors. " +
|
||||
"Be factual.\n\n" +
|
||||
svgText,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
config: { maxOutputTokens: 300 },
|
||||
});
|
||||
return { file: relPath, caption: response.text?.trim() || "" };
|
||||
}),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status === "fulfilled" && result.value.caption) {
|
||||
geminiCaptions[result.value.file] = result.value.caption;
|
||||
}
|
||||
}
|
||||
if (i + SVG_BATCH < svgFiles.length) {
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
progress(
|
||||
"design",
|
||||
`Captioned ${Math.min(i + SVG_BATCH, svgFiles.length)}/${svgFiles.length} SVGs...`,
|
||||
);
|
||||
}
|
||||
progress("design", `${Object.keys(geminiCaptions).length} total assets captioned`);
|
||||
}
|
||||
} catch (err) {
|
||||
warnings.push(`Gemini captioning failed: ${err}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user