feat(cli): caption SVGs via Gemini text analysis in capture pipeline

The vision API rejects image/svg+xml, so SVGs were silently skipped during
Gemini captioning — leaving 97% of icons/logos undescribed. This caused
asset utilization as low as 3.8% on SVG-heavy sites (Huly, Raycast).

Now sends SVG source code as text to Gemini, which reads the XML structure
and describes what it renders. Truncates SVGs >10KB to keep token costs low.

Tested on Huly: 2/52 → 37/52 assets described (3.8% → 71%).
This commit is contained in:
ukimsanov
2026-05-10 20:13:01 -07:00
parent 3eca797b2a
commit acf11413a1
+64 -1
View File
@@ -9,7 +9,7 @@
*/
import type { Page } from "puppeteer-core";
import { readdirSync, statSync, readFileSync } from "node:fs";
import { existsSync, readdirSync, statSync, readFileSync } from "node:fs";
import { join } from "node:path";
import type { CatalogedAsset } from "./assetCataloger.js";
import type { DesignTokens } from "./types.js";
@@ -231,6 +231,69 @@ export async function captionImagesWithGemini(
);
}
progress("design", `${Object.keys(geminiCaptions).length} images captioned with Gemini`);
// Caption SVGs by sending source code as text (vision API rejects image/svg+xml).
const svgFiles: Array<{ file: string; relPath: string }> = [];
const assetsDir = join(outputDir, "assets");
for (const f of readdirSync(assetsDir)) {
if (/\.svg$/i.test(f)) svgFiles.push({ file: f, relPath: f });
}
const svgsSubdir = join(assetsDir, "svgs");
if (existsSync(svgsSubdir)) {
for (const f of readdirSync(svgsSubdir)) {
if (/\.svg$/i.test(f)) svgFiles.push({ file: f, relPath: `svgs/${f}` });
}
}
if (svgFiles.length > 0) {
progress("design", `Captioning ${svgFiles.length} SVGs via code analysis...`);
const SVG_BATCH = 20;
const MAX_SVG_CHARS = 10_000;
for (let i = 0; i < svgFiles.length; i += SVG_BATCH) {
const batch = svgFiles.slice(i, i + SVG_BATCH);
const results = await Promise.allSettled(
batch.map(async ({ relPath }) => {
const filePath = join(assetsDir, relPath);
let svgText = readFileSync(filePath, "utf-8");
if (svgText.length > MAX_SVG_CHARS) {
svgText = svgText.slice(0, MAX_SVG_CHARS) + "\n<!-- truncated -->";
}
const response = await ai.models.generateContent({
model,
contents: [
{
role: "user",
parts: [
{
text:
"This SVG code is from a website. Describe what it renders in ONE short sentence " +
"for a video storyboard. Focus on: what shape/icon/illustration it is, its colors. " +
"Be factual.\n\n" +
svgText,
},
],
},
],
config: { maxOutputTokens: 300 },
});
return { file: relPath, caption: response.text?.trim() || "" };
}),
);
for (const result of results) {
if (result.status === "fulfilled" && result.value.caption) {
geminiCaptions[result.value.file] = result.value.caption;
}
}
if (i + SVG_BATCH < svgFiles.length) {
await new Promise((r) => setTimeout(r, 2000));
}
progress(
"design",
`Captioned ${Math.min(i + SVG_BATCH, svgFiles.length)}/${svgFiles.length} SVGs...`,
);
}
progress("design", `${Object.keys(geminiCaptions).length} total assets captioned`);
}
} catch (err) {
warnings.push(`Gemini captioning failed: ${err}`);
}