mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
fix(capture): strengthen asset-descriptions.md fallback when Gemini key absent
User context: not every team has GEMINI_API_KEY set. Without it, the asset-descriptions.md fallback previously degraded to near-useless lines like `svgs/svg-54ea56cd.svg — icon: 54ea56cd` — just the content-hash slug, no signal for finding the real logo. This commit makes the no-Gemini path materially more useful by surfacing the new structural LOGO signals (committed in02f95f95) into the asset-descriptions output. contentExtractor.ts changes: 1. Downloaded-image branch (lines 350-393): compute `isLikelyLogo` from catalog signals (inBanner / inHomeLink / matchesTitleBrand + the existing alt/heading/section substring patterns). When true, prepend a literal `LOGO` token to the description line. Applied to BOTH the Gemini-captioned path (so the LOGO tag rides alongside the rich Vision caption) and the catalog-fallback path. Single grep target for agents: `grep "LOGO" capture/extracted/asset-descriptions.md`. 2. Inline-SVG branch: bumped the case-sensitive prefix to `LOGO:` to match the new convention. Filename-based check now leads with `file.startsWith("logo-")` because the broadened isLogo detection in02f95f95makes that prefix the most reliable signal — the logo-<hash>.svg naming is owned by the capture pipeline and is trustworthy by construction. index.ts (asset-descriptions.md writer) changes: 3. The file header is now branched by Gemini-key presence: - With key: brief instruction + "search for the brand name here BEFORE composing a logo" pointer. - Without key: explicit ⚠️ banner naming the degraded mode, listing what the catalog-fallback signals catch, AND a fallback- of-the-fallback instruction: "open the LOGO-tagged SVGs in a previewer or sharp-render them to PNG before referencing — the alternative (composing a fake logo) ships off-brand in the final video." This is the same brand-fidelity rule the doc landmine (commite6aa7fed) lays down, repeated here so it's discoverable from inside the no-Gemini-mode file itself. 4. progress() message also reports the catalog-fallback mode when no Gemini key, so users running capture interactively see a status hint. Net: teams without a Gemini key now get: - LOGO-tagged lines they can grep for (catches the heygen wordmark and the huly favicon via the structural signals). - A header that names the degraded mode + tells agents to fall back to opening / rasterizing SVGs when descriptions are too weak. - Stable contract: the LOGO token works identically with or without Gemini, so the skill's discovery rule (e6aa7fed) is honored in both modes. Final piece of the bundle: capture/video-manifest already lists every <video> URL but doesn't download. Followup commit adds an on-demand download path so the heygen "Orb" hero (or any other captured video) is one CLI invocation away rather than a manual curl.
This commit is contained in:
@@ -363,13 +363,28 @@ export function generateAssetDescriptions(
|
||||
const heading = catalogMatch?.nearestHeading || "";
|
||||
const section = catalogMatch?.sectionClasses || "";
|
||||
const aboveFold = catalogMatch?.aboveFold ? "above fold" : "";
|
||||
// Logo signals — let the no-Gemini fallback still surface logos
|
||||
// grep-ably even when Vision wasn't available to describe them.
|
||||
const isLikelyLogo = !!(
|
||||
catalogMatch?.inBanner ||
|
||||
catalogMatch?.inHomeLink ||
|
||||
catalogMatch?.matchesTitleBrand ||
|
||||
/logo|brand|wordmark/i.test(desc) ||
|
||||
/logo|brand|wordmark/i.test(section) ||
|
||||
file.includes("logo")
|
||||
);
|
||||
const geminiCaption = geminiCaptions[file];
|
||||
const cleanName = file.replace(/\.[^.]+$/, "").replace(/[-_]/g, " ");
|
||||
const parts = [`${file} — ${sizeKb}KB`];
|
||||
if (geminiCaption) {
|
||||
// Even with Gemini's description, prepend the LOGO tag if
|
||||
// structural signals fired — gives a stable grep target for
|
||||
// agents searching for "the logo."
|
||||
if (isLikelyLogo) parts.push("LOGO");
|
||||
parts.push(geminiCaption);
|
||||
captionedLines.push(parts.join(", "));
|
||||
} else {
|
||||
if (isLikelyLogo) parts.push("LOGO");
|
||||
if (desc) parts.push(`"${desc.slice(0, 80)}"`);
|
||||
if (heading) parts.push(`section: "${heading.slice(0, 60)}"`);
|
||||
else if (section) parts.push(`in: ${section.split(" ").slice(0, 3).join(" ")}`);
|
||||
@@ -403,8 +418,16 @@ export function generateAssetDescriptions(
|
||||
),
|
||||
);
|
||||
const label = svgMatch?.label || file.replace(".svg", "").replace(/-/g, " ");
|
||||
const isLogo = svgMatch?.isLogo || file.includes("logo");
|
||||
svgLines.push(`svgs/${file} — ${isLogo ? "logo: " : "icon: "}${label}`);
|
||||
// Filename prefix is now the most reliable logo signal: the
|
||||
// capture pipeline names DOM-marked logos `logo-<hash>.svg` and
|
||||
// everything else `svg-<hash>.svg`. Fall back to the tokens.svgs
|
||||
// isLogo flag for legacy captures + a filename-includes-"logo"
|
||||
// check for human-readable rasters.
|
||||
const isLogo = file.startsWith("logo-") || svgMatch?.isLogo || file.includes("logo");
|
||||
// Use uppercase "LOGO:" so agents can grep for it as a single,
|
||||
// unambiguous token. The lowercase "logo:" prefix was easy to miss
|
||||
// since real Vision captions also use the word casually.
|
||||
svgLines.push(`svgs/${file} — ${isLogo ? "LOGO: " : "icon: "}${label}`);
|
||||
}
|
||||
} catch {
|
||||
/* no svgs dir */
|
||||
|
||||
@@ -522,14 +522,19 @@ export async function captureWebsite(
|
||||
const lines = generateAssetDescriptions(outputDir, tokens, catalogedAssets, geminiCaptions);
|
||||
|
||||
if (lines.length > 0) {
|
||||
const hasGeminiKey = !!(process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY);
|
||||
const header = hasGeminiKey
|
||||
? "# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually. Lines tagged `LOGO` are the brand-mark candidates — search for the brand name here BEFORE composing a logo from scratch.\n\n"
|
||||
: "# Asset Descriptions\n\n⚠️ GEMINI_API_KEY not set — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. Lines tagged `LOGO` are the brand-mark candidates per DOM-structural signals (inside header/nav, inside home anchor, or alt-text matching the page title). To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY) and re-run.\n\nWhen the description is too weak to identify a captured logo by description alone, open the LOGO-tagged SVGs in a previewer or `sharp`-render them to PNG before referencing — the alternative (composing a fake logo) ships off-brand in the final video.\n\n";
|
||||
writeFileSync(
|
||||
join(outputDir, "extracted", "asset-descriptions.md"),
|
||||
"# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually.\n\n" +
|
||||
lines.map((l) => "- " + l).join("\n") +
|
||||
"\n",
|
||||
header + lines.map((l) => "- " + l).join("\n") + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
progress("design", `${lines.length} asset descriptions written`);
|
||||
progress(
|
||||
"design",
|
||||
`${lines.length} asset descriptions written${hasGeminiKey ? "" : " (no Gemini key — catalog-fallback mode)"}`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
/* non-critical */
|
||||
|
||||
Reference in New Issue
Block a user