mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
* fix(cli): resolve helper programs from PATH, never the working directory On Windows a bare command name is looked up in the working directory before PATH. The CLI now sets NoDefaultCurrentDirectoryInExePath at startup, which Node's spawn and cmd.exe both honor and child processes inherit, so tools such as powershell.exe, taskkill and npx.cmd always come from PATH. The shared ffmpeg/ffprobe resolver no longer scans the working directory on Windows and skips relative PATH entries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): keep Python helpers from importing files in the working directory `python -c` and `python -m` put the working directory first on sys.path, so a file in the project named like a probed module ran instead of the installed package. Inline helper code now drops that entry before any import, and `-m` helpers (pip) run through runpy after the same step. Paths and arguments are unchanged. On Windows, findPython also skips a `where` match in the working directory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): run the skills installer and nested CLI calls independent of the project The global skills install ran `npx skills@<pin>` in the current folder, so npm could take the package or registry from that project's node_modules or .npmrc. It now runs from the home folder; the install is global either way. When the CLI starts the media-use engine it hands over its own entry file. The local voice fallback and the audio helpers then run that same CLI with the same Node instead of resolving `npx hyperframes` from the project. Without a handed-over entry they behave as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(media-use): load only provider keys from a project .env The audio scripts copied every key of a project .env into their environment except two. They now take the same kind of short allowlist the CLI uses: HeyGen, ElevenLabs and Gemini API keys plus a few harmless settings. Program overrides, runtime options, credential files and the credential store path come only from the person's own environment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): build whisper.cpp only from a pinned, verified release The source-build fallback cloned the upstream default branch. It now clones the v1.9.5 release tag and refuses to build unless the checkout is the reviewed commit for that tag. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): keep registry caches apart and refuse config install targets Registry cache file names now include a digest of the exact registry URL, and each entry records the URL it came from, so one registry's cached manifest is never read for another. Entries from older versions are refetched once. Registry installs refuse targets inside .git or any hidden directory, hidden configuration files, and package.json, hyperframes.json and hyperframes.lock.json. An example installed by `init` may still bring its own package.json and .gitignore. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(capture): keep page text to one line in agent files and validate Vertex keys Capture writes AGENTS.md, CLAUDE.md and .cursorrules from the page's title, font names and colours. Page text is now flattened to one short line and colours must be hex values. Existing agent files that capture did not write are kept, with a warning. HYPERFRAMES_VERTEX_SERVICE_ACCOUNT must now be a plain service-account key with a Google token endpoint; other credential types skip captioning with a warning. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(cli): find the digest-named registry cache and trim a new file's comment Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): match registry targets as Windows names them; skills install without HOME Windows drops trailing dots and spaces from a path segment and reads name:stream as name, so package.json. or .git./hooks reached the files the target rules refuse. The global skills install now falls back to the temp folder when HOME does not exist. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): turn off the Windows working-directory lookup in every program entry The warm-check server and client are their own tsup entries and copied the environment before cli.js set NoDefaultCurrentDirectoryInExePath, so the server spawned a bare powershell.exe that Windows could take from the project folder. Every entry that hides child consoles now also loads windowsExeSearch first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(cli): refuse Windows short-name registry targets A segment like CLAUDE~1 or PACKAG~1.JSO opens the long name on NTFS, so it reached the files the target rules refuse. The trailing-dot and stream-name normalization is removed: publication already rejects those segments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
815 lines
29 KiB
TypeScript
815 lines
29 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import {
|
|
captionImagesWithGemini,
|
|
resolveVisionPhaseCompletion,
|
|
type VisionCaptionOutcome,
|
|
} from "./contentExtractor.js";
|
|
import { loadOptionalPackage } from "../utils/optionalPackages.js";
|
|
|
|
const { generateContentMock, clientOptions, sharpState } = vi.hoisted(() => ({
|
|
generateContentMock: vi.fn(),
|
|
// How the SDK client was constructed is the whole difference between the Vertex and API-key
|
|
// paths, so the Vertex cases assert on it rather than on the request.
|
|
clientOptions: [] as Record<string, unknown>[],
|
|
// A native abort inside libvips cannot be caught, so the only defence is never running two
|
|
// renders at once. That is a property of the loop, and this records it.
|
|
sharpState: {
|
|
inFlight: 0,
|
|
maxInFlight: 0,
|
|
concurrencyCalls: [] as number[],
|
|
// Stands in for the host core count sharp reports before anything touches it, so a failure
|
|
// to restore shows up as a wrong value rather than as a coincidental match with 1.
|
|
HOST_CONCURRENCY: 8,
|
|
concurrency: 8,
|
|
renders: [] as string[],
|
|
},
|
|
}));
|
|
|
|
vi.mock("sharp", () => {
|
|
const pipeline = (filePath: string) => {
|
|
const chain = {
|
|
resize: () => chain,
|
|
flatten: () => chain,
|
|
png: () => chain,
|
|
toBuffer: async () => {
|
|
sharpState.inFlight += 1;
|
|
sharpState.maxInFlight = Math.max(sharpState.maxInFlight, sharpState.inFlight);
|
|
sharpState.renders.push(filePath);
|
|
// Yield, so an overlapping caller would be observed rather than serialized by luck.
|
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
sharpState.inFlight -= 1;
|
|
return Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
|
},
|
|
};
|
|
return chain;
|
|
};
|
|
const sharp = Object.assign(pipeline, {
|
|
// Real `sharp.concurrency()` is a getter when called with no argument and a process-global
|
|
// setter otherwise. The mock has to be both, or code that saves and restores the host value
|
|
// cannot be tested at all.
|
|
concurrency: (value?: number) => {
|
|
if (typeof value === "number") {
|
|
sharpState.concurrencyCalls.push(value);
|
|
sharpState.concurrency = value;
|
|
}
|
|
return sharpState.concurrency;
|
|
},
|
|
});
|
|
return { default: sharp };
|
|
});
|
|
|
|
vi.mock("../utils/optionalPackages.js", () => ({
|
|
loadSharp: async () => (await import("sharp")).default,
|
|
loadOptionalPackage: vi.fn(async () => ({
|
|
GoogleGenAI: class {
|
|
models = { generateContent: generateContentMock };
|
|
constructor(options: Record<string, unknown>) {
|
|
clientOptions.push(options);
|
|
}
|
|
},
|
|
})),
|
|
}));
|
|
|
|
// These tests exercise the OpenRouter provider path only — it makes a plain
|
|
// `fetch` call we can stub, with no native (`sharp`) or `@google/genai`
|
|
// dependency. OpenRouter wins over Gemini when OPENROUTER_API_KEY is set, so we
|
|
// don't need to clear the Gemini keys for the OpenRouter cases.
|
|
|
|
function makeProjectWithImages(files = ["hero.png"]): string {
|
|
const dir = mkdtempSync(join(tmpdir(), "hf-caption-"));
|
|
mkdirSync(join(dir, "assets"), { recursive: true });
|
|
// Contents are irrelevant to the OpenRouter path (it just base64-encodes the
|
|
// bytes); only the .png extension matters for the image filter.
|
|
for (const file of files) {
|
|
writeFileSync(join(dir, "assets", file), Buffer.from([0x89, 0x50, 0x4e, 0x47]));
|
|
}
|
|
return dir;
|
|
}
|
|
|
|
describe("captionImagesWithGemini — OpenRouter provider", () => {
|
|
const dirs: string[] = [];
|
|
afterEach(() => {
|
|
vi.unstubAllGlobals();
|
|
vi.unstubAllEnvs();
|
|
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
|
dirs.length = 0;
|
|
});
|
|
|
|
it("captions via OpenRouter when OPENROUTER_API_KEY is set", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-test-key");
|
|
vi.stubEnv("HYPERFRAMES_OPENROUTER_MODEL", "google/gemini-3.1-flash-lite");
|
|
|
|
// Capture the request inside the mock, where the args are well-typed —
|
|
// avoids casting `mock.calls` (and the repo's ban on `as` assertions).
|
|
let capturedUrl: string | undefined;
|
|
let capturedInit: RequestInit | undefined;
|
|
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
|
capturedUrl = url;
|
|
capturedInit = init;
|
|
return new Response(
|
|
JSON.stringify({ choices: [{ message: { content: "A dark hero with blue accents." } }] }),
|
|
{ status: 200, headers: { "content-type": "application/json" } },
|
|
);
|
|
});
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
const warnings: string[] = [];
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings);
|
|
|
|
expect(captions).toEqual({ "hero.png": "A dark hero with blue accents." });
|
|
expect(warnings).toEqual([]);
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
|
|
expect(capturedUrl).toBe("https://openrouter.ai/api/v1/chat/completions");
|
|
expect(new Headers(capturedInit?.headers).get("authorization")).toBe("Bearer or-test-key");
|
|
const body = JSON.parse(typeof capturedInit?.body === "string" ? capturedInit.body : "{}");
|
|
expect(body.model).toBe("google/gemini-3.1-flash-lite");
|
|
const image = body.messages[0].content.find((p: { type: string }) => p.type === "image_url");
|
|
expect(image?.image_url?.url).toMatch(/^data:image\/png;base64,/);
|
|
});
|
|
|
|
it("degrades gracefully (no throw, no captions) when OpenRouter returns a non-OK status", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-bad-key");
|
|
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(
|
|
async () =>
|
|
new Response("provider detail containing or-bad-key", {
|
|
status: 401,
|
|
statusText: "Unauthorized",
|
|
}),
|
|
),
|
|
);
|
|
|
|
const warnings: string[] = [];
|
|
let outcome: VisionCaptionOutcome | undefined;
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
|
|
onOutcome: (value) => {
|
|
outcome = value;
|
|
},
|
|
});
|
|
|
|
expect(captions).toEqual({});
|
|
expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]);
|
|
expect(warnings.join(" ")).not.toContain("or-bad-key");
|
|
expect(outcome).toEqual({
|
|
timedOutRequests: 0,
|
|
failedRequests: 1,
|
|
budgetExhausted: false,
|
|
});
|
|
if (!outcome) throw new Error("Expected vision caption outcome");
|
|
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
|
|
status: "degraded",
|
|
reason: "provider-error",
|
|
});
|
|
});
|
|
|
|
it("terminates captioning when the vision provider never responds", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-hanging-key");
|
|
vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
|
|
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn((_url: string, init?: RequestInit) => {
|
|
return new Promise<Response>((_resolve, reject) => {
|
|
init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), {
|
|
once: true,
|
|
});
|
|
});
|
|
}),
|
|
);
|
|
|
|
const warnings: string[] = [];
|
|
const result = await Promise.race([
|
|
captionImagesWithGemini(dir, () => {}, warnings),
|
|
new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)),
|
|
]);
|
|
|
|
expect(result).toEqual({});
|
|
});
|
|
|
|
it.each([
|
|
{ label: "success JSON", status: 200, partialBody: '{"choices":[' },
|
|
{ label: "failure text", status: 500, partialBody: "upstream failed: " },
|
|
])(
|
|
"terminates captioning when OpenRouter $label body never ends",
|
|
async ({ status, partialBody }) => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-stalled-body-key");
|
|
vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
|
|
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(async () => {
|
|
const body = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
controller.enqueue(new TextEncoder().encode(partialBody));
|
|
},
|
|
});
|
|
return new Response(body, {
|
|
status,
|
|
headers: { "content-type": "application/json" },
|
|
});
|
|
}),
|
|
);
|
|
|
|
const warnings: string[] = [];
|
|
let outcome: VisionCaptionOutcome | undefined;
|
|
const result = await Promise.race([
|
|
captionImagesWithGemini(dir, () => {}, warnings, {
|
|
onOutcome: (value) => {
|
|
outcome = value;
|
|
},
|
|
}),
|
|
new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)),
|
|
]);
|
|
|
|
expect(result).toEqual({});
|
|
expect(warnings).toEqual(["OpenRouter vision timed out for 1 asset(s); captions omitted."]);
|
|
expect(outcome).toEqual({
|
|
timedOutRequests: 1,
|
|
failedRequests: 0,
|
|
budgetExhausted: false,
|
|
});
|
|
},
|
|
);
|
|
|
|
it("keeps successful captions while reporting a failed sibling request", async () => {
|
|
const dir = makeProjectWithImages(["failed.png", "success.png"]);
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-failure-key");
|
|
|
|
let requestCount = 0;
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(async () => {
|
|
requestCount++;
|
|
if (requestCount === 1) {
|
|
return new Response("upstream exploded with or-mixed-failure-key", {
|
|
status: 500,
|
|
statusText: "Internal Server Error",
|
|
});
|
|
}
|
|
return new Response(
|
|
JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }),
|
|
{ status: 200, headers: { "content-type": "application/json" } },
|
|
);
|
|
}),
|
|
);
|
|
|
|
const warnings: string[] = [];
|
|
let outcome: VisionCaptionOutcome | undefined;
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
|
|
onOutcome: (value) => {
|
|
outcome = value;
|
|
},
|
|
});
|
|
|
|
expect(captions).toEqual({ "success.png": "Successful image." });
|
|
expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]);
|
|
expect(warnings.join(" ")).not.toContain("or-mixed-failure-key");
|
|
expect(outcome).toEqual({
|
|
timedOutRequests: 0,
|
|
failedRequests: 1,
|
|
budgetExhausted: false,
|
|
});
|
|
});
|
|
|
|
it("keeps a successful sibling caption when another request times out", async () => {
|
|
const dir = makeProjectWithImages(["hang.png", "success.png"]);
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-key");
|
|
vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
|
|
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn((_url: string, init?: RequestInit) => {
|
|
const body = JSON.parse(typeof init?.body === "string" ? init.body : "{}");
|
|
const imageUrl = body.messages?.[0]?.content?.[1]?.image_url?.url;
|
|
if (typeof imageUrl === "string" && imageUrl.includes("iVBORw==")) {
|
|
// Both fixtures contain the same bytes, so use call order: the first
|
|
// request hangs while the second succeeds independently.
|
|
const fetchMock = vi.mocked(fetch);
|
|
if (fetchMock.mock.calls.length === 1) return new Promise<Response>(() => {});
|
|
}
|
|
return Promise.resolve(
|
|
new Response(
|
|
JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }),
|
|
{
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
},
|
|
),
|
|
);
|
|
}),
|
|
);
|
|
|
|
const warnings: string[] = [];
|
|
let outcome: VisionCaptionOutcome | undefined;
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
|
|
onOutcome: (value) => {
|
|
outcome = value;
|
|
},
|
|
});
|
|
|
|
expect(captions).toEqual({ "success.png": "Successful image." });
|
|
expect(outcome).toEqual({
|
|
timedOutRequests: 1,
|
|
failedRequests: 0,
|
|
budgetExhausted: false,
|
|
});
|
|
if (!outcome) throw new Error("Expected vision caption outcome");
|
|
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
|
|
status: "degraded",
|
|
reason: "request-timeout",
|
|
});
|
|
});
|
|
|
|
it("does not call a configured provider when vision is skipped", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-unused-key");
|
|
const fetchMock = vi.fn();
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
const captions = await captionImagesWithGemini(dir, () => {}, [], { skipVision: true });
|
|
|
|
expect(captions).toEqual({});
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("does not start a vision batch after the capture budget is exhausted", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-budget-key");
|
|
const fetchMock = vi.fn();
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
const captions = await captionImagesWithGemini(dir, () => {}, [], {
|
|
remainingMs: () => 0,
|
|
});
|
|
|
|
expect(captions).toEqual({});
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("classifies non-provider pipeline failures without leaking the local path", async () => {
|
|
const dir = mkdtempSync(join(tmpdir(), "hf-caption-no-assets-"));
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-unused-key");
|
|
|
|
const warnings: string[] = [];
|
|
let outcome: VisionCaptionOutcome | undefined;
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
|
|
onOutcome: (value) => {
|
|
outcome = value;
|
|
},
|
|
});
|
|
|
|
expect(captions).toEqual({});
|
|
expect(warnings).toEqual(["OpenRouter captioning failed internally; captions omitted."]);
|
|
expect(warnings.join(" ")).not.toContain(dir);
|
|
expect(outcome).toEqual({
|
|
timedOutRequests: 0,
|
|
failedRequests: 0,
|
|
budgetExhausted: false,
|
|
internalError: true,
|
|
});
|
|
if (!outcome) throw new Error("Expected vision caption outcome");
|
|
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
|
|
status: "degraded",
|
|
reason: "internal-error",
|
|
});
|
|
});
|
|
|
|
it("skips captioning entirely when no provider key is present", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "");
|
|
vi.stubEnv("GEMINI_API_KEY", "");
|
|
vi.stubEnv("GOOGLE_API_KEY", "");
|
|
|
|
const fetchMock = vi.fn();
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
const warnings: string[] = [];
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings);
|
|
|
|
expect(captions).toEqual({});
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe("captionImagesWithGemini — Gemini provider", () => {
|
|
const dirs: string[] = [];
|
|
|
|
afterEach(() => {
|
|
generateContentMock.mockReset();
|
|
vi.unstubAllEnvs();
|
|
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
|
dirs.length = 0;
|
|
});
|
|
|
|
it("terminates captioning when Gemini never settles even if the SDK ignores abort", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "");
|
|
vi.stubEnv("GEMINI_API_KEY", "gemini-hanging-key");
|
|
vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
|
|
generateContentMock.mockImplementation(() => new Promise(() => {}));
|
|
|
|
const result = await Promise.race([
|
|
captionImagesWithGemini(dir, () => {}, []),
|
|
new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)),
|
|
]);
|
|
|
|
expect(result).toEqual({});
|
|
});
|
|
|
|
it("reports a rejected Gemini request without leaking its error detail", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "");
|
|
vi.stubEnv("GEMINI_API_KEY", "gemini-secret-key");
|
|
generateContentMock.mockRejectedValue(
|
|
new Error("provider echoed gemini-secret-key in its response"),
|
|
);
|
|
|
|
const warnings: string[] = [];
|
|
let outcome: VisionCaptionOutcome | undefined;
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
|
|
onOutcome: (value) => {
|
|
outcome = value;
|
|
},
|
|
});
|
|
|
|
expect(captions).toEqual({});
|
|
expect(warnings).toEqual(["Gemini vision failed for 1 asset(s); captions omitted."]);
|
|
expect(warnings.join(" ")).not.toContain("gemini-secret-key");
|
|
expect(outcome).toEqual({
|
|
timedOutRequests: 0,
|
|
failedRequests: 1,
|
|
budgetExhausted: false,
|
|
});
|
|
if (!outcome) throw new Error("Expected vision caption outcome");
|
|
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
|
|
status: "degraded",
|
|
reason: "provider-error",
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("captionImagesWithGemini — Vertex AI provider", () => {
|
|
const dirs: string[] = [];
|
|
const SERVICE_ACCOUNT = JSON.stringify({
|
|
type: "service_account",
|
|
project_id: "prefab-kit-000000",
|
|
private_key: "-----BEGIN PRIVATE KEY-----super-secret-material-----END PRIVATE KEY-----",
|
|
client_email: "capture@prefab-kit-000000.iam.gserviceaccount.com",
|
|
});
|
|
|
|
// Earlier describe blocks construct the SDK client too, so the record is cleared going in
|
|
// rather than only on the way out.
|
|
beforeEach(() => {
|
|
clientOptions.length = 0;
|
|
});
|
|
|
|
afterEach(() => {
|
|
generateContentMock.mockReset();
|
|
clientOptions.length = 0;
|
|
vi.unstubAllEnvs();
|
|
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
|
dirs.length = 0;
|
|
});
|
|
|
|
function vertexEnv(): void {
|
|
vi.stubEnv("OPENROUTER_API_KEY", "");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_PROJECT_ID", "prefab-kit-000000");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_SERVICE_ACCOUNT", SERVICE_ACCOUNT);
|
|
}
|
|
|
|
it("prefers a service account over a bare API key, and authenticates against the project", async () => {
|
|
// A deployment can hold a Gemini key that is present but rejected. Every request then
|
|
// returns empty text, so the capture reports "Captioned N/N" followed by "0 images
|
|
// captioned" and no error — which is exactly what production was doing.
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vertexEnv();
|
|
vi.stubEnv("GEMINI_API_KEY", "a-key-that-the-server-would-be-rejected-for");
|
|
generateContentMock.mockResolvedValue({ text: "A dark blue product screenshot." });
|
|
|
|
const stages: string[] = [];
|
|
const captions = await captionImagesWithGemini(
|
|
dir,
|
|
(stage, detail) => {
|
|
stages.push(detail ?? stage);
|
|
},
|
|
[],
|
|
);
|
|
|
|
expect(captions).toEqual({ "hero.png": "A dark blue product screenshot." });
|
|
expect(stages.join(" ")).toContain("Vertex AI");
|
|
expect(clientOptions).toHaveLength(1);
|
|
expect(clientOptions[0]).toMatchObject({
|
|
vertexai: true,
|
|
project: "prefab-kit-000000",
|
|
location: "us-central1",
|
|
googleAuthOptions: { credentials: JSON.parse(SERVICE_ACCOUNT) },
|
|
});
|
|
expect(clientOptions[0]).not.toHaveProperty("apiKey");
|
|
});
|
|
|
|
it("skips captioning with the loader's message when @google/genai cannot be installed", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vertexEnv();
|
|
vi.mocked(loadOptionalPackage).mockRejectedValueOnce(new Error("install it yourself"));
|
|
|
|
const warnings: string[] = [];
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings);
|
|
|
|
expect(captions).toEqual({});
|
|
expect(warnings).toEqual(["Skipped vision captioning: install it yourself"]);
|
|
expect(generateContentMock).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("honours an explicit region", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vertexEnv();
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_LOCATION", "europe-west4");
|
|
generateContentMock.mockResolvedValue({ text: "A caption." });
|
|
|
|
await captionImagesWithGemini(dir, () => {}, []);
|
|
|
|
expect(clientOptions[0]).toMatchObject({ location: "europe-west4" });
|
|
});
|
|
|
|
it("spends no output budget on thinking", async () => {
|
|
// Thinking tokens come out of maxOutputTokens, so a model left free to think can consume
|
|
// the whole budget and return empty text: a successful request that produces no caption.
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vertexEnv();
|
|
// Capture the request inside the mock, where the argument is well-typed — avoids
|
|
// indexing `mock.calls` (and the repo's ban on `as` assertions).
|
|
let model: unknown;
|
|
let thinkingConfig: unknown;
|
|
generateContentMock.mockImplementation(
|
|
async (request: { model: string; config?: { thinkingConfig?: unknown } }) => {
|
|
model = request.model;
|
|
thinkingConfig = request.config?.thinkingConfig;
|
|
return { text: "A caption." };
|
|
},
|
|
);
|
|
|
|
await captionImagesWithGemini(dir, () => {}, []);
|
|
|
|
expect(generateContentMock).toHaveBeenCalledTimes(1);
|
|
expect(thinkingConfig).toEqual({ thinkingBudget: 0 });
|
|
// The API's flash-lite preview id is not resolvable on Vertex, so the default differs.
|
|
expect(model).toBe("gemini-2.5-flash");
|
|
});
|
|
|
|
it("skips captioning when the service account is unparseable, without echoing it", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "");
|
|
vi.stubEnv("GEMINI_API_KEY", "");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_PROJECT_ID", "prefab-kit-000000");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_SERVICE_ACCOUNT", "{not-json super-secret-material");
|
|
|
|
const warnings: string[] = [];
|
|
let outcome: VisionCaptionOutcome | undefined;
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
|
|
onOutcome: (value) => {
|
|
outcome = value;
|
|
},
|
|
});
|
|
|
|
expect(captions).toEqual({});
|
|
expect(generateContentMock).not.toHaveBeenCalled();
|
|
expect(warnings.join(" ")).toContain("not valid JSON");
|
|
expect(warnings.join(" ")).not.toContain("super-secret-material");
|
|
if (!outcome) throw new Error("Expected vision caption outcome");
|
|
expect(outcome.internalError).toBe(true);
|
|
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
|
|
status: "degraded",
|
|
reason: "internal-error",
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
{
|
|
type: "external_account",
|
|
token_url: "https://collector.example/token",
|
|
credential_source: { file: "/home/user/.ssh/id_ed25519" },
|
|
},
|
|
{
|
|
type: "impersonated_service_account",
|
|
service_account_impersonation_url: "https://collector.example/impersonate",
|
|
source_credentials: {},
|
|
},
|
|
{ type: "service_account", token_uri: "https://collector.example/token" },
|
|
{ type: "service_account", credential_source: { url: "http://169.254.169.254/" } },
|
|
])("refuses a credential that is not a plain service-account key: $type", async (credential) => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "");
|
|
vi.stubEnv("GEMINI_API_KEY", "");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_PROJECT_ID", "prefab-kit-000000");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_SERVICE_ACCOUNT", JSON.stringify(credential));
|
|
|
|
const warnings: string[] = [];
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings);
|
|
|
|
expect(captions).toEqual({});
|
|
expect(clientOptions).toHaveLength(0);
|
|
expect(generateContentMock).not.toHaveBeenCalled();
|
|
expect(warnings.join(" ")).toContain("not a service-account key");
|
|
});
|
|
|
|
it("stays on OpenRouter when the user has opted into it explicitly", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vertexEnv();
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-key");
|
|
const fetchMock = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({ choices: [{ message: { content: "A caption." } }] }),
|
|
});
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
await captionImagesWithGemini(dir, () => {}, []);
|
|
|
|
expect(fetchMock).toHaveBeenCalled();
|
|
expect(clientOptions).toHaveLength(0);
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
it("needs both halves of the credential before it will use Vertex", async () => {
|
|
const dir = makeProjectWithImages();
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "");
|
|
vi.stubEnv("GEMINI_API_KEY", "");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_PROJECT_ID", "prefab-kit-000000");
|
|
vi.stubEnv("HYPERFRAMES_VERTEX_SERVICE_ACCOUNT", "");
|
|
|
|
const captions = await captionImagesWithGemini(dir, () => {}, []);
|
|
|
|
expect(captions).toEqual({});
|
|
expect(clientOptions).toHaveLength(0);
|
|
expect(generateContentMock).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe("captionImagesWithGemini — SVG rasterization", () => {
|
|
const dirs: string[] = [];
|
|
|
|
beforeEach(() => {
|
|
sharpState.inFlight = 0;
|
|
sharpState.maxInFlight = 0;
|
|
sharpState.concurrencyCalls.length = 0;
|
|
sharpState.concurrency = sharpState.HOST_CONCURRENCY;
|
|
sharpState.renders.length = 0;
|
|
clientOptions.length = 0;
|
|
});
|
|
|
|
afterEach(() => {
|
|
generateContentMock.mockReset();
|
|
vi.unstubAllGlobals();
|
|
vi.unstubAllEnvs();
|
|
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
|
dirs.length = 0;
|
|
});
|
|
|
|
function makeProjectWithSvgs(count: number): string {
|
|
const dir = mkdtempSync(join(tmpdir(), "hf-svg-"));
|
|
mkdirSync(join(dir, "assets", "svgs"), { recursive: true });
|
|
for (let i = 0; i < count; i++) {
|
|
writeFileSync(
|
|
join(dir, "assets", "svgs", `logo-${i}.svg`),
|
|
'<svg xmlns="http://www.w3.org/2000/svg"><path fill="#000" d="M0 0h8v8H0z"/></svg>',
|
|
);
|
|
}
|
|
return dir;
|
|
}
|
|
|
|
it("never runs two libvips renders at once", async () => {
|
|
// Production aborted here with `free(): unaligned chunk detected in tcache 2` (SIGABRT),
|
|
// twice in fourteen days, losing the whole capture. A native abort cannot be caught by the
|
|
// surrounding try/catch, so the concurrency has to be absent rather than handled — which
|
|
// makes "one at a time" the assertion, not "errors are reported".
|
|
const dir = makeProjectWithSvgs(6);
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-key");
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(async () => {
|
|
// Slower than a render, so overlapping renders would be the easy way to go faster —
|
|
// the point is that the loop does not take it.
|
|
await new Promise((resolve) => setTimeout(resolve, 15));
|
|
return new Response(
|
|
JSON.stringify({ choices: [{ message: { content: "A dark glyph." } }] }),
|
|
{
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
},
|
|
);
|
|
}),
|
|
);
|
|
|
|
const captions = await captionImagesWithGemini(dir, () => {}, []);
|
|
|
|
expect(sharpState.renders).toHaveLength(6);
|
|
expect(sharpState.maxInFlight).toBe(1);
|
|
expect(Object.keys(captions)).toHaveLength(6);
|
|
});
|
|
|
|
it("bounds libvips' own worker pool for the renders, then hands it back", async () => {
|
|
// Left at its default the pool sizes itself to the host's core count, so serializing the
|
|
// loop alone still leaves one render fanning out across every core. But `sharp.concurrency`
|
|
// is process-global: leaving it at 1 pins every later sharp caller in this process, none of
|
|
// which asked for captioning, so the bound has to be given back when the renders are done.
|
|
const dir = makeProjectWithSvgs(1);
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-key");
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(
|
|
async () =>
|
|
new Response(JSON.stringify({ choices: [{ message: { content: "A glyph." } }] }), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
),
|
|
);
|
|
|
|
await captionImagesWithGemini(dir, () => {}, []);
|
|
|
|
expect(sharpState.concurrencyCalls).toEqual([1, sharpState.HOST_CONCURRENCY]);
|
|
expect(sharpState.concurrency).toBe(sharpState.HOST_CONCURRENCY);
|
|
});
|
|
|
|
it("hands the worker pool back even when a rasterize throws", async () => {
|
|
// The restore is in a `finally`, because a skipped SVG must not cost the rest of the process
|
|
// its threads.
|
|
const dir = makeProjectWithSvgs(2);
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-key");
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(
|
|
async () =>
|
|
new Response(JSON.stringify({ choices: [{ message: { content: "A glyph." } }] }), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
),
|
|
);
|
|
// Unreadable, which is how an exotic SVG behaves through sharp.
|
|
rmSync(join(dir, "assets", "svgs", "logo-0.svg"));
|
|
mkdirSync(join(dir, "assets", "svgs", "logo-0.svg"));
|
|
|
|
await captionImagesWithGemini(dir, () => {}, []);
|
|
|
|
expect(sharpState.concurrency).toBe(sharpState.HOST_CONCURRENCY);
|
|
});
|
|
|
|
it("keeps captioning the rest when one SVG cannot be rasterized", async () => {
|
|
const dir = makeProjectWithSvgs(3);
|
|
dirs.push(dir);
|
|
vi.stubEnv("OPENROUTER_API_KEY", "or-key");
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn(
|
|
async () =>
|
|
new Response(JSON.stringify({ choices: [{ message: { content: "A glyph." } }] }), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
),
|
|
);
|
|
// One file is unreadable, which is how an exotic SVG behaves through sharp.
|
|
rmSync(join(dir, "assets", "svgs", "logo-1.svg"));
|
|
mkdirSync(join(dir, "assets", "svgs", "logo-1.svg"));
|
|
|
|
const warnings: string[] = [];
|
|
const captions = await captionImagesWithGemini(dir, () => {}, warnings);
|
|
|
|
expect(Object.keys(captions).sort()).toEqual(["svgs/logo-0.svg", "svgs/logo-2.svg"]);
|
|
expect(sharpState.maxInFlight).toBe(1);
|
|
});
|
|
});
|