mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat: add Gemini TTS models and service-account auth to media skills (#4377)
* feat: add opt-in Gemini 3.8 TTS to media skills * feat: support Gemini service accounts and older TTS models * fix(audio): isolate Gemini auth and narrow OAuth scope * fix(audio): pin Gemini Python helpers to UTF-8
This commit is contained in:
James Russo
@@ -74,9 +74,37 @@ Run `npx hyperframes doctor` to check which local dependencies are installed.
|
||||
The media workflows run `hyperframes auth status` before generation and tell
|
||||
you which path they will use.
|
||||
|
||||
For Gemini narration, explicitly ask your agent to use Gemini TTS. The shared
|
||||
audio workflow supports 3.8 Flash and Flash-Lite, 3.1 Flash Preview, and 2.5 Pro
|
||||
and Flash Preview. Gemini remains an explicit choice; configuring credentials
|
||||
does not change the automatic voice-provider order above.
|
||||
|
||||
Choose one authentication method:
|
||||
|
||||
- **API key:** set `GEMINI_API_KEY` or `GOOGLE_API_KEY` in your environment or
|
||||
project `.env`. This path needs no Python dependencies.
|
||||
- **Service account:** install `google-auth` and `requests` in your Python 3
|
||||
environment, then set `GOOGLE_APPLICATION_CREDENTIALS` to the absolute path
|
||||
of your service-account JSON file. Alternatively, inject the JSON through
|
||||
`GCS_CREDS` from your secret manager. Set `GOOGLE_CLOUD_PROJECT` to the quota
|
||||
project when it differs from the service account's project.
|
||||
|
||||
Credentials resolve in that order: Gemini key, Google key, service-account file,
|
||||
then JSON environment variable. Unset API keys when choosing service-account
|
||||
OAuth. The helper obtains a fresh token for each generation using the
|
||||
`generative-language.retriever` scope. Keys and tokens
|
||||
are never placed in compositions or saved audio metadata.
|
||||
|
||||
Both methods call the Gemini Developer Interactions API. Your project needs
|
||||
access to that API and the requested model. Cloud Text-to-Speech and Vertex AI
|
||||
have separate model availability. This helper accepts service-account JSON,
|
||||
not user ADC files or metadata-server credentials.
|
||||
|
||||
See [Gemini voiceover](/prompting/media-and-audio#gemini-voiceover).
|
||||
|
||||
<Note>
|
||||
`npx hyperframes tts` itself is the local Kokoro CLI. Hosted HeyGen and
|
||||
ElevenLabs voices are selected by the bundled media workflow helpers, not by
|
||||
ElevenLabs and Gemini voices are selected by the bundled media workflow helpers, not by
|
||||
that command.
|
||||
</Note>
|
||||
|
||||
@@ -112,7 +140,9 @@ to a shared space with `--space`.
|
||||
| `HEYGEN_API_URL` | API base URL (default `https://api.heygen.com`). |
|
||||
| `HEYGEN_CONFIG_DIR` | Credentials directory (default `~/.heygen`). |
|
||||
| `ELEVENLABS_API_KEY` | ElevenLabs TTS, used when no HeyGen credential is present. |
|
||||
| `GEMINI_API_KEY` / `GOOGLE_API_KEY` | Lyria music generation (and capture descriptions). |
|
||||
| `GEMINI_API_KEY` / `GOOGLE_API_KEY` | Explicit Gemini TTS selection and Lyria music generation; capture descriptions use `GEMINI_API_KEY`. |
|
||||
| `GOOGLE_APPLICATION_CREDENTIALS` / `GCS_CREDS` | Gemini TTS service-account JSON file path / injected JSON. Used when neither API key is set. |
|
||||
| `GOOGLE_CLOUD_PROJECT` / `GCLOUD_PROJECT_ID` | Gemini OAuth quota project; first set value wins, then the service account project. |
|
||||
| `OPENROUTER_API_KEY` | Capture descriptions; takes priority over Gemini for that step. |
|
||||
|
||||
## Related topics
|
||||
|
||||
@@ -38,6 +38,26 @@ agent choose.
|
||||
- ❌ `add a voice`
|
||||
- ✅ `warm, unhurried female narration of the quoted script` — tone and pace are what actually change the delivery
|
||||
|
||||
### Gemini voiceover
|
||||
|
||||
Choose Gemini when you want to direct a narrator's delivery with a description.
|
||||
Configure [Gemini credentials](/guides/authentication#providers-used-by-agent-workflows),
|
||||
then ask your agent:
|
||||
|
||||
> Use Gemini 3.8 Flash TTS with the Kore voice to narrate this script. Make the
|
||||
> delivery warm and unhurried. Use the media-use audio engine, transcribe the
|
||||
> generated audio, and sync the captions and visual reveals to the spoken words.
|
||||
|
||||
The shared audio workflow supports 3.8 Flash and Flash-Lite TTS,
|
||||
3.1 Flash TTS Preview, and 2.5 Pro and Flash TTS Preview. Gemini is an explicit choice; it does not replace
|
||||
the automatic provider order above. Keep the script separate from delivery
|
||||
instructions. The helper uses structured style metadata for 3.8 and a delivery
|
||||
prompt for older models; review the generated speech for accidental directions.
|
||||
|
||||
The agent saves the voiceover locally and obtains word timings through a
|
||||
separate transcription pass. Check names, numbers, and pauses against the
|
||||
audio before the final render. A new narration take needs new timings.
|
||||
|
||||
## Background music
|
||||
|
||||
Music resolves from a large catalog by mood. It should almost always sit *under*
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"files": 142
|
||||
},
|
||||
"faceless-explainer": {
|
||||
"hash": "d3e32414b9e86678",
|
||||
"hash": "995968cc3ba6faee",
|
||||
"files": 27
|
||||
},
|
||||
"figma": {
|
||||
@@ -54,8 +54,8 @@
|
||||
"files": 1
|
||||
},
|
||||
"media-use": {
|
||||
"hash": "d15857ed6f884a1f",
|
||||
"files": 91
|
||||
"hash": "b35de041b87e763d",
|
||||
"files": 98
|
||||
},
|
||||
"motion-graphics": {
|
||||
"hash": "ab9d2e9c205ce1b1",
|
||||
@@ -66,11 +66,11 @@
|
||||
"files": 169
|
||||
},
|
||||
"pr-to-video": {
|
||||
"hash": "148de3a4692bc44d",
|
||||
"hash": "b1473fb853c2ca2c",
|
||||
"files": 33
|
||||
},
|
||||
"product-launch-video": {
|
||||
"hash": "104014221c41ae13",
|
||||
"hash": "6677e7785909bf98",
|
||||
"files": 34
|
||||
},
|
||||
"remotion-to-hyperframes": {
|
||||
|
||||
@@ -173,6 +173,8 @@ function runGenerate(argv) {
|
||||
: { mode: "retrieve", query, blob: g.message || "", arc: g.arc || "" },
|
||||
};
|
||||
if (userVoice) request.voice = userVoice;
|
||||
if (flag(argv, "tts-model", null)) request.tts_model = flag(argv, "tts-model", null);
|
||||
if (flag(argv, "style", null)) request.style = flag(argv, "style", null);
|
||||
|
||||
const neutral = neutralPath(outPath);
|
||||
runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die);
|
||||
|
||||
@@ -14,9 +14,14 @@ node skills/media-use/audio/scripts/heygen-tts.mjs \
|
||||
|
||||
`narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass.
|
||||
|
||||
## Path B — ElevenLabs / Kokoro (TTS → Whisper)
|
||||
## Path B — Gemini / ElevenLabs / Kokoro (TTS → transcription)
|
||||
|
||||
These providers don't return word data. Generate the audio, then transcribe:
|
||||
These adapters supply audio without word data. The shared audio engine runs
|
||||
transcription automatically when timings are absent. For Gemini, use the
|
||||
request in [Text to speech](tts.md#gemini-narration), then consume
|
||||
`audio_meta.json` → `voices[].words`.
|
||||
|
||||
For a standalone local Kokoro generation, generate the audio, then transcribe:
|
||||
|
||||
```bash
|
||||
npx hyperframes tts script.txt --voice af_heart --output narration.wav
|
||||
@@ -24,3 +29,7 @@ npx hyperframes transcribe narration.wav --model small.en # voice af_heart is
|
||||
```
|
||||
|
||||
Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language <code>` otherwise). Then consume `transcript.json` via the caption references in `captions/`.
|
||||
|
||||
For Gemini, verify that transcription preserved the script, especially names,
|
||||
numbers, and delivery pauses. If `words` is empty, resolve the transcription
|
||||
failure before captioning. Generate and align again after changing the read.
|
||||
|
||||
@@ -36,6 +36,10 @@ Use another voice only for a documented reason, and write the reason down.
|
||||
|
||||
## Available routes
|
||||
|
||||
Gemini is an explicit alternative to the automatic provider order below. A
|
||||
request to use Gemini already chooses the provider; do not redirect that user
|
||||
to HeyGen sign-in. Read the Gemini section for its credential requirement.
|
||||
|
||||
| Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format |
|
||||
| ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- |
|
||||
| 1 | HeyGen (Starfish) | `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish` | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg |
|
||||
@@ -83,12 +87,86 @@ node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish vo
|
||||
|
||||
## When to use which provider
|
||||
|
||||
| Goal | Use |
|
||||
| --------------------------------------------------------- | --------------------------------------------------- |
|
||||
| Best voice quality + word timestamps in one call | **HeyGen** |
|
||||
| Drop-in cloud TTS, big voice catalog | **ElevenLabs** |
|
||||
| Offline, no API key, fast iteration | **Kokoro** |
|
||||
| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |
|
||||
| Goal | Use |
|
||||
| --------------------------------------------------------- | --------------------------------------------------------- |
|
||||
| Best voice quality + word timestamps in one call | **HeyGen** |
|
||||
| Drop-in cloud TTS, big voice catalog | **ElevenLabs** |
|
||||
| Offline, no API key, fast iteration | **Kokoro** |
|
||||
| Directed delivery with Gemini prebuilt or custom voices | **Gemini** (explicit selection; transcription for timing) |
|
||||
| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |
|
||||
|
||||
## Gemini narration
|
||||
|
||||
Use the shared audio engine, not `hyperframes tts`. Authenticate with either:
|
||||
|
||||
- `GEMINI_API_KEY` or `GOOGLE_API_KEY` (first set key wins).
|
||||
- A service-account JSON file at `GOOGLE_APPLICATION_CREDENTIALS`, or injected
|
||||
JSON in `GCS_CREDS`. Install `google-auth requests` in the Python 3 environment
|
||||
used by the helper. A configured file takes precedence over injected JSON.
|
||||
|
||||
API keys take precedence over service accounts: unset both key variables to use
|
||||
OAuth. Never put credentials in a request file or composition. The helper
|
||||
obtains a fresh OAuth token for each generation with the
|
||||
`generative-language.retriever` scope. The quota project resolves from
|
||||
`GOOGLE_CLOUD_PROJECT`, then `GCLOUD_PROJECT_ID`, then the service-account JSON.
|
||||
User ADC files and metadata-server authentication are not supported.
|
||||
|
||||
Both routes call the Gemini Developer Interactions API, not Cloud TTS or Vertex
|
||||
AI. Those APIs have separate model catalogs and access requirements.
|
||||
|
||||
Save this as `audio_request.json` in the project:
|
||||
|
||||
```json
|
||||
{
|
||||
"provider": "gemini",
|
||||
"tts_model": "gemini-3.8-flash-tts",
|
||||
"voice": "Kore",
|
||||
"lang": "en",
|
||||
"style": "Warm, clear, conversational. Leave a short pause between sentences.",
|
||||
"lines": [
|
||||
{ "id": "intro", "text": "Every word has a moment. Let the picture follow the voice." }
|
||||
],
|
||||
"bgm": { "mode": "none" }
|
||||
}
|
||||
```
|
||||
|
||||
```bash
|
||||
node <SKILL_DIR>/audio/scripts/audio.mjs \
|
||||
--request ./audio_request.json --hyperframes . --out ./audio_meta.json --only tts
|
||||
```
|
||||
|
||||
The engine saves `assets/voice/intro.wav`, measures its duration, and transcribes
|
||||
it into `voices[].words` in `audio_meta.json`. Check that every requested line
|
||||
has audio and nonempty word timings before building a captioned video. Review
|
||||
the timings against the actual audio; transcription is estimated alignment,
|
||||
not native TTS timestamps. Do not distribute words evenly across a clip.
|
||||
|
||||
- **Models:** `gemini-3.8-flash-tts` (default), `gemini-3.8-flash-lite-tts`,
|
||||
`gemini-3.1-flash-tts-preview`, `gemini-2.5-pro-preview-tts`, and
|
||||
`gemini-2.5-flash-preview-tts`. Use these exact Developer API IDs; Cloud TTS
|
||||
aliases such as `gemini-2.5-flash-tts` are not accepted.
|
||||
- **Voice:** `Kore` by default; pass another prebuilt voice or an existing custom
|
||||
voice ID for 3.8. Older models require prebuilt voices. Creating or replicating
|
||||
voices is outside this helper.
|
||||
- **Delivery:** Put directions in `style`, not in spoken `text`. Each line can
|
||||
override `style`. 3.8 uses structured annotations; older models receive a
|
||||
delivery prompt before the transcript. Check that directions were not spoken.
|
||||
Use style for pacing; numeric `speed` must be omitted or 1.
|
||||
- **Audio:** 3.8 returns a complete WAV. Older models return mono 16-bit PCM,
|
||||
which the helper wraps as WAV at the returned sample rate without resampling.
|
||||
No generation calls run during playback or rendering.
|
||||
- **Timing:** This adapter requests no native word timestamps. It uses the
|
||||
existing transcription pass. `lang` selects transcription language; Gemini
|
||||
infers speech language from the text.
|
||||
- **Workflow adapters:** Product-launch, faceless-explainer, and PR-video audio
|
||||
scripts accept `--provider gemini --voice Kore --tts-model gemini-3.8-flash-tts
|
||||
--style "Warm and clear"`.
|
||||
|
||||
An API error is reported as a failed line; an explicitly chosen Gemini voice
|
||||
never silently falls back to another provider. Check the engine's anomalies
|
||||
and output metadata, not only its exit code.
|
||||
|
||||
API contract: [Google's speech generation guide](https://ai.google.dev/gemini-api/docs/speech-generation).
|
||||
|
||||
## ffmpeg requirement
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
// {
|
||||
// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider)
|
||||
// "provider": "auto", // auto|heygen|elevenlabs|kokoro|gemini (override: --provider)
|
||||
// "lang": "en", "speed": 1.0,
|
||||
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
||||
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
||||
@@ -113,7 +113,6 @@ const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0;
|
||||
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
||||
loadEnvFromDir(hyperframesDir);
|
||||
const heygenOK = heygenCredential() !== null;
|
||||
const headers = heygenOK ? heygenAuthHeaders() : null;
|
||||
|
||||
// ── merge base: preserve sections not selected by --only ──────────────────────
|
||||
const audioMeta = openAudioMeta(outPath);
|
||||
@@ -153,6 +152,8 @@ if (only.has("tts") && lines.length) {
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
model: flag("tts-model", request.tts_model),
|
||||
style: flag("style", line.style ?? request.style),
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
@@ -210,7 +211,12 @@ if (only.has("bgm")) {
|
||||
console.error(`· bgm: disabled`);
|
||||
} else if (mode === "retrieve") {
|
||||
try {
|
||||
bgm = await retrieveBgm({ query: request.bgm?.query, headers, hyperframesDir, hasVoice });
|
||||
bgm = await retrieveBgm({
|
||||
query: request.bgm?.query,
|
||||
headers: heygenAuthHeaders(),
|
||||
hyperframesDir,
|
||||
hasVoice,
|
||||
});
|
||||
if (bgm) {
|
||||
bgmFields.bgm_provider = "heygen";
|
||||
bgmFields.bgm_mode = "retrieve";
|
||||
@@ -262,6 +268,7 @@ if (only.has("sfx")) {
|
||||
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
||||
.filter((c) => c.name),
|
||||
);
|
||||
const headers = heygenOK && cues.length ? heygenAuthHeaders() : null;
|
||||
const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir });
|
||||
sfx = res.sfx;
|
||||
anomalies.push(...res.anomalies);
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
// Exercise the actual engine and provider together; only the paid API and
|
||||
// external audio tools are fixtures. This is not a live synthesis/render test.
|
||||
for (const expired of [false, true]) {
|
||||
for (const only of ["tts", "tts,bgm,sfx"]) {
|
||||
test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
const config = join(dir, "heygen");
|
||||
mkdirSync(config);
|
||||
if (expired)
|
||||
writeFileSync(
|
||||
join(config, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: { access_token: "expired-fixture", expires_at: "2000-01-01T00:00:00Z" },
|
||||
}),
|
||||
);
|
||||
const bin = join(dir, "bin");
|
||||
mkdirSync(bin);
|
||||
const preload = join(dir, "fetch.mjs");
|
||||
writeFileSync(
|
||||
preload,
|
||||
`
|
||||
import assert from 'node:assert/strict';
|
||||
globalThis.fetch = async (url, options) => {
|
||||
assert.equal(url, 'https://generativelanguage.googleapis.com/v1beta/interactions');
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, 'gemini-3.8-flash-lite-tts');
|
||||
assert.equal(body.input[0].content[0].annotations[0].style, 'Line direction');
|
||||
const bytes = Buffer.alloc(48);
|
||||
bytes.write('RIFF'); bytes.write('WAVE', 8);
|
||||
return Response.json({status:'completed', steps:[{type:'model_output', content:[
|
||||
{type:'audio', mime_type:'audio/wav', data:bytes.toString('base64')}
|
||||
]}]});
|
||||
};
|
||||
`,
|
||||
);
|
||||
writeFileSync(
|
||||
join(bin, "npx"),
|
||||
`#!${process.execPath}
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const args = process.argv.slice(2);
|
||||
assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', 'assets/voice/intro.wav']);
|
||||
assert.ok(fs.existsSync(args[2]));
|
||||
assert.equal(args[args.indexOf('--model')+1], 'small.en');
|
||||
fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([
|
||||
{text:'Hello',start:0.1,end:0.4}, {text:'there.',start:0.5,end:0.9}
|
||||
]));
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 });
|
||||
writeFileSync(join(dir, ".env"), "");
|
||||
writeFileSync(
|
||||
join(dir, "audio_request.json"),
|
||||
JSON.stringify({
|
||||
provider: "gemini",
|
||||
tts_model: "gemini-3.8-flash-lite-tts",
|
||||
style: "Global direction",
|
||||
lines: [{ id: "intro", text: "Hello there.", style: "Line direction" }],
|
||||
bgm: { mode: "none" },
|
||||
}),
|
||||
);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
preload,
|
||||
new URL("./audio.mjs", import.meta.url).pathname,
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--only",
|
||||
only,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_CONFIG_DIR: config,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
GEMINI_API_KEY: "fixture-key",
|
||||
PATH: `${bin}:${process.env.PATH}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
|
||||
assert.equal(meta.tts_provider, "gemini");
|
||||
assert.equal(meta.voice_id, "Kore");
|
||||
assert.deepEqual(meta.voices, [
|
||||
{
|
||||
id: "intro",
|
||||
path: "assets/voice/intro.wav",
|
||||
duration_s: 1.25,
|
||||
words: [
|
||||
{ id: "w0", text: "Hello", start: 0.1, end: 0.4 },
|
||||
{ id: "w1", text: "there.", start: 0.5, end: 0.9 },
|
||||
],
|
||||
},
|
||||
]);
|
||||
assert.equal(meta.total_duration_s, 1.25);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
export function geminiConfigured() {
|
||||
return Boolean(
|
||||
process.env.GEMINI_API_KEY ||
|
||||
process.env.GOOGLE_API_KEY ||
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
process.env.GCS_CREDS,
|
||||
);
|
||||
}
|
||||
|
||||
export function geminiAuth({ run = spawnSync } = {}) {
|
||||
const key = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
||||
if (key) return { headers: { "x-goog-api-key": key }, secret: key };
|
||||
if (!geminiConfigured()) {
|
||||
throw new Error(
|
||||
"Gemini TTS needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
}
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth.py", import.meta.url)),
|
||||
]);
|
||||
const result = run(cmd, args, { encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 });
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(result.stdout);
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check Python and install google-auth requests",
|
||||
);
|
||||
}
|
||||
if (result.status !== 0 || !data.token) {
|
||||
// Do not echo subprocess output: it could contain credentials.
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check service-account JSON, permissions, and Python packages google-auth requests",
|
||||
);
|
||||
}
|
||||
const headers = { Authorization: `Bearer ${data.token}` };
|
||||
if (data.project) headers["x-goog-user-project"] = data.project;
|
||||
return { headers, secret: data.token };
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Obtain Gemini OAuth headers without persisting credentials or tokens."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
|
||||
def authenticate():
|
||||
try:
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2 import service_account
|
||||
except ImportError:
|
||||
return {"error": "Service-account auth needs Python packages: pip install google-auth requests"}
|
||||
|
||||
try:
|
||||
path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as source:
|
||||
info = json.load(source)
|
||||
else:
|
||||
info = json.loads(os.environ["GCS_CREDS"])
|
||||
if info.get("type") != "service_account":
|
||||
return {"error": "Gemini requires service-account JSON, not user ADC credentials"}
|
||||
# Only exchange credentials with Google's OAuth server.
|
||||
info["token_uri"] = "https://oauth2.googleapis.com/token"
|
||||
credentials = service_account.Credentials.from_service_account_info(
|
||||
info,
|
||||
scopes=[
|
||||
"https://www.googleapis.com/auth/generative-language.retriever",
|
||||
],
|
||||
)
|
||||
credentials.refresh(Request())
|
||||
project = (
|
||||
os.environ.get("GOOGLE_CLOUD_PROJECT")
|
||||
or os.environ.get("GCLOUD_PROJECT_ID")
|
||||
or info.get("project_id")
|
||||
)
|
||||
return {"token": credentials.token, "project": project}
|
||||
except Exception:
|
||||
# Library exceptions can contain credential data. Never forward them.
|
||||
return {"error": "Gemini service-account authentication failed; check credentials, permissions, and network access"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
result = authenticate()
|
||||
print(json.dumps(result))
|
||||
sys.exit(1 if "error" in result else 0)
|
||||
@@ -0,0 +1,86 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { geminiAuth, geminiConfigured } from "./gemini-auth.mjs";
|
||||
import { pickProvider } from "./tts.mjs";
|
||||
|
||||
function env(t) {
|
||||
const saved = { ...process.env };
|
||||
for (const key of [
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"GCS_CREDS",
|
||||
])
|
||||
delete process.env[key];
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
});
|
||||
}
|
||||
|
||||
test("API keys take precedence over service accounts without invoking Python", (t) => {
|
||||
env(t);
|
||||
process.env.GEMINI_API_KEY = "gemini-key";
|
||||
process.env.GOOGLE_API_KEY = "google-key";
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
const run = () => assert.fail("must not spawn Python");
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "gemini-key" });
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "google-key" });
|
||||
});
|
||||
|
||||
test("service-account configuration selects Gemini and returns scoped helper headers", (t) => {
|
||||
env(t);
|
||||
assert.equal(geminiConfigured(), false);
|
||||
assert.throws(() => geminiAuth(), /needs GEMINI_API_KEY/);
|
||||
for (const key of ["GOOGLE_APPLICATION_CREDENTIALS", "GCS_CREDS"]) {
|
||||
process.env[key] = "credential-source";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
const auth = geminiAuth({
|
||||
run: (_, args, options) => {
|
||||
assert.ok(args.at(-1).endsWith("gemini-auth.py"));
|
||||
assert.ok(!args.includes("credential-source"));
|
||||
assert.equal(options.timeout, 60000);
|
||||
return {
|
||||
status: 0,
|
||||
stdout: JSON.stringify({ token: "access-token", project: "quota-project" }),
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.deepEqual(auth.headers, {
|
||||
Authorization: "Bearer access-token",
|
||||
"x-goog-user-project": "quota-project",
|
||||
});
|
||||
delete process.env[key];
|
||||
}
|
||||
});
|
||||
|
||||
test("failed, malformed, or missing Python output cannot leak secrets or fall back", (t) => {
|
||||
env(t);
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
for (const result of [
|
||||
{ status: 1, stdout: '{"error":"private-json"}', stderr: "private-json" },
|
||||
{ status: null, stdout: "private-json", error: new Error("private-json") },
|
||||
{ status: 0, stdout: "{}" },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => geminiAuth({ run: () => result }),
|
||||
(error) => {
|
||||
assert.match(error.message, /authentication failed/);
|
||||
assert.ok(!error.message.includes("private-json"));
|
||||
return true;
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Keep the Python credential-boundary suite reachable from the normal Node CI runner.
|
||||
test("Python service-account boundary validates credentials, scope, transport and errors", () => {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth_test.py", import.meta.url)),
|
||||
]);
|
||||
const result = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000 });
|
||||
assert.equal(result.status, 0, result.stderr || result.error?.message);
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Credential-boundary tests; stub Google's transport, never use real secrets."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
spec = importlib.util.spec_from_file_location("gemini_auth", Path(__file__).with_name("gemini-auth.py"))
|
||||
auth = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(auth)
|
||||
|
||||
|
||||
class AuthenticationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.info = {"type": "service_account", "project_id": "account-project", "token_uri": "https://untrusted.invalid/token"}
|
||||
self.credentials = Mock(token="fixture-token")
|
||||
self.factory = Mock(return_value=self.credentials)
|
||||
self.request = Mock(return_value=object())
|
||||
transport = types.ModuleType("google.auth.transport.requests")
|
||||
transport.Request = self.request
|
||||
oauth = types.ModuleType("google.oauth2")
|
||||
oauth.service_account = types.SimpleNamespace(Credentials=types.SimpleNamespace(from_service_account_info=self.factory))
|
||||
modules = {name: types.ModuleType(name) for name in ("google", "google.auth", "google.auth.transport")}
|
||||
modules.update({"google.auth.transport.requests": transport, "google.oauth2": oauth})
|
||||
self.addCleanup(patch.stopall)
|
||||
patch.dict("sys.modules", modules).start()
|
||||
patch.dict(os.environ, {"GCS_CREDS": json.dumps(self.info)}, clear=True).start()
|
||||
|
||||
def test_scope_endpoint_and_refresh(self):
|
||||
self.assertEqual(auth.authenticate(), {"token": "fixture-token", "project": "account-project"})
|
||||
info = self.factory.call_args.args[0]
|
||||
self.assertEqual(info["token_uri"], "https://oauth2.googleapis.com/token")
|
||||
self.assertEqual(self.factory.call_args.kwargs["scopes"], ["https://www.googleapis.com/auth/generative-language.retriever"])
|
||||
self.credentials.refresh.assert_called_once_with(self.request.return_value)
|
||||
|
||||
def test_file_takes_precedence_over_json_environment(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "service.json"
|
||||
path.write_text(json.dumps({**self.info, "project_id": "file-project"}), encoding="utf-8")
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path)
|
||||
os.environ["GCS_CREDS"] = "invalid-json"
|
||||
self.assertEqual(auth.authenticate()["project"], "file-project")
|
||||
|
||||
def test_quota_project_precedence(self):
|
||||
os.environ["GCLOUD_PROJECT_ID"] = "alias-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "alias-project")
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "explicit-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "explicit-project")
|
||||
|
||||
def test_user_adc_is_rejected_before_exchange(self):
|
||||
os.environ["GCS_CREDS"] = json.dumps({"type": "authorized_user"})
|
||||
self.assertIn("service-account JSON", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_invalid_json_or_missing_file_does_not_fall_back(self):
|
||||
os.environ["GCS_CREDS"] = "private-invalid-json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
os.environ["GCS_CREDS"] = json.dumps(self.info)
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/missing/service-account-fixture.json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_library_errors_are_redacted(self):
|
||||
for boundary in (self.factory, self.credentials.refresh):
|
||||
with self.subTest(boundary=boundary):
|
||||
boundary.side_effect = RuntimeError("private-key-and-token")
|
||||
result = auth.authenticate()
|
||||
self.assertIn("error", result)
|
||||
self.assertNotIn("private-key-and-token", json.dumps(result))
|
||||
boundary.side_effect = None
|
||||
|
||||
def test_missing_dependency_has_actionable_error(self):
|
||||
with patch.dict("sys.modules", {"google.auth.transport.requests": None}):
|
||||
self.assertIn("pip install google-auth requests", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { geminiAuth } from "./gemini-auth.mjs";
|
||||
|
||||
export const GEMINI_TTS_MODEL = "gemini-3.8-flash-tts";
|
||||
|
||||
export const GEMINI_TTS_MODELS = [
|
||||
GEMINI_TTS_MODEL,
|
||||
"gemini-3.8-flash-lite-tts",
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
];
|
||||
|
||||
// 3.8 returns WAV; older models return PCM that we wrap without resampling.
|
||||
// The shared engine transcribes the saved audio for word timings.
|
||||
export async function synthesizeGemini(
|
||||
{ text, voiceId = "Kore", model = GEMINI_TTS_MODEL, style, speed = 1, wavAbs },
|
||||
{ fetchImpl = fetch, authenticate = geminiAuth } = {},
|
||||
) {
|
||||
let secret;
|
||||
try {
|
||||
if (!GEMINI_TTS_MODELS.includes(model)) {
|
||||
throw new Error(`Unsupported Gemini TTS model: ${model}`);
|
||||
}
|
||||
if (speed !== 1) throw new Error("Gemini TTS uses style for pacing; omit speed or use 1");
|
||||
const modern = model.startsWith("gemini-3.8-");
|
||||
if (!modern && /^(voice_|voicekey_)/.test(voiceId)) {
|
||||
throw new Error(
|
||||
"Custom Gemini voices require a 3.8 TTS model; use a prebuilt voice with older models",
|
||||
);
|
||||
}
|
||||
const auth = authenticate();
|
||||
secret = auth.secret;
|
||||
const content = {
|
||||
type: "text",
|
||||
text:
|
||||
!modern && style ? `Read the following text with this delivery: ${style}\n\n${text}` : text,
|
||||
};
|
||||
if (modern && style) content.annotations = [{ type: "speech_metadata", style }];
|
||||
const response = await fetchImpl(
|
||||
"https://generativelanguage.googleapis.com/v1beta/interactions",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...auth.headers },
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
input: [{ type: "user_input", content: [content] }],
|
||||
response_format: modern ? { type: "audio", mime_type: "audio/wav" } : { type: "audio" },
|
||||
generation_config: { speech_config: [{ voice: voiceId }] },
|
||||
store: false,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const detail = await response.text();
|
||||
throw new Error(`Gemini TTS HTTP ${response.status}: ${detail}`);
|
||||
}
|
||||
const payload = await response.json();
|
||||
if (payload.status !== "completed") {
|
||||
throw new Error(`Gemini TTS did not complete (${payload.status ?? "missing status"})`);
|
||||
}
|
||||
const audio = (payload.steps ?? [])
|
||||
.filter((step) => step.type === "model_output")
|
||||
.flatMap((step) => step.content ?? [])
|
||||
.filter((part) => part.type === "audio");
|
||||
if (audio.length !== 1 || !audio[0].data) {
|
||||
throw new Error("Gemini TTS returned no single audio block");
|
||||
}
|
||||
let bytes = Buffer.from(audio[0].data, "base64");
|
||||
if (!modern && /^audio\/l16(?:;|$)/i.test(audio[0].mime_type ?? "")) {
|
||||
bytes = pcmToWav(bytes, audio[0].mime_type);
|
||||
} else if (audio[0].mime_type !== "audio/wav") {
|
||||
throw new Error("Gemini TTS returned an unsupported audio format");
|
||||
}
|
||||
if (
|
||||
bytes.length <= 44 ||
|
||||
bytes.toString("ascii", 0, 4) !== "RIFF" ||
|
||||
bytes.toString("ascii", 8, 12) !== "WAVE"
|
||||
) {
|
||||
throw new Error("Gemini TTS returned invalid WAV audio");
|
||||
}
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
return { ok: true, words: null };
|
||||
} catch (error) {
|
||||
// Error responses must never echo the credential into logs or metadata.
|
||||
const message = String(error?.message ?? error);
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: secret ? message.split(secret).join("[redacted]") : message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function pcmToWav(pcm, mime) {
|
||||
const rate = Number(/(?:^|;)\s*rate=(\d+)(?:;|$)/i.exec(mime)?.[1]);
|
||||
const channels = /(?:^|;)\s*channels=([^;]+)/i.exec(mime)?.[1];
|
||||
const codec = /(?:^|;)\s*codec=([^;]+)/i.exec(mime)?.[1];
|
||||
if ((channels && channels.trim() !== "1") || (codec && codec.trim() !== "pcm")) {
|
||||
throw new Error("Gemini TTS returned unsupported PCM channels or codec");
|
||||
}
|
||||
if (!Number.isInteger(rate) || rate < 8000 || rate > 96000 || !pcm.length || pcm.length % 2) {
|
||||
throw new Error("Gemini TTS returned invalid PCM audio or sample rate");
|
||||
}
|
||||
const header = Buffer.alloc(44);
|
||||
header.write("RIFF");
|
||||
header.writeUInt32LE(36 + pcm.length, 4);
|
||||
header.write("WAVEfmt ", 8);
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(1, 22);
|
||||
header.writeUInt32LE(rate, 24);
|
||||
header.writeUInt32LE(rate * 2, 28);
|
||||
header.writeUInt16LE(2, 32);
|
||||
header.writeUInt16LE(16, 34);
|
||||
header.write("data", 36);
|
||||
header.writeUInt32LE(pcm.length, 40);
|
||||
return Buffer.concat([header, pcm]);
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { synthesizeGemini, GEMINI_TTS_MODEL } from "./gemini-tts.mjs";
|
||||
import { pickProvider, resolveVoiceId, synthesizeOne } from "./tts.mjs";
|
||||
|
||||
function fixture(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "gemini-tts-"));
|
||||
const saved = { ...process.env };
|
||||
process.env.GEMINI_API_KEY = "test-gemini-key";
|
||||
delete process.env.GOOGLE_API_KEY;
|
||||
delete process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
||||
delete process.env.GCS_CREDS;
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
const wav = Buffer.alloc(48);
|
||||
wav.write("RIFF");
|
||||
wav.writeUInt32LE(40, 4);
|
||||
wav.write("WAVEfmt ", 8);
|
||||
wav.writeUInt32LE(16, 16);
|
||||
wav.writeUInt16LE(1, 20);
|
||||
wav.writeUInt16LE(1, 22);
|
||||
wav.writeUInt32LE(24000, 24);
|
||||
wav.writeUInt32LE(48000, 28);
|
||||
wav.writeUInt16LE(2, 32);
|
||||
wav.writeUInt16LE(16, 34);
|
||||
wav.write("data", 36);
|
||||
wav.writeUInt32LE(4, 40);
|
||||
const audio = { type: "audio", mime_type: "audio/wav", data: wav.toString("base64") };
|
||||
const payload = { status: "completed", steps: [{ type: "model_output", content: [audio] }] };
|
||||
return {
|
||||
wav,
|
||||
audio,
|
||||
payload,
|
||||
args: { text: "Hello there.", voiceId: "Kore", wavAbs: join(dir, "voice", "one.wav") },
|
||||
};
|
||||
}
|
||||
|
||||
test("Gemini preserves verbatim text, directs style separately, saves WAV and requests transcription", async (t) => {
|
||||
const { wav, payload, args } = fixture(t);
|
||||
t.mock.method(globalThis, "fetch", async (url, options) => {
|
||||
assert.equal(url, "https://generativelanguage.googleapis.com/v1beta/interactions");
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-gemini-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, GEMINI_TTS_MODEL);
|
||||
assert.equal(body.store, false);
|
||||
assert.deepEqual(body.input[0].content, [
|
||||
{
|
||||
type: "text",
|
||||
text: args.text,
|
||||
annotations: [{ type: "speech_metadata", style: "Warm and calm" }],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(body.generation_config.speech_config, [{ voice: "Kore" }]);
|
||||
assert.equal(body.response_format.mime_type, "audio/wav");
|
||||
return Response.json(payload);
|
||||
});
|
||||
const result = await synthesizeOne({ ...args, provider: "gemini", style: "Warm and calm" });
|
||||
assert.deepEqual(result, { ok: true, words: null });
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
|
||||
test("Flash-Lite and GOOGLE_API_KEY work without a Gemini key", async (t) => {
|
||||
const { payload, args } = fixture(t);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
process.env.GOOGLE_API_KEY = "test-google-key";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
assert.equal(await resolveVoiceId({ provider: "gemini" }), "Kore");
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.8-flash-lite-tts" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-google-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, "gemini-3.8-flash-lite-tts");
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
});
|
||||
|
||||
test("Gemini remains opt-in even with a key and preserves explicitly chosen voices", async (t) => {
|
||||
fixture(t);
|
||||
assert.notEqual(pickProvider(), "gemini");
|
||||
assert.equal(
|
||||
await resolveVoiceId({ provider: "gemini", userVoice: "voice_custom" }),
|
||||
"voice_custom",
|
||||
);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.throws(() => pickProvider("gemini"), /GEMINI_API_KEY or GOOGLE_API_KEY/);
|
||||
});
|
||||
|
||||
test("invalid config fails before spending a generation request", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const deps = { fetchImpl: () => assert.fail("must not call the API") };
|
||||
for (const change of [{ model: "gemini-2.5-flash" }, { speed: 1.2 }]) {
|
||||
const result = await synthesizeGemini({ ...args, ...change }, deps);
|
||||
assert.equal(result.ok, false);
|
||||
assert.ok(result.error);
|
||||
}
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.match((await synthesizeGemini(args, deps)).error, /needs GEMINI_API_KEY/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("HTTP and network failures stay actionable without leaking the key", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const result = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => new Response("quota exceeded test-gemini-key", { status: 429 }),
|
||||
});
|
||||
assert.equal(result.ok, false);
|
||||
assert.match(result.error, /HTTP 429: quota exceeded/);
|
||||
assert.ok(!result.error.includes("test-gemini-key"));
|
||||
const timeout = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => {
|
||||
throw new Error("timed out");
|
||||
},
|
||||
});
|
||||
assert.match(timeout.error, /timed out/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("incomplete, missing, raw PCM and malformed audio cannot become a successful WAV", async (t) => {
|
||||
const { args, payload, audio } = fixture(t);
|
||||
const cases = [
|
||||
{ ...payload, status: "incomplete" },
|
||||
{ status: "completed", steps: [] },
|
||||
...[{ ...audio, mime_type: "audio/l16" }, { ...audio, data: "not audio" }, audio].map(
|
||||
(part, i) => ({
|
||||
status: "completed",
|
||||
steps: [{ type: "model_output", content: i === 2 ? [part, part] : [part] }],
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (const body of cases) {
|
||||
const result = await synthesizeGemini(args, { fetchImpl: async () => Response.json(body) });
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
});
|
||||
|
||||
for (const model of [
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
]) {
|
||||
test(`${model} uses legacy delivery prompts and wraps PCM as mono WAV`, async (t) => {
|
||||
const { args, wav } = fixture(t);
|
||||
const pcm = wav.subarray(44);
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model, style: "Warm and clear" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
const body = JSON.parse(options.body);
|
||||
assert.deepEqual(body.response_format, { type: "audio" });
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
assert.ok(body.input[0].content[0].text.endsWith(args.text));
|
||||
assert.ok(body.input[0].content[0].text.includes("Warm and clear"));
|
||||
return Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [
|
||||
{
|
||||
type: "audio",
|
||||
mime_type: "audio/L16;codec=pcm;rate=24000",
|
||||
data: pcm.toString("base64"),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true, result.error);
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
}
|
||||
|
||||
test("service-account bearer and quota project reach synthesis, and token errors are redacted", async (t) => {
|
||||
const { args, payload } = fixture(t);
|
||||
const authenticate = () => ({
|
||||
headers: { Authorization: "Bearer secret-token", "x-goog-user-project": "test-project" },
|
||||
secret: "secret-token",
|
||||
});
|
||||
const ok = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers.Authorization, "Bearer secret-token");
|
||||
assert.equal(options.headers["x-goog-user-project"], "test-project");
|
||||
assert.equal(options.headers["x-goog-api-key"], undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
});
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async () => new Response("denied secret-token", { status: 403 }),
|
||||
});
|
||||
assert.match(failed.error, /HTTP 403/);
|
||||
assert.ok(!failed.error.includes("secret-token"));
|
||||
});
|
||||
|
||||
test("older models reject unsupported PCM and custom voices", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
for (const mime of [
|
||||
"audio/l16",
|
||||
"audio/l16;rate=0",
|
||||
"audio/l16;rate=24000;channels=2",
|
||||
"audio/l16;rate=24000;codec=other",
|
||||
]) {
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.1-flash-tts-preview" },
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [{ type: "audio", mime_type: mime, data: "AAAAAA==" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-2.5-pro-preview-tts", voiceId: "voice_custom" },
|
||||
{ fetchImpl: () => assert.fail("must not generate") },
|
||||
);
|
||||
assert.match(result.error, /Custom Gemini voices require/);
|
||||
});
|
||||
@@ -20,6 +20,8 @@ import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { synthesizeGemini } from "./gemini-tts.mjs";
|
||||
import { geminiConfigured } from "./gemini-auth.mjs";
|
||||
|
||||
// ── provider detection ────────────────────────────────────────────────────────
|
||||
export function heygenAvailable() {
|
||||
@@ -37,8 +39,12 @@ export function elevenlabsAvailable() {
|
||||
// First available provider wins; an explicit choice is honored (and validated).
|
||||
export function pickProvider(userProvider) {
|
||||
if (userProvider) {
|
||||
if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider))
|
||||
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`);
|
||||
if (!["heygen", "elevenlabs", "kokoro", "gemini"].includes(userProvider))
|
||||
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro | gemini)`);
|
||||
if (userProvider === "gemini" && !geminiConfigured())
|
||||
throw new Error(
|
||||
"provider=gemini needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
if (userProvider === "heygen" && !heygenAvailable())
|
||||
throw new Error(
|
||||
"provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)",
|
||||
@@ -56,6 +62,7 @@ export function pickProvider(userProvider) {
|
||||
// their own defaults.
|
||||
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
|
||||
if (userVoice) return userVoice;
|
||||
if (provider === "gemini") return "Kore";
|
||||
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
|
||||
if (provider === "kokoro") {
|
||||
if (lang === "en") return "am_michael";
|
||||
@@ -248,10 +255,14 @@ export async function synthesizeOne({
|
||||
voiceId,
|
||||
lang = "en",
|
||||
speed = 1.0,
|
||||
model,
|
||||
style,
|
||||
wavAbs,
|
||||
hyperframesDir,
|
||||
}) {
|
||||
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
|
||||
if (provider === "gemini")
|
||||
return synthesizeGemini({ text, voiceId, model, style, speed, wavAbs });
|
||||
if (provider === "elevenlabs") {
|
||||
// The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
|
||||
// and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir,
|
||||
|
||||
@@ -9,7 +9,8 @@ shot), use the shared engine at `audio/scripts/audio.mjs`. It takes a neutral
|
||||
node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./audio_meta.json
|
||||
```
|
||||
|
||||
- **Request** `{ provider?, lang?, speed?, lines: [{ id, text, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
|
||||
- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
|
||||
- **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged.
|
||||
- **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
|
||||
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.
|
||||
- If BGM took the generate path (`bgm_pending: true`), run `audio/scripts/wait-bgm.mjs` before final render.
|
||||
|
||||
@@ -173,6 +173,8 @@ function runGenerate(argv) {
|
||||
: { mode: "retrieve", query, blob: g.message || "", arc: g.arc || "" },
|
||||
};
|
||||
if (userVoice) request.voice = userVoice;
|
||||
if (flag(argv, "tts-model", null)) request.tts_model = flag(argv, "tts-model", null);
|
||||
if (flag(argv, "style", null)) request.style = flag(argv, "style", null);
|
||||
|
||||
const neutral = neutralPath(outPath);
|
||||
runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die);
|
||||
|
||||
@@ -172,6 +172,8 @@ function runGenerate(argv) {
|
||||
: { mode: "retrieve", query, blob: g.message || "", arc: g.arc || "" },
|
||||
};
|
||||
if (userVoice) request.voice = userVoice;
|
||||
if (flag(argv, "tts-model", null)) request.tts_model = flag(argv, "tts-model", null);
|
||||
if (flag(argv, "style", null)) request.style = flag(argv, "style", null);
|
||||
|
||||
const neutral = neutralPath(outPath);
|
||||
runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die);
|
||||
|
||||
@@ -34,6 +34,22 @@ test("passes --provider to the shared audio engine", () => {
|
||||
assert.equal(runAudio({ args: ["--provider", "kokoro"] }).provider, "kokoro");
|
||||
});
|
||||
|
||||
test("passes Gemini model and delivery style through the workflow adapter", () => {
|
||||
const request = runAudio({
|
||||
args: [
|
||||
"--provider",
|
||||
"gemini",
|
||||
"--tts-model",
|
||||
"gemini-3.8-flash-lite-tts",
|
||||
"--style",
|
||||
"Warm and calm",
|
||||
],
|
||||
});
|
||||
assert.equal(request.provider, "gemini");
|
||||
assert.equal(request.tts_model, "gemini-3.8-flash-lite-tts");
|
||||
assert.equal(request.style, "Warm and calm");
|
||||
});
|
||||
|
||||
test("uses HF_TTS_PROVIDER when --provider is omitted", () => {
|
||||
assert.equal(runAudio({ env: { HF_TTS_PROVIDER: "elevenlabs" } }).provider, "elevenlabs");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user