From 01601d110569967adbfe998e81f77bdeec4b1f2d Mon Sep 17 00:00:00 2001 From: James Russo Date: Thu, 24 Sep 2026 02:21:23 +0100 Subject: [PATCH] feat: add Gemini TTS models and service-account auth to media skills (#4377) * feat: add opt-in Gemini 3.8 TTS to media skills * feat: support Gemini service accounts and older TTS models * fix(audio): isolate Gemini auth and narrow OAuth scope * fix(audio): pin Gemini Python helpers to UTF-8 --- docs/guides/authentication.mdx | 34 ++- docs/prompting/media-and-audio.mdx | 20 ++ skills-manifest.json | 10 +- skills/faceless-explainer/scripts/audio.mjs | 2 + .../audio/references/tts-to-captions.md | 13 +- skills/media-use/audio/references/tts.md | 90 ++++++- skills/media-use/audio/scripts/audio.mjs | 13 +- .../audio/scripts/gemini-pipeline.test.mjs | 113 ++++++++ .../audio/scripts/lib/gemini-auth.mjs | 43 ++++ .../audio/scripts/lib/gemini-auth.py | 52 ++++ .../audio/scripts/lib/gemini-auth.test.mjs | 86 +++++++ .../audio/scripts/lib/gemini-auth_test.py | 90 +++++++ .../audio/scripts/lib/gemini-tts.mjs | 122 +++++++++ .../audio/scripts/lib/gemini-tts.test.mjs | 243 ++++++++++++++++++ skills/media-use/audio/scripts/lib/tts.mjs | 15 +- skills/media-use/references/audio.md | 3 +- skills/pr-to-video/scripts/audio.mjs | 2 + skills/product-launch-video/scripts/audio.mjs | 2 + .../scripts/audio.test.mjs | 16 ++ 19 files changed, 948 insertions(+), 21 deletions(-) create mode 100644 skills/media-use/audio/scripts/gemini-pipeline.test.mjs create mode 100644 skills/media-use/audio/scripts/lib/gemini-auth.mjs create mode 100644 skills/media-use/audio/scripts/lib/gemini-auth.py create mode 100644 skills/media-use/audio/scripts/lib/gemini-auth.test.mjs create mode 100644 skills/media-use/audio/scripts/lib/gemini-auth_test.py create mode 100644 skills/media-use/audio/scripts/lib/gemini-tts.mjs create mode 100644 skills/media-use/audio/scripts/lib/gemini-tts.test.mjs diff --git a/docs/guides/authentication.mdx b/docs/guides/authentication.mdx index 31650ef2d0..93564714ce 100644 --- a/docs/guides/authentication.mdx +++ b/docs/guides/authentication.mdx @@ -74,9 +74,37 @@ Run `npx hyperframes doctor` to check which local dependencies are installed. The media workflows run `hyperframes auth status` before generation and tell you which path they will use. +For Gemini narration, explicitly ask your agent to use Gemini TTS. The shared +audio workflow supports 3.8 Flash and Flash-Lite, 3.1 Flash Preview, and 2.5 Pro +and Flash Preview. Gemini remains an explicit choice; configuring credentials +does not change the automatic voice-provider order above. + +Choose one authentication method: + +- **API key:** set `GEMINI_API_KEY` or `GOOGLE_API_KEY` in your environment or + project `.env`. This path needs no Python dependencies. +- **Service account:** install `google-auth` and `requests` in your Python 3 + environment, then set `GOOGLE_APPLICATION_CREDENTIALS` to the absolute path + of your service-account JSON file. Alternatively, inject the JSON through + `GCS_CREDS` from your secret manager. Set `GOOGLE_CLOUD_PROJECT` to the quota + project when it differs from the service account's project. + +Credentials resolve in that order: Gemini key, Google key, service-account file, +then JSON environment variable. Unset API keys when choosing service-account +OAuth. The helper obtains a fresh token for each generation using the +`generative-language.retriever` scope. Keys and tokens +are never placed in compositions or saved audio metadata. + +Both methods call the Gemini Developer Interactions API. Your project needs +access to that API and the requested model. Cloud Text-to-Speech and Vertex AI +have separate model availability. This helper accepts service-account JSON, +not user ADC files or metadata-server credentials. + +See [Gemini voiceover](/prompting/media-and-audio#gemini-voiceover). + `npx hyperframes tts` itself is the local Kokoro CLI. Hosted HeyGen and - ElevenLabs voices are selected by the bundled media workflow helpers, not by + ElevenLabs and Gemini voices are selected by the bundled media workflow helpers, not by that command. @@ -112,7 +140,9 @@ to a shared space with `--space`. | `HEYGEN_API_URL` | API base URL (default `https://api.heygen.com`). | | `HEYGEN_CONFIG_DIR` | Credentials directory (default `~/.heygen`). | | `ELEVENLABS_API_KEY` | ElevenLabs TTS, used when no HeyGen credential is present. | -| `GEMINI_API_KEY` / `GOOGLE_API_KEY` | Lyria music generation (and capture descriptions). | +| `GEMINI_API_KEY` / `GOOGLE_API_KEY` | Explicit Gemini TTS selection and Lyria music generation; capture descriptions use `GEMINI_API_KEY`. | +| `GOOGLE_APPLICATION_CREDENTIALS` / `GCS_CREDS` | Gemini TTS service-account JSON file path / injected JSON. Used when neither API key is set. | +| `GOOGLE_CLOUD_PROJECT` / `GCLOUD_PROJECT_ID` | Gemini OAuth quota project; first set value wins, then the service account project. | | `OPENROUTER_API_KEY` | Capture descriptions; takes priority over Gemini for that step. | ## Related topics diff --git a/docs/prompting/media-and-audio.mdx b/docs/prompting/media-and-audio.mdx index 027a43bf77..9796342208 100644 --- a/docs/prompting/media-and-audio.mdx +++ b/docs/prompting/media-and-audio.mdx @@ -38,6 +38,26 @@ agent choose. - ❌ `add a voice` - ✅ `warm, unhurried female narration of the quoted script` — tone and pace are what actually change the delivery +### Gemini voiceover + +Choose Gemini when you want to direct a narrator's delivery with a description. +Configure [Gemini credentials](/guides/authentication#providers-used-by-agent-workflows), +then ask your agent: + +> Use Gemini 3.8 Flash TTS with the Kore voice to narrate this script. Make the +> delivery warm and unhurried. Use the media-use audio engine, transcribe the +> generated audio, and sync the captions and visual reveals to the spoken words. + +The shared audio workflow supports 3.8 Flash and Flash-Lite TTS, +3.1 Flash TTS Preview, and 2.5 Pro and Flash TTS Preview. Gemini is an explicit choice; it does not replace +the automatic provider order above. Keep the script separate from delivery +instructions. The helper uses structured style metadata for 3.8 and a delivery +prompt for older models; review the generated speech for accidental directions. + +The agent saves the voiceover locally and obtains word timings through a +separate transcription pass. Check names, numbers, and pauses against the +audio before the final render. A new narration take needs new timings. + ## Background music Music resolves from a large catalog by mood. It should almost always sit *under* diff --git a/skills-manifest.json b/skills-manifest.json index 31830d85aa..d185315316 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -6,7 +6,7 @@ "files": 142 }, "faceless-explainer": { - "hash": "d3e32414b9e86678", + "hash": "995968cc3ba6faee", "files": 27 }, "figma": { @@ -54,8 +54,8 @@ "files": 1 }, "media-use": { - "hash": "d15857ed6f884a1f", - "files": 91 + "hash": "b35de041b87e763d", + "files": 98 }, "motion-graphics": { "hash": "ab9d2e9c205ce1b1", @@ -66,11 +66,11 @@ "files": 169 }, "pr-to-video": { - "hash": "148de3a4692bc44d", + "hash": "b1473fb853c2ca2c", "files": 33 }, "product-launch-video": { - "hash": "104014221c41ae13", + "hash": "6677e7785909bf98", "files": 34 }, "remotion-to-hyperframes": { diff --git a/skills/faceless-explainer/scripts/audio.mjs b/skills/faceless-explainer/scripts/audio.mjs index 84a878a97a..e2e22d585e 100644 --- a/skills/faceless-explainer/scripts/audio.mjs +++ b/skills/faceless-explainer/scripts/audio.mjs @@ -173,6 +173,8 @@ function runGenerate(argv) { : { mode: "retrieve", query, blob: g.message || "", arc: g.arc || "" }, }; if (userVoice) request.voice = userVoice; + if (flag(argv, "tts-model", null)) request.tts_model = flag(argv, "tts-model", null); + if (flag(argv, "style", null)) request.style = flag(argv, "style", null); const neutral = neutralPath(outPath); runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die); diff --git a/skills/media-use/audio/references/tts-to-captions.md b/skills/media-use/audio/references/tts-to-captions.md index e11262cae2..4b99fa0a8c 100644 --- a/skills/media-use/audio/references/tts-to-captions.md +++ b/skills/media-use/audio/references/tts-to-captions.md @@ -14,9 +14,14 @@ node skills/media-use/audio/scripts/heygen-tts.mjs \ `narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass. -## Path B — ElevenLabs / Kokoro (TTS → Whisper) +## Path B — Gemini / ElevenLabs / Kokoro (TTS → transcription) -These providers don't return word data. Generate the audio, then transcribe: +These adapters supply audio without word data. The shared audio engine runs +transcription automatically when timings are absent. For Gemini, use the +request in [Text to speech](tts.md#gemini-narration), then consume +`audio_meta.json` → `voices[].words`. + +For a standalone local Kokoro generation, generate the audio, then transcribe: ```bash npx hyperframes tts script.txt --voice af_heart --output narration.wav @@ -24,3 +29,7 @@ npx hyperframes transcribe narration.wav --model small.en # voice af_heart is ``` Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language ` otherwise). Then consume `transcript.json` via the caption references in `captions/`. + +For Gemini, verify that transcription preserved the script, especially names, +numbers, and delivery pauses. If `words` is empty, resolve the transcription +failure before captioning. Generate and align again after changing the read. diff --git a/skills/media-use/audio/references/tts.md b/skills/media-use/audio/references/tts.md index d9fbd43349..468ca78a14 100644 --- a/skills/media-use/audio/references/tts.md +++ b/skills/media-use/audio/references/tts.md @@ -36,6 +36,10 @@ Use another voice only for a documented reason, and write the reason down. ## Available routes +Gemini is an explicit alternative to the automatic provider order below. A +request to use Gemini already chooses the provider; do not redirect that user +to HeyGen sign-in. Read the Gemini section for its credential requirement. + | Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format | | ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- | | 1 | HeyGen (Starfish) | `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish` | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg | @@ -83,12 +87,86 @@ node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish vo ## When to use which provider -| Goal | Use | -| --------------------------------------------------------- | --------------------------------------------------- | -| Best voice quality + word timestamps in one call | **HeyGen** | -| Drop-in cloud TTS, big voice catalog | **ElevenLabs** | -| Offline, no API key, fast iteration | **Kokoro** | -| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) | +| Goal | Use | +| --------------------------------------------------------- | --------------------------------------------------------- | +| Best voice quality + word timestamps in one call | **HeyGen** | +| Drop-in cloud TTS, big voice catalog | **ElevenLabs** | +| Offline, no API key, fast iteration | **Kokoro** | +| Directed delivery with Gemini prebuilt or custom voices | **Gemini** (explicit selection; transcription for timing) | +| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) | + +## Gemini narration + +Use the shared audio engine, not `hyperframes tts`. Authenticate with either: + +- `GEMINI_API_KEY` or `GOOGLE_API_KEY` (first set key wins). +- A service-account JSON file at `GOOGLE_APPLICATION_CREDENTIALS`, or injected + JSON in `GCS_CREDS`. Install `google-auth requests` in the Python 3 environment + used by the helper. A configured file takes precedence over injected JSON. + +API keys take precedence over service accounts: unset both key variables to use +OAuth. Never put credentials in a request file or composition. The helper +obtains a fresh OAuth token for each generation with the +`generative-language.retriever` scope. The quota project resolves from +`GOOGLE_CLOUD_PROJECT`, then `GCLOUD_PROJECT_ID`, then the service-account JSON. +User ADC files and metadata-server authentication are not supported. + +Both routes call the Gemini Developer Interactions API, not Cloud TTS or Vertex +AI. Those APIs have separate model catalogs and access requirements. + +Save this as `audio_request.json` in the project: + +```json +{ + "provider": "gemini", + "tts_model": "gemini-3.8-flash-tts", + "voice": "Kore", + "lang": "en", + "style": "Warm, clear, conversational. Leave a short pause between sentences.", + "lines": [ + { "id": "intro", "text": "Every word has a moment. Let the picture follow the voice." } + ], + "bgm": { "mode": "none" } +} +``` + +```bash +node /audio/scripts/audio.mjs \ + --request ./audio_request.json --hyperframes . --out ./audio_meta.json --only tts +``` + +The engine saves `assets/voice/intro.wav`, measures its duration, and transcribes +it into `voices[].words` in `audio_meta.json`. Check that every requested line +has audio and nonempty word timings before building a captioned video. Review +the timings against the actual audio; transcription is estimated alignment, +not native TTS timestamps. Do not distribute words evenly across a clip. + +- **Models:** `gemini-3.8-flash-tts` (default), `gemini-3.8-flash-lite-tts`, + `gemini-3.1-flash-tts-preview`, `gemini-2.5-pro-preview-tts`, and + `gemini-2.5-flash-preview-tts`. Use these exact Developer API IDs; Cloud TTS + aliases such as `gemini-2.5-flash-tts` are not accepted. +- **Voice:** `Kore` by default; pass another prebuilt voice or an existing custom + voice ID for 3.8. Older models require prebuilt voices. Creating or replicating + voices is outside this helper. +- **Delivery:** Put directions in `style`, not in spoken `text`. Each line can + override `style`. 3.8 uses structured annotations; older models receive a + delivery prompt before the transcript. Check that directions were not spoken. + Use style for pacing; numeric `speed` must be omitted or 1. +- **Audio:** 3.8 returns a complete WAV. Older models return mono 16-bit PCM, + which the helper wraps as WAV at the returned sample rate without resampling. + No generation calls run during playback or rendering. +- **Timing:** This adapter requests no native word timestamps. It uses the + existing transcription pass. `lang` selects transcription language; Gemini + infers speech language from the text. +- **Workflow adapters:** Product-launch, faceless-explainer, and PR-video audio + scripts accept `--provider gemini --voice Kore --tts-model gemini-3.8-flash-tts +--style "Warm and clear"`. + +An API error is reported as a failed line; an explicitly chosen Gemini voice +never silently falls back to another provider. Check the engine's anomalies +and output metadata, not only its exit code. + +API contract: [Google's speech generation guide](https://ai.google.dev/gemini-api/docs/speech-generation). ## ffmpeg requirement diff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjs index 167eb679ca..a7dc6d93b2 100644 --- a/skills/media-use/audio/scripts/audio.mjs +++ b/skills/media-use/audio/scripts/audio.mjs @@ -15,7 +15,7 @@ // // ── audio_request.json (input) ──────────────────────────────────────────────── // { -// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider) +// "provider": "auto", // auto|heygen|elevenlabs|kokoro|gemini (override: --provider) // "lang": "en", "speed": 1.0, // "lines": [ // one TTS unit each; id joins back to the caller's model // { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] } @@ -113,7 +113,6 @@ const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0; // ── env + HeyGen availability (the single switch) ───────────────────────────── loadEnvFromDir(hyperframesDir); const heygenOK = heygenCredential() !== null; -const headers = heygenOK ? heygenAuthHeaders() : null; // ── merge base: preserve sections not selected by --only ────────────────────── const audioMeta = openAudioMeta(outPath); @@ -153,6 +152,8 @@ if (only.has("tts") && lines.length) { voiceId, lang, speed, + model: flag("tts-model", request.tts_model), + style: flag("style", line.style ?? request.style), wavAbs: abs, hyperframesDir, }); @@ -210,7 +211,12 @@ if (only.has("bgm")) { console.error(`· bgm: disabled`); } else if (mode === "retrieve") { try { - bgm = await retrieveBgm({ query: request.bgm?.query, headers, hyperframesDir, hasVoice }); + bgm = await retrieveBgm({ + query: request.bgm?.query, + headers: heygenAuthHeaders(), + hyperframesDir, + hasVoice, + }); if (bgm) { bgmFields.bgm_provider = "heygen"; bgmFields.bgm_mode = "retrieve"; @@ -262,6 +268,7 @@ if (only.has("sfx")) { .map((name) => ({ id: String(l.id), name: String(name).trim() })) .filter((c) => c.name), ); + const headers = heygenOK && cues.length ? heygenAuthHeaders() : null; const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir }); sfx = res.sfx; anomalies.push(...res.anomalies); diff --git a/skills/media-use/audio/scripts/gemini-pipeline.test.mjs b/skills/media-use/audio/scripts/gemini-pipeline.test.mjs new file mode 100644 index 0000000000..01231d04cf --- /dev/null +++ b/skills/media-use/audio/scripts/gemini-pipeline.test.mjs @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; + +// Exercise the actual engine and provider together; only the paid API and +// external audio tools are fixtures. This is not a live synthesis/render test. +for (const expired of [false, true]) { + for (const only of ["tts", "tts,bgm,sfx"]) { + test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => { + const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const config = join(dir, "heygen"); + mkdirSync(config); + if (expired) + writeFileSync( + join(config, "credentials"), + JSON.stringify({ + oauth: { access_token: "expired-fixture", expires_at: "2000-01-01T00:00:00Z" }, + }), + ); + const bin = join(dir, "bin"); + mkdirSync(bin); + const preload = join(dir, "fetch.mjs"); + writeFileSync( + preload, + ` +import assert from 'node:assert/strict'; +globalThis.fetch = async (url, options) => { + assert.equal(url, 'https://generativelanguage.googleapis.com/v1beta/interactions'); + const body = JSON.parse(options.body); + assert.equal(body.model, 'gemini-3.8-flash-lite-tts'); + assert.equal(body.input[0].content[0].annotations[0].style, 'Line direction'); + const bytes = Buffer.alloc(48); + bytes.write('RIFF'); bytes.write('WAVE', 8); + return Response.json({status:'completed', steps:[{type:'model_output', content:[ + {type:'audio', mime_type:'audio/wav', data:bytes.toString('base64')} + ]}]}); +}; +`, + ); + writeFileSync( + join(bin, "npx"), + `#!${process.execPath} +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const args = process.argv.slice(2); +assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', 'assets/voice/intro.wav']); +assert.ok(fs.existsSync(args[2])); +assert.equal(args[args.indexOf('--model')+1], 'small.en'); +fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([ + {text:'Hello',start:0.1,end:0.4}, {text:'there.',start:0.5,end:0.9} +])); +`, + { mode: 0o755 }, + ); + writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 }); + writeFileSync(join(dir, ".env"), ""); + writeFileSync( + join(dir, "audio_request.json"), + JSON.stringify({ + provider: "gemini", + tts_model: "gemini-3.8-flash-lite-tts", + style: "Global direction", + lines: [{ id: "intro", text: "Hello there.", style: "Line direction" }], + bgm: { mode: "none" }, + }), + ); + const result = spawnSync( + process.execPath, + [ + "--import", + preload, + new URL("./audio.mjs", import.meta.url).pathname, + "--hyperframes", + dir, + "--only", + only, + ], + { + encoding: "utf8", + env: { + ...process.env, + HEYGEN_CONFIG_DIR: config, + HEYGEN_API_KEY: "", + HYPERFRAMES_API_KEY: "", + GEMINI_API_KEY: "fixture-key", + PATH: `${bin}:${process.env.PATH}`, + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8")); + assert.equal(meta.tts_provider, "gemini"); + assert.equal(meta.voice_id, "Kore"); + assert.deepEqual(meta.voices, [ + { + id: "intro", + path: "assets/voice/intro.wav", + duration_s: 1.25, + words: [ + { id: "w0", text: "Hello", start: 0.1, end: 0.4 }, + { id: "w1", text: "there.", start: 0.5, end: 0.9 }, + ], + }, + ]); + assert.equal(meta.total_duration_s, 1.25); + }); + } +} diff --git a/skills/media-use/audio/scripts/lib/gemini-auth.mjs b/skills/media-use/audio/scripts/lib/gemini-auth.mjs new file mode 100644 index 0000000000..e20a6a44d1 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/gemini-auth.mjs @@ -0,0 +1,43 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { pythonInvocation } from "./python.mjs"; + +export function geminiConfigured() { + return Boolean( + process.env.GEMINI_API_KEY || + process.env.GOOGLE_API_KEY || + process.env.GOOGLE_APPLICATION_CREDENTIALS || + process.env.GCS_CREDS, + ); +} + +export function geminiAuth({ run = spawnSync } = {}) { + const key = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY; + if (key) return { headers: { "x-goog-api-key": key }, secret: key }; + if (!geminiConfigured()) { + throw new Error( + "Gemini TTS needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)", + ); + } + const { cmd, args } = pythonInvocation([ + fileURLToPath(new URL("./gemini-auth.py", import.meta.url)), + ]); + const result = run(cmd, args, { encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 }); + let data; + try { + data = JSON.parse(result.stdout); + } catch { + throw new Error( + "Gemini service-account authentication failed; check Python and install google-auth requests", + ); + } + if (result.status !== 0 || !data.token) { + // Do not echo subprocess output: it could contain credentials. + throw new Error( + "Gemini service-account authentication failed; check service-account JSON, permissions, and Python packages google-auth requests", + ); + } + const headers = { Authorization: `Bearer ${data.token}` }; + if (data.project) headers["x-goog-user-project"] = data.project; + return { headers, secret: data.token }; +} diff --git a/skills/media-use/audio/scripts/lib/gemini-auth.py b/skills/media-use/audio/scripts/lib/gemini-auth.py new file mode 100644 index 0000000000..db5b3b2c06 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/gemini-auth.py @@ -0,0 +1,52 @@ +"""Obtain Gemini OAuth headers without persisting credentials or tokens.""" + +import json +import os +import sys + +# Keep the credential subprocess protocol UTF-8 on every platform. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors=_stream.errors) + + +def authenticate(): + try: + from google.auth.transport.requests import Request + from google.oauth2 import service_account + except ImportError: + return {"error": "Service-account auth needs Python packages: pip install google-auth requests"} + + try: + path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if path: + with open(path, encoding="utf-8") as source: + info = json.load(source) + else: + info = json.loads(os.environ["GCS_CREDS"]) + if info.get("type") != "service_account": + return {"error": "Gemini requires service-account JSON, not user ADC credentials"} + # Only exchange credentials with Google's OAuth server. + info["token_uri"] = "https://oauth2.googleapis.com/token" + credentials = service_account.Credentials.from_service_account_info( + info, + scopes=[ + "https://www.googleapis.com/auth/generative-language.retriever", + ], + ) + credentials.refresh(Request()) + project = ( + os.environ.get("GOOGLE_CLOUD_PROJECT") + or os.environ.get("GCLOUD_PROJECT_ID") + or info.get("project_id") + ) + return {"token": credentials.token, "project": project} + except Exception: + # Library exceptions can contain credential data. Never forward them. + return {"error": "Gemini service-account authentication failed; check credentials, permissions, and network access"} + + +if __name__ == "__main__": + result = authenticate() + print(json.dumps(result)) + sys.exit(1 if "error" in result else 0) diff --git a/skills/media-use/audio/scripts/lib/gemini-auth.test.mjs b/skills/media-use/audio/scripts/lib/gemini-auth.test.mjs new file mode 100644 index 0000000000..a7720e85c1 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/gemini-auth.test.mjs @@ -0,0 +1,86 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { pythonInvocation } from "./python.mjs"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { geminiAuth, geminiConfigured } from "./gemini-auth.mjs"; +import { pickProvider } from "./tts.mjs"; + +function env(t) { + const saved = { ...process.env }; + for (const key of [ + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", + "GCS_CREDS", + ]) + delete process.env[key]; + t.after(() => { + process.env = saved; + }); +} + +test("API keys take precedence over service accounts without invoking Python", (t) => { + env(t); + process.env.GEMINI_API_KEY = "gemini-key"; + process.env.GOOGLE_API_KEY = "google-key"; + process.env.GCS_CREDS = "private-json"; + const run = () => assert.fail("must not spawn Python"); + assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "gemini-key" }); + delete process.env.GEMINI_API_KEY; + assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "google-key" }); +}); + +test("service-account configuration selects Gemini and returns scoped helper headers", (t) => { + env(t); + assert.equal(geminiConfigured(), false); + assert.throws(() => geminiAuth(), /needs GEMINI_API_KEY/); + for (const key of ["GOOGLE_APPLICATION_CREDENTIALS", "GCS_CREDS"]) { + process.env[key] = "credential-source"; + assert.equal(pickProvider("gemini"), "gemini"); + const auth = geminiAuth({ + run: (_, args, options) => { + assert.ok(args.at(-1).endsWith("gemini-auth.py")); + assert.ok(!args.includes("credential-source")); + assert.equal(options.timeout, 60000); + return { + status: 0, + stdout: JSON.stringify({ token: "access-token", project: "quota-project" }), + }; + }, + }); + assert.deepEqual(auth.headers, { + Authorization: "Bearer access-token", + "x-goog-user-project": "quota-project", + }); + delete process.env[key]; + } +}); + +test("failed, malformed, or missing Python output cannot leak secrets or fall back", (t) => { + env(t); + process.env.GCS_CREDS = "private-json"; + for (const result of [ + { status: 1, stdout: '{"error":"private-json"}', stderr: "private-json" }, + { status: null, stdout: "private-json", error: new Error("private-json") }, + { status: 0, stdout: "{}" }, + ]) { + assert.throws( + () => geminiAuth({ run: () => result }), + (error) => { + assert.match(error.message, /authentication failed/); + assert.ok(!error.message.includes("private-json")); + return true; + }, + ); + } +}); + +// Keep the Python credential-boundary suite reachable from the normal Node CI runner. +test("Python service-account boundary validates credentials, scope, transport and errors", () => { + const { cmd, args } = pythonInvocation([ + fileURLToPath(new URL("./gemini-auth_test.py", import.meta.url)), + ]); + const result = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000 }); + assert.equal(result.status, 0, result.stderr || result.error?.message); +}); diff --git a/skills/media-use/audio/scripts/lib/gemini-auth_test.py b/skills/media-use/audio/scripts/lib/gemini-auth_test.py new file mode 100644 index 0000000000..598d4b4a92 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/gemini-auth_test.py @@ -0,0 +1,90 @@ +"""Credential-boundary tests; stub Google's transport, never use real secrets.""" + +import importlib.util +import json +import os +import sys +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import Mock, patch + + +# Keep the credential subprocess protocol UTF-8 on every platform. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors=_stream.errors) + +spec = importlib.util.spec_from_file_location("gemini_auth", Path(__file__).with_name("gemini-auth.py")) +auth = importlib.util.module_from_spec(spec) +spec.loader.exec_module(auth) + + +class AuthenticationTests(unittest.TestCase): + def setUp(self): + self.info = {"type": "service_account", "project_id": "account-project", "token_uri": "https://untrusted.invalid/token"} + self.credentials = Mock(token="fixture-token") + self.factory = Mock(return_value=self.credentials) + self.request = Mock(return_value=object()) + transport = types.ModuleType("google.auth.transport.requests") + transport.Request = self.request + oauth = types.ModuleType("google.oauth2") + oauth.service_account = types.SimpleNamespace(Credentials=types.SimpleNamespace(from_service_account_info=self.factory)) + modules = {name: types.ModuleType(name) for name in ("google", "google.auth", "google.auth.transport")} + modules.update({"google.auth.transport.requests": transport, "google.oauth2": oauth}) + self.addCleanup(patch.stopall) + patch.dict("sys.modules", modules).start() + patch.dict(os.environ, {"GCS_CREDS": json.dumps(self.info)}, clear=True).start() + + def test_scope_endpoint_and_refresh(self): + self.assertEqual(auth.authenticate(), {"token": "fixture-token", "project": "account-project"}) + info = self.factory.call_args.args[0] + self.assertEqual(info["token_uri"], "https://oauth2.googleapis.com/token") + self.assertEqual(self.factory.call_args.kwargs["scopes"], ["https://www.googleapis.com/auth/generative-language.retriever"]) + self.credentials.refresh.assert_called_once_with(self.request.return_value) + + def test_file_takes_precedence_over_json_environment(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "service.json" + path.write_text(json.dumps({**self.info, "project_id": "file-project"}), encoding="utf-8") + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path) + os.environ["GCS_CREDS"] = "invalid-json" + self.assertEqual(auth.authenticate()["project"], "file-project") + + def test_quota_project_precedence(self): + os.environ["GCLOUD_PROJECT_ID"] = "alias-project" + self.assertEqual(auth.authenticate()["project"], "alias-project") + os.environ["GOOGLE_CLOUD_PROJECT"] = "explicit-project" + self.assertEqual(auth.authenticate()["project"], "explicit-project") + + def test_user_adc_is_rejected_before_exchange(self): + os.environ["GCS_CREDS"] = json.dumps({"type": "authorized_user"}) + self.assertIn("service-account JSON", auth.authenticate()["error"]) + self.factory.assert_not_called() + + def test_invalid_json_or_missing_file_does_not_fall_back(self): + os.environ["GCS_CREDS"] = "private-invalid-json" + self.assertIn("error", auth.authenticate()) + os.environ["GCS_CREDS"] = json.dumps(self.info) + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/missing/service-account-fixture.json" + self.assertIn("error", auth.authenticate()) + self.factory.assert_not_called() + + def test_library_errors_are_redacted(self): + for boundary in (self.factory, self.credentials.refresh): + with self.subTest(boundary=boundary): + boundary.side_effect = RuntimeError("private-key-and-token") + result = auth.authenticate() + self.assertIn("error", result) + self.assertNotIn("private-key-and-token", json.dumps(result)) + boundary.side_effect = None + + def test_missing_dependency_has_actionable_error(self): + with patch.dict("sys.modules", {"google.auth.transport.requests": None}): + self.assertIn("pip install google-auth requests", auth.authenticate()["error"]) + self.factory.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/media-use/audio/scripts/lib/gemini-tts.mjs b/skills/media-use/audio/scripts/lib/gemini-tts.mjs new file mode 100644 index 0000000000..bf94cb8912 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/gemini-tts.mjs @@ -0,0 +1,122 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { geminiAuth } from "./gemini-auth.mjs"; + +export const GEMINI_TTS_MODEL = "gemini-3.8-flash-tts"; + +export const GEMINI_TTS_MODELS = [ + GEMINI_TTS_MODEL, + "gemini-3.8-flash-lite-tts", + "gemini-3.1-flash-tts-preview", + "gemini-2.5-pro-preview-tts", + "gemini-2.5-flash-preview-tts", +]; + +// 3.8 returns WAV; older models return PCM that we wrap without resampling. +// The shared engine transcribes the saved audio for word timings. +export async function synthesizeGemini( + { text, voiceId = "Kore", model = GEMINI_TTS_MODEL, style, speed = 1, wavAbs }, + { fetchImpl = fetch, authenticate = geminiAuth } = {}, +) { + let secret; + try { + if (!GEMINI_TTS_MODELS.includes(model)) { + throw new Error(`Unsupported Gemini TTS model: ${model}`); + } + if (speed !== 1) throw new Error("Gemini TTS uses style for pacing; omit speed or use 1"); + const modern = model.startsWith("gemini-3.8-"); + if (!modern && /^(voice_|voicekey_)/.test(voiceId)) { + throw new Error( + "Custom Gemini voices require a 3.8 TTS model; use a prebuilt voice with older models", + ); + } + const auth = authenticate(); + secret = auth.secret; + const content = { + type: "text", + text: + !modern && style ? `Read the following text with this delivery: ${style}\n\n${text}` : text, + }; + if (modern && style) content.annotations = [{ type: "speech_metadata", style }]; + const response = await fetchImpl( + "https://generativelanguage.googleapis.com/v1beta/interactions", + { + method: "POST", + headers: { "Content-Type": "application/json", ...auth.headers }, + signal: AbortSignal.timeout(120_000), + body: JSON.stringify({ + model, + input: [{ type: "user_input", content: [content] }], + response_format: modern ? { type: "audio", mime_type: "audio/wav" } : { type: "audio" }, + generation_config: { speech_config: [{ voice: voiceId }] }, + store: false, + }), + }, + ); + if (!response.ok) { + const detail = await response.text(); + throw new Error(`Gemini TTS HTTP ${response.status}: ${detail}`); + } + const payload = await response.json(); + if (payload.status !== "completed") { + throw new Error(`Gemini TTS did not complete (${payload.status ?? "missing status"})`); + } + const audio = (payload.steps ?? []) + .filter((step) => step.type === "model_output") + .flatMap((step) => step.content ?? []) + .filter((part) => part.type === "audio"); + if (audio.length !== 1 || !audio[0].data) { + throw new Error("Gemini TTS returned no single audio block"); + } + let bytes = Buffer.from(audio[0].data, "base64"); + if (!modern && /^audio\/l16(?:;|$)/i.test(audio[0].mime_type ?? "")) { + bytes = pcmToWav(bytes, audio[0].mime_type); + } else if (audio[0].mime_type !== "audio/wav") { + throw new Error("Gemini TTS returned an unsupported audio format"); + } + if ( + bytes.length <= 44 || + bytes.toString("ascii", 0, 4) !== "RIFF" || + bytes.toString("ascii", 8, 12) !== "WAVE" + ) { + throw new Error("Gemini TTS returned invalid WAV audio"); + } + mkdirSync(dirname(wavAbs), { recursive: true }); + writeFileSync(wavAbs, bytes); + return { ok: true, words: null }; + } catch (error) { + // Error responses must never echo the credential into logs or metadata. + const message = String(error?.message ?? error); + return { + ok: false, + words: null, + error: secret ? message.split(secret).join("[redacted]") : message, + }; + } +} + +function pcmToWav(pcm, mime) { + const rate = Number(/(?:^|;)\s*rate=(\d+)(?:;|$)/i.exec(mime)?.[1]); + const channels = /(?:^|;)\s*channels=([^;]+)/i.exec(mime)?.[1]; + const codec = /(?:^|;)\s*codec=([^;]+)/i.exec(mime)?.[1]; + if ((channels && channels.trim() !== "1") || (codec && codec.trim() !== "pcm")) { + throw new Error("Gemini TTS returned unsupported PCM channels or codec"); + } + if (!Number.isInteger(rate) || rate < 8000 || rate > 96000 || !pcm.length || pcm.length % 2) { + throw new Error("Gemini TTS returned invalid PCM audio or sample rate"); + } + const header = Buffer.alloc(44); + header.write("RIFF"); + header.writeUInt32LE(36 + pcm.length, 4); + header.write("WAVEfmt ", 8); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); + header.writeUInt16LE(1, 22); + header.writeUInt32LE(rate, 24); + header.writeUInt32LE(rate * 2, 28); + header.writeUInt16LE(2, 32); + header.writeUInt16LE(16, 34); + header.write("data", 36); + header.writeUInt32LE(pcm.length, 40); + return Buffer.concat([header, pcm]); +} diff --git a/skills/media-use/audio/scripts/lib/gemini-tts.test.mjs b/skills/media-use/audio/scripts/lib/gemini-tts.test.mjs new file mode 100644 index 0000000000..7c28e8d55b --- /dev/null +++ b/skills/media-use/audio/scripts/lib/gemini-tts.test.mjs @@ -0,0 +1,243 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { synthesizeGemini, GEMINI_TTS_MODEL } from "./gemini-tts.mjs"; +import { pickProvider, resolveVoiceId, synthesizeOne } from "./tts.mjs"; + +function fixture(t) { + const dir = mkdtempSync(join(tmpdir(), "gemini-tts-")); + const saved = { ...process.env }; + process.env.GEMINI_API_KEY = "test-gemini-key"; + delete process.env.GOOGLE_API_KEY; + delete process.env.GOOGLE_APPLICATION_CREDENTIALS; + delete process.env.GCS_CREDS; + t.after(() => { + process.env = saved; + rmSync(dir, { recursive: true, force: true }); + }); + const wav = Buffer.alloc(48); + wav.write("RIFF"); + wav.writeUInt32LE(40, 4); + wav.write("WAVEfmt ", 8); + wav.writeUInt32LE(16, 16); + wav.writeUInt16LE(1, 20); + wav.writeUInt16LE(1, 22); + wav.writeUInt32LE(24000, 24); + wav.writeUInt32LE(48000, 28); + wav.writeUInt16LE(2, 32); + wav.writeUInt16LE(16, 34); + wav.write("data", 36); + wav.writeUInt32LE(4, 40); + const audio = { type: "audio", mime_type: "audio/wav", data: wav.toString("base64") }; + const payload = { status: "completed", steps: [{ type: "model_output", content: [audio] }] }; + return { + wav, + audio, + payload, + args: { text: "Hello there.", voiceId: "Kore", wavAbs: join(dir, "voice", "one.wav") }, + }; +} + +test("Gemini preserves verbatim text, directs style separately, saves WAV and requests transcription", async (t) => { + const { wav, payload, args } = fixture(t); + t.mock.method(globalThis, "fetch", async (url, options) => { + assert.equal(url, "https://generativelanguage.googleapis.com/v1beta/interactions"); + assert.equal(options.headers["x-goog-api-key"], "test-gemini-key"); + const body = JSON.parse(options.body); + assert.equal(body.model, GEMINI_TTS_MODEL); + assert.equal(body.store, false); + assert.deepEqual(body.input[0].content, [ + { + type: "text", + text: args.text, + annotations: [{ type: "speech_metadata", style: "Warm and calm" }], + }, + ]); + assert.deepEqual(body.generation_config.speech_config, [{ voice: "Kore" }]); + assert.equal(body.response_format.mime_type, "audio/wav"); + return Response.json(payload); + }); + const result = await synthesizeOne({ ...args, provider: "gemini", style: "Warm and calm" }); + assert.deepEqual(result, { ok: true, words: null }); + assert.deepEqual(readFileSync(args.wavAbs), wav); +}); + +test("Flash-Lite and GOOGLE_API_KEY work without a Gemini key", async (t) => { + const { payload, args } = fixture(t); + delete process.env.GEMINI_API_KEY; + process.env.GOOGLE_API_KEY = "test-google-key"; + assert.equal(pickProvider("gemini"), "gemini"); + assert.equal(await resolveVoiceId({ provider: "gemini" }), "Kore"); + const result = await synthesizeGemini( + { ...args, model: "gemini-3.8-flash-lite-tts" }, + { + fetchImpl: async (_, options) => { + assert.equal(options.headers["x-goog-api-key"], "test-google-key"); + const body = JSON.parse(options.body); + assert.equal(body.model, "gemini-3.8-flash-lite-tts"); + assert.equal(body.input[0].content[0].annotations, undefined); + return Response.json(payload); + }, + }, + ); + assert.equal(result.ok, true); +}); + +test("Gemini remains opt-in even with a key and preserves explicitly chosen voices", async (t) => { + fixture(t); + assert.notEqual(pickProvider(), "gemini"); + assert.equal( + await resolveVoiceId({ provider: "gemini", userVoice: "voice_custom" }), + "voice_custom", + ); + delete process.env.GEMINI_API_KEY; + assert.throws(() => pickProvider("gemini"), /GEMINI_API_KEY or GOOGLE_API_KEY/); +}); + +test("invalid config fails before spending a generation request", async (t) => { + const { args } = fixture(t); + const deps = { fetchImpl: () => assert.fail("must not call the API") }; + for (const change of [{ model: "gemini-2.5-flash" }, { speed: 1.2 }]) { + const result = await synthesizeGemini({ ...args, ...change }, deps); + assert.equal(result.ok, false); + assert.ok(result.error); + } + delete process.env.GEMINI_API_KEY; + assert.match((await synthesizeGemini(args, deps)).error, /needs GEMINI_API_KEY/); + assert.equal(existsSync(args.wavAbs), false); +}); + +test("HTTP and network failures stay actionable without leaking the key", async (t) => { + const { args } = fixture(t); + const result = await synthesizeGemini(args, { + fetchImpl: async () => new Response("quota exceeded test-gemini-key", { status: 429 }), + }); + assert.equal(result.ok, false); + assert.match(result.error, /HTTP 429: quota exceeded/); + assert.ok(!result.error.includes("test-gemini-key")); + const timeout = await synthesizeGemini(args, { + fetchImpl: async () => { + throw new Error("timed out"); + }, + }); + assert.match(timeout.error, /timed out/); + assert.equal(existsSync(args.wavAbs), false); +}); + +test("incomplete, missing, raw PCM and malformed audio cannot become a successful WAV", async (t) => { + const { args, payload, audio } = fixture(t); + const cases = [ + { ...payload, status: "incomplete" }, + { status: "completed", steps: [] }, + ...[{ ...audio, mime_type: "audio/l16" }, { ...audio, data: "not audio" }, audio].map( + (part, i) => ({ + status: "completed", + steps: [{ type: "model_output", content: i === 2 ? [part, part] : [part] }], + }), + ), + ]; + for (const body of cases) { + const result = await synthesizeGemini(args, { fetchImpl: async () => Response.json(body) }); + assert.equal(result.ok, false); + assert.equal(existsSync(args.wavAbs), false); + } +}); + +for (const model of [ + "gemini-3.1-flash-tts-preview", + "gemini-2.5-pro-preview-tts", + "gemini-2.5-flash-preview-tts", +]) { + test(`${model} uses legacy delivery prompts and wraps PCM as mono WAV`, async (t) => { + const { args, wav } = fixture(t); + const pcm = wav.subarray(44); + const result = await synthesizeGemini( + { ...args, model, style: "Warm and clear" }, + { + fetchImpl: async (_, options) => { + const body = JSON.parse(options.body); + assert.deepEqual(body.response_format, { type: "audio" }); + assert.equal(body.input[0].content[0].annotations, undefined); + assert.ok(body.input[0].content[0].text.endsWith(args.text)); + assert.ok(body.input[0].content[0].text.includes("Warm and clear")); + return Response.json({ + status: "completed", + steps: [ + { + type: "model_output", + content: [ + { + type: "audio", + mime_type: "audio/L16;codec=pcm;rate=24000", + data: pcm.toString("base64"), + }, + ], + }, + ], + }); + }, + }, + ); + assert.equal(result.ok, true, result.error); + assert.deepEqual(readFileSync(args.wavAbs), wav); + }); +} + +test("service-account bearer and quota project reach synthesis, and token errors are redacted", async (t) => { + const { args, payload } = fixture(t); + const authenticate = () => ({ + headers: { Authorization: "Bearer secret-token", "x-goog-user-project": "test-project" }, + secret: "secret-token", + }); + const ok = await synthesizeGemini(args, { + authenticate, + fetchImpl: async (_, options) => { + assert.equal(options.headers.Authorization, "Bearer secret-token"); + assert.equal(options.headers["x-goog-user-project"], "test-project"); + assert.equal(options.headers["x-goog-api-key"], undefined); + return Response.json(payload); + }, + }); + assert.equal(ok.ok, true); + const failed = await synthesizeGemini(args, { + authenticate, + fetchImpl: async () => new Response("denied secret-token", { status: 403 }), + }); + assert.match(failed.error, /HTTP 403/); + assert.ok(!failed.error.includes("secret-token")); +}); + +test("older models reject unsupported PCM and custom voices", async (t) => { + const { args } = fixture(t); + for (const mime of [ + "audio/l16", + "audio/l16;rate=0", + "audio/l16;rate=24000;channels=2", + "audio/l16;rate=24000;codec=other", + ]) { + const result = await synthesizeGemini( + { ...args, model: "gemini-3.1-flash-tts-preview" }, + { + fetchImpl: async () => + Response.json({ + status: "completed", + steps: [ + { + type: "model_output", + content: [{ type: "audio", mime_type: mime, data: "AAAAAA==" }], + }, + ], + }), + }, + ); + assert.equal(result.ok, false); + assert.equal(existsSync(args.wavAbs), false); + } + const result = await synthesizeGemini( + { ...args, model: "gemini-2.5-pro-preview-tts", voiceId: "voice_custom" }, + { fetchImpl: () => assert.fail("must not generate") }, + ); + assert.match(result.error, /Custom Gemini voices require/); +}); diff --git a/skills/media-use/audio/scripts/lib/tts.mjs b/skills/media-use/audio/scripts/lib/tts.mjs index e341bf8edb..4d5fbccf0c 100644 --- a/skills/media-use/audio/scripts/lib/tts.mjs +++ b/skills/media-use/audio/scripts/lib/tts.mjs @@ -20,6 +20,8 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs"; import { pythonInvocation } from "./python.mjs"; +import { synthesizeGemini } from "./gemini-tts.mjs"; +import { geminiConfigured } from "./gemini-auth.mjs"; // ── provider detection ──────────────────────────────────────────────────────── export function heygenAvailable() { @@ -37,8 +39,12 @@ export function elevenlabsAvailable() { // First available provider wins; an explicit choice is honored (and validated). export function pickProvider(userProvider) { if (userProvider) { - if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider)) - throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`); + if (!["heygen", "elevenlabs", "kokoro", "gemini"].includes(userProvider)) + throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro | gemini)`); + if (userProvider === "gemini" && !geminiConfigured()) + throw new Error( + "provider=gemini needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)", + ); if (userProvider === "heygen" && !heygenAvailable()) throw new Error( "provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)", @@ -56,6 +62,7 @@ export function pickProvider(userProvider) { // their own defaults. export async function resolveVoiceId({ provider, userVoice, lang = "en" }) { if (userVoice) return userVoice; + if (provider === "gemini") return "Kore"; if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel if (provider === "kokoro") { if (lang === "en") return "am_michael"; @@ -248,10 +255,14 @@ export async function synthesizeOne({ voiceId, lang = "en", speed = 1.0, + model, + style, wavAbs, hyperframesDir, }) { if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }); + if (provider === "gemini") + return synthesizeGemini({ text, voiceId, model, style, speed, wavAbs }); if (provider === "elevenlabs") { // The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav) // and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir, diff --git a/skills/media-use/references/audio.md b/skills/media-use/references/audio.md index ec27bc138c..c789da8685 100644 --- a/skills/media-use/references/audio.md +++ b/skills/media-use/references/audio.md @@ -9,7 +9,8 @@ shot), use the shared engine at `audio/scripts/audio.mjs`. It takes a neutral node /audio/scripts/audio.mjs --request ./audio_request.json --out ./audio_meta.json ``` -- **Request** `{ provider?, lang?, speed?, lines: [{ id, text, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`. +- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`. +- **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged. - **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`. - **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work. - If BGM took the generate path (`bgm_pending: true`), run `audio/scripts/wait-bgm.mjs` before final render. diff --git a/skills/pr-to-video/scripts/audio.mjs b/skills/pr-to-video/scripts/audio.mjs index 84a878a97a..e2e22d585e 100644 --- a/skills/pr-to-video/scripts/audio.mjs +++ b/skills/pr-to-video/scripts/audio.mjs @@ -173,6 +173,8 @@ function runGenerate(argv) { : { mode: "retrieve", query, blob: g.message || "", arc: g.arc || "" }, }; if (userVoice) request.voice = userVoice; + if (flag(argv, "tts-model", null)) request.tts_model = flag(argv, "tts-model", null); + if (flag(argv, "style", null)) request.style = flag(argv, "style", null); const neutral = neutralPath(outPath); runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die); diff --git a/skills/product-launch-video/scripts/audio.mjs b/skills/product-launch-video/scripts/audio.mjs index ed8526782c..026c3d1987 100644 --- a/skills/product-launch-video/scripts/audio.mjs +++ b/skills/product-launch-video/scripts/audio.mjs @@ -172,6 +172,8 @@ function runGenerate(argv) { : { mode: "retrieve", query, blob: g.message || "", arc: g.arc || "" }, }; if (userVoice) request.voice = userVoice; + if (flag(argv, "tts-model", null)) request.tts_model = flag(argv, "tts-model", null); + if (flag(argv, "style", null)) request.style = flag(argv, "style", null); const neutral = neutralPath(outPath); runEngine({ request, hyperframesDir, neutral, only: "tts,bgm" }, die); diff --git a/skills/product-launch-video/scripts/audio.test.mjs b/skills/product-launch-video/scripts/audio.test.mjs index 47f548d666..de3f419055 100644 --- a/skills/product-launch-video/scripts/audio.test.mjs +++ b/skills/product-launch-video/scripts/audio.test.mjs @@ -34,6 +34,22 @@ test("passes --provider to the shared audio engine", () => { assert.equal(runAudio({ args: ["--provider", "kokoro"] }).provider, "kokoro"); }); +test("passes Gemini model and delivery style through the workflow adapter", () => { + const request = runAudio({ + args: [ + "--provider", + "gemini", + "--tts-model", + "gemini-3.8-flash-lite-tts", + "--style", + "Warm and calm", + ], + }); + assert.equal(request.provider, "gemini"); + assert.equal(request.tts_model, "gemini-3.8-flash-lite-tts"); + assert.equal(request.style, "Warm and calm"); +}); + test("uses HF_TTS_PROVIDER when --provider is omitted", () => { assert.equal(runAudio({ env: { HF_TTS_PROVIDER: "elevenlabs" } }).provider, "elevenlabs"); });