diff --git a/packages/cli/src/commands/media-use.ts b/packages/cli/src/commands/media-use.ts index 7feeeadcf0..3543aa731e 100644 --- a/packages/cli/src/commands/media-use.ts +++ b/packages/cli/src/commands/media-use.ts @@ -7,18 +7,18 @@ import { finishCommand } from "../utils/commandResult.js"; import { withMediaAuthEnvironment } from "../auth/media-environment.js"; const MEDIA_USE_ARGS = { - type: { type: "string" }, - intent: { type: "string" }, - entity: { type: "string" }, - project: { type: "string", alias: "p" }, - adopt: { type: "boolean" }, - candidates: { type: "boolean" }, + type: { type: "string", description: "What to find: bgm, sfx, image, icon, logo, voice" }, + intent: { type: "string", description: "What it should be, in plain words" }, + entity: { type: "string", description: "The brand or thing a logo belongs to" }, + project: { type: "string", alias: "p", description: "The project folder (default: here)" }, + adopt: { type: "boolean", description: "Keep a file you found yourself, with --from" }, + candidates: { type: "boolean", description: "List reusable assets instead of resolving one" }, doctor: { type: "boolean" }, stats: { type: "boolean" }, days: { type: "string" }, "dry-run": { type: "boolean" }, reuse: { type: "string" }, - from: { type: "string" }, + from: { type: "string", description: "Bring in this file or URL instead of searching" }, params: { type: "string" }, for: { type: "string" }, analyze: { type: "boolean" }, @@ -26,7 +26,7 @@ const MEDIA_USE_ARGS = { provider: { type: "string" }, "avatar-id": { type: "string" }, "voice-id": { type: "string" }, - json: { type: "boolean" }, + json: { type: "boolean", description: "One JSON result on stdout" }, help: { type: "boolean", alias: "h" }, } as const; diff --git a/packages/cli/src/commands/media-use/generate.ts b/packages/cli/src/commands/media-use/generate.ts index e5f4fd3fd0..d518df7916 100644 --- a/packages/cli/src/commands/media-use/generate.ts +++ b/packages/cli/src/commands/media-use/generate.ts @@ -25,6 +25,14 @@ export const examples: Example[] = [ "The original's presenter re-shot clean, saying an 8 s stretch of the original voice", 'hyperframes media-use generate --type video --reference assets/presenter-clean.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165 --prompt "the woman talks to camera, seated, soft daylight" --seconds 8 --aspect 9:16 --out assets/s02-presenter.mp4', ], + [ + "That shot's mouth redone to the exact voice, which stays as it is", + "hyperframes media-use generate --type lipsync --video assets/s02-presenter.mp4 --audio assets/s02-voice.mp3 --out assets/s02-lips.mp4", + ], + [ + "The person in a photo saying a finished voice, which stays as it is", + 'hyperframes media-use generate --type avatar --photo assets/presenter.png --audio assets/vo.mp3 --prompt "seated, talks with both hands" --aspect 16:9 --out assets/host.mp4', + ], [ "The person in a photo saying a script in a chosen voice", 'hyperframes media-use generate --type avatar --photo assets/presenter.png --script "Hi, I am Ana" --voice --out assets/ana.mp4', @@ -48,7 +56,7 @@ export default defineCommand({ meta: { name: "generate", description: - "Make new media on HeyGen: an image (edited from references), a video shot (from a first frame), a person speaking from a photo, a voice, or music. --plan prices it first", + "Make new media on HeyGen: an image (edited from references), a video shot (from a first frame), a shot's mouth redone to a voice, a person speaking from a photo, a voice, or music. --plan prices it first", }, args: { type: { @@ -61,7 +69,10 @@ export default defineCommand({ description: "The project file to write, e.g. assets/plate.png", required: true, }, - prompt: { type: "string", description: "What it shows or sounds like (image, video, music)" }, + prompt: { + type: "string", + description: "What it shows or sounds like (image, video, music); avatar: how a photo moves", + }, reference: { type: "string", description: @@ -69,8 +80,10 @@ export default defineCommand({ }, audio: { type: "string", - description: "video: speech the referenced person says, lips in sync (5–15 s)", + description: + "video: speech the referenced person says (5–15 s), a reference the model re-paces; lipsync, avatar: the exact voice", }, + video: { type: "string", description: "lipsync: the shot whose mouth is redone" }, motion: { type: "string", description: "video: a clip whose camera and body motion the shot follows (up to 15 s)", @@ -82,6 +95,10 @@ export default defineCommand({ }, "first-frame": { type: "string", description: "video: the picture the shot starts from" }, photo: { type: "string", description: "avatar: the person's photo" }, + avatar: { + type: "string", + description: "avatar: the person's own HeyGen avatar look id, instead of --photo", + }, script: { type: "string", description: "avatar, voice: the words spoken" }, voice: { type: "string", description: "avatar, voice: a HeyGen voice id" }, seconds: { @@ -126,9 +143,11 @@ export default defineCommand({ .filter(Boolean), firstFrame: args["first-frame"], audio: args.audio, + video: args.video, motion: args.motion, masks: repeated(rawArgs, "--mask").map(parseMask), photo: args.photo, + avatar: args.avatar, script: args.script, voice: args.voice, seconds: num(args.seconds, "--seconds"), diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts index 1b690eeb1b..ce061d034b 100644 --- a/packages/cli/src/commands/transcribe.ts +++ b/packages/cli/src/commands/transcribe.ts @@ -56,7 +56,7 @@ export default defineCommand({ engine: { type: "string", description: - "ASR engine: auto (Parakeet if installed and it covers --language, else whisper), parakeet, whisper, or heygen (HeyGen's speech model; needs an API key, charged to API credits). Default: auto. Parakeet is more accurate and faster than whisper; install it with `hyperframes models install parakeet`.", + "ASR engine: auto (Parakeet if installed and it covers --language, else whisper), parakeet, whisper, or heygen (HeyGen's speech model; bills the HyperFrames plan when signed in, or API credits with a key). Default: auto. Parakeet is more accurate and faster than whisper; install it with `hyperframes models install parakeet`.", alias: "e", }, speakers: { diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index 413fa1b18b..9b3a04ea85 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -11,6 +11,13 @@ vi.mock("../cloud/auth.js", () => ({ resolveCloudBaseUrl: () => "https://api.test", })); +vi.mock("../cloud/index.js", () => ({ createCloudClient: async () => ({}) })); +vi.mock("../cloud/upload.js", () => ({ + uploadAssetViaDirectUpload: async ({ filename }: { filename: string }) => ({ + asset_id: `asset-${filename}`, + }), +})); + vi.stubEnv("HEYGEN_API_KEY", "k"); const { checkRequest, generate, parseMask, urlOf } = await import("./generate.js"); @@ -349,6 +356,7 @@ describe("generate", () => { }).usd, ).toBe(0.07); expect(priceOf({ type: "voice", out: "a.mp3" }).usd).toBeNull(); + expect(creditsOf(priceOf({ type: "voice", out: "a.mp3", script: "今天聊黄金" }))).toBe(1); }); it("prices only a request the real call would accept", () => { @@ -373,12 +381,74 @@ describe("generate", () => { ); }); + it.skipIf(!ffmpeg)("redoes a shot's mouth to the exact voice", async () => { + const video = shots("lips-shot.mp4", ["color=c=red:d=2"]); + const audio = join(project, "lips-voice.mp3"); + writeFileSync(audio, "voice"); + const sent: { url: string; body?: Record }[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + sent.push({ url, ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}) }); + if (url === "https://files.test/lips.mp4") return new Response(new Uint8Array([1, 2, 3])); + if (init?.method === "POST") return json({ data: { lipsync_id: "lip1" } }); + return json({ + data: { id: "lip1", status: "completed", video_url: "https://files.test/lips.mp4" }, + }); + }); + const out = join(project, "lips-out.mp4"); + const request = { type: "lipsync" as const, video, audio, out }; + const made = await generate(request); + expect(made.job).toBe("lip1"); + expect(sent[0]!.url).toBe("https://api.test/v3/lipsyncs"); + expect(sent[0]!.body).toMatchObject({ + video: { type: "asset_id", asset_id: "asset-lips-shot.mp4" }, + audio: { type: "asset_id", asset_id: "asset-lips-voice.mp3" }, + mode: "speed", + enable_dynamic_duration: false, + }); + expect(sent[1]!.url).toBe("https://api.test/v3/lipsyncs/lip1"); + expect(existsSync(out)).toBe(true); + expect(creditsOf(priceOf(request))).toBe(14); + expect(() => checkRequest({ type: "lipsync", audio, out })).toThrow("--video"); + }); + + it.skipIf(!ffmpeg)("speaks a finished voice as it is, from a photo or a look", async () => { + const photo = join(project, "avatar-photo.png"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=red:s=64x36"], + ...["-frames:v", "1", photo], + ]); + const audio = join(project, "avatar-voice.mp3"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "sine=frequency=440:duration=6", audio], + ]); + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (url === "https://files.test/a.mp4") return new Response(new Uint8Array([1])); + if (init?.body) sent.push(JSON.parse(String(init.body))); + return json({ + data: { id: "v1", status: "completed", video_url: "https://files.test/a.mp4" }, + }); + }); + const out = join(project, "avatar-out.mp4"); + await generate({ type: "avatar", photo, audio, prompt: "talks with both hands", out }); + await generate({ type: "avatar", avatar: "look1", audio, out }); + expect(sent[0]).toMatchObject({ + type: "image", + audio_asset_id: "asset-avatar-voice.mp3", + motion_prompt: "talks with both hands", + }); + expect(sent[0]).not.toHaveProperty("script"); + expect(sent[1]).toMatchObject({ type: "avatar", avatar_id: "look1" }); + expect(creditsOf(priceOf({ type: "avatar", photo, audio, out }))).toBe(117); + expect(() => checkRequest({ type: "avatar", photo, out })).toThrow("--script"); + }); + it("prices an avatar on the plan's avatar rows, not its API price", () => { const script = Array.from({ length: 30 }, () => "word").join(" "); const avatar = priceOf({ type: "avatar", out: "a.mp4", script }); - expect(creditsOf(avatar)).toBe(200); + expect(creditsOf(avatar)).toBe(289); expect(avatar.list).toBeCloseTo(1.2167); - expect(() => checkLimit(creditsOf(avatar), 220)).not.toThrow(); + expect(() => checkLimit(creditsOf(avatar), 300)).not.toThrow(); }); it("charges a reference shot's pictures after the first two", () => { @@ -389,18 +459,18 @@ describe("generate", () => { references: Array.from({ length: pictures }, (_, i) => `${i}.png`), }); const credits = [1, 2, 3, 9].map((n) => creditsOf(priceOf(shot(n)))); - expect(credits).toEqual([167, 167, 180, 260]); + expect(credits).toEqual([250, 250, 270, 390]); expect(priceOf(shot(9)).list).toBeCloseTo(0.39); - expect(() => checkLimit(credits[3]!, 250)).toThrow("about 260 credits, over --max-credits 250"); - expect(() => checkLimit(credits[3]!, 260)).not.toThrow(); + expect(() => checkLimit(credits[3]!, 380)).toThrow("about 390 credits, over --max-credits 380"); + expect(() => checkLimit(credits[3]!, 390)).not.toThrow(); }); it("keeps a call under the person's credit limit, or refuses it before sending", () => { const shot = creditsOf(priceOf({ type: "video", out: "a.mp4", seconds: 6 })); - expect(shot).toBe(72); + expect(shot).toBe(108); expect(creditsOf(priceOf({ type: "image", out: "a.png" }))).toBe(67); - expect(() => checkLimit(shot, 72)).not.toThrow(); - expect(() => checkLimit(shot, 30)).toThrow("about 72 credits, over --max-credits 30"); + expect(() => checkLimit(shot, 108)).not.toThrow(); + expect(() => checkLimit(shot, 30)).toThrow("about 108 credits, over --max-credits 30"); expect(() => checkLimit(null, 30)).toThrow("cannot be priced ahead"); expect(() => checkLimit(null, undefined)).not.toThrow(); }); diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index a2ccce360d..9fdeb5a783 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -1,17 +1,20 @@ // New media on HeyGen for `media-use generate`: a still (optionally edited from a reference), a shot (optionally from -// a first frame), a person speaking from one photo, a voice reading text, or music. Every call is priced first with -// --plan, keyed for idempotency, downloaded into the project and recorded in the media ledger as generated. +// a first frame), a shot's mouth redone to a voice, a person speaking from one photo, a voice reading text, or music. +// Every call is priced first with --plan, keyed for idempotency, downloaded into the project and recorded in the media +// ledger as generated. +import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, extname, join, relative, resolve } from "node:path"; +import { basename, dirname, extname, join, relative, resolve } from "node:path"; import { findCuts, signatures } from "../measure/cuts.js"; import { Fix, probeVideo, round, runFfmpeg, videoClip } from "../measure/media.js"; import { idempotencyKey, runJob, type Job } from "./v3.js"; -export type GenerateType = "image" | "video" | "avatar" | "voice" | "music"; +export type GenerateType = "image" | "video" | "lipsync" | "avatar" | "voice" | "music"; export const GENERATE_TYPES: readonly GenerateType[] = [ "image", "video", + "lipsync", "avatar", "voice", "music", @@ -23,8 +26,13 @@ export interface GenerateRequest { prompt?: string; /** image: pictures the result is edited from or styled after. video: the person or things to keep in the shot. */ references?: string[]; - /** video: speech the person in the references says, lips in sync (a stretch of the original's voice). */ + /** + * video: speech the person in the references says. The model takes it as a reference and re-paces it, so the lips + * follow the shot's own track, not this file. lipsync and avatar: the exact voice, which stays as it is. + */ audio?: string; + /** lipsync: the shot whose mouth is redone. */ + video?: string; /** video: a clip whose camera and body motion the shot follows (the original's same stretch, up to 15 s). */ motion?: string; /** video: regions of the motion clip blanked before it is sent (its drawn text), so the shot does not copy them. */ @@ -33,6 +41,8 @@ export interface GenerateRequest { firstFrame?: string; /** avatar: the photo of the person who speaks. */ photo?: string; + /** avatar: the person's own HeyGen avatar look, instead of a photo. */ + avatar?: string; /** avatar and voice: what is said. */ script?: string; voice?: string; @@ -85,10 +95,7 @@ interface Route { ext: string[]; } -type Media = ( - path: string, - kind: "image" | "audio" | "video", -) => { type: "base64"; media_type: string; data: string }; +type Media = (path: string, kind: "image" | "audio" | "video") => Record; const DEFAULTS = { image: "nano-banana-2", video: "heygen-video-1", music: "lyria-3-pro" } as const; const VIDEO_RESOLUTION = "768p"; @@ -124,7 +131,7 @@ const ROUTES: Record = { poll: (id) => `/v3/models/videos/${id}`, // fallow-ignore-next-line complexity body: (r, media) => { - // A person speaking given audio is a reference shot: the references show who, the audio drives the lips. + // A person speaking given audio is a reference shot: the references show who, the audio guides the speech. const spoken = !!r.audio || !!r.references?.length || !!r.motion; if (spoken && r.firstFrame) throw new Fix( @@ -153,19 +160,46 @@ const ROUTES: Record = { ledger: "video", ext: [".mp4", ".mov", ".webm"], }, + // The voice stays exactly as given and only the mouth changes, so the shot lands on the voice from its first frame. + lipsync: { + path: "/v3/lipsyncs", + poll: (id) => `/v3/lipsyncs/${id}`, + body: (r, media) => ({ + video: media(need(r.video, "--video", "--video work/s01.mp4"), "video"), + audio: media(need(r.audio, "--audio", "--audio work/s01.mp3"), "audio"), + mode: "speed", + enable_dynamic_duration: false, + fps_mode: "cfr", + keep_the_same_format: true, + }), + urls: ["video_url"], + ledger: "video", + ext: [".mp4"], + }, avatar: { path: "/v3/videos", poll: (id) => `/v3/videos/${id}`, body: (r, media) => ({ - type: "image", - image: media(need(r.photo, "--photo", "--photo assets/presenter.png"), "image"), - script: need(r.script, "--script", '--script "What they say"'), - voice_id: need( - r.voice, - "--voice", - "--voice (npx hyperframes media-use resolve --type voice lists voices)", - ), + ...(r.avatar + ? { type: "avatar", avatar_id: r.avatar } + : { + type: "image", + image: media(need(r.photo, "--photo", "--photo assets/presenter.png"), "image"), + ...(r.prompt ? { motion_prompt: r.prompt } : {}), + }), + // A finished voice stays exactly as it is and the lips land on it; a script is read in a voice. + ...(r.audio + ? { audio_asset_id: media(r.audio, "audio").asset_id } + : { + script: need(r.script, "--script", '--script "What they say"'), + voice_id: need( + r.voice, + "--voice", + "--voice (npx hyperframes media-use resolve --type voice lists voices)", + ), + }), ...(r.aspect ? { aspect_ratio: r.aspect } : {}), + resolution: "1080p", output_format: "mp4", }), urls: ["video_url"], @@ -264,6 +298,32 @@ function inline(work: string, masks: Mask[] = []): Media { }; } +/** Files a route takes as asset ids, not inline. */ +function assetsOf(r: GenerateRequest): string[] { + if (r.type === "lipsync") return [r.video!, r.audio!]; + return r.type === "avatar" && r.audio ? [r.audio] : []; +} + +/** Uploads them whole; the asset id of each path. */ +async function uploaded(paths: string[]): Promise> { + const ids = new Map(); + if (!paths.length) return ids; + const { createCloudClient } = await import("../cloud/index.js"); + const { uploadAssetViaDirectUpload } = await import("../cloud/upload.js"); + const client = await createCloudClient(); + for (const path of paths) { + const asset = await uploadAssetViaDirectUpload({ + client, + bytes: new Uint8Array(readFileSync(path)), + filename: basename(path), + contentType: extname(path).toLowerCase() === ".mp4" ? "video/mp4" : "audio/mpeg", + idempotencyKey: randomUUID(), + }); + ids.set(path, asset.asset_id); + } + return ids; +} + /** * The seconds where a new shot starts in a video's first `seconds`, and the length read. A cut within a frame of * either end leaves a stray frame, not a second shot, and does not count. @@ -346,11 +406,18 @@ export async function generate( const project = projectOf(dirname(out)); const work = mkdtempSync(join(tmpdir(), "hf-generate-")); try { - const body = route.body(r, inline(work, r.masks)); - const key = idempotencyKey(route.path, [JSON.stringify(body)]); + const sources = assetsOf(r); + const ids = await uploaded(sources); + const local = inline(work, r.masks); + const body = route.body(r, (path, kind) => + ids.has(path) ? { type: "asset_id", asset_id: ids.get(path) } : local(path, kind), + ); + // An asset id is new on every upload; the files say whether this is the same job. + const sent = [...ids.values()].reduce((t, id) => t.replaceAll(id, ""), JSON.stringify(body)); + const key = idempotencyKey(route.path, [...sources.map((p) => readFileSync(p)), sent]); const job = await runJob(route.path, body, key, (id) => route.poll(encodeURIComponent(id)), { log: { - projectDir: project ?? undefined, + projectDir: project ?? process.cwd(), what: `generate ${r.type} ${relative(project ?? ".", out)}`, }, onWait: (j) => onWait?.(j.status), @@ -398,7 +465,9 @@ export async function generate( ...(r.references?.length ? { references: r.references.map(describe) } : {}), ...(r.firstFrame ? { first_frame: describe(r.firstFrame) } : {}), ...(r.photo ? { photo: describe(r.photo) } : {}), + ...(r.avatar ? { avatar: r.avatar } : {}), ...(r.audio ? { audio: describe(r.audio) } : {}), + ...(r.video ? { video: describe(r.video) } : {}), ...(r.motion ? { motion: describe(r.motion) } : {}), ...(r.masks?.length ? { motion_masks: r.masks.map((m) => [...m.box, ...(m.window ?? [])]) } diff --git a/packages/cli/src/heygen/pricing.ts b/packages/cli/src/heygen/pricing.ts index c9132847e7..3393d25add 100644 --- a/packages/cli/src/heygen/pricing.ts +++ b/packages/cli/src/heygen/pricing.ts @@ -1,7 +1,7 @@ // Prices before a paid call, from the HyperFrames app's creation pricing (hyperframes-internal // packages/desktop/shared/creationPricing.mjs, 2026-10-22). HeyGen charges in units of $1/60, rounded up; the label // rounds up to the cent. Routes HeyGen does not price ahead say so instead of guessing. -import { Fix, probeVideo, round } from "../measure/media.js"; +import { Fix, probeSeconds, probeVideo, round } from "../measure/media.js"; import { GUIDE_MAX_S, type GenerateRequest } from "./generate.js"; export interface Price { @@ -39,14 +39,19 @@ const VIDEO_REF_768P_PER_S = 0.05; // A reference shot's first two pictures are included; each one after them bills on its own line. const VIDEO_INCLUDED_IMAGES = 2; const VIDEO_EXTRA_IMAGE = 0.02; -// The price book's HyperFrames rows (experiment-framework price_book rules.json, 2026-10-28). -const HF_VIDEO_768P_PER_S = 12; -const HF_VIDEO_REF_768P_PER_S = 100 / 3; -const HF_VIDEO_EXTRA_IMAGE = 40 / 3; -const HF_AVATAR_IV_PHOTO_PER_S = 800 / 60; +// The price book's HyperFrames rows (experiment-framework price_book rules.json, 2026-10-30). +const HF_VIDEO_768P_PER_S = 18; +const HF_VIDEO_REF_768P_PER_S = 50; +const HF_VIDEO_EXTRA_IMAGE = 20; +const HF_AVATAR_IV_PHOTO_PER_S = 1155 / 60; +const HF_AVATAR_IV_TWIN_PER_S = 2415 / 60; +// Billed on the whole video, not the voice: measured 2026-11-01, a 13.75 s shot took 97 credits. +const HF_LIPSYNC_SPEED_PER_S = 7; const MUSIC_PER_TRACK = 0.08; const AVATAR_IV_PER_MIN = 4.83; const SCRIPT_WORDS_PER_S = 2; +const HF_TTS_PER_S = 1; +const SPEECH_CHARS_PER_S = { zh: 5, en: 15 }; // fallow-ignore-next-line complexity export function priceOf(r: GenerateRequest): Price { @@ -79,24 +84,39 @@ export function priceOf(r: GenerateRequest): Price { seconds * HF_VIDEO_768P_PER_S, ); } + case "lipsync": { + const seconds = r.video ? probeVideo(r.video).pictureDuration : 0; + return { + usd: null, + list: null, + credits: seconds * HF_LIPSYNC_SPEED_PER_S, + basis: `lipsync at speed, ${round(seconds, 2)} s at ${HF_LIPSYNC_SPEED_PER_S} credits a second`, + }; + } case "avatar": { const words = (r.script ?? "").split(/\s+/).filter(Boolean).length; - const seconds = Math.max(1, Math.floor(words / SCRIPT_WORDS_PER_S)); + const seconds = r.audio + ? probeSeconds(r.audio) + : Math.max(1, Math.floor(words / SCRIPT_WORDS_PER_S)); return at( Math.ceil(seconds * AVATAR_IV_PER_MIN) / 60, - `avatar video, about ${seconds} s from ${words} words at $4.83 a minute`, - seconds * HF_AVATAR_IV_PHOTO_PER_S, + `avatar video, about ${round(seconds, 1)} s${r.audio ? "" : ` from ${words} words`} at $4.83 a minute`, + seconds * (r.avatar ? HF_AVATAR_IV_TWIN_PER_S : HF_AVATAR_IV_PHOTO_PER_S), ); } case "music": return at(MUSIC_PER_TRACK, "lyria-3-pro, $0.08 a track"); - case "voice": + case "voice": { + const text = r.script ?? ""; + const cjk = (text.match(/\p{Script=Han}/gu) ?? []).length; + const seconds = cjk / SPEECH_CHARS_PER_S.zh + (text.length - cjk) / SPEECH_CHARS_PER_S.en; return { usd: null, list: null, - credits: null, - basis: "HeyGen does not price catalog speech ahead; it is charged per character", + credits: seconds * HF_TTS_PER_S, + basis: `about ${round(seconds, 1)} s of speech at ${HF_TTS_PER_S} credit a second, an estimate: it is charged per character`, }; + } } } diff --git a/packages/cli/src/heygen/v3.ts b/packages/cli/src/heygen/v3.ts index 4ec4c15366..692a5b520a 100644 --- a/packages/cli/src/heygen/v3.ts +++ b/packages/cli/src/heygen/v3.ts @@ -179,7 +179,7 @@ async function call( const data = isRecord(value) && isRecord(value.data) ? value.data : value; if (!isRecord(data)) throw new Fix(`HeyGen answered ${route} with no job`, "run the same command again"); - const id = data.id ?? data.job_id ?? data.video_id ?? data.generation_id; + const id = data.id ?? data.job_id ?? data.video_id ?? data.generation_id ?? data.lipsync_id; return { ...data, id: typeof id === "string" ? id : "", diff --git a/packages/cli/src/measure/media.ts b/packages/cli/src/measure/media.ts index 64f6b426db..0eb96682e7 100644 --- a/packages/cli/src/measure/media.ts +++ b/packages/cli/src/measure/media.ts @@ -86,6 +86,17 @@ const rate = (text: string | undefined): number => { return b ? (a ?? 0) / b : (a ?? 0); }; +/** A sound's length in seconds. */ +export function probeSeconds(path: string): number { + if (!existsSync(path)) throw new Fix(`no file at ${path}`, "pass a sound from the project"); + const text = execFileSync( + ffprobeBin(), + ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", path], + { maxBuffer: MAX_BUFFER, windowsHide: true }, + ).toString(); + return Number(text) || 0; +} + /** Size (as displayed, rotation applied), clock and sound of a video. */ // fallow-ignore-next-line complexity export function probeVideo(path: string): VideoInfo { diff --git a/packages/cli/src/media-use/lib/registry.mjs b/packages/cli/src/media-use/lib/registry.mjs index f7562c2b95..50234c4e28 100644 --- a/packages/cli/src/media-use/lib/registry.mjs +++ b/packages/cli/src/media-use/lib/registry.mjs @@ -23,7 +23,7 @@ // (e.g. "make an image with codex"). import { bgmProvider } from "./bgm-provider.mjs"; -import { OPENVERSE_SFX_PROVIDER, sfxProvider } from "./sfx-provider.mjs"; +import { heygenSfxProvider, OPENVERSE_SFX_PROVIDER, sfxProvider } from "./sfx-provider.mjs"; import { bundledSfxProvider } from "./bundled-sfx-provider.mjs"; import { imageProvider, iconProvider } from "./image-provider.mjs"; import { brandProvider } from "./brand-provider.mjs"; @@ -48,6 +48,7 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem const REGISTRY = { bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })], sfx: [ + N("heygen.audio.sounds", { search: heygenSfxProvider.search }), N(OPENVERSE_SFX_PROVIDER, { search: sfxProvider.search }), A("bundled.sfx", { search: bundledSfxProvider.search }), ], diff --git a/packages/cli/src/media-use/lib/sfx-provider.mjs b/packages/cli/src/media-use/lib/sfx-provider.mjs index 26c8e6d762..3cdfbedb8d 100644 --- a/packages/cli/src/media-use/lib/sfx-provider.mjs +++ b/packages/cli/src/media-use/lib/sfx-provider.mjs @@ -1,7 +1,30 @@ +import { heygenSearch } from "./heygen-search.mjs"; import { OpenverseUnavailableError, searchOpenverseSfx } from "./openverse.mjs"; export const OPENVERSE_SFX_PROVIDER = "openverse.audio"; +export const heygenSfxProvider = { + async search(intent) { + // Good effects score 0.5-0.67, under the server's default floor of 0.7. + const results = heygenSearch("audio sounds list", intent, { + type: "sound_effects", + minScore: 0.4, + }); + if (!results) return null; + const best = results[0]; + return { + url: best.audio_url, + source: "search", + metadata: { + description: best.description || intent, + duration: best.duration || null, + provider: "heygen.audio.sounds", + provenance: { track_id: best.id, score: best.score, query: intent }, + }, + }; + }, +}; + export const sfxProvider = { async search(intent) { let sound; diff --git a/skills-manifest.json b/skills-manifest.json index ef16966944..c6d7827c66 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -2,7 +2,7 @@ "source": "heygen-com/hyperframes", "skills": { "creator-remix": { - "hash": "fb348339ce2fa040", + "hash": "1841826491d93706", "files": 2 }, "figma": { @@ -22,7 +22,7 @@ "files": 13 }, "media-use": { - "hash": "f049425ce5d31ec9", + "hash": "5be81ddc174ac351", "files": 117 }, "remotion-to-hyperframes": { diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index ddb316eac4..315048bb8c 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -20,7 +20,7 @@ It prints one line, `resolved → (, )`. List reusab | Need | Free | HyperFrames credits | Local or the person's own key (only when they ask) | | --- | --- | --- | --- | | Music (`bgm`) | HeyGen catalog of 10k+ tracks (signed in) | Generated track | Local MusicGen | -| Sound effects (`sfx`) | Openverse (credit CC BY sounds); 21 bundled offline | | | +| Sound effects (`sfx`) | HeyGen (signed in); Openverse (credit CC BY); bundled | | | | Images, icons | HeyGen asset search, 75k+ (signed in) | Generated images | Local FLUX (mflux), if installed | | Logos (`logo`) | theSVG, GitHub avatar, favicon; never redrawn | | | | Voiceover (`voice`) | HeyGen Voice Turbo (signed in) | Other HeyGen voices, voice cloning | `npx hyperframes tts` (Kokoro on device) | diff --git a/skills/media-use/references/heygen.md b/skills/media-use/references/heygen.md index 11abae4f06..c55407fdab 100644 --- a/skills/media-use/references/heygen.md +++ b/skills/media-use/references/heygen.md @@ -6,11 +6,11 @@ `npx hyperframes auth status` says what is signed in. The credential is read in this order: `HEYGEN_API_KEY`, `HYPERFRAMES_API_KEY`, a host app's `HEYGEN_ACCESS_TOKEN`, then `~/.hyperframes/credentials`, which `npx hyperframes auth login` writes. `heygen auth login` is a separate sign-in that only the `heygen` CLI reads. -| Credential | Pays with | Covers | -| -------------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------- | -| API key | API credits | everything below | -| Sign-in (OAuth) | the plan's allowance on subscription credits | avatar videos, video shots, images and voices; music and transcription take an API key | -| Inside the HyperFrames app | the app | whatever the app allows; nothing to set up | +| Credential | Pays with | Covers | +| -------------------------- | -------------------------------------------- | ---------------------------------------------------------------------------- | +| API key | API credits | everything below | +| Sign-in (OAuth) | the plan's allowance on subscription credits | avatar videos, video shots, lipsync, images, voices, music and transcription | +| Inside the HyperFrames app | the app | whatever the app allows; nothing to set up | A refusal names its fix: `api_key_required` needs an API key, `402` or `insufficient_credit` needs a top-up, `hf_usage_exhausted` means the plan's allowance is used up. @@ -27,10 +27,14 @@ npx hyperframes media-use generate --type video --first-frame assets/s03-first.p # in sync and moving as the guide (the original shot itself) does; --mask x,y,w,h[,a-b] (repeatable, the guide's # pixels and seconds) blanks text in the guide, which HeyGen would copy npx hyperframes media-use generate --type video --reference assets/person.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165,0.4-2.0 --prompt "talks to camera, seated" --seconds 8 --aspect 9:16 --out assets/s02-person.mp4 +# a speaking shot follows its own re-paced audio, not --audio: to land it on a set voice, redo its mouth to that voice +npx hyperframes media-use generate --type lipsync --video assets/s02-person.mp4 --audio assets/s02-voice.mp3 --out assets/s02-lips.mp4 # the same for a person who does not speak: no --audio, the guide clip carries the motion npx hyperframes media-use generate --type video --reference assets/person.png --motion assets/s05-original.mp4 --prompt "holds up a jar, smiles" --seconds 6 --aspect 9:16 --out assets/s05-person.mp4 # a person in a photo speaking a script in a HeyGen voice npx hyperframes media-use generate --type avatar --photo assets/person.png --script "…" --voice --out assets/person.mp4 +# a finished voice stays exactly as it is: --audio replaces --script, --avatar replaces --photo, --prompt moves a photo +npx hyperframes media-use generate --type avatar --photo assets/person.png --audio assets/vo.mp3 --prompt "seated, talks with both hands" --aspect 16:9 --out assets/host.mp4 # a voice reading text npx hyperframes media-use generate --type voice --script "…" --voice --out assets/vo.mp3 # music, 30, 60, 120 or 180 s (instrumental unless --vocals), trimmed to length in the composition @@ -39,13 +43,14 @@ npx hyperframes media-use generate --type music --prompt "warm lo-fi, soft keys" The file lands at `--out` and the media ledger records it as generated, with the route, model, job, prompt and the files it was made from. Pictures and shots carry no text or logos: ask for clean plates and draw words in HyperFrames. -| Type | Model | Price | -| ------ | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first | -| video | heygen-video-1, 768p | $0.02 a second from a prompt or first frame; $0.05 a second with a reference picture, voice or guide clip, and the guide's seconds count too (1080p costs about twice) | -| avatar | talking photo | $4.83 a minute of speech | -| music | lyria-3-pro | $0.08 a track | -| voice | the voice's engine | charged per character, not priced ahead | +| Type | Model | Price | +| ------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first | +| video | heygen-video-1, 768p | $0.02 a second from a prompt or first frame; $0.05 a second with a reference picture, voice or guide clip, and the guide's seconds count too (1080p costs about twice) | +| lipsync | HeyGen lipsync, speed | 7 credits a second of the video on a plan | +| avatar | talking photo | $4.83 a minute of speech | +| music | lyria-3-pro | $0.08 a track | +| voice | the voice's engine | charged per character, not priced ahead | `--plan`'s `usd` is what an API key is charged (HeyGen rounds up to units of $1/60); its `credits` is what a HyperFrames plan charges, on the plan's own rates: a video shot's seconds and guide seconds, plus each picture past the first two. Voice ids come from `heygen voice list` or `npx hyperframes media-use resolve --type voice`; a cloned voice from `heygen voice clone create` (`operations.md`). @@ -55,4 +60,4 @@ The file lands at `--out` and the media ledger records it as generated, with the npx hyperframes transcribe interview.mp4 --engine heygen --speakers ``` -Word times from HeyGen's speech model, each word with its speaker under `--speakers`, in the same `transcript.json` local engines write. It takes an API key. The local engines (`--engine auto`) are free and good for one clear voice; HeyGen is the better choice for several speakers, noise or music under the voice. +Word times from HeyGen's speech model, each word with its speaker under `--speakers`, in the same `transcript.json` local engines write. The local engines (`--engine auto`) are free and good for one clear voice; HeyGen is the better choice for several speakers, noise or music under the voice.