mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 23:47:33 +00:00
feat(cli): lipsync, and avatars that speak a finished voice
Video 1 takes audio as a reference and re-paces it, so a shot's lips follow its own track and drift up to a second from the voice. generate --type lipsync redoes the mouth to the exact voice. generate --type avatar takes --audio, and --avatar for the person's own look, so the lips land on the voice from one job. Also moves the plan rows to the 2026-10-30 price book, estimates a voice's credits so --max-credits can hold it, logs jobs from a folder with no project, and describes the resolve flags. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7066c73745
commit
6c10f1905d
@@ -7,18 +7,18 @@ import { finishCommand } from "../utils/commandResult.js";
|
||||
import { withMediaAuthEnvironment } from "../auth/media-environment.js";
|
||||
|
||||
const MEDIA_USE_ARGS = {
|
||||
type: { type: "string" },
|
||||
intent: { type: "string" },
|
||||
entity: { type: "string" },
|
||||
project: { type: "string", alias: "p" },
|
||||
adopt: { type: "boolean" },
|
||||
candidates: { type: "boolean" },
|
||||
type: { type: "string", description: "What to find: bgm, sfx, image, icon, logo, voice" },
|
||||
intent: { type: "string", description: "What it should be, in plain words" },
|
||||
entity: { type: "string", description: "The brand or thing a logo belongs to" },
|
||||
project: { type: "string", alias: "p", description: "The project folder (default: here)" },
|
||||
adopt: { type: "boolean", description: "Keep a file you found yourself, with --from" },
|
||||
candidates: { type: "boolean", description: "List reusable assets instead of resolving one" },
|
||||
doctor: { type: "boolean" },
|
||||
stats: { type: "boolean" },
|
||||
days: { type: "string" },
|
||||
"dry-run": { type: "boolean" },
|
||||
reuse: { type: "string" },
|
||||
from: { type: "string" },
|
||||
from: { type: "string", description: "Bring in this file or URL instead of searching" },
|
||||
params: { type: "string" },
|
||||
for: { type: "string" },
|
||||
analyze: { type: "boolean" },
|
||||
@@ -26,7 +26,7 @@ const MEDIA_USE_ARGS = {
|
||||
provider: { type: "string" },
|
||||
"avatar-id": { type: "string" },
|
||||
"voice-id": { type: "string" },
|
||||
json: { type: "boolean" },
|
||||
json: { type: "boolean", description: "One JSON result on stdout" },
|
||||
help: { type: "boolean", alias: "h" },
|
||||
} as const;
|
||||
|
||||
|
||||
@@ -25,6 +25,14 @@ export const examples: Example[] = [
|
||||
"The original's presenter re-shot clean, saying an 8 s stretch of the original voice",
|
||||
'hyperframes media-use generate --type video --reference assets/presenter-clean.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165 --prompt "the woman talks to camera, seated, soft daylight" --seconds 8 --aspect 9:16 --out assets/s02-presenter.mp4',
|
||||
],
|
||||
[
|
||||
"That shot's mouth redone to the exact voice, which stays as it is",
|
||||
"hyperframes media-use generate --type lipsync --video assets/s02-presenter.mp4 --audio assets/s02-voice.mp3 --out assets/s02-lips.mp4",
|
||||
],
|
||||
[
|
||||
"The person in a photo saying a finished voice, which stays as it is",
|
||||
'hyperframes media-use generate --type avatar --photo assets/presenter.png --audio assets/vo.mp3 --prompt "seated, talks with both hands" --aspect 16:9 --out assets/host.mp4',
|
||||
],
|
||||
[
|
||||
"The person in a photo saying a script in a chosen voice",
|
||||
'hyperframes media-use generate --type avatar --photo assets/presenter.png --script "Hi, I am Ana" --voice <voice-id> --out assets/ana.mp4',
|
||||
@@ -48,7 +56,7 @@ export default defineCommand({
|
||||
meta: {
|
||||
name: "generate",
|
||||
description:
|
||||
"Make new media on HeyGen: an image (edited from references), a video shot (from a first frame), a person speaking from a photo, a voice, or music. --plan prices it first",
|
||||
"Make new media on HeyGen: an image (edited from references), a video shot (from a first frame), a shot's mouth redone to a voice, a person speaking from a photo, a voice, or music. --plan prices it first",
|
||||
},
|
||||
args: {
|
||||
type: {
|
||||
@@ -61,7 +69,10 @@ export default defineCommand({
|
||||
description: "The project file to write, e.g. assets/plate.png",
|
||||
required: true,
|
||||
},
|
||||
prompt: { type: "string", description: "What it shows or sounds like (image, video, music)" },
|
||||
prompt: {
|
||||
type: "string",
|
||||
description: "What it shows or sounds like (image, video, music); avatar: how a photo moves",
|
||||
},
|
||||
reference: {
|
||||
type: "string",
|
||||
description:
|
||||
@@ -69,8 +80,10 @@ export default defineCommand({
|
||||
},
|
||||
audio: {
|
||||
type: "string",
|
||||
description: "video: speech the referenced person says, lips in sync (5–15 s)",
|
||||
description:
|
||||
"video: speech the referenced person says (5–15 s), a reference the model re-paces; lipsync, avatar: the exact voice",
|
||||
},
|
||||
video: { type: "string", description: "lipsync: the shot whose mouth is redone" },
|
||||
motion: {
|
||||
type: "string",
|
||||
description: "video: a clip whose camera and body motion the shot follows (up to 15 s)",
|
||||
@@ -82,6 +95,10 @@ export default defineCommand({
|
||||
},
|
||||
"first-frame": { type: "string", description: "video: the picture the shot starts from" },
|
||||
photo: { type: "string", description: "avatar: the person's photo" },
|
||||
avatar: {
|
||||
type: "string",
|
||||
description: "avatar: the person's own HeyGen avatar look id, instead of --photo",
|
||||
},
|
||||
script: { type: "string", description: "avatar, voice: the words spoken" },
|
||||
voice: { type: "string", description: "avatar, voice: a HeyGen voice id" },
|
||||
seconds: {
|
||||
@@ -126,9 +143,11 @@ export default defineCommand({
|
||||
.filter(Boolean),
|
||||
firstFrame: args["first-frame"],
|
||||
audio: args.audio,
|
||||
video: args.video,
|
||||
motion: args.motion,
|
||||
masks: repeated(rawArgs, "--mask").map(parseMask),
|
||||
photo: args.photo,
|
||||
avatar: args.avatar,
|
||||
script: args.script,
|
||||
voice: args.voice,
|
||||
seconds: num(args.seconds, "--seconds"),
|
||||
|
||||
@@ -56,7 +56,7 @@ export default defineCommand({
|
||||
engine: {
|
||||
type: "string",
|
||||
description:
|
||||
"ASR engine: auto (Parakeet if installed and it covers --language, else whisper), parakeet, whisper, or heygen (HeyGen's speech model; needs an API key, charged to API credits). Default: auto. Parakeet is more accurate and faster than whisper; install it with `hyperframes models install parakeet`.",
|
||||
"ASR engine: auto (Parakeet if installed and it covers --language, else whisper), parakeet, whisper, or heygen (HeyGen's speech model; bills the HyperFrames plan when signed in, or API credits with a key). Default: auto. Parakeet is more accurate and faster than whisper; install it with `hyperframes models install parakeet`.",
|
||||
alias: "e",
|
||||
},
|
||||
speakers: {
|
||||
|
||||
@@ -11,6 +11,13 @@ vi.mock("../cloud/auth.js", () => ({
|
||||
resolveCloudBaseUrl: () => "https://api.test",
|
||||
}));
|
||||
|
||||
vi.mock("../cloud/index.js", () => ({ createCloudClient: async () => ({}) }));
|
||||
vi.mock("../cloud/upload.js", () => ({
|
||||
uploadAssetViaDirectUpload: async ({ filename }: { filename: string }) => ({
|
||||
asset_id: `asset-${filename}`,
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.stubEnv("HEYGEN_API_KEY", "k");
|
||||
|
||||
const { checkRequest, generate, parseMask, urlOf } = await import("./generate.js");
|
||||
@@ -349,6 +356,7 @@ describe("generate", () => {
|
||||
}).usd,
|
||||
).toBe(0.07);
|
||||
expect(priceOf({ type: "voice", out: "a.mp3" }).usd).toBeNull();
|
||||
expect(creditsOf(priceOf({ type: "voice", out: "a.mp3", script: "今天聊黄金" }))).toBe(1);
|
||||
});
|
||||
|
||||
it("prices only a request the real call would accept", () => {
|
||||
@@ -373,12 +381,74 @@ describe("generate", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpeg)("redoes a shot's mouth to the exact voice", async () => {
|
||||
const video = shots("lips-shot.mp4", ["color=c=red:d=2"]);
|
||||
const audio = join(project, "lips-voice.mp3");
|
||||
writeFileSync(audio, "voice");
|
||||
const sent: { url: string; body?: Record<string, unknown> }[] = [];
|
||||
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
|
||||
sent.push({ url, ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}) });
|
||||
if (url === "https://files.test/lips.mp4") return new Response(new Uint8Array([1, 2, 3]));
|
||||
if (init?.method === "POST") return json({ data: { lipsync_id: "lip1" } });
|
||||
return json({
|
||||
data: { id: "lip1", status: "completed", video_url: "https://files.test/lips.mp4" },
|
||||
});
|
||||
});
|
||||
const out = join(project, "lips-out.mp4");
|
||||
const request = { type: "lipsync" as const, video, audio, out };
|
||||
const made = await generate(request);
|
||||
expect(made.job).toBe("lip1");
|
||||
expect(sent[0]!.url).toBe("https://api.test/v3/lipsyncs");
|
||||
expect(sent[0]!.body).toMatchObject({
|
||||
video: { type: "asset_id", asset_id: "asset-lips-shot.mp4" },
|
||||
audio: { type: "asset_id", asset_id: "asset-lips-voice.mp3" },
|
||||
mode: "speed",
|
||||
enable_dynamic_duration: false,
|
||||
});
|
||||
expect(sent[1]!.url).toBe("https://api.test/v3/lipsyncs/lip1");
|
||||
expect(existsSync(out)).toBe(true);
|
||||
expect(creditsOf(priceOf(request))).toBe(14);
|
||||
expect(() => checkRequest({ type: "lipsync", audio, out })).toThrow("--video");
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpeg)("speaks a finished voice as it is, from a photo or a look", async () => {
|
||||
const photo = join(project, "avatar-photo.png");
|
||||
execFileSync(ffmpeg!, [
|
||||
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=red:s=64x36"],
|
||||
...["-frames:v", "1", photo],
|
||||
]);
|
||||
const audio = join(project, "avatar-voice.mp3");
|
||||
execFileSync(ffmpeg!, [
|
||||
...["-v", "error", "-y", "-f", "lavfi", "-i", "sine=frequency=440:duration=6", audio],
|
||||
]);
|
||||
const sent: Record<string, unknown>[] = [];
|
||||
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
|
||||
if (url === "https://files.test/a.mp4") return new Response(new Uint8Array([1]));
|
||||
if (init?.body) sent.push(JSON.parse(String(init.body)));
|
||||
return json({
|
||||
data: { id: "v1", status: "completed", video_url: "https://files.test/a.mp4" },
|
||||
});
|
||||
});
|
||||
const out = join(project, "avatar-out.mp4");
|
||||
await generate({ type: "avatar", photo, audio, prompt: "talks with both hands", out });
|
||||
await generate({ type: "avatar", avatar: "look1", audio, out });
|
||||
expect(sent[0]).toMatchObject({
|
||||
type: "image",
|
||||
audio_asset_id: "asset-avatar-voice.mp3",
|
||||
motion_prompt: "talks with both hands",
|
||||
});
|
||||
expect(sent[0]).not.toHaveProperty("script");
|
||||
expect(sent[1]).toMatchObject({ type: "avatar", avatar_id: "look1" });
|
||||
expect(creditsOf(priceOf({ type: "avatar", photo, audio, out }))).toBe(117);
|
||||
expect(() => checkRequest({ type: "avatar", photo, out })).toThrow("--script");
|
||||
});
|
||||
|
||||
it("prices an avatar on the plan's avatar rows, not its API price", () => {
|
||||
const script = Array.from({ length: 30 }, () => "word").join(" ");
|
||||
const avatar = priceOf({ type: "avatar", out: "a.mp4", script });
|
||||
expect(creditsOf(avatar)).toBe(200);
|
||||
expect(creditsOf(avatar)).toBe(289);
|
||||
expect(avatar.list).toBeCloseTo(1.2167);
|
||||
expect(() => checkLimit(creditsOf(avatar), 220)).not.toThrow();
|
||||
expect(() => checkLimit(creditsOf(avatar), 300)).not.toThrow();
|
||||
});
|
||||
|
||||
it("charges a reference shot's pictures after the first two", () => {
|
||||
@@ -389,18 +459,18 @@ describe("generate", () => {
|
||||
references: Array.from({ length: pictures }, (_, i) => `${i}.png`),
|
||||
});
|
||||
const credits = [1, 2, 3, 9].map((n) => creditsOf(priceOf(shot(n))));
|
||||
expect(credits).toEqual([167, 167, 180, 260]);
|
||||
expect(credits).toEqual([250, 250, 270, 390]);
|
||||
expect(priceOf(shot(9)).list).toBeCloseTo(0.39);
|
||||
expect(() => checkLimit(credits[3]!, 250)).toThrow("about 260 credits, over --max-credits 250");
|
||||
expect(() => checkLimit(credits[3]!, 260)).not.toThrow();
|
||||
expect(() => checkLimit(credits[3]!, 380)).toThrow("about 390 credits, over --max-credits 380");
|
||||
expect(() => checkLimit(credits[3]!, 390)).not.toThrow();
|
||||
});
|
||||
|
||||
it("keeps a call under the person's credit limit, or refuses it before sending", () => {
|
||||
const shot = creditsOf(priceOf({ type: "video", out: "a.mp4", seconds: 6 }));
|
||||
expect(shot).toBe(72);
|
||||
expect(shot).toBe(108);
|
||||
expect(creditsOf(priceOf({ type: "image", out: "a.png" }))).toBe(67);
|
||||
expect(() => checkLimit(shot, 72)).not.toThrow();
|
||||
expect(() => checkLimit(shot, 30)).toThrow("about 72 credits, over --max-credits 30");
|
||||
expect(() => checkLimit(shot, 108)).not.toThrow();
|
||||
expect(() => checkLimit(shot, 30)).toThrow("about 108 credits, over --max-credits 30");
|
||||
expect(() => checkLimit(null, 30)).toThrow("cannot be priced ahead");
|
||||
expect(() => checkLimit(null, undefined)).not.toThrow();
|
||||
});
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
// New media on HeyGen for `media-use generate`: a still (optionally edited from a reference), a shot (optionally from
|
||||
// a first frame), a person speaking from one photo, a voice reading text, or music. Every call is priced first with
|
||||
// --plan, keyed for idempotency, downloaded into the project and recorded in the media ledger as generated.
|
||||
// a first frame), a shot's mouth redone to a voice, a person speaking from one photo, a voice reading text, or music.
|
||||
// Every call is priced first with --plan, keyed for idempotency, downloaded into the project and recorded in the media
|
||||
// ledger as generated.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, extname, join, relative, resolve } from "node:path";
|
||||
import { basename, dirname, extname, join, relative, resolve } from "node:path";
|
||||
import { findCuts, signatures } from "../measure/cuts.js";
|
||||
import { Fix, probeVideo, round, runFfmpeg, videoClip } from "../measure/media.js";
|
||||
import { idempotencyKey, runJob, type Job } from "./v3.js";
|
||||
|
||||
export type GenerateType = "image" | "video" | "avatar" | "voice" | "music";
|
||||
export type GenerateType = "image" | "video" | "lipsync" | "avatar" | "voice" | "music";
|
||||
export const GENERATE_TYPES: readonly GenerateType[] = [
|
||||
"image",
|
||||
"video",
|
||||
"lipsync",
|
||||
"avatar",
|
||||
"voice",
|
||||
"music",
|
||||
@@ -23,8 +26,13 @@ export interface GenerateRequest {
|
||||
prompt?: string;
|
||||
/** image: pictures the result is edited from or styled after. video: the person or things to keep in the shot. */
|
||||
references?: string[];
|
||||
/** video: speech the person in the references says, lips in sync (a stretch of the original's voice). */
|
||||
/**
|
||||
* video: speech the person in the references says. The model takes it as a reference and re-paces it, so the lips
|
||||
* follow the shot's own track, not this file. lipsync and avatar: the exact voice, which stays as it is.
|
||||
*/
|
||||
audio?: string;
|
||||
/** lipsync: the shot whose mouth is redone. */
|
||||
video?: string;
|
||||
/** video: a clip whose camera and body motion the shot follows (the original's same stretch, up to 15 s). */
|
||||
motion?: string;
|
||||
/** video: regions of the motion clip blanked before it is sent (its drawn text), so the shot does not copy them. */
|
||||
@@ -33,6 +41,8 @@ export interface GenerateRequest {
|
||||
firstFrame?: string;
|
||||
/** avatar: the photo of the person who speaks. */
|
||||
photo?: string;
|
||||
/** avatar: the person's own HeyGen avatar look, instead of a photo. */
|
||||
avatar?: string;
|
||||
/** avatar and voice: what is said. */
|
||||
script?: string;
|
||||
voice?: string;
|
||||
@@ -85,10 +95,7 @@ interface Route {
|
||||
ext: string[];
|
||||
}
|
||||
|
||||
type Media = (
|
||||
path: string,
|
||||
kind: "image" | "audio" | "video",
|
||||
) => { type: "base64"; media_type: string; data: string };
|
||||
type Media = (path: string, kind: "image" | "audio" | "video") => Record<string, unknown>;
|
||||
|
||||
const DEFAULTS = { image: "nano-banana-2", video: "heygen-video-1", music: "lyria-3-pro" } as const;
|
||||
const VIDEO_RESOLUTION = "768p";
|
||||
@@ -124,7 +131,7 @@ const ROUTES: Record<GenerateType, Route> = {
|
||||
poll: (id) => `/v3/models/videos/${id}`,
|
||||
// fallow-ignore-next-line complexity
|
||||
body: (r, media) => {
|
||||
// A person speaking given audio is a reference shot: the references show who, the audio drives the lips.
|
||||
// A person speaking given audio is a reference shot: the references show who, the audio guides the speech.
|
||||
const spoken = !!r.audio || !!r.references?.length || !!r.motion;
|
||||
if (spoken && r.firstFrame)
|
||||
throw new Fix(
|
||||
@@ -153,19 +160,46 @@ const ROUTES: Record<GenerateType, Route> = {
|
||||
ledger: "video",
|
||||
ext: [".mp4", ".mov", ".webm"],
|
||||
},
|
||||
// The voice stays exactly as given and only the mouth changes, so the shot lands on the voice from its first frame.
|
||||
lipsync: {
|
||||
path: "/v3/lipsyncs",
|
||||
poll: (id) => `/v3/lipsyncs/${id}`,
|
||||
body: (r, media) => ({
|
||||
video: media(need(r.video, "--video", "--video work/s01.mp4"), "video"),
|
||||
audio: media(need(r.audio, "--audio", "--audio work/s01.mp3"), "audio"),
|
||||
mode: "speed",
|
||||
enable_dynamic_duration: false,
|
||||
fps_mode: "cfr",
|
||||
keep_the_same_format: true,
|
||||
}),
|
||||
urls: ["video_url"],
|
||||
ledger: "video",
|
||||
ext: [".mp4"],
|
||||
},
|
||||
avatar: {
|
||||
path: "/v3/videos",
|
||||
poll: (id) => `/v3/videos/${id}`,
|
||||
body: (r, media) => ({
|
||||
type: "image",
|
||||
image: media(need(r.photo, "--photo", "--photo assets/presenter.png"), "image"),
|
||||
script: need(r.script, "--script", '--script "What they say"'),
|
||||
voice_id: need(
|
||||
r.voice,
|
||||
"--voice",
|
||||
"--voice <voice id> (npx hyperframes media-use resolve --type voice lists voices)",
|
||||
),
|
||||
...(r.avatar
|
||||
? { type: "avatar", avatar_id: r.avatar }
|
||||
: {
|
||||
type: "image",
|
||||
image: media(need(r.photo, "--photo", "--photo assets/presenter.png"), "image"),
|
||||
...(r.prompt ? { motion_prompt: r.prompt } : {}),
|
||||
}),
|
||||
// A finished voice stays exactly as it is and the lips land on it; a script is read in a voice.
|
||||
...(r.audio
|
||||
? { audio_asset_id: media(r.audio, "audio").asset_id }
|
||||
: {
|
||||
script: need(r.script, "--script", '--script "What they say"'),
|
||||
voice_id: need(
|
||||
r.voice,
|
||||
"--voice",
|
||||
"--voice <voice id> (npx hyperframes media-use resolve --type voice lists voices)",
|
||||
),
|
||||
}),
|
||||
...(r.aspect ? { aspect_ratio: r.aspect } : {}),
|
||||
resolution: "1080p",
|
||||
output_format: "mp4",
|
||||
}),
|
||||
urls: ["video_url"],
|
||||
@@ -264,6 +298,32 @@ function inline(work: string, masks: Mask[] = []): Media {
|
||||
};
|
||||
}
|
||||
|
||||
/** Files a route takes as asset ids, not inline. */
|
||||
function assetsOf(r: GenerateRequest): string[] {
|
||||
if (r.type === "lipsync") return [r.video!, r.audio!];
|
||||
return r.type === "avatar" && r.audio ? [r.audio] : [];
|
||||
}
|
||||
|
||||
/** Uploads them whole; the asset id of each path. */
|
||||
async function uploaded(paths: string[]): Promise<Map<string, string>> {
|
||||
const ids = new Map<string, string>();
|
||||
if (!paths.length) return ids;
|
||||
const { createCloudClient } = await import("../cloud/index.js");
|
||||
const { uploadAssetViaDirectUpload } = await import("../cloud/upload.js");
|
||||
const client = await createCloudClient();
|
||||
for (const path of paths) {
|
||||
const asset = await uploadAssetViaDirectUpload({
|
||||
client,
|
||||
bytes: new Uint8Array(readFileSync(path)),
|
||||
filename: basename(path),
|
||||
contentType: extname(path).toLowerCase() === ".mp4" ? "video/mp4" : "audio/mpeg",
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
ids.set(path, asset.asset_id);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* The seconds where a new shot starts in a video's first `seconds`, and the length read. A cut within a frame of
|
||||
* either end leaves a stray frame, not a second shot, and does not count.
|
||||
@@ -346,11 +406,18 @@ export async function generate(
|
||||
const project = projectOf(dirname(out));
|
||||
const work = mkdtempSync(join(tmpdir(), "hf-generate-"));
|
||||
try {
|
||||
const body = route.body(r, inline(work, r.masks));
|
||||
const key = idempotencyKey(route.path, [JSON.stringify(body)]);
|
||||
const sources = assetsOf(r);
|
||||
const ids = await uploaded(sources);
|
||||
const local = inline(work, r.masks);
|
||||
const body = route.body(r, (path, kind) =>
|
||||
ids.has(path) ? { type: "asset_id", asset_id: ids.get(path) } : local(path, kind),
|
||||
);
|
||||
// An asset id is new on every upload; the files say whether this is the same job.
|
||||
const sent = [...ids.values()].reduce((t, id) => t.replaceAll(id, ""), JSON.stringify(body));
|
||||
const key = idempotencyKey(route.path, [...sources.map((p) => readFileSync(p)), sent]);
|
||||
const job = await runJob(route.path, body, key, (id) => route.poll(encodeURIComponent(id)), {
|
||||
log: {
|
||||
projectDir: project ?? undefined,
|
||||
projectDir: project ?? process.cwd(),
|
||||
what: `generate ${r.type} ${relative(project ?? ".", out)}`,
|
||||
},
|
||||
onWait: (j) => onWait?.(j.status),
|
||||
@@ -398,7 +465,9 @@ export async function generate(
|
||||
...(r.references?.length ? { references: r.references.map(describe) } : {}),
|
||||
...(r.firstFrame ? { first_frame: describe(r.firstFrame) } : {}),
|
||||
...(r.photo ? { photo: describe(r.photo) } : {}),
|
||||
...(r.avatar ? { avatar: r.avatar } : {}),
|
||||
...(r.audio ? { audio: describe(r.audio) } : {}),
|
||||
...(r.video ? { video: describe(r.video) } : {}),
|
||||
...(r.motion ? { motion: describe(r.motion) } : {}),
|
||||
...(r.masks?.length
|
||||
? { motion_masks: r.masks.map((m) => [...m.box, ...(m.window ?? [])]) }
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Prices before a paid call, from the HyperFrames app's creation pricing (hyperframes-internal
|
||||
// packages/desktop/shared/creationPricing.mjs, 2026-10-22). HeyGen charges in units of $1/60, rounded up; the label
|
||||
// rounds up to the cent. Routes HeyGen does not price ahead say so instead of guessing.
|
||||
import { Fix, probeVideo, round } from "../measure/media.js";
|
||||
import { Fix, probeSeconds, probeVideo, round } from "../measure/media.js";
|
||||
import { GUIDE_MAX_S, type GenerateRequest } from "./generate.js";
|
||||
|
||||
export interface Price {
|
||||
@@ -39,14 +39,19 @@ const VIDEO_REF_768P_PER_S = 0.05;
|
||||
// A reference shot's first two pictures are included; each one after them bills on its own line.
|
||||
const VIDEO_INCLUDED_IMAGES = 2;
|
||||
const VIDEO_EXTRA_IMAGE = 0.02;
|
||||
// The price book's HyperFrames rows (experiment-framework price_book rules.json, 2026-10-28).
|
||||
const HF_VIDEO_768P_PER_S = 12;
|
||||
const HF_VIDEO_REF_768P_PER_S = 100 / 3;
|
||||
const HF_VIDEO_EXTRA_IMAGE = 40 / 3;
|
||||
const HF_AVATAR_IV_PHOTO_PER_S = 800 / 60;
|
||||
// The price book's HyperFrames rows (experiment-framework price_book rules.json, 2026-10-30).
|
||||
const HF_VIDEO_768P_PER_S = 18;
|
||||
const HF_VIDEO_REF_768P_PER_S = 50;
|
||||
const HF_VIDEO_EXTRA_IMAGE = 20;
|
||||
const HF_AVATAR_IV_PHOTO_PER_S = 1155 / 60;
|
||||
const HF_AVATAR_IV_TWIN_PER_S = 2415 / 60;
|
||||
// Billed on the whole video, not the voice: measured 2026-11-01, a 13.75 s shot took 97 credits.
|
||||
const HF_LIPSYNC_SPEED_PER_S = 7;
|
||||
const MUSIC_PER_TRACK = 0.08;
|
||||
const AVATAR_IV_PER_MIN = 4.83;
|
||||
const SCRIPT_WORDS_PER_S = 2;
|
||||
const HF_TTS_PER_S = 1;
|
||||
const SPEECH_CHARS_PER_S = { zh: 5, en: 15 };
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
export function priceOf(r: GenerateRequest): Price {
|
||||
@@ -79,24 +84,39 @@ export function priceOf(r: GenerateRequest): Price {
|
||||
seconds * HF_VIDEO_768P_PER_S,
|
||||
);
|
||||
}
|
||||
case "lipsync": {
|
||||
const seconds = r.video ? probeVideo(r.video).pictureDuration : 0;
|
||||
return {
|
||||
usd: null,
|
||||
list: null,
|
||||
credits: seconds * HF_LIPSYNC_SPEED_PER_S,
|
||||
basis: `lipsync at speed, ${round(seconds, 2)} s at ${HF_LIPSYNC_SPEED_PER_S} credits a second`,
|
||||
};
|
||||
}
|
||||
case "avatar": {
|
||||
const words = (r.script ?? "").split(/\s+/).filter(Boolean).length;
|
||||
const seconds = Math.max(1, Math.floor(words / SCRIPT_WORDS_PER_S));
|
||||
const seconds = r.audio
|
||||
? probeSeconds(r.audio)
|
||||
: Math.max(1, Math.floor(words / SCRIPT_WORDS_PER_S));
|
||||
return at(
|
||||
Math.ceil(seconds * AVATAR_IV_PER_MIN) / 60,
|
||||
`avatar video, about ${seconds} s from ${words} words at $4.83 a minute`,
|
||||
seconds * HF_AVATAR_IV_PHOTO_PER_S,
|
||||
`avatar video, about ${round(seconds, 1)} s${r.audio ? "" : ` from ${words} words`} at $4.83 a minute`,
|
||||
seconds * (r.avatar ? HF_AVATAR_IV_TWIN_PER_S : HF_AVATAR_IV_PHOTO_PER_S),
|
||||
);
|
||||
}
|
||||
case "music":
|
||||
return at(MUSIC_PER_TRACK, "lyria-3-pro, $0.08 a track");
|
||||
case "voice":
|
||||
case "voice": {
|
||||
const text = r.script ?? "";
|
||||
const cjk = (text.match(/\p{Script=Han}/gu) ?? []).length;
|
||||
const seconds = cjk / SPEECH_CHARS_PER_S.zh + (text.length - cjk) / SPEECH_CHARS_PER_S.en;
|
||||
return {
|
||||
usd: null,
|
||||
list: null,
|
||||
credits: null,
|
||||
basis: "HeyGen does not price catalog speech ahead; it is charged per character",
|
||||
credits: seconds * HF_TTS_PER_S,
|
||||
basis: `about ${round(seconds, 1)} s of speech at ${HF_TTS_PER_S} credit a second, an estimate: it is charged per character`,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -179,7 +179,7 @@ async function call(
|
||||
const data = isRecord(value) && isRecord(value.data) ? value.data : value;
|
||||
if (!isRecord(data))
|
||||
throw new Fix(`HeyGen answered ${route} with no job`, "run the same command again");
|
||||
const id = data.id ?? data.job_id ?? data.video_id ?? data.generation_id;
|
||||
const id = data.id ?? data.job_id ?? data.video_id ?? data.generation_id ?? data.lipsync_id;
|
||||
return {
|
||||
...data,
|
||||
id: typeof id === "string" ? id : "",
|
||||
|
||||
@@ -86,6 +86,17 @@ const rate = (text: string | undefined): number => {
|
||||
return b ? (a ?? 0) / b : (a ?? 0);
|
||||
};
|
||||
|
||||
/** A sound's length in seconds. */
|
||||
export function probeSeconds(path: string): number {
|
||||
if (!existsSync(path)) throw new Fix(`no file at ${path}`, "pass a sound from the project");
|
||||
const text = execFileSync(
|
||||
ffprobeBin(),
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", path],
|
||||
{ maxBuffer: MAX_BUFFER, windowsHide: true },
|
||||
).toString();
|
||||
return Number(text) || 0;
|
||||
}
|
||||
|
||||
/** Size (as displayed, rotation applied), clock and sound of a video. */
|
||||
// fallow-ignore-next-line complexity
|
||||
export function probeVideo(path: string): VideoInfo {
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
// (e.g. "make an image with codex").
|
||||
|
||||
import { bgmProvider } from "./bgm-provider.mjs";
|
||||
import { OPENVERSE_SFX_PROVIDER, sfxProvider } from "./sfx-provider.mjs";
|
||||
import { heygenSfxProvider, OPENVERSE_SFX_PROVIDER, sfxProvider } from "./sfx-provider.mjs";
|
||||
import { bundledSfxProvider } from "./bundled-sfx-provider.mjs";
|
||||
import { imageProvider, iconProvider } from "./image-provider.mjs";
|
||||
import { brandProvider } from "./brand-provider.mjs";
|
||||
@@ -48,6 +48,7 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem
|
||||
const REGISTRY = {
|
||||
bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })],
|
||||
sfx: [
|
||||
N("heygen.audio.sounds", { search: heygenSfxProvider.search }),
|
||||
N(OPENVERSE_SFX_PROVIDER, { search: sfxProvider.search }),
|
||||
A("bundled.sfx", { search: bundledSfxProvider.search }),
|
||||
],
|
||||
|
||||
@@ -1,7 +1,30 @@
|
||||
import { heygenSearch } from "./heygen-search.mjs";
|
||||
import { OpenverseUnavailableError, searchOpenverseSfx } from "./openverse.mjs";
|
||||
|
||||
export const OPENVERSE_SFX_PROVIDER = "openverse.audio";
|
||||
|
||||
export const heygenSfxProvider = {
|
||||
async search(intent) {
|
||||
// Good effects score 0.5-0.67, under the server's default floor of 0.7.
|
||||
const results = heygenSearch("audio sounds list", intent, {
|
||||
type: "sound_effects",
|
||||
minScore: 0.4,
|
||||
});
|
||||
if (!results) return null;
|
||||
const best = results[0];
|
||||
return {
|
||||
url: best.audio_url,
|
||||
source: "search",
|
||||
metadata: {
|
||||
description: best.description || intent,
|
||||
duration: best.duration || null,
|
||||
provider: "heygen.audio.sounds",
|
||||
provenance: { track_id: best.id, score: best.score, query: intent },
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export const sfxProvider = {
|
||||
async search(intent) {
|
||||
let sound;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"source": "heygen-com/hyperframes",
|
||||
"skills": {
|
||||
"creator-remix": {
|
||||
"hash": "fb348339ce2fa040",
|
||||
"hash": "1841826491d93706",
|
||||
"files": 2
|
||||
},
|
||||
"figma": {
|
||||
@@ -22,7 +22,7 @@
|
||||
"files": 13
|
||||
},
|
||||
"media-use": {
|
||||
"hash": "f049425ce5d31ec9",
|
||||
"hash": "5be81ddc174ac351",
|
||||
"files": 117
|
||||
},
|
||||
"remotion-to-hyperframes": {
|
||||
|
||||
@@ -20,7 +20,7 @@ It prints one line, `resolved <id> → <path> (<type>, <metadata>)`. List reusab
|
||||
| Need | Free | HyperFrames credits | Local or the person's own key (only when they ask) |
|
||||
| --- | --- | --- | --- |
|
||||
| Music (`bgm`) | HeyGen catalog of 10k+ tracks (signed in) | Generated track | Local MusicGen |
|
||||
| Sound effects (`sfx`) | Openverse (credit CC BY sounds); 21 bundled offline | | |
|
||||
| Sound effects (`sfx`) | HeyGen (signed in); Openverse (credit CC BY); bundled | | |
|
||||
| Images, icons | HeyGen asset search, 75k+ (signed in) | Generated images | Local FLUX (mflux), if installed |
|
||||
| Logos (`logo`) | theSVG, GitHub avatar, favicon; never redrawn | | |
|
||||
| Voiceover (`voice`) | HeyGen Voice Turbo (signed in) | Other HeyGen voices, voice cloning | `npx hyperframes tts` (Kokoro on device) |
|
||||
|
||||
@@ -6,11 +6,11 @@
|
||||
|
||||
`npx hyperframes auth status` says what is signed in. The credential is read in this order: `HEYGEN_API_KEY`, `HYPERFRAMES_API_KEY`, a host app's `HEYGEN_ACCESS_TOKEN`, then `~/.hyperframes/credentials`, which `npx hyperframes auth login` writes. `heygen auth login` is a separate sign-in that only the `heygen` CLI reads.
|
||||
|
||||
| Credential | Pays with | Covers |
|
||||
| -------------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------- |
|
||||
| API key | API credits | everything below |
|
||||
| Sign-in (OAuth) | the plan's allowance on subscription credits | avatar videos, video shots, images and voices; music and transcription take an API key |
|
||||
| Inside the HyperFrames app | the app | whatever the app allows; nothing to set up |
|
||||
| Credential | Pays with | Covers |
|
||||
| -------------------------- | -------------------------------------------- | ---------------------------------------------------------------------------- |
|
||||
| API key | API credits | everything below |
|
||||
| Sign-in (OAuth) | the plan's allowance on subscription credits | avatar videos, video shots, lipsync, images, voices, music and transcription |
|
||||
| Inside the HyperFrames app | the app | whatever the app allows; nothing to set up |
|
||||
|
||||
A refusal names its fix: `api_key_required` needs an API key, `402` or `insufficient_credit` needs a top-up, `hf_usage_exhausted` means the plan's allowance is used up.
|
||||
|
||||
@@ -27,10 +27,14 @@ npx hyperframes media-use generate --type video --first-frame assets/s03-first.p
|
||||
# in sync and moving as the guide (the original shot itself) does; --mask x,y,w,h[,a-b] (repeatable, the guide's
|
||||
# pixels and seconds) blanks text in the guide, which HeyGen would copy
|
||||
npx hyperframes media-use generate --type video --reference assets/person.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165,0.4-2.0 --prompt "talks to camera, seated" --seconds 8 --aspect 9:16 --out assets/s02-person.mp4
|
||||
# a speaking shot follows its own re-paced audio, not --audio: to land it on a set voice, redo its mouth to that voice
|
||||
npx hyperframes media-use generate --type lipsync --video assets/s02-person.mp4 --audio assets/s02-voice.mp3 --out assets/s02-lips.mp4
|
||||
# the same for a person who does not speak: no --audio, the guide clip carries the motion
|
||||
npx hyperframes media-use generate --type video --reference assets/person.png --motion assets/s05-original.mp4 --prompt "holds up a jar, smiles" --seconds 6 --aspect 9:16 --out assets/s05-person.mp4
|
||||
# a person in a photo speaking a script in a HeyGen voice
|
||||
npx hyperframes media-use generate --type avatar --photo assets/person.png --script "…" --voice <voice-id> --out assets/person.mp4
|
||||
# a finished voice stays exactly as it is: --audio replaces --script, --avatar <look id> replaces --photo, --prompt moves a photo
|
||||
npx hyperframes media-use generate --type avatar --photo assets/person.png --audio assets/vo.mp3 --prompt "seated, talks with both hands" --aspect 16:9 --out assets/host.mp4
|
||||
# a voice reading text
|
||||
npx hyperframes media-use generate --type voice --script "…" --voice <voice-id> --out assets/vo.mp3
|
||||
# music, 30, 60, 120 or 180 s (instrumental unless --vocals), trimmed to length in the composition
|
||||
@@ -39,13 +43,14 @@ npx hyperframes media-use generate --type music --prompt "warm lo-fi, soft keys"
|
||||
|
||||
The file lands at `--out` and the media ledger records it as generated, with the route, model, job, prompt and the files it was made from. Pictures and shots carry no text or logos: ask for clean plates and draw words in HyperFrames.
|
||||
|
||||
| Type | Model | Price |
|
||||
| ------ | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first |
|
||||
| video | heygen-video-1, 768p | $0.02 a second from a prompt or first frame; $0.05 a second with a reference picture, voice or guide clip, and the guide's seconds count too (1080p costs about twice) |
|
||||
| avatar | talking photo | $4.83 a minute of speech |
|
||||
| music | lyria-3-pro | $0.08 a track |
|
||||
| voice | the voice's engine | charged per character, not priced ahead |
|
||||
| Type | Model | Price |
|
||||
| ------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first |
|
||||
| video | heygen-video-1, 768p | $0.02 a second from a prompt or first frame; $0.05 a second with a reference picture, voice or guide clip, and the guide's seconds count too (1080p costs about twice) |
|
||||
| lipsync | HeyGen lipsync, speed | 7 credits a second of the video on a plan |
|
||||
| avatar | talking photo | $4.83 a minute of speech |
|
||||
| music | lyria-3-pro | $0.08 a track |
|
||||
| voice | the voice's engine | charged per character, not priced ahead |
|
||||
|
||||
`--plan`'s `usd` is what an API key is charged (HeyGen rounds up to units of $1/60); its `credits` is what a HyperFrames plan charges, on the plan's own rates: a video shot's seconds and guide seconds, plus each picture past the first two. Voice ids come from `heygen voice list` or `npx hyperframes media-use resolve --type voice`; a cloned voice from `heygen voice clone create` (`operations.md`).
|
||||
|
||||
@@ -55,4 +60,4 @@ The file lands at `--out` and the media ledger records it as generated, with the
|
||||
npx hyperframes transcribe interview.mp4 --engine heygen --speakers
|
||||
```
|
||||
|
||||
Word times from HeyGen's speech model, each word with its speaker under `--speakers`, in the same `transcript.json` local engines write. It takes an API key. The local engines (`--engine auto`) are free and good for one clear voice; HeyGen is the better choice for several speakers, noise or music under the voice.
|
||||
Word times from HeyGen's speech model, each word with its speaker under `--speakers`, in the same `transcript.json` local engines write. The local engines (`--engine auto`) are free and good for one clear voice; HeyGen is the better choice for several speakers, noise or music under the voice.
|
||||
|
||||
Reference in New Issue
Block a user