From 10bc426e321dcf1bdbbb00ed9dc5d54faf2ce9f9 Mon Sep 17 00:00:00 2001 From: James Russo Date: Tue, 3 Nov 2026 08:59:27 +0000 Subject: [PATCH] feat(cli): call HeyGen's v3 media routes through the generated client (#7063) * feat(cli): call HeyGen's v3 media routes through the generated client The media-use generate routes (image, video, avatar, voice, music), HeyGen transcription and the public voice list called fetch by hand. They now go through the generated cloud client, regenerated with the create and status methods for those routes. runJob takes a job route (its path, how it starts, how its status is read) in place of a path and a poll-path function. Keys, retries, timeouts, polling and error messages are unchanged. Speech answers in the request and the API takes no Idempotency-Key there, so it has no status to poll and no key header is sent. Co-Authored-By: Claude Opus 5.5 (1M context) * chore(cli): keep v3.ts comment share at the base Co-Authored-By: Claude Opus 5.5 (1M context) * chore(cli): regenerate the cloud client without API description text The generated client and types now carry names and types only, with no JSDoc taken from the API spec's summaries and descriptions. The source contract test checks that no method carries generated description text, in place of checking the uploadAsset description. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- packages/cli/src/cloud/_gen/client.ts | 261 ++++- packages/cli/src/cloud/_gen/types.ts | 915 ++++++++++++------ .../cloud/generatedClient.contract.test.ts | 10 +- packages/cli/src/heygen/generate.test.ts | 116 ++- packages/cli/src/heygen/generate.ts | 34 +- packages/cli/src/heygen/transcribe.test.ts | 46 + packages/cli/src/heygen/transcribe.ts | 17 +- packages/cli/src/heygen/v3.test.ts | 137 ++- packages/cli/src/heygen/v3.ts | 117 +-- 9 files changed, 1219 insertions(+), 434 deletions(-) diff --git a/packages/cli/src/cloud/_gen/client.ts b/packages/cli/src/cloud/_gen/client.ts index b8863c6de5..850fedefb1 100644 --- a/packages/cli/src/cloud/_gen/client.ts +++ b/packages/cli/src/cloud/_gen/client.ts @@ -5,15 +5,35 @@ * experiment-framework to regenerate. */ import type { + BackgroundRemovalJob, CompleteAssetUploadRequest, CompleteAssetUploadResponse, CreateAssetUploadRequest, CreateAssetUploadResponse, + CreateBackgroundRemovalRequest, CreateHyperframesRenderRequest, CreateHyperframesRenderResponse, + CreateLipsyncRequest, + CreateModelVideoRequest, + CreateModelVideoResponse, + CreateSpeechV3Request, + CreateSpeechV3ResponseData, + CreateVideoV3RequestBody, + CreateVideoV3Response, DeleteHyperframesRenderResponse, HyperframesRenderDetail, + ImageCreateRequest, + LipsyncCreateResponse, + LipsyncDetail, + ModelVideoResponse, + MusicCreateRequest, + MusicResponse, + ThirdPartyImageResponse, + TranscriptionCreateRequest, + TranscriptionResponse, UploadAssetV3Response, + VideoDetail, + VoiceV3Item, } from "./types.js"; export type AuthHeaders = Record; @@ -201,11 +221,6 @@ export class HyperframesCloudClient { }); } - /** - * Upload Asset - * - * Uploads a file (image, video, audio, PDF, or SRT subtitle) and returns an asset_id for use in other endpoints. Max 50 MB, or 32 MB for an image. Supported types: png, jpeg, mp4, webm, mp3, wav, pdf, srt. A file is checked against the content policy after it is uploaded; if it is blocked, the asset is removed and later requests for it return not found. - */ async uploadAsset(args: { file: Blob | Buffer | Uint8Array; filename: string; @@ -229,11 +244,6 @@ export class HyperframesCloudClient { }); } - /** - * Create Asset Upload - * - * Begin a direct-to-S3 upload. Returns an asset_id and a presigned upload_url; PUT the file bytes to upload_url, then call POST /v3/assets/{asset_id}/complete. Unlike POST /v3/assets (which proxies the bytes), this never sends the file through the API. - */ async createAssetUpload(args: { body: CreateAssetUploadRequest; idempotencyKey?: string; @@ -248,11 +258,6 @@ export class HyperframesCloudClient { }); } - /** - * Complete Asset Upload - * - * Finalize a direct-to-S3 upload into a reusable asset. Call after the upload PUT returns 200. Idempotent: repeated calls return the same finalized asset. - */ async completeAssetUpload(args: { asset_id: string; body: CompleteAssetUploadRequest; @@ -266,11 +271,6 @@ export class HyperframesCloudClient { }); } - /** - * Create HyperFrames Render - * - * Renders a HyperFrames composition (an HTML+JS+assets project bundled as a .zip) into a video. Submit the project via `url`, `asset_id` (pre-uploaded via POST /v3/assets), or inline `base64`. Returns a `render_id` to poll via GET /v3/hyperframes/renders/{render_id}. - */ async createRender(args: { body: CreateHyperframesRenderRequest; idempotencyKey?: string; @@ -285,11 +285,6 @@ export class HyperframesCloudClient { }); } - /** - * List HyperFrames Renders - * - * Returns a cursor-paginated list of HyperFrames renders in the account, newest first. - */ async listRenders(args: { limit?: number; token?: string; signal?: AbortSignal }): Promise<{ data?: Array; has_more?: boolean; @@ -312,11 +307,6 @@ export class HyperframesCloudClient { }); } - /** - * Get HyperFrames Render - * - * Returns full details for a single HyperFrames render, including status and signed video_url when complete. - */ async getRender(args: { render_id: string; signal?: AbortSignal; @@ -328,11 +318,6 @@ export class HyperframesCloudClient { }); } - /** - * Delete HyperFrames Render - * - * Soft-deletes a HyperFrames render. Subsequent GETs return 404. - */ async deleteRender(args: { render_id: string; signal?: AbortSignal; @@ -343,4 +328,210 @@ export class HyperframesCloudClient { signal: args.signal, }); } + + async createImage(args: { + body: ImageCreateRequest; + idempotencyKey: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/images", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getImage(args: { + generation_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/images/${encodeURIComponent(args.generation_id)}`, + signal: args.signal, + }); + } + + async createModelVideo(args: { + body: CreateModelVideoRequest; + idempotencyKey?: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/models/videos", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getModelVideo(args: { + video_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/models/videos/${encodeURIComponent(args.video_id)}`, + signal: args.signal, + }); + } + + async createVideo(args: { + body: CreateVideoV3RequestBody; + idempotencyKey?: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/videos", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getVideoV3(args: { video_id: string; signal?: AbortSignal }): Promise { + return await this.request({ + method: "GET", + path: `/v3/videos/${encodeURIComponent(args.video_id)}`, + signal: args.signal, + }); + } + + async listVoicesV3(args: { + type?: "public" | "private"; + engine?: string; + language?: string; + gender?: string; + limit?: number; + token?: string; + signal?: AbortSignal; + }): Promise<{ data?: Array; has_more?: boolean; next_token?: string | null }> { + const query: Record = { + type: args.type, + engine: args.engine, + language: args.language, + gender: args.gender, + limit: args.limit, + token: args.token, + }; + return await this.request<{ + data?: Array; + has_more?: boolean; + next_token?: string | null; + }>({ + method: "GET", + path: "/v3/voices", + query, + unwrapData: false, + signal: args.signal, + }); + } + + async createSpeechV3(args: { + body: CreateSpeechV3Request; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/voices/speech", + body: args.body, + signal: args.signal, + }); + } + + async createMusic(args: { + body: MusicCreateRequest; + idempotencyKey: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/audio/music", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getMusic(args: { music_id: string; signal?: AbortSignal }): Promise { + return await this.request({ + method: "GET", + path: `/v3/audio/music/${encodeURIComponent(args.music_id)}`, + signal: args.signal, + }); + } + + async createTranscription(args: { + body: TranscriptionCreateRequest; + idempotencyKey: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/audio/transcriptions", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getTranscription(args: { + transcription_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/audio/transcriptions/${encodeURIComponent(args.transcription_id)}`, + signal: args.signal, + }); + } + + async createLipsyncV3(args: { + body: CreateLipsyncRequest; + idempotencyKey?: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/lipsyncs", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getLipsyncV3(args: { lipsync_id: string; signal?: AbortSignal }): Promise { + return await this.request({ + method: "GET", + path: `/v3/lipsyncs/${encodeURIComponent(args.lipsync_id)}`, + signal: args.signal, + }); + } + + async createBackgroundRemoval(args: { + body: CreateBackgroundRemovalRequest; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/background-removals", + body: args.body, + signal: args.signal, + }); + } + + async getBackgroundRemoval(args: { + job_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/background-removals/${encodeURIComponent(args.job_id)}`, + signal: args.signal, + }); + } } diff --git a/packages/cli/src/cloud/_gen/types.ts b/packages/cli/src/cloud/_gen/types.ts index fea4d799d1..51804d8de8 100644 --- a/packages/cli/src/cloud/_gen/types.ts +++ b/packages/cli/src/cloud/_gen/types.ts @@ -9,409 +9,744 @@ // Add a new path/method to TARGET_ENDPOINTS in // scripts/generate_hyperframes_cli_client.py to extend this list. -/** - * Asset input via base64-encoded content. - */ export interface AssetBase64 { - /** - * Input type discriminator - */ type: "base64"; - /** - * MIME type of the encoded content (e.g. "image/png") - */ media_type: string; - /** - * Base64-encoded file content - */ data: string; } -/** - * Asset input via HeyGen asset ID from the asset upload endpoint. - */ export interface AssetId { - /** - * Input type discriminator - */ type: "asset_id"; - /** - * HeyGen asset ID from an asset upload or import; not a third-party file ID. - */ asset_id: string; } -/** - * Asset input via publicly accessible HTTPS URL. - */ export interface AssetUrl { - /** - * Input type discriminator - */ type: "url"; - /** - * Public http(s) URL HeyGen can download without auth or redirects. Not a - * local or sandbox path (/mnt/data/..., file://, sandbox:); upload such files - * and pass asset_id. - */ url: string; } -/** - * Finalize a presigned upload (POST /v3/assets/{asset_id}/complete). - */ +export type AudioVoiceType = "public" | "private"; + +export type AvatarFit = "contain" | "cover"; + +export interface AvatarIIIEngineConfig { + type: "avatar_iii"; +} + +export interface AvatarIVEngineConfig { + type: "avatar_iv"; +} + +export interface AvatarVEngineConfig { + type: "avatar_v"; + reference_look_id?: string | null; + more_blinking?: boolean; +} + +export interface AvatarVideoScene { + type: "avatar_video"; + input: StudioAvatarInput; +} + +export interface BackgroundRemovalJob { + id: string; + status: string; + layers?: Record; + duration?: number | null; + created_at?: number | null; + completed_at?: number | null; + error?: string | null; +} + +export interface BackgroundSetting { + type: "color" | "image"; + value?: string | null; + url?: string | null; + asset_id?: string | null; +} + +export type CaptionFileFormat = "srt"; + +export interface CaptionSetting { + file_format?: CaptionFileFormat; + style?: CaptionStyle | null; +} + +export type CaptionStyle = "default"; + +export type CinematicAvatarModel = "seedance_2" | "heygen_video_1"; + export interface CompleteAssetUploadRequest { - /** - * Optional SHA256 (hex) cross-check. - */ checksum_sha256?: string | null; } -/** - * Result of finalizing an upload. - */ export interface CompleteAssetUploadResponse { - /** - * The reusable asset identifier. - */ asset_id: string; - /** - * Public URL of the finalized asset. - */ url: string; - /** - * MIME type detected from the stored bytes. - */ mime_type: string; - /** - * Size of the stored object in bytes. - */ size_bytes: number; - /** - * Asset status, e.g. 'processing'. - */ status: "processing"; } -/** - * Request to begin a presigned direct-to-S3 upload (POST - * /v3/assets/direct-uploads). - */ export interface CreateAssetUploadRequest { - /** - * Original filename for reference/metadata. The stored object's extension is - * derived from content_type. - */ filename: string; - /** - * Declared MIME type (e.g. 'video/mp4', 'image/png', 'audio/mpeg', - * 'application/pdf', 'application/zip'). Verified against the stored bytes at - * completion. - */ content_type: string; - /** - * Exact byte size of the file. Signed into the upload URL so it cannot be - * exceeded. - */ size_bytes: number; - /** - * Optional SHA256 of the file as hex. When provided, S3 enforces it on upload. - */ checksum_sha256?: string | null; } -/** - * Presigned upload instructions. - */ export interface CreateAssetUploadResponse { - /** - * Reusable asset identifier. Becomes usable after POST - * /v3/assets/{asset_id}/complete. - */ asset_id: string; - /** - * Presigned S3 URL. PUT the raw file bytes here. - */ upload_url: string; - /** - * Headers that must be sent verbatim on the PUT request. - */ upload_headers: Record; - /** - * Seconds until the upload URL expires. - */ expires_in_seconds: number; - /** - * Maximum allowed upload size in bytes for this content type: 1.5 GB - * (1,500,000,000 bytes) per audio file and 200 MiB (209,715,200 bytes) for - * other supported files. - */ max_bytes: number; - /** - * Upload lifecycle status. Always 'pending_upload' here. - */ status: "pending_upload"; } -/** - * Request body for POST /v3/hyperframes/renders. - */ +export interface CreateBackgroundRemovalRequest { + video: AssetUrl | AssetId | AssetBase64; + layers?: Array; + title?: string | null; + request_id?: string | null; +} + export interface CreateHyperframesRenderRequest { - /** - * HyperFrames composition .zip — provide as {type: 'url', url: '...'}, {type: - * 'asset_id', asset_id: '...'} (pre-uploaded via POST /v3/assets), or {type: - * 'base64', media_type: 'application/zip', data: '...'}. Zip must contain - * index.html at the root (or the path you set in `composition`). - */ project: AssetUrl | AssetId | AssetBase64; - /** - * Output frames per second. Defaults to 30 if not provided. - */ fps?: number | null; - /** - * Render quality preset; higher quality is slower. - */ quality?: "draft" | "standard" | "high"; - /** - * Output container/codec. - */ format?: "mp4" | "webm" | "mov"; - /** - * Overrides the quality preset's constant rate factor (lower is higher quality - * and a larger file). Integer 0-51 for mp4, 0-63 for webm; ignored for mov. - * Omit to use the preset's CRF. - */ crf?: number | null; - /** - * Output resolution tier. Defaults to '1080p'. Pass '4k' for 4K renders - * (billed at 1.5x). - */ resolution?: HyperframesResolution; - /** - * Output aspect ratio. Defaults to '16:9' (landscape). Pass '9:16' for - * portrait or '1:1' for square. - */ aspect_ratio?: HyperframesAspectRatio; - /** - * Entry HTML file relative to the project root (e.g. compositions/intro.html). - * Defaults to index.html when omitted. - */ composition?: string | null; - /** - * Optional overrides for the composition's data-composition-variables. Use - * this to parameterise a single composition across multiple renders. - */ variables?: Record | null; - /** - * Free-text label for the render; echoed back in detail responses. - */ title?: string | null; - /** - * Opaque client tracking ID, echoed back in webhook payloads. - */ callback_id?: string | null; - /** - * Per-request HTTPS webhook URL the render fires when it terminates. - */ callback_url?: string | null; } -/** - * Response for POST /v3/hyperframes/renders. - */ export interface CreateHyperframesRenderResponse { - /** - * HyperFrames render identifier — poll GET /v3/hyperframes/renders/{render_id} - * for status. - */ render_id: string; } -/** - * Response for DELETE /v3/hyperframes/renders/{render_id}. - */ +export interface CreateLipsyncRequest { + video: AssetUrl | AssetId; + audio: AssetUrl | AssetId; + title?: string | null; + mode?: VideoTranslationMode; + callback_url?: string | null; + callback_id?: string | null; + enable_caption?: boolean; + keep_the_same_format?: boolean | null; + enable_dynamic_duration?: boolean; + disable_music_track?: boolean; + enable_speech_enhancement?: boolean; + enable_watermark?: boolean; + start_time?: number | null; + end_time?: number | null; + fps_mode?: string | null; + folder_id?: string | null; +} + +export type CreateModelVideoRequest = + | TextToVideoRequest + | ImageToVideoRequest + | ReferenceToVideoRequest; + +export interface CreateModelVideoResponse { + video_id: string; + status?: "pending"; + input_video_duration?: number | null; +} + +export interface CreateSpeechV3OutputFormat { + encoding: "mp3" | "wav"; + sample_rate_hertz?: number | null; + bit_depth?: number | null; + num_channels?: number; + bitrate_bps?: number | null; +} + +export interface CreateSpeechV3Request { + text: string; + voice_id: string; + input_type?: string; + speed?: number; + language?: string | null; + locale?: string | null; + force_regenerate?: boolean; + engine?: "heygen_voice" | "starfish" | "orca" | "elevenlabs" | "elevenlabs_v3" | null; + settings?: ElevenLabsSpeechSettings | null; + brand_glossary_id?: string | null; + output_format?: CreateSpeechV3OutputFormat | null; +} + +export interface CreateSpeechV3ResponseData { + audio_url: string; + duration: number; + request_id?: string | null; + word_timestamps?: Array | null; + engine: SpeechEngine; +} + +export interface CreateVideoFromAvatar { + title?: string | null; + folder_id?: string | null; + resolution?: VideoResolution | null; + aspect_ratio?: VideoAspectRatio | null; + fit?: AvatarFit | null; + background?: BackgroundSetting | null; + remove_background?: boolean | null; + callback_url?: string | null; + callback_id?: string | null; + watermark?: WatermarkInput | null; + caption?: CaptionSetting | null; + output_format?: VideoOutputFormat; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; + type: "avatar"; + avatar_id: string; + brand_glossary_id?: string | null; + motion_prompt?: string | null; + expressiveness?: Expressiveness | null; + engine?: AvatarVEngineConfig | AvatarIVEngineConfig | AvatarIIIEngineConfig | null; +} + +export interface CreateVideoFromCinematicAvatar { + type: "cinematic_avatar"; + prompt: string; + avatar_id: Array; + model?: CinematicAvatarModel | null; + references?: Array | null; + aspect_ratio?: "16:9" | "9:16" | "1:1"; + resolution?: "720p" | "1080p"; + auto_duration?: boolean; + duration?: number | null; + enhance_prompt?: boolean; + title?: string | null; + folder_id?: string | null; +} + +export interface CreateVideoFromImage { + title?: string | null; + folder_id?: string | null; + resolution?: VideoResolution | null; + aspect_ratio?: VideoAspectRatio | null; + fit?: AvatarFit | null; + background?: BackgroundSetting | null; + remove_background?: boolean | null; + callback_url?: string | null; + callback_id?: string | null; + watermark?: WatermarkInput | null; + caption?: CaptionSetting | null; + output_format?: VideoOutputFormat; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; + type: "image"; + image: AssetUrl | AssetId | AssetBase64; + brand_glossary_id?: string | null; + motion_prompt?: string | null; + expressiveness?: Expressiveness | null; +} + +export interface CreateVideoFromStudio { + type: "studio"; + title?: string | null; + folder_id?: string | null; + aspect_ratio?: VideoAspectRatio | null; + resolution?: VideoResolution | null; + brand_glossary_id?: string | null; + callback_url?: string | null; + callback_id?: string | null; + caption?: CaptionSetting | null; + watermark?: WatermarkInput | null; + scenes: Array; +} + +export type CreateVideoV3RequestBody = + | CreateVideoFromAvatar + | CreateVideoFromImage + | CreateVideoFromCinematicAvatar + | CreateVideoFromStudio; + +export interface CreateVideoV3Response { + video_id: string; + status: string; + output_format?: VideoOutputFormat; + resolution?: VideoResolution | null; + resolution_note?: string | null; +} + export interface DeleteHyperframesRenderResponse { - /** - * ID of the deleted render. - */ render_id: string; } -/** - * Output aspect ratio. Supported ratios are ``16:9`` (landscape), ``9:16`` - * (portrait), and ``1:1`` (square). - */ +export interface ElevenLabsEngineSettings { + model?: ElevenLabsModel | null; + similarity_boost?: number | null; + stability?: number | null; + style?: number | null; + use_speaker_boost?: boolean | null; + engine_type: "elevenlabs"; +} + +export type ElevenLabsModel = + | "eleven_multilingual_v2" + | "eleven_turbo_v2_5" + | "eleven_flash_v2_5" + | "eleven_v3" + | "eleven_v4"; + +export interface ElevenLabsSpeechSettings { + model_id: + | "eleven_multilingual_v2" + | "eleven_flash_v2" + | "eleven_flash_v2_5" + | "eleven_turbo_v2_5" + | "eleven_v3" + | "eleven_v4"; +} + +export type Expressiveness = "high" | "medium" | "low"; + +export interface FishEngineSettings { + model?: FishModel | null; + stability?: number | null; + similarity?: number | null; + engine_type: "fish"; +} + +export type FishModel = "s1" | "s2-pro" | "s2.1-pro"; + +export interface GptImage25Request { + model: "gpt-image-2.5"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null; + reference_images?: Array; +} + +export interface HeyGenVoiceEngineSettings { + engine_type: "heygen_voice" | "orca"; +} + export type HyperframesAspectRatio = "16:9" | "9:16" | "1:1"; -/** - * Detailed HyperFrames render resource. - */ export interface HyperframesRenderDetail { - /** - * Unique render identifier. - */ render_id: string; - /** - * Current lifecycle state. - */ status: HyperframesRenderStatus; - /** - * Caller-supplied free-text label. - */ title?: string | null; - /** - * Caller-supplied client tracking ID. - */ callback_id?: string | null; - /** - * Presigned download URL for the rendered video. Present only when status is - * 'completed'. - */ video_url?: string | null; - /** - * Presigned download URL for the auto-generated thumbnail. - */ thumbnail_url?: string | null; - /** - * Video duration in seconds; null until completed. - */ duration?: number | null; - /** - * Frames per second the render was created at. - */ fps?: number | null; - /** - * Render quality preset. - */ quality?: "draft" | "standard" | "high" | null; - /** - * Output container/codec. - */ format: "mp4" | "webm" | "mov"; - /** - * Resolution tier, if one was set. - */ resolution?: HyperframesResolution | null; - /** - * Aspect ratio, if one was set. - */ aspect_ratio?: HyperframesAspectRatio | null; - /** - * Composition entry file path. - */ composition?: string | null; - /** - * Unix timestamp when the render was created. - */ created_at?: number | null; - /** - * Unix timestamp when the render terminated. Null until status is 'completed' - * or 'failed'. - */ completed_at?: number | null; - /** - * Error description. Present only when status is 'failed'. - */ failure_message?: string | null; - /** - * Why billing refused a finished render: 'hf_usage_exhausted', - * 'hf_subscription_required' or 'api_key_required', for which nothing was - * charged and the render can be sent again on a HeyGen API key; or - * 'usage_limit_reached', when a usage limit a workspace admin set (the API - * key's credit limit or the member's usage limit) was reached; sending the - * render again does not help until the limit is raised or resets, and - * `failure_message` is the limit's own message; or - * 'chunk_frames_exceed_timeout_at_max_chunks', when the render is too long to - * export at its size (even split into the most cloud parts allowed, it would - * still take longer than the time limit): sending it again fails the same way - * until it is shortened or rendered at a lower resolution or frame rate, and - * `failure_message` says so. Present only when status is 'failed' for one of - * these reasons. - */ failure_code?: string | null; } -/** - * Lifecycle status of a HyperFrames render. - */ export type HyperframesRenderStatus = "queued" | "rendering" | "completed" | "failed"; -/** - * Output resolution tier. 4K output uses a 1.5x pricing multiplier. - */ export type HyperframesResolution = "1080p" | "4k"; +export type ImageCreateRequest = GptImage25Request | NanoBanana2Request | NanoBanana21Request; + +export interface ImageScene { + type: "image"; + source: AssetUrl | AssetId | AssetBase64; + duration?: number | null; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; +} + +export interface ImageToVideoRequest { + end_user_id?: string | null; + mode: "image_to_video"; + model: "heygen-video-1"; + prompt: string; + prompt_enhancement?: "turbo" | "quality" | "default" | "disabled" | "ultraquality"; + resolution?: "480p" | "768p" | "1080p" | "2k"; + duration?: number | "auto"; + aspect_ratio?: "21:9" | "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "adaptive" | null; + seed?: number | null; + callback_url?: string | null; + callback_id?: string | null; + image: AssetUrl | AssetId | AssetBase64; + end_image?: AssetUrl | AssetId | AssetBase64 | null; +} + +export interface LipsyncCreateResponse { + lipsync_id: string; +} + +export interface LipsyncDetail { + id: string; + title?: string | null; + status: VideoTranslationStatus; + duration?: number | null; + video_url?: string | null; + caption_url?: string | null; + callback_id?: string | null; + created_at?: number | null; + failure_message?: string | null; + failure_code?: string | null; +} + +export interface LyriaMusicRequest { + model: "lyria-3-pro"; + prompt: string; + duration_seconds: number; + bpm?: number | null; + intensity?: number | null; + instrumental?: boolean; + lyrics?: string | null; +} + +export interface ModelVideoResponse { + video_id: string; + status: "pending" | "processing" | "completed" | "failed" | "cancelled"; + model: VideoGenerationModel; + created_at: number; + video_url?: string | null; + duration?: number | null; + input_video_duration?: number | null; + aspect_ratio?: string | null; + width?: number | null; + height?: number | null; + seed?: number | null; + timings?: ModelVideoTimings | null; + failure_code?: "generation_failed" | "generation_cancelled" | "usage_limit_reached" | null; + failure_message?: string | null; +} + +export interface ModelVideoTimings { + inference: number; +} + +export type MusicCreateRequest = LyriaMusicRequest; + +export type MusicFailureCode = + | "invalid_input" + | "content_policy_violation" + | "vendor_unavailable" + | "vendor_timeout" + | "ai_vendor_access_restricted" + | "insufficient_credit" + | "api_key_required" + | "internal_error"; + +export interface MusicResponse { + id: string; + status: MusicStatus; + model: "lyria-3-pro"; + created_at?: number | null; + requested_duration_seconds: number; + duration_seconds?: number | null; + audio_url?: string | null; + format?: "mp3" | null; + failure_code?: MusicFailureCode | null; + failure_message?: string | null; +} + +export type MusicStatus = "pending" | "completed" | "failed"; + +export interface NanoBanana21Request { + model: "nano-banana-2.1"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | "3:2" | "2:3" | "21:9" | null; + reference_images?: Array; +} + +export interface NanoBanana2Request { + model: "nano-banana-2"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | "3:2" | "2:3" | "21:9" | null; + reference_images?: Array; +} + +export interface PlaybackInput { + mode?: "freeze" | "loop" | "fit_to_scene" | null; + volume?: number; + mute?: boolean; +} + +export interface ReferenceToVideoRequest { + end_user_id?: string | null; + mode: "reference_to_video"; + model: "heygen-video-1"; + prompt: string; + prompt_enhancement?: "turbo" | "quality" | "default" | "disabled" | "ultraquality"; + resolution?: "480p" | "768p" | "1080p" | "2k"; + duration?: number | "auto"; + aspect_ratio?: "21:9" | "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "adaptive"; + seed?: number | null; + callback_url?: string | null; + callback_id?: string | null; + reference_images?: Array; + reference_videos?: Array; + reference_audio?: Array; +} + +export interface ScribeV2TranscriptionRequest { + audio: AssetUrl | AssetId | AssetBase64; + model: "scribe-v2"; + language?: string | null; + word_timestamps?: boolean; + diarize?: boolean; + num_speakers?: number | null; +} + +export type SpeechEngine = "starfish" | "orca" | "elevenlabs" | "elevenlabs_v3"; + export interface StandardAPIError { - /** - * Machine-readable error code - */ code: string; - /** - * Human-readable error message - */ message: string; - /** - * Which request field caused the error - */ param?: string | null; - /** - * Link to error documentation - */ doc_url?: string | null; - /** - * Per-item error details for batch/multi-node failures (e.g. graph compilation - * errors by node). - */ errors?: Array; - /** - * Request identifier, also returned in the X-HeyGen-Request-Id response - * header. Include it when contacting support. - */ request_id?: string; } export interface StandardAPIErrorDetail { - /** - * Machine-readable error code - */ code: string; - /** - * Human-readable error message - */ message: string; - /** - * Graph node id this error pertains to, if applicable - */ node_id?: string; - /** - * Port name on the node this error pertains to, if applicable - */ port?: string; - /** - * Field path within the item this error pertains to, if applicable - */ path?: string; } -/** - * Response from uploading an asset via POST /v3/assets. - */ +export interface StarfishEngineSettings { + engine_type: "starfish"; +} + +export interface StudioAvatarInput { + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; + type: "avatar"; + avatar_id: string; + motion_prompt?: string | null; + expressiveness?: Expressiveness | null; + engine?: AvatarVEngineConfig | AvatarIVEngineConfig | AvatarIIIEngineConfig | null; + background?: + | StudioColorBackgroundInput + | StudioImageBackgroundInput + | StudioVideoBackgroundInput + | null; + remove_background?: boolean | null; +} + +export interface StudioColorBackgroundInput { + type: "color"; + color: string; +} + +export interface StudioImageBackgroundInput { + type: "image"; + source: AssetUrl | AssetId | AssetBase64; +} + +export interface StudioVideoBackgroundInput { + type: "video"; + source: AssetUrl | AssetId | AssetBase64; +} + +export interface TextToVideoRequest { + end_user_id?: string | null; + mode: "text_to_video"; + model: "heygen-video-1"; + prompt: string; + prompt_enhancement?: "turbo" | "quality" | "default" | "disabled" | "ultraquality"; + resolution?: "480p" | "768p" | "1080p" | "2k"; + duration?: number | "auto"; + aspect_ratio?: "21:9" | "16:9" | "4:3" | "1:1" | "3:4" | "9:16"; + seed?: number | null; + callback_url?: string | null; + callback_id?: string | null; +} + +export interface ThirdPartyGeneratedImage { + url: string; + width: number; + height: number; +} + +export type ThirdPartyImageFailureCode = + | "content_policy_violation" + | "partial_generation" + | "vendor_unavailable" + | "ai_vendor_access_restricted" + | "insufficient_credit" + | "api_key_required" + | "internal_error"; + +export interface ThirdPartyImageResponse { + id: string; + status: "pending" | "completed" | "failed"; + model: string; + created_at: number; + images: Array; + failure_code?: ThirdPartyImageFailureCode | null; + failure_message?: string | null; +} + +export type TranscriptionCreateRequest = ScribeV2TranscriptionRequest; + +export type TranscriptionFailureCode = + | "invalid_input" + | "vendor_unavailable" + | "vendor_timeout" + | "ai_vendor_access_restricted" + | "internal_error" + | "insufficient_credit" + | "api_key_required" + | "rate_limited"; + +export interface TranscriptionResponse { + id: string; + status: TranscriptionStatus; + model: "scribe-v2"; + created_at: number; + language?: string | null; + duration_seconds?: number | null; + text?: string | null; + segments?: Array | null; + failure_code?: TranscriptionFailureCode | null; + failure_message?: string | null; +} + +export interface TranscriptionSegment { + speaker?: string | null; + start: number; + end: number; + text: string; + words?: Array | null; +} + +export type TranscriptionStatus = "pending" | "completed" | "failed"; + +export interface TranscriptionWord { + word: string; + start: number; + end: number; + speaker?: string | null; +} + export interface UploadAssetV3Response { - /** - * Unique asset identifier for use in other endpoints like POST - * /v3/video-agents - */ asset_id: string; - /** - * Public URL of the uploaded asset - */ url: string; - /** - * Detected MIME type of the file - */ mime_type: string; - /** - * File size in bytes - */ size_bytes: number; } + +export type VideoAspectRatio = "16:9" | "9:16" | "4:5" | "5:4" | "1:1" | "auto"; + +export interface VideoDetail { + id: string; + title?: string | null; + status: VideoStatus; + created_at?: number | null; + completed_at?: number | null; + video_url?: string | null; + thumbnail_url?: string | null; + gif_url?: string | null; + captioned_video_url?: string | null; + subtitle_url?: string | null; + duration?: number | null; + folder_id?: string | null; + output_language?: string | null; + failure_code?: string | null; + failure_message?: string | null; + video_page_url?: string | null; +} + +export type VideoGenerationModel = "heygen-video-1"; + +export type VideoOutputFormat = "mp4" | "webm"; + +export type VideoResolution = "4k" | "1080p" | "720p"; + +export interface VideoScene { + type: "video"; + source: AssetUrl | AssetId | AssetBase64; + playback?: PlaybackInput | null; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; +} + +export type VideoStatus = "pending" | "processing" | "completed" | "failed"; + +export type VideoTranslationMode = "speed" | "precision"; + +export type VideoTranslationStatus = "pending" | "running" | "completed" | "failed"; + +export interface VoiceSettingsInput { + speed?: number; + pitch?: number; + volume?: number; + locale?: string | null; + engine_settings?: + | ElevenLabsEngineSettings + | FishEngineSettings + | StarfishEngineSettings + | HeyGenVoiceEngineSettings + | null; +} + +export interface VoiceV3Item { + voice_id: string; + name: string; + language: string; + gender: string; + preview_audio_url?: string | null; + support_pause: boolean; + support_locale: boolean; + type: AudioVoiceType; + default_engine?: SpeechEngine | null; + available_engines?: Array; +} + +export interface WatermarkInput { + image: AssetUrl | AssetId | AssetBase64; + scale?: number; + opacity?: number; + placement?: WatermarkPlacement | null; +} + +export interface WatermarkPlacement { + position?: WatermarkPosition; + offset_x?: number | null; + offset_y?: number | null; +} + +export type WatermarkPosition = "top_left" | "top_right" | "bottom_left" | "bottom_right"; + +export interface WordWithTimestamp { + word: string; + start: number; + end: number; +} diff --git a/packages/cli/src/cloud/generatedClient.contract.test.ts b/packages/cli/src/cloud/generatedClient.contract.test.ts index 815fb70010..0ecc416180 100644 --- a/packages/cli/src/cloud/generatedClient.contract.test.ts +++ b/packages/cli/src/cloud/generatedClient.contract.test.ts @@ -4,12 +4,8 @@ import { describe, expect, it } from "vitest"; const generatedClientSource = readFileSync(new URL("./_gen/client.ts", import.meta.url), "utf8"); describe("generated cloud client source contract", () => { - it("documents SRT subtitles on the uploadAsset endpoint", () => { - const uploadAssetDocs = generatedClientSource.match( - /\/\*\*\s*\*\s*Upload Asset[\s\S]*?\*\/\s+async uploadAsset/, - )?.[0]; - - expect(uploadAssetDocs).toContain("SRT subtitle"); - expect(uploadAssetDocs).toMatch(/Supported types:[^\n]*\bsrt\b/); + it("carries no API description text on its methods", () => { + expect(generatedClientSource).toMatch(/async uploadAsset\(/); + expect(generatedClientSource).not.toMatch(/\*\/\s+async \w+\(/); }); }); diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index b9355b6733..9ffab49957 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -2,7 +2,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { findFFmpeg } from "../browser/ffmpeg.js"; import { probeVideo } from "../measure/media.js"; @@ -41,6 +41,22 @@ afterEach(() => vi.unstubAllGlobals()); const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200 }); +/** Runs `work` to the end with the poll interval's timers fast-forwarded. */ +async function settled(work: Promise): Promise { + vi.useFakeTimers({ toFake: ["setTimeout"] }); + let done = false; + const run = work.finally(() => (done = true)); + try { + while (!done) { + await new Promise((r) => setImmediate(r)); + await vi.advanceTimersByTimeAsync(5000); + } + return await run; + } finally { + vi.useRealTimers(); + } +} + /** A 30 fps clip of these lavfi sources one after another: each new source is a cut. */ function shots(name: string, sources: string[], size = "180x320"): string { const path = join(project, name); @@ -690,6 +706,82 @@ describe("generate", () => { }); }); +describe("each route through the generated client", () => { + // Keys an earlier refusal in this file marked failed would move this file's keys on. + beforeEach(() => { + vi.stubEnv("HYPERFRAMES_CONFIG_DIR", mkdtempSync(join(tmpdir(), "hf-route-test-"))); + }); + const file = "https://files.test/route-result"; + const routes = [ + { type: "image", path: "/v3/images", ext: "png", done: { images: [{ url: file }] } }, + { type: "video", path: "/v3/models/videos", ext: "mp4", done: { video_url: file } }, + { type: "avatar", path: "/v3/videos", ext: "mp4", done: { video_url: file } }, + { type: "voice", path: "/v3/voices/speech", ext: "mp3", done: { audio_url: file } }, + { type: "music", path: "/v3/audio/music", ext: "mp3", done: { audio_url: file } }, + ] as const; + const request = (type: GenerateRequest["type"], ext: string): GenerateRequest => { + const photo = join(project, "route-photo.png"); + if (type === "avatar") + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=red:s=64x64"], + ...["-frames:v", "1", photo], + ]); + return { + type, + out: join(project, `assets/route-${type}.${ext}`), + prompt: "a desk", + script: "Hello", + voice: "c".repeat(32), + ...(type === "avatar" ? { photo } : {}), + }; + }; + + for (const { type, path, ext, done } of routes) { + const polled = type !== "voice"; + it.skipIf(!ffmpeg && type === "avatar")( + `${type}: posts ${path}${polled ? ", keyed, and polls it to completion" : " once, unkeyed"}`, + async () => { + const calls: { url: string; method?: string; key: string | null; body?: unknown }[] = []; + const answers = polled + ? [ + json({ data: { id: "j/1", status: "pending" } }), + json({ data: { id: "j/1", status: "completed", ...done } }), + ] + : [json({ data: { ...done, duration: 1 } })]; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + calls.push({ + url, + method: init?.method, + key: new Headers(init?.headers).get("Idempotency-Key"), + ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}), + }); + if (url === file) return new Response(new Uint8Array([1])); + return answers.shift()!; + }); + const r = await settled(generate(request(type, ext))); + const [post, ...rest] = calls; + expect(post).toMatchObject({ url: `https://api.test${path}`, method: "POST" }); + expect(post!.key).toEqual(polled ? expect.stringMatching(/^hf-[0-9a-f]{40}$/) : null); + expect(post!.body).toBeTypeOf("object"); + expect(rest.map((c) => [c.url, c.method])).toEqual([ + ...(polled ? [[`https://api.test${path}/j%2F1`, "GET"]] : []), + [file, undefined], + ]); + expect(r).toMatchObject({ route: path, job: polled ? "j/1" : "" }); + }, + ); + + it.skipIf(!ffmpeg && type === "avatar")(`${type}: says what a refusal needs`, async () => { + vi.stubGlobal( + "fetch", + async () => + new Response(JSON.stringify({ error: { code: "hf_usage_exhausted" } }), { status: 402 }), + ); + await expect(generate(request(type, ext))).rejects.toThrow("out of HyperFrames credits"); + }); + } +}); + describe("foldShort", () => { it("folds a piece of a few frames into its neighbour", () => { expect(foldShort([4, 4.1, 8], 10)).toEqual([4.1, 8]); @@ -713,10 +805,14 @@ describe("pickVoice", () => { vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { asked.push(url); if (url.includes("/v3/voices?")) return json({ data: catalog }); + if (url === "https://files.test/vo.mp3") return new Response(new Uint8Array([1])); if (init?.method === "POST") { const key = new Headers(init.headers).get("Idempotency-Key"); posts.push({ key, body: JSON.parse(String(init.body)) }); } + // Speech answers in the request; a video is a job to poll. + if (url.endsWith("/v3/voices/speech")) + return json({ data: { audio_url: "https://files.test/vo.mp3", duration: 1 } }); return json({ data: { id: "j1", status: "pending" } }); }); afterEach(() => { @@ -741,6 +837,18 @@ describe("pickVoice", () => { expect(asked[0]).toBe("https://api.test/v3/voices?type=public&language=Chinese&limit=1"); }); + it("says what to do when HeyGen will not list its voices", async () => { + vi.stubGlobal( + "fetch", + async () => + new Response(JSON.stringify({ error: { code: "unauthorized" } }), { status: 401 }), + ); + await expect(pickVoice({ type: "voice", out: "vo.mp3" })).rejects.toMatchObject({ + message: "HeyGen 401 listing its voices", + fix: "run the same command again", + }); + }); + it("lists voices to pick from when none is named", async () => { heygen(); await expect(pickVoice({ type: "voice", out: "vo.mp3" })).rejects.toMatchObject({ @@ -790,7 +898,11 @@ describe("pickVoice", () => { script: "你好", ...(type === "avatar" ? { photo: face } : {}), }; - expect(await runOnce(r)).toMatchObject({ message: "HeyGen job j1 is still pending" }); + expect(await runOnce(r)).toMatchObject( + type === "voice" + ? { route: "/v3/voices/speech" } + : { message: "HeyGen job j1 is still pending" }, + ); catalog = [voice("b", "Wei"), voice("a", "Lin")]; await runOnce(r); expect(posts).toHaveLength(2); diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index 704d808c0f..e5c2ad6da4 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -12,6 +12,13 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, extname, join, relative, resolve } from "node:path"; +import type { + CreateModelVideoRequest, + CreateSpeechV3Request, + CreateVideoV3RequestBody, + ImageCreateRequest, + MusicCreateRequest, +} from "../cloud/_gen/types.js"; import { findCuts, signatures } from "../measure/cuts.js"; import { Fix, probeVideo, projectOf, round, runFfmpeg, videoClip } from "../measure/media.js"; import { @@ -22,6 +29,7 @@ import { runJob, senderScope, type Job, + type JobRoute, type Voice, } from "./v3.js"; @@ -141,9 +149,7 @@ export function structureRatio( return { inside: sum[0]! / (n[0] || 1), ring: sum[1]! / (n[1] || 1) }; } -interface Route { - path: string; - poll: (id: string) => string; +interface Route extends JobRoute { body: (r: GenerateRequest, media: Media) => Record; /** Where the finished job keeps the file's URL. */ urls: string[]; @@ -171,7 +177,9 @@ const need = (value: T | undefined, flag: string, example: string): T => { const ROUTES: Record = { image: { path: "/v3/images", - poll: (id) => `/v3/images/${id}`, + start: (c, body, key, signal) => + c.createImage({ body: body as ImageCreateRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getImage({ generation_id: id, signal }), body: (r, media) => ({ model: r.model ?? DEFAULTS.image, prompt: need(r.prompt, "--prompt", '--prompt "the same room, empty, no text"'), @@ -187,7 +195,9 @@ const ROUTES: Record = { }, video: { path: "/v3/models/videos", - poll: (id) => `/v3/models/videos/${id}`, + start: (c, body, key, signal) => + c.createModelVideo({ body: body as CreateModelVideoRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getModelVideo({ video_id: id, signal }), // fallow-ignore-next-line complexity body: (r, media) => { // A person speaking given audio is a reference shot: the references show who, the audio drives the lips. @@ -221,7 +231,9 @@ const ROUTES: Record = { }, avatar: { path: "/v3/videos", - poll: (id) => `/v3/videos/${id}`, + start: (c, body, key, signal) => + c.createVideo({ body: body as CreateVideoV3RequestBody, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getVideoV3({ video_id: id, signal }), body: (r, media) => ({ type: "image", image: media(need(r.photo, "--photo", "--photo assets/presenter.png"), "image"), @@ -235,8 +247,10 @@ const ROUTES: Record = { ext: [".mp4"], }, voice: { + // Speech answers in the request and takes no Idempotency-Key, so there is no status to poll. path: "/v3/voices/speech", - poll: (id) => `/v3/voices/speech/${id}`, + start: (c, body, _key, signal) => + c.createSpeechV3({ body: body as CreateSpeechV3Request, signal }), body: (r) => ({ text: need(r.script, "--script", '--script "What the voice says"'), voice_id: need(r.voice, "--voice", "--voice "), @@ -247,7 +261,9 @@ const ROUTES: Record = { }, music: { path: "/v3/audio/music", - poll: (id) => `/v3/audio/music/${id}`, + start: (c, body, key, signal) => + c.createMusic({ body: body as MusicCreateRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getMusic({ music_id: id, signal }), body: (r) => ({ model: r.model ?? DEFAULTS.music, prompt: need(r.prompt, "--prompt", '--prompt "warm lo-fi beat, soft keys"'), @@ -551,7 +567,7 @@ export async function generate( try { const body = route.body(r, inline(work, r.masks)); const key = idempotencyKey(route.path, [JSON.stringify(body)]); - const job = await runJob(route.path, body, key, (id) => route.poll(encodeURIComponent(id)), { + const job = await runJob(route, body, key, { log: { projectDir: project ?? undefined, what: `generate ${r.type} ${relative(project ?? ".", out)}`, diff --git a/packages/cli/src/heygen/transcribe.test.ts b/packages/cli/src/heygen/transcribe.test.ts index 1078db14e9..b30204c1b5 100644 --- a/packages/cli/src/heygen/transcribe.test.ts +++ b/packages/cli/src/heygen/transcribe.test.ts @@ -69,4 +69,50 @@ describe("HeyGen transcription", () => { }); expect(JSON.parse(readFileSync(out, "utf8"))).toHaveLength(2); }); + + it.skipIf(!ffmpeg)("keys the request and polls a pending transcription to the end", async () => { + const audio = join(dir, "b.wav"); + execFileSync(ffmpeg!, ["-v", "error", "-y", "-f", "lavfi", "-i", "sine=f=200:d=1", audio]); + const calls: { url: string; method?: string; key: string | null }[] = []; + const answers = [{ id: "t1", status: "pending" }, job]; + vi.stubGlobal("fetch", async (url: string, init: RequestInit) => { + calls.push({ + url, + method: init.method, + key: new Headers(init.headers).get("Idempotency-Key"), + }); + return new Response(JSON.stringify({ data: answers.shift() }), { status: 200 }); + }); + vi.useFakeTimers({ toFake: ["setTimeout"] }); + let done = false; + const pending = transcribeWithHeygen(audio, join(dir, "b.json")).finally(() => (done = true)); + while (!done) { + await new Promise((r) => setImmediate(r)); + await vi.advanceTimersByTimeAsync(5000); + } + const r = await pending; + vi.useRealTimers(); + expect(calls).toEqual([ + { + url: "https://api.test/v3/audio/transcriptions", + method: "POST", + key: expect.stringMatching(/^hf-[0-9a-f]{40}$/), + }, + { url: "https://api.test/v3/audio/transcriptions/t1", method: "GET", key: null }, + ]); + expect(r.detectedLanguage).toBe("en"); + }); + + it.skipIf(!ffmpeg)("turns a refusal into what to do next", async () => { + const audio = join(dir, "c.wav"); + execFileSync(ffmpeg!, ["-v", "error", "-y", "-f", "lavfi", "-i", "sine=f=250:d=1", audio]); + vi.stubGlobal( + "fetch", + async () => + new Response(JSON.stringify({ error: { code: "api_key_required" } }), { status: 403 }), + ); + await expect(transcribeWithHeygen(audio, join(dir, "c.json"))).rejects.toMatchObject({ + fix: "continue without this step", + }); + }); }); diff --git a/packages/cli/src/heygen/transcribe.ts b/packages/cli/src/heygen/transcribe.ts index b98be3f6ca..68a742a941 100644 --- a/packages/cli/src/heygen/transcribe.ts +++ b/packages/cli/src/heygen/transcribe.ts @@ -3,10 +3,20 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { TranscriptionCreateRequest } from "../cloud/_gen/types.js"; import { runFfmpeg } from "../measure/media.js"; -import { idempotencyKey, runJob } from "./v3.js"; +import { idempotencyKey, runJob, type JobRoute } from "./v3.js"; -const ROUTE = "/v3/audio/transcriptions"; +const ROUTE: JobRoute = { + path: "/v3/audio/transcriptions", + start: (c, body, key, signal) => + c.createTranscription({ + body: body as TranscriptionCreateRequest, + idempotencyKey: key, + signal, + }), + status: (c, id, signal) => c.getTranscription({ transcription_id: id, signal }), +}; const MODEL = "scribe-v2"; export interface HeygenTranscript { @@ -74,8 +84,7 @@ export async function transcribeWithHeygen( audio: { type: "base64", media_type: "audio/mpeg", data: audio.toString("base64") }, diarize, }, - idempotencyKey(ROUTE, [audio, String(diarize)]), - (id) => `${ROUTE}/${encodeURIComponent(id)}`, + idempotencyKey(ROUTE.path, [audio, String(diarize)]), { log: { projectDir: opts.projectDir, what: `transcribe ${inputPath}` }, timeoutMs: 10 * 60_000, diff --git a/packages/cli/src/heygen/v3.test.ts b/packages/cli/src/heygen/v3.test.ts index c76972079c..efe32dede5 100644 --- a/packages/cli/src/heygen/v3.test.ts +++ b/packages/cli/src/heygen/v3.test.ts @@ -2,6 +2,8 @@ import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { CreateModelVideoRequest } from "../cloud/_gen/types.js"; +import type { JobRoute } from "./v3.js"; vi.mock("../cloud/auth.js", () => ({ resolveCloudAuthHeaders: async () => ({ "x-api-key": "k" }), @@ -11,6 +13,14 @@ vi.mock("../cloud/auth.js", () => ({ const { idempotencyKey, runJob } = await import("./v3.js"); +/** A job route through the generated client: the create and the status GET it polls. */ +const X: JobRoute = { + path: "/v3/models/videos", + start: (c, body, key, signal) => + c.createModelVideo({ body: body as CreateModelVideoRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getModelVideo({ video_id: id, signal }), +}; + const reply = (status: number, body: unknown) => new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); @@ -35,12 +45,12 @@ describe("runJob", () => { calls.push({ url, init }); return answers.shift()!; }); - const job = await runJob("/v3/x", { a: 1 }, "hf-key", (id) => `/v3/x/${id}`, { intervalMs: 1 }); + const job = await runJob(X, { a: 1 }, "hf-key", { intervalMs: 1 }); expect(job.output).toEqual({ json: { ok: 1 } }); expect(calls.map((c) => c.url)).toEqual([ - "https://api.test/v3/x", - "https://api.test/v3/x/j1", - "https://api.test/v3/x/j1", + "https://api.test/v3/models/videos", + "https://api.test/v3/models/videos/j1", + "https://api.test/v3/models/videos/j1", ]); expect(new Headers(calls[0]!.init.headers).get("Idempotency-Key")).toBe("hf-key"); }); @@ -49,50 +59,50 @@ describe("runJob", () => { vi.stubGlobal("fetch", async () => reply(402, { error: { code: "insufficient_credit", message: "no" } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: expect.stringContaining("top up"), }); vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_subscription_required", message: "no plan" } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: "Subscribe: https://hyperframes.studio/pricing?from=cli", }); vi.stubGlobal("fetch", async () => reply(402, { code: "hf_usage_exhausted", url: "https://pay.test/credits" }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: "Add credits: https://pay.test/credits", }); vi.stubGlobal("fetch", async () => reply(401, { message: "bad" })); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: "npx hyperframes auth login", }); vi.stubGlobal("fetch", async () => reply(200, { data: { id: "j", status: "failed", failure_code: "content_policy_violation" } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: expect.stringContaining("reword"), }); }); it("names what an out-of-credits step needs, from the call's own price when HeyGen does not say", async () => { vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_usage_exhausted" } })); - await expect(runJob("/v3/x", {}, "k", (id) => id, { credits: 782 })).rejects.toMatchObject({ + await expect(runJob(X, {}, "k", { credits: 782 })).rejects.toMatchObject({ message: expect.stringContaining("this step needs about 782 in total"), }); - await expect(runJob("/v3/x", {}, "k", (id) => id, { credits: 782 })).rejects.toMatchObject({ + await expect(runJob(X, {}, "k", { credits: 782 })).rejects.toMatchObject({ message: expect.stringContaining("billing top-up --needed 782"), fix: expect.stringContaining("needed_credits=782"), }); vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_usage_exhausted", needed_credits: 180, balance: 30 } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id, { credits: 782 })).rejects.toMatchObject({ + await expect(runJob(X, {}, "k", { credits: 782 })).rejects.toMatchObject({ message: expect.stringContaining("this step needs 180 in total, balance 30"), }); vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_usage_exhausted" } })); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ message: expect.stringContaining("this step needs more"), }); }); @@ -104,7 +114,7 @@ describe("runJob", () => { if (init.method === "POST") keys.push(new Headers(init.headers).get("Idempotency-Key")); return answers.shift()!; }); - return runJob("/v3/x", {}, key, (id) => `/v3/x/${id}`, { intervalMs: 1 }); + return runJob(X, {}, key, { intervalMs: 1 }); }; const failed = () => [ reply(200, { data: { id: "j", status: "pending" } }), @@ -146,8 +156,8 @@ describe("runJob", () => { keys.push(new Headers(init.headers).get("Idempotency-Key")); return answers.shift()!; }); - await expect(runJob("/v3/x", {}, key, (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, key, (id) => id); + await expect(runJob(X, {}, key)).rejects.toThrow(); + await runJob(X, {}, key); expect(keys).toEqual([key, key]); }, ); @@ -159,7 +169,7 @@ describe("runJob", () => { keys.push(new Headers(init.headers).get("Idempotency-Key")); return answers.shift()!; }); - return runJob("/v3/x", {}, "hf-shared", (id) => id); + return runJob(X, {}, "hf-shared"); }; vi.stubEnv("HEYGEN_API_KEY", "payer-a"); await expect(run([reply(402, { code: "hf_usage_exhausted" })])).rejects.toThrow(); @@ -214,11 +224,11 @@ describe("runJob", () => { done(), ]); signIn("a"); - await runJob("/v3/x", {}, "hf-host", (id) => id); + await runJob(X, {}, "hf-host"); signIn("b"); - await expect(runJob("/v3/x", {}, "hf-host", (id) => id)).rejects.toThrow(); + await expect(runJob(X, {}, "hf-host")).rejects.toThrow(); signIn("a"); - await runJob("/v3/x", {}, "hf-host", (id) => id); + await runJob(X, {}, "hf-host"); expect(keys).toEqual(["hf-host", "hf-host", "hf-host"]); }); @@ -234,14 +244,12 @@ describe("runJob", () => { done(), ]); signIn("b"); - await runJob("/v3/x", {}, "hf-switch", (id) => id); + await runJob(X, {}, "hf-switch"); signIn("a"); - await expect( - runJob("/v3/x", {}, "hf-switch", (id) => id, { intervalMs: 1 }), - ).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-switch", (id) => id); + await expect(runJob(X, {}, "hf-switch", { intervalMs: 1 })).rejects.toThrow(); + await runJob(X, {}, "hf-switch"); signIn("a"); - await runJob("/v3/x", {}, "hf-switch", (id) => id); + await runJob(X, {}, "hf-switch"); expect(keys).toEqual(["hf-switch", "hf-switch", "hf-switch", "hf-switch-1"]); }); @@ -257,7 +265,7 @@ describe("runJob", () => { return done(); }); signIn("a"); - await runJob("/v3/x", {}, "hf-window", (id) => id); + await runJob(X, {}, "hf-window"); expect(seen).toEqual(["Bearer a"]); }); @@ -267,28 +275,91 @@ describe("runJob", () => { done(), ]); signIn("a"); - await expect(runJob("/v3/x", {}, "hf-unknown", (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-unknown", (id) => id); + await expect(runJob(X, {}, "hf-unknown")).rejects.toThrow(); + await runJob(X, {}, "hf-unknown"); expect(keys).toEqual(["hf-unknown", "hf-unknown"]); }); it("sends a job that failed for this account as a new one", async () => { const keys = server(accounts, [failed(), done()]); signIn("a"); - await expect(runJob("/v3/x", {}, "hf-own", (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-own", (id) => id); + await expect(runJob(X, {}, "hf-own")).rejects.toThrow(); + await runJob(X, {}, "hf-own"); expect(keys).toEqual(["hf-own", "hf-own-1"]); }); it("keeps the key through a lost answer, then a new sign-in", async () => { const keys = server(accounts, [reply(401, { message: "expired" }), done()]); signIn("a"); - await expect(runJob("/v3/x", {}, "hf-resigned", (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-resigned", (id) => id); + await expect(runJob(X, {}, "hf-resigned")).rejects.toThrow(); + await runJob(X, {}, "hf-resigned"); expect(keys).toEqual(["hf-resigned", "hf-resigned"]); }); }); + it("sends the request as JSON and polls the status route with a GET", async () => { + const calls: { url: string; init: RequestInit }[] = []; + const answers = [ + reply(202, { data: { video_id: "v/1", status: "pending" } }), + reply(200, { data: { id: "v/1", status: "completed" } }), + ]; + vi.stubGlobal("fetch", async (url: string, init: RequestInit) => { + calls.push({ url, init }); + return answers.shift()!; + }); + const job = await runJob(X, { prompt: "p" }, "hf-shape", { intervalMs: 1 }); + expect(job).toMatchObject({ id: "v/1", status: "completed" }); + const [post, get] = calls; + expect(post!.init.method).toBe("POST"); + expect(JSON.parse(String(post!.init.body))).toEqual({ prompt: "p" }); + const headers = new Headers(post!.init.headers); + expect(headers.get("content-type")).toBe("application/json"); + expect(headers.get("x-api-key")).toBe("payer"); + expect(get!.url).toBe("https://api.test/v3/models/videos/v%2F1"); + expect(get!.init.method).toBe("GET"); + expect(get!.init.body).toBeUndefined(); + }); + + it("asks again for an answer it could not read or never got", async () => { + vi.stubGlobal("fetch", async () => reply(503, { error: { code: "unavailable" } })); + await expect(runJob(X, {}, "hf-503")).rejects.toMatchObject({ + message: "HeyGen 503 for /v3/models/videos with no readable answer", + fix: "run the same command again: it resumes the same job", + }); + vi.stubGlobal("fetch", async () => new Response("", { status: 400 })); + await expect(runJob(X, {}, "hf-html")).rejects.toThrow("HeyGen 400 for /v3/models/videos"); + vi.stubGlobal("fetch", async () => new Response("not json", { status: 200 })); + await expect(runJob(X, {}, "hf-text")).rejects.toThrow("HeyGen 200 for /v3/models/videos"); + vi.stubGlobal("fetch", async () => { + throw new TypeError("fetch failed"); + }); + await expect(runJob(X, {}, "hf-lost")).rejects.toMatchObject({ + message: "no answer from HeyGen for /v3/models/videos (fetch failed)", + }); + const keys: (string | null)[] = []; + vi.stubGlobal("fetch", async (_url: string, init: RequestInit) => { + keys.push(new Headers(init.headers).get("Idempotency-Key")); + return reply(200, { data: { id: "j", status: "completed" } }); + }); + await runJob(X, {}, "hf-503"); + expect(keys).toEqual(["hf-503"]); + }); + + it("takes the answer of a route with no status as final", async () => { + const calls: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + calls.push(url); + return reply(200, { data: { audio_url: "https://files.test/a.mp3", duration: 1 } }); + }); + const sync = { path: X.path, start: X.start }; + expect(await runJob(sync, {}, "hf-sync")).toMatchObject({ + id: "", + status: "completed", + audio_url: "https://files.test/a.mp3", + }); + expect(calls).toHaveLength(1); + }); + it("keys the same request the same way", () => { expect(idempotencyKey("/v3/x", ["a", Buffer.from("b")])).toBe( idempotencyKey("/v3/x", ["a", Buffer.from("b")]), diff --git a/packages/cli/src/heygen/v3.ts b/packages/cli/src/heygen/v3.ts index 4d36d2bffa..93351f9142 100644 --- a/packages/cli/src/heygen/v3.ts +++ b/packages/cli/src/heygen/v3.ts @@ -8,6 +8,7 @@ import { configDir } from "../auth/paths.js"; import { AuthClient, buildAuthHeaders } from "../auth/client.js"; import { resolveCredential } from "../auth/resolver.js"; import { refreshIfNeeded, resolveCloudAuthHeaders, resolveCloudBaseUrl } from "../cloud/auth.js"; +import { HyperframesApiError, HyperframesCloudClient } from "../cloud/_gen/client.js"; import { Fix } from "../measure/media.js"; import { pageUrl } from "./plan.js"; @@ -136,47 +137,47 @@ function refusal(status: number, body: unknown, estimate?: number): Refused { ); } +export interface JobRoute { + path: string; + start: ( + client: HyperframesCloudClient, + body: unknown, + key: string, + signal: AbortSignal, + ) => Promise; + status?: (client: HyperframesCloudClient, id: string, signal: AbortSignal) => Promise; +} + +const clientFor = (auth: Record) => + new HyperframesCloudClient({ baseUrl: resolveCloudBaseUrl(), getAuthHeaders: () => auth }); + +const resume = "run the same command again: it resumes the same job"; + // fallow-ignore-next-line complexity async function call( - method: "GET" | "POST", route: string, - body?: unknown, - key?: string, + send: (client: HyperframesCloudClient, signal: AbortSignal) => Promise, + timeoutMs: number, estimate?: number, auth?: Record, ): Promise { - const headers: Record = { - ...(auth ?? (await resolveCloudAuthHeaders().catch(noSignIn))), - accept: "application/json", - }; - if (body !== undefined) headers["content-type"] = "application/json"; - if (key) headers["Idempotency-Key"] = key; - let res: Response; + const client = clientFor(auth ?? (await resolveCloudAuthHeaders().catch(noSignIn))); + let data: unknown; try { - res = await fetch(`${resolveCloudBaseUrl()}${route}`, { - method, - headers, - ...(body !== undefined ? { body: JSON.stringify(body) } : {}), - signal: AbortSignal.timeout(method === "POST" ? 180_000 : 30_000), - }); + data = await send(client, AbortSignal.timeout(timeoutMs)); } catch (err) { - throw new Fix( - `no answer from HeyGen for ${route} (${err instanceof Error ? err.message : String(err)})`, - "run the same command again: it resumes the same job", - ); - } - const value: unknown = await res.json().catch(() => null); - if (res.status >= 500 || value === null) - throw new Fix( - `HeyGen ${res.status} for ${route} with no readable answer`, - "run the same command again: it resumes the same job", - ); - if (!res.ok) { - const refused = refusal(res.status, value, estimate); - refused.status = res.status; + if (!(err instanceof HyperframesApiError)) + throw new Fix( + `no answer from HeyGen for ${route} (${err instanceof Error ? err.message : String(err)})`, + resume, + ); + if (err.status >= 500 || err.status < 300 || err.raw == null) + throw new Fix(`HeyGen ${err.status} for ${route} with no readable answer`, resume); + const refused = refusal(err.status, err.raw, estimate); + refused.status = err.status; throw refused; } - const data = isRecord(value) && isRecord(value.data) ? value.data : value; + if (data == null) throw new Fix(`HeyGen answered ${route} with no readable answer`, resume); if (!isRecord(data)) throw new Fix(`HeyGen answered ${route} with no job`, "run the same command again"); const id = data.id ?? data.job_id ?? data.video_id ?? data.generation_id; @@ -214,26 +215,33 @@ export interface RunOptions { credits?: number; } -/** Starts a job (or picks up the one this key already started) and polls `pollRoute(id)` until it ends. */ +/** Starts a job (or picks up the one this key already started) and polls its status until it ends. */ export async function runJob( - route: string, + jobRoute: JobRoute, body: unknown, key: string, - pollRoute: (id: string) => string, opts: RunOptions = {}, ): Promise { const started = Date.now(); + const route = jobRoute.path; const { auth, scope } = await jobSender(); const sent = liveKey(key, scope); let job: Job; try { - job = await call("POST", route, body, sent, opts.credits, auth); + job = await call( + route, + (client, signal) => jobRoute.start(client, body, sent, signal), + 180_000, + opts.credits, + auth, + ); } catch (err) { if (err instanceof Refused && !KEEPS_KEY.has(err.status)) markFailed(sent, scope); throw err; } if (opts.log) log(opts.log, { route, id: job.id, key: sent, status: job.status }); - while (RUNNING.has(job.status.toLowerCase())) { + const status = jobRoute.status; + while (status && RUNNING.has(job.status.toLowerCase())) { if (Date.now() - started > (opts.timeoutMs ?? 15 * 60_000)) throw new Fix( `HeyGen job ${job.id} is still ${job.status}`, @@ -241,7 +249,12 @@ export async function runJob( ); opts.onWait?.(job); await new Promise((r) => setTimeout(r, opts.intervalMs ?? 5000)); - job = await call("GET", pollRoute(job.id)); + const id = job.id; + job = await call( + `${route}/${encodeURIComponent(id)}`, + (client, signal) => status(client, id, signal), + 30_000, + ); } if (opts.log) log(opts.log, { route, id: job.id, key: sent, status: job.status }); if (!/^(completed|succeeded|success|done)$/i.test(job.status)) { @@ -265,23 +278,19 @@ const isVoice = (v: unknown): v is Voice => /** HeyGen's public voices for a language, best first; a language it does not know gets multilingual ones. */ export async function publicVoices(language: string, limit: number): Promise { - const headers = { - ...(await resolveCloudAuthHeaders().catch(noSignIn)), - accept: "application/json", - }; - const query = new URLSearchParams({ type: "public", language, limit: String(limit) }); - const res = await fetch(`${resolveCloudBaseUrl()}/v3/voices?${query}`, { - headers, - signal: AbortSignal.timeout(30_000), - }).catch((err: unknown) => { - throw new Fix( - `no answer from HeyGen for its voices (${err instanceof Error ? err.message : String(err)})`, - "run the same command again", - ); - }); - const value: unknown = await res.json().catch(() => null); - if (!res.ok || !isRecord(value) || !Array.isArray(value.data)) - throw new Fix(`HeyGen ${res.status} listing its voices`, "run the same command again"); + const client = clientFor(await resolveCloudAuthHeaders().catch(noSignIn)); + const value = await client + .listVoicesV3({ type: "public", language, limit, signal: AbortSignal.timeout(30_000) }) + .catch((err: unknown) => { + if (err instanceof HyperframesApiError) + throw new Fix(`HeyGen ${err.status} listing its voices`, "run the same command again"); + throw new Fix( + `no answer from HeyGen for its voices (${err instanceof Error ? err.message : String(err)})`, + "run the same command again", + ); + }); + if (!isRecord(value) || !Array.isArray(value.data)) + throw new Fix("HeyGen answered its voices with no list", "run the same command again"); return value.data .filter(isVoice) .map(({ voice_id, name, language, gender }) => ({ voice_id, name, language, gender }));