diff --git a/packages/cli/src/cloud/_gen/client.ts b/packages/cli/src/cloud/_gen/client.ts index b8863c6de5..850fedefb1 100644 --- a/packages/cli/src/cloud/_gen/client.ts +++ b/packages/cli/src/cloud/_gen/client.ts @@ -5,15 +5,35 @@ * experiment-framework to regenerate. */ import type { + BackgroundRemovalJob, CompleteAssetUploadRequest, CompleteAssetUploadResponse, CreateAssetUploadRequest, CreateAssetUploadResponse, + CreateBackgroundRemovalRequest, CreateHyperframesRenderRequest, CreateHyperframesRenderResponse, + CreateLipsyncRequest, + CreateModelVideoRequest, + CreateModelVideoResponse, + CreateSpeechV3Request, + CreateSpeechV3ResponseData, + CreateVideoV3RequestBody, + CreateVideoV3Response, DeleteHyperframesRenderResponse, HyperframesRenderDetail, + ImageCreateRequest, + LipsyncCreateResponse, + LipsyncDetail, + ModelVideoResponse, + MusicCreateRequest, + MusicResponse, + ThirdPartyImageResponse, + TranscriptionCreateRequest, + TranscriptionResponse, UploadAssetV3Response, + VideoDetail, + VoiceV3Item, } from "./types.js"; export type AuthHeaders = Record; @@ -201,11 +221,6 @@ export class HyperframesCloudClient { }); } - /** - * Upload Asset - * - * Uploads a file (image, video, audio, PDF, or SRT subtitle) and returns an asset_id for use in other endpoints. Max 50 MB, or 32 MB for an image. Supported types: png, jpeg, mp4, webm, mp3, wav, pdf, srt. A file is checked against the content policy after it is uploaded; if it is blocked, the asset is removed and later requests for it return not found. - */ async uploadAsset(args: { file: Blob | Buffer | Uint8Array; filename: string; @@ -229,11 +244,6 @@ export class HyperframesCloudClient { }); } - /** - * Create Asset Upload - * - * Begin a direct-to-S3 upload. Returns an asset_id and a presigned upload_url; PUT the file bytes to upload_url, then call POST /v3/assets/{asset_id}/complete. Unlike POST /v3/assets (which proxies the bytes), this never sends the file through the API. - */ async createAssetUpload(args: { body: CreateAssetUploadRequest; idempotencyKey?: string; @@ -248,11 +258,6 @@ export class HyperframesCloudClient { }); } - /** - * Complete Asset Upload - * - * Finalize a direct-to-S3 upload into a reusable asset. Call after the upload PUT returns 200. Idempotent: repeated calls return the same finalized asset. - */ async completeAssetUpload(args: { asset_id: string; body: CompleteAssetUploadRequest; @@ -266,11 +271,6 @@ export class HyperframesCloudClient { }); } - /** - * Create HyperFrames Render - * - * Renders a HyperFrames composition (an HTML+JS+assets project bundled as a .zip) into a video. Submit the project via `url`, `asset_id` (pre-uploaded via POST /v3/assets), or inline `base64`. Returns a `render_id` to poll via GET /v3/hyperframes/renders/{render_id}. - */ async createRender(args: { body: CreateHyperframesRenderRequest; idempotencyKey?: string; @@ -285,11 +285,6 @@ export class HyperframesCloudClient { }); } - /** - * List HyperFrames Renders - * - * Returns a cursor-paginated list of HyperFrames renders in the account, newest first. - */ async listRenders(args: { limit?: number; token?: string; signal?: AbortSignal }): Promise<{ data?: Array; has_more?: boolean; @@ -312,11 +307,6 @@ export class HyperframesCloudClient { }); } - /** - * Get HyperFrames Render - * - * Returns full details for a single HyperFrames render, including status and signed video_url when complete. - */ async getRender(args: { render_id: string; signal?: AbortSignal; @@ -328,11 +318,6 @@ export class HyperframesCloudClient { }); } - /** - * Delete HyperFrames Render - * - * Soft-deletes a HyperFrames render. Subsequent GETs return 404. - */ async deleteRender(args: { render_id: string; signal?: AbortSignal; @@ -343,4 +328,210 @@ export class HyperframesCloudClient { signal: args.signal, }); } + + async createImage(args: { + body: ImageCreateRequest; + idempotencyKey: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/images", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getImage(args: { + generation_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/images/${encodeURIComponent(args.generation_id)}`, + signal: args.signal, + }); + } + + async createModelVideo(args: { + body: CreateModelVideoRequest; + idempotencyKey?: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/models/videos", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getModelVideo(args: { + video_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/models/videos/${encodeURIComponent(args.video_id)}`, + signal: args.signal, + }); + } + + async createVideo(args: { + body: CreateVideoV3RequestBody; + idempotencyKey?: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/videos", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getVideoV3(args: { video_id: string; signal?: AbortSignal }): Promise { + return await this.request({ + method: "GET", + path: `/v3/videos/${encodeURIComponent(args.video_id)}`, + signal: args.signal, + }); + } + + async listVoicesV3(args: { + type?: "public" | "private"; + engine?: string; + language?: string; + gender?: string; + limit?: number; + token?: string; + signal?: AbortSignal; + }): Promise<{ data?: Array; has_more?: boolean; next_token?: string | null }> { + const query: Record = { + type: args.type, + engine: args.engine, + language: args.language, + gender: args.gender, + limit: args.limit, + token: args.token, + }; + return await this.request<{ + data?: Array; + has_more?: boolean; + next_token?: string | null; + }>({ + method: "GET", + path: "/v3/voices", + query, + unwrapData: false, + signal: args.signal, + }); + } + + async createSpeechV3(args: { + body: CreateSpeechV3Request; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/voices/speech", + body: args.body, + signal: args.signal, + }); + } + + async createMusic(args: { + body: MusicCreateRequest; + idempotencyKey: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/audio/music", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getMusic(args: { music_id: string; signal?: AbortSignal }): Promise { + return await this.request({ + method: "GET", + path: `/v3/audio/music/${encodeURIComponent(args.music_id)}`, + signal: args.signal, + }); + } + + async createTranscription(args: { + body: TranscriptionCreateRequest; + idempotencyKey: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/audio/transcriptions", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getTranscription(args: { + transcription_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/audio/transcriptions/${encodeURIComponent(args.transcription_id)}`, + signal: args.signal, + }); + } + + async createLipsyncV3(args: { + body: CreateLipsyncRequest; + idempotencyKey?: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/lipsyncs", + body: args.body, + idempotencyKey: args.idempotencyKey, + signal: args.signal, + }); + } + + async getLipsyncV3(args: { lipsync_id: string; signal?: AbortSignal }): Promise { + return await this.request({ + method: "GET", + path: `/v3/lipsyncs/${encodeURIComponent(args.lipsync_id)}`, + signal: args.signal, + }); + } + + async createBackgroundRemoval(args: { + body: CreateBackgroundRemovalRequest; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "POST", + path: "/v3/background-removals", + body: args.body, + signal: args.signal, + }); + } + + async getBackgroundRemoval(args: { + job_id: string; + signal?: AbortSignal; + }): Promise { + return await this.request({ + method: "GET", + path: `/v3/background-removals/${encodeURIComponent(args.job_id)}`, + signal: args.signal, + }); + } } diff --git a/packages/cli/src/cloud/_gen/types.ts b/packages/cli/src/cloud/_gen/types.ts index fea4d799d1..51804d8de8 100644 --- a/packages/cli/src/cloud/_gen/types.ts +++ b/packages/cli/src/cloud/_gen/types.ts @@ -9,409 +9,744 @@ // Add a new path/method to TARGET_ENDPOINTS in // scripts/generate_hyperframes_cli_client.py to extend this list. -/** - * Asset input via base64-encoded content. - */ export interface AssetBase64 { - /** - * Input type discriminator - */ type: "base64"; - /** - * MIME type of the encoded content (e.g. "image/png") - */ media_type: string; - /** - * Base64-encoded file content - */ data: string; } -/** - * Asset input via HeyGen asset ID from the asset upload endpoint. - */ export interface AssetId { - /** - * Input type discriminator - */ type: "asset_id"; - /** - * HeyGen asset ID from an asset upload or import; not a third-party file ID. - */ asset_id: string; } -/** - * Asset input via publicly accessible HTTPS URL. - */ export interface AssetUrl { - /** - * Input type discriminator - */ type: "url"; - /** - * Public http(s) URL HeyGen can download without auth or redirects. Not a - * local or sandbox path (/mnt/data/..., file://, sandbox:); upload such files - * and pass asset_id. - */ url: string; } -/** - * Finalize a presigned upload (POST /v3/assets/{asset_id}/complete). - */ +export type AudioVoiceType = "public" | "private"; + +export type AvatarFit = "contain" | "cover"; + +export interface AvatarIIIEngineConfig { + type: "avatar_iii"; +} + +export interface AvatarIVEngineConfig { + type: "avatar_iv"; +} + +export interface AvatarVEngineConfig { + type: "avatar_v"; + reference_look_id?: string | null; + more_blinking?: boolean; +} + +export interface AvatarVideoScene { + type: "avatar_video"; + input: StudioAvatarInput; +} + +export interface BackgroundRemovalJob { + id: string; + status: string; + layers?: Record; + duration?: number | null; + created_at?: number | null; + completed_at?: number | null; + error?: string | null; +} + +export interface BackgroundSetting { + type: "color" | "image"; + value?: string | null; + url?: string | null; + asset_id?: string | null; +} + +export type CaptionFileFormat = "srt"; + +export interface CaptionSetting { + file_format?: CaptionFileFormat; + style?: CaptionStyle | null; +} + +export type CaptionStyle = "default"; + +export type CinematicAvatarModel = "seedance_2" | "heygen_video_1"; + export interface CompleteAssetUploadRequest { - /** - * Optional SHA256 (hex) cross-check. - */ checksum_sha256?: string | null; } -/** - * Result of finalizing an upload. - */ export interface CompleteAssetUploadResponse { - /** - * The reusable asset identifier. - */ asset_id: string; - /** - * Public URL of the finalized asset. - */ url: string; - /** - * MIME type detected from the stored bytes. - */ mime_type: string; - /** - * Size of the stored object in bytes. - */ size_bytes: number; - /** - * Asset status, e.g. 'processing'. - */ status: "processing"; } -/** - * Request to begin a presigned direct-to-S3 upload (POST - * /v3/assets/direct-uploads). - */ export interface CreateAssetUploadRequest { - /** - * Original filename for reference/metadata. The stored object's extension is - * derived from content_type. - */ filename: string; - /** - * Declared MIME type (e.g. 'video/mp4', 'image/png', 'audio/mpeg', - * 'application/pdf', 'application/zip'). Verified against the stored bytes at - * completion. - */ content_type: string; - /** - * Exact byte size of the file. Signed into the upload URL so it cannot be - * exceeded. - */ size_bytes: number; - /** - * Optional SHA256 of the file as hex. When provided, S3 enforces it on upload. - */ checksum_sha256?: string | null; } -/** - * Presigned upload instructions. - */ export interface CreateAssetUploadResponse { - /** - * Reusable asset identifier. Becomes usable after POST - * /v3/assets/{asset_id}/complete. - */ asset_id: string; - /** - * Presigned S3 URL. PUT the raw file bytes here. - */ upload_url: string; - /** - * Headers that must be sent verbatim on the PUT request. - */ upload_headers: Record; - /** - * Seconds until the upload URL expires. - */ expires_in_seconds: number; - /** - * Maximum allowed upload size in bytes for this content type: 1.5 GB - * (1,500,000,000 bytes) per audio file and 200 MiB (209,715,200 bytes) for - * other supported files. - */ max_bytes: number; - /** - * Upload lifecycle status. Always 'pending_upload' here. - */ status: "pending_upload"; } -/** - * Request body for POST /v3/hyperframes/renders. - */ +export interface CreateBackgroundRemovalRequest { + video: AssetUrl | AssetId | AssetBase64; + layers?: Array; + title?: string | null; + request_id?: string | null; +} + export interface CreateHyperframesRenderRequest { - /** - * HyperFrames composition .zip — provide as {type: 'url', url: '...'}, {type: - * 'asset_id', asset_id: '...'} (pre-uploaded via POST /v3/assets), or {type: - * 'base64', media_type: 'application/zip', data: '...'}. Zip must contain - * index.html at the root (or the path you set in `composition`). - */ project: AssetUrl | AssetId | AssetBase64; - /** - * Output frames per second. Defaults to 30 if not provided. - */ fps?: number | null; - /** - * Render quality preset; higher quality is slower. - */ quality?: "draft" | "standard" | "high"; - /** - * Output container/codec. - */ format?: "mp4" | "webm" | "mov"; - /** - * Overrides the quality preset's constant rate factor (lower is higher quality - * and a larger file). Integer 0-51 for mp4, 0-63 for webm; ignored for mov. - * Omit to use the preset's CRF. - */ crf?: number | null; - /** - * Output resolution tier. Defaults to '1080p'. Pass '4k' for 4K renders - * (billed at 1.5x). - */ resolution?: HyperframesResolution; - /** - * Output aspect ratio. Defaults to '16:9' (landscape). Pass '9:16' for - * portrait or '1:1' for square. - */ aspect_ratio?: HyperframesAspectRatio; - /** - * Entry HTML file relative to the project root (e.g. compositions/intro.html). - * Defaults to index.html when omitted. - */ composition?: string | null; - /** - * Optional overrides for the composition's data-composition-variables. Use - * this to parameterise a single composition across multiple renders. - */ variables?: Record | null; - /** - * Free-text label for the render; echoed back in detail responses. - */ title?: string | null; - /** - * Opaque client tracking ID, echoed back in webhook payloads. - */ callback_id?: string | null; - /** - * Per-request HTTPS webhook URL the render fires when it terminates. - */ callback_url?: string | null; } -/** - * Response for POST /v3/hyperframes/renders. - */ export interface CreateHyperframesRenderResponse { - /** - * HyperFrames render identifier — poll GET /v3/hyperframes/renders/{render_id} - * for status. - */ render_id: string; } -/** - * Response for DELETE /v3/hyperframes/renders/{render_id}. - */ +export interface CreateLipsyncRequest { + video: AssetUrl | AssetId; + audio: AssetUrl | AssetId; + title?: string | null; + mode?: VideoTranslationMode; + callback_url?: string | null; + callback_id?: string | null; + enable_caption?: boolean; + keep_the_same_format?: boolean | null; + enable_dynamic_duration?: boolean; + disable_music_track?: boolean; + enable_speech_enhancement?: boolean; + enable_watermark?: boolean; + start_time?: number | null; + end_time?: number | null; + fps_mode?: string | null; + folder_id?: string | null; +} + +export type CreateModelVideoRequest = + | TextToVideoRequest + | ImageToVideoRequest + | ReferenceToVideoRequest; + +export interface CreateModelVideoResponse { + video_id: string; + status?: "pending"; + input_video_duration?: number | null; +} + +export interface CreateSpeechV3OutputFormat { + encoding: "mp3" | "wav"; + sample_rate_hertz?: number | null; + bit_depth?: number | null; + num_channels?: number; + bitrate_bps?: number | null; +} + +export interface CreateSpeechV3Request { + text: string; + voice_id: string; + input_type?: string; + speed?: number; + language?: string | null; + locale?: string | null; + force_regenerate?: boolean; + engine?: "heygen_voice" | "starfish" | "orca" | "elevenlabs" | "elevenlabs_v3" | null; + settings?: ElevenLabsSpeechSettings | null; + brand_glossary_id?: string | null; + output_format?: CreateSpeechV3OutputFormat | null; +} + +export interface CreateSpeechV3ResponseData { + audio_url: string; + duration: number; + request_id?: string | null; + word_timestamps?: Array | null; + engine: SpeechEngine; +} + +export interface CreateVideoFromAvatar { + title?: string | null; + folder_id?: string | null; + resolution?: VideoResolution | null; + aspect_ratio?: VideoAspectRatio | null; + fit?: AvatarFit | null; + background?: BackgroundSetting | null; + remove_background?: boolean | null; + callback_url?: string | null; + callback_id?: string | null; + watermark?: WatermarkInput | null; + caption?: CaptionSetting | null; + output_format?: VideoOutputFormat; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; + type: "avatar"; + avatar_id: string; + brand_glossary_id?: string | null; + motion_prompt?: string | null; + expressiveness?: Expressiveness | null; + engine?: AvatarVEngineConfig | AvatarIVEngineConfig | AvatarIIIEngineConfig | null; +} + +export interface CreateVideoFromCinematicAvatar { + type: "cinematic_avatar"; + prompt: string; + avatar_id: Array; + model?: CinematicAvatarModel | null; + references?: Array | null; + aspect_ratio?: "16:9" | "9:16" | "1:1"; + resolution?: "720p" | "1080p"; + auto_duration?: boolean; + duration?: number | null; + enhance_prompt?: boolean; + title?: string | null; + folder_id?: string | null; +} + +export interface CreateVideoFromImage { + title?: string | null; + folder_id?: string | null; + resolution?: VideoResolution | null; + aspect_ratio?: VideoAspectRatio | null; + fit?: AvatarFit | null; + background?: BackgroundSetting | null; + remove_background?: boolean | null; + callback_url?: string | null; + callback_id?: string | null; + watermark?: WatermarkInput | null; + caption?: CaptionSetting | null; + output_format?: VideoOutputFormat; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; + type: "image"; + image: AssetUrl | AssetId | AssetBase64; + brand_glossary_id?: string | null; + motion_prompt?: string | null; + expressiveness?: Expressiveness | null; +} + +export interface CreateVideoFromStudio { + type: "studio"; + title?: string | null; + folder_id?: string | null; + aspect_ratio?: VideoAspectRatio | null; + resolution?: VideoResolution | null; + brand_glossary_id?: string | null; + callback_url?: string | null; + callback_id?: string | null; + caption?: CaptionSetting | null; + watermark?: WatermarkInput | null; + scenes: Array; +} + +export type CreateVideoV3RequestBody = + | CreateVideoFromAvatar + | CreateVideoFromImage + | CreateVideoFromCinematicAvatar + | CreateVideoFromStudio; + +export interface CreateVideoV3Response { + video_id: string; + status: string; + output_format?: VideoOutputFormat; + resolution?: VideoResolution | null; + resolution_note?: string | null; +} + export interface DeleteHyperframesRenderResponse { - /** - * ID of the deleted render. - */ render_id: string; } -/** - * Output aspect ratio. Supported ratios are ``16:9`` (landscape), ``9:16`` - * (portrait), and ``1:1`` (square). - */ +export interface ElevenLabsEngineSettings { + model?: ElevenLabsModel | null; + similarity_boost?: number | null; + stability?: number | null; + style?: number | null; + use_speaker_boost?: boolean | null; + engine_type: "elevenlabs"; +} + +export type ElevenLabsModel = + | "eleven_multilingual_v2" + | "eleven_turbo_v2_5" + | "eleven_flash_v2_5" + | "eleven_v3" + | "eleven_v4"; + +export interface ElevenLabsSpeechSettings { + model_id: + | "eleven_multilingual_v2" + | "eleven_flash_v2" + | "eleven_flash_v2_5" + | "eleven_turbo_v2_5" + | "eleven_v3" + | "eleven_v4"; +} + +export type Expressiveness = "high" | "medium" | "low"; + +export interface FishEngineSettings { + model?: FishModel | null; + stability?: number | null; + similarity?: number | null; + engine_type: "fish"; +} + +export type FishModel = "s1" | "s2-pro" | "s2.1-pro"; + +export interface GptImage25Request { + model: "gpt-image-2.5"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | null; + reference_images?: Array; +} + +export interface HeyGenVoiceEngineSettings { + engine_type: "heygen_voice" | "orca"; +} + export type HyperframesAspectRatio = "16:9" | "9:16" | "1:1"; -/** - * Detailed HyperFrames render resource. - */ export interface HyperframesRenderDetail { - /** - * Unique render identifier. - */ render_id: string; - /** - * Current lifecycle state. - */ status: HyperframesRenderStatus; - /** - * Caller-supplied free-text label. - */ title?: string | null; - /** - * Caller-supplied client tracking ID. - */ callback_id?: string | null; - /** - * Presigned download URL for the rendered video. Present only when status is - * 'completed'. - */ video_url?: string | null; - /** - * Presigned download URL for the auto-generated thumbnail. - */ thumbnail_url?: string | null; - /** - * Video duration in seconds; null until completed. - */ duration?: number | null; - /** - * Frames per second the render was created at. - */ fps?: number | null; - /** - * Render quality preset. - */ quality?: "draft" | "standard" | "high" | null; - /** - * Output container/codec. - */ format: "mp4" | "webm" | "mov"; - /** - * Resolution tier, if one was set. - */ resolution?: HyperframesResolution | null; - /** - * Aspect ratio, if one was set. - */ aspect_ratio?: HyperframesAspectRatio | null; - /** - * Composition entry file path. - */ composition?: string | null; - /** - * Unix timestamp when the render was created. - */ created_at?: number | null; - /** - * Unix timestamp when the render terminated. Null until status is 'completed' - * or 'failed'. - */ completed_at?: number | null; - /** - * Error description. Present only when status is 'failed'. - */ failure_message?: string | null; - /** - * Why billing refused a finished render: 'hf_usage_exhausted', - * 'hf_subscription_required' or 'api_key_required', for which nothing was - * charged and the render can be sent again on a HeyGen API key; or - * 'usage_limit_reached', when a usage limit a workspace admin set (the API - * key's credit limit or the member's usage limit) was reached; sending the - * render again does not help until the limit is raised or resets, and - * `failure_message` is the limit's own message; or - * 'chunk_frames_exceed_timeout_at_max_chunks', when the render is too long to - * export at its size (even split into the most cloud parts allowed, it would - * still take longer than the time limit): sending it again fails the same way - * until it is shortened or rendered at a lower resolution or frame rate, and - * `failure_message` says so. Present only when status is 'failed' for one of - * these reasons. - */ failure_code?: string | null; } -/** - * Lifecycle status of a HyperFrames render. - */ export type HyperframesRenderStatus = "queued" | "rendering" | "completed" | "failed"; -/** - * Output resolution tier. 4K output uses a 1.5x pricing multiplier. - */ export type HyperframesResolution = "1080p" | "4k"; +export type ImageCreateRequest = GptImage25Request | NanoBanana2Request | NanoBanana21Request; + +export interface ImageScene { + type: "image"; + source: AssetUrl | AssetId | AssetBase64; + duration?: number | null; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; +} + +export interface ImageToVideoRequest { + end_user_id?: string | null; + mode: "image_to_video"; + model: "heygen-video-1"; + prompt: string; + prompt_enhancement?: "turbo" | "quality" | "default" | "disabled" | "ultraquality"; + resolution?: "480p" | "768p" | "1080p" | "2k"; + duration?: number | "auto"; + aspect_ratio?: "21:9" | "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "adaptive" | null; + seed?: number | null; + callback_url?: string | null; + callback_id?: string | null; + image: AssetUrl | AssetId | AssetBase64; + end_image?: AssetUrl | AssetId | AssetBase64 | null; +} + +export interface LipsyncCreateResponse { + lipsync_id: string; +} + +export interface LipsyncDetail { + id: string; + title?: string | null; + status: VideoTranslationStatus; + duration?: number | null; + video_url?: string | null; + caption_url?: string | null; + callback_id?: string | null; + created_at?: number | null; + failure_message?: string | null; + failure_code?: string | null; +} + +export interface LyriaMusicRequest { + model: "lyria-3-pro"; + prompt: string; + duration_seconds: number; + bpm?: number | null; + intensity?: number | null; + instrumental?: boolean; + lyrics?: string | null; +} + +export interface ModelVideoResponse { + video_id: string; + status: "pending" | "processing" | "completed" | "failed" | "cancelled"; + model: VideoGenerationModel; + created_at: number; + video_url?: string | null; + duration?: number | null; + input_video_duration?: number | null; + aspect_ratio?: string | null; + width?: number | null; + height?: number | null; + seed?: number | null; + timings?: ModelVideoTimings | null; + failure_code?: "generation_failed" | "generation_cancelled" | "usage_limit_reached" | null; + failure_message?: string | null; +} + +export interface ModelVideoTimings { + inference: number; +} + +export type MusicCreateRequest = LyriaMusicRequest; + +export type MusicFailureCode = + | "invalid_input" + | "content_policy_violation" + | "vendor_unavailable" + | "vendor_timeout" + | "ai_vendor_access_restricted" + | "insufficient_credit" + | "api_key_required" + | "internal_error"; + +export interface MusicResponse { + id: string; + status: MusicStatus; + model: "lyria-3-pro"; + created_at?: number | null; + requested_duration_seconds: number; + duration_seconds?: number | null; + audio_url?: string | null; + format?: "mp3" | null; + failure_code?: MusicFailureCode | null; + failure_message?: string | null; +} + +export type MusicStatus = "pending" | "completed" | "failed"; + +export interface NanoBanana21Request { + model: "nano-banana-2.1"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | "3:2" | "2:3" | "21:9" | null; + reference_images?: Array; +} + +export interface NanoBanana2Request { + model: "nano-banana-2"; + prompt: string; + num_images?: number; + aspect_ratio?: "1:1" | "16:9" | "9:16" | "4:3" | "3:4" | "3:2" | "2:3" | "21:9" | null; + reference_images?: Array; +} + +export interface PlaybackInput { + mode?: "freeze" | "loop" | "fit_to_scene" | null; + volume?: number; + mute?: boolean; +} + +export interface ReferenceToVideoRequest { + end_user_id?: string | null; + mode: "reference_to_video"; + model: "heygen-video-1"; + prompt: string; + prompt_enhancement?: "turbo" | "quality" | "default" | "disabled" | "ultraquality"; + resolution?: "480p" | "768p" | "1080p" | "2k"; + duration?: number | "auto"; + aspect_ratio?: "21:9" | "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "adaptive"; + seed?: number | null; + callback_url?: string | null; + callback_id?: string | null; + reference_images?: Array; + reference_videos?: Array; + reference_audio?: Array; +} + +export interface ScribeV2TranscriptionRequest { + audio: AssetUrl | AssetId | AssetBase64; + model: "scribe-v2"; + language?: string | null; + word_timestamps?: boolean; + diarize?: boolean; + num_speakers?: number | null; +} + +export type SpeechEngine = "starfish" | "orca" | "elevenlabs" | "elevenlabs_v3"; + export interface StandardAPIError { - /** - * Machine-readable error code - */ code: string; - /** - * Human-readable error message - */ message: string; - /** - * Which request field caused the error - */ param?: string | null; - /** - * Link to error documentation - */ doc_url?: string | null; - /** - * Per-item error details for batch/multi-node failures (e.g. graph compilation - * errors by node). - */ errors?: Array; - /** - * Request identifier, also returned in the X-HeyGen-Request-Id response - * header. Include it when contacting support. - */ request_id?: string; } export interface StandardAPIErrorDetail { - /** - * Machine-readable error code - */ code: string; - /** - * Human-readable error message - */ message: string; - /** - * Graph node id this error pertains to, if applicable - */ node_id?: string; - /** - * Port name on the node this error pertains to, if applicable - */ port?: string; - /** - * Field path within the item this error pertains to, if applicable - */ path?: string; } -/** - * Response from uploading an asset via POST /v3/assets. - */ +export interface StarfishEngineSettings { + engine_type: "starfish"; +} + +export interface StudioAvatarInput { + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; + type: "avatar"; + avatar_id: string; + motion_prompt?: string | null; + expressiveness?: Expressiveness | null; + engine?: AvatarVEngineConfig | AvatarIVEngineConfig | AvatarIIIEngineConfig | null; + background?: + | StudioColorBackgroundInput + | StudioImageBackgroundInput + | StudioVideoBackgroundInput + | null; + remove_background?: boolean | null; +} + +export interface StudioColorBackgroundInput { + type: "color"; + color: string; +} + +export interface StudioImageBackgroundInput { + type: "image"; + source: AssetUrl | AssetId | AssetBase64; +} + +export interface StudioVideoBackgroundInput { + type: "video"; + source: AssetUrl | AssetId | AssetBase64; +} + +export interface TextToVideoRequest { + end_user_id?: string | null; + mode: "text_to_video"; + model: "heygen-video-1"; + prompt: string; + prompt_enhancement?: "turbo" | "quality" | "default" | "disabled" | "ultraquality"; + resolution?: "480p" | "768p" | "1080p" | "2k"; + duration?: number | "auto"; + aspect_ratio?: "21:9" | "16:9" | "4:3" | "1:1" | "3:4" | "9:16"; + seed?: number | null; + callback_url?: string | null; + callback_id?: string | null; +} + +export interface ThirdPartyGeneratedImage { + url: string; + width: number; + height: number; +} + +export type ThirdPartyImageFailureCode = + | "content_policy_violation" + | "partial_generation" + | "vendor_unavailable" + | "ai_vendor_access_restricted" + | "insufficient_credit" + | "api_key_required" + | "internal_error"; + +export interface ThirdPartyImageResponse { + id: string; + status: "pending" | "completed" | "failed"; + model: string; + created_at: number; + images: Array; + failure_code?: ThirdPartyImageFailureCode | null; + failure_message?: string | null; +} + +export type TranscriptionCreateRequest = ScribeV2TranscriptionRequest; + +export type TranscriptionFailureCode = + | "invalid_input" + | "vendor_unavailable" + | "vendor_timeout" + | "ai_vendor_access_restricted" + | "internal_error" + | "insufficient_credit" + | "api_key_required" + | "rate_limited"; + +export interface TranscriptionResponse { + id: string; + status: TranscriptionStatus; + model: "scribe-v2"; + created_at: number; + language?: string | null; + duration_seconds?: number | null; + text?: string | null; + segments?: Array | null; + failure_code?: TranscriptionFailureCode | null; + failure_message?: string | null; +} + +export interface TranscriptionSegment { + speaker?: string | null; + start: number; + end: number; + text: string; + words?: Array | null; +} + +export type TranscriptionStatus = "pending" | "completed" | "failed"; + +export interface TranscriptionWord { + word: string; + start: number; + end: number; + speaker?: string | null; +} + export interface UploadAssetV3Response { - /** - * Unique asset identifier for use in other endpoints like POST - * /v3/video-agents - */ asset_id: string; - /** - * Public URL of the uploaded asset - */ url: string; - /** - * Detected MIME type of the file - */ mime_type: string; - /** - * File size in bytes - */ size_bytes: number; } + +export type VideoAspectRatio = "16:9" | "9:16" | "4:5" | "5:4" | "1:1" | "auto"; + +export interface VideoDetail { + id: string; + title?: string | null; + status: VideoStatus; + created_at?: number | null; + completed_at?: number | null; + video_url?: string | null; + thumbnail_url?: string | null; + gif_url?: string | null; + captioned_video_url?: string | null; + subtitle_url?: string | null; + duration?: number | null; + folder_id?: string | null; + output_language?: string | null; + failure_code?: string | null; + failure_message?: string | null; + video_page_url?: string | null; +} + +export type VideoGenerationModel = "heygen-video-1"; + +export type VideoOutputFormat = "mp4" | "webm"; + +export type VideoResolution = "4k" | "1080p" | "720p"; + +export interface VideoScene { + type: "video"; + source: AssetUrl | AssetId | AssetBase64; + playback?: PlaybackInput | null; + script?: string | null; + voice_id?: string | null; + audio_url?: string | null; + audio_asset_id?: string | null; + voice_settings?: VoiceSettingsInput | null; +} + +export type VideoStatus = "pending" | "processing" | "completed" | "failed"; + +export type VideoTranslationMode = "speed" | "precision"; + +export type VideoTranslationStatus = "pending" | "running" | "completed" | "failed"; + +export interface VoiceSettingsInput { + speed?: number; + pitch?: number; + volume?: number; + locale?: string | null; + engine_settings?: + | ElevenLabsEngineSettings + | FishEngineSettings + | StarfishEngineSettings + | HeyGenVoiceEngineSettings + | null; +} + +export interface VoiceV3Item { + voice_id: string; + name: string; + language: string; + gender: string; + preview_audio_url?: string | null; + support_pause: boolean; + support_locale: boolean; + type: AudioVoiceType; + default_engine?: SpeechEngine | null; + available_engines?: Array; +} + +export interface WatermarkInput { + image: AssetUrl | AssetId | AssetBase64; + scale?: number; + opacity?: number; + placement?: WatermarkPlacement | null; +} + +export interface WatermarkPlacement { + position?: WatermarkPosition; + offset_x?: number | null; + offset_y?: number | null; +} + +export type WatermarkPosition = "top_left" | "top_right" | "bottom_left" | "bottom_right"; + +export interface WordWithTimestamp { + word: string; + start: number; + end: number; +} diff --git a/packages/cli/src/cloud/generatedClient.contract.test.ts b/packages/cli/src/cloud/generatedClient.contract.test.ts index 815fb70010..0ecc416180 100644 --- a/packages/cli/src/cloud/generatedClient.contract.test.ts +++ b/packages/cli/src/cloud/generatedClient.contract.test.ts @@ -4,12 +4,8 @@ import { describe, expect, it } from "vitest"; const generatedClientSource = readFileSync(new URL("./_gen/client.ts", import.meta.url), "utf8"); describe("generated cloud client source contract", () => { - it("documents SRT subtitles on the uploadAsset endpoint", () => { - const uploadAssetDocs = generatedClientSource.match( - /\/\*\*\s*\*\s*Upload Asset[\s\S]*?\*\/\s+async uploadAsset/, - )?.[0]; - - expect(uploadAssetDocs).toContain("SRT subtitle"); - expect(uploadAssetDocs).toMatch(/Supported types:[^\n]*\bsrt\b/); + it("carries no API description text on its methods", () => { + expect(generatedClientSource).toMatch(/async uploadAsset\(/); + expect(generatedClientSource).not.toMatch(/\*\/\s+async \w+\(/); }); }); diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index b9355b6733..9ffab49957 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -2,7 +2,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { findFFmpeg } from "../browser/ffmpeg.js"; import { probeVideo } from "../measure/media.js"; @@ -41,6 +41,22 @@ afterEach(() => vi.unstubAllGlobals()); const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200 }); +/** Runs `work` to the end with the poll interval's timers fast-forwarded. */ +async function settled(work: Promise): Promise { + vi.useFakeTimers({ toFake: ["setTimeout"] }); + let done = false; + const run = work.finally(() => (done = true)); + try { + while (!done) { + await new Promise((r) => setImmediate(r)); + await vi.advanceTimersByTimeAsync(5000); + } + return await run; + } finally { + vi.useRealTimers(); + } +} + /** A 30 fps clip of these lavfi sources one after another: each new source is a cut. */ function shots(name: string, sources: string[], size = "180x320"): string { const path = join(project, name); @@ -690,6 +706,82 @@ describe("generate", () => { }); }); +describe("each route through the generated client", () => { + // Keys an earlier refusal in this file marked failed would move this file's keys on. + beforeEach(() => { + vi.stubEnv("HYPERFRAMES_CONFIG_DIR", mkdtempSync(join(tmpdir(), "hf-route-test-"))); + }); + const file = "https://files.test/route-result"; + const routes = [ + { type: "image", path: "/v3/images", ext: "png", done: { images: [{ url: file }] } }, + { type: "video", path: "/v3/models/videos", ext: "mp4", done: { video_url: file } }, + { type: "avatar", path: "/v3/videos", ext: "mp4", done: { video_url: file } }, + { type: "voice", path: "/v3/voices/speech", ext: "mp3", done: { audio_url: file } }, + { type: "music", path: "/v3/audio/music", ext: "mp3", done: { audio_url: file } }, + ] as const; + const request = (type: GenerateRequest["type"], ext: string): GenerateRequest => { + const photo = join(project, "route-photo.png"); + if (type === "avatar") + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=red:s=64x64"], + ...["-frames:v", "1", photo], + ]); + return { + type, + out: join(project, `assets/route-${type}.${ext}`), + prompt: "a desk", + script: "Hello", + voice: "c".repeat(32), + ...(type === "avatar" ? { photo } : {}), + }; + }; + + for (const { type, path, ext, done } of routes) { + const polled = type !== "voice"; + it.skipIf(!ffmpeg && type === "avatar")( + `${type}: posts ${path}${polled ? ", keyed, and polls it to completion" : " once, unkeyed"}`, + async () => { + const calls: { url: string; method?: string; key: string | null; body?: unknown }[] = []; + const answers = polled + ? [ + json({ data: { id: "j/1", status: "pending" } }), + json({ data: { id: "j/1", status: "completed", ...done } }), + ] + : [json({ data: { ...done, duration: 1 } })]; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + calls.push({ + url, + method: init?.method, + key: new Headers(init?.headers).get("Idempotency-Key"), + ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}), + }); + if (url === file) return new Response(new Uint8Array([1])); + return answers.shift()!; + }); + const r = await settled(generate(request(type, ext))); + const [post, ...rest] = calls; + expect(post).toMatchObject({ url: `https://api.test${path}`, method: "POST" }); + expect(post!.key).toEqual(polled ? expect.stringMatching(/^hf-[0-9a-f]{40}$/) : null); + expect(post!.body).toBeTypeOf("object"); + expect(rest.map((c) => [c.url, c.method])).toEqual([ + ...(polled ? [[`https://api.test${path}/j%2F1`, "GET"]] : []), + [file, undefined], + ]); + expect(r).toMatchObject({ route: path, job: polled ? "j/1" : "" }); + }, + ); + + it.skipIf(!ffmpeg && type === "avatar")(`${type}: says what a refusal needs`, async () => { + vi.stubGlobal( + "fetch", + async () => + new Response(JSON.stringify({ error: { code: "hf_usage_exhausted" } }), { status: 402 }), + ); + await expect(generate(request(type, ext))).rejects.toThrow("out of HyperFrames credits"); + }); + } +}); + describe("foldShort", () => { it("folds a piece of a few frames into its neighbour", () => { expect(foldShort([4, 4.1, 8], 10)).toEqual([4.1, 8]); @@ -713,10 +805,14 @@ describe("pickVoice", () => { vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { asked.push(url); if (url.includes("/v3/voices?")) return json({ data: catalog }); + if (url === "https://files.test/vo.mp3") return new Response(new Uint8Array([1])); if (init?.method === "POST") { const key = new Headers(init.headers).get("Idempotency-Key"); posts.push({ key, body: JSON.parse(String(init.body)) }); } + // Speech answers in the request; a video is a job to poll. + if (url.endsWith("/v3/voices/speech")) + return json({ data: { audio_url: "https://files.test/vo.mp3", duration: 1 } }); return json({ data: { id: "j1", status: "pending" } }); }); afterEach(() => { @@ -741,6 +837,18 @@ describe("pickVoice", () => { expect(asked[0]).toBe("https://api.test/v3/voices?type=public&language=Chinese&limit=1"); }); + it("says what to do when HeyGen will not list its voices", async () => { + vi.stubGlobal( + "fetch", + async () => + new Response(JSON.stringify({ error: { code: "unauthorized" } }), { status: 401 }), + ); + await expect(pickVoice({ type: "voice", out: "vo.mp3" })).rejects.toMatchObject({ + message: "HeyGen 401 listing its voices", + fix: "run the same command again", + }); + }); + it("lists voices to pick from when none is named", async () => { heygen(); await expect(pickVoice({ type: "voice", out: "vo.mp3" })).rejects.toMatchObject({ @@ -790,7 +898,11 @@ describe("pickVoice", () => { script: "你好", ...(type === "avatar" ? { photo: face } : {}), }; - expect(await runOnce(r)).toMatchObject({ message: "HeyGen job j1 is still pending" }); + expect(await runOnce(r)).toMatchObject( + type === "voice" + ? { route: "/v3/voices/speech" } + : { message: "HeyGen job j1 is still pending" }, + ); catalog = [voice("b", "Wei"), voice("a", "Lin")]; await runOnce(r); expect(posts).toHaveLength(2); diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index 704d808c0f..e5c2ad6da4 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -12,6 +12,13 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, extname, join, relative, resolve } from "node:path"; +import type { + CreateModelVideoRequest, + CreateSpeechV3Request, + CreateVideoV3RequestBody, + ImageCreateRequest, + MusicCreateRequest, +} from "../cloud/_gen/types.js"; import { findCuts, signatures } from "../measure/cuts.js"; import { Fix, probeVideo, projectOf, round, runFfmpeg, videoClip } from "../measure/media.js"; import { @@ -22,6 +29,7 @@ import { runJob, senderScope, type Job, + type JobRoute, type Voice, } from "./v3.js"; @@ -141,9 +149,7 @@ export function structureRatio( return { inside: sum[0]! / (n[0] || 1), ring: sum[1]! / (n[1] || 1) }; } -interface Route { - path: string; - poll: (id: string) => string; +interface Route extends JobRoute { body: (r: GenerateRequest, media: Media) => Record; /** Where the finished job keeps the file's URL. */ urls: string[]; @@ -171,7 +177,9 @@ const need = (value: T | undefined, flag: string, example: string): T => { const ROUTES: Record = { image: { path: "/v3/images", - poll: (id) => `/v3/images/${id}`, + start: (c, body, key, signal) => + c.createImage({ body: body as ImageCreateRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getImage({ generation_id: id, signal }), body: (r, media) => ({ model: r.model ?? DEFAULTS.image, prompt: need(r.prompt, "--prompt", '--prompt "the same room, empty, no text"'), @@ -187,7 +195,9 @@ const ROUTES: Record = { }, video: { path: "/v3/models/videos", - poll: (id) => `/v3/models/videos/${id}`, + start: (c, body, key, signal) => + c.createModelVideo({ body: body as CreateModelVideoRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getModelVideo({ video_id: id, signal }), // fallow-ignore-next-line complexity body: (r, media) => { // A person speaking given audio is a reference shot: the references show who, the audio drives the lips. @@ -221,7 +231,9 @@ const ROUTES: Record = { }, avatar: { path: "/v3/videos", - poll: (id) => `/v3/videos/${id}`, + start: (c, body, key, signal) => + c.createVideo({ body: body as CreateVideoV3RequestBody, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getVideoV3({ video_id: id, signal }), body: (r, media) => ({ type: "image", image: media(need(r.photo, "--photo", "--photo assets/presenter.png"), "image"), @@ -235,8 +247,10 @@ const ROUTES: Record = { ext: [".mp4"], }, voice: { + // Speech answers in the request and takes no Idempotency-Key, so there is no status to poll. path: "/v3/voices/speech", - poll: (id) => `/v3/voices/speech/${id}`, + start: (c, body, _key, signal) => + c.createSpeechV3({ body: body as CreateSpeechV3Request, signal }), body: (r) => ({ text: need(r.script, "--script", '--script "What the voice says"'), voice_id: need(r.voice, "--voice", "--voice "), @@ -247,7 +261,9 @@ const ROUTES: Record = { }, music: { path: "/v3/audio/music", - poll: (id) => `/v3/audio/music/${id}`, + start: (c, body, key, signal) => + c.createMusic({ body: body as MusicCreateRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getMusic({ music_id: id, signal }), body: (r) => ({ model: r.model ?? DEFAULTS.music, prompt: need(r.prompt, "--prompt", '--prompt "warm lo-fi beat, soft keys"'), @@ -551,7 +567,7 @@ export async function generate( try { const body = route.body(r, inline(work, r.masks)); const key = idempotencyKey(route.path, [JSON.stringify(body)]); - const job = await runJob(route.path, body, key, (id) => route.poll(encodeURIComponent(id)), { + const job = await runJob(route, body, key, { log: { projectDir: project ?? undefined, what: `generate ${r.type} ${relative(project ?? ".", out)}`, diff --git a/packages/cli/src/heygen/transcribe.test.ts b/packages/cli/src/heygen/transcribe.test.ts index 1078db14e9..b30204c1b5 100644 --- a/packages/cli/src/heygen/transcribe.test.ts +++ b/packages/cli/src/heygen/transcribe.test.ts @@ -69,4 +69,50 @@ describe("HeyGen transcription", () => { }); expect(JSON.parse(readFileSync(out, "utf8"))).toHaveLength(2); }); + + it.skipIf(!ffmpeg)("keys the request and polls a pending transcription to the end", async () => { + const audio = join(dir, "b.wav"); + execFileSync(ffmpeg!, ["-v", "error", "-y", "-f", "lavfi", "-i", "sine=f=200:d=1", audio]); + const calls: { url: string; method?: string; key: string | null }[] = []; + const answers = [{ id: "t1", status: "pending" }, job]; + vi.stubGlobal("fetch", async (url: string, init: RequestInit) => { + calls.push({ + url, + method: init.method, + key: new Headers(init.headers).get("Idempotency-Key"), + }); + return new Response(JSON.stringify({ data: answers.shift() }), { status: 200 }); + }); + vi.useFakeTimers({ toFake: ["setTimeout"] }); + let done = false; + const pending = transcribeWithHeygen(audio, join(dir, "b.json")).finally(() => (done = true)); + while (!done) { + await new Promise((r) => setImmediate(r)); + await vi.advanceTimersByTimeAsync(5000); + } + const r = await pending; + vi.useRealTimers(); + expect(calls).toEqual([ + { + url: "https://api.test/v3/audio/transcriptions", + method: "POST", + key: expect.stringMatching(/^hf-[0-9a-f]{40}$/), + }, + { url: "https://api.test/v3/audio/transcriptions/t1", method: "GET", key: null }, + ]); + expect(r.detectedLanguage).toBe("en"); + }); + + it.skipIf(!ffmpeg)("turns a refusal into what to do next", async () => { + const audio = join(dir, "c.wav"); + execFileSync(ffmpeg!, ["-v", "error", "-y", "-f", "lavfi", "-i", "sine=f=250:d=1", audio]); + vi.stubGlobal( + "fetch", + async () => + new Response(JSON.stringify({ error: { code: "api_key_required" } }), { status: 403 }), + ); + await expect(transcribeWithHeygen(audio, join(dir, "c.json"))).rejects.toMatchObject({ + fix: "continue without this step", + }); + }); }); diff --git a/packages/cli/src/heygen/transcribe.ts b/packages/cli/src/heygen/transcribe.ts index b98be3f6ca..68a742a941 100644 --- a/packages/cli/src/heygen/transcribe.ts +++ b/packages/cli/src/heygen/transcribe.ts @@ -3,10 +3,20 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { TranscriptionCreateRequest } from "../cloud/_gen/types.js"; import { runFfmpeg } from "../measure/media.js"; -import { idempotencyKey, runJob } from "./v3.js"; +import { idempotencyKey, runJob, type JobRoute } from "./v3.js"; -const ROUTE = "/v3/audio/transcriptions"; +const ROUTE: JobRoute = { + path: "/v3/audio/transcriptions", + start: (c, body, key, signal) => + c.createTranscription({ + body: body as TranscriptionCreateRequest, + idempotencyKey: key, + signal, + }), + status: (c, id, signal) => c.getTranscription({ transcription_id: id, signal }), +}; const MODEL = "scribe-v2"; export interface HeygenTranscript { @@ -74,8 +84,7 @@ export async function transcribeWithHeygen( audio: { type: "base64", media_type: "audio/mpeg", data: audio.toString("base64") }, diarize, }, - idempotencyKey(ROUTE, [audio, String(diarize)]), - (id) => `${ROUTE}/${encodeURIComponent(id)}`, + idempotencyKey(ROUTE.path, [audio, String(diarize)]), { log: { projectDir: opts.projectDir, what: `transcribe ${inputPath}` }, timeoutMs: 10 * 60_000, diff --git a/packages/cli/src/heygen/v3.test.ts b/packages/cli/src/heygen/v3.test.ts index c76972079c..efe32dede5 100644 --- a/packages/cli/src/heygen/v3.test.ts +++ b/packages/cli/src/heygen/v3.test.ts @@ -2,6 +2,8 @@ import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { CreateModelVideoRequest } from "../cloud/_gen/types.js"; +import type { JobRoute } from "./v3.js"; vi.mock("../cloud/auth.js", () => ({ resolveCloudAuthHeaders: async () => ({ "x-api-key": "k" }), @@ -11,6 +13,14 @@ vi.mock("../cloud/auth.js", () => ({ const { idempotencyKey, runJob } = await import("./v3.js"); +/** A job route through the generated client: the create and the status GET it polls. */ +const X: JobRoute = { + path: "/v3/models/videos", + start: (c, body, key, signal) => + c.createModelVideo({ body: body as CreateModelVideoRequest, idempotencyKey: key, signal }), + status: (c, id, signal) => c.getModelVideo({ video_id: id, signal }), +}; + const reply = (status: number, body: unknown) => new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); @@ -35,12 +45,12 @@ describe("runJob", () => { calls.push({ url, init }); return answers.shift()!; }); - const job = await runJob("/v3/x", { a: 1 }, "hf-key", (id) => `/v3/x/${id}`, { intervalMs: 1 }); + const job = await runJob(X, { a: 1 }, "hf-key", { intervalMs: 1 }); expect(job.output).toEqual({ json: { ok: 1 } }); expect(calls.map((c) => c.url)).toEqual([ - "https://api.test/v3/x", - "https://api.test/v3/x/j1", - "https://api.test/v3/x/j1", + "https://api.test/v3/models/videos", + "https://api.test/v3/models/videos/j1", + "https://api.test/v3/models/videos/j1", ]); expect(new Headers(calls[0]!.init.headers).get("Idempotency-Key")).toBe("hf-key"); }); @@ -49,50 +59,50 @@ describe("runJob", () => { vi.stubGlobal("fetch", async () => reply(402, { error: { code: "insufficient_credit", message: "no" } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: expect.stringContaining("top up"), }); vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_subscription_required", message: "no plan" } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: "Subscribe: https://hyperframes.studio/pricing?from=cli", }); vi.stubGlobal("fetch", async () => reply(402, { code: "hf_usage_exhausted", url: "https://pay.test/credits" }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: "Add credits: https://pay.test/credits", }); vi.stubGlobal("fetch", async () => reply(401, { message: "bad" })); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: "npx hyperframes auth login", }); vi.stubGlobal("fetch", async () => reply(200, { data: { id: "j", status: "failed", failure_code: "content_policy_violation" } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ fix: expect.stringContaining("reword"), }); }); it("names what an out-of-credits step needs, from the call's own price when HeyGen does not say", async () => { vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_usage_exhausted" } })); - await expect(runJob("/v3/x", {}, "k", (id) => id, { credits: 782 })).rejects.toMatchObject({ + await expect(runJob(X, {}, "k", { credits: 782 })).rejects.toMatchObject({ message: expect.stringContaining("this step needs about 782 in total"), }); - await expect(runJob("/v3/x", {}, "k", (id) => id, { credits: 782 })).rejects.toMatchObject({ + await expect(runJob(X, {}, "k", { credits: 782 })).rejects.toMatchObject({ message: expect.stringContaining("billing top-up --needed 782"), fix: expect.stringContaining("needed_credits=782"), }); vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_usage_exhausted", needed_credits: 180, balance: 30 } }), ); - await expect(runJob("/v3/x", {}, "k", (id) => id, { credits: 782 })).rejects.toMatchObject({ + await expect(runJob(X, {}, "k", { credits: 782 })).rejects.toMatchObject({ message: expect.stringContaining("this step needs 180 in total, balance 30"), }); vi.stubGlobal("fetch", async () => reply(402, { error: { code: "hf_usage_exhausted" } })); - await expect(runJob("/v3/x", {}, "k", (id) => id)).rejects.toMatchObject({ + await expect(runJob(X, {}, "k")).rejects.toMatchObject({ message: expect.stringContaining("this step needs more"), }); }); @@ -104,7 +114,7 @@ describe("runJob", () => { if (init.method === "POST") keys.push(new Headers(init.headers).get("Idempotency-Key")); return answers.shift()!; }); - return runJob("/v3/x", {}, key, (id) => `/v3/x/${id}`, { intervalMs: 1 }); + return runJob(X, {}, key, { intervalMs: 1 }); }; const failed = () => [ reply(200, { data: { id: "j", status: "pending" } }), @@ -146,8 +156,8 @@ describe("runJob", () => { keys.push(new Headers(init.headers).get("Idempotency-Key")); return answers.shift()!; }); - await expect(runJob("/v3/x", {}, key, (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, key, (id) => id); + await expect(runJob(X, {}, key)).rejects.toThrow(); + await runJob(X, {}, key); expect(keys).toEqual([key, key]); }, ); @@ -159,7 +169,7 @@ describe("runJob", () => { keys.push(new Headers(init.headers).get("Idempotency-Key")); return answers.shift()!; }); - return runJob("/v3/x", {}, "hf-shared", (id) => id); + return runJob(X, {}, "hf-shared"); }; vi.stubEnv("HEYGEN_API_KEY", "payer-a"); await expect(run([reply(402, { code: "hf_usage_exhausted" })])).rejects.toThrow(); @@ -214,11 +224,11 @@ describe("runJob", () => { done(), ]); signIn("a"); - await runJob("/v3/x", {}, "hf-host", (id) => id); + await runJob(X, {}, "hf-host"); signIn("b"); - await expect(runJob("/v3/x", {}, "hf-host", (id) => id)).rejects.toThrow(); + await expect(runJob(X, {}, "hf-host")).rejects.toThrow(); signIn("a"); - await runJob("/v3/x", {}, "hf-host", (id) => id); + await runJob(X, {}, "hf-host"); expect(keys).toEqual(["hf-host", "hf-host", "hf-host"]); }); @@ -234,14 +244,12 @@ describe("runJob", () => { done(), ]); signIn("b"); - await runJob("/v3/x", {}, "hf-switch", (id) => id); + await runJob(X, {}, "hf-switch"); signIn("a"); - await expect( - runJob("/v3/x", {}, "hf-switch", (id) => id, { intervalMs: 1 }), - ).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-switch", (id) => id); + await expect(runJob(X, {}, "hf-switch", { intervalMs: 1 })).rejects.toThrow(); + await runJob(X, {}, "hf-switch"); signIn("a"); - await runJob("/v3/x", {}, "hf-switch", (id) => id); + await runJob(X, {}, "hf-switch"); expect(keys).toEqual(["hf-switch", "hf-switch", "hf-switch", "hf-switch-1"]); }); @@ -257,7 +265,7 @@ describe("runJob", () => { return done(); }); signIn("a"); - await runJob("/v3/x", {}, "hf-window", (id) => id); + await runJob(X, {}, "hf-window"); expect(seen).toEqual(["Bearer a"]); }); @@ -267,28 +275,91 @@ describe("runJob", () => { done(), ]); signIn("a"); - await expect(runJob("/v3/x", {}, "hf-unknown", (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-unknown", (id) => id); + await expect(runJob(X, {}, "hf-unknown")).rejects.toThrow(); + await runJob(X, {}, "hf-unknown"); expect(keys).toEqual(["hf-unknown", "hf-unknown"]); }); it("sends a job that failed for this account as a new one", async () => { const keys = server(accounts, [failed(), done()]); signIn("a"); - await expect(runJob("/v3/x", {}, "hf-own", (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-own", (id) => id); + await expect(runJob(X, {}, "hf-own")).rejects.toThrow(); + await runJob(X, {}, "hf-own"); expect(keys).toEqual(["hf-own", "hf-own-1"]); }); it("keeps the key through a lost answer, then a new sign-in", async () => { const keys = server(accounts, [reply(401, { message: "expired" }), done()]); signIn("a"); - await expect(runJob("/v3/x", {}, "hf-resigned", (id) => id)).rejects.toThrow(); - await runJob("/v3/x", {}, "hf-resigned", (id) => id); + await expect(runJob(X, {}, "hf-resigned")).rejects.toThrow(); + await runJob(X, {}, "hf-resigned"); expect(keys).toEqual(["hf-resigned", "hf-resigned"]); }); }); + it("sends the request as JSON and polls the status route with a GET", async () => { + const calls: { url: string; init: RequestInit }[] = []; + const answers = [ + reply(202, { data: { video_id: "v/1", status: "pending" } }), + reply(200, { data: { id: "v/1", status: "completed" } }), + ]; + vi.stubGlobal("fetch", async (url: string, init: RequestInit) => { + calls.push({ url, init }); + return answers.shift()!; + }); + const job = await runJob(X, { prompt: "p" }, "hf-shape", { intervalMs: 1 }); + expect(job).toMatchObject({ id: "v/1", status: "completed" }); + const [post, get] = calls; + expect(post!.init.method).toBe("POST"); + expect(JSON.parse(String(post!.init.body))).toEqual({ prompt: "p" }); + const headers = new Headers(post!.init.headers); + expect(headers.get("content-type")).toBe("application/json"); + expect(headers.get("x-api-key")).toBe("payer"); + expect(get!.url).toBe("https://api.test/v3/models/videos/v%2F1"); + expect(get!.init.method).toBe("GET"); + expect(get!.init.body).toBeUndefined(); + }); + + it("asks again for an answer it could not read or never got", async () => { + vi.stubGlobal("fetch", async () => reply(503, { error: { code: "unavailable" } })); + await expect(runJob(X, {}, "hf-503")).rejects.toMatchObject({ + message: "HeyGen 503 for /v3/models/videos with no readable answer", + fix: "run the same command again: it resumes the same job", + }); + vi.stubGlobal("fetch", async () => new Response("", { status: 400 })); + await expect(runJob(X, {}, "hf-html")).rejects.toThrow("HeyGen 400 for /v3/models/videos"); + vi.stubGlobal("fetch", async () => new Response("not json", { status: 200 })); + await expect(runJob(X, {}, "hf-text")).rejects.toThrow("HeyGen 200 for /v3/models/videos"); + vi.stubGlobal("fetch", async () => { + throw new TypeError("fetch failed"); + }); + await expect(runJob(X, {}, "hf-lost")).rejects.toMatchObject({ + message: "no answer from HeyGen for /v3/models/videos (fetch failed)", + }); + const keys: (string | null)[] = []; + vi.stubGlobal("fetch", async (_url: string, init: RequestInit) => { + keys.push(new Headers(init.headers).get("Idempotency-Key")); + return reply(200, { data: { id: "j", status: "completed" } }); + }); + await runJob(X, {}, "hf-503"); + expect(keys).toEqual(["hf-503"]); + }); + + it("takes the answer of a route with no status as final", async () => { + const calls: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + calls.push(url); + return reply(200, { data: { audio_url: "https://files.test/a.mp3", duration: 1 } }); + }); + const sync = { path: X.path, start: X.start }; + expect(await runJob(sync, {}, "hf-sync")).toMatchObject({ + id: "", + status: "completed", + audio_url: "https://files.test/a.mp3", + }); + expect(calls).toHaveLength(1); + }); + it("keys the same request the same way", () => { expect(idempotencyKey("/v3/x", ["a", Buffer.from("b")])).toBe( idempotencyKey("/v3/x", ["a", Buffer.from("b")]), diff --git a/packages/cli/src/heygen/v3.ts b/packages/cli/src/heygen/v3.ts index 4d36d2bffa..93351f9142 100644 --- a/packages/cli/src/heygen/v3.ts +++ b/packages/cli/src/heygen/v3.ts @@ -8,6 +8,7 @@ import { configDir } from "../auth/paths.js"; import { AuthClient, buildAuthHeaders } from "../auth/client.js"; import { resolveCredential } from "../auth/resolver.js"; import { refreshIfNeeded, resolveCloudAuthHeaders, resolveCloudBaseUrl } from "../cloud/auth.js"; +import { HyperframesApiError, HyperframesCloudClient } from "../cloud/_gen/client.js"; import { Fix } from "../measure/media.js"; import { pageUrl } from "./plan.js"; @@ -136,47 +137,47 @@ function refusal(status: number, body: unknown, estimate?: number): Refused { ); } +export interface JobRoute { + path: string; + start: ( + client: HyperframesCloudClient, + body: unknown, + key: string, + signal: AbortSignal, + ) => Promise; + status?: (client: HyperframesCloudClient, id: string, signal: AbortSignal) => Promise; +} + +const clientFor = (auth: Record) => + new HyperframesCloudClient({ baseUrl: resolveCloudBaseUrl(), getAuthHeaders: () => auth }); + +const resume = "run the same command again: it resumes the same job"; + // fallow-ignore-next-line complexity async function call( - method: "GET" | "POST", route: string, - body?: unknown, - key?: string, + send: (client: HyperframesCloudClient, signal: AbortSignal) => Promise, + timeoutMs: number, estimate?: number, auth?: Record, ): Promise { - const headers: Record = { - ...(auth ?? (await resolveCloudAuthHeaders().catch(noSignIn))), - accept: "application/json", - }; - if (body !== undefined) headers["content-type"] = "application/json"; - if (key) headers["Idempotency-Key"] = key; - let res: Response; + const client = clientFor(auth ?? (await resolveCloudAuthHeaders().catch(noSignIn))); + let data: unknown; try { - res = await fetch(`${resolveCloudBaseUrl()}${route}`, { - method, - headers, - ...(body !== undefined ? { body: JSON.stringify(body) } : {}), - signal: AbortSignal.timeout(method === "POST" ? 180_000 : 30_000), - }); + data = await send(client, AbortSignal.timeout(timeoutMs)); } catch (err) { - throw new Fix( - `no answer from HeyGen for ${route} (${err instanceof Error ? err.message : String(err)})`, - "run the same command again: it resumes the same job", - ); - } - const value: unknown = await res.json().catch(() => null); - if (res.status >= 500 || value === null) - throw new Fix( - `HeyGen ${res.status} for ${route} with no readable answer`, - "run the same command again: it resumes the same job", - ); - if (!res.ok) { - const refused = refusal(res.status, value, estimate); - refused.status = res.status; + if (!(err instanceof HyperframesApiError)) + throw new Fix( + `no answer from HeyGen for ${route} (${err instanceof Error ? err.message : String(err)})`, + resume, + ); + if (err.status >= 500 || err.status < 300 || err.raw == null) + throw new Fix(`HeyGen ${err.status} for ${route} with no readable answer`, resume); + const refused = refusal(err.status, err.raw, estimate); + refused.status = err.status; throw refused; } - const data = isRecord(value) && isRecord(value.data) ? value.data : value; + if (data == null) throw new Fix(`HeyGen answered ${route} with no readable answer`, resume); if (!isRecord(data)) throw new Fix(`HeyGen answered ${route} with no job`, "run the same command again"); const id = data.id ?? data.job_id ?? data.video_id ?? data.generation_id; @@ -214,26 +215,33 @@ export interface RunOptions { credits?: number; } -/** Starts a job (or picks up the one this key already started) and polls `pollRoute(id)` until it ends. */ +/** Starts a job (or picks up the one this key already started) and polls its status until it ends. */ export async function runJob( - route: string, + jobRoute: JobRoute, body: unknown, key: string, - pollRoute: (id: string) => string, opts: RunOptions = {}, ): Promise { const started = Date.now(); + const route = jobRoute.path; const { auth, scope } = await jobSender(); const sent = liveKey(key, scope); let job: Job; try { - job = await call("POST", route, body, sent, opts.credits, auth); + job = await call( + route, + (client, signal) => jobRoute.start(client, body, sent, signal), + 180_000, + opts.credits, + auth, + ); } catch (err) { if (err instanceof Refused && !KEEPS_KEY.has(err.status)) markFailed(sent, scope); throw err; } if (opts.log) log(opts.log, { route, id: job.id, key: sent, status: job.status }); - while (RUNNING.has(job.status.toLowerCase())) { + const status = jobRoute.status; + while (status && RUNNING.has(job.status.toLowerCase())) { if (Date.now() - started > (opts.timeoutMs ?? 15 * 60_000)) throw new Fix( `HeyGen job ${job.id} is still ${job.status}`, @@ -241,7 +249,12 @@ export async function runJob( ); opts.onWait?.(job); await new Promise((r) => setTimeout(r, opts.intervalMs ?? 5000)); - job = await call("GET", pollRoute(job.id)); + const id = job.id; + job = await call( + `${route}/${encodeURIComponent(id)}`, + (client, signal) => status(client, id, signal), + 30_000, + ); } if (opts.log) log(opts.log, { route, id: job.id, key: sent, status: job.status }); if (!/^(completed|succeeded|success|done)$/i.test(job.status)) { @@ -265,23 +278,19 @@ const isVoice = (v: unknown): v is Voice => /** HeyGen's public voices for a language, best first; a language it does not know gets multilingual ones. */ export async function publicVoices(language: string, limit: number): Promise { - const headers = { - ...(await resolveCloudAuthHeaders().catch(noSignIn)), - accept: "application/json", - }; - const query = new URLSearchParams({ type: "public", language, limit: String(limit) }); - const res = await fetch(`${resolveCloudBaseUrl()}/v3/voices?${query}`, { - headers, - signal: AbortSignal.timeout(30_000), - }).catch((err: unknown) => { - throw new Fix( - `no answer from HeyGen for its voices (${err instanceof Error ? err.message : String(err)})`, - "run the same command again", - ); - }); - const value: unknown = await res.json().catch(() => null); - if (!res.ok || !isRecord(value) || !Array.isArray(value.data)) - throw new Fix(`HeyGen ${res.status} listing its voices`, "run the same command again"); + const client = clientFor(await resolveCloudAuthHeaders().catch(noSignIn)); + const value = await client + .listVoicesV3({ type: "public", language, limit, signal: AbortSignal.timeout(30_000) }) + .catch((err: unknown) => { + if (err instanceof HyperframesApiError) + throw new Fix(`HeyGen ${err.status} listing its voices`, "run the same command again"); + throw new Fix( + `no answer from HeyGen for its voices (${err instanceof Error ? err.message : String(err)})`, + "run the same command again", + ); + }); + if (!isRecord(value) || !Array.isArray(value.data)) + throw new Fix("HeyGen answered its voices with no list", "run the same command again"); return value.data .filter(isVoice) .map(({ voice_id, name, language, gender }) => ({ voice_id, name, language, gender }));