mirror of
https://github.com/QuantumNous/new-api.git
synced 2026-09-12 15:21:09 +00:00
feat(task): replace built-in task adaptors with a sandboxed JS plugin system (#7076)
This commit is contained in:
Calcium-Ion
@@ -1,638 +0,0 @@
|
||||
package ali
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/logger"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
"github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
"github.com/samber/lo"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
// AliVideoRequest 阿里通义万相视频生成请求
|
||||
type AliVideoRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input AliVideoInput `json:"input"`
|
||||
Parameters *AliVideoParameters `json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
// AliVideoMedia describes Wan2.7 image-to-video media inputs.
|
||||
type AliVideoMedia struct {
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// AliVideoInput 视频输入参数
|
||||
type AliVideoInput struct {
|
||||
Prompt string `json:"prompt,omitempty"` // 文本提示词
|
||||
ImgURL string `json:"img_url,omitempty"` // 首帧图像URL或Base64（图生视频）
|
||||
FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频）
|
||||
LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频）
|
||||
AudioURL string `json:"audio_url,omitempty"` // 音频URL（wan2.5支持）
|
||||
Media []AliVideoMedia `json:"media,omitempty"` // 媒体列表（wan2.7-i2v新协议）
|
||||
NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词
|
||||
Template string `json:"template,omitempty"` // 视频特效模板
|
||||
}
|
||||
|
||||
// AliVideoParameters 视频参数
|
||||
type AliVideoParameters struct {
|
||||
Resolution string `json:"resolution,omitempty"` // 分辨率: 480P/720P/1080P（图生视频、首尾帧生视频）
|
||||
Size string `json:"size,omitempty"` // 尺寸: 如 "832*480"（文生视频）
|
||||
Duration int `json:"duration,omitempty"` // 时长: 3-10秒
|
||||
PromptExtend bool `json:"prompt_extend,omitempty"` // 是否开启prompt智能改写
|
||||
Watermark bool `json:"watermark,omitempty"` // 是否添加水印
|
||||
Audio *bool `json:"audio,omitempty"` // 是否添加音频（wan2.5）
|
||||
Seed int `json:"seed,omitempty"` // 随机数种子
|
||||
}
|
||||
|
||||
// AliVideoResponse 阿里通义万相响应
|
||||
type AliVideoResponse struct {
|
||||
Output AliVideoOutput `json:"output"`
|
||||
RequestID string `json:"request_id"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Usage *AliUsage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// AliVideoOutput 输出信息
|
||||
type AliVideoOutput struct {
|
||||
TaskID string `json:"task_id"`
|
||||
TaskStatus string `json:"task_status"`
|
||||
SubmitTime string `json:"submit_time,omitempty"`
|
||||
ScheduledTime string `json:"scheduled_time,omitempty"`
|
||||
EndTime string `json:"end_time,omitempty"`
|
||||
OrigPrompt string `json:"orig_prompt,omitempty"`
|
||||
ActualPrompt string `json:"actual_prompt,omitempty"`
|
||||
VideoURL string `json:"video_url,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// AliUsage 使用统计
|
||||
type AliUsage struct {
|
||||
Duration dto.IntValue `json:"duration,omitempty"`
|
||||
VideoCount dto.IntValue `json:"video_count,omitempty"`
|
||||
SR dto.IntValue `json:"SR,omitempty"`
|
||||
}
|
||||
|
||||
type AliMetadata struct {
|
||||
// Input 相关
|
||||
AudioURL string `json:"audio_url,omitempty"` // 音频URL
|
||||
ImgURL string `json:"img_url,omitempty"` // 图片URL（图生视频）
|
||||
FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频）
|
||||
LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频）
|
||||
Media []AliVideoMedia `json:"media,omitempty"` // 媒体列表（wan2.7-i2v新协议）
|
||||
NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词
|
||||
Template string `json:"template,omitempty"` // 视频特效模板
|
||||
|
||||
// Parameters 相关
|
||||
Resolution *string `json:"resolution,omitempty"` // 分辨率: 480P/720P/1080P
|
||||
Size *string `json:"size,omitempty"` // 尺寸: 如 "832*480"
|
||||
Duration *int `json:"duration,omitempty"` // 时长
|
||||
PromptExtend *bool `json:"prompt_extend,omitempty"` // 是否开启prompt智能改写
|
||||
Watermark *bool `json:"watermark,omitempty"` // 是否添加水印
|
||||
Audio *bool `json:"audio,omitempty"` // 是否添加音频
|
||||
Seed *int `json:"seed,omitempty"` // 随机数种子
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *taskdto.TaskError) {
|
||||
// ValidateMultipartDirect 负责解析并将原始 TaskSubmitReq 存入 context
|
||||
return relaycommon.ValidateMultipartDirect(c, info)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
return fmt.Sprintf("%s/api/v1/services/aigc/video-generation/video-synthesis", a.baseURL), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers for Ali API
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Authorization", "Bearer "+a.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-DashScope-Async", "enable") // 阿里异步任务必须设置
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
taskReq, err := relaycommon.GetTaskRequest(c)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "get_task_request_failed")
|
||||
}
|
||||
|
||||
aliReq, err := a.convertToAliRequest(info, taskReq)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "convert_to_ali_request_failed")
|
||||
}
|
||||
logger.LogJson(c, "ali video request body", aliReq)
|
||||
|
||||
bodyBytes, err := common.Marshal(aliReq)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "marshal_ali_request_failed")
|
||||
}
|
||||
return bytes.NewReader(bodyBytes), nil
|
||||
}
|
||||
|
||||
var (
|
||||
size480p = []string{
|
||||
"832*480",
|
||||
"480*832",
|
||||
"624*624",
|
||||
}
|
||||
size720p = []string{
|
||||
"1280*720",
|
||||
"720*1280",
|
||||
"960*960",
|
||||
"1088*832",
|
||||
"832*1088",
|
||||
}
|
||||
size1080p = []string{
|
||||
"1920*1080",
|
||||
"1080*1920",
|
||||
"1440*1440",
|
||||
"1632*1248",
|
||||
"1248*1632",
|
||||
}
|
||||
)
|
||||
|
||||
func sizeToResolution(size string) (string, error) {
|
||||
if lo.Contains(size480p, size) {
|
||||
return "480P", nil
|
||||
} else if lo.Contains(size720p, size) {
|
||||
return "720P", nil
|
||||
} else if lo.Contains(size1080p, size) {
|
||||
return "1080P", nil
|
||||
}
|
||||
return "", fmt.Errorf("invalid size: %s", size)
|
||||
}
|
||||
|
||||
func ProcessAliOtherRatios(aliReq *AliVideoRequest) (map[string]float64, error) {
|
||||
otherRatios := make(map[string]float64)
|
||||
aliRatios := map[string]map[string]float64{
|
||||
"wan2.6-i2v": {
|
||||
"720P": 1,
|
||||
"1080P": 1 / 0.6,
|
||||
},
|
||||
"wan2.5-t2v-preview": {
|
||||
"480P": 1,
|
||||
"720P": 2,
|
||||
"1080P": 1 / 0.3,
|
||||
},
|
||||
"wan2.2-t2v-plus": {
|
||||
"480P": 1,
|
||||
"1080P": 0.7 / 0.14,
|
||||
},
|
||||
"wan2.5-i2v-preview": {
|
||||
"480P": 1,
|
||||
"720P": 2,
|
||||
"1080P": 1 / 0.3,
|
||||
},
|
||||
"wan2.2-i2v-plus": {
|
||||
"480P": 1,
|
||||
"1080P": 0.7 / 0.14,
|
||||
},
|
||||
"wan2.2-kf2v-flash": {
|
||||
"480P": 1,
|
||||
"720P": 2,
|
||||
"1080P": 4.8,
|
||||
},
|
||||
"wan2.2-i2v-flash": {
|
||||
"480P": 1,
|
||||
"720P": 2,
|
||||
},
|
||||
"wan2.2-s2v": {
|
||||
"480P": 1,
|
||||
"720P": 0.9 / 0.5,
|
||||
},
|
||||
}
|
||||
var resolution string
|
||||
|
||||
// size match
|
||||
if aliReq.Parameters.Size != "" {
|
||||
toResolution, err := sizeToResolution(aliReq.Parameters.Size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolution = toResolution
|
||||
} else {
|
||||
resolution = strings.ToUpper(aliReq.Parameters.Resolution)
|
||||
if !strings.HasSuffix(resolution, "P") {
|
||||
resolution = resolution + "P"
|
||||
}
|
||||
}
|
||||
if otherRatio, ok := aliRatios[aliReq.Model]; ok {
|
||||
if ratio, ok := otherRatio[resolution]; ok {
|
||||
otherRatios[fmt.Sprintf("resolution-%s", resolution)] = ratio
|
||||
}
|
||||
}
|
||||
return otherRatios, nil
|
||||
}
|
||||
|
||||
func isWan27I2VModel(model string) bool {
|
||||
return strings.HasPrefix(model, "wan2.7-i2v")
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstTaskImage(req relaycommon.TaskSubmitReq) string {
|
||||
if image := strings.TrimSpace(req.Image); image != "" {
|
||||
return image
|
||||
}
|
||||
for _, image := range req.Images {
|
||||
if trimmed := strings.TrimSpace(image); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
if inputReference := strings.TrimSpace(req.InputReference); inputReference != "" {
|
||||
return inputReference
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func secondTaskImage(req relaycommon.TaskSubmitReq) string {
|
||||
nonEmptyImages := 0
|
||||
for _, image := range req.Images {
|
||||
trimmed := strings.TrimSpace(image)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
nonEmptyImages++
|
||||
if nonEmptyImages == 2 {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func normalizeWan27I2VInput(aliReq *AliVideoRequest, req relaycommon.TaskSubmitReq) error {
|
||||
if !isWan27I2VModel(aliReq.Model) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(aliReq.Input.Media) == 0 {
|
||||
firstFrameURL := firstNonEmpty(aliReq.Input.FirstFrameURL, aliReq.Input.ImgURL, firstTaskImage(req))
|
||||
lastFrameURL := firstNonEmpty(aliReq.Input.LastFrameURL, secondTaskImage(req))
|
||||
audioURL := aliReq.Input.AudioURL
|
||||
|
||||
if firstFrameURL != "" {
|
||||
aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{
|
||||
Type: "first_frame",
|
||||
URL: firstFrameURL,
|
||||
})
|
||||
}
|
||||
if lastFrameURL != "" {
|
||||
aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{
|
||||
Type: "last_frame",
|
||||
URL: lastFrameURL,
|
||||
})
|
||||
}
|
||||
if audioURL != "" {
|
||||
aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{
|
||||
Type: "driving_audio",
|
||||
URL: audioURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if len(aliReq.Input.Media) == 0 {
|
||||
return fmt.Errorf("wan2.7-i2v requires image, images, input_reference, or input.media")
|
||||
}
|
||||
|
||||
// Wan2.7 image-to-video uses the new input.media protocol. Avoid sending
|
||||
// legacy fields that belong to wan2.6 and earlier image-to-video APIs.
|
||||
aliReq.Input.ImgURL = ""
|
||||
aliReq.Input.FirstFrameURL = ""
|
||||
aliReq.Input.LastFrameURL = ""
|
||||
aliReq.Input.AudioURL = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relaycommon.TaskSubmitReq) (*AliVideoRequest, error) {
|
||||
upstreamModel := req.Model
|
||||
if info.IsModelMapped {
|
||||
upstreamModel = info.UpstreamModelName
|
||||
}
|
||||
aliReq := &AliVideoRequest{
|
||||
Model: upstreamModel,
|
||||
Input: AliVideoInput{
|
||||
Prompt: req.Prompt,
|
||||
ImgURL: firstTaskImage(req),
|
||||
},
|
||||
Parameters: &AliVideoParameters{
|
||||
PromptExtend: true, // 默认开启智能改写
|
||||
Watermark: false,
|
||||
},
|
||||
}
|
||||
|
||||
// 处理分辨率映射
|
||||
if req.Size != "" {
|
||||
// text to video size must be contained *
|
||||
if strings.Contains(req.Model, "t2v") && !strings.Contains(req.Size, "*") {
|
||||
return nil, fmt.Errorf("invalid size: %s, example: %s", req.Size, "1920*1080")
|
||||
}
|
||||
if strings.Contains(req.Size, "*") {
|
||||
aliReq.Parameters.Size = req.Size
|
||||
} else {
|
||||
resolution := strings.ToUpper(req.Size)
|
||||
// 支持 480p, 720p, 1080p 或 480P, 720P, 1080P
|
||||
if !strings.HasSuffix(resolution, "P") {
|
||||
resolution = resolution + "P"
|
||||
}
|
||||
aliReq.Parameters.Resolution = resolution
|
||||
}
|
||||
} else {
|
||||
// 根据模型设置默认分辨率
|
||||
if strings.Contains(req.Model, "t2v") { // image to video
|
||||
if strings.HasPrefix(req.Model, "wan2.5") {
|
||||
aliReq.Parameters.Size = "1920*1080"
|
||||
} else if strings.HasPrefix(req.Model, "wan2.2") {
|
||||
aliReq.Parameters.Size = "1920*1080"
|
||||
} else {
|
||||
aliReq.Parameters.Size = "1280*720"
|
||||
}
|
||||
} else {
|
||||
if strings.HasPrefix(req.Model, "wan2.6") {
|
||||
aliReq.Parameters.Resolution = "1080P"
|
||||
} else if strings.HasPrefix(req.Model, "wan2.5") {
|
||||
aliReq.Parameters.Resolution = "1080P"
|
||||
} else if strings.HasPrefix(req.Model, "wan2.2-i2v-flash") {
|
||||
aliReq.Parameters.Resolution = "720P"
|
||||
} else if strings.HasPrefix(req.Model, "wan2.2-i2v-plus") {
|
||||
aliReq.Parameters.Resolution = "1080P"
|
||||
} else {
|
||||
aliReq.Parameters.Resolution = "720P"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 处理时长
|
||||
if req.Duration > 0 {
|
||||
aliReq.Parameters.Duration = req.Duration
|
||||
} else if req.Seconds != "" {
|
||||
seconds, err := strconv.Atoi(req.Seconds)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "convert seconds to int failed")
|
||||
} else {
|
||||
aliReq.Parameters.Duration = seconds
|
||||
}
|
||||
}
|
||||
if aliReq.Parameters.Duration <= 0 {
|
||||
aliReq.Parameters.Duration = 5 // 默认5秒
|
||||
}
|
||||
|
||||
// 从 metadata 中提取额外参数
|
||||
if req.Metadata != nil {
|
||||
if metadataBytes, err := common.Marshal(req.Metadata); err == nil {
|
||||
err = common.Unmarshal(metadataBytes, aliReq)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal metadata failed")
|
||||
}
|
||||
} else {
|
||||
return nil, errors.Wrap(err, "marshal metadata failed")
|
||||
}
|
||||
}
|
||||
|
||||
if aliReq.Model != upstreamModel {
|
||||
return nil, errors.New("can't change model with metadata")
|
||||
}
|
||||
|
||||
if err := normalizeWan27I2VInput(aliReq, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return aliReq, nil
|
||||
}
|
||||
|
||||
// EstimateBilling 根据用户请求参数计算 OtherRatios（时长、分辨率等）。
|
||||
// 在 ValidateRequestAndSetAction 之后、价格计算之前调用。
|
||||
func (a *TaskAdaptor) EstimateBilling(c *gin.Context, info *relaycommon.RelayInfo) map[string]float64 {
|
||||
taskReq, err := relaycommon.GetTaskRequest(c)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
aliReq, err := a.convertToAliRequest(info, taskReq)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// metadata can override Duration past standard request validation;
|
||||
// cap it because it is used as a billing multiplier.
|
||||
otherRatios := map[string]float64{
|
||||
"seconds": float64(min(aliReq.Parameters.Duration, relaycommon.MaxTaskDurationSeconds)),
|
||||
}
|
||||
ratios, err := ProcessAliOtherRatios(aliReq)
|
||||
if err != nil {
|
||||
return otherRatios
|
||||
}
|
||||
for k, v := range ratios {
|
||||
otherRatios[k] = v
|
||||
}
|
||||
return otherRatios
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
// 解析阿里响应
|
||||
var aliResp AliVideoResponse
|
||||
if err := common.Unmarshal(responseBody, &aliResp); err != nil {
|
||||
taskErr = service.TaskErrorWrapper(errors.Wrapf(err, "body: %s", responseBody), "unmarshal_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// 检查错误
|
||||
if aliResp.Code != "" {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("%s: %s", aliResp.Code, aliResp.Message), "ali_api_error", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
if aliResp.Output.TaskID == "" {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("task_id is empty"), "invalid_response", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// 转换为 OpenAI 格式响应
|
||||
openAIResp := dto.NewOpenAIVideo()
|
||||
openAIResp.ID = info.PublicTaskID
|
||||
openAIResp.TaskID = info.PublicTaskID
|
||||
openAIResp.Model = c.GetString("model")
|
||||
if openAIResp.Model == "" && info != nil {
|
||||
openAIResp.Model = info.OriginModelName
|
||||
}
|
||||
openAIResp.Status = convertAliStatus(aliResp.Output.TaskStatus)
|
||||
openAIResp.CreatedAt = common.GetTimestamp()
|
||||
|
||||
// 返回 OpenAI 格式
|
||||
c.JSON(http.StatusOK, openAIResp)
|
||||
|
||||
return aliResp.Output.TaskID, responseBody, nil
|
||||
}
|
||||
|
||||
// FetchTask 查询任务状态
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
uri := fmt.Sprintf("%s/api/v1/tasks/%s", baseUrl, taskID)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, uri, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return ModelList
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return ChannelName
|
||||
}
|
||||
|
||||
// ParseTaskResult 解析任务结果
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
var aliResp AliVideoResponse
|
||||
if err := common.Unmarshal(respBody, &aliResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal task result failed")
|
||||
}
|
||||
|
||||
taskResult := relaycommon.TaskInfo{
|
||||
Code: 0,
|
||||
}
|
||||
|
||||
// 状态映射
|
||||
switch aliResp.Output.TaskStatus {
|
||||
case "PENDING":
|
||||
taskResult.Status = model.TaskStatusQueued
|
||||
case "RUNNING":
|
||||
taskResult.Status = model.TaskStatusInProgress
|
||||
case "SUCCEEDED":
|
||||
taskResult.Status = model.TaskStatusSuccess
|
||||
// 阿里直接返回视频URL，不需要额外的代理端点
|
||||
taskResult.Url = aliResp.Output.VideoURL
|
||||
case "FAILED", "CANCELED", "UNKNOWN":
|
||||
taskResult.Status = model.TaskStatusFailure
|
||||
if aliResp.Message != "" {
|
||||
taskResult.Reason = aliResp.Message
|
||||
} else if aliResp.Output.Message != "" {
|
||||
taskResult.Reason = fmt.Sprintf("task failed, code: %s , message: %s", aliResp.Output.Code, aliResp.Output.Message)
|
||||
} else {
|
||||
taskResult.Reason = "task failed"
|
||||
}
|
||||
default:
|
||||
taskResult.Status = model.TaskStatusQueued
|
||||
}
|
||||
|
||||
return &taskResult, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) {
|
||||
var aliResp AliVideoResponse
|
||||
if err := common.Unmarshal(task.Data, &aliResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal ali response failed")
|
||||
}
|
||||
|
||||
openAIResp := dto.NewOpenAIVideo()
|
||||
openAIResp.ID = task.TaskID
|
||||
openAIResp.Status = convertAliStatus(aliResp.Output.TaskStatus)
|
||||
openAIResp.Model = task.Properties.OriginModelName
|
||||
openAIResp.SetProgressStr(task.Progress)
|
||||
openAIResp.CreatedAt = task.CreatedAt
|
||||
openAIResp.CompletedAt = task.UpdatedAt
|
||||
|
||||
// 设置视频URL（核心字段）
|
||||
openAIResp.SetMetadata("url", aliResp.Output.VideoURL)
|
||||
|
||||
// 错误处理
|
||||
if aliResp.Code != "" {
|
||||
openAIResp.Error = &dto.OpenAIVideoError{
|
||||
Code: aliResp.Code,
|
||||
Message: aliResp.Message,
|
||||
}
|
||||
} else if aliResp.Output.Code != "" {
|
||||
openAIResp.Error = &dto.OpenAIVideoError{
|
||||
Code: aliResp.Output.Code,
|
||||
Message: aliResp.Output.Message,
|
||||
}
|
||||
}
|
||||
|
||||
return common.Marshal(openAIResp)
|
||||
}
|
||||
|
||||
func convertAliStatus(aliStatus string) string {
|
||||
switch aliStatus {
|
||||
case "PENDING":
|
||||
return dto.VideoStatusQueued
|
||||
case "RUNNING":
|
||||
return dto.VideoStatusInProgress
|
||||
case "SUCCEEDED":
|
||||
return dto.VideoStatusCompleted
|
||||
case "FAILED", "CANCELED", "UNKNOWN":
|
||||
return dto.VideoStatusFailed
|
||||
default:
|
||||
return dto.VideoStatusUnknown
|
||||
}
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
package ali
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testRelayInfo() *relaycommon.RelayInfo {
|
||||
return &relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{},
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertToAliRequestWan27I2VBuildsMediaFromImage(t *testing.T) {
|
||||
adaptor := &TaskAdaptor{}
|
||||
req := relaycommon.TaskSubmitReq{
|
||||
Model: "wan2.7-i2v",
|
||||
Prompt: "animate the first frame",
|
||||
Image: "https://example.com/first.png",
|
||||
Size: "720p",
|
||||
Duration: 10,
|
||||
}
|
||||
|
||||
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "wan2.7-i2v", aliReq.Model)
|
||||
require.Equal(t, "720P", aliReq.Parameters.Resolution)
|
||||
require.Equal(t, 10, aliReq.Parameters.Duration)
|
||||
require.Equal(t, []AliVideoMedia{
|
||||
{Type: "first_frame", URL: "https://example.com/first.png"},
|
||||
}, aliReq.Input.Media)
|
||||
require.Empty(t, aliReq.Input.ImgURL)
|
||||
|
||||
body, err := common.Marshal(aliReq)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(body), `"media"`)
|
||||
require.NotContains(t, string(body), `"img_url"`)
|
||||
}
|
||||
|
||||
func TestConvertToAliRequestWan27I2VBuildsFirstAndLastFrameFromImages(t *testing.T) {
|
||||
adaptor := &TaskAdaptor{}
|
||||
req := relaycommon.TaskSubmitReq{
|
||||
Model: "wan2.7-i2v",
|
||||
Prompt: "interpolate between frames",
|
||||
Images: []string{
|
||||
"https://example.com/first.png",
|
||||
"https://example.com/last.png",
|
||||
},
|
||||
}
|
||||
|
||||
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []AliVideoMedia{
|
||||
{Type: "first_frame", URL: "https://example.com/first.png"},
|
||||
{Type: "last_frame", URL: "https://example.com/last.png"},
|
||||
}, aliReq.Input.Media)
|
||||
}
|
||||
|
||||
func TestConvertToAliRequestWan27I2VPrefersImageBeforeImagesAndInputReference(t *testing.T) {
|
||||
adaptor := &TaskAdaptor{}
|
||||
req := relaycommon.TaskSubmitReq{
|
||||
Model: "wan2.7-i2v",
|
||||
Prompt: "use the direct image",
|
||||
Image: " https://example.com/direct.png ",
|
||||
Images: []string{"https://example.com/images-first.png", " https://example.com/images-last.png "},
|
||||
InputReference: "https://example.com/input-reference.png",
|
||||
}
|
||||
|
||||
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []AliVideoMedia{
|
||||
{Type: "first_frame", URL: "https://example.com/direct.png"},
|
||||
{Type: "last_frame", URL: "https://example.com/images-last.png"},
|
||||
}, aliReq.Input.Media)
|
||||
}
|
||||
|
||||
func TestConvertToAliRequestWan27I2VFallsBackToFirstNonEmptyImage(t *testing.T) {
|
||||
adaptor := &TaskAdaptor{}
|
||||
req := relaycommon.TaskSubmitReq{
|
||||
Model: "wan2.7-i2v",
|
||||
Prompt: "skip blank images",
|
||||
Image: " ",
|
||||
Images: []string{
|
||||
" ",
|
||||
" https://example.com/first.png ",
|
||||
" https://example.com/last.png ",
|
||||
},
|
||||
InputReference: "https://example.com/input-reference.png",
|
||||
}
|
||||
|
||||
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []AliVideoMedia{
|
||||
{Type: "first_frame", URL: "https://example.com/first.png"},
|
||||
{Type: "last_frame", URL: "https://example.com/last.png"},
|
||||
}, aliReq.Input.Media)
|
||||
}
|
||||
|
||||
func TestConvertToAliRequestWan27I2VKeepsExplicitMetadataMedia(t *testing.T) {
|
||||
adaptor := &TaskAdaptor{}
|
||||
req := relaycommon.TaskSubmitReq{
|
||||
Model: "wan2.7-i2v",
|
||||
Prompt: "continue the clip",
|
||||
Image: "https://example.com/direct.png",
|
||||
Images: []string{"https://example.com/images-first.png", "https://example.com/images-last.png"},
|
||||
InputReference: "https://example.com/input-reference.png",
|
||||
Metadata: map[string]interface{}{
|
||||
"input": map[string]interface{}{
|
||||
"media": []interface{}{
|
||||
map[string]interface{}{
|
||||
"type": "first_clip",
|
||||
"url": "https://example.com/input.mp4",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []AliVideoMedia{
|
||||
{Type: "first_clip", URL: "https://example.com/input.mp4"},
|
||||
}, aliReq.Input.Media)
|
||||
require.Empty(t, aliReq.Input.ImgURL)
|
||||
|
||||
body, err := common.Marshal(aliReq)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(body), `"media"`)
|
||||
require.NotContains(t, string(body), `"img_url"`)
|
||||
}
|
||||
|
||||
func TestConvertToAliRequestWan27I2VRequiresMedia(t *testing.T) {
|
||||
adaptor := &TaskAdaptor{}
|
||||
req := relaycommon.TaskSubmitReq{
|
||||
Model: "wan2.7-i2v",
|
||||
Prompt: "animate without a frame",
|
||||
}
|
||||
|
||||
_, err := adaptor.convertToAliRequest(testRelayInfo(), req)
|
||||
|
||||
require.Error(t, err)
|
||||
require.True(t, strings.Contains(err.Error(), "requires image"))
|
||||
}
|
||||
|
||||
func TestConvertToAliRequestWan25I2VKeepsLegacyImgURL(t *testing.T) {
|
||||
adaptor := &TaskAdaptor{}
|
||||
req := relaycommon.TaskSubmitReq{
|
||||
Model: "wan2.5-i2v-preview",
|
||||
Prompt: "animate the first frame",
|
||||
Image: "https://example.com/first.png",
|
||||
}
|
||||
|
||||
aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "https://example.com/first.png", aliReq.Input.ImgURL)
|
||||
require.Empty(t, aliReq.Input.Media)
|
||||
|
||||
body, err := common.Marshal(aliReq)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(body), `"img_url"`)
|
||||
require.NotContains(t, string(body), `"media"`)
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package ali
|
||||
|
||||
var ModelList = []string{
|
||||
"wan2.7-i2v", // 万相2.7图生视频（新input.media协议）
|
||||
"wan2.7-t2v", // 万相2.7文生视频
|
||||
"wan2.5-i2v-preview", // 万相2.5 preview（有声视频）推荐
|
||||
"wan2.2-i2v-flash", // 万相2.2极速版（无声视频）
|
||||
"wan2.2-i2v-plus", // 万相2.2专业版（无声视频）
|
||||
"wanx2.1-i2v-plus", // 万相2.1专业版（无声视频）
|
||||
"wanx2.1-i2v-turbo", // 万相2.1极速版（无声视频）
|
||||
}
|
||||
|
||||
var ChannelName = "ali"
|
||||
@@ -1,372 +0,0 @@
|
||||
package doubao
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
"github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/samber/lo"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
type ContentItem struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ImageURL *MediaURL `json:"image_url,omitempty"`
|
||||
VideoURL *MediaURL `json:"video_url,omitempty"`
|
||||
AudioURL *MediaURL `json:"audio_url,omitempty"`
|
||||
Role string `json:"role,omitempty"`
|
||||
}
|
||||
|
||||
type MediaURL struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
type requestPayload struct {
|
||||
Model string `json:"model"`
|
||||
Content []ContentItem `json:"content,omitempty"`
|
||||
CallbackURL string `json:"callback_url,omitempty"`
|
||||
ReturnLastFrame *dto.BoolValue `json:"return_last_frame,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ExecutionExpiresAfter *dto.IntValue `json:"execution_expires_after,omitempty"`
|
||||
GenerateAudio *dto.BoolValue `json:"generate_audio,omitempty"`
|
||||
Draft *dto.BoolValue `json:"draft,omitempty"`
|
||||
Tools []struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
} `json:"tools,omitempty"`
|
||||
SafetyIdentifier string `json:"safety_identifier,omitempty"`
|
||||
Priority *dto.IntValue `json:"priority,omitempty"`
|
||||
Resolution string `json:"resolution,omitempty"`
|
||||
Ratio string `json:"ratio,omitempty"`
|
||||
Duration *dto.IntValue `json:"duration,omitempty"`
|
||||
Frames *dto.IntValue `json:"frames,omitempty"`
|
||||
Seed *dto.IntValue `json:"seed,omitempty"`
|
||||
CameraFixed *dto.BoolValue `json:"camera_fixed,omitempty"`
|
||||
Watermark *dto.BoolValue `json:"watermark,omitempty"`
|
||||
}
|
||||
|
||||
type responsePayload struct {
|
||||
ID string `json:"id"` // task_id
|
||||
}
|
||||
|
||||
type responseTask struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Status string `json:"status"`
|
||||
Content struct {
|
||||
VideoURL string `json:"video_url"`
|
||||
} `json:"content"`
|
||||
Seed int `json:"seed"`
|
||||
Resolution string `json:"resolution"`
|
||||
Duration int `json:"duration"`
|
||||
Ratio string `json:"ratio"`
|
||||
FramesPerSecond int `json:"framespersecond"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
Tools []struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"tools"`
|
||||
Usage struct {
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
ToolUsage struct {
|
||||
WebSearch int `json:"web_search"`
|
||||
} `json:"tool_usage"`
|
||||
} `json:"usage"`
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
}
|
||||
|
||||
// ValidateRequestAndSetAction parses body, validates fields and sets default action.
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *taskdto.TaskError) {
|
||||
// Accept only POST /v1/video/generations as "generate" action.
|
||||
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionGenerate)
|
||||
}
|
||||
|
||||
// BuildRequestURL constructs the upstream URL.
|
||||
func (a *TaskAdaptor) BuildRequestURL(_ *relaycommon.RelayInfo) (string, error) {
|
||||
return fmt.Sprintf("%s/api/v3/contents/generations/tasks", a.baseURL), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers.
|
||||
func (a *TaskAdaptor) BuildRequestHeader(_ *gin.Context, req *http.Request, _ *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+a.apiKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
// EstimateBilling 根据请求 metadata 中的输出分辨率与是否包含视频输入，返回相对基准价的计费 OtherRatio。
|
||||
func (a *TaskAdaptor) EstimateBilling(c *gin.Context, info *relaycommon.RelayInfo) map[string]float64 {
|
||||
req, err := relaycommon.GetTaskRequest(c)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
hasVideo := hasVideoInMetadata(req.Metadata)
|
||||
resolution, _ := req.Metadata["resolution"].(string)
|
||||
ratio, ok := GetVideoInputRatio(info.OriginModelName, resolution, hasVideo)
|
||||
if !ok || ratio == 1.0 {
|
||||
return nil
|
||||
}
|
||||
return map[string]float64{"video_input": ratio}
|
||||
}
|
||||
|
||||
// hasVideoInMetadata 直接检查 metadata 的 content 数组是否包含 video_url 条目，
|
||||
// 避免构建完整的上游 requestPayload。
|
||||
func hasVideoInMetadata(metadata map[string]interface{}) bool {
|
||||
if metadata == nil {
|
||||
return false
|
||||
}
|
||||
contentRaw, ok := metadata["content"]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
contentSlice, ok := contentRaw.([]interface{})
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
for _, item := range contentSlice {
|
||||
itemMap, ok := item.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if itemMap["type"] == "video_url" {
|
||||
return true
|
||||
}
|
||||
if _, has := itemMap["video_url"]; has {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BuildRequestBody converts request into Doubao specific format.
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
req, err := relaycommon.GetTaskRequest(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
body, err := a.convertToRequestPayload(&req)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "convert request payload failed")
|
||||
}
|
||||
if info.IsModelMapped {
|
||||
body.Model = info.UpstreamModelName
|
||||
} else {
|
||||
info.UpstreamModelName = body.Model
|
||||
}
|
||||
data, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper.
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response, returns taskID etc.
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
// Parse Doubao response
|
||||
var dResp responsePayload
|
||||
if err := common.Unmarshal(responseBody, &dResp); err != nil {
|
||||
taskErr = service.TaskErrorWrapper(errors.Wrapf(err, "body: %s", responseBody), "unmarshal_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if dResp.ID == "" {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("task_id is empty"), "invalid_response", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
ov := dto.NewOpenAIVideo()
|
||||
ov.ID = info.PublicTaskID
|
||||
ov.TaskID = info.PublicTaskID
|
||||
ov.CreatedAt = time.Now().Unix()
|
||||
ov.Model = info.OriginModelName
|
||||
|
||||
c.JSON(http.StatusOK, ov)
|
||||
return dResp.ID, responseBody, nil
|
||||
}
|
||||
|
||||
// FetchTask fetch task status
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
uri := fmt.Sprintf("%s/api/v3/contents/generations/tasks/%s", baseUrl, taskID)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, uri, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return ModelList
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return ChannelName
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) convertToRequestPayload(req *relaycommon.TaskSubmitReq) (*requestPayload, error) {
|
||||
r := requestPayload{
|
||||
Model: req.Model,
|
||||
Content: []ContentItem{},
|
||||
}
|
||||
|
||||
// Add images if present
|
||||
if req.HasImage() {
|
||||
for _, imgURL := range req.Images {
|
||||
r.Content = append(r.Content, ContentItem{
|
||||
Type: "image_url",
|
||||
ImageURL: &MediaURL{
|
||||
URL: imgURL,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
metadata := req.Metadata
|
||||
if err := taskcommon.UnmarshalMetadata(metadata, &r); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal metadata failed")
|
||||
}
|
||||
|
||||
if sec, _ := strconv.Atoi(req.Seconds); sec > 0 {
|
||||
r.Duration = lo.ToPtr(dto.IntValue(sec))
|
||||
}
|
||||
|
||||
r.Content = lo.Reject(r.Content, func(c ContentItem, _ int) bool { return c.Type == "text" })
|
||||
r.Content = append(r.Content, ContentItem{
|
||||
Type: "text",
|
||||
Text: req.Prompt,
|
||||
})
|
||||
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
resTask := responseTask{}
|
||||
if err := common.Unmarshal(respBody, &resTask); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal task result failed")
|
||||
}
|
||||
|
||||
taskResult := relaycommon.TaskInfo{
|
||||
Code: 0,
|
||||
}
|
||||
|
||||
// Map Doubao status to internal status
|
||||
switch resTask.Status {
|
||||
case "pending", "queued":
|
||||
taskResult.Status = model.TaskStatusQueued
|
||||
taskResult.Progress = "10%"
|
||||
case "processing", "running":
|
||||
taskResult.Status = model.TaskStatusInProgress
|
||||
taskResult.Progress = "50%"
|
||||
case "succeeded":
|
||||
taskResult.Status = model.TaskStatusSuccess
|
||||
taskResult.Progress = "100%"
|
||||
taskResult.Url = resTask.Content.VideoURL
|
||||
// 解析 usage 信息用于按倍率计费
|
||||
taskResult.CompletionTokens = resTask.Usage.CompletionTokens
|
||||
taskResult.TotalTokens = resTask.Usage.TotalTokens
|
||||
case "failed":
|
||||
taskResult.Status = model.TaskStatusFailure
|
||||
taskResult.Progress = "100%"
|
||||
taskResult.Reason = resTask.Error.Message
|
||||
default:
|
||||
// Unknown status, treat as processing
|
||||
taskResult.Status = model.TaskStatusInProgress
|
||||
taskResult.Progress = "30%"
|
||||
}
|
||||
|
||||
return &taskResult, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) ([]byte, error) {
|
||||
var dResp responseTask
|
||||
if err := common.Unmarshal(originTask.Data, &dResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal doubao task data failed")
|
||||
}
|
||||
|
||||
openAIVideo := dto.NewOpenAIVideo()
|
||||
openAIVideo.ID = originTask.TaskID
|
||||
openAIVideo.TaskID = originTask.TaskID
|
||||
openAIVideo.Status = originTask.Status.ToVideoStatus()
|
||||
openAIVideo.SetProgressStr(originTask.Progress)
|
||||
openAIVideo.SetMetadata("url", dResp.Content.VideoURL)
|
||||
openAIVideo.CreatedAt = originTask.CreatedAt
|
||||
openAIVideo.CompletedAt = originTask.UpdatedAt
|
||||
openAIVideo.Model = originTask.Properties.OriginModelName
|
||||
|
||||
if dResp.Status == "failed" {
|
||||
openAIVideo.Error = &dto.OpenAIVideoError{
|
||||
Message: dResp.Error.Message,
|
||||
Code: dResp.Error.Code,
|
||||
}
|
||||
}
|
||||
|
||||
return common.Marshal(openAIVideo)
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package doubao
|
||||
|
||||
import "strings"
|
||||
|
||||
var ModelList = []string{
|
||||
"doubao-seedance-1-0-pro-250528",
|
||||
"doubao-seedance-1-0-lite-t2v",
|
||||
"doubao-seedance-1-0-lite-i2v",
|
||||
"doubao-seedance-1-5-pro-251215",
|
||||
"doubao-seedance-2-0-260128",
|
||||
"doubao-seedance-2-0-fast-260128",
|
||||
}
|
||||
|
||||
var ChannelName = "doubao-video"
|
||||
|
||||
// videoPriceKey 价格表的键：输出分辨率档（is1080p/is4k 均为 false 即 480p/720p 基准档）、输入是否含视频。
|
||||
type videoPriceKey struct {
|
||||
is1080p bool
|
||||
is4k bool
|
||||
hasVideo bool
|
||||
}
|
||||
|
||||
// videoPriceTable 各模型在不同 (输出分辨率档, 是否含视频输入) 下的单价（元/百万 token）。
|
||||
// 其中零值键 {480p/720p, 不含视频} 为基准价，等于管理员应配置的 ModelRatio；
|
||||
// 计费时取 实际单价/基准价 作为 OtherRatio。
|
||||
var videoPriceTable = map[string]map[videoPriceKey]float64{
|
||||
"doubao-seedance-2-0-260128": {
|
||||
{hasVideo: false}: 46.0,
|
||||
{hasVideo: true}: 28.0,
|
||||
{is1080p: true, hasVideo: false}: 51.0,
|
||||
{is1080p: true, hasVideo: true}: 31.0,
|
||||
{is4k: true, hasVideo: false}: 26.0,
|
||||
{is4k: true, hasVideo: true}: 16.0,
|
||||
},
|
||||
"doubao-seedance-2-0-fast-260128": {
|
||||
{hasVideo: false}: 37.0,
|
||||
{hasVideo: true}: 22.0,
|
||||
},
|
||||
}
|
||||
|
||||
// GetVideoInputRatio 返回指定模型在给定输出分辨率/是否含视频输入下，相对基准价的计费倍率。
|
||||
// 第二个返回值表示该模型是否配置了价格表；倍率为 1.0 时调用方可忽略该 OtherRatio。
|
||||
func GetVideoInputRatio(modelName, resolution string, hasVideo bool) (float64, bool) {
|
||||
prices, ok := videoPriceTable[modelName]
|
||||
base := prices[videoPriceKey{}] // 零值键 = {480p/720p, 不含视频} 基准价
|
||||
if !ok || base <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
res := strings.ToLower(strings.TrimSpace(resolution))
|
||||
price, ok := prices[videoPriceKey{is1080p: res == "1080p", is4k: res == "4k", hasVideo: hasVideo}]
|
||||
if !ok {
|
||||
// 未配置的组合（如 fast 无 1080p/4k，上游会自行报错）按基准价计费即可。
|
||||
return 1.0, true
|
||||
}
|
||||
return price / base, true
|
||||
}
|
||||
@@ -1,293 +0,0 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
"github.com/QuantumNous/new-api/setting/model_setting"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
}
|
||||
|
||||
// ValidateRequestAndSetAction parses body, validates fields and sets default action.
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *taskdto.TaskError) {
|
||||
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionTextGenerate)
|
||||
}
|
||||
|
||||
// BuildRequestURL constructs the Gemini API predictLongRunning endpoint for Veo.
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
modelName := info.UpstreamModelName
|
||||
version := model_setting.GetGeminiVersionSetting(modelName)
|
||||
|
||||
return fmt.Sprintf(
|
||||
"%s/%s/models/%s:predictLongRunning",
|
||||
a.baseURL,
|
||||
version,
|
||||
modelName,
|
||||
), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers.
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("x-goog-api-key", a.apiKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildRequestBody converts request into the Veo predictLongRunning format.
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
v, ok := c.Get("task_request")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("request not found in context")
|
||||
}
|
||||
req, ok := v.(relaycommon.TaskSubmitReq)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected task_request type")
|
||||
}
|
||||
|
||||
instance := VeoInstance{Prompt: req.Prompt}
|
||||
if img := ExtractMultipartImage(c, info); img != nil {
|
||||
instance.Image = img
|
||||
} else if len(req.Images) > 0 {
|
||||
if parsed := ParseImageInput(req.Images[0]); parsed != nil {
|
||||
instance.Image = parsed
|
||||
info.Action = constant.TaskActionGenerate
|
||||
}
|
||||
}
|
||||
|
||||
params := &VeoParameters{}
|
||||
if err := taskcommon.UnmarshalMetadata(req.Metadata, params); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal metadata failed")
|
||||
}
|
||||
if params.DurationSeconds == 0 && req.Duration > 0 {
|
||||
params.DurationSeconds = req.Duration
|
||||
}
|
||||
if params.Resolution == "" && req.Size != "" {
|
||||
params.Resolution = SizeToVeoResolution(req.Size)
|
||||
}
|
||||
if params.AspectRatio == "" && req.Size != "" {
|
||||
params.AspectRatio = SizeToVeoAspectRatio(req.Size)
|
||||
}
|
||||
params.Resolution = strings.ToLower(params.Resolution)
|
||||
params.SampleCount = 1
|
||||
|
||||
body := VeoRequestPayload{
|
||||
Instances: []VeoInstance{instance},
|
||||
Parameters: params,
|
||||
}
|
||||
|
||||
data, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper.
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response, returns taskID etc.
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", nil, service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
var s submitResponse
|
||||
if err := common.Unmarshal(responseBody, &s); err != nil {
|
||||
return "", nil, service.TaskErrorWrapper(err, "unmarshal_response_failed", http.StatusInternalServerError)
|
||||
}
|
||||
if strings.TrimSpace(s.Name) == "" {
|
||||
return "", nil, service.TaskErrorWrapper(fmt.Errorf("missing operation name"), "invalid_response", http.StatusInternalServerError)
|
||||
}
|
||||
taskID = taskcommon.EncodeLocalTaskID(s.Name)
|
||||
ov := dto.NewOpenAIVideo()
|
||||
ov.ID = info.PublicTaskID
|
||||
ov.TaskID = info.PublicTaskID
|
||||
ov.CreatedAt = time.Now().Unix()
|
||||
ov.Model = info.OriginModelName
|
||||
c.JSON(http.StatusOK, ov)
|
||||
return taskID, responseBody, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return []string{
|
||||
"veo-3.0-generate-001",
|
||||
"veo-3.0-fast-generate-001",
|
||||
"veo-3.1-generate-preview",
|
||||
"veo-3.1-fast-generate-preview",
|
||||
}
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return "gemini"
|
||||
}
|
||||
|
||||
// EstimateBilling returns OtherRatios based on durationSeconds and resolution.
|
||||
func (a *TaskAdaptor) EstimateBilling(c *gin.Context, info *relaycommon.RelayInfo) map[string]float64 {
|
||||
v, ok := c.Get("task_request")
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
req, ok := v.(relaycommon.TaskSubmitReq)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
seconds := ResolveVeoDuration(req.Metadata, req.Duration, req.Seconds)
|
||||
resolution := ResolveVeoResolution(req.Metadata, req.Size)
|
||||
resRatio := VeoResolutionRatio(info.UpstreamModelName, resolution)
|
||||
|
||||
return map[string]float64{
|
||||
"seconds": float64(seconds),
|
||||
"resolution": resRatio,
|
||||
}
|
||||
}
|
||||
|
||||
// FetchTask polls task status via the Gemini operations GET endpoint.
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
upstreamName, err := taskcommon.DecodeLocalTaskID(taskID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode task_id failed: %w", err)
|
||||
}
|
||||
|
||||
version := model_setting.GetGeminiVersionSetting("default")
|
||||
url := fmt.Sprintf("%s/%s/%s", baseUrl, version, upstreamName)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("x-goog-api-key", key)
|
||||
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
var op operationResponse
|
||||
if err := common.Unmarshal(respBody, &op); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal operation response failed: %w", err)
|
||||
}
|
||||
|
||||
ti := &relaycommon.TaskInfo{}
|
||||
|
||||
if op.Error.Message != "" {
|
||||
ti.Status = model.TaskStatusFailure
|
||||
ti.Reason = op.Error.Message
|
||||
ti.Progress = "100%"
|
||||
return ti, nil
|
||||
}
|
||||
|
||||
if !op.Done {
|
||||
ti.Status = model.TaskStatusInProgress
|
||||
ti.Progress = "50%"
|
||||
return ti, nil
|
||||
}
|
||||
|
||||
ti.Status = model.TaskStatusSuccess
|
||||
ti.Progress = "100%"
|
||||
|
||||
ti.TaskID = taskcommon.EncodeLocalTaskID(op.Name)
|
||||
|
||||
if len(op.Response.GenerateVideoResponse.GeneratedVideos) > 0 {
|
||||
if uri := op.Response.GenerateVideoResponse.GeneratedVideos[0].Video.URI; uri != "" {
|
||||
ti.RemoteUrl = uri
|
||||
}
|
||||
}
|
||||
|
||||
return ti, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) {
|
||||
upstreamTaskID := task.GetUpstreamTaskID()
|
||||
upstreamName, err := taskcommon.DecodeLocalTaskID(upstreamTaskID)
|
||||
if err != nil {
|
||||
upstreamName = ""
|
||||
}
|
||||
modelName := extractModelFromOperationName(upstreamName)
|
||||
if strings.TrimSpace(modelName) == "" {
|
||||
modelName = "veo-3.0-generate-001"
|
||||
}
|
||||
|
||||
video := dto.NewOpenAIVideo()
|
||||
video.ID = task.TaskID
|
||||
video.Model = modelName
|
||||
video.Status = task.Status.ToVideoStatus()
|
||||
video.SetProgressStr(task.Progress)
|
||||
video.CreatedAt = task.CreatedAt
|
||||
if task.FinishTime > 0 {
|
||||
video.CompletedAt = task.FinishTime
|
||||
} else if task.UpdatedAt > 0 {
|
||||
video.CompletedAt = task.UpdatedAt
|
||||
}
|
||||
|
||||
return common.Marshal(video)
|
||||
}
|
||||
|
||||
// ============================
|
||||
// helpers
|
||||
// ============================
|
||||
|
||||
var modelRe = regexp.MustCompile(`models/([^/]+)/operations/`)
|
||||
|
||||
func extractModelFromOperationName(name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
if m := modelRe.FindStringSubmatch(name); len(m) == 2 {
|
||||
return m[1]
|
||||
}
|
||||
if idx := strings.Index(name, "models/"); idx >= 0 {
|
||||
s := name[idx+len("models/"):]
|
||||
if p := strings.Index(s, "/operations/"); p > 0 {
|
||||
return s[:p]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
)
|
||||
|
||||
// ParseVeoDurationSeconds extracts durationSeconds from metadata.
|
||||
// Returns 8 (Veo default) when not specified or invalid.
|
||||
func ParseVeoDurationSeconds(metadata map[string]any) int {
|
||||
if metadata == nil {
|
||||
return 8
|
||||
}
|
||||
v, ok := metadata["durationSeconds"]
|
||||
if !ok {
|
||||
return 8
|
||||
}
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
if int(n) > 0 {
|
||||
return int(n)
|
||||
}
|
||||
case int:
|
||||
if n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 8
|
||||
}
|
||||
|
||||
// ParseVeoResolution extracts resolution from metadata.
|
||||
// Returns "720p" when not specified.
|
||||
func ParseVeoResolution(metadata map[string]any) string {
|
||||
if metadata == nil {
|
||||
return "720p"
|
||||
}
|
||||
v, ok := metadata["resolution"]
|
||||
if !ok {
|
||||
return "720p"
|
||||
}
|
||||
if s, ok := v.(string); ok && s != "" {
|
||||
return strings.ToLower(s)
|
||||
}
|
||||
return "720p"
|
||||
}
|
||||
|
||||
// ResolveVeoDuration returns the effective duration in seconds.
|
||||
// Priority: metadata["durationSeconds"] > stdDuration > stdSeconds > default (8).
|
||||
// The result is capped because it is used as a billing multiplier and the
|
||||
// metadata path bypasses standard request validation.
|
||||
func ResolveVeoDuration(metadata map[string]any, stdDuration int, stdSeconds string) int {
|
||||
if metadata != nil {
|
||||
if _, exists := metadata["durationSeconds"]; exists {
|
||||
if d := ParseVeoDurationSeconds(metadata); d > 0 {
|
||||
return min(d, relaycommon.MaxTaskDurationSeconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
if stdDuration > 0 {
|
||||
return min(stdDuration, relaycommon.MaxTaskDurationSeconds)
|
||||
}
|
||||
if s, err := strconv.Atoi(stdSeconds); err == nil && s > 0 {
|
||||
return min(s, relaycommon.MaxTaskDurationSeconds)
|
||||
}
|
||||
return 8
|
||||
}
|
||||
|
||||
// ResolveVeoResolution returns the effective resolution string (lowercase).
|
||||
// Priority: metadata["resolution"] > SizeToVeoResolution(stdSize) > default ("720p").
|
||||
func ResolveVeoResolution(metadata map[string]any, stdSize string) string {
|
||||
if metadata != nil {
|
||||
if _, exists := metadata["resolution"]; exists {
|
||||
if r := ParseVeoResolution(metadata); r != "" {
|
||||
return r
|
||||
}
|
||||
}
|
||||
}
|
||||
if stdSize != "" {
|
||||
return SizeToVeoResolution(stdSize)
|
||||
}
|
||||
return "720p"
|
||||
}
|
||||
|
||||
// SizeToVeoResolution converts a "WxH" size string to a Veo resolution label.
|
||||
func SizeToVeoResolution(size string) string {
|
||||
parts := strings.SplitN(strings.ToLower(size), "x", 2)
|
||||
if len(parts) != 2 {
|
||||
return "720p"
|
||||
}
|
||||
w, _ := strconv.Atoi(parts[0])
|
||||
h, _ := strconv.Atoi(parts[1])
|
||||
maxDim := w
|
||||
if h > maxDim {
|
||||
maxDim = h
|
||||
}
|
||||
if maxDim >= 3840 {
|
||||
return "4k"
|
||||
}
|
||||
if maxDim >= 1920 {
|
||||
return "1080p"
|
||||
}
|
||||
return "720p"
|
||||
}
|
||||
|
||||
// SizeToVeoAspectRatio converts a "WxH" size string to a Veo aspect ratio.
|
||||
func SizeToVeoAspectRatio(size string) string {
|
||||
parts := strings.SplitN(strings.ToLower(size), "x", 2)
|
||||
if len(parts) != 2 {
|
||||
return "16:9"
|
||||
}
|
||||
w, _ := strconv.Atoi(parts[0])
|
||||
h, _ := strconv.Atoi(parts[1])
|
||||
if w <= 0 || h <= 0 {
|
||||
return "16:9"
|
||||
}
|
||||
if h > w {
|
||||
return "9:16"
|
||||
}
|
||||
return "16:9"
|
||||
}
|
||||
|
||||
// VeoResolutionRatio returns the pricing multiplier for the given resolution.
|
||||
// Standard resolutions (720p, 1080p) return 1.0.
|
||||
// 4K returns a model-specific multiplier based on Google's official pricing.
|
||||
func VeoResolutionRatio(modelName, resolution string) float64 {
|
||||
if resolution != "4k" {
|
||||
return 1.0
|
||||
}
|
||||
// 4K multipliers derived from Vertex AI official pricing (video+audio base):
|
||||
// veo-3.1-generate: $0.60 / $0.40 = 1.5
|
||||
// veo-3.1-fast-generate: $0.35 / $0.15 ≈ 2.333
|
||||
// Veo 3.0 models do not support 4K; return 1.0 as fallback.
|
||||
if strings.Contains(modelName, "3.1-fast-generate") {
|
||||
return 2.333333
|
||||
}
|
||||
if strings.Contains(modelName, "3.1-generate") || strings.Contains(modelName, "3.1") {
|
||||
return 1.5
|
||||
}
|
||||
return 1.0
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package gemini
|
||||
|
||||
// VeoImageInput represents an image input for Veo image-to-video.
|
||||
// Used by both Gemini and Vertex adaptors.
|
||||
type VeoImageInput struct {
|
||||
BytesBase64Encoded string `json:"bytesBase64Encoded"`
|
||||
MimeType string `json:"mimeType"`
|
||||
}
|
||||
|
||||
// VeoInstance represents a single instance in the Veo predictLongRunning request.
|
||||
type VeoInstance struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Image *VeoImageInput `json:"image,omitempty"`
|
||||
// TODO: support referenceImages (style/asset references, up to 3 images)
|
||||
// TODO: support lastFrame (first+last frame interpolation, Veo 3.1)
|
||||
}
|
||||
|
||||
// VeoParameters represents the parameters block for Veo predictLongRunning.
|
||||
type VeoParameters struct {
|
||||
SampleCount int `json:"sampleCount"`
|
||||
DurationSeconds int `json:"durationSeconds,omitempty"`
|
||||
AspectRatio string `json:"aspectRatio,omitempty"`
|
||||
Resolution string `json:"resolution,omitempty"`
|
||||
NegativePrompt string `json:"negativePrompt,omitempty"`
|
||||
PersonGeneration string `json:"personGeneration,omitempty"`
|
||||
StorageUri string `json:"storageUri,omitempty"`
|
||||
CompressionQuality string `json:"compressionQuality,omitempty"`
|
||||
ResizeMode string `json:"resizeMode,omitempty"`
|
||||
Seed *int `json:"seed,omitempty"`
|
||||
GenerateAudio *bool `json:"generateAudio,omitempty"`
|
||||
}
|
||||
|
||||
// VeoRequestPayload is the top-level request body for the Veo
|
||||
// predictLongRunning endpoint (used by both Gemini and Vertex).
|
||||
type VeoRequestPayload struct {
|
||||
Instances []VeoInstance `json:"instances"`
|
||||
Parameters *VeoParameters `json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
type submitResponse struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type operationVideo struct {
|
||||
MimeType string `json:"mimeType"`
|
||||
BytesBase64Encoded string `json:"bytesBase64Encoded"`
|
||||
Encoding string `json:"encoding"`
|
||||
}
|
||||
|
||||
type operationResponse struct {
|
||||
Name string `json:"name"`
|
||||
Done bool `json:"done"`
|
||||
Response struct {
|
||||
Type string `json:"@type"`
|
||||
RaiMediaFilteredCount int `json:"raiMediaFilteredCount"`
|
||||
Videos []operationVideo `json:"videos"`
|
||||
BytesBase64Encoded string `json:"bytesBase64Encoded"`
|
||||
Encoding string `json:"encoding"`
|
||||
Video string `json:"video"`
|
||||
GenerateVideoResponse struct {
|
||||
GeneratedVideos []struct {
|
||||
Video struct {
|
||||
URI string `json:"uri"`
|
||||
} `json:"video"`
|
||||
} `json:"generatedVideos"`
|
||||
} `json:"generateVideoResponse"`
|
||||
} `json:"response"`
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const maxVeoImageSize = 20 * 1024 * 1024 // 20 MB
|
||||
|
||||
// ExtractMultipartImage reads the first `input_reference` file from a multipart
|
||||
// form upload and returns a VeoImageInput. Returns nil if no file is present.
|
||||
func ExtractMultipartImage(c *gin.Context, info *relaycommon.RelayInfo) *VeoImageInput {
|
||||
mf, err := c.MultipartForm()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
files, exists := mf.File["input_reference"]
|
||||
if !exists || len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
fh := files[0]
|
||||
if fh.Size > maxVeoImageSize {
|
||||
return nil
|
||||
}
|
||||
file, err := fh.Open()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
fileBytes, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
mimeType := fh.Header.Get("Content-Type")
|
||||
if mimeType == "" || mimeType == "application/octet-stream" {
|
||||
mimeType = http.DetectContentType(fileBytes)
|
||||
}
|
||||
|
||||
info.Action = constant.TaskActionGenerate
|
||||
return &VeoImageInput{
|
||||
BytesBase64Encoded: base64.StdEncoding.EncodeToString(fileBytes),
|
||||
MimeType: mimeType,
|
||||
}
|
||||
}
|
||||
|
||||
// ParseImageInput parses an image string (data URI or raw base64) into a
|
||||
// VeoImageInput. Returns nil if the input is empty or invalid.
|
||||
// TODO: support downloading HTTP URL images and converting to base64
|
||||
func ParseImageInput(imageStr string) *VeoImageInput {
|
||||
imageStr = strings.TrimSpace(imageStr)
|
||||
if imageStr == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(imageStr, "data:") {
|
||||
return parseDataURI(imageStr)
|
||||
}
|
||||
|
||||
raw, err := base64.StdEncoding.DecodeString(imageStr)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return &VeoImageInput{
|
||||
BytesBase64Encoded: imageStr,
|
||||
MimeType: http.DetectContentType(raw),
|
||||
}
|
||||
}
|
||||
|
||||
func parseDataURI(uri string) *VeoImageInput {
|
||||
// data:image/png;base64,iVBOR...
|
||||
rest := uri[len("data:"):]
|
||||
idx := strings.Index(rest, ",")
|
||||
if idx < 0 {
|
||||
return nil
|
||||
}
|
||||
meta := rest[:idx]
|
||||
b64 := rest[idx+1:]
|
||||
if b64 == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
mimeType := "application/octet-stream"
|
||||
parts := strings.SplitN(meta, ";", 2)
|
||||
if len(parts) >= 1 && parts[0] != "" {
|
||||
mimeType = parts[0]
|
||||
}
|
||||
|
||||
return &VeoImageInput{
|
||||
BytesBase64Encoded: b64,
|
||||
MimeType: mimeType,
|
||||
}
|
||||
}
|
||||
@@ -1,303 +0,0 @@
|
||||
package hailuo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
)
|
||||
|
||||
// https://platform.minimaxi.com/docs/api-reference/video-generation-intro
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *taskdto.TaskError) {
|
||||
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionGenerate)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
return fmt.Sprintf("%s%s", a.baseURL, TextToVideoEndpoint), nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+a.apiKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
v, exists := c.Get("task_request")
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("request not found in context")
|
||||
}
|
||||
req, ok := v.(relaycommon.TaskSubmitReq)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid request type in context")
|
||||
}
|
||||
|
||||
body, err := a.convertToRequestPayload(&req, info)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "convert request payload failed")
|
||||
}
|
||||
|
||||
data, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
var hResp VideoResponse
|
||||
if err := common.Unmarshal(responseBody, &hResp); err != nil {
|
||||
taskErr = service.TaskErrorWrapper(errors.Wrapf(err, "body: %s", responseBody), "unmarshal_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if hResp.BaseResp.StatusCode != StatusSuccess {
|
||||
taskErr = service.TaskErrorWrapper(
|
||||
fmt.Errorf("hailuo api error: %s", hResp.BaseResp.StatusMsg),
|
||||
strconv.Itoa(hResp.BaseResp.StatusCode),
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
ov := dto.NewOpenAIVideo()
|
||||
ov.ID = info.PublicTaskID
|
||||
ov.TaskID = info.PublicTaskID
|
||||
ov.CreatedAt = time.Now().Unix()
|
||||
ov.Model = info.OriginModelName
|
||||
|
||||
c.JSON(http.StatusOK, ov)
|
||||
return hResp.TaskID, responseBody, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
uri := fmt.Sprintf("%s%s?task_id=%s", baseUrl, QueryTaskEndpoint, taskID)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, uri, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return ModelList
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return ChannelName
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) convertToRequestPayload(req *relaycommon.TaskSubmitReq, info *relaycommon.RelayInfo) (*VideoRequest, error) {
|
||||
modelConfig := GetModelConfig(info.UpstreamModelName)
|
||||
duration := DefaultDuration
|
||||
if req.Duration > 0 {
|
||||
duration = req.Duration
|
||||
}
|
||||
resolution := modelConfig.DefaultResolution
|
||||
if req.Size != "" {
|
||||
resolution = a.parseResolutionFromSize(req.Size, modelConfig)
|
||||
}
|
||||
|
||||
videoRequest := &VideoRequest{
|
||||
Model: info.UpstreamModelName,
|
||||
Prompt: req.Prompt,
|
||||
Duration: &duration,
|
||||
Resolution: resolution,
|
||||
}
|
||||
if err := req.UnmarshalMetadata(&videoRequest); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal metadata to video request failed")
|
||||
}
|
||||
|
||||
return videoRequest, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) parseResolutionFromSize(size string, modelConfig ModelConfig) string {
|
||||
switch {
|
||||
case strings.Contains(size, "1080"):
|
||||
return Resolution1080P
|
||||
case strings.Contains(size, "768"):
|
||||
return Resolution768P
|
||||
case strings.Contains(size, "720"):
|
||||
return Resolution720P
|
||||
case strings.Contains(size, "512"):
|
||||
return Resolution512P
|
||||
default:
|
||||
return modelConfig.DefaultResolution
|
||||
}
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
resTask := QueryTaskResponse{}
|
||||
if err := common.Unmarshal(respBody, &resTask); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal task result failed")
|
||||
}
|
||||
|
||||
taskResult := relaycommon.TaskInfo{}
|
||||
|
||||
if resTask.BaseResp.StatusCode == StatusSuccess {
|
||||
taskResult.Code = 0
|
||||
} else {
|
||||
taskResult.Code = resTask.BaseResp.StatusCode
|
||||
taskResult.Reason = resTask.BaseResp.StatusMsg
|
||||
taskResult.Status = model.TaskStatusFailure
|
||||
taskResult.Progress = "100%"
|
||||
}
|
||||
|
||||
switch resTask.Status {
|
||||
case TaskStatusPreparing, TaskStatusQueueing, TaskStatusProcessing:
|
||||
taskResult.Status = model.TaskStatusInProgress
|
||||
taskResult.Progress = "30%"
|
||||
if resTask.Status == TaskStatusProcessing {
|
||||
taskResult.Progress = "50%"
|
||||
}
|
||||
case TaskStatusSuccess:
|
||||
taskResult.Status = model.TaskStatusSuccess
|
||||
taskResult.Progress = "100%"
|
||||
taskResult.Url = a.buildVideoURL(resTask.TaskID, resTask.FileID)
|
||||
case TaskStatusFailed:
|
||||
taskResult.Status = model.TaskStatusFailure
|
||||
taskResult.Progress = "100%"
|
||||
if taskResult.Reason == "" {
|
||||
taskResult.Reason = "task failed"
|
||||
}
|
||||
default:
|
||||
taskResult.Status = model.TaskStatusInProgress
|
||||
taskResult.Progress = "30%"
|
||||
}
|
||||
|
||||
return &taskResult, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) ([]byte, error) {
|
||||
var hailuoResp QueryTaskResponse
|
||||
if err := common.Unmarshal(originTask.Data, &hailuoResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal hailuo task data failed")
|
||||
}
|
||||
|
||||
openAIVideo := originTask.ToOpenAIVideo()
|
||||
if hailuoResp.BaseResp.StatusCode != StatusSuccess {
|
||||
openAIVideo.Error = &dto.OpenAIVideoError{
|
||||
Message: hailuoResp.BaseResp.StatusMsg,
|
||||
Code: strconv.Itoa(hailuoResp.BaseResp.StatusCode),
|
||||
}
|
||||
}
|
||||
|
||||
jsonData, err := common.Marshal(openAIVideo)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "marshal openai video failed")
|
||||
}
|
||||
|
||||
return jsonData, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) buildVideoURL(_, fileID string) string {
|
||||
if a.apiKey == "" || a.baseURL == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/v1/files/retrieve?file_id=%s", a.baseURL, fileID)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+a.apiKey)
|
||||
|
||||
resp, err := service.GetHttpClient().Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
var retrieveResp RetrieveFileResponse
|
||||
if err := common.Unmarshal(responseBody, &retrieveResp); err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
if retrieveResp.BaseResp.StatusCode != StatusSuccess {
|
||||
return ""
|
||||
}
|
||||
|
||||
return retrieveResp.File.DownloadURL
|
||||
}
|
||||
|
||||
func contains(slice []string, item string) bool {
|
||||
for _, s := range slice {
|
||||
if s == item {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsInt(slice []int, item int) bool {
|
||||
for _, s := range slice {
|
||||
if s == item {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package hailuo
|
||||
|
||||
const (
|
||||
ChannelName = "hailuo-video"
|
||||
)
|
||||
|
||||
var ModelList = []string{
|
||||
"MiniMax-Hailuo-2.3",
|
||||
"MiniMax-Hailuo-2.3-Fast",
|
||||
"MiniMax-Hailuo-02",
|
||||
"T2V-01-Director",
|
||||
"T2V-01",
|
||||
"I2V-01-Director",
|
||||
"I2V-01-live",
|
||||
"I2V-01",
|
||||
"S2V-01",
|
||||
}
|
||||
|
||||
const (
|
||||
TextToVideoEndpoint = "/v1/video_generation"
|
||||
QueryTaskEndpoint = "/v1/query/video_generation"
|
||||
)
|
||||
|
||||
const (
|
||||
StatusSuccess = 0
|
||||
StatusRateLimit = 1002
|
||||
StatusAuthFailed = 1004
|
||||
StatusNoBalance = 1008
|
||||
StatusSensitive = 1026
|
||||
StatusParamError = 2013
|
||||
StatusInvalidKey = 2049
|
||||
)
|
||||
|
||||
const (
|
||||
TaskStatusPreparing = "Preparing"
|
||||
TaskStatusQueueing = "Queueing"
|
||||
TaskStatusProcessing = "Processing"
|
||||
TaskStatusSuccess = "Success"
|
||||
TaskStatusFailed = "Fail"
|
||||
)
|
||||
|
||||
const (
|
||||
Resolution512P = "512P"
|
||||
Resolution720P = "720P"
|
||||
Resolution768P = "768P"
|
||||
Resolution1080P = "1080P"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultDuration = 6
|
||||
DefaultResolution = Resolution720P
|
||||
)
|
||||
@@ -1,170 +0,0 @@
|
||||
package hailuo
|
||||
|
||||
type SubjectReference struct {
|
||||
Type string `json:"type"` // Subject type, currently only supports "character"
|
||||
Image []string `json:"image"` // Array of subject reference images (currently only supports single image)
|
||||
}
|
||||
|
||||
type VideoRequest struct {
|
||||
Model string `json:"model"`
|
||||
Prompt string `json:"prompt,omitempty"`
|
||||
PromptOptimizer *bool `json:"prompt_optimizer,omitempty"`
|
||||
FastPretreatment *bool `json:"fast_pretreatment,omitempty"`
|
||||
Duration *int `json:"duration,omitempty"`
|
||||
Resolution string `json:"resolution,omitempty"`
|
||||
CallbackURL string `json:"callback_url,omitempty"`
|
||||
AigcWatermark *bool `json:"aigc_watermark,omitempty"`
|
||||
FirstFrameImage string `json:"first_frame_image,omitempty"` // For image-to-video and start-end-to-video
|
||||
LastFrameImage string `json:"last_frame_image,omitempty"` // For start-end-to-video
|
||||
SubjectReference []SubjectReference `json:"subject_reference,omitempty"` // For subject-reference-to-video
|
||||
}
|
||||
|
||||
type VideoResponse struct {
|
||||
TaskID string `json:"task_id"`
|
||||
BaseResp BaseResp `json:"base_resp"`
|
||||
}
|
||||
|
||||
type BaseResp struct {
|
||||
StatusCode int `json:"status_code"`
|
||||
StatusMsg string `json:"status_msg"`
|
||||
}
|
||||
|
||||
type QueryTaskRequest struct {
|
||||
TaskID string `json:"task_id"`
|
||||
}
|
||||
|
||||
type QueryTaskResponse struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status string `json:"status"`
|
||||
FileID string `json:"file_id,omitempty"`
|
||||
VideoWidth int `json:"video_width,omitempty"`
|
||||
VideoHeight int `json:"video_height,omitempty"`
|
||||
BaseResp BaseResp `json:"base_resp"`
|
||||
}
|
||||
|
||||
type ErrorInfo struct {
|
||||
StatusCode int `json:"status_code"`
|
||||
StatusMsg string `json:"status_msg"`
|
||||
}
|
||||
|
||||
type TaskStatusInfo struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status string `json:"status"`
|
||||
FileID string `json:"file_id,omitempty"`
|
||||
VideoURL string `json:"video_url,omitempty"`
|
||||
ErrorCode int `json:"error_code,omitempty"`
|
||||
ErrorMsg string `json:"error_msg,omitempty"`
|
||||
}
|
||||
|
||||
type ModelConfig struct {
|
||||
Name string
|
||||
DefaultResolution string
|
||||
SupportedDurations []int
|
||||
SupportedResolutions []string
|
||||
HasPromptOptimizer bool
|
||||
HasFastPretreatment bool
|
||||
}
|
||||
|
||||
type RetrieveFileResponse struct {
|
||||
File FileObject `json:"file"`
|
||||
BaseResp BaseResp `json:"base_resp"`
|
||||
}
|
||||
|
||||
type FileObject struct {
|
||||
FileID int64 `json:"file_id"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
Filename string `json:"filename"`
|
||||
Purpose string `json:"purpose"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
}
|
||||
|
||||
func GetModelConfig(model string) ModelConfig {
|
||||
configs := map[string]ModelConfig{
|
||||
"MiniMax-Hailuo-2.3": {
|
||||
Name: "MiniMax-Hailuo-2.3",
|
||||
DefaultResolution: Resolution768P,
|
||||
SupportedDurations: []int{6, 10},
|
||||
SupportedResolutions: []string{Resolution768P, Resolution1080P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: true,
|
||||
},
|
||||
"MiniMax-Hailuo-2.3-Fast": {
|
||||
Name: "MiniMax-Hailuo-2.3-Fast",
|
||||
DefaultResolution: Resolution768P,
|
||||
SupportedDurations: []int{6, 10},
|
||||
SupportedResolutions: []string{Resolution768P, Resolution1080P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: true,
|
||||
},
|
||||
"MiniMax-Hailuo-02": {
|
||||
Name: "MiniMax-Hailuo-02",
|
||||
DefaultResolution: Resolution768P,
|
||||
SupportedDurations: []int{6, 10},
|
||||
SupportedResolutions: []string{Resolution512P, Resolution768P, Resolution1080P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: true,
|
||||
},
|
||||
"T2V-01-Director": {
|
||||
Name: "T2V-01-Director",
|
||||
DefaultResolution: Resolution768P,
|
||||
SupportedDurations: []int{6},
|
||||
SupportedResolutions: []string{Resolution768P, Resolution1080P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: false,
|
||||
},
|
||||
"T2V-01": {
|
||||
Name: "T2V-01",
|
||||
DefaultResolution: Resolution720P,
|
||||
SupportedDurations: []int{6},
|
||||
SupportedResolutions: []string{Resolution720P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: false,
|
||||
},
|
||||
"I2V-01-Director": {
|
||||
Name: "I2V-01-Director",
|
||||
DefaultResolution: Resolution720P,
|
||||
SupportedDurations: []int{6},
|
||||
SupportedResolutions: []string{Resolution720P, Resolution1080P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: false,
|
||||
},
|
||||
"I2V-01-live": {
|
||||
Name: "I2V-01-live",
|
||||
DefaultResolution: Resolution720P,
|
||||
SupportedDurations: []int{6},
|
||||
SupportedResolutions: []string{Resolution720P, Resolution1080P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: false,
|
||||
},
|
||||
"I2V-01": {
|
||||
Name: "I2V-01",
|
||||
DefaultResolution: Resolution720P,
|
||||
SupportedDurations: []int{6},
|
||||
SupportedResolutions: []string{Resolution720P, Resolution1080P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: false,
|
||||
},
|
||||
"S2V-01": {
|
||||
Name: "S2V-01",
|
||||
DefaultResolution: Resolution720P,
|
||||
SupportedDurations: []int{6},
|
||||
SupportedResolutions: []string{Resolution720P},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: false,
|
||||
},
|
||||
}
|
||||
|
||||
if config, exists := configs[model]; exists {
|
||||
return config
|
||||
}
|
||||
|
||||
return ModelConfig{
|
||||
Name: model,
|
||||
DefaultResolution: DefaultResolution,
|
||||
SupportedDurations: []int{6},
|
||||
SupportedResolutions: []string{DefaultResolution},
|
||||
HasPromptOptimizer: true,
|
||||
HasFastPretreatment: false,
|
||||
}
|
||||
}
|
||||
@@ -1,481 +0,0 @@
|
||||
package jimeng
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/samber/lo"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
type requestPayload struct {
|
||||
ReqKey string `json:"req_key"`
|
||||
BinaryDataBase64 []string `json:"binary_data_base64,omitempty"`
|
||||
ImageUrls []string `json:"image_urls,omitempty"`
|
||||
Prompt string `json:"prompt,omitempty"`
|
||||
Seed int64 `json:"seed"`
|
||||
AspectRatio string `json:"aspect_ratio"`
|
||||
Frames int `json:"frames,omitempty"`
|
||||
}
|
||||
|
||||
type responsePayload struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
RequestId string `json:"request_id"`
|
||||
Data struct {
|
||||
TaskID string `json:"task_id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type responseTask struct {
|
||||
Code int `json:"code"`
|
||||
Data struct {
|
||||
BinaryDataBase64 []interface{} `json:"binary_data_base64"`
|
||||
ImageUrls interface{} `json:"image_urls"`
|
||||
RespData string `json:"resp_data"`
|
||||
Status string `json:"status"`
|
||||
VideoUrl string `json:"video_url"`
|
||||
} `json:"data"`
|
||||
Message string `json:"message"`
|
||||
RequestId string `json:"request_id"`
|
||||
Status int `json:"status"`
|
||||
TimeElapsed string `json:"time_elapsed"`
|
||||
}
|
||||
|
||||
const (
|
||||
// 即梦限制单个文件最大4.7MB https://www.volcengine.com/docs/85621/1747301
|
||||
MaxFileSize int64 = 4*1024*1024 + 700*1024 // 4.7MB (4MB + 724KB)
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
accessKey string
|
||||
secretKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
|
||||
// apiKey format: "access_key|secret_key"
|
||||
keyParts := strings.Split(info.ApiKey, "|")
|
||||
if len(keyParts) == 2 {
|
||||
a.accessKey = strings.TrimSpace(keyParts[0])
|
||||
a.secretKey = strings.TrimSpace(keyParts[1])
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateRequestAndSetAction parses body, validates fields and sets default action.
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *taskdto.TaskError) {
|
||||
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionGenerate)
|
||||
}
|
||||
|
||||
// BuildRequestURL constructs the upstream URL.
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
if isNewAPIRelay(info.ApiKey) {
|
||||
return fmt.Sprintf("%s/jimeng/?Action=CVSync2AsyncSubmitTask&Version=2022-08-31", a.baseURL), nil
|
||||
}
|
||||
return fmt.Sprintf("%s/?Action=CVSync2AsyncSubmitTask&Version=2022-08-31", a.baseURL), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers.
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if isNewAPIRelay(info.ApiKey) {
|
||||
req.Header.Set("Authorization", "Bearer "+info.ApiKey)
|
||||
} else {
|
||||
return a.signRequest(req, a.accessKey, a.secretKey)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
v, exists := c.Get("task_request")
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("request not found in context")
|
||||
}
|
||||
req, ok := v.(relaycommon.TaskSubmitReq)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid request type in context")
|
||||
}
|
||||
// 支持openai sdk的图片上传方式
|
||||
if mf, err := c.MultipartForm(); err == nil {
|
||||
if files, exists := mf.File["input_reference"]; exists && len(files) > 0 {
|
||||
if len(files) == 1 {
|
||||
info.Action = constant.TaskActionGenerate
|
||||
} else if len(files) > 1 {
|
||||
info.Action = constant.TaskActionFirstTailGenerate
|
||||
}
|
||||
|
||||
// 将上传的文件转换为base64格式
|
||||
var images []string
|
||||
|
||||
for _, fileHeader := range files {
|
||||
// 检查文件大小
|
||||
if fileHeader.Size > MaxFileSize {
|
||||
return nil, fmt.Errorf("文件 %s 大小超过限制，最大允许 %d MB", fileHeader.Filename, MaxFileSize/(1024*1024))
|
||||
}
|
||||
|
||||
file, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fileBytes, err := io.ReadAll(file)
|
||||
file.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// 将文件内容转换为base64
|
||||
base64Str := base64.StdEncoding.EncodeToString(fileBytes)
|
||||
images = append(images, base64Str)
|
||||
}
|
||||
req.Images = images
|
||||
}
|
||||
}
|
||||
|
||||
body, err := a.convertToRequestPayload(&req, info)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "convert request payload failed")
|
||||
}
|
||||
data, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper.
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response, returns taskID etc.
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
// Parse Jimeng response
|
||||
var jResp responsePayload
|
||||
if err := common.Unmarshal(responseBody, &jResp); err != nil {
|
||||
taskErr = service.TaskErrorWrapper(errors.Wrapf(err, "body: %s", responseBody), "unmarshal_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if jResp.Code != 10000 {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("%s", jResp.Message), fmt.Sprintf("%d", jResp.Code), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
ov := dto.NewOpenAIVideo()
|
||||
ov.ID = info.PublicTaskID
|
||||
ov.TaskID = info.PublicTaskID
|
||||
ov.CreatedAt = time.Now().Unix()
|
||||
ov.Model = info.OriginModelName
|
||||
c.JSON(http.StatusOK, ov)
|
||||
return jResp.Data.TaskID, responseBody, nil
|
||||
}
|
||||
|
||||
// FetchTask fetch task status
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
uri := fmt.Sprintf("%s/?Action=CVSync2AsyncGetResult&Version=2022-08-31", baseUrl)
|
||||
if isNewAPIRelay(key) {
|
||||
uri = fmt.Sprintf("%s/jimeng/?Action=CVSync2AsyncGetResult&Version=2022-08-31", a.baseURL)
|
||||
}
|
||||
payload := map[string]string{
|
||||
"req_key": "jimeng_vgfm_t2v_l20", // This is fixed value from doc: https://www.volcengine.com/docs/85621/1544774
|
||||
"task_id": taskID,
|
||||
}
|
||||
payloadBytes, err := common.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "marshal fetch task payload failed")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, uri, bytes.NewBuffer(payloadBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
if isNewAPIRelay(key) {
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
} else {
|
||||
keyParts := strings.Split(key, "|")
|
||||
if len(keyParts) != 2 {
|
||||
return nil, fmt.Errorf("invalid api key format for jimeng: expected 'ak|sk'")
|
||||
}
|
||||
accessKey := strings.TrimSpace(keyParts[0])
|
||||
secretKey := strings.TrimSpace(keyParts[1])
|
||||
|
||||
if err := a.signRequest(req, accessKey, secretKey); err != nil {
|
||||
return nil, errors.Wrap(err, "sign request failed")
|
||||
}
|
||||
}
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return []string{"jimeng_vgfm_t2v_l20"}
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return "jimeng"
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) signRequest(req *http.Request, accessKey, secretKey string) error {
|
||||
var bodyBytes []byte
|
||||
var err error
|
||||
|
||||
if req.Body != nil {
|
||||
bodyBytes, err = io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "read request body failed")
|
||||
}
|
||||
_ = req.Body.Close()
|
||||
req.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) // Rewind
|
||||
} else {
|
||||
bodyBytes = []byte{}
|
||||
}
|
||||
|
||||
payloadHash := sha256.Sum256(bodyBytes)
|
||||
hexPayloadHash := hex.EncodeToString(payloadHash[:])
|
||||
|
||||
t := time.Now().UTC()
|
||||
xDate := t.Format("20060102T150405Z")
|
||||
shortDate := t.Format("20060102")
|
||||
|
||||
req.Header.Set("Host", req.URL.Host)
|
||||
req.Header.Set("X-Date", xDate)
|
||||
req.Header.Set("X-Content-Sha256", hexPayloadHash)
|
||||
|
||||
// Sort and encode query parameters to create canonical query string
|
||||
queryParams := req.URL.Query()
|
||||
sortedKeys := make([]string, 0, len(queryParams))
|
||||
for k := range queryParams {
|
||||
sortedKeys = append(sortedKeys, k)
|
||||
}
|
||||
sort.Strings(sortedKeys)
|
||||
var queryParts []string
|
||||
for _, k := range sortedKeys {
|
||||
values := queryParams[k]
|
||||
sort.Strings(values)
|
||||
for _, v := range values {
|
||||
queryParts = append(queryParts, fmt.Sprintf("%s=%s", url.QueryEscape(k), url.QueryEscape(v)))
|
||||
}
|
||||
}
|
||||
canonicalQueryString := strings.Join(queryParts, "&")
|
||||
|
||||
headersToSign := map[string]string{
|
||||
"host": req.URL.Host,
|
||||
"x-date": xDate,
|
||||
"x-content-sha256": hexPayloadHash,
|
||||
}
|
||||
if req.Header.Get("Content-Type") != "" {
|
||||
headersToSign["content-type"] = req.Header.Get("Content-Type")
|
||||
}
|
||||
|
||||
var signedHeaderKeys []string
|
||||
for k := range headersToSign {
|
||||
signedHeaderKeys = append(signedHeaderKeys, k)
|
||||
}
|
||||
sort.Strings(signedHeaderKeys)
|
||||
|
||||
var canonicalHeaders strings.Builder
|
||||
for _, k := range signedHeaderKeys {
|
||||
canonicalHeaders.WriteString(k)
|
||||
canonicalHeaders.WriteString(":")
|
||||
canonicalHeaders.WriteString(strings.TrimSpace(headersToSign[k]))
|
||||
canonicalHeaders.WriteString("\n")
|
||||
}
|
||||
signedHeaders := strings.Join(signedHeaderKeys, ";")
|
||||
|
||||
canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s",
|
||||
req.Method,
|
||||
req.URL.Path,
|
||||
canonicalQueryString,
|
||||
canonicalHeaders.String(),
|
||||
signedHeaders,
|
||||
hexPayloadHash,
|
||||
)
|
||||
|
||||
hashedCanonicalRequest := sha256.Sum256([]byte(canonicalRequest))
|
||||
hexHashedCanonicalRequest := hex.EncodeToString(hashedCanonicalRequest[:])
|
||||
|
||||
region := "cn-north-1"
|
||||
serviceName := "cv"
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/request", shortDate, region, serviceName)
|
||||
stringToSign := fmt.Sprintf("HMAC-SHA256\n%s\n%s\n%s",
|
||||
xDate,
|
||||
credentialScope,
|
||||
hexHashedCanonicalRequest,
|
||||
)
|
||||
|
||||
kDate := hmacSHA256([]byte(secretKey), []byte(shortDate))
|
||||
kRegion := hmacSHA256(kDate, []byte(region))
|
||||
kService := hmacSHA256(kRegion, []byte(serviceName))
|
||||
kSigning := hmacSHA256(kService, []byte("request"))
|
||||
signature := hex.EncodeToString(hmacSHA256(kSigning, []byte(stringToSign)))
|
||||
|
||||
authorization := fmt.Sprintf("HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
|
||||
accessKey,
|
||||
credentialScope,
|
||||
signedHeaders,
|
||||
signature,
|
||||
)
|
||||
req.Header.Set("Authorization", authorization)
|
||||
return nil
|
||||
}
|
||||
|
||||
func hmacSHA256(key []byte, data []byte) []byte {
|
||||
h := hmac.New(sha256.New, key)
|
||||
h.Write(data)
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) convertToRequestPayload(req *relaycommon.TaskSubmitReq, info *relaycommon.RelayInfo) (*requestPayload, error) {
|
||||
r := requestPayload{
|
||||
ReqKey: info.UpstreamModelName,
|
||||
Prompt: req.Prompt,
|
||||
}
|
||||
|
||||
switch req.Duration {
|
||||
case 10:
|
||||
r.Frames = 241 // 24*10+1 = 241
|
||||
default:
|
||||
r.Frames = 121 // 24*5+1 = 121
|
||||
}
|
||||
|
||||
// Handle one-of image_urls or binary_data_base64
|
||||
if req.HasImage() {
|
||||
if strings.HasPrefix(req.Images[0], "http") {
|
||||
r.ImageUrls = req.Images
|
||||
} else {
|
||||
r.BinaryDataBase64 = req.Images
|
||||
}
|
||||
}
|
||||
if err := taskcommon.UnmarshalMetadata(req.Metadata, &r); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal metadata failed")
|
||||
}
|
||||
|
||||
// 即梦视频3.0 ReqKey转换
|
||||
// https://www.volcengine.com/docs/85621/1792707
|
||||
imageLen := lo.Max([]int{len(req.Images), len(r.BinaryDataBase64), len(r.ImageUrls)})
|
||||
if strings.Contains(r.ReqKey, "jimeng_v30") {
|
||||
if r.ReqKey == "jimeng_v30_pro" {
|
||||
// 3.0 pro只有固定的jimeng_ti2v_v30_pro
|
||||
r.ReqKey = "jimeng_ti2v_v30_pro"
|
||||
} else if imageLen > 1 {
|
||||
// 多张图片：首尾帧生成
|
||||
r.ReqKey = strings.TrimSuffix(strings.Replace(r.ReqKey, "jimeng_v30", "jimeng_i2v_first_tail_v30", 1), "p")
|
||||
} else if imageLen == 1 {
|
||||
// 单张图片：图生视频
|
||||
r.ReqKey = strings.TrimSuffix(strings.Replace(r.ReqKey, "jimeng_v30", "jimeng_i2v_first_v30", 1), "p")
|
||||
} else {
|
||||
// 无图片：文生视频
|
||||
r.ReqKey = strings.Replace(r.ReqKey, "jimeng_v30", "jimeng_t2v_v30", 1)
|
||||
}
|
||||
}
|
||||
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
resTask := responseTask{}
|
||||
if err := common.Unmarshal(respBody, &resTask); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal task result failed")
|
||||
}
|
||||
taskResult := relaycommon.TaskInfo{}
|
||||
if resTask.Code == 10000 {
|
||||
taskResult.Code = 0
|
||||
} else {
|
||||
taskResult.Code = resTask.Code // todo uni code
|
||||
taskResult.Reason = resTask.Message
|
||||
taskResult.Status = model.TaskStatusFailure
|
||||
taskResult.Progress = "100%"
|
||||
}
|
||||
switch resTask.Data.Status {
|
||||
case "in_queue":
|
||||
taskResult.Status = model.TaskStatusQueued
|
||||
taskResult.Progress = "10%"
|
||||
case "done":
|
||||
taskResult.Status = model.TaskStatusSuccess
|
||||
taskResult.Progress = "100%"
|
||||
}
|
||||
taskResult.Url = resTask.Data.VideoUrl
|
||||
return &taskResult, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) ([]byte, error) {
|
||||
var jimengResp responseTask
|
||||
if err := common.Unmarshal(originTask.Data, &jimengResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal jimeng task data failed")
|
||||
}
|
||||
|
||||
openAIVideo := dto.NewOpenAIVideo()
|
||||
openAIVideo.ID = originTask.TaskID
|
||||
openAIVideo.Status = originTask.Status.ToVideoStatus()
|
||||
openAIVideo.SetProgressStr(originTask.Progress)
|
||||
openAIVideo.SetMetadata("url", jimengResp.Data.VideoUrl)
|
||||
openAIVideo.CreatedAt = originTask.CreatedAt
|
||||
openAIVideo.CompletedAt = originTask.UpdatedAt
|
||||
|
||||
if jimengResp.Code != 10000 {
|
||||
openAIVideo.Error = &dto.OpenAIVideoError{
|
||||
Message: jimengResp.Message,
|
||||
Code: fmt.Sprintf("%d", jimengResp.Code),
|
||||
}
|
||||
}
|
||||
|
||||
return common.Marshal(openAIVideo)
|
||||
}
|
||||
|
||||
func isNewAPIRelay(apiKey string) bool {
|
||||
return strings.HasPrefix(apiKey, "sk-")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
package jsplugin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
pluginruntime "github.com/QuantumNous/new-api/pkg/jsplugin"
|
||||
vertexcore "github.com/QuantumNous/new-api/relay/channel/vertex"
|
||||
)
|
||||
|
||||
type cachedAuth struct {
|
||||
header, projectID string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
var pluginAuthCache sync.Map
|
||||
var acquireAccessToken = vertexcore.AcquireAccessToken
|
||||
|
||||
func resolveAuth(meta pluginruntime.AuthMeta, apiKey, proxy string) (map[string]any, error) {
|
||||
typeName := strings.TrimSpace(meta.Type)
|
||||
if typeName == "" || typeName == "none" || typeName == "api_key" {
|
||||
return map[string]any{"authHeader": apiKey}, nil
|
||||
}
|
||||
cacheKey := apiKey + "\x00" + proxy
|
||||
if value, ok := pluginAuthCache.Load(cacheKey); ok {
|
||||
entry := value.(cachedAuth)
|
||||
if time.Now().Before(entry.expiresAt) {
|
||||
return map[string]any{"authHeader": entry.header, "projectId": entry.projectID}, nil
|
||||
}
|
||||
}
|
||||
var credentials vertexcore.Credentials
|
||||
if err := common.Unmarshal([]byte(apiKey), &credentials); err != nil {
|
||||
return nil, fmt.Errorf("decode oauth2_jwt credentials: %w", err)
|
||||
}
|
||||
token, err := acquireAccessToken(credentials, proxy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry := cachedAuth{header: "Bearer " + token, projectID: credentials.ProjectID, expiresAt: time.Now().Add(25 * time.Minute)}
|
||||
pluginAuthCache.Store(cacheKey, entry)
|
||||
return map[string]any{"authHeader": entry.header, "projectId": entry.projectID}, nil
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package jsplugin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
pluginruntime "github.com/QuantumNous/new-api/pkg/jsplugin"
|
||||
vertexcore "github.com/QuantumNous/new-api/relay/channel/vertex"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestOAuth2JWTAuthCachesAndRefreshes(t *testing.T) {
|
||||
pluginAuthCache = sync.Map{}
|
||||
original := acquireAccessToken
|
||||
t.Cleanup(func() { acquireAccessToken = original; pluginAuthCache = sync.Map{} })
|
||||
calls := 0
|
||||
acquireAccessToken = func(_ vertexcore.Credentials, _ string) (string, error) {
|
||||
calls++
|
||||
return fmt.Sprintf("token-%d", calls), nil
|
||||
}
|
||||
credentials, err := common.Marshal(vertexcore.Credentials{ProjectID: "project", ClientEmail: "a@example.com", PrivateKey: "secret"})
|
||||
require.NoError(t, err)
|
||||
meta := pluginruntime.AuthMeta{Type: "oauth2_jwt"}
|
||||
first, err := resolveAuth(meta, string(credentials), "")
|
||||
require.NoError(t, err)
|
||||
second, err := resolveAuth(meta, string(credentials), "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Bearer token-1", first["authHeader"])
|
||||
assert.Equal(t, first, second)
|
||||
assert.Equal(t, 1, calls)
|
||||
pluginAuthCache.Store(string(credentials)+"\x00", cachedAuth{expiresAt: time.Now().Add(-time.Second)})
|
||||
refreshed, err := resolveAuth(meta, string(credentials), "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Bearer token-2", refreshed["authHeader"])
|
||||
assert.Equal(t, 2, calls)
|
||||
}
|
||||
|
||||
func TestOAuth2JWTContextDoesNotExposeServiceAccountKey(t *testing.T) {
|
||||
pluginAuthCache = sync.Map{}
|
||||
original := acquireAccessToken
|
||||
t.Cleanup(func() { acquireAccessToken = original; pluginAuthCache = sync.Map{} })
|
||||
acquireAccessToken = func(_ vertexcore.Credentials, _ string) (string, error) {
|
||||
return "access-token", nil
|
||||
}
|
||||
credentials, err := common.Marshal(vertexcore.Credentials{ProjectID: "project", ClientEmail: "a@example.com", PrivateKey: "secret"})
|
||||
require.NoError(t, err)
|
||||
source := `
|
||||
export const meta = {apiVersion:1,key:"oauth",name:"OAuth",version:"1.0.0",author:{name:"Test"},models:["m"],fetchMode:"per_task",auth:{type:"oauth2_jwt"}};
|
||||
export function buildSubmitRequest(ctx) {
|
||||
if (ctx.apiKey !== undefined) throw new Error("raw key exposed");
|
||||
return {url:ctx.baseUrl+"/submit",headers:{Authorization:ctx.authHeader}};
|
||||
}
|
||||
export function parseSubmitResponse(){return {taskId:"1"}}
|
||||
export function buildQueryRequest(){return {url:"https://example.com"}}
|
||||
export function parseTaskResult(){return {status:"SUCCESS"}}
|
||||
`
|
||||
plugin, err := pluginruntime.NewRegistry().Register(source, pluginruntime.Options{})
|
||||
require.NoError(t, err)
|
||||
adaptor := New(plugin)
|
||||
info := &relaycommon.RelayInfo{ChannelMeta: &relaycommon.ChannelMeta{ChannelBaseUrl: "https://provider.example", ApiKey: string(credentials)}, TaskRelayInfo: &relaycommon.TaskRelayInfo{}}
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/videos", nil)
|
||||
c.Set("task_request", relaycommon.TaskSubmitReq{Prompt: "p"})
|
||||
require.Nil(t, adaptor.ValidateRequestAndSetAction(c, info))
|
||||
req := httptest.NewRequest(http.MethodPost, "https://provider.example/submit", nil)
|
||||
require.NoError(t, adaptor.BuildRequestHeader(c, req, info))
|
||||
assert.Equal(t, "Bearer access-token", req.Header.Get("Authorization"))
|
||||
}
|
||||
@@ -1,418 +0,0 @@
|
||||
package kling
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
|
||||
"github.com/samber/lo"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
type TrajectoryPoint struct {
|
||||
X int `json:"x"`
|
||||
Y int `json:"y"`
|
||||
}
|
||||
|
||||
type DynamicMask struct {
|
||||
Mask string `json:"mask,omitempty"`
|
||||
Trajectories []TrajectoryPoint `json:"trajectories,omitempty"`
|
||||
}
|
||||
|
||||
type CameraConfig struct {
|
||||
Horizontal float64 `json:"horizontal,omitempty"`
|
||||
Vertical float64 `json:"vertical,omitempty"`
|
||||
Pan float64 `json:"pan,omitempty"`
|
||||
Tilt float64 `json:"tilt,omitempty"`
|
||||
Roll float64 `json:"roll,omitempty"`
|
||||
Zoom float64 `json:"zoom,omitempty"`
|
||||
}
|
||||
|
||||
type CameraControl struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
Config *CameraConfig `json:"config,omitempty"`
|
||||
}
|
||||
|
||||
type requestPayload struct {
|
||||
Prompt string `json:"prompt,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
ImageTail string `json:"image_tail,omitempty"`
|
||||
NegativePrompt string `json:"negative_prompt,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Duration string `json:"duration,omitempty"`
|
||||
AspectRatio string `json:"aspect_ratio,omitempty"`
|
||||
ModelName string `json:"model_name,omitempty"`
|
||||
Model string `json:"model,omitempty"` // Compatible with upstreams that only recognize "model"
|
||||
CfgScale float64 `json:"cfg_scale,omitempty"`
|
||||
StaticMask string `json:"static_mask,omitempty"`
|
||||
DynamicMasks []DynamicMask `json:"dynamic_masks,omitempty"`
|
||||
CameraControl *CameraControl `json:"camera_control,omitempty"`
|
||||
CallbackUrl string `json:"callback_url,omitempty"`
|
||||
ExternalTaskId string `json:"external_task_id,omitempty"`
|
||||
}
|
||||
|
||||
type responsePayload struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
TaskId string `json:"task_id"`
|
||||
RequestId string `json:"request_id"`
|
||||
Data struct {
|
||||
TaskId string `json:"task_id"`
|
||||
TaskStatus string `json:"task_status"`
|
||||
TaskStatusMsg string `json:"task_status_msg"`
|
||||
TaskInfo struct {
|
||||
ExternalTaskId string `json:"external_task_id"`
|
||||
} `json:"task_info"`
|
||||
WatermarkInfo struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
} `json:"watermark_info"`
|
||||
TaskResult struct {
|
||||
Videos []struct {
|
||||
Id string `json:"id"`
|
||||
Url string `json:"url"`
|
||||
WatermarkUrl string `json:"watermark_url"`
|
||||
Duration string `json:"duration"`
|
||||
} `json:"videos"`
|
||||
Images []struct {
|
||||
Index int `json:"index"`
|
||||
Url string `json:"url"`
|
||||
WatermarkUrl string `json:"watermark_url"`
|
||||
} `json:"images"`
|
||||
} `json:"task_result"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
FinalUnitDeduction string `json:"final_unit_deduction"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
|
||||
// apiKey format: "access_key|secret_key"
|
||||
}
|
||||
|
||||
// ValidateRequestAndSetAction parses body, validates fields and sets default action.
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *taskdto.TaskError) {
|
||||
// Use the standard validation method for TaskSubmitReq
|
||||
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionGenerate)
|
||||
}
|
||||
|
||||
// BuildRequestURL constructs the upstream URL.
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
path := lo.Ternary(info.Action == constant.TaskActionGenerate, "/v1/videos/image2video", "/v1/videos/text2video")
|
||||
|
||||
if isNewAPIRelay(info.ApiKey) {
|
||||
return fmt.Sprintf("%s/kling%s", a.baseURL, path), nil
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s%s", a.baseURL, path), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers.
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
token, err := a.createJWTToken()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create JWT token: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("User-Agent", "kling-sdk/1.0")
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildRequestBody converts request into Kling specific format.
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
v, exists := c.Get("task_request")
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("request not found in context")
|
||||
}
|
||||
req := v.(relaycommon.TaskSubmitReq)
|
||||
|
||||
body, err := a.convertToRequestPayload(&req, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if body.Image == "" && body.ImageTail == "" {
|
||||
c.Set("action", constant.TaskActionTextGenerate)
|
||||
}
|
||||
data, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper.
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
if action := c.GetString("action"); action != "" {
|
||||
info.Action = action
|
||||
}
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response, returns taskID etc.
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var kResp responsePayload
|
||||
err = common.Unmarshal(responseBody, &kResp)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "unmarshal_response_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if kResp.Code != 0 {
|
||||
taskErr = service.TaskErrorWrapperLocal(fmt.Errorf("%s", kResp.Message), "task_failed", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
ov := dto.NewOpenAIVideo()
|
||||
ov.ID = info.PublicTaskID
|
||||
ov.TaskID = info.PublicTaskID
|
||||
ov.CreatedAt = time.Now().Unix()
|
||||
ov.Model = info.OriginModelName
|
||||
c.JSON(http.StatusOK, ov)
|
||||
return kResp.Data.TaskId, responseBody, nil
|
||||
}
|
||||
|
||||
// FetchTask fetch task status
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
action, ok := body["action"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid action")
|
||||
}
|
||||
path := lo.Ternary(action == constant.TaskActionGenerate, "/v1/videos/image2video", "/v1/videos/text2video")
|
||||
url := fmt.Sprintf("%s%s/%s", baseUrl, path, taskID)
|
||||
if isNewAPIRelay(key) {
|
||||
url = fmt.Sprintf("%s/kling%s/%s", baseUrl, path, taskID)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
token, err := a.createJWTTokenWithKey(key)
|
||||
if err != nil {
|
||||
token = key
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("User-Agent", "kling-sdk/1.0")
|
||||
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return []string{"kling-v1", "kling-v1-6", "kling-v2-master"}
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return "kling"
|
||||
}
|
||||
|
||||
// ============================
|
||||
// helpers
|
||||
// ============================
|
||||
|
||||
func (a *TaskAdaptor) convertToRequestPayload(req *relaycommon.TaskSubmitReq, info *relaycommon.RelayInfo) (*requestPayload, error) {
|
||||
r := requestPayload{
|
||||
Prompt: req.Prompt,
|
||||
Image: req.Image,
|
||||
Mode: taskcommon.DefaultString(req.Mode, "std"),
|
||||
Duration: fmt.Sprintf("%d", taskcommon.DefaultInt(req.Duration, 5)),
|
||||
AspectRatio: a.getAspectRatio(req.Size),
|
||||
ModelName: info.UpstreamModelName,
|
||||
Model: info.UpstreamModelName,
|
||||
CfgScale: 0.5,
|
||||
StaticMask: "",
|
||||
DynamicMasks: []DynamicMask{},
|
||||
CameraControl: nil,
|
||||
CallbackUrl: "",
|
||||
ExternalTaskId: "",
|
||||
}
|
||||
if r.ModelName == "" {
|
||||
r.ModelName = "kling-v1"
|
||||
r.Model = "kling-v1"
|
||||
}
|
||||
if err := taskcommon.UnmarshalMetadata(req.Metadata, &r); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal metadata failed")
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) getAspectRatio(size string) string {
|
||||
switch size {
|
||||
case "1024x1024", "512x512":
|
||||
return "1:1"
|
||||
case "1280x720", "1920x1080":
|
||||
return "16:9"
|
||||
case "720x1280", "1080x1920":
|
||||
return "9:16"
|
||||
default:
|
||||
return "1:1"
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// JWT helpers
|
||||
// ============================
|
||||
|
||||
func (a *TaskAdaptor) createJWTToken() (string, error) {
|
||||
return a.createJWTTokenWithKey(a.apiKey)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) createJWTTokenWithKey(apiKey string) (string, error) {
|
||||
if isNewAPIRelay(apiKey) {
|
||||
return apiKey, nil // new api relay
|
||||
}
|
||||
keyParts := strings.Split(apiKey, "|")
|
||||
if len(keyParts) != 2 {
|
||||
return "", errors.New("invalid api_key, required format is accessKey|secretKey")
|
||||
}
|
||||
accessKey := strings.TrimSpace(keyParts[0])
|
||||
if len(keyParts) == 1 {
|
||||
return accessKey, nil
|
||||
}
|
||||
secretKey := strings.TrimSpace(keyParts[1])
|
||||
now := time.Now().Unix()
|
||||
claims := jwt.MapClaims{
|
||||
"iss": accessKey,
|
||||
"exp": now + 1800, // 30 minutes
|
||||
"nbf": now - 5,
|
||||
}
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||
token.Header["typ"] = "JWT"
|
||||
return token.SignedString([]byte(secretKey))
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
taskInfo := &relaycommon.TaskInfo{}
|
||||
resPayload := responsePayload{}
|
||||
err := common.Unmarshal(respBody, &resPayload)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to unmarshal response body")
|
||||
}
|
||||
taskInfo.Code = resPayload.Code
|
||||
taskInfo.TaskID = resPayload.Data.TaskId
|
||||
taskInfo.Reason = resPayload.Data.TaskStatusMsg
|
||||
//任务状态，枚举值：submitted（已提交）、processing（处理中）、succeed（成功）、failed（失败）
|
||||
status := resPayload.Data.TaskStatus
|
||||
switch status {
|
||||
case "submitted":
|
||||
taskInfo.Status = model.TaskStatusSubmitted
|
||||
case "processing":
|
||||
taskInfo.Status = model.TaskStatusInProgress
|
||||
case "succeed":
|
||||
taskInfo.Status = model.TaskStatusSuccess
|
||||
if videos := resPayload.Data.TaskResult.Videos; len(videos) > 0 {
|
||||
video := videos[0]
|
||||
taskInfo.Url = video.Url
|
||||
}
|
||||
if tokens, err := strconv.ParseFloat(resPayload.Data.FinalUnitDeduction, 64); err == nil {
|
||||
// 上游返回的扣费数值，饱和转换防止超大数值回绕成负数
|
||||
rounded := common.QuotaFromFloat(math.Ceil(tokens))
|
||||
if rounded > 0 {
|
||||
taskInfo.CompletionTokens = rounded
|
||||
taskInfo.TotalTokens = rounded
|
||||
}
|
||||
}
|
||||
case "failed":
|
||||
taskInfo.Status = model.TaskStatusFailure
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown task status: %s", status)
|
||||
}
|
||||
return taskInfo, nil
|
||||
}
|
||||
|
||||
func isNewAPIRelay(apiKey string) bool {
|
||||
return strings.HasPrefix(apiKey, "sk-")
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) ([]byte, error) {
|
||||
var klingResp responsePayload
|
||||
if err := common.Unmarshal(originTask.Data, &klingResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal kling task data failed")
|
||||
}
|
||||
|
||||
openAIVideo := dto.NewOpenAIVideo()
|
||||
openAIVideo.ID = originTask.TaskID
|
||||
openAIVideo.Status = originTask.Status.ToVideoStatus()
|
||||
openAIVideo.SetProgressStr(originTask.Progress)
|
||||
openAIVideo.CreatedAt = klingResp.Data.CreatedAt
|
||||
openAIVideo.CompletedAt = klingResp.Data.UpdatedAt
|
||||
|
||||
if len(klingResp.Data.TaskResult.Videos) > 0 {
|
||||
video := klingResp.Data.TaskResult.Videos[0]
|
||||
if video.Url != "" {
|
||||
openAIVideo.SetMetadata("url", video.Url)
|
||||
}
|
||||
if video.Duration != "" {
|
||||
openAIVideo.Seconds = video.Duration
|
||||
}
|
||||
}
|
||||
|
||||
if klingResp.Code != 0 && klingResp.Message != "" {
|
||||
openAIVideo.Error = &dto.OpenAIVideoError{
|
||||
Message: klingResp.Message,
|
||||
Code: fmt.Sprintf("%d", klingResp.Code),
|
||||
}
|
||||
}
|
||||
|
||||
// https://app.klingai.com/cn/dev/document-api/apiReference/model/textToVideo
|
||||
if data := klingResp.Data; data.TaskStatus == "failed" {
|
||||
openAIVideo.Error = &dto.OpenAIVideoError{
|
||||
Message: data.TaskStatusMsg,
|
||||
}
|
||||
}
|
||||
return common.Marshal(openAIVideo)
|
||||
}
|
||||
@@ -1,331 +0,0 @@
|
||||
package sora
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
"github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tidwall/sjson"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
type ContentItem struct {
|
||||
Type string `json:"type"` // "text" or "image_url"
|
||||
Text string `json:"text,omitempty"` // for text type
|
||||
ImageURL *ImageURL `json:"image_url,omitempty"` // for image_url type
|
||||
}
|
||||
|
||||
type ImageURL struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type responseTask struct {
|
||||
ID string `json:"id"`
|
||||
TaskID string `json:"task_id,omitempty"` //兼容旧接口
|
||||
Object string `json:"object"`
|
||||
Model string `json:"model"`
|
||||
Status string `json:"status"`
|
||||
Progress int `json:"progress"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
CompletedAt int64 `json:"completed_at,omitempty"`
|
||||
ExpiresAt int64 `json:"expires_at,omitempty"`
|
||||
Seconds string `json:"seconds,omitempty"`
|
||||
Size string `json:"size,omitempty"`
|
||||
RemixedFromVideoID string `json:"remixed_from_video_id,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
Code string `json:"code"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
}
|
||||
|
||||
func validateRemixRequest(c *gin.Context) *dto.TaskError {
|
||||
var req relaycommon.TaskSubmitReq
|
||||
if err := common.UnmarshalBodyReusable(c, &req); err != nil {
|
||||
return service.TaskErrorWrapperLocal(err, "invalid_request", http.StatusBadRequest)
|
||||
}
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
return service.TaskErrorWrapperLocal(fmt.Errorf("field prompt is required"), "invalid_request", http.StatusBadRequest)
|
||||
}
|
||||
// 存储原始请求到 context，与 ValidateMultipartDirect 路径保持一致
|
||||
c.Set("task_request", req)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *dto.TaskError) {
|
||||
if info.Action == constant.TaskActionRemix {
|
||||
return validateRemixRequest(c)
|
||||
}
|
||||
return relaycommon.ValidateMultipartDirect(c, info)
|
||||
}
|
||||
|
||||
// EstimateBilling 根据用户请求的 seconds 和 size 计算 OtherRatios。
|
||||
func (a *TaskAdaptor) EstimateBilling(c *gin.Context, info *relaycommon.RelayInfo) map[string]float64 {
|
||||
// remix 路径的 OtherRatios 已在 ResolveOriginTask 中设置
|
||||
if info.Action == constant.TaskActionRemix {
|
||||
return nil
|
||||
}
|
||||
|
||||
req, err := relaycommon.GetTaskRequest(c)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
seconds, _ := strconv.Atoi(req.Seconds)
|
||||
if seconds == 0 {
|
||||
seconds = req.Duration
|
||||
}
|
||||
if seconds <= 0 {
|
||||
seconds = 4
|
||||
}
|
||||
|
||||
size := req.Size
|
||||
if size == "" {
|
||||
size = "720x1280"
|
||||
}
|
||||
|
||||
ratios := map[string]float64{
|
||||
"seconds": float64(seconds),
|
||||
"size": 1,
|
||||
}
|
||||
if size == "1792x1024" || size == "1024x1792" {
|
||||
ratios["size"] = 1.666667
|
||||
}
|
||||
return ratios
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
if info.Action == constant.TaskActionRemix {
|
||||
return fmt.Sprintf("%s/v1/videos/%s/remix", a.baseURL, info.OriginTaskID), nil
|
||||
}
|
||||
return fmt.Sprintf("%s/v1/videos", a.baseURL), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers.
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Authorization", "Bearer "+a.apiKey)
|
||||
req.Header.Set("Content-Type", c.Request.Header.Get("Content-Type"))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
storage, err := common.GetBodyStorage(c)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "get_request_body_failed")
|
||||
}
|
||||
cachedBody, err := storage.Bytes()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "read_body_bytes_failed")
|
||||
}
|
||||
contentType := c.GetHeader("Content-Type")
|
||||
|
||||
if strings.HasPrefix(contentType, "application/json") {
|
||||
var bodyMap map[string]interface{}
|
||||
if err := common.Unmarshal(cachedBody, &bodyMap); err == nil {
|
||||
bodyMap["model"] = info.UpstreamModelName
|
||||
if newBody, err := common.Marshal(bodyMap); err == nil {
|
||||
return bytes.NewReader(newBody), nil
|
||||
}
|
||||
}
|
||||
return bytes.NewReader(cachedBody), nil
|
||||
}
|
||||
|
||||
if strings.Contains(contentType, "multipart/form-data") {
|
||||
formData, err := common.ParseMultipartFormReusable(c)
|
||||
if err != nil {
|
||||
return bytes.NewReader(cachedBody), nil
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
writer := multipart.NewWriter(&buf)
|
||||
writer.WriteField("model", info.UpstreamModelName)
|
||||
for key, values := range formData.Value {
|
||||
if key == "model" {
|
||||
continue
|
||||
}
|
||||
for _, v := range values {
|
||||
writer.WriteField(key, v)
|
||||
}
|
||||
}
|
||||
for fieldName, fileHeaders := range formData.File {
|
||||
for _, fh := range fileHeaders {
|
||||
f, err := fh.Open()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ct := fh.Header.Get("Content-Type")
|
||||
if ct == "" || ct == "application/octet-stream" {
|
||||
buf512 := make([]byte, 512)
|
||||
n, _ := io.ReadFull(f, buf512)
|
||||
ct = http.DetectContentType(buf512[:n])
|
||||
// Re-open after sniffing so the full content is copied below
|
||||
f.Close()
|
||||
f, err = fh.Open()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
h := make(textproto.MIMEHeader)
|
||||
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldName, fh.Filename))
|
||||
h.Set("Content-Type", ct)
|
||||
part, err := writer.CreatePart(h)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
continue
|
||||
}
|
||||
io.Copy(part, f)
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
writer.Close()
|
||||
c.Request.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
return &buf, nil
|
||||
}
|
||||
|
||||
return common.NewReplayableBodyReader(storage), nil
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper.
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response, returns taskID etc.
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *dto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
// Parse Sora response
|
||||
var dResp responseTask
|
||||
if err := common.Unmarshal(responseBody, &dResp); err != nil {
|
||||
taskErr = service.TaskErrorWrapper(errors.Wrapf(err, "body: %s", responseBody), "unmarshal_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
upstreamID := dResp.ID
|
||||
if upstreamID == "" {
|
||||
upstreamID = dResp.TaskID
|
||||
}
|
||||
if upstreamID == "" {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("task_id is empty"), "invalid_response", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// 使用公开 task_xxxx ID 返回给客户端
|
||||
dResp.ID = info.PublicTaskID
|
||||
dResp.TaskID = info.PublicTaskID
|
||||
c.JSON(http.StatusOK, dResp)
|
||||
return upstreamID, responseBody, nil
|
||||
}
|
||||
|
||||
// FetchTask fetch task status
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
uri := fmt.Sprintf("%s/v1/videos/%s", baseUrl, taskID)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, uri, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return ModelList
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return ChannelName
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
resTask := responseTask{}
|
||||
if err := common.Unmarshal(respBody, &resTask); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal task result failed")
|
||||
}
|
||||
|
||||
taskResult := relaycommon.TaskInfo{
|
||||
Code: 0,
|
||||
}
|
||||
|
||||
switch resTask.Status {
|
||||
case "queued", "pending":
|
||||
taskResult.Status = model.TaskStatusQueued
|
||||
case "processing", "in_progress":
|
||||
taskResult.Status = model.TaskStatusInProgress
|
||||
case "completed":
|
||||
taskResult.Status = model.TaskStatusSuccess
|
||||
// Url intentionally left empty — the caller constructs the proxy URL using the public task ID
|
||||
case "failed", "cancelled":
|
||||
taskResult.Status = model.TaskStatusFailure
|
||||
if resTask.Error != nil {
|
||||
taskResult.Reason = resTask.Error.Message
|
||||
} else {
|
||||
taskResult.Reason = "task failed"
|
||||
}
|
||||
default:
|
||||
}
|
||||
if resTask.Progress > 0 && resTask.Progress < 100 {
|
||||
taskResult.Progress = fmt.Sprintf("%d%%", resTask.Progress)
|
||||
}
|
||||
|
||||
return &taskResult, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) {
|
||||
data := task.Data
|
||||
var err error
|
||||
if data, err = sjson.SetBytes(data, "id", task.TaskID); err != nil {
|
||||
return nil, errors.Wrap(err, "set id failed")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package sora
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSoraBuildRequestBodyReturnsReplayablePassThroughBody(t *testing.T) {
|
||||
payload := []byte("opaque-sora-request-body")
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/videos", bytes.NewReader(payload))
|
||||
c.Request.Header.Set("Content-Type", "application/octet-stream")
|
||||
defer common.CleanupBodyStorage(c)
|
||||
|
||||
info := &relaycommon.RelayInfo{}
|
||||
body, err := (&TaskAdaptor{}).BuildRequestBody(c, info)
|
||||
require.NoError(t, err)
|
||||
replayable, ok := body.(common.ReplayableBody)
|
||||
require.True(t, ok)
|
||||
|
||||
sent, err := io.ReadAll(body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, payload, sent)
|
||||
assert.EqualValues(t, len(payload), replayable.Size())
|
||||
|
||||
replayBody, err := replayable.NewReader()
|
||||
require.NoError(t, err)
|
||||
replay, err := io.ReadAll(replayBody)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, replayBody.Close())
|
||||
assert.Equal(t, payload, replay)
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
package sora
|
||||
|
||||
var ModelList = []string{
|
||||
"sora-2",
|
||||
"sora-2-pro",
|
||||
}
|
||||
|
||||
var ChannelName = "sora"
|
||||
@@ -1,167 +0,0 @@
|
||||
package suno
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
"github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
}
|
||||
|
||||
// ParseTaskResult is not used for Suno tasks.
|
||||
// Suno polling uses a dedicated batch-fetch path (service.UpdateSunoTasks) that
|
||||
// receives dto.TaskResponse[[]dto.SunoDataResponse] from the upstream /fetch API.
|
||||
// This differs from the per-task polling used by video adaptors.
|
||||
func (a *TaskAdaptor) ParseTaskResult([]byte) (*relaycommon.TaskInfo, error) {
|
||||
return nil, fmt.Errorf("suno uses batch polling via UpdateSunoTasks, ParseTaskResult is not applicable")
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *dto.TaskError) {
|
||||
action := strings.ToUpper(c.Param("action"))
|
||||
|
||||
var sunoRequest *dto.SunoSubmitReq
|
||||
err := common.UnmarshalBodyReusable(c, &sunoRequest)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapperLocal(err, "invalid_request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
err = actionValidate(c, sunoRequest, action)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapperLocal(err, "invalid_request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
//if sunoRequest.ContinueClipId != "" {
|
||||
// if sunoRequest.TaskID == "" {
|
||||
// taskErr = service.TaskErrorWrapperLocal(fmt.Errorf("task id is empty"), "invalid_request", http.StatusBadRequest)
|
||||
// return
|
||||
// }
|
||||
// info.OriginTaskID = sunoRequest.TaskID
|
||||
//}
|
||||
|
||||
info.Action = action
|
||||
c.Set("task_request", sunoRequest)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
baseURL := info.ChannelBaseUrl
|
||||
fullRequestURL := fmt.Sprintf("%s%s", baseURL, "/suno/submit/"+info.Action)
|
||||
return fullRequestURL, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Content-Type", c.Request.Header.Get("Content-Type"))
|
||||
req.Header.Set("Accept", c.Request.Header.Get("Accept"))
|
||||
req.Header.Set("Authorization", "Bearer "+info.ApiKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
sunoRequest, ok := c.Get("task_request")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("task_request not found in context")
|
||||
}
|
||||
data, err := common.Marshal(sunoRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *dto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var sunoResponse dto.TaskResponse[string]
|
||||
err = common.Unmarshal(responseBody, &sunoResponse)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !sunoResponse.IsSuccess() {
|
||||
taskErr = service.TaskErrorWrapper(fmt.Errorf("%s", sunoResponse.Message), sunoResponse.Code, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// 使用公开 task_xxxx ID 替换上游 ID 返回给客户端
|
||||
publicResponse := dto.TaskResponse[string]{
|
||||
Code: sunoResponse.Code,
|
||||
Message: sunoResponse.Message,
|
||||
Data: info.PublicTaskID,
|
||||
}
|
||||
c.JSON(http.StatusOK, publicResponse)
|
||||
|
||||
return sunoResponse.Data, nil, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return ModelList
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return ChannelName
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
requestUrl := fmt.Sprintf("%s/suno/fetch", baseUrl)
|
||||
byteBody, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", requestUrl, bytes.NewBuffer(byteBody))
|
||||
if err != nil {
|
||||
common.SysLog(fmt.Sprintf("Get Task error: %v", err))
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func actionValidate(c *gin.Context, sunoRequest *dto.SunoSubmitReq, action string) (err error) {
|
||||
switch action {
|
||||
case constant.SunoActionMusic:
|
||||
if sunoRequest.Mv == "" {
|
||||
sunoRequest.Mv = "chirp-v3-0"
|
||||
}
|
||||
case constant.SunoActionLyrics:
|
||||
if sunoRequest.Prompt == "" {
|
||||
err = fmt.Errorf("prompt_empty")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = fmt.Errorf("invalid_action")
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package suno
|
||||
|
||||
var ModelList = []string{
|
||||
"suno_music", "suno_lyrics",
|
||||
}
|
||||
|
||||
var ChannelName = "suno"
|
||||
@@ -1,417 +0,0 @@
|
||||
package vertex
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
geminitask "github.com/QuantumNous/new-api/relay/channel/task/gemini"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
vertexcore "github.com/QuantumNous/new-api/relay/channel/vertex"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
type fetchOperationPayload struct {
|
||||
OperationName string `json:"operationName"`
|
||||
}
|
||||
|
||||
type submitResponse struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type operationVideo struct {
|
||||
MimeType string `json:"mimeType"`
|
||||
BytesBase64Encoded string `json:"bytesBase64Encoded"`
|
||||
Encoding string `json:"encoding"`
|
||||
}
|
||||
|
||||
type operationResponse struct {
|
||||
Name string `json:"name"`
|
||||
Done bool `json:"done"`
|
||||
Response struct {
|
||||
Type string `json:"@type"`
|
||||
RaiMediaFilteredCount int `json:"raiMediaFilteredCount"`
|
||||
Videos []operationVideo `json:"videos"`
|
||||
BytesBase64Encoded string `json:"bytesBase64Encoded"`
|
||||
Encoding string `json:"encoding"`
|
||||
Video string `json:"video"`
|
||||
} `json:"response"`
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
apiKey string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
a.apiKey = info.ApiKey
|
||||
}
|
||||
|
||||
// ValidateRequestAndSetAction parses body, validates fields and sets default action.
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *taskdto.TaskError) {
|
||||
// Use the standard validation method for TaskSubmitReq
|
||||
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionTextGenerate)
|
||||
}
|
||||
|
||||
// BuildRequestURL constructs the upstream URL.
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
adc := &vertexcore.Credentials{}
|
||||
if err := common.Unmarshal([]byte(a.apiKey), adc); err != nil {
|
||||
return "", fmt.Errorf("failed to decode credentials: %w", err)
|
||||
}
|
||||
modelName := info.UpstreamModelName
|
||||
if modelName == "" {
|
||||
modelName = "veo-3.0-generate-001"
|
||||
}
|
||||
|
||||
region := vertexcore.GetModelRegion(info.ApiVersion, modelName)
|
||||
if strings.TrimSpace(region) == "" {
|
||||
region = "global"
|
||||
}
|
||||
return vertexcore.BuildGoogleModelURL(a.baseURL, vertexcore.DefaultAPIVersion, adc.ProjectID, region, modelName, "predictLongRunning"), nil
|
||||
}
|
||||
|
||||
// BuildRequestHeader sets required headers.
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
adc := &vertexcore.Credentials{}
|
||||
if err := common.Unmarshal([]byte(a.apiKey), adc); err != nil {
|
||||
return fmt.Errorf("failed to decode credentials: %w", err)
|
||||
}
|
||||
|
||||
proxy := ""
|
||||
if info != nil {
|
||||
proxy = info.ChannelSetting.Proxy
|
||||
}
|
||||
token, err := vertexcore.AcquireAccessToken(*adc, proxy)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to acquire access token: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("x-goog-user-project", adc.ProjectID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// EstimateBilling returns OtherRatios based on durationSeconds and resolution.
|
||||
func (a *TaskAdaptor) EstimateBilling(c *gin.Context, info *relaycommon.RelayInfo) map[string]float64 {
|
||||
v, ok := c.Get("task_request")
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
req := v.(relaycommon.TaskSubmitReq)
|
||||
|
||||
seconds := geminitask.ResolveVeoDuration(req.Metadata, req.Duration, req.Seconds)
|
||||
resolution := geminitask.ResolveVeoResolution(req.Metadata, req.Size)
|
||||
resRatio := geminitask.VeoResolutionRatio(info.UpstreamModelName, resolution)
|
||||
|
||||
return map[string]float64{
|
||||
"seconds": float64(seconds),
|
||||
"resolution": resRatio,
|
||||
}
|
||||
}
|
||||
|
||||
// BuildRequestBody converts request into Vertex specific format.
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
v, ok := c.Get("task_request")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("request not found in context")
|
||||
}
|
||||
req := v.(relaycommon.TaskSubmitReq)
|
||||
|
||||
instance := geminitask.VeoInstance{Prompt: req.Prompt}
|
||||
if img := geminitask.ExtractMultipartImage(c, info); img != nil {
|
||||
instance.Image = img
|
||||
} else if len(req.Images) > 0 {
|
||||
if parsed := geminitask.ParseImageInput(req.Images[0]); parsed != nil {
|
||||
instance.Image = parsed
|
||||
info.Action = constant.TaskActionGenerate
|
||||
}
|
||||
}
|
||||
|
||||
params := &geminitask.VeoParameters{}
|
||||
if err := taskcommon.UnmarshalMetadata(req.Metadata, params); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal metadata failed: %w", err)
|
||||
}
|
||||
if params.DurationSeconds == 0 && req.Duration > 0 {
|
||||
params.DurationSeconds = req.Duration
|
||||
}
|
||||
if params.Resolution == "" && req.Size != "" {
|
||||
params.Resolution = geminitask.SizeToVeoResolution(req.Size)
|
||||
}
|
||||
if params.AspectRatio == "" && req.Size != "" {
|
||||
params.AspectRatio = geminitask.SizeToVeoAspectRatio(req.Size)
|
||||
}
|
||||
params.Resolution = strings.ToLower(params.Resolution)
|
||||
params.SampleCount = 1
|
||||
|
||||
body := geminitask.VeoRequestPayload{
|
||||
Instances: []geminitask.VeoInstance{instance},
|
||||
Parameters: params,
|
||||
}
|
||||
|
||||
data, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
// DoRequest delegates to common helper.
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
// DoResponse handles upstream response, returns taskID etc.
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", nil, service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
var s submitResponse
|
||||
if err := common.Unmarshal(responseBody, &s); err != nil {
|
||||
return "", nil, service.TaskErrorWrapper(err, "unmarshal_response_failed", http.StatusInternalServerError)
|
||||
}
|
||||
if strings.TrimSpace(s.Name) == "" {
|
||||
return "", nil, service.TaskErrorWrapper(fmt.Errorf("missing operation name"), "invalid_response", http.StatusInternalServerError)
|
||||
}
|
||||
localID := taskcommon.EncodeLocalTaskID(s.Name)
|
||||
ov := dto.NewOpenAIVideo()
|
||||
ov.ID = info.PublicTaskID
|
||||
ov.TaskID = info.PublicTaskID
|
||||
ov.CreatedAt = time.Now().Unix()
|
||||
ov.Model = info.OriginModelName
|
||||
c.JSON(http.StatusOK, ov)
|
||||
return localID, responseBody, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return []string{
|
||||
"veo-3.0-generate-001",
|
||||
"veo-3.0-fast-generate-001",
|
||||
"veo-3.1-generate-preview",
|
||||
"veo-3.1-fast-generate-preview",
|
||||
}
|
||||
}
|
||||
func (a *TaskAdaptor) GetChannelName() string { return "vertex" }
|
||||
|
||||
func buildFetchOperationURL(baseURL, upstreamName string) (string, error) {
|
||||
region := extractRegionFromOperationName(upstreamName)
|
||||
if region == "" {
|
||||
region = "us-central1"
|
||||
}
|
||||
project := extractProjectFromOperationName(upstreamName)
|
||||
modelName := extractModelFromOperationName(upstreamName)
|
||||
if strings.TrimSpace(modelName) == "" {
|
||||
return "", fmt.Errorf("cannot extract model from operation name")
|
||||
}
|
||||
if strings.TrimSpace(project) == "" {
|
||||
return "", fmt.Errorf("cannot extract project from operation name")
|
||||
}
|
||||
return vertexcore.BuildGoogleModelURL(baseURL, vertexcore.DefaultAPIVersion, project, region, modelName, "fetchPredictOperation"), nil
|
||||
}
|
||||
|
||||
// FetchTask fetch task status
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
upstreamName, err := taskcommon.DecodeLocalTaskID(taskID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode task_id failed: %w", err)
|
||||
}
|
||||
url, err := buildFetchOperationURL(baseUrl, upstreamName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
payload := fetchOperationPayload{OperationName: upstreamName}
|
||||
data, err := common.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
adc := &vertexcore.Credentials{}
|
||||
if err := common.Unmarshal([]byte(key), adc); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode credentials: %w", err)
|
||||
}
|
||||
token, err := vertexcore.AcquireAccessToken(*adc, proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire access token: %w", err)
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("x-goog-user-project", adc.ProjectID)
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
var op operationResponse
|
||||
if err := common.Unmarshal(respBody, &op); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal operation response failed: %w", err)
|
||||
}
|
||||
ti := &relaycommon.TaskInfo{}
|
||||
if op.Error.Message != "" {
|
||||
ti.Status = model.TaskStatusFailure
|
||||
ti.Reason = op.Error.Message
|
||||
ti.Progress = "100%"
|
||||
return ti, nil
|
||||
}
|
||||
if !op.Done {
|
||||
ti.Status = model.TaskStatusInProgress
|
||||
ti.Progress = "50%"
|
||||
return ti, nil
|
||||
}
|
||||
ti.Status = model.TaskStatusSuccess
|
||||
ti.Progress = "100%"
|
||||
if len(op.Response.Videos) > 0 {
|
||||
v0 := op.Response.Videos[0]
|
||||
if v0.BytesBase64Encoded != "" {
|
||||
mime := strings.TrimSpace(v0.MimeType)
|
||||
if mime == "" {
|
||||
enc := strings.TrimSpace(v0.Encoding)
|
||||
if enc == "" {
|
||||
enc = "mp4"
|
||||
}
|
||||
if strings.Contains(enc, "/") {
|
||||
mime = enc
|
||||
} else {
|
||||
mime = "video/" + enc
|
||||
}
|
||||
}
|
||||
ti.Url = "data:" + mime + ";base64," + v0.BytesBase64Encoded
|
||||
return ti, nil
|
||||
}
|
||||
}
|
||||
if op.Response.BytesBase64Encoded != "" {
|
||||
enc := strings.TrimSpace(op.Response.Encoding)
|
||||
if enc == "" {
|
||||
enc = "mp4"
|
||||
}
|
||||
mime := enc
|
||||
if !strings.Contains(enc, "/") {
|
||||
mime = "video/" + enc
|
||||
}
|
||||
ti.Url = "data:" + mime + ";base64," + op.Response.BytesBase64Encoded
|
||||
return ti, nil
|
||||
}
|
||||
if op.Response.Video != "" { // some variants use `video` as base64
|
||||
enc := strings.TrimSpace(op.Response.Encoding)
|
||||
if enc == "" {
|
||||
enc = "mp4"
|
||||
}
|
||||
mime := enc
|
||||
if !strings.Contains(enc, "/") {
|
||||
mime = "video/" + enc
|
||||
}
|
||||
ti.Url = "data:" + mime + ";base64," + op.Response.Video
|
||||
return ti, nil
|
||||
}
|
||||
return ti, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) {
|
||||
// Use GetUpstreamTaskID() to get the real upstream operation name for model extraction.
|
||||
// task.TaskID is now a public task_xxxx ID, no longer a base64-encoded upstream name.
|
||||
upstreamTaskID := task.GetUpstreamTaskID()
|
||||
upstreamName, err := taskcommon.DecodeLocalTaskID(upstreamTaskID)
|
||||
if err != nil {
|
||||
upstreamName = ""
|
||||
}
|
||||
modelName := extractModelFromOperationName(upstreamName)
|
||||
if strings.TrimSpace(modelName) == "" {
|
||||
modelName = "veo-3.0-generate-001"
|
||||
}
|
||||
v := dto.NewOpenAIVideo()
|
||||
v.ID = task.TaskID
|
||||
v.Model = modelName
|
||||
v.Status = task.Status.ToVideoStatus()
|
||||
v.SetProgressStr(task.Progress)
|
||||
v.CreatedAt = task.CreatedAt
|
||||
v.CompletedAt = task.UpdatedAt
|
||||
if resultURL := task.GetResultURL(); strings.HasPrefix(resultURL, "data:") && len(resultURL) > 0 {
|
||||
v.SetMetadata("url", resultURL)
|
||||
}
|
||||
|
||||
return common.Marshal(v)
|
||||
}
|
||||
|
||||
// ============================
|
||||
// helpers
|
||||
// ============================
|
||||
|
||||
var regionRe = regexp.MustCompile(`locations/([a-z0-9-]+)/`)
|
||||
|
||||
func extractRegionFromOperationName(name string) string {
|
||||
m := regionRe.FindStringSubmatch(name)
|
||||
if len(m) == 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var modelRe = regexp.MustCompile(`models/([^/]+)/operations/`)
|
||||
|
||||
func extractModelFromOperationName(name string) string {
|
||||
m := modelRe.FindStringSubmatch(name)
|
||||
if len(m) == 2 {
|
||||
return m[1]
|
||||
}
|
||||
idx := strings.Index(name, "models/")
|
||||
if idx >= 0 {
|
||||
s := name[idx+len("models/"):]
|
||||
if p := strings.Index(s, "/operations/"); p > 0 {
|
||||
return s[:p]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var projectRe = regexp.MustCompile(`projects/([^/]+)/locations/`)
|
||||
|
||||
func extractProjectFromOperationName(name string) string {
|
||||
m := projectRe.FindStringSubmatch(name)
|
||||
if len(m) == 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,301 +0,0 @@
|
||||
package vidu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
taskdto "github.com/QuantumNous/new-api/dto"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/relaykit/dto"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ============================
|
||||
// Request / Response structures
|
||||
// ============================
|
||||
|
||||
type requestPayload struct {
|
||||
Model string `json:"model"`
|
||||
Images []string `json:"images"`
|
||||
Prompt string `json:"prompt,omitempty"`
|
||||
Duration int `json:"duration,omitempty"`
|
||||
Seed int `json:"seed,omitempty"`
|
||||
Resolution string `json:"resolution,omitempty"`
|
||||
MovementAmplitude string `json:"movement_amplitude,omitempty"`
|
||||
Bgm bool `json:"bgm,omitempty"`
|
||||
Payload string `json:"payload,omitempty"`
|
||||
CallbackUrl string `json:"callback_url,omitempty"`
|
||||
}
|
||||
|
||||
type responsePayload struct {
|
||||
TaskId string `json:"task_id"`
|
||||
State string `json:"state"`
|
||||
Model string `json:"model"`
|
||||
Images []string `json:"images"`
|
||||
Prompt string `json:"prompt"`
|
||||
Duration int `json:"duration"`
|
||||
Seed int `json:"seed"`
|
||||
Resolution string `json:"resolution"`
|
||||
Bgm bool `json:"bgm"`
|
||||
MovementAmplitude string `json:"movement_amplitude"`
|
||||
Payload string `json:"payload"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
type taskResultResponse struct {
|
||||
State string `json:"state"`
|
||||
ErrCode string `json:"err_code"`
|
||||
Credits int `json:"credits"`
|
||||
Payload string `json:"payload"`
|
||||
Creations []creation `json:"creations"`
|
||||
}
|
||||
|
||||
type creation struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
CoverURL string `json:"cover_url"`
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Adaptor implementation
|
||||
// ============================
|
||||
|
||||
type TaskAdaptor struct {
|
||||
taskcommon.BaseBilling
|
||||
ChannelType int
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
|
||||
a.ChannelType = info.ChannelType
|
||||
a.baseURL = info.ChannelBaseUrl
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) *taskdto.TaskError {
|
||||
if err := relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionGenerate); err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := relaycommon.GetTaskRequest(c)
|
||||
if err != nil {
|
||||
return service.TaskErrorWrapper(err, "get_task_request_failed", http.StatusBadRequest)
|
||||
}
|
||||
action := constant.TaskActionTextGenerate
|
||||
if meatAction, ok := req.Metadata["action"]; ok {
|
||||
action, _ = meatAction.(string)
|
||||
} else if req.HasImage() {
|
||||
action = constant.TaskActionGenerate
|
||||
if info.ChannelType == constant.ChannelTypeVidu {
|
||||
// vidu 增加 首尾帧生视频和参考图生视频
|
||||
if len(req.Images) == 2 {
|
||||
action = constant.TaskActionFirstTailGenerate
|
||||
} else if len(req.Images) > 2 {
|
||||
action = constant.TaskActionReferenceGenerate
|
||||
}
|
||||
}
|
||||
}
|
||||
info.Action = action
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
|
||||
v, exists := c.Get("task_request")
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("request not found in context")
|
||||
}
|
||||
req := v.(relaycommon.TaskSubmitReq)
|
||||
|
||||
body, err := a.convertToRequestPayload(&req, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.Action == constant.TaskActionReferenceGenerate {
|
||||
if strings.Contains(body.Model, "viduq2") {
|
||||
// 参考图生视频只能用 viduq2 模型, 不能带有pro或turbo后缀 https://platform.vidu.cn/docs/reference-to-video
|
||||
body.Model = "viduq2"
|
||||
}
|
||||
}
|
||||
|
||||
data, err := common.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(data), nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
|
||||
var path string
|
||||
switch info.Action {
|
||||
case constant.TaskActionGenerate:
|
||||
path = "/img2video"
|
||||
case constant.TaskActionFirstTailGenerate:
|
||||
path = "/start-end2video"
|
||||
case constant.TaskActionReferenceGenerate:
|
||||
path = "/reference2video"
|
||||
default:
|
||||
path = "/text2video"
|
||||
}
|
||||
return fmt.Sprintf("%s/ent/v2%s", a.baseURL, path), nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Token "+info.ApiKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
|
||||
return channel.DoTaskApiRequest(a, c, info, requestBody)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *taskdto.TaskError) {
|
||||
responseBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var vResp responsePayload
|
||||
err = common.Unmarshal(responseBody, &vResp)
|
||||
if err != nil {
|
||||
taskErr = service.TaskErrorWrapper(errors.Wrap(err, fmt.Sprintf("%s", responseBody)), "unmarshal_response_failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if vResp.State == "failed" {
|
||||
taskErr = service.TaskErrorWrapperLocal(fmt.Errorf("task failed"), "task_failed", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
ov := dto.NewOpenAIVideo()
|
||||
ov.ID = info.PublicTaskID
|
||||
ov.TaskID = info.PublicTaskID
|
||||
ov.CreatedAt = time.Now().Unix()
|
||||
ov.Model = info.OriginModelName
|
||||
c.JSON(http.StatusOK, ov)
|
||||
return vResp.TaskId, responseBody, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
|
||||
taskID, ok := body["task_id"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid task_id")
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/ent/v2/tasks/%s/creations", baseUrl, taskID)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Token "+key)
|
||||
|
||||
client, err := service.GetHttpClientWithProxy(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("new proxy http client failed: %w", err)
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetModelList() []string {
|
||||
return []string{"viduq2", "viduq1", "vidu2.0", "vidu1.5"}
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) GetChannelName() string {
|
||||
return "vidu"
|
||||
}
|
||||
|
||||
// ============================
|
||||
// helpers
|
||||
// ============================
|
||||
|
||||
func (a *TaskAdaptor) convertToRequestPayload(req *relaycommon.TaskSubmitReq, info *relaycommon.RelayInfo) (*requestPayload, error) {
|
||||
r := requestPayload{
|
||||
Model: taskcommon.DefaultString(info.UpstreamModelName, "viduq1"),
|
||||
Images: req.Images,
|
||||
Prompt: req.Prompt,
|
||||
Duration: taskcommon.DefaultInt(req.Duration, 5),
|
||||
Resolution: taskcommon.DefaultString(req.Size, "1080p"),
|
||||
MovementAmplitude: "auto",
|
||||
Bgm: false,
|
||||
}
|
||||
if err := taskcommon.UnmarshalMetadata(req.Metadata, &r); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal metadata failed")
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
|
||||
taskInfo := &relaycommon.TaskInfo{}
|
||||
|
||||
var taskResp taskResultResponse
|
||||
err := common.Unmarshal(respBody, &taskResp)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to unmarshal response body")
|
||||
}
|
||||
|
||||
state := taskResp.State
|
||||
switch state {
|
||||
case "created", "queueing":
|
||||
taskInfo.Status = model.TaskStatusSubmitted
|
||||
case "processing":
|
||||
taskInfo.Status = model.TaskStatusInProgress
|
||||
case "success":
|
||||
taskInfo.Status = model.TaskStatusSuccess
|
||||
if len(taskResp.Creations) > 0 {
|
||||
taskInfo.Url = taskResp.Creations[0].URL
|
||||
}
|
||||
case "failed":
|
||||
taskInfo.Status = model.TaskStatusFailure
|
||||
if taskResp.ErrCode != "" {
|
||||
taskInfo.Reason = taskResp.ErrCode
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown task state: %s", state)
|
||||
}
|
||||
|
||||
return taskInfo, nil
|
||||
}
|
||||
|
||||
func (a *TaskAdaptor) ConvertToOpenAIVideo(originTask *model.Task) ([]byte, error) {
|
||||
var viduResp taskResultResponse
|
||||
if err := common.Unmarshal(originTask.Data, &viduResp); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal vidu task data failed")
|
||||
}
|
||||
|
||||
openAIVideo := dto.NewOpenAIVideo()
|
||||
openAIVideo.ID = originTask.TaskID
|
||||
openAIVideo.Status = originTask.Status.ToVideoStatus()
|
||||
openAIVideo.SetProgressStr(originTask.Progress)
|
||||
openAIVideo.CreatedAt = originTask.CreatedAt
|
||||
openAIVideo.CompletedAt = originTask.UpdatedAt
|
||||
|
||||
if len(viduResp.Creations) > 0 && viduResp.Creations[0].URL != "" {
|
||||
openAIVideo.SetMetadata("url", viduResp.Creations[0].URL)
|
||||
}
|
||||
|
||||
if viduResp.State == "failed" && viduResp.ErrCode != "" {
|
||||
openAIVideo.Error = &dto.OpenAIVideoError{
|
||||
Message: viduResp.ErrCode,
|
||||
Code: viduResp.ErrCode,
|
||||
}
|
||||
}
|
||||
|
||||
return common.Marshal(openAIVideo)
|
||||
}
|
||||
Reference in New Issue
Block a user