BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 支持 MiniMax 音色克隆和 2.8 语音模型

Browse Source
This commit is contained in:
wangbo
2026-06-17 02:13:21 +08:00
parent 02ba5d3cdd
commit c4341335d7
20 changed files with 1645 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/httpapi/handlers.go
+12
View File
@@ -962,6 +962,7 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque
// @Router /api/v1/song/generations [post]
// @Router /api/v1/music/generations [post]
// @Router /api/v1/speech/generations [post]
// @Router /api/v1/voice_clone [post]
// @Router /chat/completions [post]
// @Router /v1/chat/completions [post]
// @Router /responses [post]
@@ -980,6 +981,8 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque
// @Router /v1/music/generations [post]
// @Router /speech/generations [post]
// @Router /v1/speech/generations [post]
// @Router /voice_clone [post]
// @Router /v1/voice_clone [post]
func (s *Server) createTask(kind string, compatible bool) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, ok := auth.UserFromContext(r.Context())
@@ -1250,6 +1253,9 @@ func apiKeyScopeAllowed(user *auth.User, kind string) bool {
if required == "audio" && (scope == "text_to_speech" || scope == "speech" || scope == "tts") {
return true
}
if required == "voice_clone" && (scope == "audio" || scope == "text_to_speech" || scope == "speech" || scope == "tts") {
return true
}
}
return false
}
@@ -1291,6 +1297,8 @@ func scopeForTaskKind(kind string) string {
return "music"
case "speech.generations":
return "audio"
case "voice.clone":
return "voice_clone"
default:
return kind
}
@@ -1298,6 +1306,10 @@ func scopeForTaskKind(kind string) string {
func statusFromRunError(err error) int {
switch {
case clients.ErrorCode(err) == "bad_request" || clients.ErrorCode(err) == "cloned_voice_expired" || clients.ErrorCode(err) == "cloned_voice_unavailable":
return http.StatusBadRequest
case clients.ErrorCode(err) == "cloned_voice_not_found":
return http.StatusNotFound
case store.ModelCandidateErrorCode(err) == "platform_cooling_down" || store.ModelCandidateErrorCode(err) == "model_cooling_down":
return http.StatusTooManyRequests
case errors.Is(err, store.ErrNoModelCandidate):
apps/api/internal/httpapi/model_catalog.go
+2
View File
@@ -1023,6 +1023,7 @@ func modelCatalogCapabilityDefinitions() []ModelCatalogFilterOption {
{Value: "video_understanding", Label: "视频理解"},
{Value: "audio_generate", Label: "音频生成"},
{Value: "text_to_speech", Label: "语音合成"},
{Value: "voice_clone", Label: "音色克隆"},
{Value: "audio_understanding", Label: "音频理解"},
{Value: "text_embedding", Label: "Embedding"},
{Value: "text_rerank", Label: "重排序"},
@@ -1183,6 +1184,7 @@ func capabilityLabel(value string) string {
"video_understanding": "视频理解",
"audio_generate": "音频生成",
"text_to_speech": "语音合成",
"voice_clone": "音色克隆",
"audio_understanding": "音频理解",
"tools_call": "工具调用",
"omni": "全模态",
apps/api/internal/httpapi/server.go
+6
View File
@@ -143,6 +143,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("POST /api/v1/song/generations", server.auth.Require(auth.PermissionBasic, server.createTask("song.generations", true)))
mux.Handle("POST /api/v1/music/generations", server.auth.Require(auth.PermissionBasic, server.createTask("music.generations", true)))
mux.Handle("POST /api/v1/speech/generations", server.auth.Require(auth.PermissionBasic, server.createTask("speech.generations", true)))
mux.Handle("POST /api/v1/voice_clone", server.auth.Require(auth.PermissionBasic, server.createTask("voice.clone", true)))
mux.Handle("GET /api/v1/voice_clone/voices", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.listClonedVoices)))
mux.Handle("POST /api/v1/files/upload", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.uploadFile)))
mux.Handle("GET /api/v1/tasks", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.listTasks)))
mux.Handle("GET /api/v1/tasks/{taskID}", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.getTask)))
@@ -172,6 +174,10 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("POST /v1/music/generations", server.auth.Require(auth.PermissionBasic, server.createTask("music.generations", true)))
mux.Handle("POST /speech/generations", server.auth.Require(auth.PermissionBasic, server.createTask("speech.generations", true)))
mux.Handle("POST /v1/speech/generations", server.auth.Require(auth.PermissionBasic, server.createTask("speech.generations", true)))
mux.Handle("POST /voice_clone", server.auth.Require(auth.PermissionBasic, server.createTask("voice.clone", true)))
mux.Handle("POST /v1/voice_clone", server.auth.Require(auth.PermissionBasic, server.createTask("voice.clone", true)))
mux.Handle("GET /voice_clone/voices", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.listClonedVoices)))
mux.Handle("GET /v1/voice_clone/voices", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.listClonedVoices)))
mux.Handle("POST /v1/files/upload", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.uploadFile)))
mux.Handle("POST /v1/tasks/{taskID}/cancel", server.auth.Require(auth.PermissionBasic, http.HandlerFunc(server.cancelTask)))
apps/api/internal/httpapi/task_multipart.go
+198 -3
View File
@@ -7,6 +7,7 @@ import (
"mime"
"mime/multipart"
"net/http"
"path/filepath"
"strconv"
"strings"
@@ -16,13 +17,18 @@ import (
const multipartTaskMemoryBytes = 32 << 20
type imageEditMultipartAssetUploader func(context.Context, string, *multipart.FileHeader) (map[string]any, error)
type voiceCloneMultipartAssetUploader func(context.Context, string, *multipart.FileHeader) (map[string]any, error)
func (s *Server) decodeTaskRequestBody(ctx context.Context, w http.ResponseWriter, r *http.Request, kind string) (map[string]any, error) {
if requestIsMultipartForm(r) {
if kind != "images.edits" {
return nil, &clients.ClientError{Code: "unsupported_multipart_body", Message: "multipart/form-data is only supported for image edit tasks", Retryable: false}
switch kind {
case "images.edits":
return s.decodeImageEditMultipartBody(ctx, w, r)
case "voice.clone":
return s.decodeVoiceCloneMultipartBody(ctx, w, r)
default:
return nil, &clients.ClientError{Code: "unsupported_multipart_body", Message: "multipart/form-data is only supported for image edit and voice clone tasks", Retryable: false}
}
return s.decodeImageEditMultipartBody(ctx, w, r)
}
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
@@ -259,6 +265,195 @@ func (s *Server) uploadImageEditMultipartAsset(ctx context.Context, field string
return requestAssetWrapper(ref), nil
}
func (s *Server) decodeVoiceCloneMultipartBody(ctx context.Context, w http.ResponseWriter, r *http.Request) (map[string]any, error) {
r.Body = http.MaxBytesReader(w, r.Body, maxGatewayUploadBytes)
if err := r.ParseMultipartForm(multipartTaskMemoryBytes); err != nil {
return nil, &clients.ClientError{Code: "invalid_multipart_body", Message: "invalid multipart form-data body", Retryable: false}
}
if r.MultipartForm == nil {
return map[string]any{}, nil
}
defer r.MultipartForm.RemoveAll()
return voiceCloneMultipartFormBody(ctx, r.MultipartForm, s.uploadVoiceCloneMultipartAsset)
}
func voiceCloneMultipartFormBody(ctx context.Context, form *multipart.Form, upload voiceCloneMultipartAssetUploader) (map[string]any, error) {
body := map[string]any{}
if form == nil {
return body, nil
}
for key, values := range form.Value {
addVoiceCloneMultipartFieldValues(body, key, values)
}
if upload == nil {
return body, nil
}
if err := addVoiceCloneMultipartFiles(ctx, body, form.File, upload); err != nil {
return nil, err
}
return body, nil
}
func addVoiceCloneMultipartFieldValues(body map[string]any, rawKey string, values []string) {
key := normalizeVoiceCloneMultipartFieldName(rawKey)
parsed := make([]any, 0, len(values))
for _, value := range values {
if strings.TrimSpace(value) == "" {
continue
}
parsed = append(parsed, parseVoiceCloneMultipartFieldValue(key, value))
}
if len(parsed) == 0 {
return
}
if len(parsed) == 1 {
body[key] = parsed[0]
return
}
body[key] = parsed
}
func normalizeVoiceCloneMultipartFieldName(key string) string {
switch strings.TrimSpace(key) {
case "voiceId":
return "voice_id"
case "audioUrl":
return "audio_url"
case "promptAudioUrl":
return "prompt_audio_url"
case "promptText":
return "prompt_text"
case "previewModel":
return "preview_model"
case "textValidation":
return "text_validation"
case "languageBoost":
return "language_boost"
case "needNoiseReduction":
return "need_noise_reduction"
case "needVolumeNormalization":
return "need_volume_normalization"
case "aigcWatermark":
return "aigc_watermark"
case "fileId":
return "file_id"
case "promptFileId":
return "prompt_file_id"
case "displayName":
return "display_name"
default:
return strings.TrimSpace(key)
}
}
func parseVoiceCloneMultipartFieldValue(key string, value string) any {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
if parsed, ok := parseImageEditMultipartJSONValue(trimmed); ok {
return parsed
}
switch key {
case "need_noise_reduction", "need_volume_normalization", "aigc_watermark":
if parsed, err := strconv.ParseBool(trimmed); err == nil {
return parsed
}
case "file_id", "prompt_file_id":
if parsed, err := strconv.ParseInt(trimmed, 10, 64); err == nil {
return parsed
}
case "accuracy":
if parsed, err := strconv.ParseFloat(trimmed, 64); err == nil {
return parsed
}
}
return trimmed
}
func addVoiceCloneMultipartFiles(ctx context.Context, body map[string]any, files map[string][]*multipart.FileHeader, upload voiceCloneMultipartAssetUploader) error {
sourceFiles := collectVoiceCloneMultipartFiles(files, "file", "audio", "source_audio", "sourceAudio")
if len(sourceFiles) > 0 {
value, err := upload(ctx, "audio", sourceFiles[0])
if err != nil {
return err
}
body["audio"] = value
}
promptFiles := collectVoiceCloneMultipartFiles(files, "prompt_audio", "promptAudio")
if len(promptFiles) > 0 {
value, err := upload(ctx, "prompt_audio", promptFiles[0])
if err != nil {
return err
}
body["prompt_audio"] = value
}
return nil
}
func collectVoiceCloneMultipartFiles(files map[string][]*multipart.FileHeader, keys ...string) []*multipart.FileHeader {
out := make([]*multipart.FileHeader, 0)
for _, key := range keys {
out = append(out, files[key]...)
}
return out
}
func (s *Server) uploadVoiceCloneMultipartAsset(ctx context.Context, field string, header *multipart.FileHeader) (map[string]any, error) {
file, err := header.Open()
if err != nil {
return nil, &clients.ClientError{Code: "invalid_multipart_file", Message: err.Error(), Retryable: false}
}
defer file.Close()
payload, err := io.ReadAll(file)
if err != nil {
return nil, &clients.ClientError{Code: "invalid_multipart_file", Message: err.Error(), Retryable: false}
}
contentType := strings.TrimSpace(header.Header.Get("Content-Type"))
detectedContentType := ""
if len(payload) > 0 {
detectedContentType = http.DetectContentType(payload)
}
if !voiceCloneMultipartAudioAllowed(contentType, detectedContentType, header.Filename) {
return nil, &clients.ClientError{Code: "invalid_multipart_audio", Message: "voice clone multipart files must be mp3, m4a, or wav audio", Retryable: false}
}
contentType = requestAssetContentType(contentType, payload, field, []string{field}, nil)
if !voiceCloneMultipartAudioAllowed(contentType, detectedContentType, header.Filename) {
contentType = voiceCloneContentTypeFromExtension(header.Filename)
}
ref, err := s.ensureRequestAsset(ctx, decodedRequestAsset{
Bytes: payload,
ContentType: contentType,
})
if err != nil {
return nil, err
}
return requestAssetWrapper(ref), nil
}
func voiceCloneMultipartAudioAllowed(contentType string, detectedContentType string, filename string) bool {
for _, value := range []string{contentType, detectedContentType} {
normalized := strings.ToLower(strings.TrimSpace(value))
if strings.HasPrefix(normalized, "audio/") {
return true
}
}
return voiceCloneContentTypeFromExtension(filename) != ""
}
func voiceCloneContentTypeFromExtension(filename string) string {
switch strings.ToLower(filepath.Ext(strings.TrimSpace(filename))) {
case ".mp3":
return "audio/mpeg"
case ".m4a":
return "audio/mp4"
case ".wav":
return "audio/wav"
default:
return ""
}
}
func appendImageEditMultipartList(body map[string]any, key string, values ...any) {
list := flattenImageEditMultipartValues([]any{body[key]})
list = append(list, flattenImageEditMultipartValues(values)...)
apps/api/internal/httpapi/voice_clone_handlers.go
+38
View File
@@ -0,0 +1,38 @@
package httpapi
import (
"net/http"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
)
// listClonedVoices godoc
// @Summary 列出当前用户克隆音色
// @Description 返回当前用户在网关中维护的克隆音色，以及克隆时绑定的平台与平台模型。
// @Tags voice-clone
// @Produce json
// @Security BearerAuth
// @Success 200 {object} map[string]any
// @Failure 401 {object} ErrorEnvelope
// @Failure 500 {object} ErrorEnvelope
// @Router /api/v1/voice_clone/voices [get]
// @Router /v1/voice_clone/voices [get]
// @Router /voice_clone/voices [get]
func (s *Server) listClonedVoices(w http.ResponseWriter, r *http.Request) {
user, ok := auth.UserFromContext(r.Context())
if !ok {
writeError(w, http.StatusUnauthorized, "unauthorized")
return
}
if !apiKeyScopeAllowed(user, "voice.clone") {
writeError(w, http.StatusForbidden, "api key scope does not allow this capability")
return
}
items, err := s.store.ListClonedVoices(r.Context(), user)
if err != nil {
s.logger.Error("list cloned voices failed", "error", err)
writeError(w, http.StatusInternalServerError, "list cloned voices failed")
return
}
writeJSON(w, http.StatusOK, map[string]any{"items": items})
}