easyai-ai-gateway/apps/api/internal/clients/openai_chat_audio.go

package clients

import (
	"net/http"
	"net/url"
	"strings"

	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)

var openAIChatAudioFormatByMIME = map[string]string{
	"audio/aac":   "aac",
	"audio/flac":  "flac",
	"audio/m4a":   "m4a",
	"audio/mp4":   "m4a",
	"audio/mpeg":  "mp3",
	"audio/ogg":   "ogg",
	"audio/opus":  "opus",
	"audio/wav":   "wav",
	"audio/webm":  "webm",
	"audio/x-m4a": "m4a",
	"audio/x-wav": "wav",
}

var openAIChatAudioFormatByExtension = map[string]string{
	"aac":  "aac",
	"flac": "flac",
	"m4a":  "m4a",
	"mp3":  "mp3",
	"mp4":  "m4a",
	"oga":  "ogg",
	"ogg":  "ogg",
	"opus": "opus",
	"wav":  "wav",
	"webm": "webm",
}

func normalizeOpenAIChatAudioParts(body map[string]any, candidate store.RuntimeModelCandidate) (map[string]any, error) {
	if !isAliyunBailianOpenAI(candidate) {
		return body, nil
	}
	messages, ok := body["messages"].([]any)
	if !ok {
		return body, nil
	}

	normalizedMessages := make([]any, 0, len(messages))
	for _, rawMessage := range messages {
		message, ok := rawMessage.(map[string]any)
		if !ok {
			normalizedMessages = append(normalizedMessages, rawMessage)
			continue
		}
		content, ok := message["content"].([]any)
		if !ok {
			normalizedMessages = append(normalizedMessages, rawMessage)
			continue
		}

		normalizedContent := make([]any, 0, len(content))
		for _, rawPart := range content {
			part, ok := rawPart.(map[string]any)
			if !ok {
				normalizedContent = append(normalizedContent, rawPart)
				continue
			}
			normalizedPart, err := normalizeAliyunOpenAIChatAudioPart(part)
			if err != nil {
				return nil, err
			}
			normalizedContent = append(normalizedContent, normalizedPart)
		}
		copiedMessage := cloneMapAny(message)
		copiedMessage["content"] = normalizedContent
		normalizedMessages = append(normalizedMessages, copiedMessage)
	}

	out := cloneMapAny(body)
	out["messages"] = normalizedMessages
	return out, nil
}

func normalizeAliyunOpenAIChatAudioPart(part map[string]any) (map[string]any, error) {
	switch strings.TrimSpace(stringFromAny(part["type"])) {
	case "audio_url":
		audioURL, _ := part["audio_url"].(map[string]any)
		data := strings.TrimSpace(stringFromAny(audioURL["url"]))
		format := resolveOpenAIChatAudioFormat(data, audioURL["format"], audioURL["mime_type"], audioURL["mimeType"])
		if data == "" || format == "" {
			return nil, invalidOpenAIChatAudioError()
		}
		return map[string]any{
			"type": "input_audio",
			"input_audio": map[string]any{
				"data":   data,
				"format": format,
			},
		}, nil
	case "input_audio":
		inputAudio, _ := part["input_audio"].(map[string]any)
		data := firstNonEmptyString(inputAudio["data"], inputAudio["url"])
		format := resolveOpenAIChatAudioFormat(data, inputAudio["format"], inputAudio["mime_type"], inputAudio["mimeType"])
		if data == "" || format == "" {
			return nil, invalidOpenAIChatAudioError()
		}
		copiedInput := cloneMapAny(inputAudio)
		copiedInput["data"] = data
		copiedInput["format"] = format
		delete(copiedInput, "url")
		copiedPart := cloneMapAny(part)
		copiedPart["input_audio"] = copiedInput
		return copiedPart, nil
	default:
		return part, nil
	}
}

func resolveOpenAIChatAudioFormat(source string, values ...any) string {
	if explicit := strings.ToLower(strings.TrimSpace(firstNonEmptyString(values...))); explicit != "" {
		if normalized := openAIChatAudioFormatByExtension[explicit]; normalized != "" {
			return normalized
		}
		if normalized := openAIChatAudioFormatByMIME[strings.TrimSpace(strings.Split(explicit, ";")[0])]; normalized != "" {
			return normalized
		}
		return explicit
	}
	if dataMIME := openAIChatAudioDataMIME(source); dataMIME != "" {
		if normalized := openAIChatAudioFormatByMIME[dataMIME]; normalized != "" {
			return normalized
		}
	}
	if extension := openAIChatAudioSourceExtension(source); extension != "" {
		return openAIChatAudioFormatByExtension[extension]
	}
	return ""
}

func openAIChatAudioDataMIME(source string) string {
	if !strings.HasPrefix(strings.ToLower(source), "data:") {
		return ""
	}
	header := strings.TrimPrefix(source[:strings.Index(source+",", ",")], "data:")
	return strings.ToLower(strings.TrimSpace(strings.Split(header, ";")[0]))
}

func openAIChatAudioSourceExtension(source string) string {
	if source == "" || strings.HasPrefix(strings.ToLower(source), "data:") {
		return ""
	}
	if parsed, err := url.Parse(source); err == nil && parsed.Path != "" {
		source = parsed.Path
	}
	source = strings.TrimSuffix(strings.Split(strings.Split(source, "?")[0], "#")[0], "/")
	if index := strings.LastIndex(source, "."); index >= 0 && index+1 < len(source) {
		return strings.ToLower(source[index+1:])
	}
	return ""
}

func invalidOpenAIChatAudioError() error {
	return &ClientError{
		Code:       "invalid_parameter",
		Message:    "input_audio requires data and a resolvable format",
		Param:      "messages.content.input_audio",
		StatusCode: http.StatusBadRequest,
		Retryable:  false,
	}
}