233 lines
8.3 KiB
Go
233 lines
8.3 KiB
Go
package runner
|
||
|
||
import (
|
||
"fmt"
|
||
"math"
|
||
"strings"
|
||
|
||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||
)
|
||
|
||
const (
|
||
volcesOutputTokenThreshold = 10240
|
||
volcesOutputCapabilityErrorCode = "model_capability_configuration_error"
|
||
volcesOutputInvalidParameterCode = "invalid_parameter"
|
||
)
|
||
|
||
type outputTokenLimitProcessor struct{}
|
||
|
||
func (outputTokenLimitProcessor) Name() string { return "OutputTokenLimitProcessor" }
|
||
|
||
func (outputTokenLimitProcessor) ShouldProcess(_ map[string]any, modelType string, context *paramProcessContext) bool {
|
||
return context != nil && isOpenAITextGenerationKind(context.kind) && isTextOutputModelType(modelType) && isVolcesCandidate(context.candidate)
|
||
}
|
||
|
||
func (outputTokenLimitProcessor) Process(params map[string]any, modelType string, context *paramProcessContext) bool {
|
||
modelMax, sourceType, capabilityValue, ok := candidateMaxOutputTokens(context.candidate, modelType)
|
||
path := capabilityPath(sourceType, "max_output_tokens")
|
||
if !ok {
|
||
return context.reject(
|
||
"OutputTokenLimitProcessor",
|
||
outputTokenParameter(context.kind),
|
||
nil,
|
||
"火山引擎文本候选未配置正整数 max_output_tokens，已禁止执行该候选。",
|
||
path,
|
||
capabilityValue,
|
||
)
|
||
}
|
||
|
||
if context.kind == "chat.completions" {
|
||
if value, explicit := nonNullParameter(params, "max_tokens"); explicit {
|
||
if parsed, valid := positiveInteger(value); !valid || parsed > modelMax {
|
||
return context.reject("OutputTokenLimitProcessor", "max_tokens", value, fmt.Sprintf("max_tokens must be a positive integer no greater than the selected Volcengine model limit (%d)", modelMax), path, capabilityValue)
|
||
}
|
||
return true
|
||
}
|
||
if _, explicit := nonNullParameter(params, "max_completion_tokens"); explicit {
|
||
return true
|
||
}
|
||
value := defaultVolcesOutputTokens(modelMax)
|
||
params["max_tokens"] = value
|
||
context.recordChange(
|
||
"OutputTokenLimitProcessor", "set", "max_tokens", nil, value,
|
||
fmt.Sprintf("火山候选未显式设置输出上限：floor(%d/3)=%d，阈值=%d，注入候选级默认值。", modelMax, modelMax/3, volcesOutputTokenThreshold),
|
||
path, capabilityValue,
|
||
)
|
||
return true
|
||
}
|
||
|
||
if value, explicit := nonNullParameter(params, "max_output_tokens"); explicit {
|
||
if parsed, valid := positiveInteger(value); !valid || parsed > modelMax {
|
||
return context.reject("OutputTokenLimitProcessor", "max_output_tokens", value, fmt.Sprintf("max_output_tokens must be a positive integer no greater than the selected Volcengine model limit (%d)", modelMax), path, capabilityValue)
|
||
}
|
||
return true
|
||
}
|
||
value := defaultVolcesOutputTokens(modelMax)
|
||
params["max_output_tokens"] = value
|
||
context.recordChange(
|
||
"OutputTokenLimitProcessor", "set", "max_output_tokens", nil, value,
|
||
fmt.Sprintf("火山候选未显式设置输出上限：floor(%d/3)=%d，阈值=%d，注入候选级默认值。", modelMax, modelMax/3, volcesOutputTokenThreshold),
|
||
path, capabilityValue,
|
||
)
|
||
return true
|
||
}
|
||
|
||
func filterRuntimeCandidatesByOutputTokens(kind string, requestedModel string, modelType string, body map[string]any, candidates []store.RuntimeModelCandidate) ([]store.RuntimeModelCandidate, map[string]any, error) {
|
||
if !isOpenAITextGenerationKind(kind) || !isTextOutputModelType(modelType) || len(candidates) == 0 {
|
||
return candidates, nil, nil
|
||
}
|
||
filtered := make([]store.RuntimeModelCandidate, 0, len(candidates))
|
||
rejected := make([]map[string]any, 0)
|
||
invalidExplicit := false
|
||
for _, candidate := range candidates {
|
||
if !isVolcesCandidate(candidate) {
|
||
filtered = append(filtered, candidate)
|
||
continue
|
||
}
|
||
modelMax, sourceType, raw, configured := candidateMaxOutputTokens(candidate, modelType)
|
||
detail := map[string]any{
|
||
"platformId": candidate.PlatformID, "platformKey": candidate.PlatformKey, "provider": candidate.Provider,
|
||
"platformModelId": candidate.PlatformModelID, "providerModelName": candidate.ProviderModelName,
|
||
"modelType": modelType, "capabilityPath": capabilityPath(sourceType, "max_output_tokens"), "capabilityValue": raw,
|
||
}
|
||
if !configured {
|
||
detail["reason"] = "max_output_tokens_missing"
|
||
rejected = append(rejected, detail)
|
||
continue
|
||
}
|
||
if parameter, value, explicit := explicitOutputTokenParameter(kind, body); explicit {
|
||
parsed, valid := positiveInteger(value)
|
||
if !valid || (parameter != "max_completion_tokens" && parsed > modelMax) {
|
||
detail["reason"] = "requested_output_tokens_exceed_capability"
|
||
detail["parameter"] = parameter
|
||
detail["requestedValue"] = value
|
||
invalidExplicit = true
|
||
rejected = append(rejected, detail)
|
||
continue
|
||
}
|
||
}
|
||
filtered = append(filtered, candidate)
|
||
}
|
||
if len(rejected) == 0 {
|
||
return filtered, nil, nil
|
||
}
|
||
summary := map[string]any{
|
||
"filter": "volces_output_token_limit", "kind": kind, "requestedModel": requestedModel, "modelType": modelType,
|
||
"candidateCount": len(candidates), "supportedCandidateCount": len(filtered), "filteredCandidateCount": len(rejected),
|
||
"threshold": volcesOutputTokenThreshold, "formula": "third=floor(modelMaxOutputTokens/3); default=third>=10240?third:modelMaxOutputTokens",
|
||
"rejectedCandidates": rejected,
|
||
}
|
||
if len(filtered) > 0 {
|
||
return filtered, summary, nil
|
||
}
|
||
code := volcesOutputCapabilityErrorCode
|
||
message := "所有火山引擎文本候选都缺少有效的 max_output_tokens 能力配置"
|
||
if invalidExplicit {
|
||
code = volcesOutputInvalidParameterCode
|
||
message = "请求的输出 token 上限超过所有可用火山引擎候选的模型能力"
|
||
}
|
||
summary["code"] = code
|
||
return nil, summary, &store.ModelCandidateUnavailableError{Code: code, Message: message, Details: summary}
|
||
}
|
||
|
||
func defaultVolcesOutputTokens(modelMax int) int {
|
||
third := modelMax / 3
|
||
if third >= volcesOutputTokenThreshold {
|
||
return third
|
||
}
|
||
return modelMax
|
||
}
|
||
|
||
func isVolcesCandidate(candidate store.RuntimeModelCandidate) bool {
|
||
provider := strings.ToLower(strings.TrimSpace(candidate.Provider))
|
||
baseURL := strings.ToLower(strings.TrimSpace(candidate.BaseURL))
|
||
return provider == "volces-openai" || strings.Contains(baseURL, "volces.com") || strings.Contains(baseURL, "byteplus.com")
|
||
}
|
||
|
||
func candidateMaxOutputTokens(candidate store.RuntimeModelCandidate, modelType string) (int, string, any, bool) {
|
||
capabilities := effectiveModelCapability(candidate)
|
||
seen := map[string]struct{}{}
|
||
for _, candidateType := range []string{candidate.ModelType, modelType, "text_generate"} {
|
||
candidateType = strings.TrimSpace(candidateType)
|
||
if candidateType == "" {
|
||
continue
|
||
}
|
||
if _, ok := seen[candidateType]; ok {
|
||
continue
|
||
}
|
||
seen[candidateType] = struct{}{}
|
||
capability := capabilityForType(capabilities, candidateType)
|
||
if capability == nil {
|
||
continue
|
||
}
|
||
raw, exists := capability["max_output_tokens"]
|
||
if !exists {
|
||
continue
|
||
}
|
||
value, ok := positiveInteger(raw)
|
||
return value, candidateType, raw, ok
|
||
}
|
||
return 0, firstNonEmptyString(candidate.ModelType, modelType, "text_generate"), nil, false
|
||
}
|
||
|
||
func positiveInteger(value any) (int, bool) {
|
||
number := floatFromAny(value)
|
||
if number <= 0 || math.Trunc(number) != number || number > float64(math.MaxInt) {
|
||
return 0, false
|
||
}
|
||
return int(number), true
|
||
}
|
||
|
||
func nonNullParameter(body map[string]any, key string) (any, bool) {
|
||
value, ok := body[key]
|
||
return value, ok && value != nil
|
||
}
|
||
|
||
func explicitOutputTokenParameter(kind string, body map[string]any) (string, any, bool) {
|
||
if kind == "responses" {
|
||
value, ok := nonNullParameter(body, "max_output_tokens")
|
||
return "max_output_tokens", value, ok
|
||
}
|
||
if value, ok := nonNullParameter(body, "max_tokens"); ok {
|
||
return "max_tokens", value, true
|
||
}
|
||
value, ok := nonNullParameter(body, "max_completion_tokens")
|
||
return "max_completion_tokens", value, ok
|
||
}
|
||
|
||
func outputTokenParameter(kind string) string {
|
||
if kind == "responses" {
|
||
return "max_output_tokens"
|
||
}
|
||
return "max_tokens"
|
||
}
|
||
|
||
func isOpenAITextGenerationKind(kind string) bool {
|
||
return kind == "chat.completions" || kind == "responses"
|
||
}
|
||
|
||
func isTextOutputModelType(modelType string) bool {
|
||
switch strings.TrimSpace(modelType) {
|
||
case "", "text_generate", "chat", "responses", "text":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func mergeCandidateFilterSummaries(summaries ...map[string]any) map[string]any {
|
||
nonEmpty := make([]any, 0, len(summaries))
|
||
for _, summary := range summaries {
|
||
if len(summary) > 0 {
|
||
nonEmpty = append(nonEmpty, summary)
|
||
}
|
||
}
|
||
if len(nonEmpty) == 0 {
|
||
return nil
|
||
}
|
||
if len(nonEmpty) == 1 {
|
||
return nonEmpty[0].(map[string]any)
|
||
}
|
||
return map[string]any{"filters": nonEmpty}
|
||
}
|