233 lines
8.3 KiB
Go
233 lines
8.3 KiB
Go
package runner
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"strings"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
const (
|
|
volcesOutputTokenThreshold = 10240
|
|
volcesOutputCapabilityErrorCode = "model_capability_configuration_error"
|
|
volcesOutputInvalidParameterCode = "invalid_parameter"
|
|
)
|
|
|
|
type outputTokenLimitProcessor struct{}
|
|
|
|
func (outputTokenLimitProcessor) Name() string { return "OutputTokenLimitProcessor" }
|
|
|
|
func (outputTokenLimitProcessor) ShouldProcess(_ map[string]any, modelType string, context *paramProcessContext) bool {
|
|
return context != nil && isOpenAITextGenerationKind(context.kind) && isTextOutputModelType(modelType) && isVolcesCandidate(context.candidate)
|
|
}
|
|
|
|
func (outputTokenLimitProcessor) Process(params map[string]any, modelType string, context *paramProcessContext) bool {
|
|
modelMax, sourceType, capabilityValue, ok := candidateMaxOutputTokens(context.candidate, modelType)
|
|
path := capabilityPath(sourceType, "max_output_tokens")
|
|
if !ok {
|
|
return context.reject(
|
|
"OutputTokenLimitProcessor",
|
|
outputTokenParameter(context.kind),
|
|
nil,
|
|
"火山引擎文本候选未配置正整数 max_output_tokens，已禁止执行该候选。",
|
|
path,
|
|
capabilityValue,
|
|
)
|
|
}
|
|
|
|
if context.kind == "chat.completions" {
|
|
if value, explicit := nonNullParameter(params, "max_tokens"); explicit {
|
|
if parsed, valid := positiveInteger(value); !valid || parsed > modelMax {
|
|
return context.reject("OutputTokenLimitProcessor", "max_tokens", value, fmt.Sprintf("max_tokens must be a positive integer no greater than the selected Volcengine model limit (%d)", modelMax), path, capabilityValue)
|
|
}
|
|
return true
|
|
}
|
|
if _, explicit := nonNullParameter(params, "max_completion_tokens"); explicit {
|
|
return true
|
|
}
|
|
value := defaultVolcesOutputTokens(modelMax)
|
|
params["max_tokens"] = value
|
|
context.recordChange(
|
|
"OutputTokenLimitProcessor", "set", "max_tokens", nil, value,
|
|
fmt.Sprintf("火山候选未显式设置输出上限：floor(%d/3)=%d，阈值=%d，注入候选级默认值。", modelMax, modelMax/3, volcesOutputTokenThreshold),
|
|
path, capabilityValue,
|
|
)
|
|
return true
|
|
}
|
|
|
|
if value, explicit := nonNullParameter(params, "max_output_tokens"); explicit {
|
|
if parsed, valid := positiveInteger(value); !valid || parsed > modelMax {
|
|
return context.reject("OutputTokenLimitProcessor", "max_output_tokens", value, fmt.Sprintf("max_output_tokens must be a positive integer no greater than the selected Volcengine model limit (%d)", modelMax), path, capabilityValue)
|
|
}
|
|
return true
|
|
}
|
|
value := defaultVolcesOutputTokens(modelMax)
|
|
params["max_output_tokens"] = value
|
|
context.recordChange(
|
|
"OutputTokenLimitProcessor", "set", "max_output_tokens", nil, value,
|
|
fmt.Sprintf("火山候选未显式设置输出上限：floor(%d/3)=%d，阈值=%d，注入候选级默认值。", modelMax, modelMax/3, volcesOutputTokenThreshold),
|
|
path, capabilityValue,
|
|
)
|
|
return true
|
|
}
|
|
|
|
func filterRuntimeCandidatesByOutputTokens(kind string, requestedModel string, modelType string, body map[string]any, candidates []store.RuntimeModelCandidate) ([]store.RuntimeModelCandidate, map[string]any, error) {
|
|
if !isOpenAITextGenerationKind(kind) || !isTextOutputModelType(modelType) || len(candidates) == 0 {
|
|
return candidates, nil, nil
|
|
}
|
|
filtered := make([]store.RuntimeModelCandidate, 0, len(candidates))
|
|
rejected := make([]map[string]any, 0)
|
|
invalidExplicit := false
|
|
for _, candidate := range candidates {
|
|
if !isVolcesCandidate(candidate) {
|
|
filtered = append(filtered, candidate)
|
|
continue
|
|
}
|
|
modelMax, sourceType, raw, configured := candidateMaxOutputTokens(candidate, modelType)
|
|
detail := map[string]any{
|
|
"platformId": candidate.PlatformID, "platformKey": candidate.PlatformKey, "provider": candidate.Provider,
|
|
"platformModelId": candidate.PlatformModelID, "providerModelName": candidate.ProviderModelName,
|
|
"modelType": modelType, "capabilityPath": capabilityPath(sourceType, "max_output_tokens"), "capabilityValue": raw,
|
|
}
|
|
if !configured {
|
|
detail["reason"] = "max_output_tokens_missing"
|
|
rejected = append(rejected, detail)
|
|
continue
|
|
}
|
|
if parameter, value, explicit := explicitOutputTokenParameter(kind, body); explicit {
|
|
parsed, valid := positiveInteger(value)
|
|
if !valid || (parameter != "max_completion_tokens" && parsed > modelMax) {
|
|
detail["reason"] = "requested_output_tokens_exceed_capability"
|
|
detail["parameter"] = parameter
|
|
detail["requestedValue"] = value
|
|
invalidExplicit = true
|
|
rejected = append(rejected, detail)
|
|
continue
|
|
}
|
|
}
|
|
filtered = append(filtered, candidate)
|
|
}
|
|
if len(rejected) == 0 {
|
|
return filtered, nil, nil
|
|
}
|
|
summary := map[string]any{
|
|
"filter": "volces_output_token_limit", "kind": kind, "requestedModel": requestedModel, "modelType": modelType,
|
|
"candidateCount": len(candidates), "supportedCandidateCount": len(filtered), "filteredCandidateCount": len(rejected),
|
|
"threshold": volcesOutputTokenThreshold, "formula": "third=floor(modelMaxOutputTokens/3); default=third>=10240?third:modelMaxOutputTokens",
|
|
"rejectedCandidates": rejected,
|
|
}
|
|
if len(filtered) > 0 {
|
|
return filtered, summary, nil
|
|
}
|
|
code := volcesOutputCapabilityErrorCode
|
|
message := "所有火山引擎文本候选都缺少有效的 max_output_tokens 能力配置"
|
|
if invalidExplicit {
|
|
code = volcesOutputInvalidParameterCode
|
|
message = "请求的输出 token 上限超过所有可用火山引擎候选的模型能力"
|
|
}
|
|
summary["code"] = code
|
|
return nil, summary, &store.ModelCandidateUnavailableError{Code: code, Message: message, Details: summary}
|
|
}
|
|
|
|
func defaultVolcesOutputTokens(modelMax int) int {
|
|
third := modelMax / 3
|
|
if third >= volcesOutputTokenThreshold {
|
|
return third
|
|
}
|
|
return modelMax
|
|
}
|
|
|
|
func isVolcesCandidate(candidate store.RuntimeModelCandidate) bool {
|
|
provider := strings.ToLower(strings.TrimSpace(candidate.Provider))
|
|
baseURL := strings.ToLower(strings.TrimSpace(candidate.BaseURL))
|
|
return provider == "volces-openai" || strings.Contains(baseURL, "volces.com") || strings.Contains(baseURL, "byteplus.com")
|
|
}
|
|
|
|
func candidateMaxOutputTokens(candidate store.RuntimeModelCandidate, modelType string) (int, string, any, bool) {
|
|
capabilities := effectiveModelCapability(candidate)
|
|
seen := map[string]struct{}{}
|
|
for _, candidateType := range []string{candidate.ModelType, modelType, "text_generate"} {
|
|
candidateType = strings.TrimSpace(candidateType)
|
|
if candidateType == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[candidateType]; ok {
|
|
continue
|
|
}
|
|
seen[candidateType] = struct{}{}
|
|
capability := capabilityForType(capabilities, candidateType)
|
|
if capability == nil {
|
|
continue
|
|
}
|
|
raw, exists := capability["max_output_tokens"]
|
|
if !exists {
|
|
continue
|
|
}
|
|
value, ok := positiveInteger(raw)
|
|
return value, candidateType, raw, ok
|
|
}
|
|
return 0, firstNonEmptyString(candidate.ModelType, modelType, "text_generate"), nil, false
|
|
}
|
|
|
|
func positiveInteger(value any) (int, bool) {
|
|
number := floatFromAny(value)
|
|
if number <= 0 || math.Trunc(number) != number || number > float64(math.MaxInt) {
|
|
return 0, false
|
|
}
|
|
return int(number), true
|
|
}
|
|
|
|
func nonNullParameter(body map[string]any, key string) (any, bool) {
|
|
value, ok := body[key]
|
|
return value, ok && value != nil
|
|
}
|
|
|
|
func explicitOutputTokenParameter(kind string, body map[string]any) (string, any, bool) {
|
|
if kind == "responses" {
|
|
value, ok := nonNullParameter(body, "max_output_tokens")
|
|
return "max_output_tokens", value, ok
|
|
}
|
|
if value, ok := nonNullParameter(body, "max_tokens"); ok {
|
|
return "max_tokens", value, true
|
|
}
|
|
value, ok := nonNullParameter(body, "max_completion_tokens")
|
|
return "max_completion_tokens", value, ok
|
|
}
|
|
|
|
func outputTokenParameter(kind string) string {
|
|
if kind == "responses" {
|
|
return "max_output_tokens"
|
|
}
|
|
return "max_tokens"
|
|
}
|
|
|
|
func isOpenAITextGenerationKind(kind string) bool {
|
|
return kind == "chat.completions" || kind == "responses"
|
|
}
|
|
|
|
func isTextOutputModelType(modelType string) bool {
|
|
switch strings.TrimSpace(modelType) {
|
|
case "", "text_generate", "chat", "responses", "text":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func mergeCandidateFilterSummaries(summaries ...map[string]any) map[string]any {
|
|
nonEmpty := make([]any, 0, len(summaries))
|
|
for _, summary := range summaries {
|
|
if len(summary) > 0 {
|
|
nonEmpty = append(nonEmpty, summary)
|
|
}
|
|
}
|
|
if len(nonEmpty) == 0 {
|
|
return nil
|
|
}
|
|
if len(nonEmpty) == 1 {
|
|
return nonEmpty[0].(map[string]any)
|
|
}
|
|
return map[string]any{"filters": nonEmpty}
|
|
}
|