完善文档页文本向量与重排序调用支持
This commit is contained in:
@@ -151,6 +151,107 @@ func TestOpenAIClientChatContract(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClientEmbeddingsContract(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotModel string
|
||||
var gotDimensions float64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
gotModel, _ = body["model"].(string)
|
||||
gotDimensions, _ = body["dimensions"].(float64)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"id": "embd-test",
|
||||
"object": "list",
|
||||
"model": gotModel,
|
||||
"data": []any{map[string]any{
|
||||
"object": "embedding",
|
||||
"index": 0,
|
||||
"embedding": []any{0.1, 0.2, 0.3},
|
||||
}},
|
||||
"usage": map[string]any{"prompt_tokens": 3, "total_tokens": 3},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
response, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||
Kind: "embeddings",
|
||||
Model: "aliyun-bailian-openai:text-embedding-v4",
|
||||
Body: map[string]any{
|
||||
"model": "aliyun-bailian-openai:text-embedding-v4",
|
||||
"input": []any{"hello"},
|
||||
"dimensions": 3,
|
||||
},
|
||||
Candidate: store.RuntimeModelCandidate{
|
||||
BaseURL: server.URL,
|
||||
ProviderModelName: "text-embedding-v4",
|
||||
Credentials: map[string]any{"apiKey": "test-key"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run embeddings client: %v", err)
|
||||
}
|
||||
if gotPath != "/embeddings" || gotModel != "text-embedding-v4" || gotDimensions != 3 {
|
||||
t.Fatalf("unexpected embeddings request path=%s model=%s dimensions=%v", gotPath, gotModel, gotDimensions)
|
||||
}
|
||||
if response.Usage.InputTokens != 3 || response.Usage.TotalTokens != 3 || response.Result["id"] != "embd-test" {
|
||||
t.Fatalf("unexpected embeddings response: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClientAliyunRerankUsesCompatibleAPIBase(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotModel string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
gotModel, _ = body["model"].(string)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"id": "rerank-test",
|
||||
"object": "list",
|
||||
"model": gotModel,
|
||||
"results": []any{
|
||||
map[string]any{"index": 0, "relevance_score": 0.93},
|
||||
map[string]any{"index": 2, "relevance_score": 0.34},
|
||||
},
|
||||
"usage": map[string]any{"total_tokens": 9},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
response, err := (OpenAIClient{HTTPClient: server.Client()}).Run(context.Background(), Request{
|
||||
Kind: "reranks",
|
||||
Model: "aliyun-bailian-openai:qwen3-rerank",
|
||||
Body: map[string]any{
|
||||
"model": "aliyun-bailian-openai:qwen3-rerank",
|
||||
"query": "what is rerank",
|
||||
"documents": []any{"rerank sorts documents", "unrelated"},
|
||||
"top_n": 2,
|
||||
},
|
||||
Candidate: store.RuntimeModelCandidate{
|
||||
Provider: "aliyun-bailian-openai",
|
||||
BaseURL: server.URL + "/compatible-mode/v1",
|
||||
ProviderModelName: "qwen3-rerank",
|
||||
Credentials: map[string]any{"apiKey": "test-key"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run rerank client: %v", err)
|
||||
}
|
||||
if gotPath != "/compatible-api/v1/reranks" || gotModel != "qwen3-rerank" {
|
||||
t.Fatalf("unexpected rerank request path=%s model=%s", gotPath, gotModel)
|
||||
}
|
||||
if response.Usage.TotalTokens != 9 || response.Result["id"] != "rerank-test" {
|
||||
t.Fatalf("unexpected rerank response: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIClientChatRequestNormalizesToolContext(t *testing.T) {
|
||||
var captured map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
type OpenAIClient struct {
|
||||
@@ -27,10 +29,10 @@ func (c OpenAIClient) Run(ctx context.Context, request Request) (Response, error
|
||||
body = NormalizeChatCompletionRequestBody(body)
|
||||
}
|
||||
body["model"] = upstreamModelName(request.Candidate)
|
||||
stream := request.Stream || boolValue(body, "stream")
|
||||
stream := openAIEndpointSupportsStream(request.Kind) && (request.Stream || boolValue(body, "stream"))
|
||||
ensureOpenAIStreamUsage(body, request.Kind, stream)
|
||||
raw, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(request.Candidate.BaseURL, endpoint), bytes.NewReader(raw))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(openAIBaseURL(request.Kind, request.Candidate), endpoint), bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return Response{}, err
|
||||
}
|
||||
@@ -81,6 +83,10 @@ func openAIEndpoint(kind string) string {
|
||||
return "/chat/completions"
|
||||
case "responses":
|
||||
return "/responses"
|
||||
case "embeddings":
|
||||
return "/embeddings"
|
||||
case "reranks":
|
||||
return "/reranks"
|
||||
case "images.generations":
|
||||
return "/images/generations"
|
||||
case "images.edits":
|
||||
@@ -90,6 +96,24 @@ func openAIEndpoint(kind string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func openAIEndpointSupportsStream(kind string) bool {
|
||||
return kind == "chat.completions" || kind == "responses"
|
||||
}
|
||||
|
||||
func openAIBaseURL(kind string, candidate store.RuntimeModelCandidate) string {
|
||||
base := strings.TrimSpace(candidate.BaseURL)
|
||||
if kind != "reranks" {
|
||||
return base
|
||||
}
|
||||
if strings.Contains(base, "/compatible-mode/") && (strings.EqualFold(candidate.Provider, "aliyun-bailian-openai") || strings.Contains(base, "dashscope")) {
|
||||
return strings.Replace(base, "/compatible-mode/", "/compatible-api/", 1)
|
||||
}
|
||||
if base == "" && strings.EqualFold(candidate.Provider, "aliyun-bailian-openai") {
|
||||
return "https://dashscope.aliyuncs.com/compatible-api/v1"
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func cloneBody(body map[string]any) map[string]any {
|
||||
out := map[string]any{}
|
||||
for key, value := range body {
|
||||
|
||||
@@ -131,6 +131,10 @@ func simulatedResult(request Request) map[string]any {
|
||||
"output_text": fmt.Sprintf("simulation response from %s", request.Candidate.Provider),
|
||||
"usage": map[string]any{"input_tokens": 12, "output_tokens": 8, "total_tokens": 20},
|
||||
}
|
||||
case "embeddings":
|
||||
return simulatedEmbeddingResult(request)
|
||||
case "reranks":
|
||||
return simulatedRerankResult(request)
|
||||
case "images.edits":
|
||||
return map[string]any{
|
||||
"id": "img-edit-simulated",
|
||||
@@ -172,6 +176,106 @@ func simulatedResult(request Request) map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
func simulatedEmbeddingResult(request Request) map[string]any {
|
||||
inputCount := simulatedEmbeddingInputCount(request.Body["input"])
|
||||
dimensions := intValue(request.Body, "dimensions", 3)
|
||||
if dimensions <= 0 {
|
||||
dimensions = 3
|
||||
}
|
||||
if dimensions > 2048 {
|
||||
dimensions = 2048
|
||||
}
|
||||
data := make([]any, 0, inputCount)
|
||||
for index := 0; index < inputCount; index += 1 {
|
||||
embedding := make([]any, 0, dimensions)
|
||||
for dimension := 0; dimension < dimensions; dimension += 1 {
|
||||
embedding = append(embedding, float64(index+1)/10+float64(dimension)/100)
|
||||
}
|
||||
data = append(data, map[string]any{
|
||||
"object": "embedding",
|
||||
"index": index,
|
||||
"embedding": embedding,
|
||||
})
|
||||
}
|
||||
usage := simulatedUsage(request)
|
||||
return map[string]any{
|
||||
"id": "embd-simulated",
|
||||
"object": "list",
|
||||
"model": request.Model,
|
||||
"data": data,
|
||||
"usage": map[string]any{"prompt_tokens": usage.InputTokens, "total_tokens": usage.TotalTokens},
|
||||
}
|
||||
}
|
||||
|
||||
func simulatedEmbeddingInputCount(value any) int {
|
||||
switch typed := value.(type) {
|
||||
case []any:
|
||||
if len(typed) > 0 {
|
||||
return len(typed)
|
||||
}
|
||||
case []string:
|
||||
if len(typed) > 0 {
|
||||
return len(typed)
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func simulatedRerankResult(request Request) map[string]any {
|
||||
documents := simulatedRerankDocuments(request.Body["documents"])
|
||||
topN := intValue(request.Body, "top_n", len(documents))
|
||||
if topN <= 0 || topN > len(documents) {
|
||||
topN = len(documents)
|
||||
}
|
||||
results := make([]any, 0, topN)
|
||||
for index := 0; index < topN; index += 1 {
|
||||
score := 0.95 - float64(index)*0.1
|
||||
if score < 0 {
|
||||
score = 0
|
||||
}
|
||||
result := map[string]any{
|
||||
"index": index,
|
||||
"relevance_score": score,
|
||||
}
|
||||
if boolValue(request.Body, "return_documents") {
|
||||
result["document"] = map[string]any{"text": documents[index]}
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
usage := simulatedUsage(request)
|
||||
return map[string]any{
|
||||
"id": "rerank-simulated",
|
||||
"object": "list",
|
||||
"model": request.Model,
|
||||
"results": results,
|
||||
"usage": map[string]any{"total_tokens": usage.TotalTokens},
|
||||
}
|
||||
}
|
||||
|
||||
func simulatedRerankDocuments(value any) []string {
|
||||
switch typed := value.(type) {
|
||||
case []any:
|
||||
out := make([]string, 0, len(typed))
|
||||
for _, item := range typed {
|
||||
text := stringValue(map[string]any{"value": item}, "value")
|
||||
if text == "" {
|
||||
if record, ok := item.(map[string]any); ok {
|
||||
text = firstNonEmptyString(stringValue(record, "text"), stringValue(record, "content"))
|
||||
}
|
||||
}
|
||||
out = append(out, text)
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out
|
||||
}
|
||||
case []string:
|
||||
if len(typed) > 0 {
|
||||
return typed
|
||||
}
|
||||
}
|
||||
return []string{"simulated document"}
|
||||
}
|
||||
|
||||
func simulatedImageData(request Request, url string, fallbackPrompt string) []any {
|
||||
count := simulatedOutputCount(request.Body)
|
||||
items := make([]any, 0, count)
|
||||
@@ -207,6 +311,9 @@ func simulatedUsage(request Request) Usage {
|
||||
if request.ModelType == "chat" || request.ModelType == "text_generate" || request.Kind == "responses" {
|
||||
return Usage{InputTokens: 12, OutputTokens: 8, TotalTokens: 20}
|
||||
}
|
||||
if request.ModelType == "text_embedding" || request.ModelType == "text_rerank" || request.Kind == "embeddings" || request.Kind == "reranks" {
|
||||
return Usage{InputTokens: 16, TotalTokens: 16}
|
||||
}
|
||||
return Usage{}
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,33 @@ func TestPlanTaskResponseTreatsAPIV1ChatCompletionsAsSynchronousCompatibleRespon
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanTaskResponseTreatsAPIV1EmbeddingAndRerankAsSynchronousCompatibleResponse(t *testing.T) {
|
||||
for _, item := range []struct {
|
||||
kind string
|
||||
path string
|
||||
}{
|
||||
{kind: "embeddings", path: "/api/v1/embeddings"},
|
||||
{kind: "reranks", path: "/api/v1/reranks"},
|
||||
} {
|
||||
t.Run(item.kind, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, item.path, nil)
|
||||
req.Header.Set("X-Async", "true")
|
||||
|
||||
plan := planTaskResponse(item.kind, false, map[string]any{"stream": true}, req)
|
||||
|
||||
if plan.asyncMode {
|
||||
t.Fatalf("%s must not enter async task mode", item.path)
|
||||
}
|
||||
if !plan.compatibleMode {
|
||||
t.Fatalf("%s should return compatible response payloads", item.path)
|
||||
}
|
||||
if plan.streamMode {
|
||||
t.Fatal("embedding and rerank endpoints should stay JSON-only even when stream=true is present")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanTaskResponseKeepsAsyncTaskModeForOtherAPIV1Tasks(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/images/generations", nil)
|
||||
req.Header.Set("X-Async", "true")
|
||||
|
||||
@@ -876,6 +876,8 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque
|
||||
// @Failure 429 {object} ErrorEnvelope
|
||||
// @Failure 502 {object} ErrorEnvelope
|
||||
// @Router /api/v1/responses [post]
|
||||
// @Router /api/v1/embeddings [post]
|
||||
// @Router /api/v1/reranks [post]
|
||||
// @Router /api/v1/images/generations [post]
|
||||
// @Router /api/v1/images/edits [post]
|
||||
// @Router /api/v1/videos/generations [post]
|
||||
@@ -883,6 +885,10 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque
|
||||
// @Router /v1/chat/completions [post]
|
||||
// @Router /responses [post]
|
||||
// @Router /v1/responses [post]
|
||||
// @Router /embeddings [post]
|
||||
// @Router /v1/embeddings [post]
|
||||
// @Router /reranks [post]
|
||||
// @Router /v1/reranks [post]
|
||||
// @Router /images/generations [post]
|
||||
// @Router /v1/images/generations [post]
|
||||
// @Router /images/edits [post]
|
||||
@@ -1085,17 +1091,25 @@ type taskResponsePlan struct {
|
||||
func planTaskResponse(kind string, compatible bool, body map[string]any, r *http.Request) taskResponsePlan {
|
||||
asyncMode := asyncRequest(r)
|
||||
compatibleMode := compatible
|
||||
if kind == "chat.completions" && !compatible {
|
||||
if synchronousCompatibleKind(kind) && !compatible {
|
||||
asyncMode = false
|
||||
compatibleMode = true
|
||||
}
|
||||
return taskResponsePlan{
|
||||
asyncMode: asyncMode,
|
||||
compatibleMode: compatibleMode,
|
||||
streamMode: boolValue(body, "stream"),
|
||||
streamMode: streamCompatibleKind(kind) && boolValue(body, "stream"),
|
||||
}
|
||||
}
|
||||
|
||||
func synchronousCompatibleKind(kind string) bool {
|
||||
return kind == "chat.completions" || kind == "embeddings" || kind == "reranks"
|
||||
}
|
||||
|
||||
func streamCompatibleKind(kind string) bool {
|
||||
return kind == "chat.completions" || kind == "responses"
|
||||
}
|
||||
|
||||
func writeTaskAccepted(w http.ResponseWriter, task store.GatewayTask) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"taskId": task.ID,
|
||||
@@ -1120,6 +1134,12 @@ func apiKeyScopeAllowed(user *auth.User, kind string) bool {
|
||||
if required == "chat" && (scope == "text" || scope == "text_generate") {
|
||||
return true
|
||||
}
|
||||
if required == "embedding" && scope == "text_embedding" {
|
||||
return true
|
||||
}
|
||||
if required == "rerank" && scope == "text_rerank" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1128,6 +1148,10 @@ func scopeForTaskKind(kind string) string {
|
||||
switch kind {
|
||||
case "chat.completions", "responses":
|
||||
return "chat"
|
||||
case "embeddings":
|
||||
return "embedding"
|
||||
case "reranks":
|
||||
return "rerank"
|
||||
case "images.generations", "images.edits":
|
||||
return "image"
|
||||
case "videos.generations":
|
||||
|
||||
@@ -1025,6 +1025,7 @@ func modelCatalogCapabilityDefinitions() []ModelCatalogFilterOption {
|
||||
{Value: "text_to_speech", Label: "语音合成"},
|
||||
{Value: "audio_understanding", Label: "音频理解"},
|
||||
{Value: "text_embedding", Label: "Embedding"},
|
||||
{Value: "text_rerank", Label: "重排序"},
|
||||
{Value: "omni", Label: "全模态"},
|
||||
{Value: "omni_video", Label: "全模态视频"},
|
||||
{Value: "multimodal", Label: "多模态"},
|
||||
@@ -1120,6 +1121,8 @@ func canonicalCapabilityFilterValue(value string) string {
|
||||
switch normalized {
|
||||
case "embedding":
|
||||
return "text_embedding"
|
||||
case "rerank", "reranks":
|
||||
return "text_rerank"
|
||||
case "model":
|
||||
return "model_3d"
|
||||
default:
|
||||
@@ -1143,6 +1146,8 @@ func capabilityFilterValueForTag(tag string) string {
|
||||
return "structured_output"
|
||||
case "数字人":
|
||||
return "digital_human"
|
||||
case "重排序":
|
||||
return "text_rerank"
|
||||
case "3D 模型":
|
||||
return "model_3d"
|
||||
case "文生 3D":
|
||||
@@ -1165,6 +1170,9 @@ func capabilityLabel(value string) string {
|
||||
"responses": "Responses",
|
||||
"text_embedding": "Embedding",
|
||||
"embedding": "Embedding",
|
||||
"text_rerank": "重排序",
|
||||
"rerank": "重排序",
|
||||
"reranks": "重排序",
|
||||
"image_generate": "图像生成",
|
||||
"image_edit": "图像编辑",
|
||||
"image_analysis": "图像分析",
|
||||
|
||||
@@ -128,6 +128,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
|
||||
mux.Handle("GET /api/admin/runtime/model-rate-limits", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listModelRateLimitStatuses)))
|
||||
mux.Handle("POST /api/v1/chat/completions", server.auth.Require(auth.PermissionBasic, server.createAPIV1ChatCompletions()))
|
||||
mux.Handle("POST /api/v1/responses", server.auth.Require(auth.PermissionBasic, server.createTask("responses", false)))
|
||||
mux.Handle("POST /api/v1/embeddings", server.auth.Require(auth.PermissionBasic, server.createTask("embeddings", false)))
|
||||
mux.Handle("POST /api/v1/reranks", server.auth.Require(auth.PermissionBasic, server.createTask("reranks", false)))
|
||||
mux.Handle("POST /api/v1/images/generations", server.auth.Require(auth.PermissionBasic, server.createTask("images.generations", false)))
|
||||
mux.Handle("POST /api/v1/images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", false)))
|
||||
mux.Handle("POST /api/v1/videos/generations", server.auth.Require(auth.PermissionBasic, server.createTask("videos.generations", false)))
|
||||
@@ -140,6 +142,10 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
|
||||
mux.Handle("POST /v1/chat/completions", server.auth.Require(auth.PermissionBasic, server.createTask("chat.completions", true)))
|
||||
mux.Handle("POST /responses", server.auth.Require(auth.PermissionBasic, server.createTask("responses", true)))
|
||||
mux.Handle("POST /v1/responses", server.auth.Require(auth.PermissionBasic, server.createTask("responses", true)))
|
||||
mux.Handle("POST /embeddings", server.auth.Require(auth.PermissionBasic, server.createTask("embeddings", true)))
|
||||
mux.Handle("POST /v1/embeddings", server.auth.Require(auth.PermissionBasic, server.createTask("embeddings", true)))
|
||||
mux.Handle("POST /reranks", server.auth.Require(auth.PermissionBasic, server.createTask("reranks", true)))
|
||||
mux.Handle("POST /v1/reranks", server.auth.Require(auth.PermissionBasic, server.createTask("reranks", true)))
|
||||
mux.Handle("POST /images/generations", server.auth.Require(auth.PermissionBasic, server.createTask("images.generations", true)))
|
||||
mux.Handle("POST /v1/images/generations", server.auth.Require(auth.PermissionBasic, server.createTask("images.generations", true)))
|
||||
mux.Handle("POST /images/edits", server.auth.Require(auth.PermissionBasic, server.createTask("images.edits", true)))
|
||||
|
||||
@@ -159,16 +159,14 @@ func hasRules(policy map[string]any) bool {
|
||||
}
|
||||
|
||||
func estimateRequestTokens(body map[string]any) int {
|
||||
text := ""
|
||||
if prompt := stringFromMap(body, "prompt"); prompt != "" {
|
||||
text += prompt
|
||||
}
|
||||
if input := stringFromMap(body, "input"); input != "" {
|
||||
text += input
|
||||
}
|
||||
var text strings.Builder
|
||||
appendTokenEstimateText(&text, body["prompt"])
|
||||
appendTokenEstimateText(&text, body["input"])
|
||||
appendTokenEstimateText(&text, body["query"])
|
||||
appendTokenEstimateText(&text, body["documents"])
|
||||
for _, item := range contentItems(body["content"]) {
|
||||
if stringFromAny(item["type"]) == "text" {
|
||||
text += stringFromAny(item["text"])
|
||||
appendTokenEstimateText(&text, item["text"])
|
||||
}
|
||||
}
|
||||
if messages, ok := body["messages"].([]any); ok {
|
||||
@@ -176,19 +174,41 @@ func estimateRequestTokens(body map[string]any) int {
|
||||
message, _ := raw.(map[string]any)
|
||||
switch content := message["content"].(type) {
|
||||
case string:
|
||||
text += content
|
||||
appendTokenEstimateText(&text, content)
|
||||
case []any:
|
||||
for _, rawPart := range content {
|
||||
part, _ := rawPart.(map[string]any)
|
||||
text += stringFromMap(part, "text")
|
||||
appendTokenEstimateText(&text, part["text"])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if text == "" {
|
||||
estimatedText := text.String()
|
||||
if estimatedText == "" {
|
||||
return 1
|
||||
}
|
||||
return len([]rune(text))/4 + 1
|
||||
return len([]rune(estimatedText))/4 + 1
|
||||
}
|
||||
|
||||
func appendTokenEstimateText(out *strings.Builder, value any) {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
out.WriteString(typed)
|
||||
case []string:
|
||||
for _, item := range typed {
|
||||
out.WriteString(item)
|
||||
}
|
||||
case []any:
|
||||
for _, item := range typed {
|
||||
appendTokenEstimateText(out, item)
|
||||
}
|
||||
case map[string]any:
|
||||
for _, key := range []string{"text", "content", "query", "document"} {
|
||||
if text := stringFromAny(typed[key]); text != "" {
|
||||
out.WriteString(text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func tokenUsageAmounts(usage clients.Usage) map[string]float64 {
|
||||
|
||||
@@ -40,8 +40,11 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body
|
||||
}
|
||||
|
||||
func (s *Service) estimatedBillings(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate) []any {
|
||||
usage := clients.Usage{InputTokens: estimateRequestTokens(body), OutputTokens: int(floatFromAny(body["max_tokens"]))}
|
||||
if usage.OutputTokens == 0 {
|
||||
usage := clients.Usage{InputTokens: estimateRequestTokens(body)}
|
||||
if isTextGenerationKind(kind) {
|
||||
usage.OutputTokens = int(floatFromAny(body["max_tokens"]))
|
||||
}
|
||||
if isTextGenerationKind(kind) && usage.OutputTokens == 0 {
|
||||
usage.OutputTokens = 64
|
||||
}
|
||||
usage.TotalTokens = usage.InputTokens + usage.OutputTokens
|
||||
@@ -56,19 +59,25 @@ func (s *Service) estimatedBillings(ctx context.Context, user *auth.User, kind s
|
||||
func (s *Service) billings(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate, response clients.Response, simulated bool) []any {
|
||||
config := s.effectiveBillingConfig(ctx, candidate)
|
||||
discount := effectiveDiscount(ctx, s.store, user, candidate)
|
||||
if isTextGenerationKind(kind) {
|
||||
if isTextBillingKind(kind) {
|
||||
inputTokens := response.Usage.InputTokens
|
||||
outputTokens := response.Usage.OutputTokens
|
||||
if isTextInputOnlyKind(kind) && inputTokens == 0 && response.Usage.TotalTokens > 0 {
|
||||
inputTokens = response.Usage.TotalTokens
|
||||
}
|
||||
if inputTokens == 0 && outputTokens == 0 {
|
||||
inputTokens = estimateRequestTokens(body)
|
||||
outputTokens = 1
|
||||
if isTextGenerationKind(kind) {
|
||||
outputTokens = 1
|
||||
}
|
||||
}
|
||||
inputAmount := roundPrice(float64(inputTokens) / 1000 * resourcePrice(config, "text", "textInputPer1k", "inputTokenPrice", "basePrice") * discount)
|
||||
outputAmount := roundPrice(float64(outputTokens) / 1000 * resourcePrice(config, "text", "textOutputPer1k", "outputTokenPrice", "basePrice") * discount)
|
||||
return []any{
|
||||
billingLine(candidate, "text_input", "1k_tokens", inputTokens, inputAmount, discount, simulated),
|
||||
billingLine(candidate, "text_output", "1k_tokens", outputTokens, outputAmount, discount, simulated),
|
||||
lines := []any{billingLine(candidate, "text_input", "1k_tokens", inputTokens, inputAmount, discount, simulated)}
|
||||
if isTextGenerationKind(kind) {
|
||||
outputAmount := roundPrice(float64(outputTokens) / 1000 * resourcePrice(config, "text", "textOutputPer1k", "outputTokenPrice", "basePrice") * discount)
|
||||
lines = append(lines, billingLine(candidate, "text_output", "1k_tokens", outputTokens, outputAmount, discount, simulated))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
count := requestOutputCount(body)
|
||||
resource := "image"
|
||||
|
||||
@@ -698,7 +698,7 @@ func (s *Service) recordTaskParameterPreprocessing(ctx context.Context, taskID s
|
||||
|
||||
func skipTaskParameterPreprocessingLog(modelType string) bool {
|
||||
switch strings.TrimSpace(modelType) {
|
||||
case "text_generate", "chat", "responses", "text":
|
||||
case "text_generate", "text_embedding", "text_rerank", "chat", "responses", "text":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -923,6 +923,10 @@ func modelTypeFromKind(kind string, body map[string]any) string {
|
||||
switch kind {
|
||||
case "chat.completions", "responses":
|
||||
return "text_generate"
|
||||
case "embeddings":
|
||||
return "text_embedding"
|
||||
case "reranks":
|
||||
return "text_rerank"
|
||||
case "images.generations", "images.edits":
|
||||
if kind == "images.edits" {
|
||||
return "image_edit"
|
||||
@@ -943,7 +947,7 @@ func modelTypeFromKind(kind string, body map[string]any) string {
|
||||
|
||||
func requestedModelTypeFromBody(body map[string]any) string {
|
||||
for _, key := range []string{"modelType", "model_type", "capability", "capabilityType"} {
|
||||
value := strings.TrimSpace(stringFromMap(body, key))
|
||||
value := canonicalModelType(strings.TrimSpace(stringFromMap(body, key)))
|
||||
if isKnownModelType(value) {
|
||||
return value
|
||||
}
|
||||
@@ -951,9 +955,21 @@ func requestedModelTypeFromBody(body map[string]any) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func canonicalModelType(value string) string {
|
||||
normalized := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(value)), "-", "_")
|
||||
switch normalized {
|
||||
case "embedding":
|
||||
return "text_embedding"
|
||||
case "rerank", "reranks":
|
||||
return "text_rerank"
|
||||
default:
|
||||
return normalized
|
||||
}
|
||||
}
|
||||
|
||||
func isKnownModelType(value string) bool {
|
||||
switch value {
|
||||
case "text_generate", "image_generate", "image_edit", "video_generate", "image_to_video", "text_to_video", "video_edit", "video_reference", "video_first_last_frame", "omni_video", "omni":
|
||||
case "text_generate", "text_embedding", "text_rerank", "image_generate", "image_edit", "video_generate", "image_to_video", "text_to_video", "video_edit", "video_reference", "video_first_last_frame", "omni_video", "omni":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -1001,6 +1017,14 @@ func isTextGenerationKind(kind string) bool {
|
||||
return kind == "chat.completions" || kind == "responses"
|
||||
}
|
||||
|
||||
func isTextInputOnlyKind(kind string) bool {
|
||||
return kind == "embeddings" || kind == "reranks"
|
||||
}
|
||||
|
||||
func isTextBillingKind(kind string) bool {
|
||||
return isTextGenerationKind(kind) || isTextInputOnlyKind(kind)
|
||||
}
|
||||
|
||||
func isSimulation(task store.GatewayTask, candidate store.RuntimeModelCandidate) bool {
|
||||
if task.RunMode == "simulation" {
|
||||
return true
|
||||
@@ -1115,6 +1139,17 @@ func validateRequest(kind string, body map[string]any) error {
|
||||
if body["input"] == nil && body["messages"] == nil {
|
||||
return errors.New("input or messages is required")
|
||||
}
|
||||
case "embeddings":
|
||||
if body["input"] == nil {
|
||||
return errors.New("input is required")
|
||||
}
|
||||
case "reranks":
|
||||
if body["query"] == nil {
|
||||
return errors.New("query is required")
|
||||
}
|
||||
if !hasRerankDocuments(body["documents"]) {
|
||||
return errors.New("documents is required")
|
||||
}
|
||||
case "images.generations", "images.edits":
|
||||
if strings.TrimSpace(stringFromMap(body, "prompt")) == "" {
|
||||
return errors.New("prompt is required")
|
||||
@@ -1123,6 +1158,17 @@ func validateRequest(kind string, body map[string]any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasRerankDocuments(value any) bool {
|
||||
switch typed := value.(type) {
|
||||
case []any:
|
||||
return len(typed) > 0
|
||||
case []string:
|
||||
return len(typed) > 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func parameterPreprocessClientError(err error) *clients.ClientError {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
@@ -47,7 +47,7 @@ func billingResourcesForModelTypes(modelTypes []string) map[string]bool {
|
||||
resources := map[string]bool{}
|
||||
for _, modelType := range modelTypes {
|
||||
switch normalizeBillingType(modelType) {
|
||||
case "chat", "text", "responses", "text_generate", "text_embedding", "embedding",
|
||||
case "chat", "text", "responses", "text_generate", "text_embedding", "embedding", "text_rerank", "rerank",
|
||||
"image_analysis", "video_understanding", "audio_understanding", "omni", "tools_call":
|
||||
resources["text"] = true
|
||||
case "image", "images.generations", "image_generate":
|
||||
|
||||
Reference in New Issue
Block a user