feat: add Responses API compatibility
This commit is contained in:
@@ -70,6 +70,15 @@ func taskMetrics(task store.GatewayTask, user *auth.User, body map[string]any, c
|
||||
if response.ResponseDurationMS > 0 {
|
||||
metrics["responseDurationMs"] = response.ResponseDurationMS
|
||||
}
|
||||
if response.UpstreamProtocol != "" {
|
||||
metrics["upstreamProtocol"] = response.UpstreamProtocol
|
||||
metrics["upstreamEndpoint"] = response.UpstreamEndpoint
|
||||
metrics["responseConverted"] = response.ResponseConverted
|
||||
metrics["publicResponseId"] = response.PublicResponseID
|
||||
metrics["upstreamResponseId"] = response.UpstreamResponseID
|
||||
metrics["parentResponseId"] = response.ParentResponseID
|
||||
metrics["responseChainDepth"] = response.ResponseChainDepth
|
||||
}
|
||||
switch task.Kind {
|
||||
case "chat.completions", "responses":
|
||||
metrics["stream"] = boolFromMap(body, "stream")
|
||||
@@ -119,6 +128,16 @@ func attemptMetrics(candidate store.RuntimeModelCandidate, attemptNo int, simula
|
||||
"loadAvoided": candidate.LoadAvoided,
|
||||
"simulated": simulated,
|
||||
}
|
||||
if candidate.ResponseProtocol != "" {
|
||||
metrics["upstreamProtocol"] = candidate.ResponseProtocol
|
||||
if candidate.ResponseProtocol == clients.ProtocolOpenAIResponses {
|
||||
metrics["upstreamEndpoint"] = "/responses"
|
||||
metrics["responseConverted"] = false
|
||||
} else if candidate.ResponseProtocol == clients.ProtocolOpenAIChatCompletions {
|
||||
metrics["upstreamEndpoint"] = "/chat/completions"
|
||||
metrics["responseConverted"] = true
|
||||
}
|
||||
}
|
||||
if candidate.LoadLimited {
|
||||
metrics["loadMetrics"] = map[string]any{
|
||||
"rpm": map[string]any{
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
responseChainMaxTTL = 30 * 24 * time.Hour
|
||||
responseChainMaxDepth = 100
|
||||
)
|
||||
|
||||
type responseExecutionContext struct {
|
||||
PublicResponseID string
|
||||
PublicPreviousResponseID string
|
||||
UpstreamPreviousResponseID string
|
||||
Protocol string
|
||||
Store bool
|
||||
PreviousChain *store.ResponseChain
|
||||
PreviousTurns []clients.ResponseTurn
|
||||
ChainDepth int
|
||||
}
|
||||
|
||||
func (s *Service) prepareResponseExecution(ctx context.Context, task store.GatewayTask, body map[string]any) (responseExecutionContext, error) {
|
||||
result := responseExecutionContext{
|
||||
PublicResponseID: gatewayResponseID(task.ID),
|
||||
Store: true,
|
||||
}
|
||||
if value, ok := body["store"].(bool); ok {
|
||||
result.Store = value
|
||||
}
|
||||
previousID := strings.TrimSpace(stringFromAny(body["previous_response_id"]))
|
||||
if previousID == "" {
|
||||
// No Gateway response chain was requested. The caller owns the complete
|
||||
// conversation state in input/messages, so it must be forwarded as-is.
|
||||
return result, nil
|
||||
}
|
||||
chain, err := s.store.GetResponseChain(ctx, previousID, task)
|
||||
if err != nil {
|
||||
return responseExecutionContext{}, &clients.ClientError{
|
||||
Code: "invalid_previous_response_id", Message: "previous_response_id is invalid or expired", StatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
if !sameResponseModel(chain.RequestedModel, task.Model) {
|
||||
return responseExecutionContext{}, &clients.ClientError{
|
||||
Code: "invalid_previous_response_id", Message: "previous_response_id belongs to a different model", StatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
history, historyErr := s.store.ListResponseChainHistory(ctx, chain)
|
||||
if historyErr != nil {
|
||||
if errors.Is(historyErr, store.ErrResponseChainTooDeep) {
|
||||
return responseExecutionContext{}, &clients.ClientError{Code: "response_chain_too_deep", Message: historyErr.Error(), StatusCode: http.StatusBadRequest}
|
||||
}
|
||||
return responseExecutionContext{}, historyErr
|
||||
}
|
||||
chainDepth, depthErr := responseHistoryDepth(history)
|
||||
if depthErr != nil {
|
||||
return responseExecutionContext{}, depthErr
|
||||
}
|
||||
result.PublicPreviousResponseID = chain.PublicResponseID
|
||||
result.UpstreamPreviousResponseID = chain.UpstreamResponseID
|
||||
result.Protocol = chain.UpstreamProtocol
|
||||
result.PreviousChain = &chain
|
||||
result.ChainDepth = chainDepth
|
||||
if chain.UpstreamProtocol == clients.ProtocolOpenAIChatCompletions {
|
||||
// Chat upstreams are stateless from the Responses API perspective. Only
|
||||
// this Gateway-managed mode replays stored turns; native Responses keeps
|
||||
// the provider's previous_response_id state authoritative.
|
||||
result.PreviousTurns = make([]clients.ResponseTurn, 0, len(history))
|
||||
for _, turn := range history {
|
||||
result.PreviousTurns = append(result.PreviousTurns, clients.ResponseTurn{
|
||||
Request: turn.RequestSnapshot, Response: turn.ResponseSnapshot, Internal: turn.InternalSnapshot,
|
||||
})
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func prepareResponseCandidates(candidates []store.RuntimeModelCandidate, execution responseExecutionContext) ([]store.RuntimeModelCandidate, error) {
|
||||
if execution.PreviousChain != nil {
|
||||
for _, candidate := range candidates {
|
||||
if candidate.PlatformModelID != execution.PreviousChain.PlatformModelID {
|
||||
continue
|
||||
}
|
||||
if !candidateSupportsProtocol(candidate, execution.PreviousChain.UpstreamProtocol) {
|
||||
break
|
||||
}
|
||||
candidate.ResponseProtocol = execution.PreviousChain.UpstreamProtocol
|
||||
return []store.RuntimeModelCandidate{candidate}, nil
|
||||
}
|
||||
return nil, responseChainUnavailableError()
|
||||
}
|
||||
|
||||
type indexedCandidate struct {
|
||||
candidate store.RuntimeModelCandidate
|
||||
index int
|
||||
group int
|
||||
}
|
||||
items := make([]indexedCandidate, 0, len(candidates))
|
||||
hasAnthropicOnly := false
|
||||
hasDeclaredUnsupported := false
|
||||
for index, candidate := range candidates {
|
||||
protocols := candidateSupportedProtocols(candidate)
|
||||
group := 1
|
||||
protocol := clients.ProtocolOpenAIChatCompletions
|
||||
if containsString(protocols, clients.ProtocolOpenAIResponses) {
|
||||
group = 0
|
||||
protocol = clients.ProtocolOpenAIResponses
|
||||
} else if !containsString(protocols, clients.ProtocolOpenAIChatCompletions) {
|
||||
hasDeclaredUnsupported = true
|
||||
if containsString(protocols, clients.ProtocolAnthropicMessages) {
|
||||
hasAnthropicOnly = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
candidate.ResponseProtocol = protocol
|
||||
items = append(items, indexedCandidate{candidate: candidate, index: index, group: group})
|
||||
}
|
||||
if len(items) == 0 {
|
||||
if hasAnthropicOnly {
|
||||
return nil, &clients.ClientError{Code: "unsupported_model_protocol", Message: "the selected model only supports Anthropic Messages; the public Anthropic adapter is not available", StatusCode: http.StatusBadRequest}
|
||||
}
|
||||
if hasDeclaredUnsupported {
|
||||
return nil, &clients.ClientError{Code: "unsupported_model_protocol", Message: "the selected model does not declare an OpenAI-compatible protocol", StatusCode: http.StatusBadRequest}
|
||||
}
|
||||
return nil, responseChainUnavailableError()
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
if items[i].group != items[j].group {
|
||||
return items[i].group < items[j].group
|
||||
}
|
||||
return items[i].index < items[j].index
|
||||
})
|
||||
out := make([]store.RuntimeModelCandidate, 0, len(items))
|
||||
for _, item := range items {
|
||||
out = append(out, item.candidate)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func candidateSupportedProtocols(candidate store.RuntimeModelCandidate) []string {
|
||||
capability := effectiveModelCapability(candidate)
|
||||
textCapability, _ := capability["text_generate"].(map[string]any)
|
||||
value, present := textCapability["supportedApiProtocols"]
|
||||
if !present {
|
||||
value, present = capability["supportedApiProtocols"]
|
||||
}
|
||||
if !present {
|
||||
return []string{clients.ProtocolOpenAIChatCompletions}
|
||||
}
|
||||
protocols := stringListFromAny(value)
|
||||
out := make([]string, 0, len(protocols))
|
||||
for _, protocol := range protocols {
|
||||
switch protocol {
|
||||
case clients.ProtocolOpenAIChatCompletions, clients.ProtocolOpenAIResponses, clients.ProtocolAnthropicMessages:
|
||||
out = append(out, protocol)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func candidateSupportsProtocol(candidate store.RuntimeModelCandidate, protocol string) bool {
|
||||
return containsString(candidateSupportedProtocols(candidate), protocol)
|
||||
}
|
||||
|
||||
func responseChainUnavailableError() error {
|
||||
return &clients.ClientError{
|
||||
Code: "response_chain_unavailable", Message: "the platform model pinned to this response chain is unavailable", StatusCode: http.StatusServiceUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
func gatewayResponseID(taskID string) string {
|
||||
return "resp_" + strings.ReplaceAll(strings.TrimSpace(taskID), "-", "")
|
||||
}
|
||||
|
||||
func (s *Service) persistResponseChain(ctx context.Context, task store.GatewayTask, body map[string]any, candidate store.RuntimeModelCandidate, execution responseExecutionContext, response clients.Response) error {
|
||||
if task.Kind != "responses" || !execution.Store {
|
||||
return nil
|
||||
}
|
||||
expiresAt := responseChainExpiry(response.Result, candidate.ResponseProtocol)
|
||||
publicResponseID := strings.TrimSpace(response.PublicResponseID)
|
||||
if publicResponseID == "" {
|
||||
publicResponseID = strings.TrimSpace(stringFromAny(response.Result["id"]))
|
||||
}
|
||||
if publicResponseID == "" {
|
||||
return &clients.ClientError{Code: "invalid_response", Message: "Responses result is missing id", StatusCode: http.StatusBadGateway}
|
||||
}
|
||||
return s.store.CreateResponseChain(ctx, store.CreateResponseChainInput{ResponseChain: store.ResponseChain{
|
||||
PublicResponseID: publicResponseID,
|
||||
ParentResponseID: execution.PublicPreviousResponseID,
|
||||
GatewayTaskID: task.ID,
|
||||
GatewayUserID: task.GatewayUserID,
|
||||
UserID: task.UserID,
|
||||
UserSource: task.UserSource,
|
||||
GatewayTenantID: task.GatewayTenantID,
|
||||
TenantID: task.TenantID,
|
||||
TenantKey: task.TenantKey,
|
||||
APIKeyID: task.APIKeyID,
|
||||
PlatformID: candidate.PlatformID,
|
||||
PlatformModelID: candidate.PlatformModelID,
|
||||
ClientID: candidate.ClientID,
|
||||
UpstreamProtocol: candidate.ResponseProtocol,
|
||||
UpstreamEndpoint: response.UpstreamEndpoint,
|
||||
UpstreamResponseID: response.UpstreamResponseID,
|
||||
RequestSnapshot: cloneMap(body),
|
||||
ResponseSnapshot: cloneMap(response.Result),
|
||||
InternalSnapshot: cloneMap(response.InternalResult),
|
||||
ExpiresAt: expiresAt,
|
||||
}})
|
||||
}
|
||||
|
||||
func sameResponseModel(previous string, current string) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(previous), strings.TrimSpace(current))
|
||||
}
|
||||
|
||||
func responseHistoryDepth(history []store.ResponseChain) (int, error) {
|
||||
if len(history) >= responseChainMaxDepth {
|
||||
return 0, &clients.ClientError{Code: "response_chain_too_deep", Message: "response chain exceeds the maximum depth", StatusCode: http.StatusBadRequest}
|
||||
}
|
||||
return len(history), nil
|
||||
}
|
||||
|
||||
func responseChainExpiry(result map[string]any, protocol string) time.Time {
|
||||
now := time.Now()
|
||||
maximum := now.Add(responseChainMaxTTL)
|
||||
if protocol != clients.ProtocolOpenAIResponses {
|
||||
return maximum
|
||||
}
|
||||
var parsed time.Time
|
||||
switch value := result["expire_at"].(type) {
|
||||
case float64:
|
||||
parsed = time.Unix(int64(value), 0)
|
||||
case int64:
|
||||
parsed = time.Unix(value, 0)
|
||||
case string:
|
||||
parsed, _ = time.Parse(time.RFC3339, value)
|
||||
}
|
||||
if parsed.IsZero() || parsed.After(maximum) {
|
||||
return maximum
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func responseExecutionFailure(err error) (string, string) {
|
||||
if errors.Is(err, store.ErrInvalidPreviousResponseID) {
|
||||
return "invalid_previous_response_id", "previous_response_id is invalid or expired"
|
||||
}
|
||||
return clients.ErrorCode(err), err.Error()
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestCandidateSupportedProtocolsDefaultsToChat(t *testing.T) {
|
||||
candidate := store.RuntimeModelCandidate{Capabilities: map[string]any{"text_generate": map[string]any{"supportTool": true}}}
|
||||
protocols := candidateSupportedProtocols(candidate)
|
||||
if len(protocols) != 1 || protocols[0] != clients.ProtocolOpenAIChatCompletions {
|
||||
t.Fatalf("legacy capability must default to Chat: %+v", protocols)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResponseCandidatesPrioritizesNativeAndKeepsGroupOrder(t *testing.T) {
|
||||
candidates := []store.RuntimeModelCandidate{
|
||||
{PlatformModelID: "chat-1", Capabilities: responseProtocolCapability(clients.ProtocolOpenAIChatCompletions)},
|
||||
{PlatformModelID: "native-1", Capabilities: responseProtocolCapability(clients.ProtocolOpenAIChatCompletions, clients.ProtocolOpenAIResponses)},
|
||||
{PlatformModelID: "native-2", Capabilities: responseProtocolCapability(clients.ProtocolOpenAIResponses)},
|
||||
{PlatformModelID: "chat-2", Capabilities: responseProtocolCapability(clients.ProtocolOpenAIChatCompletions)},
|
||||
}
|
||||
prepared, err := prepareResponseCandidates(candidates, responseExecutionContext{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := []string{prepared[0].PlatformModelID, prepared[1].PlatformModelID, prepared[2].PlatformModelID, prepared[3].PlatformModelID}
|
||||
want := []string{"native-1", "native-2", "chat-1", "chat-2"}
|
||||
for index := range want {
|
||||
if got[index] != want[index] {
|
||||
t.Fatalf("unexpected protocol ordering got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
if prepared[0].ResponseProtocol != clients.ProtocolOpenAIResponses || prepared[2].ResponseProtocol != clients.ProtocolOpenAIChatCompletions {
|
||||
t.Fatalf("protocol assignment mismatch: %+v", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResponseCandidatesPinsPreviousPlatformModelAndProtocol(t *testing.T) {
|
||||
chain := store.ResponseChain{PlatformModelID: "pinned", UpstreamProtocol: clients.ProtocolOpenAIChatCompletions}
|
||||
candidates := []store.RuntimeModelCandidate{
|
||||
{PlatformModelID: "other", Capabilities: responseProtocolCapability(clients.ProtocolOpenAIChatCompletions)},
|
||||
{PlatformModelID: "pinned", Capabilities: responseProtocolCapability(clients.ProtocolOpenAIChatCompletions, clients.ProtocolOpenAIResponses)},
|
||||
}
|
||||
prepared, err := prepareResponseCandidates(candidates, responseExecutionContext{PreviousChain: &chain})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(prepared) != 1 || prepared[0].PlatformModelID != "pinned" || prepared[0].ResponseProtocol != clients.ProtocolOpenAIChatCompletions {
|
||||
t.Fatalf("previous chain was not pinned: %+v", prepared)
|
||||
}
|
||||
|
||||
_, err = prepareResponseCandidates(candidates[:1], responseExecutionContext{PreviousChain: &chain})
|
||||
if clients.ErrorCode(err) != "response_chain_unavailable" {
|
||||
t.Fatalf("expected response_chain_unavailable, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResponseCandidatesRejectsAnthropicOnly(t *testing.T) {
|
||||
_, err := prepareResponseCandidates([]store.RuntimeModelCandidate{{Capabilities: responseProtocolCapability(clients.ProtocolAnthropicMessages)}}, responseExecutionContext{})
|
||||
if clients.ErrorCode(err) != "unsupported_model_protocol" {
|
||||
t.Fatalf("expected unsupported_model_protocol, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewayResponseIDAndExpiryPolicy(t *testing.T) {
|
||||
id := gatewayResponseID("01234567-89ab-cdef-0123-456789abcdef")
|
||||
if id != "resp_0123456789abcdef0123456789abcdef" {
|
||||
t.Fatalf("unexpected public response id: %s", id)
|
||||
}
|
||||
now := time.Now()
|
||||
chatExpiry := responseChainExpiry(map[string]any{}, clients.ProtocolOpenAIChatCompletions)
|
||||
if chatExpiry.Before(now.Add(responseChainMaxTTL-time.Minute)) || chatExpiry.After(now.Add(responseChainMaxTTL+time.Minute)) {
|
||||
t.Fatalf("Chat chain must use the 30-day retention policy: %v", chatExpiry)
|
||||
}
|
||||
upstreamExpiry := now.Add(24 * time.Hour).Unix()
|
||||
nativeExpiry := responseChainExpiry(map[string]any{"expire_at": float64(upstreamExpiry)}, clients.ProtocolOpenAIResponses)
|
||||
if nativeExpiry.Unix() != upstreamExpiry {
|
||||
t.Fatalf("native chain must honor an earlier upstream expiry: %v", nativeExpiry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseModelAndDepthValidation(t *testing.T) {
|
||||
if !sameResponseModel("DeepSeek-V4-Pro", "deepseek-v4-pro") || sameResponseModel("Qwen3.7-Plus", "DeepSeek-V4-Pro") {
|
||||
t.Fatal("response continuation model matching is not strict")
|
||||
}
|
||||
history := make([]store.ResponseChain, responseChainMaxDepth)
|
||||
if _, err := responseHistoryDepth(history); clients.ErrorCode(err) != "response_chain_too_deep" {
|
||||
t.Fatalf("expected response_chain_too_deep, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateProtocolOverrideAndExplicitEmpty(t *testing.T) {
|
||||
candidate := store.RuntimeModelCandidate{
|
||||
Capabilities: responseProtocolCapability(clients.ProtocolOpenAIChatCompletions, clients.ProtocolOpenAIResponses),
|
||||
CapabilityOverride: responseProtocolCapability(clients.ProtocolAnthropicMessages),
|
||||
}
|
||||
protocols := candidateSupportedProtocols(candidate)
|
||||
if len(protocols) != 1 || protocols[0] != clients.ProtocolAnthropicMessages {
|
||||
t.Fatalf("platform capability override was not applied: %v", protocols)
|
||||
}
|
||||
empty := store.RuntimeModelCandidate{Capabilities: responseProtocolCapability()}
|
||||
if protocols := candidateSupportedProtocols(empty); len(protocols) != 0 {
|
||||
t.Fatalf("an explicitly empty protocol list must not become legacy Chat: %v", protocols)
|
||||
}
|
||||
if _, err := prepareResponseCandidates([]store.RuntimeModelCandidate{empty}, responseExecutionContext{}); clients.ErrorCode(err) != "unsupported_model_protocol" {
|
||||
t.Fatalf("expected unsupported_model_protocol, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistResponseChainSkipsStoreFalse(t *testing.T) {
|
||||
service := &Service{}
|
||||
err := service.persistResponseChain(context.Background(), store.GatewayTask{Kind: "responses"}, nil, store.RuntimeModelCandidate{}, responseExecutionContext{Store: false}, clients.Response{})
|
||||
if err != nil {
|
||||
t.Fatalf("store:false must not require a response-chain store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func responseProtocolCapability(protocols ...string) map[string]any {
|
||||
items := make([]any, len(protocols))
|
||||
for index, protocol := range protocols {
|
||||
items[index] = protocol
|
||||
}
|
||||
return map[string]any{"text_generate": map[string]any{"supportedApiProtocols": items}}
|
||||
}
|
||||
@@ -93,6 +93,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
|
||||
return Result{}, err
|
||||
}
|
||||
body := normalizeRequest(task.Kind, restoredRequest)
|
||||
responseExecution := responseExecutionContext{}
|
||||
modelType := modelTypeFromKind(task.Kind, body)
|
||||
if err := s.store.MarkTaskRunning(ctx, task.ID, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil {
|
||||
return Result{}, err
|
||||
@@ -145,6 +146,18 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
if task.Kind == "responses" {
|
||||
responseExecution, err = s.prepareResponseExecution(ctx, task, body)
|
||||
if err != nil {
|
||||
code, message := responseExecutionFailure(err)
|
||||
s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_chain", Reason: code, ModelType: modelType})
|
||||
failed, finishErr := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", err)
|
||||
if finishErr != nil {
|
||||
return Result{}, finishErr
|
||||
}
|
||||
return Result{Task: failed, Output: failed.Result}, err
|
||||
}
|
||||
}
|
||||
runnerPolicy, err := s.store.GetActiveRunnerPolicy(ctx)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
@@ -156,6 +169,9 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
|
||||
CacheAffinityPolicy: runnerPolicy.CacheAffinityPolicy,
|
||||
})
|
||||
if err != nil {
|
||||
if task.Kind == "responses" && responseExecution.PreviousChain != nil {
|
||||
err = responseChainUnavailableError()
|
||||
}
|
||||
s.recordFailedAttempt(ctx, failedAttemptRecord{
|
||||
Task: task,
|
||||
Body: body,
|
||||
@@ -214,6 +230,18 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
|
||||
}
|
||||
return Result{Task: failed, Output: failed.Result}, err
|
||||
}
|
||||
if task.Kind == "responses" {
|
||||
candidates, err = prepareResponseCandidates(candidates, responseExecution)
|
||||
if err != nil {
|
||||
code, message := responseExecutionFailure(err)
|
||||
s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_protocol", Reason: code, ModelType: modelType})
|
||||
failed, finishErr := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", err)
|
||||
if finishErr != nil {
|
||||
return Result{}, finishErr
|
||||
}
|
||||
return Result{Task: failed, Output: failed.Result}, err
|
||||
}
|
||||
}
|
||||
firstCandidateBody := body
|
||||
normalizedModelType := modelType
|
||||
attemptNo := task.AttemptCount
|
||||
@@ -325,7 +353,7 @@ candidatesLoop:
|
||||
break candidatesLoop
|
||||
}
|
||||
candidateBody := preprocessing.Body
|
||||
response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record)
|
||||
response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record)
|
||||
if err == nil {
|
||||
attemptNo = nextAttemptNo
|
||||
billings := s.billings(ctx, user, task.Kind, candidateBody, candidate, response, isSimulation(task, candidate))
|
||||
@@ -536,7 +564,7 @@ candidatesLoop:
|
||||
return Result{Task: failed, Output: failed.Result}, lastErr
|
||||
}
|
||||
|
||||
func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) {
|
||||
func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) {
|
||||
simulated := isSimulation(task, candidate)
|
||||
baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing))
|
||||
reservations := s.rateLimitReservations(ctx, user, candidate, body)
|
||||
@@ -628,6 +656,12 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
return clients.Response{}, err
|
||||
}
|
||||
callStartedAt := time.Now()
|
||||
publicResponseID := ""
|
||||
publicPreviousResponseID := ""
|
||||
if task.Kind == "responses" && candidate.ResponseProtocol == clients.ProtocolOpenAIChatCompletions {
|
||||
publicResponseID = responseExecution.PublicResponseID
|
||||
publicPreviousResponseID = responseExecution.PublicPreviousResponseID
|
||||
}
|
||||
response, err := client.Run(ctx, clients.Request{
|
||||
Kind: task.Kind,
|
||||
ModelType: candidate.ModelType,
|
||||
@@ -643,8 +677,13 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
}
|
||||
return s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, attemptID, remoteTaskID, payload)
|
||||
},
|
||||
Stream: boolFromMap(providerBody, "stream"),
|
||||
StreamDelta: onDelta,
|
||||
Stream: boolFromMap(providerBody, "stream"),
|
||||
StreamDelta: onDelta,
|
||||
UpstreamProtocol: candidate.ResponseProtocol,
|
||||
PublicResponseID: publicResponseID,
|
||||
PublicPreviousResponseID: publicPreviousResponseID,
|
||||
UpstreamPreviousResponseID: responseExecution.UpstreamPreviousResponseID,
|
||||
PreviousResponseTurns: responseExecution.PreviousTurns,
|
||||
})
|
||||
callFinishedAt := time.Now()
|
||||
if response.ResponseStartedAt.IsZero() {
|
||||
@@ -715,6 +754,20 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
return clients.Response{}, err
|
||||
}
|
||||
response.Result = uploadedResult
|
||||
if task.Kind == "responses" {
|
||||
response.UpstreamProtocol = candidate.ResponseProtocol
|
||||
response.ParentResponseID = responseExecution.PublicPreviousResponseID
|
||||
response.ResponseChainDepth = responseExecution.ChainDepth
|
||||
if candidate.ResponseProtocol == clients.ProtocolOpenAIChatCompletions {
|
||||
response.Result["id"] = responseExecution.PublicResponseID
|
||||
response.PublicResponseID = responseExecution.PublicResponseID
|
||||
} else {
|
||||
response.PublicResponseID = strings.TrimSpace(stringFromAny(response.Result["id"]))
|
||||
}
|
||||
if err := s.persistResponseChain(ctx, task, body, candidate, responseExecution, response); err != nil {
|
||||
return clients.Response{}, fmt.Errorf("persist response chain: %w", err)
|
||||
}
|
||||
}
|
||||
if task.Kind == "voice.clone" {
|
||||
voice, err := s.persistVoiceCloneResult(ctx, task, user, candidate, attemptID, body, response.Result)
|
||||
if err != nil {
|
||||
@@ -1283,9 +1336,6 @@ func loadAvoidanceFallbackDecision(err error) failoverDecision {
|
||||
|
||||
func normalizeRequest(kind string, body map[string]any) map[string]any {
|
||||
out := cloneMap(body)
|
||||
if kind == "responses" && out["messages"] == nil && out["input"] != nil {
|
||||
out["messages"] = []any{map[string]any{"role": "user", "content": out["input"]}}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user