BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix gateway loopback validation chains

Browse Source
This commit is contained in:
wangbo
2026-05-11 08:48:02 +08:00
parent ff666b1ece
commit ca7e76e815
42 changed files with 1641 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/runner/limits.go
+25 -4
View File
@@ -18,15 +18,36 @@ func (s *Service) rateLimitReservations(ctx context.Context, user *auth.User, ca
}
func effectiveRateLimitPolicy(candidate store.RuntimeModelCandidate) map[string]any {
if hasRules(candidate.ModelRateLimitPolicy) {
return candidate.ModelRateLimitPolicy
policy := candidate.PlatformRateLimitPolicy
if hasRules(candidate.RuntimeRateLimitPolicy) {
policy = mergeMap(policy, candidate.RuntimeRateLimitPolicy)
}
if hasRules(candidate.PlatformRateLimitPolicy) {
return candidate.PlatformRateLimitPolicy
if nested, ok := candidate.RuntimePolicyOverride["rateLimitPolicy"].(map[string]any); ok && len(nested) > 0 {
policy = mergeMap(policy, nested)
}
if hasRules(candidate.ModelRateLimitPolicy) {
policy = mergeMap(policy, candidate.ModelRateLimitPolicy)
}
if hasRules(policy) {
return policy
}
return nil
}
func effectiveRetryPolicy(candidate store.RuntimeModelCandidate) map[string]any {
policy := candidate.PlatformRetryPolicy
if len(candidate.RuntimeRetryPolicy) > 0 {
policy = mergeMap(policy, candidate.RuntimeRetryPolicy)
}
if nested, ok := candidate.RuntimePolicyOverride["retryPolicy"].(map[string]any); ok && len(nested) > 0 {
policy = mergeMap(policy, nested)
}
if len(candidate.ModelRetryPolicy) > 0 {
policy = mergeMap(policy, candidate.ModelRetryPolicy)
}
return policy
}
func reservationsFromPolicy(scopeType string, scopeKey string, policy map[string]any, body map[string]any) []store.RateLimitReservation {
if scopeKey == "" || !hasRules(policy) {
return nil
apps/api/internal/runner/pricing.go
+13 -3
View File
@@ -16,7 +16,7 @@ type EstimateResult struct {
}
func (s *Service) Estimate(ctx context.Context, kind string, model string, body map[string]any, user *auth.User) (EstimateResult, error) {
candidates, err := s.store.ListModelCandidates(ctx, model, modelTypeFromKind(kind), user)
candidates, err := s.store.ListModelCandidates(ctx, model, modelTypeFromKind(kind, body), user)
if err != nil {
return EstimateResult{}, err
}
@@ -38,7 +38,7 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body
}
func (s *Service) billings(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate, response clients.Response, simulated bool) []any {
config := effectiveBillingConfig(candidate)
config := s.effectiveBillingConfig(ctx, candidate)
discount := effectiveDiscount(ctx, s.store, user, candidate)
if isTextGenerationKind(kind) {
inputTokens := response.Usage.InputTokens
@@ -74,11 +74,21 @@ func (s *Service) billings(ctx context.Context, user *auth.User, kind string, bo
return []any{billingLine(candidate, resource, unit, count, roundPrice(amount), discount, simulated)}
}
func effectiveBillingConfig(candidate store.RuntimeModelCandidate) map[string]any {
func (s *Service) effectiveBillingConfig(ctx context.Context, candidate store.RuntimeModelCandidate) map[string]any {
base := candidate.BaseBillingConfig
if ruleSetID := firstNonEmptyString(candidate.BasePricingRuleSetID, candidate.PlatformPricingRuleSetID); ruleSetID != "" {
if ruleSetConfig, err := s.store.PricingRuleSetBillingConfig(ctx, ruleSetID); err == nil && len(ruleSetConfig) > 0 {
base = ruleSetConfig
}
}
if len(candidate.BillingConfig) > 0 {
base = candidate.BillingConfig
}
if candidate.ModelPricingRuleSetID != "" {
if ruleSetConfig, err := s.store.PricingRuleSetBillingConfig(ctx, candidate.ModelPricingRuleSetID); err == nil && len(ruleSetConfig) > 0 {
base = ruleSetConfig
}
}
if len(candidate.BillingConfigOverride) > 0 {
base = mergeMap(base, candidate.BillingConfigOverride)
}
apps/api/internal/runner/runtime_policy.go
+84
View File
@@ -0,0 +1,84 @@
package runner
import (
"context"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func (s *Service) applyCandidateFailurePolicies(ctx context.Context, taskID string, candidate store.RuntimeModelCandidate, cause error, simulated bool) {
code := clients.ErrorCode(cause)
message := ""
if cause != nil {
message = cause.Error()
}
autoDisablePolicy := effectiveRuntimePolicy(candidate.AutoDisablePolicy, candidate.RuntimePolicyOverride, "autoDisablePolicy")
if failurePolicyMatches(autoDisablePolicy, code, message) && intFromPolicy(autoDisablePolicy, "threshold") <= 1 {
if err := s.store.DisableCandidatePlatform(ctx, candidate.PlatformID); err == nil {
_ = s.emit(ctx, taskID, "task.policy.auto_disabled", "running", "auto_disable", 0.48, "candidate platform disabled by failure policy", map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"code": code,
}, simulated)
}
}
degradePolicy := effectiveRuntimePolicy(candidate.DegradePolicy, candidate.RuntimePolicyOverride, "degradePolicy")
if failurePolicyMatches(degradePolicy, code, message) {
cooldownSeconds := intFromPolicy(degradePolicy, "cooldownSeconds")
if err := s.store.CooldownCandidatePlatform(ctx, candidate.PlatformID, cooldownSeconds); err == nil {
_ = s.emit(ctx, taskID, "task.policy.degraded", "running", "degrade", 0.5, "candidate platform cooled down by failure policy", map[string]any{
"platformId": candidate.PlatformID,
"platformModelId": candidate.PlatformModelID,
"cooldownSeconds": cooldownSeconds,
"code": code,
}, simulated)
}
}
}
func effectiveRuntimePolicy(base map[string]any, override map[string]any, key string) map[string]any {
policy := base
if nested, ok := override[key].(map[string]any); ok && len(nested) > 0 {
policy = mergeMap(policy, nested)
}
return policy
}
func failurePolicyMatches(policy map[string]any, code string, message string) bool {
if len(policy) == 0 || !boolFromMap(policy, "enabled") {
return false
}
keywords := stringListFromPolicy(policy, "keywords")
if len(keywords) == 0 {
return false
}
target := strings.ToLower(strings.TrimSpace(code + " " + message))
for _, keyword := range keywords {
keyword = strings.ToLower(strings.TrimSpace(keyword))
if keyword != "" && strings.Contains(target, keyword) {
return true
}
}
return false
}
func stringListFromPolicy(values map[string]any, key string) []string {
raw, ok := values[key].([]any)
if !ok {
if typed, ok := values[key].([]string); ok {
return typed
}
return nil
}
out := make([]string, 0, len(raw))
for _, item := range raw {
if text, ok := item.(string); ok && strings.TrimSpace(text) != "" {
out = append(out, text)
}
}
return out
}
apps/api/internal/runner/service.go
+35 -10
View File
@@ -50,8 +50,8 @@ func (s *Service) ExecuteStream(ctx context.Context, task store.GatewayTask, use
}
func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
modelType := modelTypeFromKind(task.Kind)
body := normalizeRequest(task.Kind, task.Request)
modelType := modelTypeFromKind(task.Kind, body)
if err := validateRequest(task.Kind, body); err != nil {
failed, finishErr := s.failTask(ctx, task.ID, "bad_request", err.Error(), task.RunMode == "simulation", err)
if finishErr != nil {
@@ -102,6 +102,17 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
if finishErr != nil {
return Result{}, finishErr
}
if settleErr := s.store.SettleTaskBilling(ctx, finished); settleErr != nil {
return Result{}, settleErr
}
if finished.FinalChargeAmount > 0 {
if err := s.emit(ctx, task.ID, "task.billing.settled", "succeeded", "billing", 0.98, "task billing settled", map[string]any{
"amount": finished.FinalChargeAmount,
"currency": stringFromAny(record.BillingSummary["currency"]),
}, isSimulation(task, candidate)); err != nil {
return Result{}, err
}
}
if err := s.emit(ctx, task.ID, "task.completed", "succeeded", "completed", 1, "task completed", map[string]any{
"result": response.Result,
"billings": billings,
@@ -226,6 +237,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
ErrorMessage: err.Error(),
})
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "metrics": metrics}, simulated)
s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated)
return clients.Response{}, err
}
uploadedResult, err := s.uploadGeneratedAssets(ctx, response.Result)
@@ -317,7 +329,7 @@ func (s *Service) emit(ctx context.Context, taskID string, eventType string, sta
return nil
}
func modelTypeFromKind(kind string) string {
func modelTypeFromKind(kind string, body map[string]any) string {
switch kind {
case "chat.completions", "responses":
return "text_generate"
@@ -327,12 +339,30 @@ func modelTypeFromKind(kind string) string {
}
return "image_generate"
case "videos.generations":
if videoRequestHasReferenceImage(body) {
return "image_to_video"
}
return "video_generate"
default:
return "task"
}
}
func videoRequestHasReferenceImage(body map[string]any) bool {
if body == nil {
return false
}
for _, key := range []string{
"image", "images", "image_url", "imageUrl", "image_urls", "imageUrls",
"reference_image", "referenceImage", "first_frame", "firstFrame", "last_frame", "lastFrame",
} {
if hasAnyString(body, key) {
return true
}
}
return false
}
func isTextGenerationKind(kind string) bool {
return kind == "chat.completions" || kind == "responses"
}
@@ -345,10 +375,8 @@ func isSimulation(task store.GatewayTask, candidate store.RuntimeModelCandidate)
}
func retryEnabled(candidate store.RuntimeModelCandidate) bool {
if enabled, ok := candidate.ModelRetryPolicy["enabled"].(bool); ok {
return enabled
}
if enabled, ok := candidate.PlatformRetryPolicy["enabled"].(bool); ok {
policy := effectiveRetryPolicy(candidate)
if enabled, ok := policy["enabled"].(bool); ok {
return enabled
}
return true
@@ -360,10 +388,7 @@ func maxAttemptsForCandidates(candidates []store.RuntimeModelCandidate) int {
}
maxAttempts := len(candidates)
for _, candidate := range candidates {
if value := intFromPolicy(candidate.ModelRetryPolicy, "maxAttempts"); value > 0 && value < maxAttempts {
maxAttempts = value
}
if value := intFromPolicy(candidate.PlatformRetryPolicy, "maxAttempts"); value > 0 && value < maxAttempts {
if value := intFromPolicy(effectiveRetryPolicy(candidate), "maxAttempts"); value > 0 && value < maxAttempts {
maxAttempts = value
}
}