fix(rate-limits): treat empty policies as unlimited
This commit is contained in:
@@ -83,11 +83,18 @@ func (s *Service) rateLimitReservations(ctx context.Context, user *auth.User, ca
|
||||
|
||||
func effectiveRateLimitPolicy(candidate store.RuntimeModelCandidate) map[string]any {
|
||||
policy := candidate.PlatformRateLimitPolicy
|
||||
if hasRules(candidate.RuntimeRateLimitPolicy) {
|
||||
if strings.TrimSpace(candidate.RuntimePolicySetID) != "" {
|
||||
policy = candidate.RuntimeRateLimitPolicy
|
||||
} else if hasRules(candidate.RuntimeRateLimitPolicy) {
|
||||
policy = mergeMap(policy, candidate.RuntimeRateLimitPolicy)
|
||||
}
|
||||
if nested, ok := candidate.RuntimePolicyOverride["rateLimitPolicy"].(map[string]any); ok && len(nested) > 0 {
|
||||
policy = mergeMap(policy, nested)
|
||||
if _, hasOverride := candidate.RuntimePolicyOverride["rateLimitPolicy"]; hasOverride {
|
||||
nested, _ := candidate.RuntimePolicyOverride["rateLimitPolicy"].(map[string]any)
|
||||
if len(nested) == 0 {
|
||||
policy = nil
|
||||
} else {
|
||||
policy = mergeMap(policy, nested)
|
||||
}
|
||||
}
|
||||
if hasRules(candidate.ModelRateLimitPolicy) {
|
||||
policy = mergeMap(policy, candidate.ModelRateLimitPolicy)
|
||||
@@ -123,6 +130,9 @@ func reservationsFromPolicy(scopeType string, scopeKey string, scopeName string,
|
||||
rule, _ := rawRule.(map[string]any)
|
||||
metric := strings.TrimSpace(stringFromMap(rule, "metric"))
|
||||
limit := floatFromAny(rule["limit"])
|
||||
if metric == "" || limit <= 0 {
|
||||
continue
|
||||
}
|
||||
amount := 1.0
|
||||
if strings.HasPrefix(metric, "tpm") {
|
||||
amount = float64(estimatedTokens)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestTokenUsageAmountsUsesActualUsageForTPM(t *testing.T) {
|
||||
@@ -27,3 +28,39 @@ func TestTokenUsageAmountsFallsBackToInputOutputTotal(t *testing.T) {
|
||||
t.Fatalf("expected total token fallback 8, got %v", got["tpm_total"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveRateLimitPolicyTreatsEmptyRuntimePolicyAsUnlimited(t *testing.T) {
|
||||
policy := effectiveRateLimitPolicy(store.RuntimeModelCandidate{
|
||||
PlatformRateLimitPolicy: map[string]any{"rules": []any{
|
||||
map[string]any{"metric": "rpm", "limit": 500},
|
||||
}},
|
||||
RuntimePolicySetID: "runtime-policy-1",
|
||||
RuntimeRateLimitPolicy: map[string]any{"rules": []any{}},
|
||||
})
|
||||
|
||||
if hasRules(policy) {
|
||||
t.Fatalf("expected empty runtime policy to clear inherited limits, got %+v", policy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReservationsFromPolicySkipsNonPositiveLimits(t *testing.T) {
|
||||
reservations := reservationsFromPolicy(
|
||||
"platform_model",
|
||||
"model-1",
|
||||
"Model 1",
|
||||
nil,
|
||||
map[string]any{"rules": []any{
|
||||
map[string]any{"metric": "rpm", "limit": -1},
|
||||
map[string]any{"metric": "tpm_total", "limit": 0},
|
||||
map[string]any{"metric": "concurrent", "limit": 2},
|
||||
}},
|
||||
map[string]any{"prompt": "hello"},
|
||||
)
|
||||
|
||||
if len(reservations) != 1 {
|
||||
t.Fatalf("expected only the positive concurrent rule to reserve, got %+v", reservations)
|
||||
}
|
||||
if reservations[0].Metric != "concurrent" || reservations[0].Limit != 2 {
|
||||
t.Fatalf("expected concurrent reservation with limit 2, got %+v", reservations[0])
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user