BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(routing): 完善平台满载避让与故障轮转

Browse Source 
为未配置并发上限的平台增加基于运行和等待任务数的软负载，并按非满载、有效优先级、缓存亲和与负载稳定排序。\n\n平台模型 RPM、TPM 和并发额度竞争失败时只轮转候选，不触发冷却、禁用或降级；用户组额度保持不可绕过。补齐异步冷却排队恢复、满载原因、选择原因与低基数指标。\n\n验证：go test ./...；go vet ./...；PostgreSQL 原子额度、准入队列、Worker 容量回收及故障策略 HTTP acceptance。
This commit is contained in:
wangbo
2026-08-03 17:42:34 +08:00
parent b125599354
commit 8362d6d27a
15 changed files with 468 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/httpapi/failure_policy_http_acceptance_integration_test.go
+2 -4
View File
@@ -774,7 +774,6 @@ func (f *failurePolicyAcceptanceFixture) testAllCoolingSynchronousContract(t *te
if status != http.StatusTooManyRequests || stubA.calls.Load() != 1 || stubB.calls.Load() != 1 {
t.Fatalf("initial cooling request status=%d calls=%d/%d", status, stubA.calls.Load(), stubB.calls.Load())
}
startedAt := time.Now()
status, headers, body := f.postJSON(t, "/api/v1/images/generations", map[string]any{"model": model, "prompt": "all cooling"}, nil)
if status != http.StatusTooManyRequests {
t.Fatalf("cooldown contract status=%d body=%s", status, body)
@@ -801,9 +800,8 @@ func (f *failurePolicyAcceptanceFixture) testAllCoolingSynchronousContract(t *te
if envelope.Error.Code != "model_cooling_down" || len(envelope.Error.Details) != 2 {
t.Fatalf("unexpected cooldown envelope: %+v", envelope)
}
detail := f.loadTask(t, f.taskIDForModel(t, model, startedAt))
if len(detail.Attempts) != 0 {
t.Fatalf("candidate selection cooldown must not create provider attempts: %+v", detail.Attempts)
if stubA.calls.Load() != 1 || stubB.calls.Load() != 1 {
t.Fatalf("candidate selection cooldown must not create provider attempts: calls=%d/%d", stubA.calls.Load(), stubB.calls.Load())
}
}
apps/api/internal/httpapi/handlers.go
+8 -4
View File
@@ -1202,14 +1202,18 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
return
}
if err := s.runner.ValidateModelAccess(r.Context(), kind, model, prepared.Body, user); err != nil {
if errors.Is(err, store.ErrNoModelCandidate) {
if responsePlan.asyncMode && store.ModelCandidateRetryAfter(err) > 0 {
// Persist recoverable asynchronous requests so the shared queue can
// wake them when the earliest candidate cooldown expires.
} else if errors.Is(err, store.ErrNoModelCandidate) {
applyRunErrorHeaders(w, err)
writeTaskError(statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), runErrorCode(err))
return
} else {
s.logger.Error("validate model access failed", "kind", kind, "error_category", "model_access_validation_failed", "error", err)
writeTaskError(http.StatusInternalServerError, "validate model access failed", nil, "model_access_validation_failed")
return
}
s.logger.Error("validate model access failed", "kind", kind, "error_category", "model_access_validation_failed", "error", err)
writeTaskError(http.StatusInternalServerError, "validate model access failed", nil, "model_access_validation_failed")
return
}
runMode, err := s.admittedTaskRunMode(admission, prepared.Body)