fix(acceptance): 隔离容量压测并稳定数据库连接

将协议模拟验收的供应商并发限额与 Worker 容量门禁分离,真实金丝雀继续保留生产限流。readyz 改用关键连接池并预热关键及 River 池,延长生产连接空闲周期,降低跨地域连接抖动。失败 Run 现在可以原子替换且失败报告记录任务数量,避免 validation 之间短暂放开正式流量。\n\n验证:Go 全量测试、go vet、PostgreSQL 集成测试、ShellCheck、迁移安全、发布脚本、pnpm lint/test/build、OpenAPI 无漂移均通过。
This commit is contained in:
2026-08-01 19:29:08 +08:00
parent 89a0ff7e99
commit 92e328a575
11 changed files with 277 additions and 42 deletions
+37 -3
View File
@@ -146,7 +146,11 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
ModelType: modelType,
}, nil
}
if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidate, body)); err != nil {
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidate, body),
)
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
return taskAdmissionPlan{}, err
}
return taskAdmissionPlan{
@@ -206,8 +210,17 @@ func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionS
}
out := append([]store.AdmissionScope(nil), scopes...)
for index := range out {
// Protocol-emulated acceptance measures Gateway and Worker capacity, so
// the isolated Run is bounded by the worker_capacity scope instead of a
// production supplier quota. The real acceptance_canary path deliberately
// retains the production platform-model concurrency limit.
if task.RunMode == "acceptance" && out[index].ScopeType == "platform_model" {
out[index].ConcurrentLimit = 0
}
if out[index].ConcurrentLimit <= 0 {
continue
if task.RunMode != "acceptance" || out[index].ScopeType != "platform_model" {
continue
}
}
out[index].QueueLimit = acceptanceQueueLimit
out[index].MaxWaitSeconds = acceptanceQueueMaxWait
@@ -215,6 +228,23 @@ func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionS
return out
}
func acceptanceInfrastructureReservations(
task store.GatewayTask,
reservations []store.RateLimitReservation,
) []store.RateLimitReservation {
if task.RunMode != "acceptance" {
return reservations
}
out := make([]store.RateLimitReservation, 0, len(reservations))
for _, reservation := range reservations {
if reservation.ScopeType == "platform_model" && reservation.Metric == "concurrent" {
continue
}
out = append(out, reservation)
}
return out
}
func (s *Service) loadAsyncTaskAdmission(ctx context.Context, task store.GatewayTask) (*store.TaskAdmission, error) {
if !task.AsyncMode {
return nil, nil
@@ -420,7 +450,11 @@ func (s *Service) ensureCandidateAdmission(
}
return store.TaskAdmissionResult{}, false, nil
}
if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidate, body)); err != nil {
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidate, body),
)
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
return store.TaskAdmissionResult{}, true, err
}
plan := taskAdmissionPlan{
+41 -3
View File
@@ -24,28 +24,66 @@ func TestDistributedAdmissionModelTypeBoundary(t *testing.T) {
}
}
func TestAcceptanceAdmissionScopesEnableBoundedQueueWithoutChangingConcurrency(t *testing.T) {
func TestAcceptanceAdmissionScopesUseWorkerCapacityInsteadOfSupplierConcurrency(t *testing.T) {
input := []store.AdmissionScope{{
ScopeType: "platform_model",
ScopeKey: "model-1",
ConcurrentLimit: 10,
QueueLimit: 0,
MaxWaitSeconds: 0,
}, {
ScopeType: "worker_capacity",
ScopeKey: "global",
ConcurrentLimit: 48,
}}
got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance"}, input)
if got[0].ConcurrentLimit != 10 {
t.Fatalf("acceptance must preserve the production concurrency limit, got %+v", got[0])
if got[0].ConcurrentLimit != 0 {
t.Fatalf("protocol-emulated acceptance must defer to worker capacity, got %+v", got[0])
}
if got[0].QueueLimit != acceptanceQueueLimit || got[0].MaxWaitSeconds != acceptanceQueueMaxWait {
t.Fatalf("acceptance must enable a bounded queue, got %+v", got[0])
}
if got[1].ConcurrentLimit != 48 {
t.Fatalf("acceptance worker capacity changed, got %+v", got[1])
}
if input[0].QueueLimit != 0 || input[0].MaxWaitSeconds != 0 {
t.Fatalf("acceptance queue overlay must not mutate the production scopes, got %+v", input[0])
}
}
func TestAcceptanceCanaryPreservesProductionConcurrency(t *testing.T) {
input := []store.AdmissionScope{{
ScopeType: "platform_model",
ScopeKey: "model-1",
ConcurrentLimit: 10,
}}
got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance_canary"}, input)
if got[0].ConcurrentLimit != 10 {
t.Fatalf("real acceptance canary must preserve production concurrency, got %+v", got[0])
}
}
func TestAcceptanceInfrastructureReservationsOnlyRemoveSupplierConcurrency(t *testing.T) {
input := []store.RateLimitReservation{
{ScopeType: "platform_model", Metric: "concurrent", Limit: 10},
{ScopeType: "platform_model", Metric: "rpm", Limit: 600},
{ScopeType: "user_group", Metric: "concurrent", Limit: 20},
}
got := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance"}, input)
if len(got) != 2 || got[0].Metric != "rpm" || got[1].ScopeType != "user_group" {
t.Fatalf("unexpected acceptance reservations: %+v", got)
}
canary := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance_canary"}, input)
if len(canary) != len(input) {
t.Fatalf("real canary reservations changed: %+v", canary)
}
}
func TestAcceptanceAdmissionScopesLeaveProductionPolicyUnchanged(t *testing.T) {
input := []store.AdmissionScope{{
ScopeType: "platform_model",
+5 -1
View File
@@ -601,7 +601,11 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
}
}
if hasConcurrentLimit {
if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidates[0], body)); err != nil {
reservations := acceptanceInfrastructureReservations(
task,
s.rateLimitReservations(ctx, user, candidates[0], body),
)
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
if queueErr != nil {