fix(acceptance): 隔离容量压测并稳定数据库连接
将协议模拟验收的供应商并发限额与 Worker 容量门禁分离,真实金丝雀继续保留生产限流。readyz 改用关键连接池并预热关键及 River 池,延长生产连接空闲周期,降低跨地域连接抖动。失败 Run 现在可以原子替换且失败报告记录任务数量,避免 validation 之间短暂放开正式流量。\n\n验证:Go 全量测试、go vet、PostgreSQL 集成测试、ShellCheck、迁移安全、发布脚本、pnpm lint/test/build、OpenAPI 无漂移均通过。
This commit is contained in:
@@ -146,7 +146,11 @@ func (s *Service) buildTaskAdmissionPlanForCurrentBinding(
|
||||
ModelType: modelType,
|
||||
}, nil
|
||||
}
|
||||
if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidate, body)); err != nil {
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidate, body),
|
||||
)
|
||||
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
|
||||
return taskAdmissionPlan{}, err
|
||||
}
|
||||
return taskAdmissionPlan{
|
||||
@@ -206,8 +210,17 @@ func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionS
|
||||
}
|
||||
out := append([]store.AdmissionScope(nil), scopes...)
|
||||
for index := range out {
|
||||
// Protocol-emulated acceptance measures Gateway and Worker capacity, so
|
||||
// the isolated Run is bounded by the worker_capacity scope instead of a
|
||||
// production supplier quota. The real acceptance_canary path deliberately
|
||||
// retains the production platform-model concurrency limit.
|
||||
if task.RunMode == "acceptance" && out[index].ScopeType == "platform_model" {
|
||||
out[index].ConcurrentLimit = 0
|
||||
}
|
||||
if out[index].ConcurrentLimit <= 0 {
|
||||
continue
|
||||
if task.RunMode != "acceptance" || out[index].ScopeType != "platform_model" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out[index].QueueLimit = acceptanceQueueLimit
|
||||
out[index].MaxWaitSeconds = acceptanceQueueMaxWait
|
||||
@@ -215,6 +228,23 @@ func acceptanceAdmissionScopes(task store.GatewayTask, scopes []store.AdmissionS
|
||||
return out
|
||||
}
|
||||
|
||||
func acceptanceInfrastructureReservations(
|
||||
task store.GatewayTask,
|
||||
reservations []store.RateLimitReservation,
|
||||
) []store.RateLimitReservation {
|
||||
if task.RunMode != "acceptance" {
|
||||
return reservations
|
||||
}
|
||||
out := make([]store.RateLimitReservation, 0, len(reservations))
|
||||
for _, reservation := range reservations {
|
||||
if reservation.ScopeType == "platform_model" && reservation.Metric == "concurrent" {
|
||||
continue
|
||||
}
|
||||
out = append(out, reservation)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Service) loadAsyncTaskAdmission(ctx context.Context, task store.GatewayTask) (*store.TaskAdmission, error) {
|
||||
if !task.AsyncMode {
|
||||
return nil, nil
|
||||
@@ -420,7 +450,11 @@ func (s *Service) ensureCandidateAdmission(
|
||||
}
|
||||
return store.TaskAdmissionResult{}, false, nil
|
||||
}
|
||||
if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidate, body)); err != nil {
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidate, body),
|
||||
)
|
||||
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
|
||||
return store.TaskAdmissionResult{}, true, err
|
||||
}
|
||||
plan := taskAdmissionPlan{
|
||||
|
||||
@@ -24,28 +24,66 @@ func TestDistributedAdmissionModelTypeBoundary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceAdmissionScopesEnableBoundedQueueWithoutChangingConcurrency(t *testing.T) {
|
||||
func TestAcceptanceAdmissionScopesUseWorkerCapacityInsteadOfSupplierConcurrency(t *testing.T) {
|
||||
input := []store.AdmissionScope{{
|
||||
ScopeType: "platform_model",
|
||||
ScopeKey: "model-1",
|
||||
ConcurrentLimit: 10,
|
||||
QueueLimit: 0,
|
||||
MaxWaitSeconds: 0,
|
||||
}, {
|
||||
ScopeType: "worker_capacity",
|
||||
ScopeKey: "global",
|
||||
ConcurrentLimit: 48,
|
||||
}}
|
||||
|
||||
got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance"}, input)
|
||||
|
||||
if got[0].ConcurrentLimit != 10 {
|
||||
t.Fatalf("acceptance must preserve the production concurrency limit, got %+v", got[0])
|
||||
if got[0].ConcurrentLimit != 0 {
|
||||
t.Fatalf("protocol-emulated acceptance must defer to worker capacity, got %+v", got[0])
|
||||
}
|
||||
if got[0].QueueLimit != acceptanceQueueLimit || got[0].MaxWaitSeconds != acceptanceQueueMaxWait {
|
||||
t.Fatalf("acceptance must enable a bounded queue, got %+v", got[0])
|
||||
}
|
||||
if got[1].ConcurrentLimit != 48 {
|
||||
t.Fatalf("acceptance worker capacity changed, got %+v", got[1])
|
||||
}
|
||||
if input[0].QueueLimit != 0 || input[0].MaxWaitSeconds != 0 {
|
||||
t.Fatalf("acceptance queue overlay must not mutate the production scopes, got %+v", input[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceCanaryPreservesProductionConcurrency(t *testing.T) {
|
||||
input := []store.AdmissionScope{{
|
||||
ScopeType: "platform_model",
|
||||
ScopeKey: "model-1",
|
||||
ConcurrentLimit: 10,
|
||||
}}
|
||||
|
||||
got := acceptanceAdmissionScopes(store.GatewayTask{RunMode: "acceptance_canary"}, input)
|
||||
|
||||
if got[0].ConcurrentLimit != 10 {
|
||||
t.Fatalf("real acceptance canary must preserve production concurrency, got %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceInfrastructureReservationsOnlyRemoveSupplierConcurrency(t *testing.T) {
|
||||
input := []store.RateLimitReservation{
|
||||
{ScopeType: "platform_model", Metric: "concurrent", Limit: 10},
|
||||
{ScopeType: "platform_model", Metric: "rpm", Limit: 600},
|
||||
{ScopeType: "user_group", Metric: "concurrent", Limit: 20},
|
||||
}
|
||||
|
||||
got := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance"}, input)
|
||||
if len(got) != 2 || got[0].Metric != "rpm" || got[1].ScopeType != "user_group" {
|
||||
t.Fatalf("unexpected acceptance reservations: %+v", got)
|
||||
}
|
||||
canary := acceptanceInfrastructureReservations(store.GatewayTask{RunMode: "acceptance_canary"}, input)
|
||||
if len(canary) != len(input) {
|
||||
t.Fatalf("real canary reservations changed: %+v", canary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceAdmissionScopesLeaveProductionPolicyUnchanged(t *testing.T) {
|
||||
input := []store.AdmissionScope{{
|
||||
ScopeType: "platform_model",
|
||||
|
||||
@@ -601,7 +601,11 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
|
||||
}
|
||||
}
|
||||
if hasConcurrentLimit {
|
||||
if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidates[0], body)); err != nil {
|
||||
reservations := acceptanceInfrastructureReservations(
|
||||
task,
|
||||
s.rateLimitReservations(ctx, user, candidates[0], body),
|
||||
)
|
||||
if err := s.store.CheckRateLimits(ctx, reservations); err != nil {
|
||||
if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
|
||||
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
|
||||
if queueErr != nil {
|
||||
|
||||
Reference in New Issue
Block a user