fix(routing): 避免路由探测耗尽数据库连接

将跨实例探测协调从持有会话级 advisory lock 改为带过期时间的数据库短租约,避免网络探测期间占用关键 PostgreSQL 连接。新增可配置的探测并发上限和真实 PostgreSQL 回归测试,确保租约生效期间连接池仍可全部获取。\n\n验证:go test ./... -count=1;go vet ./...;真实 PostgreSQL 17 集成测试;迁移安全检查。
This commit is contained in:
2026-08-05 23:13:32 +08:00
parent 21f72da7a8
commit 335b21d1f8
6 changed files with 127 additions and 4 deletions
+25 -4
View File
@@ -27,6 +27,11 @@ func (s *Service) runRouteHealthProber(ctx context.Context) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
states := make(map[string]routeProbeState)
probeConcurrency := s.cfg.RouteProbeConcurrency
if probeConcurrency <= 0 {
probeConcurrency = 8
}
probeSlots := make(chan struct{}, probeConcurrency)
for {
select {
case <-ctx.Done():
@@ -58,19 +63,35 @@ func (s *Service) runRouteHealthProber(ctx context.Context) {
if target.Hot {
interval = time.Duration(s.cfg.RouteProbeHotIntervalSeconds) * time.Second
}
states[profile.Key] = routeProbeState{next: now.Add(interval), lastRequestObserved: requestedAt}
go s.probeRouteTarget(ctx, target, profile)
select {
case probeSlots <- struct{}{}:
states[profile.Key] = routeProbeState{next: now.Add(interval), lastRequestObserved: requestedAt}
case <-ctx.Done():
return
default:
continue
}
go func(target store.RouteProbeTarget, profile executionpool.RouteProfile) {
defer func() { <-probeSlots }()
s.probeRouteTarget(ctx, target, profile)
}(target, profile)
}
}
}
}
func (s *Service) probeRouteTarget(ctx context.Context, target store.RouteProbeTarget, profile executionpool.RouteProfile) {
leadership, locked, err := s.coordinationStore.TryAcquireRouteProbeLeadership(ctx, s.cfg.ExecutionPoolID, profile.Key)
leaseToken, locked, err := s.coordinationStore.TryClaimRouteProbe(ctx, s.cfg.ExecutionPoolID, profile.Key, time.Duration(s.cfg.RouteProbeTimeoutMS)*time.Millisecond+5*time.Second)
if err != nil || !locked {
return
}
defer leadership.Release()
defer func() {
releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.coordinationStore.ReleaseRouteProbe(releaseCtx, s.cfg.ExecutionPoolID, profile.Key, leaseToken); err != nil {
s.logRouteProbeFailure("release_lease", err)
}
}()
client, err := s.httpClientForCandidate(target.Candidate, false, "")
if err != nil {
s.logRouteProbeFailure("prepare_client", err)