fix(routing): 避免路由探测耗尽数据库连接
将跨实例探测协调从持有会话级 advisory lock 改为带过期时间的数据库短租约,避免网络探测期间占用关键 PostgreSQL 连接。新增可配置的探测并发上限和真实 PostgreSQL 回归测试,确保租约生效期间连接池仍可全部获取。\n\n验证:go test ./... -count=1;go vet ./...;真实 PostgreSQL 17 集成测试;迁移安全检查。
This commit is contained in:
@@ -27,6 +27,11 @@ func (s *Service) runRouteHealthProber(ctx context.Context) {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
states := make(map[string]routeProbeState)
|
||||
probeConcurrency := s.cfg.RouteProbeConcurrency
|
||||
if probeConcurrency <= 0 {
|
||||
probeConcurrency = 8
|
||||
}
|
||||
probeSlots := make(chan struct{}, probeConcurrency)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -58,19 +63,35 @@ func (s *Service) runRouteHealthProber(ctx context.Context) {
|
||||
if target.Hot {
|
||||
interval = time.Duration(s.cfg.RouteProbeHotIntervalSeconds) * time.Second
|
||||
}
|
||||
states[profile.Key] = routeProbeState{next: now.Add(interval), lastRequestObserved: requestedAt}
|
||||
go s.probeRouteTarget(ctx, target, profile)
|
||||
select {
|
||||
case probeSlots <- struct{}{}:
|
||||
states[profile.Key] = routeProbeState{next: now.Add(interval), lastRequestObserved: requestedAt}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
continue
|
||||
}
|
||||
go func(target store.RouteProbeTarget, profile executionpool.RouteProfile) {
|
||||
defer func() { <-probeSlots }()
|
||||
s.probeRouteTarget(ctx, target, profile)
|
||||
}(target, profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) probeRouteTarget(ctx context.Context, target store.RouteProbeTarget, profile executionpool.RouteProfile) {
|
||||
leadership, locked, err := s.coordinationStore.TryAcquireRouteProbeLeadership(ctx, s.cfg.ExecutionPoolID, profile.Key)
|
||||
leaseToken, locked, err := s.coordinationStore.TryClaimRouteProbe(ctx, s.cfg.ExecutionPoolID, profile.Key, time.Duration(s.cfg.RouteProbeTimeoutMS)*time.Millisecond+5*time.Second)
|
||||
if err != nil || !locked {
|
||||
return
|
||||
}
|
||||
defer leadership.Release()
|
||||
defer func() {
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := s.coordinationStore.ReleaseRouteProbe(releaseCtx, s.cfg.ExecutionPoolID, profile.Key, leaseToken); err != nil {
|
||||
s.logRouteProbeFailure("release_lease", err)
|
||||
}
|
||||
}()
|
||||
client, err := s.httpClientForCandidate(target.Candidate, false, "")
|
||||
if err != nil {
|
||||
s.logRouteProbeFailure("prepare_client", err)
|
||||
|
||||
Reference in New Issue
Block a user