fix(routing): 避免路由探测耗尽数据库连接
将跨实例探测协调从持有会话级 advisory lock 改为带过期时间的数据库短租约,避免网络探测期间占用关键 PostgreSQL 连接。新增可配置的探测并发上限和真实 PostgreSQL 回归测试,确保租约生效期间连接池仍可全部获取。\n\n验证:go test ./... -count=1;go vet ./...;真实 PostgreSQL 17 集成测试;迁移安全检查。
This commit is contained in:
@@ -222,6 +222,49 @@ WHERE expires_at > now()`)
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
// TryClaimRouteProbe coordinates probes without holding a database connection
|
||||
// during DNS/TCP/TLS/HTTP I/O. The previous session advisory lock pinned one
|
||||
// critical-pool connection per in-flight probe and could starve readiness.
|
||||
func (s *Store) TryClaimRouteProbe(ctx context.Context, poolID, routeProfileKey string, ttl time.Duration) (string, bool, error) {
|
||||
poolID = strings.TrimSpace(poolID)
|
||||
routeProfileKey = strings.TrimSpace(routeProfileKey)
|
||||
if poolID == "" || routeProfileKey == "" {
|
||||
return "", false, errors.New("route probe pool and profile are required")
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = 15 * time.Second
|
||||
}
|
||||
token := uuid.NewString()
|
||||
var claimed string
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
INSERT INTO gateway_route_probe_leases (
|
||||
pool_id, route_profile_key, lease_token, expires_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3::uuid, now() + $4::interval, now())
|
||||
ON CONFLICT (pool_id, route_profile_key) DO UPDATE
|
||||
SET lease_token = EXCLUDED.lease_token,
|
||||
expires_at = EXCLUDED.expires_at,
|
||||
updated_at = now()
|
||||
WHERE gateway_route_probe_leases.expires_at <= now()
|
||||
RETURNING lease_token::text`, poolID, routeProfileKey, token, ttl.String()).Scan(&claimed)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return claimed, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) ReleaseRouteProbe(ctx context.Context, poolID, routeProfileKey, token string) error {
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
DELETE FROM gateway_route_probe_leases
|
||||
WHERE pool_id = $1
|
||||
AND route_profile_key = $2
|
||||
AND lease_token = $3::uuid`, strings.TrimSpace(poolID), strings.TrimSpace(routeProfileKey), strings.TrimSpace(token))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) ListRouteHealth(ctx context.Context, routeProfileKey string, _ time.Time) ([]executionpool.RouteHealth, error) {
|
||||
return s.listRouteHealth(ctx, "WHERE route_profile_key = $1", strings.TrimSpace(routeProfileKey))
|
||||
}
|
||||
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/executionpool"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestExecutionPoolQueriesUseTimestampCutoff(t *testing.T) {
|
||||
@@ -18,3 +21,42 @@ func TestExecutionPoolQueriesUseTimestampCutoff(t *testing.T) {
|
||||
t.Fatalf("list capacity with timestamp cutoff: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRouteProbeLeaseDoesNotPinDatabaseConnection(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := db.UpsertExecutionPool(ctx, executionpool.ExecutionPool{ID: "probe-pool", State: executionpool.PoolActive}); err != nil {
|
||||
t.Fatalf("upsert execution pool: %v", err)
|
||||
}
|
||||
token, claimed, err := db.TryClaimRouteProbe(ctx, "probe-pool", "route-profile", 15*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("claim route probe: %v", err)
|
||||
}
|
||||
if !claimed || token == "" {
|
||||
t.Fatal("route probe was not claimed")
|
||||
}
|
||||
|
||||
connections := make([]*pgxpool.Conn, 0, db.Pool().Stat().MaxConns())
|
||||
defer func() {
|
||||
for _, connection := range connections {
|
||||
connection.Release()
|
||||
}
|
||||
}()
|
||||
for range db.Pool().Stat().MaxConns() {
|
||||
connection, err := db.Pool().Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire all pool connections while lease is active: %v", err)
|
||||
}
|
||||
connections = append(connections, connection)
|
||||
}
|
||||
for _, connection := range connections {
|
||||
connection.Release()
|
||||
}
|
||||
connections = connections[:0]
|
||||
|
||||
if err := db.ReleaseRouteProbe(ctx, "probe-pool", "route-profile", token); err != nil {
|
||||
t.Fatalf("release route probe: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user