perf(queue): 按策略动态扩缩异步 Worker

将 River 执行容量按平台模型与用户组的有效并发策略动态调整，并为长任务续租、并发租约原子抢占和限流退避补充保护。\n\n统一平台模型限流继承语义，兼容历史 platformLimits/modelLimits，并为三个迁移图像模型建立独立并发租约。补充管理端显式继承/覆盖配置、指标、单元测试及隔离 PostgreSQL 验收。\n\n验证：go test ./... -count=1；pnpm lint；pnpm test；pnpm build；隔离 PostgreSQL 并发原子性/续租测试；128 任务与三分钟长任务动态 Worker 验收。
2026-07-24 12:26:56 +08:00
parent 290b8c1854
commit 6c5daf29ca
37 changed files with 2365 additions and 205 deletions
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"time"

 	"github.com/jackc/pgx/v5"
@@ -17,6 +18,8 @@ type RuntimeRecoveryResult struct {
 	RequeuedAsyncTasks        int64 `json:"requeuedAsyncTasks"`
 }

+var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
+
 func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID string, reservations []RateLimitReservation) (RateLimitResult, error) {
 	tx, err := s.pool.Begin(ctx)
 	if err != nil {
@@ -24,6 +27,26 @@ func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID
 	}
 	defer tx.Rollback(ctx)

+	lockKeys := make([]string, 0)
+	lockKeySet := make(map[string]struct{})
+	for _, reservation := range reservations {
+		if reservation.Metric != "concurrent" || reservation.Limit <= 0 || reservation.Amount <= 0 {
+			continue
+		}
+		key := fmt.Sprintf("%d:%s%d:%s", len(reservation.ScopeType), reservation.ScopeType, len(reservation.ScopeKey), reservation.ScopeKey)
+		if _, exists := lockKeySet[key]; exists {
+			continue
+		}
+		lockKeySet[key] = struct{}{}
+		lockKeys = append(lockKeys, key)
+	}
+	sort.Strings(lockKeys)
+	for _, key := range lockKeys {
+		if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, key); err != nil {
+			return RateLimitResult{}, err
+		}
+	}
+
 	result := RateLimitResult{}
 	for _, reservation := range reservations {
 		if reservation.Limit <= 0 || reservation.Amount <= 0 {
@@ -49,11 +72,11 @@ func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID
 			reservation.WindowSeconds = 60
 		}
 		if reservation.Metric == "concurrent" {
-			leaseID, err := reserveConcurrencyLease(ctx, tx, taskID, attemptID, reservation)
+			lease, err := reserveConcurrencyLease(ctx, tx, taskID, attemptID, reservation)
 			if err != nil {
 				return RateLimitResult{}, err
 			}
-			result.LeaseIDs = append(result.LeaseIDs, leaseID)
+			result.Leases = append(result.Leases, lease)
 			continue
 		}
 		normalized, err := reserveCounterWindow(ctx, tx, taskID, attemptID, reservation)
@@ -65,7 +88,7 @@ func (s *Store) ReserveRateLimits(ctx context.Context, taskID string, attemptID
 	return result, tx.Commit(ctx)
 }

-func reserveConcurrencyLease(ctx context.Context, tx pgx.Tx, taskID string, attemptID string, reservation RateLimitReservation) (string, error) {
+func reserveConcurrencyLease(ctx context.Context, tx pgx.Tx, taskID string, attemptID string, reservation RateLimitReservation) (ConcurrencyLease, error) {
 	if reservation.LeaseTTLSeconds <= 0 {
 		reservation.LeaseTTLSeconds = 120
 	}
@@ -83,10 +106,10 @@ WHERE scope_type = $1
 		reservation.ScopeKey,
 		reservation.LeaseTTLSeconds,
 	).Scan(&active, &nextAvailableAt); err != nil {
-		return "", err
+		return ConcurrencyLease{}, err
 	}
 	if active+reservation.Amount > reservation.Limit {
-		return "", &RateLimitExceededError{
+		return ConcurrencyLease{}, &RateLimitExceededError{
 			ScopeType:     reservation.ScopeType,
 			ScopeKey:      reservation.ScopeKey,
 			ScopeName:     reservation.ScopeName,
@@ -117,9 +140,9 @@ RETURNING id::text`,
 		reservation.Amount,
 		reservation.LeaseTTLSeconds,
 	).Scan(&leaseID); err != nil {
-		return "", err
+		return ConcurrencyLease{}, err
 	}
-	return leaseID, nil
+	return ConcurrencyLease{ID: leaseID, TTL: time.Duration(reservation.LeaseTTLSeconds) * time.Second}, nil
 }

 func reserveCounterWindow(ctx context.Context, tx pgx.Tx, taskID string, attemptID string, reservation RateLimitReservation) (RateLimitReservation, error) {
@@ -232,13 +255,16 @@ func retryAfterUntil(when time.Time) time.Duration {

 func concurrencyRetryAfter(leaseExpiresAt time.Time) time.Duration {
 	if leaseExpiresAt.IsZero() {
-		return time.Second
+		return 5 * time.Second
 	}
 	duration := time.Until(leaseExpiresAt)
 	if duration <= time.Second {
 		return time.Second
 	}
-	return time.Second
+	if duration > 5*time.Second {
+		return 5 * time.Second
+	}
+	return duration
 }

 func (s *Store) CommitRateLimitReservations(ctx context.Context, reservations []RateLimitReservation, actualByMetric map[string]float64) error {
@@ -249,26 +275,57 @@ func (s *Store) ReleaseRateLimitReservations(ctx context.Context, reservations [
 	return s.finishRateLimitReservations(ctx, reservations, nil, "released", reason)
 }

-func (s *Store) ReleaseConcurrencyLeases(ctx context.Context, leaseIDs []string) error {
+func (s *Store) ReleaseConcurrencyLeases(ctx context.Context, leases []ConcurrencyLease) error {
+	leaseIDs := concurrencyLeaseIDs(leases)
 	if len(leaseIDs) == 0 {
 		return nil
 	}
-	for _, leaseID := range leaseIDs {
-		if leaseID == "" {
-			continue
-		}
-		if _, err := s.pool.Exec(ctx, `
+	_, err := s.pool.Exec(ctx, `
 UPDATE gateway_concurrency_leases
 SET released_at = now()
-WHERE id = $1::uuid AND released_at IS NULL`, leaseID); err != nil && !errors.Is(err, ErrRateLimited) {
-			return err
+WHERE id = ANY($1::uuid[]) AND released_at IS NULL`, leaseIDs)
+	return err
+}
+
+func (s *Store) RenewConcurrencyLeases(ctx context.Context, leases []ConcurrencyLease) error {
+	leaseIDs := make([]string, 0, len(leases))
+	ttlSeconds := make([]int32, 0, len(leases))
+	for _, lease := range leases {
+		if lease.ID == "" {
+			continue
 		}
+		ttl := lease.TTL
+		if ttl <= 0 {
+			ttl = 120 * time.Second
+		}
+		seconds := int32(ttl / time.Second)
+		if seconds < 1 {
+			seconds = 1
+		}
+		leaseIDs = append(leaseIDs, lease.ID)
+		ttlSeconds = append(ttlSeconds, seconds)
+	}
+	if len(leaseIDs) == 0 {
+		return nil
+	}
+	tag, err := s.pool.Exec(ctx, `
+UPDATE gateway_concurrency_leases lease
+SET expires_at = now() + (renewal.ttl_seconds * interval '1 second')
+FROM unnest($1::uuid[], $2::int[]) AS renewal(id, ttl_seconds)
+WHERE lease.id = renewal.id
+  AND lease.released_at IS NULL
+  AND lease.expires_at > now()`, leaseIDs, ttlSeconds)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() != int64(len(leaseIDs)) {
+		return fmt.Errorf("%w: renewed %d of %d leases", ErrConcurrencyLeaseLost, tag.RowsAffected(), len(leaseIDs))
 	}
 	return nil
 }

 func (s *Store) AttachRateLimitResultToAttempt(ctx context.Context, attemptID string, result RateLimitResult) error {
-	if attemptID == "" || (len(result.Reservations) == 0 && len(result.LeaseIDs) == 0) {
+	if attemptID == "" || (len(result.Reservations) == 0 && len(result.Leases) == 0) {
 		return nil
 	}
 	tx, err := s.pool.Begin(ctx)
@@ -289,7 +346,8 @@ WHERE id = $1::uuid`, reservation.ReservationID, attemptID); err != nil {
 			return err
 		}
 	}
-	for _, leaseID := range result.LeaseIDs {
+	for _, lease := range result.Leases {
+		leaseID := lease.ID
 		if leaseID == "" {
 			continue
 		}
@@ -303,6 +361,16 @@ WHERE id = $1::uuid`, leaseID, attemptID); err != nil {
 	return tx.Commit(ctx)
 }

+func concurrencyLeaseIDs(leases []ConcurrencyLease) []string {
+	ids := make([]string, 0, len(leases))
+	for _, lease := range leases {
+		if lease.ID != "" {
+			ids = append(ids, lease.ID)
+		}
+	}
+	return ids
+}
+
 func (s *Store) RecoverInterruptedRuntimeState(ctx context.Context) (RuntimeRecoveryResult, error) {
 	tx, err := s.pool.Begin(ctx)
 	if err != nil {