fix(cluster): 修复身份事务阻塞与 Worker 容量抖动

将身份协调和安全事件心跳改为 PostgreSQL 单 Leader 执行，并从独立 Worker 进程中移除身份运行时，避免多副本重复写同一状态。为安全事件事务增加锁等待、空闲事务超时及独立回滚上下文；Worker 需连续丢失六次心跳后才判定失效，降低跨节点抖动导致的容量反复扩缩。验证：Go 全量测试、go vet、Race 聚焦测试、PostgreSQL 18 Leader/安全事件/Worker 分配集成测试及迁移测试通过。
2026-07-29 23:04:41 +08:00
parent 3886048e0f
commit 91451b3c86
10 changed files with 497 additions and 76 deletions
@@ -7,9 +7,11 @@ import (
 	"time"

 	"github.com/easyai/easyai-ai-gateway/apps/api/internal/identity"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
 )

 const identityPairingReconcileInterval = 5 * time.Second
+const identityPairingLeadershipTimeout = 5 * time.Second

 type identityPairingReconciliationStore interface {
 	PendingIdentityPairingExchanges(context.Context) ([]identity.PairingExchange, error)
@@ -21,9 +23,9 @@ type identityPairingRestorer interface {
 	RestoreCompletedSecurityEvents(context.Context, string) error
 }

-func (s *Server) reconcileCanonicalIdentityPairing() {
+func (s *Server) reconcileCanonicalIdentityPairing(ctx context.Context) {
 	reconcileCanonicalIdentityPairing(
-		s.ctx,
+		ctx,
 		s.store,
 		s.identityPairing,
 		&s.identityRestoredPairings,
@@ -34,15 +36,57 @@ func (s *Server) reconcileCanonicalIdentityPairing() {
 }

 func (s *Server) runIdentityPairingCoordinator() {
-	ticker := time.NewTicker(identityPairingReconcileInterval)
-	defer ticker.Stop()
+	retry := time.NewTicker(identityPairingReconcileInterval)
+	defer retry.Stop()
 	for {
+		acquireCtx, cancel := context.WithTimeout(s.ctx, identityPairingLeadershipTimeout)
+		leadership, acquired, err := s.store.TryAcquireIdentityCoordinatorLeadership(acquireCtx)
+		cancel()
+		if err != nil && s.logger != nil {
+			s.logger.Warn("identity pairing coordinator leader election failed",
+				"error_category", "identity_coordinator_leader_election_failed",
+				"error", err,
+			)
+		}
+		if acquired {
+			if s.logger != nil {
+				s.logger.Info("identity pairing coordinator leadership acquired")
+			}
+			if !s.runIdentityPairingLeader(leadership) {
+				return
+			}
+		}
 		select {
 		case <-s.ctx.Done():
 			return
+		case <-retry.C:
+		}
+	}
+}
+
+func (s *Server) runIdentityPairingLeader(leadership store.Leadership) bool {
+	defer leadership.Release()
+	ticker := time.NewTicker(identityPairingReconcileInterval)
+	defer ticker.Stop()
+	for {
+		cycleCtx, cancel := context.WithTimeout(s.ctx, identityPairingLeadershipTimeout)
+		if err := leadership.KeepAlive(cycleCtx); err != nil {
+			cancel()
+			if s.logger != nil {
+				s.logger.Warn("identity pairing coordinator leadership lost",
+					"error_category", "identity_coordinator_leadership_lost",
+					"error", err,
+				)
+			}
+			return true
+		}
+		s.reconcileIdentityRuntime(cycleCtx)
+		s.reconcileCanonicalIdentityPairing(cycleCtx)
+		cancel()
+		select {
+		case <-s.ctx.Done():
+			return false
 		case <-ticker.C:
-			s.reconcileIdentityRuntime()
-			s.reconcileCanonicalIdentityPairing()
 		}
 	}
 }
@@ -105,14 +149,15 @@ func reconcileCanonicalIdentityPairing(
 	}
 }

-func (s *Server) reconcileIdentityRuntime() {
+func (s *Server) reconcileIdentityRuntime(ctx context.Context) {
 	if s.identityRuntime == nil || !s.identityRuntime.ReconciliationRequired() {
 		return
 	}
-	ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second)
-	defer cancel()
 	if err := s.identityRuntime.ReconcileActive(ctx); err != nil && s.logger != nil {
-		s.logger.Warn("identity runtime reconciliation failed and will retry", "error_category", "identity_runtime_reconciliation_failed")
+		s.logger.Warn("identity runtime reconciliation failed and will retry",
+			"error_category", "identity_runtime_reconciliation_failed",
+			"error", err,
+		)
 	}
 }

@@ -81,43 +81,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
 	}
 	server.auth.ServerMainInternalKey = cfg.ServerMainInternalKey
 	server.auth.ServerMainInternalSecret = cfg.ServerMainInternalSecret
-	secretStore, err := identitySecretStore(cfg)
-	if err != nil {
-		panic("invalid identity SecretStore: " + err.Error())
-	}
-	go ssfreceiver.RunSecurityEventRetirementWorker(ctx, db)
-	go ssfreceiver.RunIdentitySecretCleanupWorker(ctx, db, secretStore)
-	runtimeBuilder := identityruntime.NewRuntimeBuilder(ctx, db, secretStore, identityruntime.RuntimeBuilderConfig{
-		AppEnv: cfg.AppEnv, JWKSCacheTTL: 5 * time.Minute,
-		HeartbeatInterval: time.Duration(cfg.IdentitySecurityEventHeartbeatIntervalSeconds) * time.Second,
-		StaleAfter:        time.Duration(cfg.IdentitySecurityEventStaleAfterSeconds) * time.Second,
-		ClockSkew:         time.Duration(cfg.IdentitySecurityEventClockSkewSeconds) * time.Second,
-	}, securityEventMetrics)
-	server.identityRuntime = identityruntime.NewManager(db, runtimeBuilder)
-	if err := server.identityRuntime.LoadActive(ctx); err != nil && logger != nil {
-		logger.Error("load active identity runtime failed; local management login remains available", "error_category", "identity_runtime_load_failed")
-	}
-	server.identityPairing = identity.NewPairingService(db, secretStore, func(baseURL string) (identity.OnboardingRemote, error) {
-		return identity.NewOnboardingClient(baseURL, nil, cfg.AppEnv)
-	}, server.identityRuntime, cfg.AppEnv)
-	server.reconcileCanonicalIdentityPairing()
-	go server.runIdentityPairingCoordinator()
-	server.auth.OIDCVerifierProvider = func() *auth.OIDCVerifier {
-		if runtime := server.identityRuntime.Current(); runtime != nil {
-			return runtime.Verifier
-		}
-		return nil
-	}
-	server.auth.OIDCSessionResolverProvider = func(ctx context.Context, sessionID string) (*auth.User, error) {
-		runtime := server.identityRuntime.Current()
-		if runtime == nil || runtime.Sessions == nil {
-			return nil, auth.ErrUnauthorized
-		}
-		user, resolveErr := runtime.Sessions.Resolve(ctx, sessionID)
-		return user, oidcSessionRequestError(resolveErr)
-	}
-	server.auth.LegacyJWTEnabledProvider = func() bool {
-		return server.identityRuntime.LegacyJWTEnabled()
+	if cfg.RunsPublicHTTP() {
+		server.initializeIdentityRuntime(ctx, db, securityEventMetrics)
 	}
 	server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey
 	server.runner.StartAdmissionNotifier(ctx)
@@ -359,6 +324,54 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
 	return server.recover(server.cors(server.protectOIDCSessionCookie(mux)))
 }

+func (server *Server) initializeIdentityRuntime(
+	ctx context.Context,
+	db *store.Store,
+	securityEventMetrics *ssfreceiver.Metrics,
+) {
+	secretStore, err := identitySecretStore(server.cfg)
+	if err != nil {
+		panic("invalid identity SecretStore: " + err.Error())
+	}
+	go ssfreceiver.RunSecurityEventRetirementWorker(ctx, db)
+	go ssfreceiver.RunIdentitySecretCleanupWorker(ctx, db, secretStore)
+	runtimeBuilder := identityruntime.NewRuntimeBuilder(ctx, db, secretStore, identityruntime.RuntimeBuilderConfig{
+		AppEnv: server.cfg.AppEnv, JWKSCacheTTL: 5 * time.Minute,
+		HeartbeatInterval: time.Duration(server.cfg.IdentitySecurityEventHeartbeatIntervalSeconds) * time.Second,
+		StaleAfter:        time.Duration(server.cfg.IdentitySecurityEventStaleAfterSeconds) * time.Second,
+		ClockSkew:         time.Duration(server.cfg.IdentitySecurityEventClockSkewSeconds) * time.Second,
+	}, securityEventMetrics)
+	server.identityRuntime = identityruntime.NewManager(db, runtimeBuilder)
+	if err := server.identityRuntime.LoadActive(ctx); err != nil && server.logger != nil {
+		server.logger.Error(
+			"load active identity runtime failed; local management login remains available",
+			"error_category", "identity_runtime_load_failed",
+			"error", err,
+		)
+	}
+	server.identityPairing = identity.NewPairingService(db, secretStore, func(baseURL string) (identity.OnboardingRemote, error) {
+		return identity.NewOnboardingClient(baseURL, nil, server.cfg.AppEnv)
+	}, server.identityRuntime, server.cfg.AppEnv)
+	go server.runIdentityPairingCoordinator()
+	server.auth.OIDCVerifierProvider = func() *auth.OIDCVerifier {
+		if runtime := server.identityRuntime.Current(); runtime != nil {
+			return runtime.Verifier
+		}
+		return nil
+	}
+	server.auth.OIDCSessionResolverProvider = func(ctx context.Context, sessionID string) (*auth.User, error) {
+		runtime := server.identityRuntime.Current()
+		if runtime == nil || runtime.Sessions == nil {
+			return nil, auth.ErrUnauthorized
+		}
+		user, resolveErr := runtime.Sessions.Resolve(ctx, sessionID)
+		return user, oidcSessionRequestError(resolveErr)
+	}
+	server.auth.LegacyJWTEnabledProvider = func() bool {
+		return server.identityRuntime.LegacyJWTEnabled()
+	}
+}
+
 func identitySecretStore(cfg config.Config) (ssfreceiver.SecretStore, error) {
 	switch strings.ToLower(strings.TrimSpace(cfg.IdentitySecretStore)) {
 	case "", "file":