fix(cluster): 修复身份事务阻塞与 Worker 容量抖动

将身份协调和安全事件心跳改为 PostgreSQL 单 Leader 执行,并从独立 Worker 进程中移除身份运行时,避免多副本重复写同一状态。

为安全事件事务增加锁等待、空闲事务超时及独立回滚上下文;Worker 需连续丢失六次心跳后才判定失效,降低跨节点抖动导致的容量反复扩缩。

验证:Go 全量测试、go vet、Race 聚焦测试、PostgreSQL 18 Leader/安全事件/Worker 分配集成测试及迁移测试通过。
This commit is contained in:
2026-07-29 23:04:41 +08:00
parent 3886048e0f
commit 91451b3c86
10 changed files with 497 additions and 76 deletions
+70 -2
View File
@@ -52,6 +52,16 @@ type Service struct {
metrics *Metrics
}
type heartbeatLeadershipRepository interface {
TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error)
}
const (
heartbeatLeadershipRetryInterval = 5 * time.Second
heartbeatLeadershipKeepAlive = 5 * time.Second
heartbeatExecutionTimeout = 30 * time.Second
)
func (s *Service) SetMetrics(metrics *Metrics) { s.metrics = metrics }
func NewService(repository StateRepository, config ServiceConfig) (*Service, error) {
@@ -108,7 +118,34 @@ func (s *Service) Evaluate(ctx context.Context, identity auth.OIDCSecurityEventI
}
func (s *Service) run(ctx context.Context) {
s.sendVerification(ctx)
elector, distributed := s.repository.(heartbeatLeadershipRepository)
if !distributed {
s.runHeartbeatLoop(ctx)
return
}
retry := time.NewTicker(heartbeatLeadershipRetryInterval)
defer retry.Stop()
for {
acquireCtx, cancel := context.WithTimeout(ctx, heartbeatLeadershipKeepAlive)
leadership, acquired, err := elector.TryAcquireSecurityEventHeartbeatLeadership(acquireCtx)
cancel()
if err != nil {
_ = s.repository.RecordSecurityEventHeartbeatFailure(ctx, s.config.Issuer, s.config.Audience, "leader_election_failed")
} else if acquired {
if !s.runLeaderHeartbeatLoop(ctx, leadership) {
return
}
}
select {
case <-ctx.Done():
return
case <-retry.C:
}
}
}
func (s *Service) runHeartbeatLoop(ctx context.Context) {
s.sendScheduledVerification(ctx)
ticker := time.NewTicker(s.config.HeartbeatInterval)
defer ticker.Stop()
for {
@@ -116,11 +153,42 @@ func (s *Service) run(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
s.sendVerification(ctx)
s.sendScheduledVerification(ctx)
}
}
}
func (s *Service) runLeaderHeartbeatLoop(ctx context.Context, leadership store.Leadership) bool {
defer leadership.Release()
s.sendScheduledVerification(ctx)
heartbeat := time.NewTicker(s.config.HeartbeatInterval)
defer heartbeat.Stop()
keepalive := time.NewTicker(heartbeatLeadershipKeepAlive)
defer keepalive.Stop()
for {
select {
case <-ctx.Done():
return false
case <-heartbeat.C:
s.sendScheduledVerification(ctx)
case <-keepalive.C:
keepaliveCtx, cancel := context.WithTimeout(ctx, heartbeatLeadershipKeepAlive)
err := leadership.KeepAlive(keepaliveCtx)
cancel()
if err != nil {
_ = s.repository.RecordSecurityEventHeartbeatFailure(ctx, s.config.Issuer, s.config.Audience, "leader_keepalive_failed")
return true
}
}
}
}
func (s *Service) sendScheduledVerification(ctx context.Context) {
verificationCtx, cancel := context.WithTimeout(ctx, heartbeatExecutionTimeout)
defer cancel()
s.sendVerification(verificationCtx)
}
func (s *Service) sendVerification(ctx context.Context) {
outcome := "failed"
defer func() {