fix(cluster): 修复身份事务阻塞与 Worker 容量抖动
将身份协调和安全事件心跳改为 PostgreSQL 单 Leader 执行,并从独立 Worker 进程中移除身份运行时,避免多副本重复写同一状态。 为安全事件事务增加锁等待、空闲事务超时及独立回滚上下文;Worker 需连续丢失六次心跳后才判定失效,降低跨节点抖动导致的容量反复扩缩。 验证:Go 全量测试、go vet、Race 聚焦测试、PostgreSQL 18 Leader/安全事件/Worker 分配集成测试及迁移测试通过。
This commit is contained in:
@@ -52,6 +52,16 @@ type Service struct {
|
||||
metrics *Metrics
|
||||
}
|
||||
|
||||
type heartbeatLeadershipRepository interface {
|
||||
TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error)
|
||||
}
|
||||
|
||||
const (
|
||||
heartbeatLeadershipRetryInterval = 5 * time.Second
|
||||
heartbeatLeadershipKeepAlive = 5 * time.Second
|
||||
heartbeatExecutionTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
func (s *Service) SetMetrics(metrics *Metrics) { s.metrics = metrics }
|
||||
|
||||
func NewService(repository StateRepository, config ServiceConfig) (*Service, error) {
|
||||
@@ -108,7 +118,34 @@ func (s *Service) Evaluate(ctx context.Context, identity auth.OIDCSecurityEventI
|
||||
}
|
||||
|
||||
func (s *Service) run(ctx context.Context) {
|
||||
s.sendVerification(ctx)
|
||||
elector, distributed := s.repository.(heartbeatLeadershipRepository)
|
||||
if !distributed {
|
||||
s.runHeartbeatLoop(ctx)
|
||||
return
|
||||
}
|
||||
retry := time.NewTicker(heartbeatLeadershipRetryInterval)
|
||||
defer retry.Stop()
|
||||
for {
|
||||
acquireCtx, cancel := context.WithTimeout(ctx, heartbeatLeadershipKeepAlive)
|
||||
leadership, acquired, err := elector.TryAcquireSecurityEventHeartbeatLeadership(acquireCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
_ = s.repository.RecordSecurityEventHeartbeatFailure(ctx, s.config.Issuer, s.config.Audience, "leader_election_failed")
|
||||
} else if acquired {
|
||||
if !s.runLeaderHeartbeatLoop(ctx, leadership) {
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-retry.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) runHeartbeatLoop(ctx context.Context) {
|
||||
s.sendScheduledVerification(ctx)
|
||||
ticker := time.NewTicker(s.config.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
@@ -116,11 +153,42 @@ func (s *Service) run(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.sendVerification(ctx)
|
||||
s.sendScheduledVerification(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) runLeaderHeartbeatLoop(ctx context.Context, leadership store.Leadership) bool {
|
||||
defer leadership.Release()
|
||||
s.sendScheduledVerification(ctx)
|
||||
heartbeat := time.NewTicker(s.config.HeartbeatInterval)
|
||||
defer heartbeat.Stop()
|
||||
keepalive := time.NewTicker(heartbeatLeadershipKeepAlive)
|
||||
defer keepalive.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-heartbeat.C:
|
||||
s.sendScheduledVerification(ctx)
|
||||
case <-keepalive.C:
|
||||
keepaliveCtx, cancel := context.WithTimeout(ctx, heartbeatLeadershipKeepAlive)
|
||||
err := leadership.KeepAlive(keepaliveCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
_ = s.repository.RecordSecurityEventHeartbeatFailure(ctx, s.config.Issuer, s.config.Audience, "leader_keepalive_failed")
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) sendScheduledVerification(ctx context.Context) {
|
||||
verificationCtx, cancel := context.WithTimeout(ctx, heartbeatExecutionTimeout)
|
||||
defer cancel()
|
||||
s.sendVerification(verificationCtx)
|
||||
}
|
||||
|
||||
func (s *Service) sendVerification(ctx context.Context) {
|
||||
outcome := "failed"
|
||||
defer func() {
|
||||
|
||||
Reference in New Issue
Block a user