fix(cluster): 修复身份事务阻塞与 Worker 容量抖动
将身份协调和安全事件心跳改为 PostgreSQL 单 Leader 执行，并从独立 Worker 进程中移除身份运行时，避免多副本重复写同一状态。 为安全事件事务增加锁等待、空闲事务超时及独立回滚上下文；Worker 需连续丢失六次心跳后才判定失效，降低跨节点抖动导致的容量反复扩缩。 验证：Go 全量测试、go vet、Race 聚焦测试、PostgreSQL 18 Leader/安全事件/Worker 分配集成测试及迁移测试通过。
This commit is contained in:
@@ -52,6 +52,16 @@ type Service struct {
|
||||
metrics *Metrics
|
||||
}
|
||||
|
||||
type heartbeatLeadershipRepository interface {
|
||||
TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error)
|
||||
}
|
||||
|
||||
const (
|
||||
heartbeatLeadershipRetryInterval = 5 * time.Second
|
||||
heartbeatLeadershipKeepAlive = 5 * time.Second
|
||||
heartbeatExecutionTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
func (s *Service) SetMetrics(metrics *Metrics) { s.metrics = metrics }
|
||||
|
||||
func NewService(repository StateRepository, config ServiceConfig) (*Service, error) {
|
||||
@@ -108,7 +118,34 @@ func (s *Service) Evaluate(ctx context.Context, identity auth.OIDCSecurityEventI
|
||||
}
|
||||
|
||||
func (s *Service) run(ctx context.Context) {
|
||||
s.sendVerification(ctx)
|
||||
elector, distributed := s.repository.(heartbeatLeadershipRepository)
|
||||
if !distributed {
|
||||
s.runHeartbeatLoop(ctx)
|
||||
return
|
||||
}
|
||||
retry := time.NewTicker(heartbeatLeadershipRetryInterval)
|
||||
defer retry.Stop()
|
||||
for {
|
||||
acquireCtx, cancel := context.WithTimeout(ctx, heartbeatLeadershipKeepAlive)
|
||||
leadership, acquired, err := elector.TryAcquireSecurityEventHeartbeatLeadership(acquireCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
_ = s.repository.RecordSecurityEventHeartbeatFailure(ctx, s.config.Issuer, s.config.Audience, "leader_election_failed")
|
||||
} else if acquired {
|
||||
if !s.runLeaderHeartbeatLoop(ctx, leadership) {
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-retry.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) runHeartbeatLoop(ctx context.Context) {
|
||||
s.sendScheduledVerification(ctx)
|
||||
ticker := time.NewTicker(s.config.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
@@ -116,11 +153,42 @@ func (s *Service) run(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.sendVerification(ctx)
|
||||
s.sendScheduledVerification(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) runLeaderHeartbeatLoop(ctx context.Context, leadership store.Leadership) bool {
|
||||
defer leadership.Release()
|
||||
s.sendScheduledVerification(ctx)
|
||||
heartbeat := time.NewTicker(s.config.HeartbeatInterval)
|
||||
defer heartbeat.Stop()
|
||||
keepalive := time.NewTicker(heartbeatLeadershipKeepAlive)
|
||||
defer keepalive.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-heartbeat.C:
|
||||
s.sendScheduledVerification(ctx)
|
||||
case <-keepalive.C:
|
||||
keepaliveCtx, cancel := context.WithTimeout(ctx, heartbeatLeadershipKeepAlive)
|
||||
err := leadership.KeepAlive(keepaliveCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
_ = s.repository.RecordSecurityEventHeartbeatFailure(ctx, s.config.Issuer, s.config.Audience, "leader_keepalive_failed")
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) sendScheduledVerification(ctx context.Context) {
|
||||
verificationCtx, cancel := context.WithTimeout(ctx, heartbeatExecutionTimeout)
|
||||
defer cancel()
|
||||
s.sendVerification(verificationCtx)
|
||||
}
|
||||
|
||||
func (s *Service) sendVerification(ctx context.Context) {
|
||||
outcome := "failed"
|
||||
defer func() {
|
||||
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
type fakeStateRepository struct {
|
||||
mutex sync.Mutex
|
||||
stateHash []byte
|
||||
beginCount int
|
||||
beginSignal chan struct{}
|
||||
verification chan struct{}
|
||||
evaluation store.SecurityEventEvaluation
|
||||
}
|
||||
@@ -25,6 +27,13 @@ func (*fakeStateRepository) EnsureSecurityEventStreamState(context.Context, stri
|
||||
func (f *fakeStateRepository) BeginSecurityEventVerification(_ context.Context, _, _ string, hash []byte, _ time.Time) error {
|
||||
f.mutex.Lock()
|
||||
f.stateHash = append([]byte(nil), hash...)
|
||||
f.beginCount++
|
||||
if f.beginSignal != nil {
|
||||
select {
|
||||
case f.beginSignal <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
f.mutex.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -38,6 +47,105 @@ func (f *fakeStateRepository) EvaluateOIDCSecurityEvent(context.Context, string,
|
||||
return f.evaluation, nil
|
||||
}
|
||||
|
||||
type deniedHeartbeatLeadershipRepository struct {
|
||||
*fakeStateRepository
|
||||
}
|
||||
|
||||
func (*deniedHeartbeatLeadershipRepository) TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
type fakeHeartbeatLeadership struct {
|
||||
released chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (*fakeHeartbeatLeadership) KeepAlive(context.Context) error { return nil }
|
||||
func (leadership *fakeHeartbeatLeadership) Release() {
|
||||
leadership.once.Do(func() { close(leadership.released) })
|
||||
}
|
||||
|
||||
type grantedHeartbeatLeadershipRepository struct {
|
||||
*fakeStateRepository
|
||||
leadership *fakeHeartbeatLeadership
|
||||
}
|
||||
|
||||
func (repository *grantedHeartbeatLeadershipRepository) TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error) {
|
||||
return repository.leadership, true, nil
|
||||
}
|
||||
|
||||
func TestServiceFollowerDoesNotRunVerificationHeartbeat(t *testing.T) {
|
||||
repository := &deniedHeartbeatLeadershipRepository{fakeStateRepository: &fakeStateRepository{}}
|
||||
service, err := NewService(repository, ServiceConfig{
|
||||
Issuer: "https://auth.test.example/ssf", Audience: "urn:easyai:ssf:receiver:app",
|
||||
StreamID: "00000000-0000-4000-8000-000000000001",
|
||||
ManagementTokenURL: "https://auth.test.example/token",
|
||||
ManagementClientID: "gateway-ssf", ManagementSecret: "management-secret",
|
||||
HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
service.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
<-done
|
||||
repository.mutex.Lock()
|
||||
defer repository.mutex.Unlock()
|
||||
if repository.beginCount != 0 {
|
||||
t.Fatalf("follower persisted %d verification heartbeats", repository.beginCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceLeaderRunsHeartbeatAndReleasesLeadership(t *testing.T) {
|
||||
repository := &grantedHeartbeatLeadershipRepository{
|
||||
fakeStateRepository: &fakeStateRepository{beginSignal: make(chan struct{}, 1)},
|
||||
leadership: &fakeHeartbeatLeadership{released: make(chan struct{})},
|
||||
}
|
||||
tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/token" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"access_token":"management-token","token_type":"Bearer","expires_in":300}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer tokenServer.Close()
|
||||
service, err := NewService(repository, ServiceConfig{
|
||||
Issuer: tokenServer.URL, Audience: "urn:easyai:ssf:receiver:app",
|
||||
StreamID: "00000000-0000-4000-8000-000000000001",
|
||||
ManagementTokenURL: tokenServer.URL + "/token",
|
||||
ManagementClientID: "gateway-ssf", ManagementSecret: "management-secret",
|
||||
HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour, HTTPClient: tokenServer.Client(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
service.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-repository.beginSignal:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("leader did not persist a verification heartbeat")
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
select {
|
||||
case <-repository.leadership.released:
|
||||
default:
|
||||
t.Fatal("leader did not release PostgreSQL leadership")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceRequestsVerificationWithMachineTokenAndPersistsStateFirst(t *testing.T) {
|
||||
repository := &fakeStateRepository{verification: make(chan struct{}, 1)}
|
||||
var server *httptest.Server
|
||||
|
||||
Reference in New Issue
Block a user