fix(cluster): 修复身份事务阻塞与 Worker 容量抖动
将身份协调和安全事件心跳改为 PostgreSQL 单 Leader 执行,并从独立 Worker 进程中移除身份运行时,避免多副本重复写同一状态。 为安全事件事务增加锁等待、空闲事务超时及独立回滚上下文;Worker 需连续丢失六次心跳后才判定失效,降低跨节点抖动导致的容量反复扩缩。 验证:Go 全量测试、go vet、Race 聚焦测试、PostgreSQL 18 Leader/安全事件/Worker 分配集成测试及迁移测试通过。
This commit is contained in:
@@ -15,6 +15,8 @@ import (
|
||||
type fakeStateRepository struct {
|
||||
mutex sync.Mutex
|
||||
stateHash []byte
|
||||
beginCount int
|
||||
beginSignal chan struct{}
|
||||
verification chan struct{}
|
||||
evaluation store.SecurityEventEvaluation
|
||||
}
|
||||
@@ -25,6 +27,13 @@ func (*fakeStateRepository) EnsureSecurityEventStreamState(context.Context, stri
|
||||
func (f *fakeStateRepository) BeginSecurityEventVerification(_ context.Context, _, _ string, hash []byte, _ time.Time) error {
|
||||
f.mutex.Lock()
|
||||
f.stateHash = append([]byte(nil), hash...)
|
||||
f.beginCount++
|
||||
if f.beginSignal != nil {
|
||||
select {
|
||||
case f.beginSignal <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
f.mutex.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -38,6 +47,105 @@ func (f *fakeStateRepository) EvaluateOIDCSecurityEvent(context.Context, string,
|
||||
return f.evaluation, nil
|
||||
}
|
||||
|
||||
type deniedHeartbeatLeadershipRepository struct {
|
||||
*fakeStateRepository
|
||||
}
|
||||
|
||||
func (*deniedHeartbeatLeadershipRepository) TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
type fakeHeartbeatLeadership struct {
|
||||
released chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (*fakeHeartbeatLeadership) KeepAlive(context.Context) error { return nil }
|
||||
func (leadership *fakeHeartbeatLeadership) Release() {
|
||||
leadership.once.Do(func() { close(leadership.released) })
|
||||
}
|
||||
|
||||
type grantedHeartbeatLeadershipRepository struct {
|
||||
*fakeStateRepository
|
||||
leadership *fakeHeartbeatLeadership
|
||||
}
|
||||
|
||||
func (repository *grantedHeartbeatLeadershipRepository) TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error) {
|
||||
return repository.leadership, true, nil
|
||||
}
|
||||
|
||||
func TestServiceFollowerDoesNotRunVerificationHeartbeat(t *testing.T) {
|
||||
repository := &deniedHeartbeatLeadershipRepository{fakeStateRepository: &fakeStateRepository{}}
|
||||
service, err := NewService(repository, ServiceConfig{
|
||||
Issuer: "https://auth.test.example/ssf", Audience: "urn:easyai:ssf:receiver:app",
|
||||
StreamID: "00000000-0000-4000-8000-000000000001",
|
||||
ManagementTokenURL: "https://auth.test.example/token",
|
||||
ManagementClientID: "gateway-ssf", ManagementSecret: "management-secret",
|
||||
HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
service.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
<-done
|
||||
repository.mutex.Lock()
|
||||
defer repository.mutex.Unlock()
|
||||
if repository.beginCount != 0 {
|
||||
t.Fatalf("follower persisted %d verification heartbeats", repository.beginCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceLeaderRunsHeartbeatAndReleasesLeadership(t *testing.T) {
|
||||
repository := &grantedHeartbeatLeadershipRepository{
|
||||
fakeStateRepository: &fakeStateRepository{beginSignal: make(chan struct{}, 1)},
|
||||
leadership: &fakeHeartbeatLeadership{released: make(chan struct{})},
|
||||
}
|
||||
tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/token" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"access_token":"management-token","token_type":"Bearer","expires_in":300}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer tokenServer.Close()
|
||||
service, err := NewService(repository, ServiceConfig{
|
||||
Issuer: tokenServer.URL, Audience: "urn:easyai:ssf:receiver:app",
|
||||
StreamID: "00000000-0000-4000-8000-000000000001",
|
||||
ManagementTokenURL: tokenServer.URL + "/token",
|
||||
ManagementClientID: "gateway-ssf", ManagementSecret: "management-secret",
|
||||
HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour, HTTPClient: tokenServer.Client(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
service.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-repository.beginSignal:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("leader did not persist a verification heartbeat")
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
select {
|
||||
case <-repository.leadership.released:
|
||||
default:
|
||||
t.Fatal("leader did not release PostgreSQL leadership")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceRequestsVerificationWithMachineTokenAndPersistsStateFirst(t *testing.T) {
|
||||
repository := &fakeStateRepository{verification: make(chan struct{}, 1)}
|
||||
var server *httptest.Server
|
||||
|
||||
Reference in New Issue
Block a user