Files
easyai-ai-gateway/apps/api/internal/securityevents/service_test.go
T
wangbo 91451b3c86 fix(cluster): 修复身份事务阻塞与 Worker 容量抖动
将身份协调和安全事件心跳改为 PostgreSQL 单 Leader 执行,并从独立 Worker 进程中移除身份运行时,避免多副本重复写同一状态。

为安全事件事务增加锁等待、空闲事务超时及独立回滚上下文;Worker 需连续丢失六次心跳后才判定失效,降低跨节点抖动导致的容量反复扩缩。

验证:Go 全量测试、go vet、Race 聚焦测试、PostgreSQL 18 Leader/安全事件/Worker 分配集成测试及迁移测试通过。
2026-07-29 23:04:41 +08:00

195 lines
6.3 KiB
Go

package securityevents
import (
"context"
"crypto/sha256"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
type fakeStateRepository struct {
mutex sync.Mutex
stateHash []byte
beginCount int
beginSignal chan struct{}
verification chan struct{}
evaluation store.SecurityEventEvaluation
}
func (*fakeStateRepository) EnsureSecurityEventStreamState(context.Context, string, string, string) error {
return nil
}
func (f *fakeStateRepository) BeginSecurityEventVerification(_ context.Context, _, _ string, hash []byte, _ time.Time) error {
f.mutex.Lock()
f.stateHash = append([]byte(nil), hash...)
f.beginCount++
if f.beginSignal != nil {
select {
case f.beginSignal <- struct{}{}:
default:
}
}
f.mutex.Unlock()
return nil
}
func (*fakeStateRepository) RecordSecurityEventHeartbeatFailure(context.Context, string, string, string) error {
return nil
}
func (*fakeStateRepository) AdvanceSecurityEventStreamState(context.Context, string, string, time.Time, time.Duration) error {
return nil
}
func (f *fakeStateRepository) EvaluateOIDCSecurityEvent(context.Context, string, string, string, string, string, string, time.Time, time.Time, time.Duration) (store.SecurityEventEvaluation, error) {
return f.evaluation, nil
}
type deniedHeartbeatLeadershipRepository struct {
*fakeStateRepository
}
func (*deniedHeartbeatLeadershipRepository) TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error) {
return nil, false, nil
}
type fakeHeartbeatLeadership struct {
released chan struct{}
once sync.Once
}
func (*fakeHeartbeatLeadership) KeepAlive(context.Context) error { return nil }
func (leadership *fakeHeartbeatLeadership) Release() {
leadership.once.Do(func() { close(leadership.released) })
}
type grantedHeartbeatLeadershipRepository struct {
*fakeStateRepository
leadership *fakeHeartbeatLeadership
}
func (repository *grantedHeartbeatLeadershipRepository) TryAcquireSecurityEventHeartbeatLeadership(context.Context) (store.Leadership, bool, error) {
return repository.leadership, true, nil
}
func TestServiceFollowerDoesNotRunVerificationHeartbeat(t *testing.T) {
repository := &deniedHeartbeatLeadershipRepository{fakeStateRepository: &fakeStateRepository{}}
service, err := NewService(repository, ServiceConfig{
Issuer: "https://auth.test.example/ssf", Audience: "urn:easyai:ssf:receiver:app",
StreamID: "00000000-0000-4000-8000-000000000001",
ManagementTokenURL: "https://auth.test.example/token",
ManagementClientID: "gateway-ssf", ManagementSecret: "management-secret",
HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour,
})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
service.Run(ctx)
close(done)
}()
time.Sleep(100 * time.Millisecond)
cancel()
<-done
repository.mutex.Lock()
defer repository.mutex.Unlock()
if repository.beginCount != 0 {
t.Fatalf("follower persisted %d verification heartbeats", repository.beginCount)
}
}
func TestServiceLeaderRunsHeartbeatAndReleasesLeadership(t *testing.T) {
repository := &grantedHeartbeatLeadershipRepository{
fakeStateRepository: &fakeStateRepository{beginSignal: make(chan struct{}, 1)},
leadership: &fakeHeartbeatLeadership{released: make(chan struct{})},
}
tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/token" {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"access_token":"management-token","token_type":"Bearer","expires_in":300}`))
return
}
w.WriteHeader(http.StatusNoContent)
}))
defer tokenServer.Close()
service, err := NewService(repository, ServiceConfig{
Issuer: tokenServer.URL, Audience: "urn:easyai:ssf:receiver:app",
StreamID: "00000000-0000-4000-8000-000000000001",
ManagementTokenURL: tokenServer.URL + "/token",
ManagementClientID: "gateway-ssf", ManagementSecret: "management-secret",
HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour, HTTPClient: tokenServer.Client(),
})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
service.Run(ctx)
close(done)
}()
select {
case <-repository.beginSignal:
case <-time.After(2 * time.Second):
t.Fatal("leader did not persist a verification heartbeat")
}
cancel()
<-done
select {
case <-repository.leadership.released:
default:
t.Fatal("leader did not release PostgreSQL leadership")
}
}
func TestServiceRequestsVerificationWithMachineTokenAndPersistsStateFirst(t *testing.T) {
repository := &fakeStateRepository{verification: make(chan struct{}, 1)}
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/token":
clientID, secret, ok := r.BasicAuth()
if !ok || clientID != "gateway-ssf" || secret != "management-secret" {
t.Fatal("machine client authentication is missing")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"access_token":"management-token","token_type":"Bearer","expires_in":300}`))
case "/ssf/v1/verify":
if r.Header.Get("Authorization") != "Bearer management-token" {
t.Fatal("verification bearer is missing")
}
repository.mutex.Lock()
persisted := len(repository.stateHash) == sha256.Size
repository.mutex.Unlock()
if !persisted {
t.Fatal("verification state was not persisted before request")
}
w.WriteHeader(http.StatusNoContent)
repository.verification <- struct{}{}
}
}))
defer server.Close()
service, err := NewService(repository, ServiceConfig{
Issuer: server.URL + "/ssf", Audience: "urn:easyai:ssf:receiver:app", StreamID: "00000000-0000-4000-8000-000000000001",
ManagementTokenURL: server.URL + "/token", ManagementClientID: "gateway-ssf", ManagementSecret: "management-secret",
HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour, HTTPClient: server.Client(),
})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := service.Start(ctx); err != nil {
t.Fatal(err)
}
select {
case <-repository.verification:
case <-time.After(2 * time.Second):
t.Fatal("verification was not requested")
}
}