BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(cluster): 修复身份事务阻塞与 Worker 容量抖动

Browse Source 
将身份协调和安全事件心跳改为 PostgreSQL 单 Leader 执行，并从独立 Worker 进程中移除身份运行时，避免多副本重复写同一状态。

为安全事件事务增加锁等待、空闲事务超时及独立回滚上下文；Worker 需连续丢失六次心跳后才判定失效，降低跨节点抖动导致的容量反复扩缩。

验证：Go 全量测试、go vet、Race 聚焦测试、PostgreSQL 18 Leader/安全事件/Worker 分配集成测试及迁移测试通过。
This commit is contained in:
wangbo
2026-07-29 23:04:41 +08:00
parent 3886048e0f
commit 91451b3c86
10 changed files with 497 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/httpapi/server.go
+50 -37
View File
@@ -81,43 +81,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
}
server.auth.ServerMainInternalKey = cfg.ServerMainInternalKey
server.auth.ServerMainInternalSecret = cfg.ServerMainInternalSecret
secretStore, err := identitySecretStore(cfg)
if err != nil {
panic("invalid identity SecretStore: " + err.Error())
}
go ssfreceiver.RunSecurityEventRetirementWorker(ctx, db)
go ssfreceiver.RunIdentitySecretCleanupWorker(ctx, db, secretStore)
runtimeBuilder := identityruntime.NewRuntimeBuilder(ctx, db, secretStore, identityruntime.RuntimeBuilderConfig{
AppEnv: cfg.AppEnv, JWKSCacheTTL: 5 * time.Minute,
HeartbeatInterval: time.Duration(cfg.IdentitySecurityEventHeartbeatIntervalSeconds) * time.Second,
StaleAfter: time.Duration(cfg.IdentitySecurityEventStaleAfterSeconds) * time.Second,
ClockSkew: time.Duration(cfg.IdentitySecurityEventClockSkewSeconds) * time.Second,
}, securityEventMetrics)
server.identityRuntime = identityruntime.NewManager(db, runtimeBuilder)
if err := server.identityRuntime.LoadActive(ctx); err != nil && logger != nil {
logger.Error("load active identity runtime failed; local management login remains available", "error_category", "identity_runtime_load_failed")
}
server.identityPairing = identity.NewPairingService(db, secretStore, func(baseURL string) (identity.OnboardingRemote, error) {
return identity.NewOnboardingClient(baseURL, nil, cfg.AppEnv)
}, server.identityRuntime, cfg.AppEnv)
server.reconcileCanonicalIdentityPairing()
go server.runIdentityPairingCoordinator()
server.auth.OIDCVerifierProvider = func() *auth.OIDCVerifier {
if runtime := server.identityRuntime.Current(); runtime != nil {
return runtime.Verifier
}
return nil
}
server.auth.OIDCSessionResolverProvider = func(ctx context.Context, sessionID string) (*auth.User, error) {
runtime := server.identityRuntime.Current()
if runtime == nil || runtime.Sessions == nil {
return nil, auth.ErrUnauthorized
}
user, resolveErr := runtime.Sessions.Resolve(ctx, sessionID)
return user, oidcSessionRequestError(resolveErr)
}
server.auth.LegacyJWTEnabledProvider = func() bool {
return server.identityRuntime.LegacyJWTEnabled()
if cfg.RunsPublicHTTP() {
server.initializeIdentityRuntime(ctx, db, securityEventMetrics)
}
server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey
server.runner.StartAdmissionNotifier(ctx)
@@ -359,6 +324,54 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
return server.recover(server.cors(server.protectOIDCSessionCookie(mux)))
}
func (server *Server) initializeIdentityRuntime(
ctx context.Context,
db *store.Store,
securityEventMetrics *ssfreceiver.Metrics,
) {
secretStore, err := identitySecretStore(server.cfg)
if err != nil {
panic("invalid identity SecretStore: " + err.Error())
}
go ssfreceiver.RunSecurityEventRetirementWorker(ctx, db)
go ssfreceiver.RunIdentitySecretCleanupWorker(ctx, db, secretStore)
runtimeBuilder := identityruntime.NewRuntimeBuilder(ctx, db, secretStore, identityruntime.RuntimeBuilderConfig{
AppEnv: server.cfg.AppEnv, JWKSCacheTTL: 5 * time.Minute,
HeartbeatInterval: time.Duration(server.cfg.IdentitySecurityEventHeartbeatIntervalSeconds) * time.Second,
StaleAfter: time.Duration(server.cfg.IdentitySecurityEventStaleAfterSeconds) * time.Second,
ClockSkew: time.Duration(server.cfg.IdentitySecurityEventClockSkewSeconds) * time.Second,
}, securityEventMetrics)
server.identityRuntime = identityruntime.NewManager(db, runtimeBuilder)
if err := server.identityRuntime.LoadActive(ctx); err != nil && server.logger != nil {
server.logger.Error(
"load active identity runtime failed; local management login remains available",
"error_category", "identity_runtime_load_failed",
"error", err,
)
}
server.identityPairing = identity.NewPairingService(db, secretStore, func(baseURL string) (identity.OnboardingRemote, error) {
return identity.NewOnboardingClient(baseURL, nil, server.cfg.AppEnv)
}, server.identityRuntime, server.cfg.AppEnv)
go server.runIdentityPairingCoordinator()
server.auth.OIDCVerifierProvider = func() *auth.OIDCVerifier {
if runtime := server.identityRuntime.Current(); runtime != nil {
return runtime.Verifier
}
return nil
}
server.auth.OIDCSessionResolverProvider = func(ctx context.Context, sessionID string) (*auth.User, error) {
runtime := server.identityRuntime.Current()
if runtime == nil || runtime.Sessions == nil {
return nil, auth.ErrUnauthorized
}
user, resolveErr := runtime.Sessions.Resolve(ctx, sessionID)
return user, oidcSessionRequestError(resolveErr)
}
server.auth.LegacyJWTEnabledProvider = func() bool {
return server.identityRuntime.LegacyJWTEnabled()
}
}
func identitySecretStore(cfg config.Config) (ssfreceiver.SecretStore, error) {
switch strings.ToLower(strings.TrimSpace(cfg.IdentitySecretStore)) {
case "", "file":