fix(identity): 完善统一认证配对恢复与安全退役

修复 credentials_saved 状态无法恢复、配对与激活并发冲突,以及 SSF 和身份 Secret 生命周期不完整的问题。新增持久化协调器、取消与清理状态机、事务级并发门禁、受控 SSF 凭据交接、禁用后的延迟 Secret 清理,并对生产环境统一认证及 Discovery 端点强制 HTTPS。

验证:go test ./...;go test -race ./internal/auth ./internal/identity ./internal/identityruntime ./internal/securityevents ./internal/httpapi ./internal/store -count=1;go vet ./...;真实 PostgreSQL 并发及清理成功/冲突回滚测试;pnpm openapi。
This commit is contained in:
2026-07-17 18:31:12 +08:00
parent cdfca61304
commit a312ad880d
55 changed files with 9225 additions and 419 deletions
+231 -21
View File
@@ -3,8 +3,10 @@ package identityruntime
import (
"context"
"errors"
"net/http"
"net/url"
"slices"
"strings"
"sync"
"time"
@@ -24,10 +26,23 @@ type RuntimeBuilderConfig struct {
}
type preparedSecurityRuntime struct {
manager *securityevents.ConnectionManager
cancel context.CancelFunc
manager *securityevents.ConnectionManager
cancel context.CancelFunc
receiverReady bool
blockedCategory string
}
type safeRuntimeError struct {
category string
cause error
}
func (err safeRuntimeError) Error() string {
return "identity runtime operation failed: " + err.category
}
func (err safeRuntimeError) Unwrap() error { return err.cause }
func (err safeRuntimeError) SafeErrorCategory() string { return err.category }
type RuntimeBuilder struct {
ctx context.Context
store *store.Store
@@ -54,7 +69,36 @@ func NewRuntimeBuilder(ctx context.Context, data *store.Store, secrets PairingSe
return &RuntimeBuilder{ctx: ctx, store: data, secrets: secrets, config: config, metrics: metrics, prepared: map[string]preparedSecurityRuntime{}}
}
func (builder *RuntimeBuilder) PreparedSecurityEventReceiver() http.Handler {
builder.mutex.Lock()
defer builder.mutex.Unlock()
if len(builder.prepared) != 1 {
return nil
}
for _, prepared := range builder.prepared {
if prepared.receiverReady {
return prepared.manager
}
}
return nil
}
func (builder *RuntimeBuilder) PreparedSecurityEventManager() *securityevents.ConnectionManager {
builder.mutex.Lock()
defer builder.mutex.Unlock()
if len(builder.prepared) != 1 {
return nil
}
for _, prepared := range builder.prepared {
return prepared.manager
}
return nil
}
func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revision) (*Runtime, error) {
if err := identity.ValidateRevisionURLs(revision, builder.config.AppEnv); err != nil {
return nil, err
}
if builder.store == nil || builder.secrets == nil || revision.Issuer == "" || revision.TenantID == "" ||
revision.Audience == "" || revision.RolePrefix == "" {
return nil, errors.New("identity runtime configuration is incomplete")
@@ -69,23 +113,28 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi
var securityManager *securityevents.ConnectionManager
if revision.SessionRevocation {
prepared := builder.takePreparedSecurityRuntime(revision.ID)
prepared := builder.preparedSecurityRuntime(revision.ID)
securityManager = prepared.manager
if prepared.cancel != nil {
runtime.close = func() {
if securityManager != nil {
if err := securityManager.ValidateConfiguredConnectionBinding(ctx); err != nil {
cancel()
prepared.cancel()
return nil, err
}
if !prepared.receiverReady {
cancel()
return nil, safeRuntimeError{category: "security_event_not_ready", cause: store.ErrSecurityEventConnectionConflict}
}
}
if securityManager == nil {
var err error
securityManager, err = builder.newSecurityEventManager(runtimeCtx, revision)
securityManager, err = builder.newSecurityEventManager(runtimeCtx, revision, true)
if err != nil {
cancel()
return nil, err
}
}
runtime.SecurityEvents = securityManager
runtime.securityEventDisconnector = securityManager
}
var evaluator func(context.Context, auth.OIDCSecurityEventIdentity) (auth.OIDCSecurityEventEvaluation, error)
@@ -100,6 +149,7 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi
return revision.MachineClientID, secret, err
}
verifier, err := auth.NewOIDCVerifier(auth.OIDCConfig{
AppEnv: builder.config.AppEnv,
Issuer: revision.Issuer, Audience: revision.Audience, TenantID: revision.TenantID,
RolePrefix: revision.RolePrefix, RequiredScopes: append([]string(nil), revision.Scopes...),
JWKSCacheTTL: builder.config.JWKSCacheTTL, IntrospectionEnabled: revision.TokenIntrospection,
@@ -134,6 +184,7 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi
return nil, err
}
client, err := auth.NewOIDCPublicClient(auth.OIDCPublicClientConfig{
AppEnv: builder.config.AppEnv,
Issuer: revision.Issuer, ClientID: revision.BrowserClientID,
RedirectURI: revision.PublicBaseURL + "/api/v1/auth/oidc/callback",
PostLogoutRedirectURI: revision.WebBaseURL + "/", Scopes: append([]string{"openid", "profile"}, revision.Scopes...),
@@ -161,46 +212,205 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi
}
func (builder *RuntimeBuilder) PrepareSecurityEvents(ctx context.Context, revision identity.Revision, managementSecret []byte) error {
if !revision.SessionRevocation || revision.SecurityEventIssuer == "" || revision.MachineClientID == "" {
if err := identity.ValidateRevisionURLs(revision, builder.config.AppEnv); err != nil {
return safeRuntimeError{category: "configuration_invalid", cause: err}
}
if !revision.SessionRevocation || revision.SecurityEventIssuer == "" || revision.SecurityEventAudience == "" || revision.MachineClientID == "" {
return errors.New("security event configuration is incomplete")
}
if prepared := builder.preparedSecurityRuntime(revision.ID); prepared.manager != nil && prepared.blockedCategory != "" {
view, err := prepared.manager.Get(ctx)
if err == nil {
category := prepared.blockedCategory
if view.LifecycleStatus == "retiring" || view.LifecycleStatus == "disconnect_pending" {
category = "retirement_pending"
}
return safeRuntimeError{category: category, cause: store.ErrSecurityEventConnectionConflict}
}
if !errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return safeSecurityEventRuntimeError(err)
}
builder.removePreparedSecurityRuntime(revision.ID, prepared)
}
runtimeCtx, cancel := context.WithCancel(builder.ctx)
manager, err := builder.newSecurityEventManager(runtimeCtx, revision)
manager, err := builder.newSecurityEventManager(runtimeCtx, revision, false)
if err != nil {
cancel()
return err
return safeRuntimeError{category: "configuration_invalid", cause: err}
}
secretCopy := append([]byte(nil), managementSecret...)
_, err = manager.Connect(ctx, revision.SecurityEventIssuer, revision.MachineClientID, secretCopy, "identity-pairing-ssf-"+revision.ID)
clear(secretCopy)
if err != nil {
// A conflicting persisted connection is itself the resource an
// administrator must safely retire. Keep this manager reachable by the
// recovery API even when there is no Active identity Runtime yet.
if errors.Is(err, store.ErrSecurityEventConnectionConflict) {
builder.replacePreparedSecurityRuntime(revision.ID, preparedSecurityRuntime{
manager: manager, cancel: cancel, blockedCategory: safeSecurityEventCategory(err, "connection_conflict"),
})
return safeSecurityEventRuntimeError(err)
}
cancel()
return err
return safeSecurityEventRuntimeError(err)
}
if err := manager.ValidateConfiguredConnectionBinding(ctx); err != nil {
builder.replacePreparedSecurityRuntime(revision.ID, preparedSecurityRuntime{
manager: manager, cancel: cancel, blockedCategory: safeSecurityEventCategory(err, "connection_binding_mismatch"),
})
return safeSecurityEventRuntimeError(err)
}
builder.replacePreparedSecurityRuntime(revision.ID, preparedSecurityRuntime{manager: manager, cancel: cancel, receiverReady: true})
return nil
}
func (builder *RuntimeBuilder) removePreparedSecurityRuntime(revisionID string, prepared preparedSecurityRuntime) {
builder.mutex.Lock()
previous := builder.prepared[revision.ID]
builder.prepared[revision.ID] = preparedSecurityRuntime{manager: manager, cancel: cancel}
current := builder.prepared[revisionID]
if current.manager == prepared.manager {
delete(builder.prepared, revisionID)
}
builder.mutex.Unlock()
if prepared.cancel != nil {
prepared.cancel()
}
}
func (builder *RuntimeBuilder) replacePreparedSecurityRuntime(revisionID string, replacement preparedSecurityRuntime) {
builder.mutex.Lock()
previous := builder.prepared[revisionID]
builder.prepared[revisionID] = replacement
builder.mutex.Unlock()
if previous.cancel != nil {
previous.cancel()
}
}
func (builder *RuntimeBuilder) CleanupPreparedSecurityEvents(ctx context.Context, revision identity.Revision) error {
builder.mutex.Lock()
prepared, exists := builder.prepared[revision.ID]
builder.mutex.Unlock()
if !exists {
runtimeCtx, cancel := context.WithCancel(builder.ctx)
manager, err := builder.newSecurityEventManager(runtimeCtx, revision, false)
if err != nil {
cancel()
return safeRuntimeError{category: "configuration_invalid", cause: err}
}
prepared = preparedSecurityRuntime{manager: manager, cancel: cancel}
}
err := prepared.manager.DiscardPreparedConnection(ctx, "identity-pairing-ssf-"+revision.ID)
if err != nil {
builder.mutex.Lock()
if _, alreadyStored := builder.prepared[revision.ID]; !alreadyStored {
builder.prepared[revision.ID] = prepared
}
builder.mutex.Unlock()
return safeSecurityEventRuntimeError(err)
}
builder.mutex.Lock()
current := builder.prepared[revision.ID]
if current.manager == prepared.manager {
delete(builder.prepared, revision.ID)
}
builder.mutex.Unlock()
if prepared.cancel != nil {
prepared.cancel()
}
return nil
}
func (builder *RuntimeBuilder) newSecurityEventManager(ctx context.Context, revision identity.Revision) (*securityevents.ConnectionManager, error) {
return securityevents.NewConnectionManager(ctx, builder.store, builder.secrets, securityevents.ConnectionManagerConfig{
AppEnv: builder.config.AppEnv, OIDCEnabled: true, OIDCIssuer: revision.Issuer, OIDCTenantID: revision.TenantID,
ManagementClientID: revision.MachineClientID, PublicBaseURL: revision.PublicBaseURL,
HeartbeatInterval: builder.config.HeartbeatInterval, StaleAfter: builder.config.StaleAfter, ClockSkew: builder.config.ClockSkew,
}, builder.metrics)
func (builder *RuntimeBuilder) RetireConflictingSecurityEvents(ctx context.Context, revision identity.Revision) error {
prepared := builder.preparedSecurityRuntime(revision.ID)
if prepared.manager == nil {
runtimeCtx, cancel := context.WithCancel(builder.ctx)
manager, err := builder.newSecurityEventManager(runtimeCtx, revision, false)
if err != nil {
cancel()
return safeRuntimeError{category: "configuration_invalid", cause: err}
}
prepared = preparedSecurityRuntime{manager: manager, cancel: cancel}
builder.replacePreparedSecurityRuntime(revision.ID, prepared)
}
if err := prepared.manager.RetireConflictingConnection(ctx, "identity-pairing-ssf-"+revision.ID); err != nil {
return safeSecurityEventRuntimeError(err)
}
return nil
}
func (builder *RuntimeBuilder) takePreparedSecurityRuntime(revisionID string) preparedSecurityRuntime {
// RecoverSecurityEventDisconnector reconstructs only the SSF management
// surface needed to disable a fail-closed Active Revision. It deliberately
// avoids OIDC discovery, JIT, BFF Session and tenant Runtime construction.
func (builder *RuntimeBuilder) RecoverSecurityEventDisconnector(_ context.Context, revision identity.Revision) (SecurityEventDisconnector, error) {
prepared := builder.preparedSecurityRuntime(revision.ID)
if prepared.manager != nil {
if err := prepared.manager.ValidateConfiguredConnectionBinding(builder.ctx); err != nil {
return nil, err
}
return prepared.manager, nil
}
manager, err := builder.newSecurityEventManager(builder.ctx, revision, true)
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return manager, nil
}
// AdoptPreparedSecurityEvents transfers the prepared manager lifetime to an
// activated Runtime. Validation only peeks at prepared state, so Verification
// callbacks remain available between validation and activation.
func (builder *RuntimeBuilder) AdoptPreparedSecurityEvents(revisionID string) context.CancelFunc {
builder.mutex.Lock()
defer builder.mutex.Unlock()
prepared := builder.prepared[revisionID]
delete(builder.prepared, revisionID)
return prepared
return prepared.cancel
}
func safeSecurityEventRuntimeError(err error) error {
var categorized interface{ SafeErrorCategory() string }
if errors.As(err, &categorized) {
return err
}
if errors.Is(err, store.ErrSecurityEventConnectionConflict) {
return safeRuntimeError{category: "connection_conflict", cause: err}
}
return safeRuntimeError{category: "preparation_failed", cause: err}
}
func safeSecurityEventCategory(err error, fallback string) string {
var categorized interface{ SafeErrorCategory() string }
if errors.As(err, &categorized) && categorized.SafeErrorCategory() != "" {
return categorized.SafeErrorCategory()
}
return fallback
}
func (builder *RuntimeBuilder) newSecurityEventManager(ctx context.Context, revision identity.Revision, strictRevisionBinding bool) (*securityevents.ConnectionManager, error) {
return securityevents.NewConnectionManager(ctx, builder.store, builder.secrets, builder.securityEventManagerConfig(revision, strictRevisionBinding), builder.metrics)
}
func (builder *RuntimeBuilder) securityEventManagerConfig(revision identity.Revision, strictRevisionBinding bool) securityevents.ConnectionManagerConfig {
return securityevents.ConnectionManagerConfig{
AppEnv: builder.config.AppEnv, OIDCEnabled: true, OIDCIssuer: revision.Issuer, OIDCTenantID: revision.TenantID,
ManagementClientID: revision.MachineClientID, PublicBaseURL: revision.PublicBaseURL,
ExpectedTransmitterIssuer: strings.TrimRight(strings.TrimSpace(revision.SecurityEventIssuer), "/"),
ExpectedAudience: revision.SecurityEventAudience,
ExpectedOwnerKey: "identity-pairing-ssf-" + revision.ID,
StrictRevisionBinding: strictRevisionBinding,
HeartbeatInterval: builder.config.HeartbeatInterval,
StaleAfter: builder.config.StaleAfter,
ClockSkew: builder.config.ClockSkew,
}
}
func (builder *RuntimeBuilder) preparedSecurityRuntime(revisionID string) preparedSecurityRuntime {
builder.mutex.Lock()
defer builder.mutex.Unlock()
return builder.prepared[revisionID]
}
func secureCookieFor(baseURL string) bool {
@@ -0,0 +1,81 @@
package identityruntime
import (
"context"
"strings"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/identity"
)
func TestRuntimeBuilderRejectsLoopbackHTTPOutsideLocalEnvironments(t *testing.T) {
revision := identity.Revision{
AuthCenterURL: "https://auth.example.com", Issuer: "https://auth.example.com/issuer/easyai",
PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com",
SecurityEventIssuer: "https://auth.example.com/ssf", SecurityEventConfigURL: "https://auth.example.com/.well-known/ssf-configuration/ssf",
SessionRevocation: true,
}
production := &RuntimeBuilder{config: RuntimeBuilderConfig{AppEnv: "production"}}
for _, test := range []struct {
name string
mutate func(*identity.Revision)
}{
{name: "auth center", mutate: func(value *identity.Revision) { value.AuthCenterURL = "http://localhost:18000" }},
{name: "OIDC issuer", mutate: func(value *identity.Revision) { value.Issuer = "http://localhost:18003/issuer/easyai" }},
{name: "public base", mutate: func(value *identity.Revision) { value.PublicBaseURL = "http://127.0.0.1:18089" }},
{name: "web base", mutate: func(value *identity.Revision) { value.WebBaseURL = "http://localhost:5178" }},
{name: "SSF issuer", mutate: func(value *identity.Revision) { value.SecurityEventIssuer = "http://localhost:18004/ssf" }},
{name: "SSF configuration", mutate: func(value *identity.Revision) {
value.SecurityEventConfigURL = "http://localhost:18004/.well-known/ssf-configuration/ssf"
}},
} {
t.Run(test.name, func(t *testing.T) {
candidate := revision
test.mutate(&candidate)
if _, err := production.Build(context.Background(), candidate); err == nil || !strings.Contains(err.Error(), "HTTPS") {
t.Fatalf("production runtime did not reject loopback HTTP %s URL: %v", test.name, err)
}
})
}
localRevision := revision
localRevision.AuthCenterURL = "http://localhost:18000"
localRevision.Issuer = "http://localhost:18003/issuer/easyai"
localRevision.PublicBaseURL = "http://127.0.0.1:18089"
localRevision.WebBaseURL = "http://localhost:5178"
localRevision.SecurityEventIssuer = "http://localhost:18004/ssf"
localRevision.SecurityEventConfigURL = "http://localhost:18004/.well-known/ssf-configuration/ssf"
development := &RuntimeBuilder{config: RuntimeBuilderConfig{AppEnv: "development"}}
if _, err := development.Build(context.Background(), localRevision); err == nil || strings.Contains(err.Error(), "HTTPS") {
t.Fatalf("development runtime did not pass URL policy before completeness checks: %v", err)
}
}
func TestSecurityEventManagerConfigBindsRuntimeToTargetRevision(t *testing.T) {
builder := &RuntimeBuilder{config: RuntimeBuilderConfig{AppEnv: "test"}}
revision := identity.Revision{
ID: "revision-target",
Issuer: "https://auth.example/issuer",
TenantID: "tenant-public-id",
MachineClientID: "gateway-machine",
PublicBaseURL: "https://gateway.example",
SecurityEventIssuer: "https://auth.example/ssf/",
SecurityEventAudience: "urn:easyai:ssf:receiver:target",
}
runtimeConfig := builder.securityEventManagerConfig(revision, true)
if !runtimeConfig.StrictRevisionBinding ||
runtimeConfig.ExpectedTransmitterIssuer != "https://auth.example/ssf" ||
runtimeConfig.ExpectedAudience != revision.SecurityEventAudience ||
runtimeConfig.ManagementClientID != revision.MachineClientID ||
runtimeConfig.ExpectedOwnerKey != "identity-pairing-ssf-revision-target" {
t.Fatalf("runtime SSF binding config=%#v", runtimeConfig)
}
recoveryConfig := builder.securityEventManagerConfig(revision, false)
if recoveryConfig.StrictRevisionBinding {
t.Fatal("conflict recovery was incorrectly blocked by strict Revision binding")
}
if recoveryConfig.ExpectedOwnerKey != runtimeConfig.ExpectedOwnerKey {
t.Fatal("recovery manager lost the target owner needed for explicit validation")
}
}
+328 -47
View File
@@ -3,6 +3,8 @@ package identityruntime
import (
"context"
"errors"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
@@ -27,15 +29,20 @@ type Builder interface {
Build(context.Context, identity.Revision) (*Runtime, error)
}
type SecurityEventDisconnector interface {
Disconnect(context.Context) (securityevents.ConnectionView, error)
}
type Runtime struct {
Revision identity.Revision
Verifier *auth.OIDCVerifier
PublicClient *auth.OIDCPublicClient
Sessions *oidcsession.Service
SessionCipher *oidcsession.Cipher
SecurityEvents *securityevents.ConnectionManager
CookieSecure bool
close func()
Revision identity.Revision
Verifier *auth.OIDCVerifier
PublicClient *auth.OIDCPublicClient
Sessions *oidcsession.Service
SessionCipher *oidcsession.Cipher
SecurityEvents *securityevents.ConnectionManager
CookieSecure bool
close func()
securityEventDisconnector SecurityEventDisconnector
}
func (runtime *Runtime) Close() {
@@ -45,36 +52,126 @@ func (runtime *Runtime) Close() {
}
type Manager struct {
repository Repository
builder Builder
operation sync.Mutex
current atomic.Pointer[Runtime]
repository Repository
builder Builder
operation sync.Mutex
current atomic.Pointer[Runtime]
reconciliationRequired atomic.Bool
legacyJWTAllowed atomic.Bool
trustedWebBaseURL atomic.Pointer[string]
}
const identityRuntimeReconciliationTimeout = 5 * time.Second
func NewManager(repository Repository, builder Builder) *Manager {
return &Manager{repository: repository, builder: builder}
manager := &Manager{repository: repository, builder: builder}
manager.legacyJWTAllowed.Store(true)
return manager
}
func (manager *Manager) Current() *Runtime {
return manager.current.Load()
}
func (manager *Manager) ReconciliationRequired() bool {
return manager.reconciliationRequired.Load()
}
func (manager *Manager) LegacyJWTEnabled() bool {
return manager.legacyJWTAllowed.Load()
}
// TrustedWebBaseURL is the persisted Active Revision's exact browser origin.
// It intentionally survives a fail-closed Runtime build so the local
// break-glass manager can still repair or disable a broken Active Revision.
func (manager *Manager) TrustedWebBaseURL() string {
value := manager.trustedWebBaseURL.Load()
if value == nil {
return ""
}
return *value
}
// SecurityEventReceiver resolves the request-time receiver. During first
// onboarding there is no Active Runtime yet, so the builder-owned prepared
// receiver must remain reachable for SSF Verification callbacks.
func (manager *Manager) SecurityEventReceiver() http.Handler {
provider, ok := manager.builder.(interface{ PreparedSecurityEventReceiver() http.Handler })
if ok {
if prepared := provider.PreparedSecurityEventReceiver(); prepared != nil {
return prepared
}
}
if runtime := manager.Current(); runtime != nil && runtime.SecurityEvents != nil {
return runtime.SecurityEvents
}
return manager.SecurityEventManager()
}
// SecurityEventManager resolves the manager used by administrative recovery
// operations. A prepared manager may represent an older persisted connection
// that must be retired before the first identity Runtime can be activated.
func (manager *Manager) SecurityEventManager() *securityevents.ConnectionManager {
if runtime := manager.Current(); runtime != nil && runtime.SecurityEvents != nil {
return runtime.SecurityEvents
}
managerProvider, ok := manager.builder.(interface {
PreparedSecurityEventManager() *securityevents.ConnectionManager
})
if ok {
return managerProvider.PreparedSecurityEventManager()
}
return nil
}
func (manager *Manager) LoadActive(ctx context.Context) error {
manager.operation.Lock()
defer manager.operation.Unlock()
revision, err := manager.repository.ActiveIdentityConfigurationRevision(ctx)
if errors.Is(err, identity.ErrRevisionNotFound) {
manager.publishDisabledRuntime()
return nil
}
if err != nil {
manager.failClosedRuntime()
return err
}
manager.rememberTrustedWebBaseURL(revision.WebBaseURL)
runtime, err := manager.builder.Build(ctx, revision)
if err != nil {
manager.failClosedRuntime()
return err
}
manager.publishRuntime(runtime, revision)
return nil
}
// ReconcileActive restores a fail-closed Runtime after an uncertain mutation
// outcome. It is safe to call periodically: a matching Active Runtime is left
// untouched, while a changed or missing Active Revision is published atomically.
func (manager *Manager) ReconcileActive(ctx context.Context) error {
manager.operation.Lock()
defer manager.operation.Unlock()
revision, err := manager.repository.ActiveIdentityConfigurationRevision(ctx)
if errors.Is(err, identity.ErrRevisionNotFound) {
manager.publishDisabledRuntime()
return nil
}
if err != nil {
manager.reconciliationRequired.Store(true)
return err
}
manager.rememberTrustedWebBaseURL(revision.WebBaseURL)
if current := manager.current.Load(); current != nil && current.Revision.ID == revision.ID && current.Revision.Version == revision.Version {
manager.reconciliationRequired.Store(false)
return nil
}
runtime, err := manager.builder.Build(ctx, revision)
if err != nil {
manager.reconciliationRequired.Store(true)
return err
}
runtime.Revision = revision
manager.current.Store(runtime)
manager.publishRuntime(runtime, revision)
return nil
}
@@ -85,7 +182,7 @@ func (manager *Manager) Validate(ctx context.Context, id string, expectedVersion
if err != nil {
return identity.Revision{}, err
}
if revision.Version != expectedVersion || revision.State != identity.RevisionDraft && revision.State != identity.RevisionSuperseded && revision.State != identity.RevisionActive {
if revision.Version != expectedVersion || revision.State != identity.RevisionDraft && revision.State != identity.RevisionActive {
return identity.Revision{}, identity.ErrRevisionConflict
}
candidate, buildErr := manager.builder.Build(ctx, revision)
@@ -101,9 +198,7 @@ func (manager *Manager) Validate(ctx context.Context, id string, expectedVersion
candidate.Close()
return identity.Revision{}, err
}
candidate.Revision = revalidated
old := manager.current.Swap(candidate)
retireRuntime(old)
manager.publishRuntime(candidate, revalidated)
return revalidated, nil
}
candidate.Close()
@@ -126,12 +221,9 @@ func (manager *Manager) Activate(ctx context.Context, id string, expectedVersion
}
activated, _, err := manager.repository.ActivateIdentityRevision(ctx, id, expectedVersion, traceID, auditID)
if err != nil {
candidate.Close()
return identity.Revision{}, err
return manager.reconcileActivationMutation(ctx, id, candidate, err)
}
candidate.Revision = activated
old := manager.current.Swap(candidate)
retireRuntime(old)
manager.publishRuntime(candidate, activated)
return activated, nil
}
@@ -145,46 +237,235 @@ func (manager *Manager) Rollback(ctx context.Context, id string, expectedVersion
if revision.Version != expectedVersion || revision.State != identity.RevisionSuperseded {
return identity.Revision{}, identity.ErrRevisionConflict
}
candidate, err := manager.builder.Build(ctx, revision)
if err != nil {
return identity.Revision{}, err
}
validated, err := manager.repository.MarkIdentityRevisionValidated(ctx, id, expectedVersion, traceID, auditID)
if err != nil {
candidate.Close()
return identity.Revision{}, err
}
activated, _, err := manager.repository.ActivateIdentityRevision(ctx, id, validated.Version, traceID, auditID)
if err != nil {
candidate.Close()
return identity.Revision{}, err
}
candidate.Revision = activated
old := manager.current.Swap(candidate)
retireRuntime(old)
return activated, nil
// Auth Center currently reuses and mutates OAuth/SSF resources. An older
// local Revision therefore cannot prove that its remote redirect URIs,
// scopes, audiences, or credentials still match. Require a fresh onboarding
// exchange until a versioned remote-resource handoff exists.
return identity.Revision{}, identity.ErrRollbackConfigurationHandoffRequired
}
func (manager *Manager) Disable(ctx context.Context, expectedVersion int64, traceID, auditID string) (identity.Revision, error) {
manager.operation.Lock()
defer manager.operation.Unlock()
disabled, err := manager.repository.DisableActiveIdentityRevision(ctx, expectedVersion, traceID, auditID)
if err != nil {
if err := manager.retireActiveSecurityEventsBeforeDisable(ctx, expectedVersion); err != nil {
return identity.Revision{}, err
}
old := manager.current.Swap(nil)
retireRuntime(old)
disabled, err := manager.repository.DisableActiveIdentityRevision(ctx, expectedVersion, traceID, auditID)
if err != nil {
return manager.reconcileDisableMutation(ctx, expectedVersion, err)
}
manager.publishDisabledRuntime()
return disabled, nil
}
func (manager *Manager) retireActiveSecurityEventsBeforeDisable(ctx context.Context, expectedVersion int64) error {
active, err := manager.repository.ActiveIdentityConfigurationRevision(ctx)
if err != nil {
return err
}
if active.Version != expectedVersion {
return identity.ErrRevisionConflict
}
if !active.SessionRevocation {
return nil
}
runtime := manager.current.Load()
var disconnector SecurityEventDisconnector
if runtime != nil && runtime.Revision.ID == active.ID {
disconnector = runtime.securityEventDisconnector
if disconnector == nil && runtime.SecurityEvents != nil {
disconnector = runtime.SecurityEvents
}
}
if disconnector == nil {
recoverer, ok := manager.builder.(interface {
RecoverSecurityEventDisconnector(context.Context, identity.Revision) (SecurityEventDisconnector, error)
})
if !ok {
return identity.ErrSecurityEventRetirementPending
}
var recoverErr error
disconnector, recoverErr = recoverer.RecoverSecurityEventDisconnector(ctx, active)
if recoverErr != nil {
return recoverErr
}
// A missing local connection means there is no bound local Stream or
// credential left to retire. The already-required audit record makes
// this fail-closed recovery decision traceable.
if disconnector == nil {
return nil
}
}
connection, err := disconnector.Disconnect(ctx)
if err != nil {
return err
}
if connection.LifecycleStatus != "retiring" {
return identity.ErrSecurityEventRetirementPending
}
return nil
}
func (manager *Manager) reconcileActivationMutation(ctx context.Context, targetID string, candidate *Runtime, mutationErr error) (identity.Revision, error) {
reconcileCtx, cancel := identityRuntimeReconciliationContext(ctx)
defer cancel()
active, err := manager.repository.ActiveIdentityConfigurationRevision(reconcileCtx)
if err == nil && active.ID == targetID && active.State == identity.RevisionActive {
manager.publishRuntime(candidate, active)
return active, nil
}
candidate.Close()
if errors.Is(err, identity.ErrRevisionNotFound) {
manager.publishDisabledRuntime()
return identity.Revision{}, mutationErr
}
if err != nil {
manager.failClosedRuntime()
return identity.Revision{}, mutationErr
}
current := manager.current.Load()
if current == nil || current.Revision.ID != active.ID {
manager.failClosedRuntime()
}
return identity.Revision{}, mutationErr
}
func (manager *Manager) reconcileRollbackValidation(ctx context.Context, targetID string, expectedVersion int64, mutationErr error) (identity.Revision, error) {
reconcileCtx, cancel := identityRuntimeReconciliationContext(ctx)
defer cancel()
revision, err := manager.repository.IdentityConfigurationRevision(reconcileCtx, targetID)
if err == nil && revision.State == identity.RevisionValidated && revision.Version == expectedVersion+1 {
return revision, nil
}
return identity.Revision{}, mutationErr
}
func (manager *Manager) reconcileDisableMutation(ctx context.Context, expectedVersion int64, mutationErr error) (identity.Revision, error) {
previous := manager.current.Load()
reconcileCtx, cancel := identityRuntimeReconciliationContext(ctx)
defer cancel()
active, err := manager.repository.ActiveIdentityConfigurationRevision(reconcileCtx)
if err == nil {
if previous == nil || active.ID != previous.Revision.ID {
manager.failClosedRuntime()
}
return identity.Revision{}, mutationErr
}
if !errors.Is(err, identity.ErrRevisionNotFound) {
manager.failClosedRuntime()
return identity.Revision{}, mutationErr
}
manager.publishDisabledRuntime()
if previous == nil || previous.Revision.ID == "" {
return identity.Revision{}, mutationErr
}
disabled, lookupErr := manager.repository.IdentityConfigurationRevision(reconcileCtx, previous.Revision.ID)
if lookupErr == nil && disabled.State == identity.RevisionSuperseded && disabled.Version == expectedVersion+1 {
return disabled, nil
}
return identity.Revision{}, mutationErr
}
func identityRuntimeReconciliationContext(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(context.WithoutCancel(ctx), identityRuntimeReconciliationTimeout)
}
func (manager *Manager) publishRuntime(candidate *Runtime, revision identity.Revision) {
candidate.Revision = revision
manager.rememberTrustedWebBaseURL(revision.WebBaseURL)
manager.legacyJWTAllowed.Store(revision.LegacyJWTEnabled)
old := manager.current.Swap(candidate)
manager.adoptPreparedSecurityEvents(candidate, revision.ID)
manager.reconciliationRequired.Store(false)
retireRuntime(old)
}
func (manager *Manager) publishDisabledRuntime() {
old := manager.current.Swap(nil)
manager.trustedWebBaseURL.Store(nil)
manager.legacyJWTAllowed.Store(true)
manager.reconciliationRequired.Store(false)
retireRuntime(old)
}
func (manager *Manager) failClosedRuntime() {
manager.legacyJWTAllowed.Store(false)
manager.reconciliationRequired.Store(true)
old := manager.current.Swap(nil)
retireRuntime(old)
}
func (manager *Manager) rememberTrustedWebBaseURL(value string) {
normalized := strings.TrimSpace(value)
if normalized == "" {
manager.trustedWebBaseURL.Store(nil)
return
}
manager.trustedWebBaseURL.Store(&normalized)
}
func (manager *Manager) PrepareSecurityEvents(ctx context.Context, revision identity.Revision, secret []byte) error {
manager.operation.Lock()
defer manager.operation.Unlock()
current, err := manager.repository.IdentityConfigurationRevision(ctx, revision.ID)
if err != nil {
return err
}
if current.State == identity.RevisionActive {
// Activation already crossed the prepared-manager adoption point. A
// stale restart restore must not insert a second Receiver afterwards.
return nil
}
if current.Version != revision.Version || current.State != identity.RevisionDraft && current.State != identity.RevisionValidated {
return identity.ErrRevisionConflict
}
preparer, ok := manager.builder.(interface {
PrepareSecurityEvents(context.Context, identity.Revision, []byte) error
})
if !ok {
return errors.New("security event runtime preparation is unavailable")
}
return preparer.PrepareSecurityEvents(ctx, revision, secret)
return preparer.PrepareSecurityEvents(ctx, current, secret)
}
func (manager *Manager) CleanupPreparedSecurityEvents(ctx context.Context, revision identity.Revision) error {
cleaner, ok := manager.builder.(interface {
CleanupPreparedSecurityEvents(context.Context, identity.Revision) error
})
if !ok {
return errors.New("security event runtime cleanup is unavailable")
}
return cleaner.CleanupPreparedSecurityEvents(ctx, revision)
}
func (manager *Manager) RetireConflictingSecurityEvents(ctx context.Context, revision identity.Revision) error {
resolver, ok := manager.builder.(interface {
RetireConflictingSecurityEvents(context.Context, identity.Revision) error
})
if !ok {
return errors.New("security event conflict recovery is unavailable")
}
return resolver.RetireConflictingSecurityEvents(ctx, revision)
}
func (manager *Manager) adoptPreparedSecurityEvents(runtime *Runtime, revisionID string) {
adopter, ok := manager.builder.(interface {
AdoptPreparedSecurityEvents(string) context.CancelFunc
})
if !ok {
return
}
cancel := adopter.AdoptPreparedSecurityEvents(revisionID)
if cancel == nil {
return
}
closeRuntime := runtime.close
runtime.close = func() {
if closeRuntime != nil {
closeRuntime()
}
cancel()
}
}
func retireRuntime(runtime *Runtime) {
+513 -10
View File
@@ -3,15 +3,25 @@ package identityruntime
import (
"context"
"errors"
"net/http"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/identity"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/securityevents"
)
type runtimeRepositoryFake struct {
revisions map[string]identity.Revision
active identity.Revision
activateCalled bool
revisions map[string]identity.Revision
active identity.Revision
activateCalled bool
activeLookupErr error
markValidatedErr error
markValidatedErrAfterApply bool
activateErr error
activateErrAfterApply bool
disableErr error
disableErrAfterApply bool
}
func (f *runtimeRepositoryFake) IdentityConfigurationRevision(_ context.Context, id string) (identity.Revision, error) {
@@ -22,18 +32,27 @@ func (f *runtimeRepositoryFake) IdentityConfigurationRevision(_ context.Context,
return revision, nil
}
func (f *runtimeRepositoryFake) ActiveIdentityConfigurationRevision(context.Context) (identity.Revision, error) {
if f.activeLookupErr != nil {
return identity.Revision{}, f.activeLookupErr
}
if f.active.ID == "" {
return identity.Revision{}, identity.ErrRevisionNotFound
}
return f.active, nil
}
func (f *runtimeRepositoryFake) MarkIdentityRevisionValidated(_ context.Context, id string, expected int64, _, _ string) (identity.Revision, error) {
if f.markValidatedErr != nil && !f.markValidatedErrAfterApply {
return identity.Revision{}, f.markValidatedErr
}
revision := f.revisions[id]
if revision.Version != expected {
return identity.Revision{}, identity.ErrRevisionConflict
}
revision.State, revision.Version = identity.RevisionValidated, revision.Version+1
f.revisions[id] = revision
if f.markValidatedErr != nil {
return identity.Revision{}, f.markValidatedErr
}
return revision, nil
}
func (f *runtimeRepositoryFake) RevalidateActiveIdentityRevision(_ context.Context, id string, expected int64, traceID, auditID string) (identity.Revision, error) {
@@ -56,6 +75,9 @@ func (f *runtimeRepositoryFake) MarkIdentityRevisionFailed(_ context.Context, id
}
func (f *runtimeRepositoryFake) ActivateIdentityRevision(_ context.Context, id string, expected int64, traceID, auditID string) (identity.Revision, bool, error) {
f.activateCalled = true
if f.activateErr != nil && !f.activateErrAfterApply {
return identity.Revision{}, false, f.activateErr
}
revision := f.revisions[id]
if revision.Version != expected || revision.State != identity.RevisionValidated {
return identity.Revision{}, false, identity.ErrRevisionConflict
@@ -67,9 +89,15 @@ func (f *runtimeRepositoryFake) ActivateIdentityRevision(_ context.Context, id s
}
revision.State, revision.Version, revision.LastTraceID, revision.LastAuditID = identity.RevisionActive, revision.Version+1, traceID, auditID
f.active, f.revisions[id] = revision, revision
if f.activateErr != nil {
return identity.Revision{}, false, f.activateErr
}
return revision, true, nil
}
func (f *runtimeRepositoryFake) DisableActiveIdentityRevision(_ context.Context, expected int64, traceID, auditID string) (identity.Revision, error) {
if f.disableErr != nil && !f.disableErrAfterApply {
return identity.Revision{}, f.disableErr
}
if f.active.Version != expected {
return identity.Revision{}, identity.ErrRevisionConflict
}
@@ -77,22 +105,87 @@ func (f *runtimeRepositoryFake) DisableActiveIdentityRevision(_ context.Context,
disabled.State, disabled.Version, disabled.LastTraceID, disabled.LastAuditID = identity.RevisionSuperseded, disabled.Version+1, traceID, auditID
f.revisions[disabled.ID] = disabled
f.active = identity.Revision{}
if f.disableErr != nil {
return identity.Revision{}, f.disableErr
}
return disabled, nil
}
type runtimeBuilderFake struct {
err error
builtIDs []string
err error
builtIDs []string
buildStarted chan struct{}
buildRelease chan struct{}
prepareStarted chan struct{}
prepareRelease chan struct{}
prepareCalls int
cleanupCalledFor string
adoptedRevision string
preparedCancel context.CancelFunc
preparedManager *securityevents.ConnectionManager
preparedReceiver http.Handler
recoveredSecurityEventDisconnector SecurityEventDisconnector
recoverSecurityEventErr error
}
type runtimeSecurityEventDisconnector struct {
lifecycle string
err error
calls int
}
func (f *runtimeSecurityEventDisconnector) Disconnect(context.Context) (securityevents.ConnectionView, error) {
f.calls++
return securityevents.ConnectionView{LifecycleStatus: f.lifecycle}, f.err
}
func (f *runtimeBuilderFake) Build(_ context.Context, revision identity.Revision) (*Runtime, error) {
f.builtIDs = append(f.builtIDs, revision.ID)
if f.buildStarted != nil {
close(f.buildStarted)
}
if f.buildRelease != nil {
<-f.buildRelease
}
if f.err != nil {
return nil, f.err
}
return &Runtime{Revision: revision}, nil
}
func (f *runtimeBuilderFake) PrepareSecurityEvents(_ context.Context, _ identity.Revision, _ []byte) error {
f.prepareCalls++
if f.prepareStarted != nil {
close(f.prepareStarted)
}
if f.prepareRelease != nil {
<-f.prepareRelease
}
return f.err
}
func (f *runtimeBuilderFake) CleanupPreparedSecurityEvents(_ context.Context, revision identity.Revision) error {
f.cleanupCalledFor = revision.ID
return f.err
}
func (f *runtimeBuilderFake) AdoptPreparedSecurityEvents(revisionID string) context.CancelFunc {
f.adoptedRevision = revisionID
return f.preparedCancel
}
func (f *runtimeBuilderFake) PreparedSecurityEventManager() *securityevents.ConnectionManager {
return f.preparedManager
}
func (f *runtimeBuilderFake) PreparedSecurityEventReceiver() http.Handler {
return f.preparedReceiver
}
func (f *runtimeBuilderFake) RecoverSecurityEventDisconnector(context.Context, identity.Revision) (SecurityEventDisconnector, error) {
return f.recoveredSecurityEventDisconnector, f.recoverSecurityEventErr
}
func TestValidationFailureKeepsCurrentRuntimeAndDoesNotActivate(t *testing.T) {
active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4}
draft := identity.Revision{ID: "draft", State: identity.RevisionDraft, Version: 1}
@@ -112,6 +205,39 @@ func TestValidationFailureKeepsCurrentRuntimeAndDoesNotActivate(t *testing.T) {
}
}
func TestLoadActiveMarksTransientFailureForAutomaticReconciliation(t *testing.T) {
active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4}
repository := &runtimeRepositoryFake{
revisions: map[string]identity.Revision{"active": active},
active: active,
activeLookupErr: errors.New("database temporarily unavailable"),
}
builder := &runtimeBuilderFake{}
manager := NewManager(repository, builder)
if err := manager.LoadActive(context.Background()); err == nil {
t.Fatal("transient active lookup failure was ignored")
}
if !manager.ReconciliationRequired() || manager.Current() != nil {
t.Fatal("failed startup load was not left fail-closed and retryable")
}
repository.activeLookupErr = nil
builder.err = errors.New("SecretStore temporarily unavailable")
if err := manager.ReconcileActive(context.Background()); err == nil {
t.Fatal("transient runtime build failure was ignored")
}
if !manager.ReconciliationRequired() {
t.Fatal("failed runtime build cleared automatic reconciliation")
}
builder.err = nil
if err := manager.ReconcileActive(context.Background()); err != nil {
t.Fatal(err)
}
if manager.ReconciliationRequired() || manager.Current() == nil || manager.Current().Revision.ID != active.ID {
t.Fatalf("startup runtime was not recovered: current=%#v required=%v", manager.Current(), manager.ReconciliationRequired())
}
}
func TestActivationSwapsRuntimeOnlyAfterRepositoryActivation(t *testing.T) {
active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2}
candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3}
@@ -128,21 +254,209 @@ func TestActivationSwapsRuntimeOnlyAfterRepositoryActivation(t *testing.T) {
}
}
func TestRollbackValidatesAndAtomicallyReplacesActiveRuntime(t *testing.T) {
func TestActivationReconcilesCommitAppliedThenResponseLost(t *testing.T) {
active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2}
candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3}
repository := &runtimeRepositoryFake{
revisions: map[string]identity.Revision{"old": active, "new": candidate},
active: active,
activateErr: errors.New("commit response lost"),
activateErrAfterApply: true,
}
manager := NewManager(repository, &runtimeBuilderFake{})
manager.current.Store(&Runtime{Revision: active})
activated, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit")
if err != nil {
t.Fatal(err)
}
if activated.State != identity.RevisionActive || manager.Current() == nil || manager.Current().Revision.ID != candidate.ID {
t.Fatalf("committed activation was not reconciled: activated=%#v current=%#v", activated, manager.Current())
}
}
func TestActivationKeepsCurrentRuntimeWhenMutationWasNotApplied(t *testing.T) {
active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2}
candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3}
mutationErr := errors.New("activation rejected before commit")
repository := &runtimeRepositoryFake{
revisions: map[string]identity.Revision{"old": active, "new": candidate},
active: active,
activateErr: mutationErr,
}
manager := NewManager(repository, &runtimeBuilderFake{})
original := &Runtime{Revision: active}
manager.current.Store(original)
if _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit"); !errors.Is(err, mutationErr) {
t.Fatalf("activation error = %v, want %v", err, mutationErr)
}
if manager.Current() != original {
t.Fatal("definitely uncommitted activation replaced the current runtime")
}
}
func TestActivationFailsClosedWhenCommitOutcomeCannotBeReconciled(t *testing.T) {
active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2}
candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3}
repository := &runtimeRepositoryFake{
revisions: map[string]identity.Revision{"old": active, "new": candidate},
active: active,
activateErr: errors.New("commit outcome unknown"),
activeLookupErr: errors.New("database unavailable during reconciliation"),
}
builder := &runtimeBuilderFake{}
manager := NewManager(repository, builder)
manager.current.Store(&Runtime{Revision: active})
if _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit"); err == nil {
t.Fatal("unknown activation outcome was reported as successful")
}
if manager.Current() != nil {
t.Fatal("unknown activation outcome did not fail OIDC closed")
}
if !manager.ReconciliationRequired() {
t.Fatal("unknown activation outcome did not request background reconciliation")
}
repository.activeLookupErr = nil
builder.err = errors.New("runtime dependency temporarily unavailable")
if err := manager.ReconcileActive(context.Background()); err == nil {
t.Fatal("runtime reconciliation unexpectedly ignored build failure")
}
if !manager.ReconciliationRequired() || manager.Current() != nil {
t.Fatal("failed runtime rebuild cleared fail-closed reconciliation state")
}
builder.err = nil
if err := manager.ReconcileActive(context.Background()); err != nil {
t.Fatal(err)
}
if manager.ReconciliationRequired() || manager.Current() == nil || manager.Current().Revision.ID != active.ID {
t.Fatalf("runtime did not recover after reconciliation: current=%#v required=%v", manager.Current(), manager.ReconciliationRequired())
}
}
func TestRollbackRejectsOIDCOnlyRevisionBeforeRuntimeBuild(t *testing.T) {
active := identity.Revision{ID: "current", State: identity.RevisionActive, Version: 5}
previous := identity.Revision{ID: "previous", State: identity.RevisionSuperseded, Version: 3}
repository := &runtimeRepositoryFake{
revisions: map[string]identity.Revision{"current": active, "previous": previous}, active: active,
}
manager := NewManager(repository, &runtimeBuilderFake{})
builder := &runtimeBuilderFake{}
manager := NewManager(repository, builder)
manager.current.Store(&Runtime{Revision: active})
rolledBack, err := manager.Rollback(context.Background(), previous.ID, previous.Version, "trace", "audit")
if _, err := manager.Rollback(context.Background(), previous.ID, previous.Version, "trace", "audit"); !errors.Is(err, identity.ErrRollbackConfigurationHandoffRequired) {
t.Fatalf("rollback error=%v, want remote resource handoff required", err)
}
if len(builder.builtIDs) != 0 || repository.activateCalled || manager.Current() == nil || manager.Current().Revision.ID != active.ID {
t.Fatalf("unsafe rollback changed runtime: built=%v activate=%t current=%#v", builder.builtIDs, repository.activateCalled, manager.Current())
}
}
func TestRollbackRejectsSupersededMachineCredentialBeforeRuntimeBuild(t *testing.T) {
active := identity.Revision{ID: "current", State: identity.RevisionActive, Version: 5}
previous := identity.Revision{
ID: "previous", State: identity.RevisionSuperseded, Version: 3,
MachineCredentialRef: "identity-machine-previous", TokenIntrospection: true,
}
repository := &runtimeRepositoryFake{
revisions: map[string]identity.Revision{"current": active, "previous": previous}, active: active,
}
builder := &runtimeBuilderFake{}
manager := NewManager(repository, builder)
manager.current.Store(&Runtime{Revision: active})
if _, err := manager.Rollback(context.Background(), previous.ID, previous.Version, "trace", "audit"); !errors.Is(err, identity.ErrRollbackConfigurationHandoffRequired) {
t.Fatalf("rollback error=%v, want credential handoff required", err)
}
if len(builder.builtIDs) != 0 || repository.activateCalled {
t.Fatal("unsafe rollback reached runtime build or activation")
}
}
func TestValidateRejectsSupersededRevisionBeforeRuntimeBuild(t *testing.T) {
previous := identity.Revision{ID: "previous", State: identity.RevisionSuperseded, Version: 3}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"previous": previous}}
builder := &runtimeBuilderFake{}
manager := NewManager(repository, builder)
if _, err := manager.Validate(context.Background(), previous.ID, previous.Version, "trace", "audit"); !errors.Is(err, identity.ErrRevisionConflict) {
t.Fatalf("validate error=%v, want revision conflict", err)
}
if len(builder.builtIDs) != 0 || repository.revisions[previous.ID].State != identity.RevisionSuperseded {
t.Fatalf("superseded revision reached runtime validation: built=%v revision=%#v", builder.builtIDs, repository.revisions[previous.ID])
}
}
func TestDisableKeepsActiveRevisionUntilSecurityEventStreamCanRetire(t *testing.T) {
active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4, SessionRevocation: true}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active}
disconnector := &runtimeSecurityEventDisconnector{lifecycle: "disconnect_pending"}
manager := NewManager(repository, &runtimeBuilderFake{})
original := &Runtime{Revision: active, securityEventDisconnector: disconnector}
manager.current.Store(original)
if _, err := manager.Disable(context.Background(), active.Version, "trace", "audit"); !errors.Is(err, identity.ErrSecurityEventRetirementPending) {
t.Fatalf("disable error=%v, want security event retirement pending", err)
}
if disconnector.calls != 1 || repository.active.ID != active.ID || manager.Current() != original {
t.Fatalf("pending retirement changed Active state: calls=%d repository=%#v current=%#v", disconnector.calls, repository.active, manager.Current())
}
}
func TestDisableRecoversSSFWithoutBuildingBrokenFullIdentityRuntime(t *testing.T) {
active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4, SessionRevocation: true}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active}
disconnector := &runtimeSecurityEventDisconnector{lifecycle: "retiring"}
builder := &runtimeBuilderFake{
err: errors.New("OIDC discovery unavailable"),
recoveredSecurityEventDisconnector: disconnector,
}
manager := NewManager(repository, builder)
disabled, err := manager.Disable(context.Background(), active.Version, "trace", "audit")
if err != nil {
t.Fatal(err)
}
if rolledBack.State != identity.RevisionActive || manager.Current().Revision.ID != previous.ID || repository.active.ID != previous.ID {
t.Fatalf("rollback did not replace active runtime: revision=%#v current=%#v", rolledBack, manager.Current())
if disabled.State != identity.RevisionSuperseded || disconnector.calls != 1 || len(builder.builtIDs) != 0 || manager.Current() != nil {
t.Fatalf("SSF-only recovery did not disable safely: disabled=%#v calls=%d built=%v current=%#v", disabled, disconnector.calls, builder.builtIDs, manager.Current())
}
}
func TestDisableRetiresSecurityEventStreamBeforeSupersedingActiveRevision(t *testing.T) {
active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4, SessionRevocation: true}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active}
disconnector := &runtimeSecurityEventDisconnector{lifecycle: "retiring"}
manager := NewManager(repository, &runtimeBuilderFake{})
manager.current.Store(&Runtime{Revision: active, securityEventDisconnector: disconnector})
disabled, err := manager.Disable(context.Background(), active.Version, "trace", "audit")
if err != nil {
t.Fatal(err)
}
if disconnector.calls != 1 || disabled.State != identity.RevisionSuperseded || repository.active.ID != "" || manager.Current() != nil {
t.Fatalf("safe disable state: calls=%d disabled=%#v repository=%#v current=%#v", disconnector.calls, disabled, repository.active, manager.Current())
}
}
func TestDisableReconcilesCommitAppliedThenResponseLost(t *testing.T) {
active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4}
repository := &runtimeRepositoryFake{
revisions: map[string]identity.Revision{"active": active},
active: active,
disableErr: errors.New("commit response lost"),
disableErrAfterApply: true,
}
manager := NewManager(repository, &runtimeBuilderFake{})
manager.current.Store(&Runtime{Revision: active})
disabled, err := manager.Disable(context.Background(), active.Version, "trace", "audit")
if err != nil {
t.Fatal(err)
}
if disabled.State != identity.RevisionSuperseded || manager.Current() != nil {
t.Fatalf("committed disable was not reconciled: disabled=%#v current=%#v", disabled, manager.Current())
}
}
@@ -162,3 +476,192 @@ func TestActiveRevalidationSwapsOnlyAfterSuccessfulValidation(t *testing.T) {
t.Fatalf("active runtime was not safely revalidated: %#v", revalidated)
}
}
func TestLoadFailureKeepsPersistedActiveWebOriginForBreakGlassRecovery(t *testing.T) {
active := identity.Revision{
ID: "active", State: identity.RevisionActive, Version: 4,
WebBaseURL: "https://gateway.example.com",
}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active}
manager := NewManager(repository, &runtimeBuilderFake{err: errors.New("OIDC discovery unavailable")})
if err := manager.LoadActive(context.Background()); err == nil {
t.Fatal("active runtime load unexpectedly succeeded")
}
if manager.Current() != nil || manager.TrustedWebBaseURL() != active.WebBaseURL {
t.Fatalf("failed Active load lost recovery origin: current=%#v origin=%q", manager.Current(), manager.TrustedWebBaseURL())
}
repository.active = identity.Revision{}
if err := manager.ReconcileActive(context.Background()); err != nil {
t.Fatal(err)
}
if manager.TrustedWebBaseURL() != "" {
t.Fatalf("confirmed disable retained stale recovery origin %q", manager.TrustedWebBaseURL())
}
}
func TestActivationFinishingFirstPreventsStalePreparedSecurityEventRestore(t *testing.T) {
candidate := identity.Revision{ID: "candidate", State: identity.RevisionValidated, Version: 3}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"candidate": candidate}}
builder := &runtimeBuilderFake{buildStarted: make(chan struct{}), buildRelease: make(chan struct{})}
manager := NewManager(repository, builder)
activationDone := make(chan error, 1)
go func() {
_, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit")
activationDone <- err
}()
<-builder.buildStarted
restoreDone := make(chan error, 1)
go func() {
restoreDone <- manager.PrepareSecurityEvents(context.Background(), candidate, []byte("secret"))
}()
close(builder.buildRelease)
if err := <-activationDone; err != nil {
t.Fatal(err)
}
if err := <-restoreDone; err != nil {
t.Fatal(err)
}
if builder.prepareCalls != 0 || manager.Current() == nil || manager.Current().Revision.ID != candidate.ID {
t.Fatalf("stale restore survived activation: prepare_calls=%d current=%#v", builder.prepareCalls, manager.Current())
}
}
func TestPreparedSecurityEventRestoreFinishingFirstIsAdoptedByActivation(t *testing.T) {
candidate := identity.Revision{ID: "candidate", State: identity.RevisionValidated, Version: 3}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"candidate": candidate}}
builder := &runtimeBuilderFake{
buildStarted: make(chan struct{}), buildRelease: make(chan struct{}),
prepareStarted: make(chan struct{}), prepareRelease: make(chan struct{}),
}
manager := NewManager(repository, builder)
restoreDone := make(chan error, 1)
go func() {
restoreDone <- manager.PrepareSecurityEvents(context.Background(), candidate, []byte("secret"))
}()
<-builder.prepareStarted
activationDone := make(chan error, 1)
go func() {
_, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit")
activationDone <- err
}()
select {
case <-builder.buildStarted:
t.Fatal("activation did not wait for prepared Receiver restoration")
case <-time.After(50 * time.Millisecond):
}
close(builder.prepareRelease)
if err := <-restoreDone; err != nil {
t.Fatal(err)
}
<-builder.buildStarted
close(builder.buildRelease)
if err := <-activationDone; err != nil {
t.Fatal(err)
}
if builder.prepareCalls != 1 || builder.adoptedRevision != candidate.ID {
t.Fatalf("prepared restore was not adopted: prepare_calls=%d adopted=%q", builder.prepareCalls, builder.adoptedRevision)
}
}
func TestManagerDelegatesPreparedSecurityEventCleanupWithoutChangingActiveRuntime(t *testing.T) {
active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4}
draft := identity.Revision{ID: "draft", State: identity.RevisionFailed, Version: 2}
builder := &runtimeBuilderFake{}
manager := NewManager(&runtimeRepositoryFake{}, builder)
manager.current.Store(&Runtime{Revision: active})
if err := manager.CleanupPreparedSecurityEvents(context.Background(), draft); err != nil {
t.Fatal(err)
}
if builder.cleanupCalledFor != draft.ID || manager.Current().Revision.ID != active.ID {
t.Fatalf("cleanup changed active runtime or skipped the draft: called=%q current=%#v", builder.cleanupCalledFor, manager.Current())
}
}
func TestActivationAdoptsPreparedSecurityEventLifetime(t *testing.T) {
candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3}
repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"new": candidate}}
preparedContext, preparedCancel := context.WithCancel(context.Background())
builder := &runtimeBuilderFake{preparedCancel: preparedCancel}
manager := NewManager(repository, builder)
if _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit"); err != nil {
t.Fatal(err)
}
if builder.adoptedRevision != candidate.ID {
t.Fatalf("prepared security events were not adopted: %q", builder.adoptedRevision)
}
manager.Current().Close()
select {
case <-preparedContext.Done():
default:
t.Fatal("closing the active runtime did not release its prepared security event manager")
}
}
func TestPreparedSecurityEventReceiverRemainsAvailableUntilAdoption(t *testing.T) {
cancelled := false
builder := &RuntimeBuilder{prepared: map[string]preparedSecurityRuntime{
"draft": {manager: &securityevents.ConnectionManager{}, cancel: func() { cancelled = true }, receiverReady: true},
}}
if builder.PreparedSecurityEventReceiver() == nil {
t.Fatal("prepared receiver was unavailable before activation")
}
cancel := builder.AdoptPreparedSecurityEvents("draft")
if cancel == nil || builder.PreparedSecurityEventReceiver() != nil {
t.Fatal("prepared receiver ownership was not transferred exactly once")
}
cancel()
if !cancelled {
t.Fatal("adopted receiver lifetime could not be released")
}
}
func TestSecurityEventManagerExposesPreparedRecoveryManagerWithoutActiveRuntime(t *testing.T) {
prepared := &securityevents.ConnectionManager{}
manager := NewManager(&runtimeRepositoryFake{}, &runtimeBuilderFake{preparedManager: prepared})
if manager.SecurityEventManager() != prepared || manager.SecurityEventReceiver() != prepared {
t.Fatal("prepared security event manager was unavailable to recovery handlers")
}
}
func TestSecurityEventManagerPrefersActiveRuntime(t *testing.T) {
active := &securityevents.ConnectionManager{}
prepared := &securityevents.ConnectionManager{}
manager := NewManager(&runtimeRepositoryFake{}, &runtimeBuilderFake{preparedManager: prepared})
manager.current.Store(&Runtime{SecurityEvents: active})
if manager.SecurityEventManager() != active {
t.Fatal("prepared recovery manager replaced the active runtime manager")
}
}
func TestSecurityEventReceiverPrefersReadyDraftDuringRepairing(t *testing.T) {
active := &securityevents.ConnectionManager{}
prepared := &securityevents.ConnectionManager{}
manager := NewManager(&runtimeRepositoryFake{}, &runtimeBuilderFake{preparedManager: prepared, preparedReceiver: prepared})
manager.current.Store(&Runtime{SecurityEvents: active})
if manager.SecurityEventReceiver() != prepared {
t.Fatal("verification callback was not routed to the ready draft receiver")
}
}
func TestConflictingPreparedManagerIsRecoverableButNotUsedAsReceiver(t *testing.T) {
prepared := &securityevents.ConnectionManager{}
builder := &RuntimeBuilder{prepared: map[string]preparedSecurityRuntime{
"draft": {manager: prepared},
}}
if builder.PreparedSecurityEventManager() != prepared {
t.Fatal("conflicting connection manager was unavailable for retirement")
}
if builder.PreparedSecurityEventReceiver() != nil {
t.Fatal("unprepared conflict manager was exposed as a verification receiver")
}
}