fix(identity): 完善统一认证配对恢复与安全退役

修复 credentials_saved 状态无法恢复、配对与激活并发冲突,以及 SSF 和身份 Secret 生命周期不完整的问题。新增持久化协调器、取消与清理状态机、事务级并发门禁、受控 SSF 凭据交接、禁用后的延迟 Secret 清理,并对生产环境统一认证及 Discovery 端点强制 HTTPS。

验证:go test ./...;go test -race ./internal/auth ./internal/identity ./internal/identityruntime ./internal/securityevents ./internal/httpapi ./internal/store -count=1;go vet ./...;真实 PostgreSQL 并发及清理成功/冲突回滚测试;pnpm openapi。
This commit is contained in:
2026-07-17 18:31:12 +08:00
parent cdfca61304
commit a312ad880d
55 changed files with 9225 additions and 419 deletions
@@ -34,12 +34,15 @@ type ConnectionRepository interface {
CreateSecurityEventConnection(context.Context, store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error)
SecurityEventConnection(context.Context) (store.SecurityEventConnection, error)
BindSecurityEventConnection(context.Context, string, string, string, string, int64) (store.SecurityEventConnection, error)
SetSecurityEventManagementCredential(context.Context, string, string, string) (store.SecurityEventConnection, error)
UpdateSecurityEventConnectionLifecycle(context.Context, string, string, *string) (store.SecurityEventConnection, error)
SetSecurityEventNextCredential(context.Context, string, string) (store.SecurityEventConnection, error)
SetSecurityEventManagementCredential(context.Context, string, string, string, int64) (store.SecurityEventConnection, error)
TransitionSecurityEventConnectionLifecycle(context.Context, string, string, int64, string, *string) (store.SecurityEventConnection, error)
SetSecurityEventNextCredential(context.Context, string, string, int64) (store.SecurityEventConnection, error)
PromoteSecurityEventCredential(context.Context, string, string) (store.SecurityEventConnection, error)
DeleteSecurityEventConnection(context.Context, string) error
DiscardPreparedSecurityEventConnection(context.Context, string, string, int64) error
FinalizeRetiringSecurityEventConnection(context.Context, string, int64) error
SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error)
QueueIdentitySecretCleanup(context.Context, string, time.Time) error
RemovePendingIdentitySecretCleanup(context.Context, string) (bool, error)
}
type ConnectionManagerConfig struct {
@@ -49,6 +52,10 @@ type ConnectionManagerConfig struct {
OIDCTenantID string
ManagementClientID string
ManagementClientSecret string
ExpectedTransmitterIssuer string
ExpectedAudience string
ExpectedOwnerKey string
StrictRevisionBinding bool
PublicBaseURL string
HeartbeatInterval time.Duration
StaleAfter time.Duration
@@ -79,15 +86,23 @@ type connectionRuntime struct {
cancel context.CancelFunc
}
type connectionRetirementAuthorization struct {
connectionID string
credentialRef string
ownerKey string
}
type ConnectionManager struct {
ctx context.Context
repository ConnectionRepository
secrets SecretStore
config ConnectionManagerConfig
metrics *Metrics
mutex sync.RWMutex
operation sync.Mutex
runtime *connectionRuntime
ctx context.Context
repository ConnectionRepository
secrets SecretStore
config ConnectionManagerConfig
metrics *Metrics
mutex sync.RWMutex
operation sync.Mutex
runtime *connectionRuntime
retirement *connectionRetirementAuthorization
bindingMismatch bool
}
type transmitterConfiguration struct {
@@ -118,6 +133,17 @@ func (e remoteHTTPError) Error() string {
return fmt.Sprintf("SSF endpoint returned HTTP %d", e.status)
}
type safeConnectionError struct {
category string
cause error
}
func (err safeConnectionError) Error() string {
return "security event operation failed: " + err.category
}
func (err safeConnectionError) Unwrap() error { return err.cause }
func (err safeConnectionError) SafeErrorCategory() string { return err.category }
func NewConnectionManager(ctx context.Context, repository ConnectionRepository, secrets SecretStore, config ConnectionManagerConfig, metrics *Metrics) (*ConnectionManager, error) {
if repository == nil || secrets == nil || !config.OIDCEnabled || config.OIDCIssuer == "" || config.OIDCTenantID == "" {
return nil, errors.New("security event connection prerequisites are incomplete")
@@ -138,16 +164,35 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository,
config.CredentialOverlapDuration = 3 * time.Minute
}
if config.RetirementDuration == 0 {
config.RetirementDuration = 6 * time.Minute
config.RetirementDuration = defaultSecurityEventRetirementDuration
}
manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: config, metrics: metrics}
connection, err := repository.SecurityEventConnection(ctx)
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
if config.StrictRevisionBinding {
return nil, safeConnectionError{category: "connection_binding_missing", cause: err}
}
return manager, nil
}
if err != nil {
return nil, fmt.Errorf("load security event connection: %w", err)
}
if config.StrictRevisionBinding {
if err := manager.validateConnectionBinding(connection); err != nil {
return nil, err
}
} else if hasRevisionBindingExpectation(config) {
if err := manager.validateConnectionBinding(connection); err != nil {
// A recovery manager may inspect a foreign singleton connection, but
// it must not activate it: activate() would combine the old persisted
// Secret with the new Revision's token endpoint.
if connection.LifecycleStatus == "retiring" {
go manager.finishRetirement(connection)
}
manager.bindingMismatch = true
return manager, nil
}
}
if connection.LifecycleStatus == "retiring" {
if connection.StreamID != nil && connection.Audience != nil {
_ = manager.activate(ctx, connection)
@@ -165,7 +210,9 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository,
if connection.StreamID != nil && connection.Audience != nil {
if err := manager.activate(ctx, connection); err != nil {
category := "credential_unavailable"
_, _ = repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "degraded", &category)
_, _ = repository.TransitionSecurityEventConnectionLifecycle(
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "degraded", &category,
)
}
}
if connection.LifecycleStatus == "bootstrap" {
@@ -174,6 +221,42 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository,
return manager, nil
}
func hasRevisionBindingExpectation(config ConnectionManagerConfig) bool {
return strings.TrimSpace(config.ExpectedTransmitterIssuer) != "" ||
strings.TrimSpace(config.ExpectedAudience) != "" ||
strings.TrimSpace(config.ExpectedOwnerKey) != ""
}
// ValidateConfiguredConnectionBinding verifies that the singleton persisted
// SSF connection belongs to the identity Revision represented by this manager.
// Recovery managers call this explicitly only when they are about to become an
// Active Runtime; cleanup and retirement remain able to inspect old bindings.
func (m *ConnectionManager) ValidateConfiguredConnectionBinding(ctx context.Context) error {
connection, err := m.repository.SecurityEventConnection(ctx)
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return safeConnectionError{category: "connection_binding_missing", cause: err}
}
if err != nil {
return safeConnectionError{category: "connection_binding_unavailable", cause: err}
}
return m.validateConnectionBinding(connection)
}
func (m *ConnectionManager) validateConnectionBinding(connection store.SecurityEventConnection) error {
expectedIssuer := strings.TrimRight(strings.TrimSpace(m.config.ExpectedTransmitterIssuer), "/")
expectedAudience := strings.TrimSpace(m.config.ExpectedAudience)
expectedClientID := strings.TrimSpace(m.config.ManagementClientID)
expectedOwner := strings.TrimSpace(m.config.ExpectedOwnerKey)
if expectedIssuer == "" || expectedAudience == "" || expectedClientID == "" || expectedOwner == "" {
return safeConnectionError{category: "connection_binding_invalid", cause: errors.New("security event Revision binding is incomplete")}
}
if connection.TransmitterIssuer != expectedIssuer || connection.Audience == nil || *connection.Audience != expectedAudience ||
connection.ManagementClientID != expectedClientID || connection.IdempotencyKey != expectedOwner {
return safeConnectionError{category: "connection_binding_mismatch", cause: store.ErrSecurityEventConnectionConflict}
}
return nil
}
func (m *ConnectionManager) Get(ctx context.Context) (ConnectionView, error) {
connection, err := m.repository.SecurityEventConnection(ctx)
if err != nil {
@@ -183,6 +266,9 @@ func (m *ConnectionManager) Get(ctx context.Context) (ConnectionView, error) {
}
func (m *ConnectionManager) IntrospectionCredential(ctx context.Context) (string, []byte, error) {
if err := m.requireOperationalBinding(); err != nil {
return "", nil, err
}
connection, err := m.repository.SecurityEventConnection(ctx)
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
if m.config.ManagementClientID != "" && m.config.ManagementClientSecret != "" {
@@ -212,6 +298,10 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
if (managementClientID == "") != (len(managementSecret) == 0) {
return ConnectionView{}, errors.New("machine client id and secret must be provided together")
}
if err == nil && (existing.LifecycleStatus == "retiring" ||
existing.LifecycleStatus == "disconnect_pending" && existing.IdempotencyKey == idempotencyKey) {
return ConnectionView{}, safeConnectionError{category: "retirement_pending", cause: store.ErrSecurityEventConnectionConflict}
}
if !credentialsProvided {
if err == nil {
managementClientID, managementSecret, _ = m.managementCredential(ctx, existing)
@@ -225,10 +315,46 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
return ConnectionView{}, err
}
if err == nil {
if m.bindingMismatch && existing.IdempotencyKey == idempotencyKey {
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
}
if existing.TransmitterIssuer != issuer {
return ConnectionView{}, store.ErrSecurityEventConnectionConflict
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
}
if credentialsProvided {
if existing.IdempotencyKey != idempotencyKey {
if existing.ManagementClientID == "" || existing.ManagementClientID != managementClientID {
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
}
if existing.StreamID == nil {
m.retirement = &connectionRetirementAuthorization{
connectionID: existing.ConnectionID, credentialRef: valueOrEmpty(existing.ManagementCredentialRef), ownerKey: idempotencyKey,
}
return ConnectionView{}, safeConnectionError{category: "connection_conflict", cause: store.ErrSecurityEventConnectionConflict}
}
if m.retirement == nil || m.retirement.connectionID != existing.ConnectionID ||
m.retirement.credentialRef != valueOrEmpty(existing.ManagementCredentialRef) ||
m.retirement.ownerKey != idempotencyKey {
if err := m.proveManagementCredentialHandoff(ctx, existing, managementClientID, managementSecret); err != nil {
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: err}
}
existing, err = m.persistManagementCredential(ctx, existing, managementClientID, managementSecret)
if err != nil {
return ConnectionView{}, err
}
m.retirement = &connectionRetirementAuthorization{
connectionID: existing.ConnectionID, credentialRef: valueOrEmpty(existing.ManagementCredentialRef), ownerKey: idempotencyKey,
}
}
if existing.LifecycleStatus == "disconnect_pending" {
go m.retryDisconnect(existing.ConnectionID)
return ConnectionView{}, safeConnectionError{category: "retirement_pending", cause: store.ErrSecurityEventConnectionConflict}
}
return ConnectionView{}, safeConnectionError{category: "connection_conflict", cause: store.ErrSecurityEventConnectionConflict}
}
if existing.ManagementClientID != "" && existing.ManagementClientID != managementClientID {
return ConnectionView{}, store.ErrSecurityEventConnectionConflict
}
existing, err = m.persistManagementCredential(ctx, existing, managementClientID, managementSecret)
if err != nil {
return ConnectionView{}, err
@@ -247,17 +373,17 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
connectionID := uuid.NewString()
reference := "ssf-push-" + connectionID
managementReference := "ssf-management-" + connectionID
if err := m.secrets.Put(ctx, managementReference, managementSecret); err != nil {
if err := m.stageIdentitySecret(ctx, managementReference, managementSecret); err != nil {
return ConnectionView{}, err
}
secret, err := randomPushBearer()
if err != nil {
_ = m.secrets.Delete(ctx, managementReference)
_ = m.retireIdentitySecret(ctx, managementReference)
return ConnectionView{}, err
}
defer clear(secret)
if err := m.secrets.Put(ctx, reference, secret); err != nil {
_ = m.secrets.Delete(ctx, managementReference)
if err := m.stageIdentitySecret(ctx, reference, secret); err != nil {
_ = m.retireIdentitySecret(ctx, managementReference)
return ConnectionView{}, err
}
endpoint := strings.TrimRight(m.config.PublicBaseURL, "/") + "/api/v1/security-events/ssf"
@@ -267,8 +393,9 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
ManagementCredentialRef: managementReference, IdempotencyKey: idempotencyKey,
})
if err != nil {
_ = m.secrets.Delete(ctx, reference)
_ = m.secrets.Delete(ctx, managementReference)
// Commit outcome can be ambiguous: rollback keeps the staging rows,
// while commit atomically removes them. Requeueing here could delete
// credentials already referenced by the committed connection.
return ConnectionView{}, err
}
return m.resumeConnecting(ctx, connection)
@@ -276,17 +403,14 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
func (m *ConnectionManager) persistManagementCredential(ctx context.Context, connection store.SecurityEventConnection, clientID string, secret []byte) (store.SecurityEventConnection, error) {
reference := "ssf-management-" + connection.ConnectionID + "-" + uuid.NewString()
if err := m.secrets.Put(ctx, reference, secret); err != nil {
if err := m.stageIdentitySecret(ctx, reference, secret); err != nil {
return store.SecurityEventConnection{}, err
}
updated, err := m.repository.SetSecurityEventManagementCredential(ctx, connection.ConnectionID, clientID, reference)
updated, err := m.repository.SetSecurityEventManagementCredential(ctx, connection.ConnectionID, clientID, reference, connection.Version)
if err != nil {
_ = m.secrets.Delete(ctx, reference)
// The staging row is the source of truth when commit outcome is unknown.
return store.SecurityEventConnection{}, err
}
if connection.ManagementCredentialRef != nil {
_ = m.secrets.Delete(ctx, *connection.ManagementCredentialRef)
}
return updated, nil
}
@@ -337,6 +461,9 @@ func (m *ConnectionManager) resumeConnecting(ctx context.Context, connection sto
}
func (m *ConnectionManager) Verify(ctx context.Context) (ConnectionView, error) {
if err := m.requireOperationalBinding(); err != nil {
return ConnectionView{}, err
}
connection, err := m.repository.SecurityEventConnection(ctx)
if err != nil {
return ConnectionView{}, err
@@ -366,6 +493,9 @@ func (m *ConnectionManager) Verify(ctx context.Context) (ConnectionView, error)
func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionView, error) {
m.operation.Lock()
defer m.operation.Unlock()
if err := m.requireOperationalBinding(); err != nil {
return ConnectionView{}, err
}
connection, err := m.repository.SecurityEventConnection(ctx)
if err != nil {
return ConnectionView{}, err
@@ -381,14 +511,14 @@ func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionVie
if err != nil {
return ConnectionView{}, err
}
if err := m.secrets.Put(ctx, nextReference, next); err != nil {
if err := m.stageIdentitySecret(ctx, nextReference, next); err != nil {
clear(next)
return ConnectionView{}, err
}
connection, err = m.repository.SetSecurityEventNextCredential(ctx, connection.ConnectionID, nextReference)
connection, err = m.repository.SetSecurityEventNextCredential(ctx, connection.ConnectionID, nextReference, connection.Version)
if err != nil {
clear(next)
_ = m.secrets.Delete(ctx, nextReference)
// Never recreate cleanup intent after an ambiguous adoption commit.
return ConnectionView{}, err
}
} else {
@@ -424,10 +554,56 @@ func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionVie
func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, error) {
m.operation.Lock()
defer m.operation.Unlock()
if err := m.requireOperationalBinding(); err != nil {
return ConnectionView{}, err
}
connection, err := m.repository.SecurityEventConnection(ctx)
if err != nil {
return ConnectionView{}, err
}
return m.disconnect(ctx, connection)
}
// RetireConflictingConnection is a pairing-scoped recovery operation. It may
// retire only a connection owned by another workflow; a stale request becomes
// a no-op once the current pairing owns the singleton connection.
func (m *ConnectionManager) RetireConflictingConnection(ctx context.Context, currentOwnerKey string) error {
m.operation.Lock()
defer m.operation.Unlock()
connection, err := m.repository.SecurityEventConnection(ctx)
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return nil
}
if err != nil {
return err
}
if strings.TrimSpace(currentOwnerKey) == "" {
return errors.New("current security event owner is required")
}
if connection.IdempotencyKey == currentOwnerKey {
return nil
}
if connection.LifecycleStatus == "retiring" {
go m.finishRetirement(connection)
return nil
}
if m.retirement == nil || m.retirement.connectionID != connection.ConnectionID ||
m.retirement.credentialRef != valueOrEmpty(connection.ManagementCredentialRef) ||
m.retirement.ownerKey != currentOwnerKey {
return safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
}
if connection.LifecycleStatus == "disconnect_pending" {
go m.retryDisconnect(connection.ConnectionID)
return nil
}
_, err = m.disconnect(ctx, connection)
return err
}
func (m *ConnectionManager) disconnect(ctx context.Context, connection store.SecurityEventConnection) (ConnectionView, error) {
if connection.LifecycleStatus == "retiring" || connection.LifecycleStatus == "disconnect_pending" {
return m.view(connection), nil
}
if connection.StreamID != nil {
configuration, client, discoverErr := m.discover(ctx, connection.TransmitterIssuer)
if discoverErr == nil {
@@ -441,12 +617,20 @@ func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, err
}
if discoverErr != nil {
category := "remote_disconnect_failed"
connection, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "disconnect_pending", &category)
transitioned, transitionErr := m.repository.TransitionSecurityEventConnectionLifecycle(
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "disconnect_pending", &category,
)
if transitionErr != nil {
return ConnectionView{}, transitionErr
}
connection = transitioned
go m.retryDisconnect(connection.ConnectionID)
return m.view(connection), nil
}
}
connection, err = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "retiring", nil)
connection, err := m.repository.TransitionSecurityEventConnectionLifecycle(
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "retiring", nil,
)
if err != nil {
return ConnectionView{}, err
}
@@ -454,6 +638,51 @@ func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, err
return m.view(connection), nil
}
// DiscardPreparedConnection removes only the connection created by one identity
// pairing. Bound streams continue through the normal retirement window so
// revocation watermarks and introspection fallback are never bypassed.
func (m *ConnectionManager) DiscardPreparedConnection(ctx context.Context, ownerKey string) error {
m.operation.Lock()
defer m.operation.Unlock()
connection, err := m.repository.SecurityEventConnection(ctx)
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return nil
}
if err != nil {
return safeConnectionError{category: "connection_cleanup_failed", cause: err}
}
if strings.TrimSpace(ownerKey) == "" || connection.IdempotencyKey != ownerKey {
return nil
}
if connection.StreamID != nil {
if connection.LifecycleStatus != "retiring" && connection.LifecycleStatus != "disconnect_pending" {
if _, err := m.disconnect(ctx, connection); err != nil {
return safeConnectionError{category: "connection_cleanup_failed", cause: err}
}
}
return safeConnectionError{category: "retirement_pending", cause: errors.New("security event retirement is pending")}
}
err = m.repository.DiscardPreparedSecurityEventConnection(
ctx, connection.ConnectionID, ownerKey, connection.Version,
)
if err != nil && !errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return safeConnectionError{category: "connection_cleanup_failed", cause: err}
}
m.stopRuntime()
return nil
}
func connectionSecretReferences(connection store.SecurityEventConnection) []string {
references := []string{connection.CredentialRef}
if connection.ManagementCredentialRef != nil {
references = append(references, *connection.ManagementCredentialRef)
}
if connection.NextCredentialRef != nil {
references = append(references, *connection.NextCredentialRef)
}
return references
}
func (m *ConnectionManager) retryDisconnect(connectionID string) {
delay := time.Second
for {
@@ -478,7 +707,9 @@ func (m *ConnectionManager) retryDisconnect(connectionID string) {
}
}
if err == nil {
connection, err = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "retiring", nil)
connection, err = m.repository.TransitionSecurityEventConnectionLifecycle(
m.ctx, connectionID, "disconnect_pending", connection.Version, "retiring", nil,
)
if err == nil {
go m.finishRetirement(connection)
}
@@ -625,49 +856,57 @@ func (m *ConnectionManager) finishAutomaticVerification(connectionID string, con
defer cancel()
ticker := time.NewTicker(250 * time.Millisecond)
defer ticker.Stop()
expectedLifecycle := "verifying"
if rotating {
expectedLifecycle = "rotating"
}
for {
connection, err := m.repository.SecurityEventConnection(ctx)
if err != nil || connection.ConnectionID != connectionID || connection.StreamID == nil {
if err != nil || connection.ConnectionID != connectionID || connection.StreamID == nil || connection.LifecycleStatus != expectedLifecycle {
return
}
_, verifiedAt, _ := m.repository.SecurityEventMetrics(ctx, connection.TransmitterIssuer, valueOrEmpty(connection.Audience))
if verifiedAt != nil && !verifiedAt.Before(started) {
if err := m.setStreamStatus(ctx, client, configuration, token, *connection.StreamID, "enabled", "verification_succeeded"); err != nil {
category := "stream_enable_failed"
lifecycle := "verifying"
if rotating {
lifecycle = "rotating"
}
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connectionID, lifecycle, &category)
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
ctx, connectionID, expectedLifecycle, connection.Version, expectedLifecycle, &category,
)
return
}
if rotating && connection.NextCredentialRef != nil {
oldReference, nextReference := connection.CredentialRef, *connection.NextCredentialRef
nextReference := *connection.NextCredentialRef
select {
case <-m.ctx.Done():
return
case <-time.After(m.config.CredentialOverlapDuration):
}
current, currentErr := m.repository.SecurityEventConnection(m.ctx)
if currentErr != nil || current.ConnectionID != connectionID || current.LifecycleStatus != "rotating" ||
current.NextCredentialRef == nil || *current.NextCredentialRef != nextReference {
return
}
connection, err = m.repository.PromoteSecurityEventCredential(m.ctx, connectionID, nextReference)
if err == nil {
_ = m.secrets.Delete(m.ctx, oldReference)
_ = m.activate(m.ctx, connection)
go m.finishBootstrap(connectionID)
}
return
}
connection, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connectionID, "bootstrap", nil)
go m.finishBootstrap(connectionID)
connection, err = m.repository.TransitionSecurityEventConnectionLifecycle(
ctx, connectionID, expectedLifecycle, connection.Version, "bootstrap", nil,
)
if err == nil {
go m.finishBootstrap(connectionID)
}
return
}
select {
case <-ctx.Done():
category := "verification_timeout"
lifecycle := "verifying"
if rotating {
lifecycle = "rotating"
}
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, lifecycle, &category)
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
m.ctx, connectionID, expectedLifecycle, connection.Version, expectedLifecycle, &category,
)
return
case <-ticker.C:
}
@@ -691,46 +930,89 @@ func (m *ConnectionManager) finishBootstrap(connectionID string) {
if err == nil && connection.ConnectionID == connectionID && connection.LifecycleStatus == "bootstrap" {
mode, _, metricsErr := m.repository.SecurityEventMetrics(m.ctx, connection.TransmitterIssuer, valueOrEmpty(connection.Audience))
if metricsErr == nil && mode == "push_healthy" {
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "enabled", nil)
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
m.ctx, connectionID, "bootstrap", connection.Version, "enabled", nil,
)
return
}
category := "bootstrap_verification_stale"
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "degraded", &category)
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
m.ctx, connectionID, "bootstrap", connection.Version, "degraded", &category,
)
}
}
func (m *ConnectionManager) finishRetirement(connection store.SecurityEventConnection) {
elapsed := time.Since(connection.UpdatedAt)
delay := m.config.RetirementDuration - elapsed
if delay > 0 {
select {
case <-m.ctx.Done():
return
case <-time.After(delay):
}
}
delay = time.Second
func (m *ConnectionManager) finishRetirement(retirement store.SecurityEventConnection) {
retryDelay := time.Second
for {
if err := m.repository.DeleteSecurityEventConnection(m.ctx, connection.ConnectionID); err == nil || errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
break
}
select {
case <-m.ctx.Done():
connection, err := m.repository.SecurityEventConnection(m.ctx)
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return
case <-time.After(delay):
}
if delay < time.Minute {
delay *= 2
if err != nil {
if !waitForConnectionRetry(m.ctx, retryDelay) {
return
}
retryDelay = nextConnectionRetryDelay(retryDelay)
continue
}
if connection.ConnectionID != retirement.ConnectionID || connection.LifecycleStatus != "retiring" {
return
}
if remaining := m.config.RetirementDuration - time.Since(connection.UpdatedAt); remaining > 0 {
select {
case <-m.ctx.Done():
return
case <-time.After(remaining):
}
continue
}
m.stopRuntime()
err = m.repository.FinalizeRetiringSecurityEventConnection(m.ctx, connection.ConnectionID, connection.Version)
if err == nil || errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return
}
if !waitForConnectionRetry(m.ctx, retryDelay) {
return
}
retryDelay = nextConnectionRetryDelay(retryDelay)
}
m.stopRuntime()
_ = m.secrets.Delete(m.ctx, connection.CredentialRef)
if connection.ManagementCredentialRef != nil {
_ = m.secrets.Delete(m.ctx, *connection.ManagementCredentialRef)
}
func waitForConnectionRetry(ctx context.Context, delay time.Duration) bool {
select {
case <-ctx.Done():
return false
case <-time.After(delay):
return true
}
if connection.NextCredentialRef != nil {
_ = m.secrets.Delete(m.ctx, *connection.NextCredentialRef)
}
func nextConnectionRetryDelay(delay time.Duration) time.Duration {
if delay < time.Minute {
return delay * 2
}
return delay
}
const identitySecretStagingTTL = 10 * time.Minute
func (m *ConnectionManager) stageIdentitySecret(ctx context.Context, reference string, value []byte) error {
if err := m.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC().Add(identitySecretStagingTTL)); err != nil {
return err
}
if err := m.secrets.Put(ctx, reference, value); err != nil {
deleteErr := m.secrets.Delete(ctx, reference)
if deleteErr == nil || errors.Is(deleteErr, ErrSecretNotFound) {
_, _ = m.repository.RemovePendingIdentitySecretCleanup(ctx, reference)
}
return err
}
return nil
}
func (m *ConnectionManager) retireIdentitySecret(ctx context.Context, reference string) error {
return m.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC())
}
func (m *ConnectionManager) validatePrerequisites(issuer, managementClientID string, managementSecret []byte, requirePublicBaseURL bool) error {
@@ -794,6 +1076,10 @@ func (m *ConnectionManager) managementToken(ctx context.Context, client *http.Cl
return "", err
}
defer clear(managementSecret)
return m.requestManagementToken(ctx, client, scopes, managementClientID, managementSecret)
}
func (m *ConnectionManager) requestManagementToken(ctx context.Context, client *http.Client, scopes, managementClientID string, managementSecret []byte) (string, error) {
form := url.Values{"grant_type": {"client_credentials"}, "scope": {scopes}}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(m.config.OIDCIssuer, "/")+"/token", strings.NewReader(form.Encode()))
if err != nil {
@@ -820,6 +1106,44 @@ func (m *ConnectionManager) managementToken(ctx context.Context, client *http.Cl
return payload.AccessToken, nil
}
func (m *ConnectionManager) proveManagementCredentialHandoff(ctx context.Context, connection store.SecurityEventConnection, managementClientID string, managementSecret []byte) error {
if connection.StreamID == nil || connection.Audience == nil {
return store.ErrSecurityEventConnectionConflict
}
configuration, client, err := m.discover(ctx, connection.TransmitterIssuer)
if err != nil {
return err
}
token, err := m.requestManagementToken(ctx, client, "ssf.stream.read ssf.stream.manage", managementClientID, managementSecret)
if err != nil {
return err
}
var streams []streamConfiguration
if err := requestJSON(ctx, client, http.MethodGet, configuration.ConfigurationEndpoint, token, nil, &streams); err != nil {
return err
}
for _, stream := range streams {
if stream.StreamID != *connection.StreamID {
continue
}
if err := validateCreatedStream(stream, connection.TransmitterIssuer, connection.EndpointURL); err != nil || stream.Audience != *connection.Audience {
return store.ErrSecurityEventConnectionConflict
}
return nil
}
// An authenticated empty list proves that the new credential belongs to a
// token issuer trusted by the old transmitter. Disconnect treats the absent
// Stream as an idempotent remote deletion.
return nil
}
func (m *ConnectionManager) requireOperationalBinding() error {
if m.bindingMismatch {
return safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
}
return nil
}
func (m *ConnectionManager) managementCredential(ctx context.Context, connection store.SecurityEventConnection) (string, []byte, error) {
if connection.ManagementCredentialRef != nil && connection.ManagementClientID != "" {
secret, err := m.secrets.Get(ctx, *connection.ManagementCredentialRef)
@@ -882,8 +1206,10 @@ func (m *ConnectionManager) deleteStream(ctx context.Context, client *http.Clien
}
func (m *ConnectionManager) connectionError(ctx context.Context, connection store.SecurityEventConnection, category string, cause error) error {
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "error", &category)
return cause
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "error", &category,
)
return safeConnectionError{category: category, cause: cause}
}
func (m *ConnectionManager) view(connection store.SecurityEventConnection) ConnectionView {
@@ -1054,7 +1380,12 @@ var blockedNetworkPrefixes = []netip.Prefix{
}
func isLocalHostname(value string) bool {
return value == "localhost" || value == "127.0.0.1" || value == "::1"
value = strings.ToLower(strings.TrimSpace(value))
if value == "localhost" {
return true
}
ip := net.ParseIP(value)
return ip != nil && ip.IsLoopback()
}
func isLocalEnvironment(value string) bool {
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,52 @@
package securityevents
import (
"context"
"errors"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const (
identitySecretCleanupInterval = time.Minute
identitySecretCleanupLease = 5 * time.Minute
identitySecretCleanupBatch = 100
)
type IdentitySecretCleanupRepository interface {
ClaimIdentitySecretCleanups(context.Context, int, time.Duration) ([]store.IdentitySecretCleanupClaim, error)
CompleteIdentitySecretCleanup(context.Context, string, string) (bool, error)
}
// RunIdentitySecretCleanupWorker owns cleanup at the server lifecycle rather
// than at any individual Active or Prepared identity Runtime. PostgreSQL claim
// tokens make running one worker per server replica safe.
func RunIdentitySecretCleanupWorker(ctx context.Context, repository IdentitySecretCleanupRepository, secrets SecretStore) {
if repository == nil || secrets == nil {
return
}
ticker := time.NewTicker(identitySecretCleanupInterval)
defer ticker.Stop()
for {
cleanupClaimedIdentitySecrets(ctx, repository, secrets)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func cleanupClaimedIdentitySecrets(ctx context.Context, repository IdentitySecretCleanupRepository, secrets SecretStore) {
claims, err := repository.ClaimIdentitySecretCleanups(ctx, identitySecretCleanupBatch, identitySecretCleanupLease)
if err != nil {
return
}
for _, claim := range claims {
if err := secrets.Delete(ctx, claim.Reference); err != nil && !errors.Is(err, ErrSecretNotFound) {
continue
}
_, _ = repository.CompleteIdentitySecretCleanup(ctx, claim.Reference, claim.ClaimToken)
}
}
@@ -0,0 +1,79 @@
package securityevents
import (
"context"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const (
defaultSecurityEventRetirementDuration = 6 * time.Minute
securityEventRetirementWorkerInterval = time.Minute
)
type SecurityEventRetirementRepository interface {
SecurityEventConnection(context.Context) (store.SecurityEventConnection, error)
FinalizeRetiringSecurityEventConnection(context.Context, string, int64) error
}
// RunSecurityEventRetirementWorker owns the local end of SSF retirement at the
// server lifecycle. It intentionally does not share an Active identity
// Runtime's context, because that Runtime is closed before the RFC 8935 overlap
// period expires. Store-level generation CAS makes one worker per replica safe.
func RunSecurityEventRetirementWorker(ctx context.Context, repository SecurityEventRetirementRepository) {
runSecurityEventRetirementWorker(
ctx,
repository,
defaultSecurityEventRetirementDuration,
securityEventRetirementWorkerInterval,
)
}
func runSecurityEventRetirementWorker(
ctx context.Context,
repository SecurityEventRetirementRepository,
retirementDuration time.Duration,
interval time.Duration,
) {
if repository == nil {
return
}
if retirementDuration <= 0 {
retirementDuration = defaultSecurityEventRetirementDuration
}
if interval <= 0 {
interval = securityEventRetirementWorkerInterval
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
finalizeDueSecurityEventRetirement(ctx, repository, retirementDuration)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func finalizeDueSecurityEventRetirement(
ctx context.Context,
repository SecurityEventRetirementRepository,
retirementDuration time.Duration,
) {
connection, err := repository.SecurityEventConnection(ctx)
if err != nil || connection.LifecycleStatus != "retiring" {
return
}
if retirementDuration <= 0 {
retirementDuration = defaultSecurityEventRetirementDuration
}
if time.Since(connection.UpdatedAt) < retirementDuration {
return
}
// Finalization atomically queues every Secret reference and removes only the
// exact retiring generation. A concurrent worker or changed generation is a
// benign CAS miss and will be observed on the next loop.
_ = repository.FinalizeRetiringSecurityEventConnection(ctx, connection.ConnectionID, connection.Version)
}
@@ -0,0 +1,178 @@
package securityevents
import (
"context"
"errors"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
func TestSecurityEventRetirementWorkerOutlivesCanceledRuntime(t *testing.T) {
connection := retiringWorkerTestConnection(time.Now().UTC())
repository := &memoryConnectionRepository{connection: &connection}
secrets := &memorySecretStore{values: retirementWorkerTestSecrets(connection)}
runtimeCtx, cancelRuntime := context.WithCancel(context.Background())
manager := &ConnectionManager{
ctx: runtimeCtx, repository: repository, secrets: secrets,
config: ConnectionManagerConfig{RetirementDuration: 40 * time.Millisecond},
}
go manager.finishRetirement(connection)
cancelRuntime()
time.Sleep(10 * time.Millisecond)
if _, err := repository.SecurityEventConnection(context.Background()); err != nil {
t.Fatalf("runtime cancellation unexpectedly finalized retirement: %v", err)
}
serverCtx, cancelServer := context.WithCancel(context.Background())
defer cancelServer()
go runSecurityEventRetirementWorker(serverCtx, repository, 40*time.Millisecond, 5*time.Millisecond)
waitForRetirementFinalization(t, repository)
for reference := range retirementWorkerTestSecrets(connection) {
repository.mutex.Lock()
_, queued := repository.cleanupDue[reference]
repository.mutex.Unlock()
if !queued {
t.Fatalf("retired Secret %q was not durably queued", reference)
}
if !secrets.Has(reference) {
t.Fatalf("retirement worker directly deleted Secret %q before the durable cleanup worker claimed it", reference)
}
}
}
func TestSecurityEventRetirementWorkerResumesAfterServerRestart(t *testing.T) {
connection := retiringWorkerTestConnection(time.Now().UTC())
repository := &memoryConnectionRepository{connection: &connection}
firstCtx, stopFirst := context.WithCancel(context.Background())
go runSecurityEventRetirementWorker(firstCtx, repository, 80*time.Millisecond, 5*time.Millisecond)
time.Sleep(15 * time.Millisecond)
stopFirst()
if _, err := repository.SecurityEventConnection(context.Background()); err != nil {
t.Fatalf("not-yet-due retirement disappeared before restart: %v", err)
}
time.Sleep(75 * time.Millisecond)
secondCtx, stopSecond := context.WithCancel(context.Background())
defer stopSecond()
go runSecurityEventRetirementWorker(secondCtx, repository, 80*time.Millisecond, 5*time.Millisecond)
waitForRetirementFinalization(t, repository)
}
func TestConcurrentSecurityEventRetirementWorkersFinalizeOneGenerationOnce(t *testing.T) {
connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute))
repository := &memoryConnectionRepository{connection: &connection}
start := make(chan struct{})
done := make(chan struct{}, 2)
for range 2 {
go func() {
<-start
finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond)
done <- struct{}{}
}()
}
close(start)
<-done
<-done
if _, err := repository.SecurityEventConnection(context.Background()); !errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
t.Fatalf("current retiring generation still exists: %v", err)
}
repository.mutex.Lock()
finalizeCalls := repository.finalizeCalls
queued := len(repository.cleanupDue)
repository.mutex.Unlock()
if finalizeCalls != 1 {
t.Fatalf("retirement generation finalized %d times, want 1", finalizeCalls)
}
if queued != len(retirementWorkerTestSecrets(connection)) {
t.Fatalf("queued Secret references=%d want=%d", queued, len(retirementWorkerTestSecrets(connection)))
}
}
func TestSecurityEventRetirementWorkerPreservesChangedGeneration(t *testing.T) {
connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute))
initialVersion := connection.Version
repository := &memoryConnectionRepository{connection: &connection}
repository.beforeFinalize = func(current *store.SecurityEventConnection) {
current.LifecycleStatus = "enabled"
current.Version++
}
finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond)
current, err := repository.SecurityEventConnection(context.Background())
if err != nil || current.LifecycleStatus != "enabled" || current.Version != initialVersion+1 {
t.Fatalf("changed generation was not preserved: lifecycle=%q version=%d err=%v", current.LifecycleStatus, current.Version, err)
}
repository.mutex.Lock()
queued := len(repository.cleanupDue)
repository.mutex.Unlock()
if queued != 0 {
t.Fatalf("changed generation queued %d active Secret references", queued)
}
}
func TestSecurityEventRetirementWorkerKeepsConnectionWhenCleanupQueueCannotAdoptReferences(t *testing.T) {
connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute))
repository := &memoryConnectionRepository{
connection: &connection,
cleanupClaims: map[string]string{connection.CredentialRef: uuid.NewString()},
}
finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond)
current, err := repository.SecurityEventConnection(context.Background())
if err != nil || current.ConnectionID != connection.ConnectionID || current.LifecycleStatus != "retiring" {
t.Fatalf("failed durable handoff lost retiring connection: lifecycle=%q err=%v", current.LifecycleStatus, err)
}
repository.mutex.Lock()
finalizeCalls := repository.finalizeCalls
repository.mutex.Unlock()
if finalizeCalls != 0 {
t.Fatalf("failed durable handoff finalized generation %d times", finalizeCalls)
}
}
func retiringWorkerTestConnection(updatedAt time.Time) store.SecurityEventConnection {
managementReference := "ssf-management-retirement-worker-" + uuid.NewString()
nextReference := "ssf-push-next-retirement-worker-" + uuid.NewString()
return store.SecurityEventConnection{
ConnectionID: uuid.NewString(),
CredentialRef: "ssf-push-retirement-worker-" + uuid.NewString(),
NextCredentialRef: &nextReference,
ManagementCredentialRef: &managementReference,
LifecycleStatus: "retiring",
Version: 9,
UpdatedAt: updatedAt,
}
}
func retirementWorkerTestSecrets(connection store.SecurityEventConnection) map[string][]byte {
secrets := map[string][]byte{connection.CredentialRef: []byte("retired-push-secret-value")}
if connection.NextCredentialRef != nil {
secrets[*connection.NextCredentialRef] = []byte("retired-next-push-secret-value")
}
if connection.ManagementCredentialRef != nil {
secrets[*connection.ManagementCredentialRef] = []byte("retired-management-secret-value")
}
return secrets
}
func waitForRetirementFinalization(t *testing.T, repository *memoryConnectionRepository) {
t.Helper()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if _, err := repository.SecurityEventConnection(context.Background()); errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("retiring connection was not finalized by the server-lifecycle worker")
}