fix(identity): 完善统一认证配对恢复与安全退役
修复 credentials_saved 状态无法恢复、配对与激活并发冲突,以及 SSF 和身份 Secret 生命周期不完整的问题。新增持久化协调器、取消与清理状态机、事务级并发门禁、受控 SSF 凭据交接、禁用后的延迟 Secret 清理,并对生产环境统一认证及 Discovery 端点强制 HTTPS。 验证:go test ./...;go test -race ./internal/auth ./internal/identity ./internal/identityruntime ./internal/securityevents ./internal/httpapi ./internal/store -count=1;go vet ./...;真实 PostgreSQL 并发及清理成功/冲突回滚测试;pnpm openapi。
This commit is contained in:
@@ -34,12 +34,15 @@ type ConnectionRepository interface {
|
||||
CreateSecurityEventConnection(context.Context, store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error)
|
||||
SecurityEventConnection(context.Context) (store.SecurityEventConnection, error)
|
||||
BindSecurityEventConnection(context.Context, string, string, string, string, int64) (store.SecurityEventConnection, error)
|
||||
SetSecurityEventManagementCredential(context.Context, string, string, string) (store.SecurityEventConnection, error)
|
||||
UpdateSecurityEventConnectionLifecycle(context.Context, string, string, *string) (store.SecurityEventConnection, error)
|
||||
SetSecurityEventNextCredential(context.Context, string, string) (store.SecurityEventConnection, error)
|
||||
SetSecurityEventManagementCredential(context.Context, string, string, string, int64) (store.SecurityEventConnection, error)
|
||||
TransitionSecurityEventConnectionLifecycle(context.Context, string, string, int64, string, *string) (store.SecurityEventConnection, error)
|
||||
SetSecurityEventNextCredential(context.Context, string, string, int64) (store.SecurityEventConnection, error)
|
||||
PromoteSecurityEventCredential(context.Context, string, string) (store.SecurityEventConnection, error)
|
||||
DeleteSecurityEventConnection(context.Context, string) error
|
||||
DiscardPreparedSecurityEventConnection(context.Context, string, string, int64) error
|
||||
FinalizeRetiringSecurityEventConnection(context.Context, string, int64) error
|
||||
SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error)
|
||||
QueueIdentitySecretCleanup(context.Context, string, time.Time) error
|
||||
RemovePendingIdentitySecretCleanup(context.Context, string) (bool, error)
|
||||
}
|
||||
|
||||
type ConnectionManagerConfig struct {
|
||||
@@ -49,6 +52,10 @@ type ConnectionManagerConfig struct {
|
||||
OIDCTenantID string
|
||||
ManagementClientID string
|
||||
ManagementClientSecret string
|
||||
ExpectedTransmitterIssuer string
|
||||
ExpectedAudience string
|
||||
ExpectedOwnerKey string
|
||||
StrictRevisionBinding bool
|
||||
PublicBaseURL string
|
||||
HeartbeatInterval time.Duration
|
||||
StaleAfter time.Duration
|
||||
@@ -79,15 +86,23 @@ type connectionRuntime struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type connectionRetirementAuthorization struct {
|
||||
connectionID string
|
||||
credentialRef string
|
||||
ownerKey string
|
||||
}
|
||||
|
||||
type ConnectionManager struct {
|
||||
ctx context.Context
|
||||
repository ConnectionRepository
|
||||
secrets SecretStore
|
||||
config ConnectionManagerConfig
|
||||
metrics *Metrics
|
||||
mutex sync.RWMutex
|
||||
operation sync.Mutex
|
||||
runtime *connectionRuntime
|
||||
ctx context.Context
|
||||
repository ConnectionRepository
|
||||
secrets SecretStore
|
||||
config ConnectionManagerConfig
|
||||
metrics *Metrics
|
||||
mutex sync.RWMutex
|
||||
operation sync.Mutex
|
||||
runtime *connectionRuntime
|
||||
retirement *connectionRetirementAuthorization
|
||||
bindingMismatch bool
|
||||
}
|
||||
|
||||
type transmitterConfiguration struct {
|
||||
@@ -118,6 +133,17 @@ func (e remoteHTTPError) Error() string {
|
||||
return fmt.Sprintf("SSF endpoint returned HTTP %d", e.status)
|
||||
}
|
||||
|
||||
type safeConnectionError struct {
|
||||
category string
|
||||
cause error
|
||||
}
|
||||
|
||||
func (err safeConnectionError) Error() string {
|
||||
return "security event operation failed: " + err.category
|
||||
}
|
||||
func (err safeConnectionError) Unwrap() error { return err.cause }
|
||||
func (err safeConnectionError) SafeErrorCategory() string { return err.category }
|
||||
|
||||
func NewConnectionManager(ctx context.Context, repository ConnectionRepository, secrets SecretStore, config ConnectionManagerConfig, metrics *Metrics) (*ConnectionManager, error) {
|
||||
if repository == nil || secrets == nil || !config.OIDCEnabled || config.OIDCIssuer == "" || config.OIDCTenantID == "" {
|
||||
return nil, errors.New("security event connection prerequisites are incomplete")
|
||||
@@ -138,16 +164,35 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository,
|
||||
config.CredentialOverlapDuration = 3 * time.Minute
|
||||
}
|
||||
if config.RetirementDuration == 0 {
|
||||
config.RetirementDuration = 6 * time.Minute
|
||||
config.RetirementDuration = defaultSecurityEventRetirementDuration
|
||||
}
|
||||
manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: config, metrics: metrics}
|
||||
connection, err := repository.SecurityEventConnection(ctx)
|
||||
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
if config.StrictRevisionBinding {
|
||||
return nil, safeConnectionError{category: "connection_binding_missing", cause: err}
|
||||
}
|
||||
return manager, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load security event connection: %w", err)
|
||||
}
|
||||
if config.StrictRevisionBinding {
|
||||
if err := manager.validateConnectionBinding(connection); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if hasRevisionBindingExpectation(config) {
|
||||
if err := manager.validateConnectionBinding(connection); err != nil {
|
||||
// A recovery manager may inspect a foreign singleton connection, but
|
||||
// it must not activate it: activate() would combine the old persisted
|
||||
// Secret with the new Revision's token endpoint.
|
||||
if connection.LifecycleStatus == "retiring" {
|
||||
go manager.finishRetirement(connection)
|
||||
}
|
||||
manager.bindingMismatch = true
|
||||
return manager, nil
|
||||
}
|
||||
}
|
||||
if connection.LifecycleStatus == "retiring" {
|
||||
if connection.StreamID != nil && connection.Audience != nil {
|
||||
_ = manager.activate(ctx, connection)
|
||||
@@ -165,7 +210,9 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository,
|
||||
if connection.StreamID != nil && connection.Audience != nil {
|
||||
if err := manager.activate(ctx, connection); err != nil {
|
||||
category := "credential_unavailable"
|
||||
_, _ = repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "degraded", &category)
|
||||
_, _ = repository.TransitionSecurityEventConnectionLifecycle(
|
||||
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "degraded", &category,
|
||||
)
|
||||
}
|
||||
}
|
||||
if connection.LifecycleStatus == "bootstrap" {
|
||||
@@ -174,6 +221,42 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository,
|
||||
return manager, nil
|
||||
}
|
||||
|
||||
func hasRevisionBindingExpectation(config ConnectionManagerConfig) bool {
|
||||
return strings.TrimSpace(config.ExpectedTransmitterIssuer) != "" ||
|
||||
strings.TrimSpace(config.ExpectedAudience) != "" ||
|
||||
strings.TrimSpace(config.ExpectedOwnerKey) != ""
|
||||
}
|
||||
|
||||
// ValidateConfiguredConnectionBinding verifies that the singleton persisted
|
||||
// SSF connection belongs to the identity Revision represented by this manager.
|
||||
// Recovery managers call this explicitly only when they are about to become an
|
||||
// Active Runtime; cleanup and retirement remain able to inspect old bindings.
|
||||
func (m *ConnectionManager) ValidateConfiguredConnectionBinding(ctx context.Context) error {
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
return safeConnectionError{category: "connection_binding_missing", cause: err}
|
||||
}
|
||||
if err != nil {
|
||||
return safeConnectionError{category: "connection_binding_unavailable", cause: err}
|
||||
}
|
||||
return m.validateConnectionBinding(connection)
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) validateConnectionBinding(connection store.SecurityEventConnection) error {
|
||||
expectedIssuer := strings.TrimRight(strings.TrimSpace(m.config.ExpectedTransmitterIssuer), "/")
|
||||
expectedAudience := strings.TrimSpace(m.config.ExpectedAudience)
|
||||
expectedClientID := strings.TrimSpace(m.config.ManagementClientID)
|
||||
expectedOwner := strings.TrimSpace(m.config.ExpectedOwnerKey)
|
||||
if expectedIssuer == "" || expectedAudience == "" || expectedClientID == "" || expectedOwner == "" {
|
||||
return safeConnectionError{category: "connection_binding_invalid", cause: errors.New("security event Revision binding is incomplete")}
|
||||
}
|
||||
if connection.TransmitterIssuer != expectedIssuer || connection.Audience == nil || *connection.Audience != expectedAudience ||
|
||||
connection.ManagementClientID != expectedClientID || connection.IdempotencyKey != expectedOwner {
|
||||
return safeConnectionError{category: "connection_binding_mismatch", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) Get(ctx context.Context) (ConnectionView, error) {
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if err != nil {
|
||||
@@ -183,6 +266,9 @@ func (m *ConnectionManager) Get(ctx context.Context) (ConnectionView, error) {
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) IntrospectionCredential(ctx context.Context) (string, []byte, error) {
|
||||
if err := m.requireOperationalBinding(); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
if m.config.ManagementClientID != "" && m.config.ManagementClientSecret != "" {
|
||||
@@ -212,6 +298,10 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
|
||||
if (managementClientID == "") != (len(managementSecret) == 0) {
|
||||
return ConnectionView{}, errors.New("machine client id and secret must be provided together")
|
||||
}
|
||||
if err == nil && (existing.LifecycleStatus == "retiring" ||
|
||||
existing.LifecycleStatus == "disconnect_pending" && existing.IdempotencyKey == idempotencyKey) {
|
||||
return ConnectionView{}, safeConnectionError{category: "retirement_pending", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
if !credentialsProvided {
|
||||
if err == nil {
|
||||
managementClientID, managementSecret, _ = m.managementCredential(ctx, existing)
|
||||
@@ -225,10 +315,46 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
if err == nil {
|
||||
if m.bindingMismatch && existing.IdempotencyKey == idempotencyKey {
|
||||
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
if existing.TransmitterIssuer != issuer {
|
||||
return ConnectionView{}, store.ErrSecurityEventConnectionConflict
|
||||
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
if credentialsProvided {
|
||||
if existing.IdempotencyKey != idempotencyKey {
|
||||
if existing.ManagementClientID == "" || existing.ManagementClientID != managementClientID {
|
||||
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
if existing.StreamID == nil {
|
||||
m.retirement = &connectionRetirementAuthorization{
|
||||
connectionID: existing.ConnectionID, credentialRef: valueOrEmpty(existing.ManagementCredentialRef), ownerKey: idempotencyKey,
|
||||
}
|
||||
return ConnectionView{}, safeConnectionError{category: "connection_conflict", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
if m.retirement == nil || m.retirement.connectionID != existing.ConnectionID ||
|
||||
m.retirement.credentialRef != valueOrEmpty(existing.ManagementCredentialRef) ||
|
||||
m.retirement.ownerKey != idempotencyKey {
|
||||
if err := m.proveManagementCredentialHandoff(ctx, existing, managementClientID, managementSecret); err != nil {
|
||||
return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: err}
|
||||
}
|
||||
existing, err = m.persistManagementCredential(ctx, existing, managementClientID, managementSecret)
|
||||
if err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
m.retirement = &connectionRetirementAuthorization{
|
||||
connectionID: existing.ConnectionID, credentialRef: valueOrEmpty(existing.ManagementCredentialRef), ownerKey: idempotencyKey,
|
||||
}
|
||||
}
|
||||
if existing.LifecycleStatus == "disconnect_pending" {
|
||||
go m.retryDisconnect(existing.ConnectionID)
|
||||
return ConnectionView{}, safeConnectionError{category: "retirement_pending", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
return ConnectionView{}, safeConnectionError{category: "connection_conflict", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
if existing.ManagementClientID != "" && existing.ManagementClientID != managementClientID {
|
||||
return ConnectionView{}, store.ErrSecurityEventConnectionConflict
|
||||
}
|
||||
existing, err = m.persistManagementCredential(ctx, existing, managementClientID, managementSecret)
|
||||
if err != nil {
|
||||
return ConnectionView{}, err
|
||||
@@ -247,17 +373,17 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
|
||||
connectionID := uuid.NewString()
|
||||
reference := "ssf-push-" + connectionID
|
||||
managementReference := "ssf-management-" + connectionID
|
||||
if err := m.secrets.Put(ctx, managementReference, managementSecret); err != nil {
|
||||
if err := m.stageIdentitySecret(ctx, managementReference, managementSecret); err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
secret, err := randomPushBearer()
|
||||
if err != nil {
|
||||
_ = m.secrets.Delete(ctx, managementReference)
|
||||
_ = m.retireIdentitySecret(ctx, managementReference)
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
defer clear(secret)
|
||||
if err := m.secrets.Put(ctx, reference, secret); err != nil {
|
||||
_ = m.secrets.Delete(ctx, managementReference)
|
||||
if err := m.stageIdentitySecret(ctx, reference, secret); err != nil {
|
||||
_ = m.retireIdentitySecret(ctx, managementReference)
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
endpoint := strings.TrimRight(m.config.PublicBaseURL, "/") + "/api/v1/security-events/ssf"
|
||||
@@ -267,8 +393,9 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
|
||||
ManagementCredentialRef: managementReference, IdempotencyKey: idempotencyKey,
|
||||
})
|
||||
if err != nil {
|
||||
_ = m.secrets.Delete(ctx, reference)
|
||||
_ = m.secrets.Delete(ctx, managementReference)
|
||||
// Commit outcome can be ambiguous: rollback keeps the staging rows,
|
||||
// while commit atomically removes them. Requeueing here could delete
|
||||
// credentials already referenced by the committed connection.
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
return m.resumeConnecting(ctx, connection)
|
||||
@@ -276,17 +403,14 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien
|
||||
|
||||
func (m *ConnectionManager) persistManagementCredential(ctx context.Context, connection store.SecurityEventConnection, clientID string, secret []byte) (store.SecurityEventConnection, error) {
|
||||
reference := "ssf-management-" + connection.ConnectionID + "-" + uuid.NewString()
|
||||
if err := m.secrets.Put(ctx, reference, secret); err != nil {
|
||||
if err := m.stageIdentitySecret(ctx, reference, secret); err != nil {
|
||||
return store.SecurityEventConnection{}, err
|
||||
}
|
||||
updated, err := m.repository.SetSecurityEventManagementCredential(ctx, connection.ConnectionID, clientID, reference)
|
||||
updated, err := m.repository.SetSecurityEventManagementCredential(ctx, connection.ConnectionID, clientID, reference, connection.Version)
|
||||
if err != nil {
|
||||
_ = m.secrets.Delete(ctx, reference)
|
||||
// The staging row is the source of truth when commit outcome is unknown.
|
||||
return store.SecurityEventConnection{}, err
|
||||
}
|
||||
if connection.ManagementCredentialRef != nil {
|
||||
_ = m.secrets.Delete(ctx, *connection.ManagementCredentialRef)
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
@@ -337,6 +461,9 @@ func (m *ConnectionManager) resumeConnecting(ctx context.Context, connection sto
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) Verify(ctx context.Context) (ConnectionView, error) {
|
||||
if err := m.requireOperationalBinding(); err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if err != nil {
|
||||
return ConnectionView{}, err
|
||||
@@ -366,6 +493,9 @@ func (m *ConnectionManager) Verify(ctx context.Context) (ConnectionView, error)
|
||||
func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionView, error) {
|
||||
m.operation.Lock()
|
||||
defer m.operation.Unlock()
|
||||
if err := m.requireOperationalBinding(); err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if err != nil {
|
||||
return ConnectionView{}, err
|
||||
@@ -381,14 +511,14 @@ func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionVie
|
||||
if err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
if err := m.secrets.Put(ctx, nextReference, next); err != nil {
|
||||
if err := m.stageIdentitySecret(ctx, nextReference, next); err != nil {
|
||||
clear(next)
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
connection, err = m.repository.SetSecurityEventNextCredential(ctx, connection.ConnectionID, nextReference)
|
||||
connection, err = m.repository.SetSecurityEventNextCredential(ctx, connection.ConnectionID, nextReference, connection.Version)
|
||||
if err != nil {
|
||||
clear(next)
|
||||
_ = m.secrets.Delete(ctx, nextReference)
|
||||
// Never recreate cleanup intent after an ambiguous adoption commit.
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
} else {
|
||||
@@ -424,10 +554,56 @@ func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionVie
|
||||
func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, error) {
|
||||
m.operation.Lock()
|
||||
defer m.operation.Unlock()
|
||||
if err := m.requireOperationalBinding(); err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
return m.disconnect(ctx, connection)
|
||||
}
|
||||
|
||||
// RetireConflictingConnection is a pairing-scoped recovery operation. It may
|
||||
// retire only a connection owned by another workflow; a stale request becomes
|
||||
// a no-op once the current pairing owns the singleton connection.
|
||||
func (m *ConnectionManager) RetireConflictingConnection(ctx context.Context, currentOwnerKey string) error {
|
||||
m.operation.Lock()
|
||||
defer m.operation.Unlock()
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(currentOwnerKey) == "" {
|
||||
return errors.New("current security event owner is required")
|
||||
}
|
||||
if connection.IdempotencyKey == currentOwnerKey {
|
||||
return nil
|
||||
}
|
||||
if connection.LifecycleStatus == "retiring" {
|
||||
go m.finishRetirement(connection)
|
||||
return nil
|
||||
}
|
||||
if m.retirement == nil || m.retirement.connectionID != connection.ConnectionID ||
|
||||
m.retirement.credentialRef != valueOrEmpty(connection.ManagementCredentialRef) ||
|
||||
m.retirement.ownerKey != currentOwnerKey {
|
||||
return safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
if connection.LifecycleStatus == "disconnect_pending" {
|
||||
go m.retryDisconnect(connection.ConnectionID)
|
||||
return nil
|
||||
}
|
||||
_, err = m.disconnect(ctx, connection)
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) disconnect(ctx context.Context, connection store.SecurityEventConnection) (ConnectionView, error) {
|
||||
if connection.LifecycleStatus == "retiring" || connection.LifecycleStatus == "disconnect_pending" {
|
||||
return m.view(connection), nil
|
||||
}
|
||||
if connection.StreamID != nil {
|
||||
configuration, client, discoverErr := m.discover(ctx, connection.TransmitterIssuer)
|
||||
if discoverErr == nil {
|
||||
@@ -441,12 +617,20 @@ func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, err
|
||||
}
|
||||
if discoverErr != nil {
|
||||
category := "remote_disconnect_failed"
|
||||
connection, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "disconnect_pending", &category)
|
||||
transitioned, transitionErr := m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "disconnect_pending", &category,
|
||||
)
|
||||
if transitionErr != nil {
|
||||
return ConnectionView{}, transitionErr
|
||||
}
|
||||
connection = transitioned
|
||||
go m.retryDisconnect(connection.ConnectionID)
|
||||
return m.view(connection), nil
|
||||
}
|
||||
}
|
||||
connection, err = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "retiring", nil)
|
||||
connection, err := m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "retiring", nil,
|
||||
)
|
||||
if err != nil {
|
||||
return ConnectionView{}, err
|
||||
}
|
||||
@@ -454,6 +638,51 @@ func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, err
|
||||
return m.view(connection), nil
|
||||
}
|
||||
|
||||
// DiscardPreparedConnection removes only the connection created by one identity
|
||||
// pairing. Bound streams continue through the normal retirement window so
|
||||
// revocation watermarks and introspection fallback are never bypassed.
|
||||
func (m *ConnectionManager) DiscardPreparedConnection(ctx context.Context, ownerKey string) error {
|
||||
m.operation.Lock()
|
||||
defer m.operation.Unlock()
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return safeConnectionError{category: "connection_cleanup_failed", cause: err}
|
||||
}
|
||||
if strings.TrimSpace(ownerKey) == "" || connection.IdempotencyKey != ownerKey {
|
||||
return nil
|
||||
}
|
||||
if connection.StreamID != nil {
|
||||
if connection.LifecycleStatus != "retiring" && connection.LifecycleStatus != "disconnect_pending" {
|
||||
if _, err := m.disconnect(ctx, connection); err != nil {
|
||||
return safeConnectionError{category: "connection_cleanup_failed", cause: err}
|
||||
}
|
||||
}
|
||||
return safeConnectionError{category: "retirement_pending", cause: errors.New("security event retirement is pending")}
|
||||
}
|
||||
err = m.repository.DiscardPreparedSecurityEventConnection(
|
||||
ctx, connection.ConnectionID, ownerKey, connection.Version,
|
||||
)
|
||||
if err != nil && !errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
return safeConnectionError{category: "connection_cleanup_failed", cause: err}
|
||||
}
|
||||
m.stopRuntime()
|
||||
return nil
|
||||
}
|
||||
|
||||
func connectionSecretReferences(connection store.SecurityEventConnection) []string {
|
||||
references := []string{connection.CredentialRef}
|
||||
if connection.ManagementCredentialRef != nil {
|
||||
references = append(references, *connection.ManagementCredentialRef)
|
||||
}
|
||||
if connection.NextCredentialRef != nil {
|
||||
references = append(references, *connection.NextCredentialRef)
|
||||
}
|
||||
return references
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) retryDisconnect(connectionID string) {
|
||||
delay := time.Second
|
||||
for {
|
||||
@@ -478,7 +707,9 @@ func (m *ConnectionManager) retryDisconnect(connectionID string) {
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
connection, err = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "retiring", nil)
|
||||
connection, err = m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
m.ctx, connectionID, "disconnect_pending", connection.Version, "retiring", nil,
|
||||
)
|
||||
if err == nil {
|
||||
go m.finishRetirement(connection)
|
||||
}
|
||||
@@ -625,49 +856,57 @@ func (m *ConnectionManager) finishAutomaticVerification(connectionID string, con
|
||||
defer cancel()
|
||||
ticker := time.NewTicker(250 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
expectedLifecycle := "verifying"
|
||||
if rotating {
|
||||
expectedLifecycle = "rotating"
|
||||
}
|
||||
for {
|
||||
connection, err := m.repository.SecurityEventConnection(ctx)
|
||||
if err != nil || connection.ConnectionID != connectionID || connection.StreamID == nil {
|
||||
if err != nil || connection.ConnectionID != connectionID || connection.StreamID == nil || connection.LifecycleStatus != expectedLifecycle {
|
||||
return
|
||||
}
|
||||
_, verifiedAt, _ := m.repository.SecurityEventMetrics(ctx, connection.TransmitterIssuer, valueOrEmpty(connection.Audience))
|
||||
if verifiedAt != nil && !verifiedAt.Before(started) {
|
||||
if err := m.setStreamStatus(ctx, client, configuration, token, *connection.StreamID, "enabled", "verification_succeeded"); err != nil {
|
||||
category := "stream_enable_failed"
|
||||
lifecycle := "verifying"
|
||||
if rotating {
|
||||
lifecycle = "rotating"
|
||||
}
|
||||
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connectionID, lifecycle, &category)
|
||||
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
ctx, connectionID, expectedLifecycle, connection.Version, expectedLifecycle, &category,
|
||||
)
|
||||
return
|
||||
}
|
||||
if rotating && connection.NextCredentialRef != nil {
|
||||
oldReference, nextReference := connection.CredentialRef, *connection.NextCredentialRef
|
||||
nextReference := *connection.NextCredentialRef
|
||||
select {
|
||||
case <-m.ctx.Done():
|
||||
return
|
||||
case <-time.After(m.config.CredentialOverlapDuration):
|
||||
}
|
||||
current, currentErr := m.repository.SecurityEventConnection(m.ctx)
|
||||
if currentErr != nil || current.ConnectionID != connectionID || current.LifecycleStatus != "rotating" ||
|
||||
current.NextCredentialRef == nil || *current.NextCredentialRef != nextReference {
|
||||
return
|
||||
}
|
||||
connection, err = m.repository.PromoteSecurityEventCredential(m.ctx, connectionID, nextReference)
|
||||
if err == nil {
|
||||
_ = m.secrets.Delete(m.ctx, oldReference)
|
||||
_ = m.activate(m.ctx, connection)
|
||||
go m.finishBootstrap(connectionID)
|
||||
}
|
||||
return
|
||||
}
|
||||
connection, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connectionID, "bootstrap", nil)
|
||||
go m.finishBootstrap(connectionID)
|
||||
connection, err = m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
ctx, connectionID, expectedLifecycle, connection.Version, "bootstrap", nil,
|
||||
)
|
||||
if err == nil {
|
||||
go m.finishBootstrap(connectionID)
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
category := "verification_timeout"
|
||||
lifecycle := "verifying"
|
||||
if rotating {
|
||||
lifecycle = "rotating"
|
||||
}
|
||||
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, lifecycle, &category)
|
||||
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
m.ctx, connectionID, expectedLifecycle, connection.Version, expectedLifecycle, &category,
|
||||
)
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
@@ -691,46 +930,89 @@ func (m *ConnectionManager) finishBootstrap(connectionID string) {
|
||||
if err == nil && connection.ConnectionID == connectionID && connection.LifecycleStatus == "bootstrap" {
|
||||
mode, _, metricsErr := m.repository.SecurityEventMetrics(m.ctx, connection.TransmitterIssuer, valueOrEmpty(connection.Audience))
|
||||
if metricsErr == nil && mode == "push_healthy" {
|
||||
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "enabled", nil)
|
||||
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
m.ctx, connectionID, "bootstrap", connection.Version, "enabled", nil,
|
||||
)
|
||||
return
|
||||
}
|
||||
category := "bootstrap_verification_stale"
|
||||
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "degraded", &category)
|
||||
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
m.ctx, connectionID, "bootstrap", connection.Version, "degraded", &category,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) finishRetirement(connection store.SecurityEventConnection) {
|
||||
elapsed := time.Since(connection.UpdatedAt)
|
||||
delay := m.config.RetirementDuration - elapsed
|
||||
if delay > 0 {
|
||||
select {
|
||||
case <-m.ctx.Done():
|
||||
return
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
delay = time.Second
|
||||
func (m *ConnectionManager) finishRetirement(retirement store.SecurityEventConnection) {
|
||||
retryDelay := time.Second
|
||||
for {
|
||||
if err := m.repository.DeleteSecurityEventConnection(m.ctx, connection.ConnectionID); err == nil || errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-m.ctx.Done():
|
||||
connection, err := m.repository.SecurityEventConnection(m.ctx)
|
||||
if errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
return
|
||||
case <-time.After(delay):
|
||||
}
|
||||
if delay < time.Minute {
|
||||
delay *= 2
|
||||
if err != nil {
|
||||
if !waitForConnectionRetry(m.ctx, retryDelay) {
|
||||
return
|
||||
}
|
||||
retryDelay = nextConnectionRetryDelay(retryDelay)
|
||||
continue
|
||||
}
|
||||
if connection.ConnectionID != retirement.ConnectionID || connection.LifecycleStatus != "retiring" {
|
||||
return
|
||||
}
|
||||
if remaining := m.config.RetirementDuration - time.Since(connection.UpdatedAt); remaining > 0 {
|
||||
select {
|
||||
case <-m.ctx.Done():
|
||||
return
|
||||
case <-time.After(remaining):
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.stopRuntime()
|
||||
err = m.repository.FinalizeRetiringSecurityEventConnection(m.ctx, connection.ConnectionID, connection.Version)
|
||||
if err == nil || errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
return
|
||||
}
|
||||
if !waitForConnectionRetry(m.ctx, retryDelay) {
|
||||
return
|
||||
}
|
||||
retryDelay = nextConnectionRetryDelay(retryDelay)
|
||||
}
|
||||
m.stopRuntime()
|
||||
_ = m.secrets.Delete(m.ctx, connection.CredentialRef)
|
||||
if connection.ManagementCredentialRef != nil {
|
||||
_ = m.secrets.Delete(m.ctx, *connection.ManagementCredentialRef)
|
||||
}
|
||||
|
||||
func waitForConnectionRetry(ctx context.Context, delay time.Duration) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(delay):
|
||||
return true
|
||||
}
|
||||
if connection.NextCredentialRef != nil {
|
||||
_ = m.secrets.Delete(m.ctx, *connection.NextCredentialRef)
|
||||
}
|
||||
|
||||
func nextConnectionRetryDelay(delay time.Duration) time.Duration {
|
||||
if delay < time.Minute {
|
||||
return delay * 2
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
const identitySecretStagingTTL = 10 * time.Minute
|
||||
|
||||
func (m *ConnectionManager) stageIdentitySecret(ctx context.Context, reference string, value []byte) error {
|
||||
if err := m.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC().Add(identitySecretStagingTTL)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.secrets.Put(ctx, reference, value); err != nil {
|
||||
deleteErr := m.secrets.Delete(ctx, reference)
|
||||
if deleteErr == nil || errors.Is(deleteErr, ErrSecretNotFound) {
|
||||
_, _ = m.repository.RemovePendingIdentitySecretCleanup(ctx, reference)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) retireIdentitySecret(ctx context.Context, reference string) error {
|
||||
return m.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC())
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) validatePrerequisites(issuer, managementClientID string, managementSecret []byte, requirePublicBaseURL bool) error {
|
||||
@@ -794,6 +1076,10 @@ func (m *ConnectionManager) managementToken(ctx context.Context, client *http.Cl
|
||||
return "", err
|
||||
}
|
||||
defer clear(managementSecret)
|
||||
return m.requestManagementToken(ctx, client, scopes, managementClientID, managementSecret)
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) requestManagementToken(ctx context.Context, client *http.Client, scopes, managementClientID string, managementSecret []byte) (string, error) {
|
||||
form := url.Values{"grant_type": {"client_credentials"}, "scope": {scopes}}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(m.config.OIDCIssuer, "/")+"/token", strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
@@ -820,6 +1106,44 @@ func (m *ConnectionManager) managementToken(ctx context.Context, client *http.Cl
|
||||
return payload.AccessToken, nil
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) proveManagementCredentialHandoff(ctx context.Context, connection store.SecurityEventConnection, managementClientID string, managementSecret []byte) error {
|
||||
if connection.StreamID == nil || connection.Audience == nil {
|
||||
return store.ErrSecurityEventConnectionConflict
|
||||
}
|
||||
configuration, client, err := m.discover(ctx, connection.TransmitterIssuer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token, err := m.requestManagementToken(ctx, client, "ssf.stream.read ssf.stream.manage", managementClientID, managementSecret)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var streams []streamConfiguration
|
||||
if err := requestJSON(ctx, client, http.MethodGet, configuration.ConfigurationEndpoint, token, nil, &streams); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, stream := range streams {
|
||||
if stream.StreamID != *connection.StreamID {
|
||||
continue
|
||||
}
|
||||
if err := validateCreatedStream(stream, connection.TransmitterIssuer, connection.EndpointURL); err != nil || stream.Audience != *connection.Audience {
|
||||
return store.ErrSecurityEventConnectionConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// An authenticated empty list proves that the new credential belongs to a
|
||||
// token issuer trusted by the old transmitter. Disconnect treats the absent
|
||||
// Stream as an idempotent remote deletion.
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) requireOperationalBinding() error {
|
||||
if m.bindingMismatch {
|
||||
return safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) managementCredential(ctx context.Context, connection store.SecurityEventConnection) (string, []byte, error) {
|
||||
if connection.ManagementCredentialRef != nil && connection.ManagementClientID != "" {
|
||||
secret, err := m.secrets.Get(ctx, *connection.ManagementCredentialRef)
|
||||
@@ -882,8 +1206,10 @@ func (m *ConnectionManager) deleteStream(ctx context.Context, client *http.Clien
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) connectionError(ctx context.Context, connection store.SecurityEventConnection, category string, cause error) error {
|
||||
_, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "error", &category)
|
||||
return cause
|
||||
_, _ = m.repository.TransitionSecurityEventConnectionLifecycle(
|
||||
ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "error", &category,
|
||||
)
|
||||
return safeConnectionError{category: category, cause: cause}
|
||||
}
|
||||
|
||||
func (m *ConnectionManager) view(connection store.SecurityEventConnection) ConnectionView {
|
||||
@@ -1054,7 +1380,12 @@ var blockedNetworkPrefixes = []netip.Prefix{
|
||||
}
|
||||
|
||||
func isLocalHostname(value string) bool {
|
||||
return value == "localhost" || value == "127.0.0.1" || value == "::1"
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "localhost" {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(value)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
|
||||
func isLocalEnvironment(value string) bool {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,52 @@
|
||||
package securityevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
identitySecretCleanupInterval = time.Minute
|
||||
identitySecretCleanupLease = 5 * time.Minute
|
||||
identitySecretCleanupBatch = 100
|
||||
)
|
||||
|
||||
type IdentitySecretCleanupRepository interface {
|
||||
ClaimIdentitySecretCleanups(context.Context, int, time.Duration) ([]store.IdentitySecretCleanupClaim, error)
|
||||
CompleteIdentitySecretCleanup(context.Context, string, string) (bool, error)
|
||||
}
|
||||
|
||||
// RunIdentitySecretCleanupWorker owns cleanup at the server lifecycle rather
|
||||
// than at any individual Active or Prepared identity Runtime. PostgreSQL claim
|
||||
// tokens make running one worker per server replica safe.
|
||||
func RunIdentitySecretCleanupWorker(ctx context.Context, repository IdentitySecretCleanupRepository, secrets SecretStore) {
|
||||
if repository == nil || secrets == nil {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(identitySecretCleanupInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
cleanupClaimedIdentitySecrets(ctx, repository, secrets)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupClaimedIdentitySecrets(ctx context.Context, repository IdentitySecretCleanupRepository, secrets SecretStore) {
|
||||
claims, err := repository.ClaimIdentitySecretCleanups(ctx, identitySecretCleanupBatch, identitySecretCleanupLease)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, claim := range claims {
|
||||
if err := secrets.Delete(ctx, claim.Reference); err != nil && !errors.Is(err, ErrSecretNotFound) {
|
||||
continue
|
||||
}
|
||||
_, _ = repository.CompleteIdentitySecretCleanup(ctx, claim.Reference, claim.ClaimToken)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package securityevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultSecurityEventRetirementDuration = 6 * time.Minute
|
||||
securityEventRetirementWorkerInterval = time.Minute
|
||||
)
|
||||
|
||||
type SecurityEventRetirementRepository interface {
|
||||
SecurityEventConnection(context.Context) (store.SecurityEventConnection, error)
|
||||
FinalizeRetiringSecurityEventConnection(context.Context, string, int64) error
|
||||
}
|
||||
|
||||
// RunSecurityEventRetirementWorker owns the local end of SSF retirement at the
|
||||
// server lifecycle. It intentionally does not share an Active identity
|
||||
// Runtime's context, because that Runtime is closed before the RFC 8935 overlap
|
||||
// period expires. Store-level generation CAS makes one worker per replica safe.
|
||||
func RunSecurityEventRetirementWorker(ctx context.Context, repository SecurityEventRetirementRepository) {
|
||||
runSecurityEventRetirementWorker(
|
||||
ctx,
|
||||
repository,
|
||||
defaultSecurityEventRetirementDuration,
|
||||
securityEventRetirementWorkerInterval,
|
||||
)
|
||||
}
|
||||
|
||||
func runSecurityEventRetirementWorker(
|
||||
ctx context.Context,
|
||||
repository SecurityEventRetirementRepository,
|
||||
retirementDuration time.Duration,
|
||||
interval time.Duration,
|
||||
) {
|
||||
if repository == nil {
|
||||
return
|
||||
}
|
||||
if retirementDuration <= 0 {
|
||||
retirementDuration = defaultSecurityEventRetirementDuration
|
||||
}
|
||||
if interval <= 0 {
|
||||
interval = securityEventRetirementWorkerInterval
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
finalizeDueSecurityEventRetirement(ctx, repository, retirementDuration)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func finalizeDueSecurityEventRetirement(
|
||||
ctx context.Context,
|
||||
repository SecurityEventRetirementRepository,
|
||||
retirementDuration time.Duration,
|
||||
) {
|
||||
connection, err := repository.SecurityEventConnection(ctx)
|
||||
if err != nil || connection.LifecycleStatus != "retiring" {
|
||||
return
|
||||
}
|
||||
if retirementDuration <= 0 {
|
||||
retirementDuration = defaultSecurityEventRetirementDuration
|
||||
}
|
||||
if time.Since(connection.UpdatedAt) < retirementDuration {
|
||||
return
|
||||
}
|
||||
// Finalization atomically queues every Secret reference and removes only the
|
||||
// exact retiring generation. A concurrent worker or changed generation is a
|
||||
// benign CAS miss and will be observed on the next loop.
|
||||
_ = repository.FinalizeRetiringSecurityEventConnection(ctx, connection.ConnectionID, connection.Version)
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package securityevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestSecurityEventRetirementWorkerOutlivesCanceledRuntime(t *testing.T) {
|
||||
connection := retiringWorkerTestConnection(time.Now().UTC())
|
||||
repository := &memoryConnectionRepository{connection: &connection}
|
||||
secrets := &memorySecretStore{values: retirementWorkerTestSecrets(connection)}
|
||||
|
||||
runtimeCtx, cancelRuntime := context.WithCancel(context.Background())
|
||||
manager := &ConnectionManager{
|
||||
ctx: runtimeCtx, repository: repository, secrets: secrets,
|
||||
config: ConnectionManagerConfig{RetirementDuration: 40 * time.Millisecond},
|
||||
}
|
||||
go manager.finishRetirement(connection)
|
||||
cancelRuntime()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if _, err := repository.SecurityEventConnection(context.Background()); err != nil {
|
||||
t.Fatalf("runtime cancellation unexpectedly finalized retirement: %v", err)
|
||||
}
|
||||
|
||||
serverCtx, cancelServer := context.WithCancel(context.Background())
|
||||
defer cancelServer()
|
||||
go runSecurityEventRetirementWorker(serverCtx, repository, 40*time.Millisecond, 5*time.Millisecond)
|
||||
waitForRetirementFinalization(t, repository)
|
||||
|
||||
for reference := range retirementWorkerTestSecrets(connection) {
|
||||
repository.mutex.Lock()
|
||||
_, queued := repository.cleanupDue[reference]
|
||||
repository.mutex.Unlock()
|
||||
if !queued {
|
||||
t.Fatalf("retired Secret %q was not durably queued", reference)
|
||||
}
|
||||
if !secrets.Has(reference) {
|
||||
t.Fatalf("retirement worker directly deleted Secret %q before the durable cleanup worker claimed it", reference)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityEventRetirementWorkerResumesAfterServerRestart(t *testing.T) {
|
||||
connection := retiringWorkerTestConnection(time.Now().UTC())
|
||||
repository := &memoryConnectionRepository{connection: &connection}
|
||||
|
||||
firstCtx, stopFirst := context.WithCancel(context.Background())
|
||||
go runSecurityEventRetirementWorker(firstCtx, repository, 80*time.Millisecond, 5*time.Millisecond)
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
stopFirst()
|
||||
if _, err := repository.SecurityEventConnection(context.Background()); err != nil {
|
||||
t.Fatalf("not-yet-due retirement disappeared before restart: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(75 * time.Millisecond)
|
||||
secondCtx, stopSecond := context.WithCancel(context.Background())
|
||||
defer stopSecond()
|
||||
go runSecurityEventRetirementWorker(secondCtx, repository, 80*time.Millisecond, 5*time.Millisecond)
|
||||
waitForRetirementFinalization(t, repository)
|
||||
}
|
||||
|
||||
func TestConcurrentSecurityEventRetirementWorkersFinalizeOneGenerationOnce(t *testing.T) {
|
||||
connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute))
|
||||
repository := &memoryConnectionRepository{connection: &connection}
|
||||
|
||||
start := make(chan struct{})
|
||||
done := make(chan struct{}, 2)
|
||||
for range 2 {
|
||||
go func() {
|
||||
<-start
|
||||
finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond)
|
||||
done <- struct{}{}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
<-done
|
||||
<-done
|
||||
|
||||
if _, err := repository.SecurityEventConnection(context.Background()); !errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
t.Fatalf("current retiring generation still exists: %v", err)
|
||||
}
|
||||
repository.mutex.Lock()
|
||||
finalizeCalls := repository.finalizeCalls
|
||||
queued := len(repository.cleanupDue)
|
||||
repository.mutex.Unlock()
|
||||
if finalizeCalls != 1 {
|
||||
t.Fatalf("retirement generation finalized %d times, want 1", finalizeCalls)
|
||||
}
|
||||
if queued != len(retirementWorkerTestSecrets(connection)) {
|
||||
t.Fatalf("queued Secret references=%d want=%d", queued, len(retirementWorkerTestSecrets(connection)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityEventRetirementWorkerPreservesChangedGeneration(t *testing.T) {
|
||||
connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute))
|
||||
initialVersion := connection.Version
|
||||
repository := &memoryConnectionRepository{connection: &connection}
|
||||
repository.beforeFinalize = func(current *store.SecurityEventConnection) {
|
||||
current.LifecycleStatus = "enabled"
|
||||
current.Version++
|
||||
}
|
||||
|
||||
finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond)
|
||||
|
||||
current, err := repository.SecurityEventConnection(context.Background())
|
||||
if err != nil || current.LifecycleStatus != "enabled" || current.Version != initialVersion+1 {
|
||||
t.Fatalf("changed generation was not preserved: lifecycle=%q version=%d err=%v", current.LifecycleStatus, current.Version, err)
|
||||
}
|
||||
repository.mutex.Lock()
|
||||
queued := len(repository.cleanupDue)
|
||||
repository.mutex.Unlock()
|
||||
if queued != 0 {
|
||||
t.Fatalf("changed generation queued %d active Secret references", queued)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityEventRetirementWorkerKeepsConnectionWhenCleanupQueueCannotAdoptReferences(t *testing.T) {
|
||||
connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute))
|
||||
repository := &memoryConnectionRepository{
|
||||
connection: &connection,
|
||||
cleanupClaims: map[string]string{connection.CredentialRef: uuid.NewString()},
|
||||
}
|
||||
|
||||
finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond)
|
||||
|
||||
current, err := repository.SecurityEventConnection(context.Background())
|
||||
if err != nil || current.ConnectionID != connection.ConnectionID || current.LifecycleStatus != "retiring" {
|
||||
t.Fatalf("failed durable handoff lost retiring connection: lifecycle=%q err=%v", current.LifecycleStatus, err)
|
||||
}
|
||||
repository.mutex.Lock()
|
||||
finalizeCalls := repository.finalizeCalls
|
||||
repository.mutex.Unlock()
|
||||
if finalizeCalls != 0 {
|
||||
t.Fatalf("failed durable handoff finalized generation %d times", finalizeCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func retiringWorkerTestConnection(updatedAt time.Time) store.SecurityEventConnection {
|
||||
managementReference := "ssf-management-retirement-worker-" + uuid.NewString()
|
||||
nextReference := "ssf-push-next-retirement-worker-" + uuid.NewString()
|
||||
return store.SecurityEventConnection{
|
||||
ConnectionID: uuid.NewString(),
|
||||
CredentialRef: "ssf-push-retirement-worker-" + uuid.NewString(),
|
||||
NextCredentialRef: &nextReference,
|
||||
ManagementCredentialRef: &managementReference,
|
||||
LifecycleStatus: "retiring",
|
||||
Version: 9,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func retirementWorkerTestSecrets(connection store.SecurityEventConnection) map[string][]byte {
|
||||
secrets := map[string][]byte{connection.CredentialRef: []byte("retired-push-secret-value")}
|
||||
if connection.NextCredentialRef != nil {
|
||||
secrets[*connection.NextCredentialRef] = []byte("retired-next-push-secret-value")
|
||||
}
|
||||
if connection.ManagementCredentialRef != nil {
|
||||
secrets[*connection.ManagementCredentialRef] = []byte("retired-management-secret-value")
|
||||
}
|
||||
return secrets
|
||||
}
|
||||
|
||||
func waitForRetirementFinalization(t *testing.T, repository *memoryConnectionRepository) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := repository.SecurityEventConnection(context.Background()); errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("retiring connection was not finalized by the server-lifecycle worker")
|
||||
}
|
||||
Reference in New Issue
Block a user