package securityevents import ( "context" "encoding/json" "errors" "fmt" "net" "net/http" "net/http/httptest" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" "github.com/google/uuid" ) type memoryConnectionRepository struct { mutex sync.Mutex connection *store.SecurityEventConnection beforeDiscard func(*store.SecurityEventConnection) beforeFinalize func(*store.SecurityEventConnection) finalizeCalls int cleanupDue map[string]time.Time cleanupClaims map[string]string cleanupClaimed chan struct{} mode string verifiedAt *time.Time evaluation store.SecurityEventEvaluation } func (r *memoryConnectionRepository) QueueIdentitySecretCleanup(_ context.Context, reference string, notBefore time.Time) error { r.mutex.Lock() defer r.mutex.Unlock() if r.cleanupDue == nil { r.cleanupDue = map[string]time.Time{} } if r.cleanupClaims[reference] != "" { return store.ErrIdentitySecretCleanupConflict } current, exists := r.cleanupDue[reference] if !exists || notBefore.Before(current) { r.cleanupDue[reference] = notBefore } return nil } func (r *memoryConnectionRepository) RemovePendingIdentitySecretCleanup(_ context.Context, reference string) (bool, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.cleanupClaims[reference] != "" { return false, nil } _, exists := r.cleanupDue[reference] delete(r.cleanupDue, reference) return exists, nil } func (r *memoryConnectionRepository) ClaimIdentitySecretCleanups(_ context.Context, limit int, lease time.Duration) ([]store.IdentitySecretCleanupClaim, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.cleanupClaims == nil { r.cleanupClaims = map[string]string{} } if r.cleanupClaimed != nil { select { case r.cleanupClaimed <- struct{}{}: default: } } now := time.Now().UTC() claims := make([]store.IdentitySecretCleanupClaim, 0) for reference, notBefore := range r.cleanupDue { if !notBefore.After(now) && r.cleanupClaims[reference] == "" { token := uuid.NewString() r.cleanupClaims[reference] = token claims = append(claims, store.IdentitySecretCleanupClaim{ Reference: reference, ClaimToken: token, LeaseExpiresAt: now.Add(lease), }) if len(claims) == limit { break } } } return claims, nil } func (r *memoryConnectionRepository) CompleteIdentitySecretCleanup(_ context.Context, reference, claimToken string) (bool, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.cleanupClaims[reference] != claimToken { return false, nil } delete(r.cleanupClaims, reference) delete(r.cleanupDue, reference) return true, nil } func (*memoryConnectionRepository) ApplySessionRevoked(context.Context, store.ApplySessionRevokedInput) (store.ApplySecurityEventResult, error) { return store.ApplySecurityEventResult{}, nil } func (*memoryConnectionRepository) ApplySecurityEventStreamUpdated(context.Context, string, string, string, string, string, string, time.Time) (bool, error) { return true, nil } func (*memoryConnectionRepository) ConfirmSecurityEventVerification(context.Context, string, string, string, string, []byte, time.Time) (bool, error) { return true, nil } func (*memoryConnectionRepository) RecordSecurityEventReceipt(context.Context, string, string, string, string, string) (bool, error) { return true, nil } func (r *memoryConnectionRepository) EnsureSecurityEventStreamState(context.Context, string, string, string) error { r.mode = "bootstrap" return nil } func (*memoryConnectionRepository) BeginSecurityEventVerification(context.Context, string, string, []byte, time.Time) error { return nil } func (*memoryConnectionRepository) RecordSecurityEventHeartbeatFailure(context.Context, string, string, string) error { return nil } func (*memoryConnectionRepository) AdvanceSecurityEventStreamState(context.Context, string, string, time.Time, time.Duration) error { return nil } func (r *memoryConnectionRepository) EvaluateOIDCSecurityEvent(context.Context, string, string, string, string, string, time.Time, time.Time, time.Duration) (store.SecurityEventEvaluation, error) { if r.evaluation.Mode == "" { return store.SecurityEventEvaluation{Mode: "bootstrap", RequireIntrospection: true}, nil } return r.evaluation, nil } func (r *memoryConnectionRepository) CreateSecurityEventConnection(_ context.Context, input store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.connection != nil { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict } now := time.Now().UTC() value := store.SecurityEventConnection{ConnectionID: input.ConnectionID, TransmitterIssuer: input.TransmitterIssuer, EndpointURL: input.EndpointURL, CredentialRef: input.CredentialRef, ManagementClientID: input.ManagementClientID, ManagementCredentialRef: &input.ManagementCredentialRef, LifecycleStatus: "connecting", Version: 1, IdempotencyKey: input.IdempotencyKey, CreatedAt: now, UpdatedAt: now, HealthMode: "disabled"} r.connection = &value delete(r.cleanupDue, input.CredentialRef) delete(r.cleanupDue, input.ManagementCredentialRef) return value, nil } func (r *memoryConnectionRepository) SetSecurityEventManagementCredential(_ context.Context, id, clientID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil || r.connection.ConnectionID != id { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound } if r.connection.Version != expectedVersion || r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending" { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict } oldReference := r.connection.ManagementCredentialRef r.connection.ManagementClientID, r.connection.ManagementCredentialRef = clientID, &reference r.connection.Version++ delete(r.cleanupDue, reference) if oldReference != nil && *oldReference != reference { if r.cleanupDue == nil { r.cleanupDue = map[string]time.Time{} } r.cleanupDue[*oldReference] = time.Now().UTC() } return *r.connection, nil } func (r *memoryConnectionRepository) SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound } value := *r.connection value.HealthMode, value.LastVerificationAt = r.mode, r.verifiedAt return value, nil } func (r *memoryConnectionRepository) BindSecurityEventConnection(_ context.Context, id, audience, streamID, lifecycle string, expected int64) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil || r.connection.ConnectionID != id || r.connection.Version != expected { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict } r.connection.Audience, r.connection.StreamID = &audience, &streamID r.connection.LifecycleStatus, r.connection.Version = lifecycle, r.connection.Version+1 return *r.connection, nil } func (r *memoryConnectionRepository) TransitionSecurityEventConnectionLifecycle(_ context.Context, id, expectedLifecycle string, expectedVersion int64, lifecycle string, category *string) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil || r.connection.ConnectionID != id || r.connection.LifecycleStatus != expectedLifecycle || r.connection.Version != expectedVersion { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict } if (r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending") && lifecycle != "retiring" && lifecycle != "disconnect_pending" { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict } r.connection.LifecycleStatus, r.connection.LastErrorCategory = lifecycle, category r.connection.Version++ r.connection.UpdatedAt = time.Now().UTC() return *r.connection, nil } func (r *memoryConnectionRepository) SetSecurityEventNextCredential(_ context.Context, id, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil || r.connection.ConnectionID != id { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound } if r.connection.Version != expectedVersion || r.connection.NextCredentialRef != nil || r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending" { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict } r.connection.NextCredentialRef = &reference r.connection.LifecycleStatus = "rotating" r.connection.Version++ delete(r.cleanupDue, reference) return *r.connection, nil } func TestConnectionManagerBootstrapOnlyBecomesEnabledWhenPushIsHealthy(t *testing.T) { for _, test := range []struct { mode, wantLifecycle string }{ {mode: "push_healthy", wantLifecycle: "enabled"}, {mode: "introspection_fallback", wantLifecycle: "degraded"}, } { t.Run(test.mode, func(t *testing.T) { audience := "urn:easyai:ssf:receiver:" + uuid.NewString() repository := &memoryConnectionRepository{mode: test.mode, connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", Audience: &audience, LifecycleStatus: "bootstrap", Version: 1, UpdatedAt: time.Now().UTC(), }} manager := &ConnectionManager{ctx: context.Background(), repository: repository, config: ConnectionManagerConfig{BootstrapDuration: time.Millisecond}} manager.finishBootstrap(repository.connection.ConnectionID) connection, err := repository.SecurityEventConnection(context.Background()) if err != nil || connection.LifecycleStatus != test.wantLifecycle { t.Fatalf("connection=%#v err=%v", connection, err) } }) } } func TestConnectionManagerResumesBootstrapAfterRestart(t *testing.T) { repository := &memoryConnectionRepository{mode: "push_healthy", connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", LifecycleStatus: "bootstrap", Version: 1, UpdatedAt: time.Now().Add(-time.Second), }} ctx, cancel := context.WithCancel(context.Background()) defer cancel() if _, err := NewConnectionManager(ctx, repository, &memorySecretStore{}, ConnectionManagerConfig{ OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), BootstrapDuration: 10 * time.Millisecond, }, &Metrics{}); err != nil { t.Fatal(err) } deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) { connection, err := repository.SecurityEventConnection(ctx) if err == nil && connection.LifecycleStatus == "enabled" { return } time.Sleep(10 * time.Millisecond) } t.Fatal("bootstrap lifecycle did not resume after process restart") } func TestNewConnectionManagerStrictRevisionBindingRejectsDifferentPersistedConnection(t *testing.T) { audience := "urn:easyai:ssf:receiver:target" streamID := uuid.NewString() base := store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", Audience: &audience, StreamID: &streamID, CredentialRef: "missing-credential-must-not-be-read", ManagementClientID: "gateway-machine", LifecycleStatus: "enabled", IdempotencyKey: "identity-pairing-ssf-target-revision", Version: 1, } config := ConnectionManagerConfig{ OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), ManagementClientID: "gateway-machine", ExpectedTransmitterIssuer: "https://auth.example/ssf", ExpectedAudience: audience, ExpectedOwnerKey: "identity-pairing-ssf-target-revision", StrictRevisionBinding: true, } tests := []struct { name string mutate func(*store.SecurityEventConnection) }{ {name: "transmitter issuer", mutate: func(connection *store.SecurityEventConnection) { connection.TransmitterIssuer = "https://old-auth.example/ssf" }}, {name: "audience", mutate: func(connection *store.SecurityEventConnection) { other := "urn:easyai:ssf:receiver:old" connection.Audience = &other }}, {name: "missing audience", mutate: func(connection *store.SecurityEventConnection) { connection.Audience = nil }}, {name: "management client", mutate: func(connection *store.SecurityEventConnection) { connection.ManagementClientID = "old-machine" }}, {name: "owner", mutate: func(connection *store.SecurityEventConnection) { connection.IdempotencyKey = "identity-pairing-ssf-old-revision" }}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { connection := base test.mutate(&connection) repository := &memoryConnectionRepository{connection: &connection} manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, config, &Metrics{}) if manager != nil { t.Fatal("strict runtime construction returned a manager for a different Revision binding") } var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_mismatch" { t.Fatalf("error=%v category=%v", err, categorized) } }) } } func TestNewConnectionManagerStrictRevisionBindingRequiresPersistedConnection(t *testing.T) { manager, err := NewConnectionManager(context.Background(), &memoryConnectionRepository{}, &memorySecretStore{}, ConnectionManagerConfig{ OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), ManagementClientID: "gateway-machine", ExpectedTransmitterIssuer: "https://auth.example/ssf", ExpectedAudience: "urn:easyai:ssf:receiver:target", ExpectedOwnerKey: "identity-pairing-ssf-target-revision", StrictRevisionBinding: true, }, &Metrics{}) if manager != nil { t.Fatal("strict runtime construction succeeded without a persisted SSF connection") } var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_missing" { t.Fatalf("error=%v category=%v", err, categorized) } } func TestNewConnectionManagerStrictRevisionBindingAcceptsExactPersistedConnection(t *testing.T) { audience := "urn:easyai:ssf:receiver:target" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", Audience: &audience, ManagementClientID: "gateway-machine", LifecycleStatus: "enabled", IdempotencyKey: "identity-pairing-ssf-target-revision", Version: 1, }} manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, ConnectionManagerConfig{ OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), ManagementClientID: "gateway-machine", ExpectedTransmitterIssuer: "https://auth.example/ssf/", ExpectedAudience: audience, ExpectedOwnerKey: "identity-pairing-ssf-target-revision", StrictRevisionBinding: true, }, &Metrics{}) if err != nil || manager == nil { t.Fatalf("exact Revision binding was rejected: manager=%v err=%v", manager != nil, err) } } func TestConnectionManagerRecoveryCanLoadDifferentPersistedConnection(t *testing.T) { oldAudience := "urn:easyai:ssf:receiver:old" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://old-auth.example/ssf", Audience: &oldAudience, ManagementClientID: "old-machine", LifecycleStatus: "error", IdempotencyKey: "identity-pairing-ssf-old-revision", Version: 1, }} config := ConnectionManagerConfig{ OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), ManagementClientID: "gateway-machine", ExpectedTransmitterIssuer: "https://auth.example/ssf", ExpectedAudience: "urn:easyai:ssf:receiver:target", ExpectedOwnerKey: "identity-pairing-ssf-target-revision", } manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, config, &Metrics{}) if err != nil || manager == nil { t.Fatalf("recovery manager could not load the conflicting connection: manager=%v err=%v", manager != nil, err) } err = manager.ValidateConfiguredConnectionBinding(context.Background()) var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_mismatch" { t.Fatalf("explicit runtime binding validation did not reject recovery connection: err=%v category=%v", err, categorized) } } func TestForeignRevisionRecoveryNeverUsesPersistedCredentialOnGenericOperations(t *testing.T) { var requests atomic.Int32 endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1) http.Error(w, "unexpected request", http.StatusInternalServerError) })) defer endpoint.Close() audience, streamID := "urn:easyai:ssf:receiver:old", uuid.NewString() managementReference := "ssf-management-old" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: endpoint.URL + "/old-ssf", EndpointURL: endpoint.URL + "/receiver", Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-old", ManagementClientID: "shared-machine", ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 3, IdempotencyKey: "identity-pairing-ssf-old-revision", }} secrets := &memorySecretStore{values: map[string][]byte{ managementReference: []byte("old-machine-secret-must-never-leave"), "ssf-push-old": []byte("old-push-secret-value-000000000000"), }} manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: endpoint.URL + "/new-oidc", OIDCTenantID: uuid.NewString(), ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: endpoint.URL + "/new-ssf", ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision", PublicBaseURL: endpoint.URL, HTTPClient: endpoint.Client(), }, &Metrics{}) if err != nil { t.Fatal(err) } assertUnsafe := func(name string, err error) { t.Helper() var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "credential_handoff_unsafe" { t.Fatalf("%s error=%v", name, err) } } _, _, credentialErr := manager.IntrospectionCredential(context.Background()) assertUnsafe("introspection credential", credentialErr) _, verifyErr := manager.Verify(context.Background()) assertUnsafe("verify", verifyErr) _, rotateErr := manager.RotateCredential(context.Background()) assertUnsafe("rotate", rotateErr) _, disconnectErr := manager.Disconnect(context.Background()) assertUnsafe("disconnect", disconnectErr) _, connectErr := manager.Connect(context.Background(), endpoint.URL+"/new-ssf", "shared-machine", []byte("new-machine-secret-value-000000000"), "identity-pairing-ssf-new-revision") assertUnsafe("cross-issuer connect", connectErr) assertUnsafe("retire without proved handoff", manager.RetireConflictingConnection(context.Background(), "identity-pairing-ssf-new-revision")) if requests.Load() != 0 { t.Fatalf("foreign recovery sent %d authenticated or discovery requests", requests.Load()) } connection, err := repository.SecurityEventConnection(context.Background()) if err != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference { t.Fatalf("foreign recovery changed persisted credential: connection=%#v err=%v", connection, err) } repository.mutex.Lock() queued := len(repository.cleanupDue) repository.mutex.Unlock() if queued != 0 || !secrets.Has(managementReference) { t.Fatalf("foreign recovery queued=%d old_secret_present=%t", queued, secrets.Has(managementReference)) } } func TestPairingProvesRotatedCredentialBeforeRetiringLegacyStream(t *testing.T) { const oldSecret = "old-machine-secret-value-0000000000" const newSecret = "new-machine-secret-value-0000000000" audience, streamID := "urn:easyai:ssf:receiver:legacy", uuid.NewString() managementReference := "ssf-management-legacy" var oldSecretSeen atomic.Bool var newSecretRequests atomic.Int32 var transmitter *httptest.Server transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/.well-known/ssf-configuration/ssf": _ = json.NewEncoder(w).Encode(map[string]any{ "spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json", "configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status", "verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod}, }) case "/oidc/token": _, secret, _ := r.BasicAuth() if secret == oldSecret { oldSecretSeen.Store(true) } if secret != newSecret { http.Error(w, "invalid client", http.StatusUnauthorized) return } newSecretRequests.Add(1) _ = json.NewEncoder(w).Encode(map[string]any{"access_token": "proved-management-token", "token_type": "Bearer"}) case "/ssf/v1/stream": if r.Header.Get("Authorization") != "Bearer proved-management-token" { http.Error(w, "invalid token", http.StatusUnauthorized) return } if r.Method == http.MethodDelete { w.WriteHeader(http.StatusNoContent) return } description := "easyai-gateway:legacy" _ = json.NewEncoder(w).Encode([]map[string]any{{ "stream_id": streamID, "iss": transmitter.URL + "/ssf", "aud": audience, "events_requested": []string{sessionRevokedEvent}, "delivery": map[string]any{"method": pushDeliveryMethod, "endpoint_url": transmitter.URL + "/receiver"}, "description": description, }}) default: http.NotFound(w, r) } })) defer transmitter.Close() repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: "legacy", TransmitterIssuer: transmitter.URL + "/ssf", EndpointURL: transmitter.URL + "/receiver", Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-legacy", ManagementClientID: "shared-machine", ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 2, IdempotencyKey: "manual-legacy-owner", }} secrets := &memorySecretStore{values: map[string][]byte{ managementReference: []byte(oldSecret), "ssf-push-legacy": []byte("legacy-push-secret-value-000000000"), }} manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: transmitter.URL + "/oidc", OIDCTenantID: uuid.NewString(), ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: transmitter.URL + "/ssf", ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision", PublicBaseURL: transmitter.URL, HTTPClient: transmitter.Client(), RetirementDuration: time.Hour, }, &Metrics{}) if err != nil { t.Fatal(err) } _, err = manager.Connect(context.Background(), transmitter.URL+"/ssf", "shared-machine", []byte(newSecret), "identity-pairing-ssf-new-revision") var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_conflict" { t.Fatalf("proved handoff error=%v", err) } connection, err := repository.SecurityEventConnection(context.Background()) if err != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef == managementReference { t.Fatalf("rotated credential was not adopted: connection=%#v err=%v", connection, err) } rotatedSecret, err := secrets.Get(context.Background(), *connection.ManagementCredentialRef) if err != nil || string(rotatedSecret) != newSecret { t.Fatalf("adopted credential invalid: present=%t err=%v", len(rotatedSecret) > 0, err) } clear(rotatedSecret) if err := manager.RetireConflictingConnection(context.Background(), "identity-pairing-ssf-new-revision"); err != nil { t.Fatal(err) } connection, err = repository.SecurityEventConnection(context.Background()) if err != nil || connection.LifecycleStatus != "retiring" { t.Fatalf("legacy connection was not retired: connection=%#v err=%v", connection, err) } if oldSecretSeen.Load() || newSecretRequests.Load() < 2 { t.Fatalf("old_secret_sent=%t new_secret_requests=%d", oldSecretSeen.Load(), newSecretRequests.Load()) } } func TestFailedCredentialProofDoesNotReplaceLegacyCredential(t *testing.T) { const oldSecret = "old-machine-secret-value-0000000000" const newSecret = "new-machine-secret-value-0000000000" var oldSecretSeen atomic.Bool var tokenRequests atomic.Int32 evilIssuer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, secret, _ := r.BasicAuth() if secret == oldSecret { oldSecretSeen.Store(true) } if secret == newSecret { tokenRequests.Add(1) } _ = json.NewEncoder(w).Encode(map[string]any{"access_token": "untrusted-token", "token_type": "Bearer"}) })) defer evilIssuer.Close() audience, streamID := "urn:easyai:ssf:receiver:legacy", uuid.NewString() managementReference := "ssf-management-legacy" var transmitter *httptest.Server transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/.well-known/ssf-configuration/ssf": _ = json.NewEncoder(w).Encode(map[string]any{ "spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json", "configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status", "verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod}, }) case "/ssf/v1/stream": http.Error(w, "untrusted token issuer", http.StatusUnauthorized) default: http.NotFound(w, r) } })) defer transmitter.Close() repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: transmitter.URL + "/ssf", EndpointURL: transmitter.URL + "/receiver", Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-legacy", ManagementClientID: "shared-machine", ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 2, IdempotencyKey: "manual-legacy-owner", }} secrets := &memorySecretStore{values: map[string][]byte{managementReference: []byte(oldSecret)}} manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: evilIssuer.URL, OIDCTenantID: uuid.NewString(), ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: transmitter.URL + "/ssf", ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision", PublicBaseURL: transmitter.URL, HTTPClient: &http.Client{}, }, &Metrics{}) if err != nil { t.Fatal(err) } _, err = manager.Connect(context.Background(), transmitter.URL+"/ssf", "shared-machine", []byte(newSecret), "identity-pairing-ssf-new-revision") var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "credential_handoff_unsafe" { t.Fatalf("unproved handoff error=%v", err) } connection, getErr := repository.SecurityEventConnection(context.Background()) if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference { t.Fatalf("failed proof replaced credential: connection=%#v err=%v", connection, getErr) } repository.mutex.Lock() queued := len(repository.cleanupDue) repository.mutex.Unlock() if queued != 0 || !secrets.Has(managementReference) || oldSecretSeen.Load() || tokenRequests.Load() != 1 { t.Fatalf("queued=%d old_present=%t old_sent=%t new_requests=%d", queued, secrets.Has(managementReference), oldSecretSeen.Load(), tokenRequests.Load()) } } func TestRemoteNotFoundIsRecognizedForIdempotentDisconnect(t *testing.T) { if !isRemoteHTTPStatus(remoteHTTPError{status: http.StatusNotFound}, http.StatusNotFound) { t.Fatal("remote 404 was not recognized as an already-deleted Stream") } } func TestDiscardPreparedConnectionOnlyDeletesOwnedUnboundResources(t *testing.T) { for _, test := range []struct { name string owner string wantGone bool }{ {name: "owned", owner: "identity-pairing-ssf-revision", wantGone: true}, {name: "different owner", owner: "identity-pairing-ssf-other", wantGone: false}, } { t.Run(test.name, func(t *testing.T) { managementReference := "ssf-management-connection" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", CredentialRef: "ssf-push-connection", ManagementCredentialRef: &managementReference, LifecycleStatus: "error", LastErrorCategory: pointerTo("discovery_failed"), IdempotencyKey: "identity-pairing-ssf-revision", Version: 2, }} secrets := &memorySecretStore{values: map[string][]byte{ "ssf-push-connection": []byte("push-secret-value-for-cleanup-00000000"), "ssf-management-connection": []byte("machine-secret-value-for-cleanup-00000"), }} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets} if err := manager.DiscardPreparedConnection(context.Background(), test.owner); err != nil { t.Fatal(err) } _, getErr := repository.SecurityEventConnection(context.Background()) gone := errors.Is(getErr, store.ErrSecurityEventConnectionNotFound) if gone != test.wantGone { t.Fatalf("connection gone=%t want=%t", gone, test.wantGone) } if test.wantGone && secrets.Len() != 2 { t.Fatalf("owned connection Secrets were deleted before durable cleanup: %d", secrets.Len()) } if !test.wantGone && secrets.Len() != 2 { t.Fatal("a different pairing deleted shared security event resources") } repository.mutex.Lock() queued := len(repository.cleanupDue) repository.mutex.Unlock() if test.wantGone && queued != 2 { t.Fatalf("owned connection queued %d Secrets for cleanup, want 2", queued) } if !test.wantGone && queued != 0 { t.Fatal("a different pairing queued shared security event Secrets for cleanup") } }) } } func TestDiscardPreparedConnectionKeepsBoundRetirementResumable(t *testing.T) { audience, streamID := "urn:easyai:ssf:receiver:"+uuid.NewString(), uuid.NewString() repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-connection", LifecycleStatus: "retiring", IdempotencyKey: "identity-pairing-ssf-revision", Version: 3, }} secrets := &memorySecretStore{values: map[string][]byte{"ssf-push-connection": []byte("push-secret-value-for-retirement-00000")}} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets} err := manager.DiscardPreparedConnection(context.Background(), "identity-pairing-ssf-revision") var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" { t.Fatalf("retirement did not return a safe resumable category: %v", err) } if _, getErr := repository.SecurityEventConnection(context.Background()); getErr != nil || secrets.Len() != 1 { t.Fatal("retirement cleanup deleted local state before the safety window completed") } } func TestDiscardPreparedConnectionCannotDeleteConnectionBoundAfterSnapshot(t *testing.T) { managementReference := "ssf-management-connection" connection := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", CredentialRef: "ssf-push-connection", ManagementCredentialRef: &managementReference, LifecycleStatus: "error", LastErrorCategory: pointerTo("discovery_failed"), IdempotencyKey: "identity-pairing-ssf-revision", Version: 2, } repository := &memoryConnectionRepository{connection: connection} repository.beforeDiscard = func(current *store.SecurityEventConnection) { audience := "urn:easyai:ssf:receiver:" + uuid.NewString() streamID := uuid.NewString() current.Audience = &audience current.StreamID = &streamID current.LifecycleStatus = "verifying" current.Version++ } secrets := &memorySecretStore{values: map[string][]byte{ "ssf-push-connection": []byte("push-secret-value-for-cleanup-00000000"), "ssf-management-connection": []byte("machine-secret-value-for-cleanup-00000"), }} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets} err := manager.DiscardPreparedConnection(context.Background(), "identity-pairing-ssf-revision") var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_cleanup_failed" { t.Fatalf("stale cleanup error=%v, want safe connection_cleanup_failed", err) } persisted, getErr := repository.SecurityEventConnection(context.Background()) if getErr != nil || persisted.StreamID == nil || persisted.Version != 3 { t.Fatalf("stale cleanup removed the newly bound connection: connection=%+v err=%v", persisted, getErr) } if secrets.Len() != 2 { t.Fatalf("stale cleanup removed Secrets used by the newly bound connection: %d", secrets.Len()) } repository.mutex.Lock() queued := len(repository.cleanupDue) repository.mutex.Unlock() if queued != 0 { t.Fatalf("stale cleanup queued %d Secrets used by the newly bound connection", queued) } } func TestConnectionFailureReturnsOnlySafeCategory(t *testing.T) { repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", LifecycleStatus: "connecting", Version: 1, }} manager := &ConnectionManager{repository: repository} err := manager.connectionError(context.Background(), *repository.connection, "discovery_failed", errors.New("sensitive-marker token-value https://internal.example")) var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "discovery_failed" { t.Fatalf("connection failure category=%v", err) } if strings.Contains(err.Error(), "sensitive-marker") || strings.Contains(err.Error(), "internal.example") || strings.Contains(err.Error(), "token-value") { t.Fatalf("connection failure exposed its cause: %q", err.Error()) } } func pointerTo(value string) *string { return &value } func TestConnectionLifecycleCanResumeOnlyIncompleteOrFailedConnections(t *testing.T) { for _, test := range []struct { status string want bool }{ {status: "connecting", want: true}, {status: "verifying", want: true}, {status: "degraded", want: true}, {status: "error", want: true}, {status: "bootstrap"}, {status: "enabled"}, {status: "rotating"}, {status: "disconnect_pending"}, {status: "retiring"}, } { t.Run(test.status, func(t *testing.T) { if got := resumableConnectionLifecycle(test.status); got != test.want { t.Fatalf("resumableConnectionLifecycle(%q)=%t want %t", test.status, got, test.want) } }) } } func TestExistingConnectionCredentialRepairDoesNotRequirePublicBaseURL(t *testing.T) { manager := &ConnectionManager{config: ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCTenantID: uuid.NewString(), }} secret := []byte("machine-secret-for-existing-connection") defer clear(secret) if err := manager.validatePrerequisites("https://auth.example/ssf", "gateway-machine", secret, false); err != nil { t.Fatalf("existing connection credential repair unexpectedly required a public base URL: %v", err) } if err := manager.validatePrerequisites("https://auth.example/ssf", "gateway-machine", secret, true); err == nil { t.Fatal("a new connection must still require AI_GATEWAY_PUBLIC_BASE_URL") } } func TestTransmitterIssuerAllowsLoopbackHTTPOnlyInLocalEnvironments(t *testing.T) { for _, appEnv := range []string{"", "production", "staging"} { if _, err := validatedIssuerURL("http://127.0.0.2:18004/ssf", appEnv); err == nil { t.Fatalf("%s accepted loopback HTTP transmitter issuer", appEnv) } } for _, appEnv := range []string{"local", "development", "dev", "test"} { if _, err := validatedIssuerURL("http://127.0.0.2:18004/ssf", appEnv); err != nil { t.Fatalf("%s rejected loopback HTTP transmitter issuer: %v", appEnv, err) } } } func TestDraftConnectionCannotReplaceCredentialOwnedByAnotherRevision(t *testing.T) { managementReference := "ssf-management-active" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", CredentialRef: "ssf-push-active", ManagementClientID: "active-machine", ManagementCredentialRef: &managementReference, LifecycleStatus: "error", Version: 2, IdempotencyKey: "identity-pairing-ssf-active", }} secrets := &memorySecretStore{values: map[string][]byte{ "ssf-push-active": []byte("push-secret-value-for-active-000000000"), "ssf-management-active": []byte("active-machine-secret-value-000000000"), }} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets, config: ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), }} newSecret := []byte("draft-machine-secret-value-0000000000") defer clear(newSecret) _, err := manager.Connect(context.Background(), "https://auth.example/ssf", "draft-machine", newSecret, "identity-pairing-ssf-draft") if !errors.Is(err, store.ErrSecurityEventConnectionConflict) { t.Fatalf("draft credential replacement error=%v", err) } connection, getErr := repository.SecurityEventConnection(context.Background()) if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference || connection.ManagementClientID != "active-machine" { t.Fatalf("active connection credential metadata changed: %#v err=%v", connection, getErr) } if !secrets.Has(managementReference) || secrets.Len() != 2 { t.Fatal("active connection credential was deleted or a draft credential was retained") } } func TestDraftConnectionReportsExistingRetirementAsResumable(t *testing.T) { managementReference := "ssf-management-active" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", CredentialRef: "ssf-push-active", ManagementClientID: "active-machine", ManagementCredentialRef: &managementReference, LifecycleStatus: "retiring", Version: 3, IdempotencyKey: "identity-pairing-ssf-active", }} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), }} secret := []byte("draft-machine-secret-value-0000000000") defer clear(secret) _, err := manager.Connect(context.Background(), "https://other-auth.example/ssf", "draft-machine", secret, "identity-pairing-ssf-draft") var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" { t.Fatalf("retirement conflict was not classified as resumable: %v", err) } } func TestRetiringConnectionRejectsSameOwnerReconnectBeforeReadingOrReplacingSecrets(t *testing.T) { for _, status := range []string{"retiring", "disconnect_pending"} { for _, credentialsProvided := range []bool{false, true} { t.Run(fmt.Sprintf("%s/credentials=%t", status, credentialsProvided), func(t *testing.T) { managementReference := "ssf-management-retiring" owner := "identity-pairing-ssf-current" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", CredentialRef: "ssf-push-retiring", ManagementClientID: "machine-client", ManagementCredentialRef: &managementReference, LifecycleStatus: status, Version: 3, IdempotencyKey: owner, }} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), }} var clientID string var secret []byte if credentialsProvided { clientID = "machine-client" secret = []byte("replacement-machine-secret-value-000000") defer clear(secret) } _, err := manager.Connect(context.Background(), "https://auth.example/ssf", clientID, secret, owner) var categorized interface{ SafeErrorCategory() string } if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" { t.Fatalf("same-owner reconnect during retirement error=%v", err) } connection, getErr := repository.SecurityEventConnection(context.Background()) if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference { t.Fatalf("retiring connection credential metadata changed: %v", getErr) } repository.mutex.Lock() queued := len(repository.cleanupDue) repository.mutex.Unlock() if queued != 0 { t.Fatal("retiring reconnect staged or retired a Secret") } }) } } } func TestConflictingConnectionRetirementNeverRetiresCurrentPairingOwner(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() oldOwner := "identity-pairing-ssf-active" currentOwner := "identity-pairing-ssf-draft" repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", CredentialRef: "ssf-push-active", ManagementClientID: "machine-client", LifecycleStatus: "enabled", Version: 3, IdempotencyKey: oldOwner, UpdatedAt: time.Now().UTC(), }} manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), RetirementDuration: time.Hour, }} secret := []byte("replacement-machine-secret-value-000000") defer clear(secret) _, connectErr := manager.Connect(ctx, "https://auth.example/ssf", "machine-client", secret, currentOwner) var categorized interface{ SafeErrorCategory() string } if !errors.As(connectErr, &categorized) || categorized.SafeErrorCategory() != "connection_conflict" { t.Fatalf("local-only retirement handoff error=%v", connectErr) } if err := manager.RetireConflictingConnection(ctx, currentOwner); err != nil { t.Fatal(err) } connection, err := repository.SecurityEventConnection(ctx) if err != nil || connection.LifecycleStatus != "retiring" { t.Fatalf("old owner was not safely retired: connection=%#v err=%v", connection, err) } repository.connection.IdempotencyKey = currentOwner repository.connection.LifecycleStatus = "enabled" if err := manager.RetireConflictingConnection(ctx, currentOwner); err != nil { t.Fatal(err) } connection, err = repository.SecurityEventConnection(ctx) if err != nil || connection.LifecycleStatus != "enabled" { t.Fatalf("current pairing connection was retired by a stale recovery action: connection=%#v err=%v", connection, err) } } func TestDisconnectDoesNotReturnSuccessWhenLifecycleCASFails(t *testing.T) { streamID := uuid.NewString() current := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "://invalid", StreamID: &streamID, CredentialRef: "ssf-push", LifecycleStatus: "verifying", Version: 4, } repository := &memoryConnectionRepository{connection: current} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{AppEnv: "test"}} stale := *current stale.LifecycleStatus = "enabled" if _, err := manager.disconnect(context.Background(), stale); !errors.Is(err, store.ErrSecurityEventConnectionConflict) { t.Fatalf("disconnect CAS conflict was reported as success: %v", err) } connection, err := repository.SecurityEventConnection(context.Background()) if err != nil || connection.LifecycleStatus != "verifying" { t.Fatalf("disconnect CAS conflict changed lifecycle: connection=%#v err=%v", connection, err) } } func TestDisconnectTreatsRetirementStatesAsIdempotent(t *testing.T) { for _, lifecycle := range []string{"disconnect_pending", "retiring"} { t.Run(lifecycle, func(t *testing.T) { connection := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-disconnect-idempotent", LifecycleStatus: lifecycle, Version: 9, } repository := &memoryConnectionRepository{connection: connection} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}} view, err := manager.disconnect(context.Background(), *connection) if err != nil { t.Fatal(err) } current, getErr := repository.SecurityEventConnection(context.Background()) if getErr != nil || current.LifecycleStatus != lifecycle || current.Version != 9 || view.Version != 9 { t.Fatalf("idempotent disconnect changed retirement generation: lifecycle=%q version=%d view=%d err=%v", current.LifecycleStatus, current.Version, view.Version, getErr) } }) } } func TestTransmitterSSRFProtectionBlocksReservedNetworks(t *testing.T) { for _, address := range []string{ "127.0.0.1", "10.0.0.1", "169.254.169.254", "100.64.0.1", "192.0.2.1", "198.51.100.1", "203.0.113.1", "2001:db8::1", } { if !blockedAddress(net.ParseIP(address)) { t.Fatalf("reserved address %s was allowed", address) } } if blockedAddress(net.ParseIP("8.8.8.8")) { t.Fatal("public address was blocked") } } func TestDialResolvedAddressesFallsBackFromIPv6ToIPv4(t *testing.T) { listener, err := net.Listen("tcp4", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer listener.Close() port := listener.Addr().(*net.TCPAddr).Port accepted := make(chan struct{}) go func() { connection, acceptErr := listener.Accept() if acceptErr == nil { _ = connection.Close() close(accepted) } }() ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() connection, err := dialResolvedAddresses(ctx, "tcp", strconv.Itoa(port), []net.IPAddr{ {IP: net.ParseIP("::1")}, {IP: net.ParseIP("127.0.0.1")}, }, &net.Dialer{Timeout: 250 * time.Millisecond}) if err != nil { t.Fatalf("IPv4 fallback was not attempted: %v", err) } _ = connection.Close() select { case <-accepted: case <-ctx.Done(): t.Fatal("IPv4 listener did not receive the fallback connection") } } func TestRetiringConnectionStillAppliesRevocationWatermark(t *testing.T) { audience := "urn:easyai:ssf:receiver:" + uuid.NewString() repository := &memoryConnectionRepository{ evaluation: store.SecurityEventEvaluation{Mode: "introspection_fallback", Revoked: true, RequireIntrospection: true}, connection: &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", Audience: &audience, LifecycleStatus: "retiring", Version: 1, }, } manager := &ConnectionManager{ctx: context.Background(), repository: repository, config: ConnectionManagerConfig{StaleAfter: 3 * time.Minute}} evaluation, err := manager.Evaluate(context.Background(), auth.OIDCSecurityEventIdentity{IssuedAt: time.Now().Add(-time.Minute)}) if err != nil || !evaluation.Enabled || !evaluation.Revoked || !evaluation.RequireIntrospection { t.Fatalf("retiring evaluation=%#v err=%v", evaluation, err) } } func TestRetirementDurablyQueuesReferencesBeforeRemovingConnection(t *testing.T) { ctx := context.Background() connection := store.SecurityEventConnection{ ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-retiring", LifecycleStatus: "retiring", Version: 1, UpdatedAt: time.Now().Add(-time.Minute), } repository := &memoryConnectionRepository{connection: &connection} secrets := &memorySecretStore{values: map[string][]byte{ connection.CredentialRef: []byte("push-secret-value-for-retirement-00000"), }} manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: ConnectionManagerConfig{RetirementDuration: time.Millisecond}} manager.finishRetirement(connection) if _, err := repository.SecurityEventConnection(context.Background()); !errors.Is(err, store.ErrSecurityEventConnectionNotFound) { t.Fatalf("finalized retiring connection still exists: %v", err) } repository.mutex.Lock() _, queued := repository.cleanupDue[connection.CredentialRef] repository.mutex.Unlock() if !queued { t.Fatal("retirement removed the connection before durably queueing its Secret reference") } if !secrets.Has(connection.CredentialRef) { t.Fatal("retirement bypassed the durable Secret cleanup worker") } } type retireDuringSecretReadStore struct { once sync.Once onRetire func() } func (*retireDuringSecretReadStore) Put(context.Context, string, []byte) error { return nil } func (s *retireDuringSecretReadStore) Get(context.Context, string) ([]byte, error) { s.once.Do(s.onRetire) return nil, ErrSecretNotFound } func (*retireDuringSecretReadStore) Delete(context.Context, string) error { return nil } func retireMemoryConnection(repository *memoryConnectionRepository, connectionID string) { repository.mutex.Lock() defer repository.mutex.Unlock() if repository.connection == nil || repository.connection.ConnectionID != connectionID { return } repository.connection.LifecycleStatus = "retiring" repository.connection.Version++ repository.connection.UpdatedAt = time.Now().UTC() } func TestConnectionManagerConstructionFailureCannotReviveConcurrentRetirement(t *testing.T) { audience := "urn:easyai:ssf:receiver:" + uuid.NewString() streamID := uuid.NewString() connection := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", EndpointURL: "https://gateway.example/api/v1/security-events/ssf", Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-constructor-race", LifecycleStatus: "enabled", Version: 7, } initialVersion := connection.Version repository := &memoryConnectionRepository{connection: connection} secrets := &retireDuringSecretReadStore{onRetire: func() { retireMemoryConnection(repository, connection.ConnectionID) }} if _, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), }, &Metrics{}); err != nil { t.Fatal(err) } current, err := repository.SecurityEventConnection(context.Background()) if err != nil || current.LifecycleStatus != "retiring" || current.Version != initialVersion+1 { t.Fatalf("constructor failure revived concurrent retirement: lifecycle=%q version=%d err=%v", current.LifecycleStatus, current.Version, err) } } func TestConnectionOperationFailureCannotReviveConcurrentRetirement(t *testing.T) { connection := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-operation-race", LifecycleStatus: "verifying", Version: 11, } repository := &memoryConnectionRepository{connection: connection} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}} staleOperationSnapshot := *connection retireMemoryConnection(repository, connection.ConnectionID) _ = manager.connectionError(context.Background(), staleOperationSnapshot, "discovery_failed", errors.New("remote operation failed")) current, err := repository.SecurityEventConnection(context.Background()) if err != nil || current.LifecycleStatus != "retiring" || current.Version != staleOperationSnapshot.Version+1 { t.Fatalf("operation failure revived concurrent retirement: lifecycle=%q version=%d err=%v", current.LifecycleStatus, current.Version, err) } } func TestConnectionErrorCannotMoveCurrentRetirementGenerationBackToError(t *testing.T) { connection := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-current-retirement", LifecycleStatus: "retiring", Version: 5, } repository := &memoryConnectionRepository{connection: connection} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}} _ = manager.connectionError(context.Background(), *connection, "discovery_failed", errors.New("late remote operation failed")) current, err := repository.SecurityEventConnection(context.Background()) if err != nil || current.LifecycleStatus != "retiring" || current.Version != connection.Version { t.Fatalf("late operation failure revived current retirement generation: lifecycle=%q version=%d err=%v", current.LifecycleStatus, current.Version, err) } } func TestFinishRetirementDoesNotDeleteAConnectionRevivedAfterItsSnapshot(t *testing.T) { connection := store.SecurityEventConnection{ ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-retirement-generation", LifecycleStatus: "retiring", Version: 3, UpdatedAt: time.Now().Add(-time.Minute), } staleRetirementSnapshot := connection connection.LifecycleStatus = "enabled" connection.Version++ repository := &memoryConnectionRepository{connection: &connection} secrets := &memorySecretStore{values: map[string][]byte{ connection.CredentialRef: []byte("push-secret-value-for-retirement-generation"), }} manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets, config: ConnectionManagerConfig{RetirementDuration: time.Millisecond}} manager.finishRetirement(staleRetirementSnapshot) current, err := repository.SecurityEventConnection(context.Background()) if err != nil || current.LifecycleStatus != "enabled" || current.Version != connection.Version { t.Fatalf("stale retirement deleted or changed the current connection: lifecycle=%q version=%d err=%v", current.LifecycleStatus, current.Version, err) } if !secrets.Has(connection.CredentialRef) { t.Fatal("stale retirement deleted a Secret referenced by the current connection") } } func TestIdentitySecretCleanupWorkerResumesWithoutConnectionManager(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() reference := "ssf-management-staged-restart" repository := &memoryConnectionRepository{cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}} secrets := &memorySecretStore{values: map[string][]byte{ reference: []byte("staged-machine-secret-value-for-restart"), }} go RunIdentitySecretCleanupWorker(ctx, repository, secrets) deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) { repository.mutex.Lock() _, queued := repository.cleanupDue[reference] repository.mutex.Unlock() secretExists := secrets.Has(reference) if !queued && !secretExists { return } time.Sleep(10 * time.Millisecond) } t.Fatal("queued identity Secret cleanup did not resume after restart") } func TestIdentitySecretCleanupWorkerKeepsClaimWhenDeleteIsUncertain(t *testing.T) { reference := "ssf-management-worker-delete-uncertain" repository := &memoryConnectionRepository{cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}} secrets := &failedStagingSecretStore{deleteErr: errors.New("SecretStore delete result is uncertain")} cleanupClaimedIdentitySecrets(context.Background(), repository, secrets) repository.mutex.Lock() _, pending := repository.cleanupDue[reference] claimToken := repository.cleanupClaims[reference] repository.mutex.Unlock() if !pending || claimToken == "" { t.Fatal("uncertain worker deletion released its durable claim") } } func TestConnectionManagerDoesNotStartIdentitySecretCleanupWorker(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() reference := "ssf-management-manager-must-not-clean" claimed := make(chan struct{}, 1) repository := &memoryConnectionRepository{ cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}, cleanupClaimed: claimed, } if _, err := NewConnectionManager(ctx, repository, &memorySecretStore{}, ConnectionManagerConfig{ OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), }, &Metrics{}); err != nil { t.Fatal(err) } select { case <-claimed: t.Fatal("ConnectionManager started an identity Secret cleanup worker") case <-time.After(50 * time.Millisecond): } } type failedStagingSecretStore struct { deleteErr error deletes atomic.Int64 } func (*failedStagingSecretStore) Put(context.Context, string, []byte) error { return errors.New("SecretStore write result is uncertain") } func (*failedStagingSecretStore) Get(context.Context, string) ([]byte, error) { return nil, ErrSecretNotFound } func (s *failedStagingSecretStore) Delete(context.Context, string) error { s.deletes.Add(1) return s.deleteErr } func TestStageIdentitySecretKeepsPendingCleanupWhenPutAndDeleteAreUncertain(t *testing.T) { reference := "ssf-management-uncertain-put" repository := &memoryConnectionRepository{} secrets := &failedStagingSecretStore{deleteErr: errors.New("SecretStore delete result is uncertain")} manager := &ConnectionManager{repository: repository, secrets: secrets} if err := manager.stageIdentitySecret(context.Background(), reference, []byte("machine-secret-value-for-uncertain-put")); err == nil { t.Fatal("uncertain SecretStore Put unexpectedly succeeded") } if secrets.deletes.Load() != 1 { t.Fatalf("cleanup delete attempts=%d want=1", secrets.deletes.Load()) } repository.mutex.Lock() _, pending := repository.cleanupDue[reference] repository.mutex.Unlock() if !pending { t.Fatal("uncertain SecretStore result lost the durable cleanup intent") } } func TestStageIdentitySecretRemovesPendingCleanupAfterConfirmedDelete(t *testing.T) { reference := "ssf-management-confirmed-delete" repository := &memoryConnectionRepository{} secrets := &failedStagingSecretStore{} manager := &ConnectionManager{repository: repository, secrets: secrets} if err := manager.stageIdentitySecret(context.Background(), reference, []byte("machine-secret-value-for-confirmed-delete")); err == nil { t.Fatal("failed SecretStore Put unexpectedly succeeded") } repository.mutex.Lock() _, pending := repository.cleanupDue[reference] repository.mutex.Unlock() if pending { t.Fatal("confirmed Secret deletion retained a stale cleanup intent") } } type ambiguousAdoptionRepository struct { *memoryConnectionRepository failCreate bool failManagement bool failNext bool } func (r *ambiguousAdoptionRepository) CreateSecurityEventConnection(ctx context.Context, input store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error) { connection, err := r.memoryConnectionRepository.CreateSecurityEventConnection(ctx, input) if err == nil && r.failCreate { return store.SecurityEventConnection{}, errors.New("database commit result is uncertain") } return connection, err } func (r *ambiguousAdoptionRepository) SetSecurityEventManagementCredential(ctx context.Context, connectionID, clientID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { connection, err := r.memoryConnectionRepository.SetSecurityEventManagementCredential(ctx, connectionID, clientID, reference, expectedVersion) if err == nil && r.failManagement { return store.SecurityEventConnection{}, errors.New("database commit result is uncertain") } return connection, err } func (r *ambiguousAdoptionRepository) SetSecurityEventNextCredential(ctx context.Context, connectionID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { connection, err := r.memoryConnectionRepository.SetSecurityEventNextCredential(ctx, connectionID, reference, expectedVersion) if err == nil && r.failNext { return store.SecurityEventConnection{}, errors.New("database commit result is uncertain") } return connection, err } func TestConnectDoesNotRequeueSecretsAfterAmbiguousAdoptionCommit(t *testing.T) { ctx := context.Background() base := &memoryConnectionRepository{} repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failCreate: true} secrets := &memorySecretStore{} manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), PublicBaseURL: "http://localhost:18089", }} managementSecret := []byte("machine-secret-value-for-ambiguous-create") defer clear(managementSecret) if _, err := manager.Connect(ctx, "https://auth.example/ssf", "machine-client", managementSecret, "ambiguous-create"); err == nil { t.Fatal("ambiguous connection commit unexpectedly reported success") } connection, err := base.SecurityEventConnection(ctx) if err != nil || connection.ManagementCredentialRef == nil { t.Fatalf("simulated committed connection was not retained: %v", err) } base.mutex.Lock() _, pushQueued := base.cleanupDue[connection.CredentialRef] _, managementQueued := base.cleanupDue[*connection.ManagementCredentialRef] base.mutex.Unlock() if pushQueued || managementQueued { t.Fatal("ambiguous committed connection credentials were requeued for deletion") } if !secrets.Has(connection.CredentialRef) || !secrets.Has(*connection.ManagementCredentialRef) { t.Fatal("ambiguous committed connection lost an active Secret") } } func TestManagementCredentialDoesNotRequeueSecretAfterAmbiguousAdoptionCommit(t *testing.T) { ctx := context.Background() oldReference := "ssf-management-before-ambiguous-commit" connection := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-active", ManagementCredentialRef: &oldReference, ManagementClientID: "machine-client", LifecycleStatus: "enabled", } base := &memoryConnectionRepository{connection: connection} repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failManagement: true} secrets := &memorySecretStore{values: map[string][]byte{oldReference: []byte("old-machine-secret-value-before-commit")}} manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets} newSecret := []byte("new-machine-secret-value-after-commit") defer clear(newSecret) if _, err := manager.persistManagementCredential(ctx, *connection, "machine-client", newSecret); err == nil { t.Fatal("ambiguous management credential commit unexpectedly reported success") } committed, err := base.SecurityEventConnection(ctx) if err != nil || committed.ManagementCredentialRef == nil || *committed.ManagementCredentialRef == oldReference { t.Fatalf("simulated committed management credential was not retained: %v", err) } newReference := *committed.ManagementCredentialRef base.mutex.Lock() _, newQueued := base.cleanupDue[newReference] _, oldQueued := base.cleanupDue[oldReference] base.mutex.Unlock() if newQueued || !oldQueued { t.Fatal("ambiguous commit queued the active credential or lost retirement of the old credential") } if !secrets.Has(newReference) { t.Fatal("ambiguous committed management credential lost its Secret") } } func TestNextCredentialDoesNotRequeueSecretAfterAmbiguousAdoptionCommit(t *testing.T) { ctx := context.Background() audience := "urn:easyai:ssf:receiver:" + uuid.NewString() streamID := uuid.NewString() connection := &store.SecurityEventConnection{ ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", EndpointURL: "https://gateway.example/api/v1/security-events/ssf", Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-active", LifecycleStatus: "enabled", } base := &memoryConnectionRepository{connection: connection} repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failNext: true} secrets := &memorySecretStore{} manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets} if _, err := manager.RotateCredential(ctx); err == nil { t.Fatal("ambiguous next credential commit unexpectedly reported success") } committed, err := base.SecurityEventConnection(ctx) if err != nil || committed.NextCredentialRef == nil { t.Fatalf("simulated committed next credential was not retained: %v", err) } nextReference := *committed.NextCredentialRef base.mutex.Lock() _, queued := base.cleanupDue[nextReference] base.mutex.Unlock() if queued { t.Fatal("ambiguous commit requeued the active next credential") } if !secrets.Has(nextReference) { t.Fatal("ambiguous commit deleted the active next credential") } } func (r *memoryConnectionRepository) PromoteSecurityEventCredential(context.Context, string, string) (store.SecurityEventConnection, error) { return store.SecurityEventConnection{}, errors.New("not used") } func (r *memoryConnectionRepository) DiscardPreparedSecurityEventConnection(_ context.Context, id, ownerKey string, expectedVersion int64) error { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil { return store.ErrSecurityEventConnectionConflict } if r.beforeDiscard != nil { r.beforeDiscard(r.connection) } if r.connection.ConnectionID != id || r.connection.IdempotencyKey != ownerKey || r.connection.Version != expectedVersion || r.connection.StreamID != nil { return store.ErrSecurityEventConnectionConflict } if r.cleanupDue == nil { r.cleanupDue = map[string]time.Time{} } references := connectionSecretReferences(*r.connection) for _, reference := range references { if r.cleanupClaims[reference] != "" { return store.ErrIdentitySecretCleanupConflict } } now := time.Now().UTC() for _, reference := range references { r.cleanupDue[reference] = now } r.connection = nil return nil } func (r *memoryConnectionRepository) FinalizeRetiringSecurityEventConnection(_ context.Context, id string, expectedVersion int64) error { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil { return store.ErrSecurityEventConnectionNotFound } if r.beforeFinalize != nil { r.beforeFinalize(r.connection) } if r.connection.ConnectionID != id || r.connection.LifecycleStatus != "retiring" || r.connection.Version != expectedVersion { return store.ErrSecurityEventConnectionConflict } if r.cleanupDue == nil { r.cleanupDue = map[string]time.Time{} } for _, reference := range connectionSecretReferences(*r.connection) { if r.cleanupClaims[reference] != "" { return store.ErrIdentitySecretCleanupConflict } } now := time.Now().UTC() for _, reference := range connectionSecretReferences(*r.connection) { r.cleanupDue[reference] = now } r.finalizeCalls++ r.connection = nil return nil } func (r *memoryConnectionRepository) SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error) { return r.mode, r.verifiedAt, nil } type memorySecretStore struct { mutex sync.Mutex values map[string][]byte } func (s *memorySecretStore) Put(_ context.Context, reference string, value []byte) error { s.mutex.Lock() defer s.mutex.Unlock() if s.values == nil { s.values = map[string][]byte{} } s.values[reference] = append([]byte(nil), value...) return nil } func (s *memorySecretStore) Get(_ context.Context, reference string) ([]byte, error) { s.mutex.Lock() defer s.mutex.Unlock() value, ok := s.values[reference] if !ok { return nil, ErrSecretNotFound } return append([]byte(nil), value...), nil } func (s *memorySecretStore) Delete(_ context.Context, reference string) error { s.mutex.Lock() defer s.mutex.Unlock() delete(s.values, reference) return nil } func (s *memorySecretStore) Has(reference string) bool { s.mutex.Lock() defer s.mutex.Unlock() _, ok := s.values[reference] return ok } func (s *memorySecretStore) Len() int { s.mutex.Lock() defer s.mutex.Unlock() return len(s.values) } func TestConnectionManagerCreatesPausedStreamWithBackendGeneratedBearer(t *testing.T) { repository := &memoryConnectionRepository{} secrets := &memorySecretStore{} ctx, cancel := context.WithCancel(context.Background()) defer cancel() var transmitter *httptest.Server verificationRequested := make(chan struct{}, 1) var managementScopeRequested atomic.Bool transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/.well-known/ssf-configuration/ssf": _ = json.NewEncoder(w).Encode(map[string]any{ "spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json", "configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status", "verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod}, }) case "/oidc/token": clientID, secret, ok := r.BasicAuth() if !ok || clientID != "gateway-machine" || secret != "machine-secret-for-test" { t.Fatal("RFC 7662 machine client was not reused") } _ = r.ParseForm() if strings.Contains(r.Form.Get("scope"), "ssf.stream.manage") { managementScopeRequested.Store(true) } _ = json.NewEncoder(w).Encode(map[string]any{"access_token": "machine-token", "token_type": "Bearer"}) case "/ssf/v1/stream": if r.Method == http.MethodGet { _, _ = w.Write([]byte(`[]`)) return } var request struct { Delivery struct { AuthorizationHeader string `json:"authorization_header"` EndpointURL string `json:"endpoint_url"` } `json:"delivery"` Description string `json:"description"` } _ = json.NewDecoder(r.Body).Decode(&request) if len(strings.TrimPrefix(request.Delivery.AuthorizationHeader, "Bearer ")) < 43 { t.Fatal("generated Push Bearer has less than 256 bits") } _ = json.NewEncoder(w).Encode(map[string]any{ "stream_id": uuid.NewString(), "iss": transmitter.URL + "/ssf", "aud": "urn:easyai:ssf:receiver:" + uuid.NewString(), "events_requested": []string{sessionRevokedEvent}, "delivery": map[string]any{"method": pushDeliveryMethod, "endpoint_url": request.Delivery.EndpointURL}, "description": request.Description, }) case "/ssf/v1/verify": w.WriteHeader(http.StatusNoContent) select { case verificationRequested <- struct{}{}: default: } default: http.NotFound(w, r) } })) defer transmitter.Close() manager, err := NewConnectionManager(ctx, repository, secrets, ConnectionManagerConfig{ AppEnv: "test", OIDCEnabled: true, OIDCIssuer: transmitter.URL + "/oidc", OIDCTenantID: uuid.NewString(), ManagementClientID: "gateway-machine", ManagementClientSecret: "machine-secret-for-test", PublicBaseURL: transmitter.URL, HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour, HTTPClient: transmitter.Client(), }, &Metrics{}) if err != nil { t.Fatal(err) } evaluation, err := manager.Evaluate(ctx, auth.OIDCSecurityEventIdentity{}) if err != nil || evaluation.Enabled { t.Fatalf("unconfigured manager evaluation=%#v err=%v", evaluation, err) } view, err := manager.Connect(ctx, transmitter.URL+"/ssf", "", nil, "connect-once") if err != nil { t.Fatal(err) } if view.LifecycleStatus != "verifying" || view.StreamID == nil || view.Audience == nil { t.Fatalf("connection view=%#v", view) } encoded, _ := json.Marshal(view) if strings.Contains(string(encoded), "credential") || strings.Contains(string(encoded), "machine-secret-for-test") || strings.Contains(string(encoded), "ssf-push-") { t.Fatalf("secret metadata escaped in response: %s", encoded) } if secrets.Len() != 2 { t.Fatalf("secret count=%d", secrets.Len()) } restarted := &ConnectionManager{repository: repository, secrets: secrets, config: ConnectionManagerConfig{}} restartedClientID, restartedSecret, err := restarted.IntrospectionCredential(ctx) if err != nil || restartedClientID != "gateway-machine" || string(restartedSecret) != "machine-secret-for-test" { t.Fatalf("persisted management credential was not recoverable after restart: client=%q secret_present=%v err=%v", restartedClientID, len(restartedSecret) > 0, err) } clear(restartedSecret) if !managementScopeRequested.Load() { t.Fatal("management scopes were not requested automatically") } select { case <-verificationRequested: case <-time.After(2 * time.Second): t.Fatal("verification was not started without a restart") } }