修复 credentials_saved 状态无法恢复、配对与激活并发冲突,以及 SSF 和身份 Secret 生命周期不完整的问题。新增持久化协调器、取消与清理状态机、事务级并发门禁、受控 SSF 凭据交接、禁用后的延迟 Secret 清理,并对生产环境统一认证及 Discovery 端点强制 HTTPS。 验证:go test ./...;go test -race ./internal/auth ./internal/identity ./internal/identityruntime ./internal/securityevents ./internal/httpapi ./internal/store -count=1;go vet ./...;真实 PostgreSQL 并发及清理成功/冲突回滚测试;pnpm openapi。
1651 lines
73 KiB
Go
1651 lines
73 KiB
Go
package securityevents
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type memoryConnectionRepository struct {
|
|
mutex sync.Mutex
|
|
connection *store.SecurityEventConnection
|
|
beforeDiscard func(*store.SecurityEventConnection)
|
|
beforeFinalize func(*store.SecurityEventConnection)
|
|
finalizeCalls int
|
|
cleanupDue map[string]time.Time
|
|
cleanupClaims map[string]string
|
|
cleanupClaimed chan struct{}
|
|
mode string
|
|
verifiedAt *time.Time
|
|
evaluation store.SecurityEventEvaluation
|
|
}
|
|
|
|
func (r *memoryConnectionRepository) QueueIdentitySecretCleanup(_ context.Context, reference string, notBefore time.Time) error {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.cleanupDue == nil {
|
|
r.cleanupDue = map[string]time.Time{}
|
|
}
|
|
if r.cleanupClaims[reference] != "" {
|
|
return store.ErrIdentitySecretCleanupConflict
|
|
}
|
|
current, exists := r.cleanupDue[reference]
|
|
if !exists || notBefore.Before(current) {
|
|
r.cleanupDue[reference] = notBefore
|
|
}
|
|
return nil
|
|
}
|
|
func (r *memoryConnectionRepository) RemovePendingIdentitySecretCleanup(_ context.Context, reference string) (bool, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.cleanupClaims[reference] != "" {
|
|
return false, nil
|
|
}
|
|
_, exists := r.cleanupDue[reference]
|
|
delete(r.cleanupDue, reference)
|
|
return exists, nil
|
|
}
|
|
func (r *memoryConnectionRepository) ClaimIdentitySecretCleanups(_ context.Context, limit int, lease time.Duration) ([]store.IdentitySecretCleanupClaim, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.cleanupClaims == nil {
|
|
r.cleanupClaims = map[string]string{}
|
|
}
|
|
if r.cleanupClaimed != nil {
|
|
select {
|
|
case r.cleanupClaimed <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
now := time.Now().UTC()
|
|
claims := make([]store.IdentitySecretCleanupClaim, 0)
|
|
for reference, notBefore := range r.cleanupDue {
|
|
if !notBefore.After(now) && r.cleanupClaims[reference] == "" {
|
|
token := uuid.NewString()
|
|
r.cleanupClaims[reference] = token
|
|
claims = append(claims, store.IdentitySecretCleanupClaim{
|
|
Reference: reference, ClaimToken: token, LeaseExpiresAt: now.Add(lease),
|
|
})
|
|
if len(claims) == limit {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return claims, nil
|
|
}
|
|
func (r *memoryConnectionRepository) CompleteIdentitySecretCleanup(_ context.Context, reference, claimToken string) (bool, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.cleanupClaims[reference] != claimToken {
|
|
return false, nil
|
|
}
|
|
delete(r.cleanupClaims, reference)
|
|
delete(r.cleanupDue, reference)
|
|
return true, nil
|
|
}
|
|
|
|
func (*memoryConnectionRepository) ApplySessionRevoked(context.Context, store.ApplySessionRevokedInput) (store.ApplySecurityEventResult, error) {
|
|
return store.ApplySecurityEventResult{}, nil
|
|
}
|
|
func (*memoryConnectionRepository) ApplySecurityEventStreamUpdated(context.Context, string, string, string, string, string, string, time.Time) (bool, error) {
|
|
return true, nil
|
|
}
|
|
func (*memoryConnectionRepository) ConfirmSecurityEventVerification(context.Context, string, string, string, string, []byte, time.Time) (bool, error) {
|
|
return true, nil
|
|
}
|
|
func (*memoryConnectionRepository) RecordSecurityEventReceipt(context.Context, string, string, string, string, string) (bool, error) {
|
|
return true, nil
|
|
}
|
|
func (r *memoryConnectionRepository) EnsureSecurityEventStreamState(context.Context, string, string, string) error {
|
|
r.mode = "bootstrap"
|
|
return nil
|
|
}
|
|
func (*memoryConnectionRepository) BeginSecurityEventVerification(context.Context, string, string, []byte, time.Time) error {
|
|
return nil
|
|
}
|
|
func (*memoryConnectionRepository) RecordSecurityEventHeartbeatFailure(context.Context, string, string, string) error {
|
|
return nil
|
|
}
|
|
func (*memoryConnectionRepository) AdvanceSecurityEventStreamState(context.Context, string, string, time.Time, time.Duration) error {
|
|
return nil
|
|
}
|
|
func (r *memoryConnectionRepository) EvaluateOIDCSecurityEvent(context.Context, string, string, string, string, string, time.Time, time.Time, time.Duration) (store.SecurityEventEvaluation, error) {
|
|
if r.evaluation.Mode == "" {
|
|
return store.SecurityEventEvaluation{Mode: "bootstrap", RequireIntrospection: true}, nil
|
|
}
|
|
return r.evaluation, nil
|
|
}
|
|
func (r *memoryConnectionRepository) CreateSecurityEventConnection(_ context.Context, input store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection != nil {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict
|
|
}
|
|
now := time.Now().UTC()
|
|
value := store.SecurityEventConnection{ConnectionID: input.ConnectionID, TransmitterIssuer: input.TransmitterIssuer,
|
|
EndpointURL: input.EndpointURL, CredentialRef: input.CredentialRef, ManagementClientID: input.ManagementClientID,
|
|
ManagementCredentialRef: &input.ManagementCredentialRef, LifecycleStatus: "connecting", Version: 1,
|
|
IdempotencyKey: input.IdempotencyKey, CreatedAt: now, UpdatedAt: now, HealthMode: "disabled"}
|
|
r.connection = &value
|
|
delete(r.cleanupDue, input.CredentialRef)
|
|
delete(r.cleanupDue, input.ManagementCredentialRef)
|
|
return value, nil
|
|
}
|
|
func (r *memoryConnectionRepository) SetSecurityEventManagementCredential(_ context.Context, id, clientID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection == nil || r.connection.ConnectionID != id {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound
|
|
}
|
|
if r.connection.Version != expectedVersion || r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending" {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict
|
|
}
|
|
oldReference := r.connection.ManagementCredentialRef
|
|
r.connection.ManagementClientID, r.connection.ManagementCredentialRef = clientID, &reference
|
|
r.connection.Version++
|
|
delete(r.cleanupDue, reference)
|
|
if oldReference != nil && *oldReference != reference {
|
|
if r.cleanupDue == nil {
|
|
r.cleanupDue = map[string]time.Time{}
|
|
}
|
|
r.cleanupDue[*oldReference] = time.Now().UTC()
|
|
}
|
|
return *r.connection, nil
|
|
}
|
|
func (r *memoryConnectionRepository) SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection == nil {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound
|
|
}
|
|
value := *r.connection
|
|
value.HealthMode, value.LastVerificationAt = r.mode, r.verifiedAt
|
|
return value, nil
|
|
}
|
|
func (r *memoryConnectionRepository) BindSecurityEventConnection(_ context.Context, id, audience, streamID, lifecycle string, expected int64) (store.SecurityEventConnection, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection == nil || r.connection.ConnectionID != id || r.connection.Version != expected {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict
|
|
}
|
|
r.connection.Audience, r.connection.StreamID = &audience, &streamID
|
|
r.connection.LifecycleStatus, r.connection.Version = lifecycle, r.connection.Version+1
|
|
return *r.connection, nil
|
|
}
|
|
func (r *memoryConnectionRepository) TransitionSecurityEventConnectionLifecycle(_ context.Context, id, expectedLifecycle string, expectedVersion int64, lifecycle string, category *string) (store.SecurityEventConnection, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection == nil || r.connection.ConnectionID != id || r.connection.LifecycleStatus != expectedLifecycle || r.connection.Version != expectedVersion {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict
|
|
}
|
|
if (r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending") &&
|
|
lifecycle != "retiring" && lifecycle != "disconnect_pending" {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict
|
|
}
|
|
r.connection.LifecycleStatus, r.connection.LastErrorCategory = lifecycle, category
|
|
r.connection.Version++
|
|
r.connection.UpdatedAt = time.Now().UTC()
|
|
return *r.connection, nil
|
|
}
|
|
func (r *memoryConnectionRepository) SetSecurityEventNextCredential(_ context.Context, id, reference string, expectedVersion int64) (store.SecurityEventConnection, error) {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection == nil || r.connection.ConnectionID != id {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound
|
|
}
|
|
if r.connection.Version != expectedVersion || r.connection.NextCredentialRef != nil || r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending" {
|
|
return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict
|
|
}
|
|
r.connection.NextCredentialRef = &reference
|
|
r.connection.LifecycleStatus = "rotating"
|
|
r.connection.Version++
|
|
delete(r.cleanupDue, reference)
|
|
return *r.connection, nil
|
|
}
|
|
|
|
func TestConnectionManagerBootstrapOnlyBecomesEnabledWhenPushIsHealthy(t *testing.T) {
|
|
for _, test := range []struct {
|
|
mode, wantLifecycle string
|
|
}{
|
|
{mode: "push_healthy", wantLifecycle: "enabled"},
|
|
{mode: "introspection_fallback", wantLifecycle: "degraded"},
|
|
} {
|
|
t.Run(test.mode, func(t *testing.T) {
|
|
audience := "urn:easyai:ssf:receiver:" + uuid.NewString()
|
|
repository := &memoryConnectionRepository{mode: test.mode, connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", Audience: &audience,
|
|
LifecycleStatus: "bootstrap", Version: 1, UpdatedAt: time.Now().UTC(),
|
|
}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository,
|
|
config: ConnectionManagerConfig{BootstrapDuration: time.Millisecond}}
|
|
manager.finishBootstrap(repository.connection.ConnectionID)
|
|
connection, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || connection.LifecycleStatus != test.wantLifecycle {
|
|
t.Fatalf("connection=%#v err=%v", connection, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConnectionManagerResumesBootstrapAfterRestart(t *testing.T) {
|
|
repository := &memoryConnectionRepository{mode: "push_healthy", connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
LifecycleStatus: "bootstrap", Version: 1, UpdatedAt: time.Now().Add(-time.Second),
|
|
}}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
if _, err := NewConnectionManager(ctx, repository, &memorySecretStore{}, ConnectionManagerConfig{
|
|
OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
BootstrapDuration: 10 * time.Millisecond,
|
|
}, &Metrics{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
connection, err := repository.SecurityEventConnection(ctx)
|
|
if err == nil && connection.LifecycleStatus == "enabled" {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatal("bootstrap lifecycle did not resume after process restart")
|
|
}
|
|
|
|
func TestNewConnectionManagerStrictRevisionBindingRejectsDifferentPersistedConnection(t *testing.T) {
|
|
audience := "urn:easyai:ssf:receiver:target"
|
|
streamID := uuid.NewString()
|
|
base := store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(),
|
|
TransmitterIssuer: "https://auth.example/ssf",
|
|
Audience: &audience,
|
|
StreamID: &streamID,
|
|
CredentialRef: "missing-credential-must-not-be-read",
|
|
ManagementClientID: "gateway-machine",
|
|
LifecycleStatus: "enabled",
|
|
IdempotencyKey: "identity-pairing-ssf-target-revision",
|
|
Version: 1,
|
|
}
|
|
config := ConnectionManagerConfig{
|
|
OIDCEnabled: true,
|
|
OIDCIssuer: "https://auth.example/issuer",
|
|
OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "gateway-machine",
|
|
ExpectedTransmitterIssuer: "https://auth.example/ssf",
|
|
ExpectedAudience: audience,
|
|
ExpectedOwnerKey: "identity-pairing-ssf-target-revision",
|
|
StrictRevisionBinding: true,
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*store.SecurityEventConnection)
|
|
}{
|
|
{name: "transmitter issuer", mutate: func(connection *store.SecurityEventConnection) {
|
|
connection.TransmitterIssuer = "https://old-auth.example/ssf"
|
|
}},
|
|
{name: "audience", mutate: func(connection *store.SecurityEventConnection) {
|
|
other := "urn:easyai:ssf:receiver:old"
|
|
connection.Audience = &other
|
|
}},
|
|
{name: "missing audience", mutate: func(connection *store.SecurityEventConnection) {
|
|
connection.Audience = nil
|
|
}},
|
|
{name: "management client", mutate: func(connection *store.SecurityEventConnection) {
|
|
connection.ManagementClientID = "old-machine"
|
|
}},
|
|
{name: "owner", mutate: func(connection *store.SecurityEventConnection) {
|
|
connection.IdempotencyKey = "identity-pairing-ssf-old-revision"
|
|
}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
connection := base
|
|
test.mutate(&connection)
|
|
repository := &memoryConnectionRepository{connection: &connection}
|
|
|
|
manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, config, &Metrics{})
|
|
if manager != nil {
|
|
t.Fatal("strict runtime construction returned a manager for a different Revision binding")
|
|
}
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_mismatch" {
|
|
t.Fatalf("error=%v category=%v", err, categorized)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewConnectionManagerStrictRevisionBindingRequiresPersistedConnection(t *testing.T) {
|
|
manager, err := NewConnectionManager(context.Background(), &memoryConnectionRepository{}, &memorySecretStore{}, ConnectionManagerConfig{
|
|
OIDCEnabled: true,
|
|
OIDCIssuer: "https://auth.example/issuer",
|
|
OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "gateway-machine",
|
|
ExpectedTransmitterIssuer: "https://auth.example/ssf",
|
|
ExpectedAudience: "urn:easyai:ssf:receiver:target",
|
|
ExpectedOwnerKey: "identity-pairing-ssf-target-revision",
|
|
StrictRevisionBinding: true,
|
|
}, &Metrics{})
|
|
if manager != nil {
|
|
t.Fatal("strict runtime construction succeeded without a persisted SSF connection")
|
|
}
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_missing" {
|
|
t.Fatalf("error=%v category=%v", err, categorized)
|
|
}
|
|
}
|
|
|
|
func TestNewConnectionManagerStrictRevisionBindingAcceptsExactPersistedConnection(t *testing.T) {
|
|
audience := "urn:easyai:ssf:receiver:target"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(),
|
|
TransmitterIssuer: "https://auth.example/ssf",
|
|
Audience: &audience,
|
|
ManagementClientID: "gateway-machine",
|
|
LifecycleStatus: "enabled",
|
|
IdempotencyKey: "identity-pairing-ssf-target-revision",
|
|
Version: 1,
|
|
}}
|
|
|
|
manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, ConnectionManagerConfig{
|
|
OIDCEnabled: true,
|
|
OIDCIssuer: "https://auth.example/issuer",
|
|
OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "gateway-machine",
|
|
ExpectedTransmitterIssuer: "https://auth.example/ssf/",
|
|
ExpectedAudience: audience,
|
|
ExpectedOwnerKey: "identity-pairing-ssf-target-revision",
|
|
StrictRevisionBinding: true,
|
|
}, &Metrics{})
|
|
if err != nil || manager == nil {
|
|
t.Fatalf("exact Revision binding was rejected: manager=%v err=%v", manager != nil, err)
|
|
}
|
|
}
|
|
|
|
func TestConnectionManagerRecoveryCanLoadDifferentPersistedConnection(t *testing.T) {
|
|
oldAudience := "urn:easyai:ssf:receiver:old"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(),
|
|
TransmitterIssuer: "https://old-auth.example/ssf",
|
|
Audience: &oldAudience,
|
|
ManagementClientID: "old-machine",
|
|
LifecycleStatus: "error",
|
|
IdempotencyKey: "identity-pairing-ssf-old-revision",
|
|
Version: 1,
|
|
}}
|
|
config := ConnectionManagerConfig{
|
|
OIDCEnabled: true,
|
|
OIDCIssuer: "https://auth.example/issuer",
|
|
OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "gateway-machine",
|
|
ExpectedTransmitterIssuer: "https://auth.example/ssf",
|
|
ExpectedAudience: "urn:easyai:ssf:receiver:target",
|
|
ExpectedOwnerKey: "identity-pairing-ssf-target-revision",
|
|
}
|
|
|
|
manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, config, &Metrics{})
|
|
if err != nil || manager == nil {
|
|
t.Fatalf("recovery manager could not load the conflicting connection: manager=%v err=%v", manager != nil, err)
|
|
}
|
|
err = manager.ValidateConfiguredConnectionBinding(context.Background())
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_mismatch" {
|
|
t.Fatalf("explicit runtime binding validation did not reject recovery connection: err=%v category=%v", err, categorized)
|
|
}
|
|
}
|
|
|
|
func TestForeignRevisionRecoveryNeverUsesPersistedCredentialOnGenericOperations(t *testing.T) {
|
|
var requests atomic.Int32
|
|
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests.Add(1)
|
|
http.Error(w, "unexpected request", http.StatusInternalServerError)
|
|
}))
|
|
defer endpoint.Close()
|
|
|
|
audience, streamID := "urn:easyai:ssf:receiver:old", uuid.NewString()
|
|
managementReference := "ssf-management-old"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: endpoint.URL + "/old-ssf", EndpointURL: endpoint.URL + "/receiver",
|
|
Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-old", ManagementClientID: "shared-machine",
|
|
ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 3,
|
|
IdempotencyKey: "identity-pairing-ssf-old-revision",
|
|
}}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
managementReference: []byte("old-machine-secret-must-never-leave"),
|
|
"ssf-push-old": []byte("old-push-secret-value-000000000000"),
|
|
}}
|
|
manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: endpoint.URL + "/new-oidc", OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: endpoint.URL + "/new-ssf",
|
|
ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision",
|
|
PublicBaseURL: endpoint.URL, HTTPClient: endpoint.Client(),
|
|
}, &Metrics{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
assertUnsafe := func(name string, err error) {
|
|
t.Helper()
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "credential_handoff_unsafe" {
|
|
t.Fatalf("%s error=%v", name, err)
|
|
}
|
|
}
|
|
_, _, credentialErr := manager.IntrospectionCredential(context.Background())
|
|
assertUnsafe("introspection credential", credentialErr)
|
|
_, verifyErr := manager.Verify(context.Background())
|
|
assertUnsafe("verify", verifyErr)
|
|
_, rotateErr := manager.RotateCredential(context.Background())
|
|
assertUnsafe("rotate", rotateErr)
|
|
_, disconnectErr := manager.Disconnect(context.Background())
|
|
assertUnsafe("disconnect", disconnectErr)
|
|
_, connectErr := manager.Connect(context.Background(), endpoint.URL+"/new-ssf", "shared-machine", []byte("new-machine-secret-value-000000000"), "identity-pairing-ssf-new-revision")
|
|
assertUnsafe("cross-issuer connect", connectErr)
|
|
assertUnsafe("retire without proved handoff", manager.RetireConflictingConnection(context.Background(), "identity-pairing-ssf-new-revision"))
|
|
|
|
if requests.Load() != 0 {
|
|
t.Fatalf("foreign recovery sent %d authenticated or discovery requests", requests.Load())
|
|
}
|
|
connection, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference {
|
|
t.Fatalf("foreign recovery changed persisted credential: connection=%#v err=%v", connection, err)
|
|
}
|
|
repository.mutex.Lock()
|
|
queued := len(repository.cleanupDue)
|
|
repository.mutex.Unlock()
|
|
if queued != 0 || !secrets.Has(managementReference) {
|
|
t.Fatalf("foreign recovery queued=%d old_secret_present=%t", queued, secrets.Has(managementReference))
|
|
}
|
|
}
|
|
|
|
func TestPairingProvesRotatedCredentialBeforeRetiringLegacyStream(t *testing.T) {
|
|
const oldSecret = "old-machine-secret-value-0000000000"
|
|
const newSecret = "new-machine-secret-value-0000000000"
|
|
audience, streamID := "urn:easyai:ssf:receiver:legacy", uuid.NewString()
|
|
managementReference := "ssf-management-legacy"
|
|
var oldSecretSeen atomic.Bool
|
|
var newSecretRequests atomic.Int32
|
|
var transmitter *httptest.Server
|
|
transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/.well-known/ssf-configuration/ssf":
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json",
|
|
"configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status",
|
|
"verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod},
|
|
})
|
|
case "/oidc/token":
|
|
_, secret, _ := r.BasicAuth()
|
|
if secret == oldSecret {
|
|
oldSecretSeen.Store(true)
|
|
}
|
|
if secret != newSecret {
|
|
http.Error(w, "invalid client", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
newSecretRequests.Add(1)
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"access_token": "proved-management-token", "token_type": "Bearer"})
|
|
case "/ssf/v1/stream":
|
|
if r.Header.Get("Authorization") != "Bearer proved-management-token" {
|
|
http.Error(w, "invalid token", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method == http.MethodDelete {
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
description := "easyai-gateway:legacy"
|
|
_ = json.NewEncoder(w).Encode([]map[string]any{{
|
|
"stream_id": streamID, "iss": transmitter.URL + "/ssf", "aud": audience,
|
|
"events_requested": []string{sessionRevokedEvent},
|
|
"delivery": map[string]any{"method": pushDeliveryMethod, "endpoint_url": transmitter.URL + "/receiver"},
|
|
"description": description,
|
|
}})
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
defer transmitter.Close()
|
|
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: "legacy", TransmitterIssuer: transmitter.URL + "/ssf", EndpointURL: transmitter.URL + "/receiver",
|
|
Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-legacy", ManagementClientID: "shared-machine",
|
|
ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 2, IdempotencyKey: "manual-legacy-owner",
|
|
}}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
managementReference: []byte(oldSecret), "ssf-push-legacy": []byte("legacy-push-secret-value-000000000"),
|
|
}}
|
|
manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: transmitter.URL + "/oidc", OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: transmitter.URL + "/ssf",
|
|
ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision",
|
|
PublicBaseURL: transmitter.URL, HTTPClient: transmitter.Client(), RetirementDuration: time.Hour,
|
|
}, &Metrics{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, err = manager.Connect(context.Background(), transmitter.URL+"/ssf", "shared-machine", []byte(newSecret), "identity-pairing-ssf-new-revision")
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_conflict" {
|
|
t.Fatalf("proved handoff error=%v", err)
|
|
}
|
|
connection, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef == managementReference {
|
|
t.Fatalf("rotated credential was not adopted: connection=%#v err=%v", connection, err)
|
|
}
|
|
rotatedSecret, err := secrets.Get(context.Background(), *connection.ManagementCredentialRef)
|
|
if err != nil || string(rotatedSecret) != newSecret {
|
|
t.Fatalf("adopted credential invalid: present=%t err=%v", len(rotatedSecret) > 0, err)
|
|
}
|
|
clear(rotatedSecret)
|
|
if err := manager.RetireConflictingConnection(context.Background(), "identity-pairing-ssf-new-revision"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
connection, err = repository.SecurityEventConnection(context.Background())
|
|
if err != nil || connection.LifecycleStatus != "retiring" {
|
|
t.Fatalf("legacy connection was not retired: connection=%#v err=%v", connection, err)
|
|
}
|
|
if oldSecretSeen.Load() || newSecretRequests.Load() < 2 {
|
|
t.Fatalf("old_secret_sent=%t new_secret_requests=%d", oldSecretSeen.Load(), newSecretRequests.Load())
|
|
}
|
|
}
|
|
|
|
func TestFailedCredentialProofDoesNotReplaceLegacyCredential(t *testing.T) {
|
|
const oldSecret = "old-machine-secret-value-0000000000"
|
|
const newSecret = "new-machine-secret-value-0000000000"
|
|
var oldSecretSeen atomic.Bool
|
|
var tokenRequests atomic.Int32
|
|
evilIssuer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, secret, _ := r.BasicAuth()
|
|
if secret == oldSecret {
|
|
oldSecretSeen.Store(true)
|
|
}
|
|
if secret == newSecret {
|
|
tokenRequests.Add(1)
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"access_token": "untrusted-token", "token_type": "Bearer"})
|
|
}))
|
|
defer evilIssuer.Close()
|
|
audience, streamID := "urn:easyai:ssf:receiver:legacy", uuid.NewString()
|
|
managementReference := "ssf-management-legacy"
|
|
var transmitter *httptest.Server
|
|
transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/.well-known/ssf-configuration/ssf":
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json",
|
|
"configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status",
|
|
"verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod},
|
|
})
|
|
case "/ssf/v1/stream":
|
|
http.Error(w, "untrusted token issuer", http.StatusUnauthorized)
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
defer transmitter.Close()
|
|
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: transmitter.URL + "/ssf", EndpointURL: transmitter.URL + "/receiver",
|
|
Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-legacy", ManagementClientID: "shared-machine",
|
|
ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 2, IdempotencyKey: "manual-legacy-owner",
|
|
}}
|
|
secrets := &memorySecretStore{values: map[string][]byte{managementReference: []byte(oldSecret)}}
|
|
manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: evilIssuer.URL, OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: transmitter.URL + "/ssf",
|
|
ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision",
|
|
PublicBaseURL: transmitter.URL, HTTPClient: &http.Client{},
|
|
}, &Metrics{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = manager.Connect(context.Background(), transmitter.URL+"/ssf", "shared-machine", []byte(newSecret), "identity-pairing-ssf-new-revision")
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "credential_handoff_unsafe" {
|
|
t.Fatalf("unproved handoff error=%v", err)
|
|
}
|
|
connection, getErr := repository.SecurityEventConnection(context.Background())
|
|
if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference {
|
|
t.Fatalf("failed proof replaced credential: connection=%#v err=%v", connection, getErr)
|
|
}
|
|
repository.mutex.Lock()
|
|
queued := len(repository.cleanupDue)
|
|
repository.mutex.Unlock()
|
|
if queued != 0 || !secrets.Has(managementReference) || oldSecretSeen.Load() || tokenRequests.Load() != 1 {
|
|
t.Fatalf("queued=%d old_present=%t old_sent=%t new_requests=%d", queued, secrets.Has(managementReference), oldSecretSeen.Load(), tokenRequests.Load())
|
|
}
|
|
}
|
|
|
|
func TestRemoteNotFoundIsRecognizedForIdempotentDisconnect(t *testing.T) {
|
|
if !isRemoteHTTPStatus(remoteHTTPError{status: http.StatusNotFound}, http.StatusNotFound) {
|
|
t.Fatal("remote 404 was not recognized as an already-deleted Stream")
|
|
}
|
|
}
|
|
|
|
func TestDiscardPreparedConnectionOnlyDeletesOwnedUnboundResources(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
owner string
|
|
wantGone bool
|
|
}{
|
|
{name: "owned", owner: "identity-pairing-ssf-revision", wantGone: true},
|
|
{name: "different owner", owner: "identity-pairing-ssf-other", wantGone: false},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
managementReference := "ssf-management-connection"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
CredentialRef: "ssf-push-connection", ManagementCredentialRef: &managementReference,
|
|
LifecycleStatus: "error", LastErrorCategory: pointerTo("discovery_failed"),
|
|
IdempotencyKey: "identity-pairing-ssf-revision", Version: 2,
|
|
}}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
"ssf-push-connection": []byte("push-secret-value-for-cleanup-00000000"),
|
|
"ssf-management-connection": []byte("machine-secret-value-for-cleanup-00000"),
|
|
}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets}
|
|
|
|
if err := manager.DiscardPreparedConnection(context.Background(), test.owner); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, getErr := repository.SecurityEventConnection(context.Background())
|
|
gone := errors.Is(getErr, store.ErrSecurityEventConnectionNotFound)
|
|
if gone != test.wantGone {
|
|
t.Fatalf("connection gone=%t want=%t", gone, test.wantGone)
|
|
}
|
|
if test.wantGone && secrets.Len() != 2 {
|
|
t.Fatalf("owned connection Secrets were deleted before durable cleanup: %d", secrets.Len())
|
|
}
|
|
if !test.wantGone && secrets.Len() != 2 {
|
|
t.Fatal("a different pairing deleted shared security event resources")
|
|
}
|
|
repository.mutex.Lock()
|
|
queued := len(repository.cleanupDue)
|
|
repository.mutex.Unlock()
|
|
if test.wantGone && queued != 2 {
|
|
t.Fatalf("owned connection queued %d Secrets for cleanup, want 2", queued)
|
|
}
|
|
if !test.wantGone && queued != 0 {
|
|
t.Fatal("a different pairing queued shared security event Secrets for cleanup")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDiscardPreparedConnectionKeepsBoundRetirementResumable(t *testing.T) {
|
|
audience, streamID := "urn:easyai:ssf:receiver:"+uuid.NewString(), uuid.NewString()
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-connection",
|
|
LifecycleStatus: "retiring", IdempotencyKey: "identity-pairing-ssf-revision", Version: 3,
|
|
}}
|
|
secrets := &memorySecretStore{values: map[string][]byte{"ssf-push-connection": []byte("push-secret-value-for-retirement-00000")}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets}
|
|
|
|
err := manager.DiscardPreparedConnection(context.Background(), "identity-pairing-ssf-revision")
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" {
|
|
t.Fatalf("retirement did not return a safe resumable category: %v", err)
|
|
}
|
|
if _, getErr := repository.SecurityEventConnection(context.Background()); getErr != nil || secrets.Len() != 1 {
|
|
t.Fatal("retirement cleanup deleted local state before the safety window completed")
|
|
}
|
|
}
|
|
|
|
func TestDiscardPreparedConnectionCannotDeleteConnectionBoundAfterSnapshot(t *testing.T) {
|
|
managementReference := "ssf-management-connection"
|
|
connection := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
CredentialRef: "ssf-push-connection", ManagementCredentialRef: &managementReference,
|
|
LifecycleStatus: "error", LastErrorCategory: pointerTo("discovery_failed"),
|
|
IdempotencyKey: "identity-pairing-ssf-revision", Version: 2,
|
|
}
|
|
repository := &memoryConnectionRepository{connection: connection}
|
|
repository.beforeDiscard = func(current *store.SecurityEventConnection) {
|
|
audience := "urn:easyai:ssf:receiver:" + uuid.NewString()
|
|
streamID := uuid.NewString()
|
|
current.Audience = &audience
|
|
current.StreamID = &streamID
|
|
current.LifecycleStatus = "verifying"
|
|
current.Version++
|
|
}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
"ssf-push-connection": []byte("push-secret-value-for-cleanup-00000000"),
|
|
"ssf-management-connection": []byte("machine-secret-value-for-cleanup-00000"),
|
|
}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets}
|
|
|
|
err := manager.DiscardPreparedConnection(context.Background(), "identity-pairing-ssf-revision")
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_cleanup_failed" {
|
|
t.Fatalf("stale cleanup error=%v, want safe connection_cleanup_failed", err)
|
|
}
|
|
persisted, getErr := repository.SecurityEventConnection(context.Background())
|
|
if getErr != nil || persisted.StreamID == nil || persisted.Version != 3 {
|
|
t.Fatalf("stale cleanup removed the newly bound connection: connection=%+v err=%v", persisted, getErr)
|
|
}
|
|
if secrets.Len() != 2 {
|
|
t.Fatalf("stale cleanup removed Secrets used by the newly bound connection: %d", secrets.Len())
|
|
}
|
|
repository.mutex.Lock()
|
|
queued := len(repository.cleanupDue)
|
|
repository.mutex.Unlock()
|
|
if queued != 0 {
|
|
t.Fatalf("stale cleanup queued %d Secrets used by the newly bound connection", queued)
|
|
}
|
|
}
|
|
|
|
func TestConnectionFailureReturnsOnlySafeCategory(t *testing.T) {
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", LifecycleStatus: "connecting", Version: 1,
|
|
}}
|
|
manager := &ConnectionManager{repository: repository}
|
|
err := manager.connectionError(context.Background(), *repository.connection, "discovery_failed", errors.New("sensitive-marker token-value https://internal.example"))
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "discovery_failed" {
|
|
t.Fatalf("connection failure category=%v", err)
|
|
}
|
|
if strings.Contains(err.Error(), "sensitive-marker") || strings.Contains(err.Error(), "internal.example") || strings.Contains(err.Error(), "token-value") {
|
|
t.Fatalf("connection failure exposed its cause: %q", err.Error())
|
|
}
|
|
}
|
|
|
|
func pointerTo(value string) *string { return &value }
|
|
|
|
func TestConnectionLifecycleCanResumeOnlyIncompleteOrFailedConnections(t *testing.T) {
|
|
for _, test := range []struct {
|
|
status string
|
|
want bool
|
|
}{
|
|
{status: "connecting", want: true},
|
|
{status: "verifying", want: true},
|
|
{status: "degraded", want: true},
|
|
{status: "error", want: true},
|
|
{status: "bootstrap"},
|
|
{status: "enabled"},
|
|
{status: "rotating"},
|
|
{status: "disconnect_pending"},
|
|
{status: "retiring"},
|
|
} {
|
|
t.Run(test.status, func(t *testing.T) {
|
|
if got := resumableConnectionLifecycle(test.status); got != test.want {
|
|
t.Fatalf("resumableConnectionLifecycle(%q)=%t want %t", test.status, got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExistingConnectionCredentialRepairDoesNotRequirePublicBaseURL(t *testing.T) {
|
|
manager := &ConnectionManager{config: ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCTenantID: uuid.NewString(),
|
|
}}
|
|
secret := []byte("machine-secret-for-existing-connection")
|
|
defer clear(secret)
|
|
if err := manager.validatePrerequisites("https://auth.example/ssf", "gateway-machine", secret, false); err != nil {
|
|
t.Fatalf("existing connection credential repair unexpectedly required a public base URL: %v", err)
|
|
}
|
|
if err := manager.validatePrerequisites("https://auth.example/ssf", "gateway-machine", secret, true); err == nil {
|
|
t.Fatal("a new connection must still require AI_GATEWAY_PUBLIC_BASE_URL")
|
|
}
|
|
}
|
|
|
|
func TestTransmitterIssuerAllowsLoopbackHTTPOnlyInLocalEnvironments(t *testing.T) {
|
|
for _, appEnv := range []string{"", "production", "staging"} {
|
|
if _, err := validatedIssuerURL("http://127.0.0.2:18004/ssf", appEnv); err == nil {
|
|
t.Fatalf("%s accepted loopback HTTP transmitter issuer", appEnv)
|
|
}
|
|
}
|
|
for _, appEnv := range []string{"local", "development", "dev", "test"} {
|
|
if _, err := validatedIssuerURL("http://127.0.0.2:18004/ssf", appEnv); err != nil {
|
|
t.Fatalf("%s rejected loopback HTTP transmitter issuer: %v", appEnv, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDraftConnectionCannotReplaceCredentialOwnedByAnotherRevision(t *testing.T) {
|
|
managementReference := "ssf-management-active"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
CredentialRef: "ssf-push-active", ManagementClientID: "active-machine",
|
|
ManagementCredentialRef: &managementReference, LifecycleStatus: "error", Version: 2,
|
|
IdempotencyKey: "identity-pairing-ssf-active",
|
|
}}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
"ssf-push-active": []byte("push-secret-value-for-active-000000000"),
|
|
"ssf-management-active": []byte("active-machine-secret-value-000000000"),
|
|
}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets, config: ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
}}
|
|
newSecret := []byte("draft-machine-secret-value-0000000000")
|
|
defer clear(newSecret)
|
|
|
|
_, err := manager.Connect(context.Background(), "https://auth.example/ssf", "draft-machine", newSecret, "identity-pairing-ssf-draft")
|
|
if !errors.Is(err, store.ErrSecurityEventConnectionConflict) {
|
|
t.Fatalf("draft credential replacement error=%v", err)
|
|
}
|
|
connection, getErr := repository.SecurityEventConnection(context.Background())
|
|
if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference || connection.ManagementClientID != "active-machine" {
|
|
t.Fatalf("active connection credential metadata changed: %#v err=%v", connection, getErr)
|
|
}
|
|
if !secrets.Has(managementReference) || secrets.Len() != 2 {
|
|
t.Fatal("active connection credential was deleted or a draft credential was retained")
|
|
}
|
|
}
|
|
|
|
func TestDraftConnectionReportsExistingRetirementAsResumable(t *testing.T) {
|
|
managementReference := "ssf-management-active"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
CredentialRef: "ssf-push-active", ManagementClientID: "active-machine",
|
|
ManagementCredentialRef: &managementReference, LifecycleStatus: "retiring", Version: 3,
|
|
IdempotencyKey: "identity-pairing-ssf-active",
|
|
}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
}}
|
|
secret := []byte("draft-machine-secret-value-0000000000")
|
|
defer clear(secret)
|
|
|
|
_, err := manager.Connect(context.Background(), "https://other-auth.example/ssf", "draft-machine", secret, "identity-pairing-ssf-draft")
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" {
|
|
t.Fatalf("retirement conflict was not classified as resumable: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRetiringConnectionRejectsSameOwnerReconnectBeforeReadingOrReplacingSecrets(t *testing.T) {
|
|
for _, status := range []string{"retiring", "disconnect_pending"} {
|
|
for _, credentialsProvided := range []bool{false, true} {
|
|
t.Run(fmt.Sprintf("%s/credentials=%t", status, credentialsProvided), func(t *testing.T) {
|
|
managementReference := "ssf-management-retiring"
|
|
owner := "identity-pairing-ssf-current"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
CredentialRef: "ssf-push-retiring", ManagementClientID: "machine-client",
|
|
ManagementCredentialRef: &managementReference, LifecycleStatus: status, Version: 3,
|
|
IdempotencyKey: owner,
|
|
}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
}}
|
|
var clientID string
|
|
var secret []byte
|
|
if credentialsProvided {
|
|
clientID = "machine-client"
|
|
secret = []byte("replacement-machine-secret-value-000000")
|
|
defer clear(secret)
|
|
}
|
|
|
|
_, err := manager.Connect(context.Background(), "https://auth.example/ssf", clientID, secret, owner)
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" {
|
|
t.Fatalf("same-owner reconnect during retirement error=%v", err)
|
|
}
|
|
connection, getErr := repository.SecurityEventConnection(context.Background())
|
|
if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference {
|
|
t.Fatalf("retiring connection credential metadata changed: %v", getErr)
|
|
}
|
|
repository.mutex.Lock()
|
|
queued := len(repository.cleanupDue)
|
|
repository.mutex.Unlock()
|
|
if queued != 0 {
|
|
t.Fatal("retiring reconnect staged or retired a Secret")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestConflictingConnectionRetirementNeverRetiresCurrentPairingOwner(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
oldOwner := "identity-pairing-ssf-active"
|
|
currentOwner := "identity-pairing-ssf-draft"
|
|
repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
CredentialRef: "ssf-push-active", ManagementClientID: "machine-client", LifecycleStatus: "enabled", Version: 3,
|
|
IdempotencyKey: oldOwner, UpdatedAt: time.Now().UTC(),
|
|
}}
|
|
manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
RetirementDuration: time.Hour,
|
|
}}
|
|
secret := []byte("replacement-machine-secret-value-000000")
|
|
defer clear(secret)
|
|
_, connectErr := manager.Connect(ctx, "https://auth.example/ssf", "machine-client", secret, currentOwner)
|
|
var categorized interface{ SafeErrorCategory() string }
|
|
if !errors.As(connectErr, &categorized) || categorized.SafeErrorCategory() != "connection_conflict" {
|
|
t.Fatalf("local-only retirement handoff error=%v", connectErr)
|
|
}
|
|
|
|
if err := manager.RetireConflictingConnection(ctx, currentOwner); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
connection, err := repository.SecurityEventConnection(ctx)
|
|
if err != nil || connection.LifecycleStatus != "retiring" {
|
|
t.Fatalf("old owner was not safely retired: connection=%#v err=%v", connection, err)
|
|
}
|
|
|
|
repository.connection.IdempotencyKey = currentOwner
|
|
repository.connection.LifecycleStatus = "enabled"
|
|
if err := manager.RetireConflictingConnection(ctx, currentOwner); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
connection, err = repository.SecurityEventConnection(ctx)
|
|
if err != nil || connection.LifecycleStatus != "enabled" {
|
|
t.Fatalf("current pairing connection was retired by a stale recovery action: connection=%#v err=%v", connection, err)
|
|
}
|
|
}
|
|
|
|
func TestDisconnectDoesNotReturnSuccessWhenLifecycleCASFails(t *testing.T) {
|
|
streamID := uuid.NewString()
|
|
current := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "://invalid", StreamID: &streamID,
|
|
CredentialRef: "ssf-push", LifecycleStatus: "verifying", Version: 4,
|
|
}
|
|
repository := &memoryConnectionRepository{connection: current}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{AppEnv: "test"}}
|
|
stale := *current
|
|
stale.LifecycleStatus = "enabled"
|
|
|
|
if _, err := manager.disconnect(context.Background(), stale); !errors.Is(err, store.ErrSecurityEventConnectionConflict) {
|
|
t.Fatalf("disconnect CAS conflict was reported as success: %v", err)
|
|
}
|
|
connection, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || connection.LifecycleStatus != "verifying" {
|
|
t.Fatalf("disconnect CAS conflict changed lifecycle: connection=%#v err=%v", connection, err)
|
|
}
|
|
}
|
|
|
|
func TestDisconnectTreatsRetirementStatesAsIdempotent(t *testing.T) {
|
|
for _, lifecycle := range []string{"disconnect_pending", "retiring"} {
|
|
t.Run(lifecycle, func(t *testing.T) {
|
|
connection := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-disconnect-idempotent",
|
|
LifecycleStatus: lifecycle, Version: 9,
|
|
}
|
|
repository := &memoryConnectionRepository{connection: connection}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}}
|
|
|
|
view, err := manager.disconnect(context.Background(), *connection)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, getErr := repository.SecurityEventConnection(context.Background())
|
|
if getErr != nil || current.LifecycleStatus != lifecycle || current.Version != 9 || view.Version != 9 {
|
|
t.Fatalf("idempotent disconnect changed retirement generation: lifecycle=%q version=%d view=%d err=%v",
|
|
current.LifecycleStatus, current.Version, view.Version, getErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTransmitterSSRFProtectionBlocksReservedNetworks(t *testing.T) {
|
|
for _, address := range []string{
|
|
"127.0.0.1", "10.0.0.1", "169.254.169.254", "100.64.0.1", "192.0.2.1", "198.51.100.1", "203.0.113.1", "2001:db8::1",
|
|
} {
|
|
if !blockedAddress(net.ParseIP(address)) {
|
|
t.Fatalf("reserved address %s was allowed", address)
|
|
}
|
|
}
|
|
if blockedAddress(net.ParseIP("8.8.8.8")) {
|
|
t.Fatal("public address was blocked")
|
|
}
|
|
}
|
|
|
|
func TestDialResolvedAddressesFallsBackFromIPv6ToIPv4(t *testing.T) {
|
|
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer listener.Close()
|
|
port := listener.Addr().(*net.TCPAddr).Port
|
|
accepted := make(chan struct{})
|
|
go func() {
|
|
connection, acceptErr := listener.Accept()
|
|
if acceptErr == nil {
|
|
_ = connection.Close()
|
|
close(accepted)
|
|
}
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
connection, err := dialResolvedAddresses(ctx, "tcp", strconv.Itoa(port), []net.IPAddr{
|
|
{IP: net.ParseIP("::1")},
|
|
{IP: net.ParseIP("127.0.0.1")},
|
|
}, &net.Dialer{Timeout: 250 * time.Millisecond})
|
|
if err != nil {
|
|
t.Fatalf("IPv4 fallback was not attempted: %v", err)
|
|
}
|
|
_ = connection.Close()
|
|
select {
|
|
case <-accepted:
|
|
case <-ctx.Done():
|
|
t.Fatal("IPv4 listener did not receive the fallback connection")
|
|
}
|
|
}
|
|
|
|
func TestRetiringConnectionStillAppliesRevocationWatermark(t *testing.T) {
|
|
audience := "urn:easyai:ssf:receiver:" + uuid.NewString()
|
|
repository := &memoryConnectionRepository{
|
|
evaluation: store.SecurityEventEvaluation{Mode: "introspection_fallback", Revoked: true, RequireIntrospection: true},
|
|
connection: &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", Audience: &audience,
|
|
LifecycleStatus: "retiring", Version: 1,
|
|
},
|
|
}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, config: ConnectionManagerConfig{StaleAfter: 3 * time.Minute}}
|
|
evaluation, err := manager.Evaluate(context.Background(), auth.OIDCSecurityEventIdentity{IssuedAt: time.Now().Add(-time.Minute)})
|
|
if err != nil || !evaluation.Enabled || !evaluation.Revoked || !evaluation.RequireIntrospection {
|
|
t.Fatalf("retiring evaluation=%#v err=%v", evaluation, err)
|
|
}
|
|
}
|
|
|
|
func TestRetirementDurablyQueuesReferencesBeforeRemovingConnection(t *testing.T) {
|
|
ctx := context.Background()
|
|
connection := store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-retiring", LifecycleStatus: "retiring",
|
|
Version: 1, UpdatedAt: time.Now().Add(-time.Minute),
|
|
}
|
|
repository := &memoryConnectionRepository{connection: &connection}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
connection.CredentialRef: []byte("push-secret-value-for-retirement-00000"),
|
|
}}
|
|
manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: ConnectionManagerConfig{RetirementDuration: time.Millisecond}}
|
|
|
|
manager.finishRetirement(connection)
|
|
|
|
if _, err := repository.SecurityEventConnection(context.Background()); !errors.Is(err, store.ErrSecurityEventConnectionNotFound) {
|
|
t.Fatalf("finalized retiring connection still exists: %v", err)
|
|
}
|
|
repository.mutex.Lock()
|
|
_, queued := repository.cleanupDue[connection.CredentialRef]
|
|
repository.mutex.Unlock()
|
|
if !queued {
|
|
t.Fatal("retirement removed the connection before durably queueing its Secret reference")
|
|
}
|
|
if !secrets.Has(connection.CredentialRef) {
|
|
t.Fatal("retirement bypassed the durable Secret cleanup worker")
|
|
}
|
|
}
|
|
|
|
type retireDuringSecretReadStore struct {
|
|
once sync.Once
|
|
onRetire func()
|
|
}
|
|
|
|
func (*retireDuringSecretReadStore) Put(context.Context, string, []byte) error { return nil }
|
|
func (s *retireDuringSecretReadStore) Get(context.Context, string) ([]byte, error) {
|
|
s.once.Do(s.onRetire)
|
|
return nil, ErrSecretNotFound
|
|
}
|
|
func (*retireDuringSecretReadStore) Delete(context.Context, string) error { return nil }
|
|
|
|
func retireMemoryConnection(repository *memoryConnectionRepository, connectionID string) {
|
|
repository.mutex.Lock()
|
|
defer repository.mutex.Unlock()
|
|
if repository.connection == nil || repository.connection.ConnectionID != connectionID {
|
|
return
|
|
}
|
|
repository.connection.LifecycleStatus = "retiring"
|
|
repository.connection.Version++
|
|
repository.connection.UpdatedAt = time.Now().UTC()
|
|
}
|
|
|
|
func TestConnectionManagerConstructionFailureCannotReviveConcurrentRetirement(t *testing.T) {
|
|
audience := "urn:easyai:ssf:receiver:" + uuid.NewString()
|
|
streamID := uuid.NewString()
|
|
connection := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
EndpointURL: "https://gateway.example/api/v1/security-events/ssf", Audience: &audience, StreamID: &streamID,
|
|
CredentialRef: "ssf-push-constructor-race", LifecycleStatus: "enabled", Version: 7,
|
|
}
|
|
initialVersion := connection.Version
|
|
repository := &memoryConnectionRepository{connection: connection}
|
|
secrets := &retireDuringSecretReadStore{onRetire: func() {
|
|
retireMemoryConnection(repository, connection.ConnectionID)
|
|
}}
|
|
|
|
if _, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
}, &Metrics{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || current.LifecycleStatus != "retiring" || current.Version != initialVersion+1 {
|
|
t.Fatalf("constructor failure revived concurrent retirement: lifecycle=%q version=%d err=%v",
|
|
current.LifecycleStatus, current.Version, err)
|
|
}
|
|
}
|
|
|
|
func TestConnectionOperationFailureCannotReviveConcurrentRetirement(t *testing.T) {
|
|
connection := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-operation-race",
|
|
LifecycleStatus: "verifying", Version: 11,
|
|
}
|
|
repository := &memoryConnectionRepository{connection: connection}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}}
|
|
staleOperationSnapshot := *connection
|
|
retireMemoryConnection(repository, connection.ConnectionID)
|
|
|
|
_ = manager.connectionError(context.Background(), staleOperationSnapshot, "discovery_failed", errors.New("remote operation failed"))
|
|
current, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || current.LifecycleStatus != "retiring" || current.Version != staleOperationSnapshot.Version+1 {
|
|
t.Fatalf("operation failure revived concurrent retirement: lifecycle=%q version=%d err=%v",
|
|
current.LifecycleStatus, current.Version, err)
|
|
}
|
|
}
|
|
|
|
func TestConnectionErrorCannotMoveCurrentRetirementGenerationBackToError(t *testing.T) {
|
|
connection := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-current-retirement",
|
|
LifecycleStatus: "retiring", Version: 5,
|
|
}
|
|
repository := &memoryConnectionRepository{connection: connection}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}}
|
|
|
|
_ = manager.connectionError(context.Background(), *connection, "discovery_failed", errors.New("late remote operation failed"))
|
|
current, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || current.LifecycleStatus != "retiring" || current.Version != connection.Version {
|
|
t.Fatalf("late operation failure revived current retirement generation: lifecycle=%q version=%d err=%v",
|
|
current.LifecycleStatus, current.Version, err)
|
|
}
|
|
}
|
|
|
|
func TestFinishRetirementDoesNotDeleteAConnectionRevivedAfterItsSnapshot(t *testing.T) {
|
|
connection := store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-retirement-generation",
|
|
LifecycleStatus: "retiring", Version: 3, UpdatedAt: time.Now().Add(-time.Minute),
|
|
}
|
|
staleRetirementSnapshot := connection
|
|
connection.LifecycleStatus = "enabled"
|
|
connection.Version++
|
|
repository := &memoryConnectionRepository{connection: &connection}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
connection.CredentialRef: []byte("push-secret-value-for-retirement-generation"),
|
|
}}
|
|
manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets,
|
|
config: ConnectionManagerConfig{RetirementDuration: time.Millisecond}}
|
|
|
|
manager.finishRetirement(staleRetirementSnapshot)
|
|
|
|
current, err := repository.SecurityEventConnection(context.Background())
|
|
if err != nil || current.LifecycleStatus != "enabled" || current.Version != connection.Version {
|
|
t.Fatalf("stale retirement deleted or changed the current connection: lifecycle=%q version=%d err=%v",
|
|
current.LifecycleStatus, current.Version, err)
|
|
}
|
|
if !secrets.Has(connection.CredentialRef) {
|
|
t.Fatal("stale retirement deleted a Secret referenced by the current connection")
|
|
}
|
|
}
|
|
|
|
func TestIdentitySecretCleanupWorkerResumesWithoutConnectionManager(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
reference := "ssf-management-staged-restart"
|
|
repository := &memoryConnectionRepository{cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}}
|
|
secrets := &memorySecretStore{values: map[string][]byte{
|
|
reference: []byte("staged-machine-secret-value-for-restart"),
|
|
}}
|
|
go RunIdentitySecretCleanupWorker(ctx, repository, secrets)
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
repository.mutex.Lock()
|
|
_, queued := repository.cleanupDue[reference]
|
|
repository.mutex.Unlock()
|
|
secretExists := secrets.Has(reference)
|
|
if !queued && !secretExists {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatal("queued identity Secret cleanup did not resume after restart")
|
|
}
|
|
|
|
func TestIdentitySecretCleanupWorkerKeepsClaimWhenDeleteIsUncertain(t *testing.T) {
|
|
reference := "ssf-management-worker-delete-uncertain"
|
|
repository := &memoryConnectionRepository{cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}}
|
|
secrets := &failedStagingSecretStore{deleteErr: errors.New("SecretStore delete result is uncertain")}
|
|
|
|
cleanupClaimedIdentitySecrets(context.Background(), repository, secrets)
|
|
|
|
repository.mutex.Lock()
|
|
_, pending := repository.cleanupDue[reference]
|
|
claimToken := repository.cleanupClaims[reference]
|
|
repository.mutex.Unlock()
|
|
if !pending || claimToken == "" {
|
|
t.Fatal("uncertain worker deletion released its durable claim")
|
|
}
|
|
}
|
|
|
|
func TestConnectionManagerDoesNotStartIdentitySecretCleanupWorker(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
reference := "ssf-management-manager-must-not-clean"
|
|
claimed := make(chan struct{}, 1)
|
|
repository := &memoryConnectionRepository{
|
|
cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)},
|
|
cleanupClaimed: claimed,
|
|
}
|
|
if _, err := NewConnectionManager(ctx, repository, &memorySecretStore{}, ConnectionManagerConfig{
|
|
OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
}, &Metrics{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-claimed:
|
|
t.Fatal("ConnectionManager started an identity Secret cleanup worker")
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
}
|
|
|
|
type failedStagingSecretStore struct {
|
|
deleteErr error
|
|
deletes atomic.Int64
|
|
}
|
|
|
|
func (*failedStagingSecretStore) Put(context.Context, string, []byte) error {
|
|
return errors.New("SecretStore write result is uncertain")
|
|
}
|
|
func (*failedStagingSecretStore) Get(context.Context, string) ([]byte, error) {
|
|
return nil, ErrSecretNotFound
|
|
}
|
|
func (s *failedStagingSecretStore) Delete(context.Context, string) error {
|
|
s.deletes.Add(1)
|
|
return s.deleteErr
|
|
}
|
|
|
|
func TestStageIdentitySecretKeepsPendingCleanupWhenPutAndDeleteAreUncertain(t *testing.T) {
|
|
reference := "ssf-management-uncertain-put"
|
|
repository := &memoryConnectionRepository{}
|
|
secrets := &failedStagingSecretStore{deleteErr: errors.New("SecretStore delete result is uncertain")}
|
|
manager := &ConnectionManager{repository: repository, secrets: secrets}
|
|
|
|
if err := manager.stageIdentitySecret(context.Background(), reference, []byte("machine-secret-value-for-uncertain-put")); err == nil {
|
|
t.Fatal("uncertain SecretStore Put unexpectedly succeeded")
|
|
}
|
|
if secrets.deletes.Load() != 1 {
|
|
t.Fatalf("cleanup delete attempts=%d want=1", secrets.deletes.Load())
|
|
}
|
|
repository.mutex.Lock()
|
|
_, pending := repository.cleanupDue[reference]
|
|
repository.mutex.Unlock()
|
|
if !pending {
|
|
t.Fatal("uncertain SecretStore result lost the durable cleanup intent")
|
|
}
|
|
}
|
|
|
|
func TestStageIdentitySecretRemovesPendingCleanupAfterConfirmedDelete(t *testing.T) {
|
|
reference := "ssf-management-confirmed-delete"
|
|
repository := &memoryConnectionRepository{}
|
|
secrets := &failedStagingSecretStore{}
|
|
manager := &ConnectionManager{repository: repository, secrets: secrets}
|
|
|
|
if err := manager.stageIdentitySecret(context.Background(), reference, []byte("machine-secret-value-for-confirmed-delete")); err == nil {
|
|
t.Fatal("failed SecretStore Put unexpectedly succeeded")
|
|
}
|
|
repository.mutex.Lock()
|
|
_, pending := repository.cleanupDue[reference]
|
|
repository.mutex.Unlock()
|
|
if pending {
|
|
t.Fatal("confirmed Secret deletion retained a stale cleanup intent")
|
|
}
|
|
}
|
|
|
|
type ambiguousAdoptionRepository struct {
|
|
*memoryConnectionRepository
|
|
failCreate bool
|
|
failManagement bool
|
|
failNext bool
|
|
}
|
|
|
|
func (r *ambiguousAdoptionRepository) CreateSecurityEventConnection(ctx context.Context, input store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error) {
|
|
connection, err := r.memoryConnectionRepository.CreateSecurityEventConnection(ctx, input)
|
|
if err == nil && r.failCreate {
|
|
return store.SecurityEventConnection{}, errors.New("database commit result is uncertain")
|
|
}
|
|
return connection, err
|
|
}
|
|
|
|
func (r *ambiguousAdoptionRepository) SetSecurityEventManagementCredential(ctx context.Context, connectionID, clientID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) {
|
|
connection, err := r.memoryConnectionRepository.SetSecurityEventManagementCredential(ctx, connectionID, clientID, reference, expectedVersion)
|
|
if err == nil && r.failManagement {
|
|
return store.SecurityEventConnection{}, errors.New("database commit result is uncertain")
|
|
}
|
|
return connection, err
|
|
}
|
|
|
|
func (r *ambiguousAdoptionRepository) SetSecurityEventNextCredential(ctx context.Context, connectionID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) {
|
|
connection, err := r.memoryConnectionRepository.SetSecurityEventNextCredential(ctx, connectionID, reference, expectedVersion)
|
|
if err == nil && r.failNext {
|
|
return store.SecurityEventConnection{}, errors.New("database commit result is uncertain")
|
|
}
|
|
return connection, err
|
|
}
|
|
|
|
func TestConnectDoesNotRequeueSecretsAfterAmbiguousAdoptionCommit(t *testing.T) {
|
|
ctx := context.Background()
|
|
base := &memoryConnectionRepository{}
|
|
repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failCreate: true}
|
|
secrets := &memorySecretStore{}
|
|
manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(),
|
|
PublicBaseURL: "http://localhost:18089",
|
|
}}
|
|
managementSecret := []byte("machine-secret-value-for-ambiguous-create")
|
|
defer clear(managementSecret)
|
|
|
|
if _, err := manager.Connect(ctx, "https://auth.example/ssf", "machine-client", managementSecret, "ambiguous-create"); err == nil {
|
|
t.Fatal("ambiguous connection commit unexpectedly reported success")
|
|
}
|
|
connection, err := base.SecurityEventConnection(ctx)
|
|
if err != nil || connection.ManagementCredentialRef == nil {
|
|
t.Fatalf("simulated committed connection was not retained: %v", err)
|
|
}
|
|
base.mutex.Lock()
|
|
_, pushQueued := base.cleanupDue[connection.CredentialRef]
|
|
_, managementQueued := base.cleanupDue[*connection.ManagementCredentialRef]
|
|
base.mutex.Unlock()
|
|
if pushQueued || managementQueued {
|
|
t.Fatal("ambiguous committed connection credentials were requeued for deletion")
|
|
}
|
|
if !secrets.Has(connection.CredentialRef) || !secrets.Has(*connection.ManagementCredentialRef) {
|
|
t.Fatal("ambiguous committed connection lost an active Secret")
|
|
}
|
|
}
|
|
|
|
func TestManagementCredentialDoesNotRequeueSecretAfterAmbiguousAdoptionCommit(t *testing.T) {
|
|
ctx := context.Background()
|
|
oldReference := "ssf-management-before-ambiguous-commit"
|
|
connection := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-active", ManagementCredentialRef: &oldReference,
|
|
ManagementClientID: "machine-client", LifecycleStatus: "enabled",
|
|
}
|
|
base := &memoryConnectionRepository{connection: connection}
|
|
repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failManagement: true}
|
|
secrets := &memorySecretStore{values: map[string][]byte{oldReference: []byte("old-machine-secret-value-before-commit")}}
|
|
manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets}
|
|
newSecret := []byte("new-machine-secret-value-after-commit")
|
|
defer clear(newSecret)
|
|
|
|
if _, err := manager.persistManagementCredential(ctx, *connection, "machine-client", newSecret); err == nil {
|
|
t.Fatal("ambiguous management credential commit unexpectedly reported success")
|
|
}
|
|
committed, err := base.SecurityEventConnection(ctx)
|
|
if err != nil || committed.ManagementCredentialRef == nil || *committed.ManagementCredentialRef == oldReference {
|
|
t.Fatalf("simulated committed management credential was not retained: %v", err)
|
|
}
|
|
newReference := *committed.ManagementCredentialRef
|
|
base.mutex.Lock()
|
|
_, newQueued := base.cleanupDue[newReference]
|
|
_, oldQueued := base.cleanupDue[oldReference]
|
|
base.mutex.Unlock()
|
|
if newQueued || !oldQueued {
|
|
t.Fatal("ambiguous commit queued the active credential or lost retirement of the old credential")
|
|
}
|
|
if !secrets.Has(newReference) {
|
|
t.Fatal("ambiguous committed management credential lost its Secret")
|
|
}
|
|
}
|
|
|
|
func TestNextCredentialDoesNotRequeueSecretAfterAmbiguousAdoptionCommit(t *testing.T) {
|
|
ctx := context.Background()
|
|
audience := "urn:easyai:ssf:receiver:" + uuid.NewString()
|
|
streamID := uuid.NewString()
|
|
connection := &store.SecurityEventConnection{
|
|
ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf",
|
|
EndpointURL: "https://gateway.example/api/v1/security-events/ssf", Audience: &audience, StreamID: &streamID,
|
|
CredentialRef: "ssf-push-active", LifecycleStatus: "enabled",
|
|
}
|
|
base := &memoryConnectionRepository{connection: connection}
|
|
repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failNext: true}
|
|
secrets := &memorySecretStore{}
|
|
manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets}
|
|
|
|
if _, err := manager.RotateCredential(ctx); err == nil {
|
|
t.Fatal("ambiguous next credential commit unexpectedly reported success")
|
|
}
|
|
committed, err := base.SecurityEventConnection(ctx)
|
|
if err != nil || committed.NextCredentialRef == nil {
|
|
t.Fatalf("simulated committed next credential was not retained: %v", err)
|
|
}
|
|
nextReference := *committed.NextCredentialRef
|
|
base.mutex.Lock()
|
|
_, queued := base.cleanupDue[nextReference]
|
|
base.mutex.Unlock()
|
|
if queued {
|
|
t.Fatal("ambiguous commit requeued the active next credential")
|
|
}
|
|
if !secrets.Has(nextReference) {
|
|
t.Fatal("ambiguous commit deleted the active next credential")
|
|
}
|
|
}
|
|
|
|
func (r *memoryConnectionRepository) PromoteSecurityEventCredential(context.Context, string, string) (store.SecurityEventConnection, error) {
|
|
return store.SecurityEventConnection{}, errors.New("not used")
|
|
}
|
|
func (r *memoryConnectionRepository) DiscardPreparedSecurityEventConnection(_ context.Context, id, ownerKey string, expectedVersion int64) error {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection == nil {
|
|
return store.ErrSecurityEventConnectionConflict
|
|
}
|
|
if r.beforeDiscard != nil {
|
|
r.beforeDiscard(r.connection)
|
|
}
|
|
if r.connection.ConnectionID != id || r.connection.IdempotencyKey != ownerKey ||
|
|
r.connection.Version != expectedVersion || r.connection.StreamID != nil {
|
|
return store.ErrSecurityEventConnectionConflict
|
|
}
|
|
if r.cleanupDue == nil {
|
|
r.cleanupDue = map[string]time.Time{}
|
|
}
|
|
references := connectionSecretReferences(*r.connection)
|
|
for _, reference := range references {
|
|
if r.cleanupClaims[reference] != "" {
|
|
return store.ErrIdentitySecretCleanupConflict
|
|
}
|
|
}
|
|
now := time.Now().UTC()
|
|
for _, reference := range references {
|
|
r.cleanupDue[reference] = now
|
|
}
|
|
r.connection = nil
|
|
return nil
|
|
}
|
|
func (r *memoryConnectionRepository) FinalizeRetiringSecurityEventConnection(_ context.Context, id string, expectedVersion int64) error {
|
|
r.mutex.Lock()
|
|
defer r.mutex.Unlock()
|
|
if r.connection == nil {
|
|
return store.ErrSecurityEventConnectionNotFound
|
|
}
|
|
if r.beforeFinalize != nil {
|
|
r.beforeFinalize(r.connection)
|
|
}
|
|
if r.connection.ConnectionID != id || r.connection.LifecycleStatus != "retiring" || r.connection.Version != expectedVersion {
|
|
return store.ErrSecurityEventConnectionConflict
|
|
}
|
|
if r.cleanupDue == nil {
|
|
r.cleanupDue = map[string]time.Time{}
|
|
}
|
|
for _, reference := range connectionSecretReferences(*r.connection) {
|
|
if r.cleanupClaims[reference] != "" {
|
|
return store.ErrIdentitySecretCleanupConflict
|
|
}
|
|
}
|
|
now := time.Now().UTC()
|
|
for _, reference := range connectionSecretReferences(*r.connection) {
|
|
r.cleanupDue[reference] = now
|
|
}
|
|
r.finalizeCalls++
|
|
r.connection = nil
|
|
return nil
|
|
}
|
|
func (r *memoryConnectionRepository) SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error) {
|
|
return r.mode, r.verifiedAt, nil
|
|
}
|
|
|
|
type memorySecretStore struct {
|
|
mutex sync.Mutex
|
|
values map[string][]byte
|
|
}
|
|
|
|
func (s *memorySecretStore) Put(_ context.Context, reference string, value []byte) error {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
if s.values == nil {
|
|
s.values = map[string][]byte{}
|
|
}
|
|
s.values[reference] = append([]byte(nil), value...)
|
|
return nil
|
|
}
|
|
func (s *memorySecretStore) Get(_ context.Context, reference string) ([]byte, error) {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
value, ok := s.values[reference]
|
|
if !ok {
|
|
return nil, ErrSecretNotFound
|
|
}
|
|
return append([]byte(nil), value...), nil
|
|
}
|
|
func (s *memorySecretStore) Delete(_ context.Context, reference string) error {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
delete(s.values, reference)
|
|
return nil
|
|
}
|
|
|
|
func (s *memorySecretStore) Has(reference string) bool {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
_, ok := s.values[reference]
|
|
return ok
|
|
}
|
|
|
|
func (s *memorySecretStore) Len() int {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
return len(s.values)
|
|
}
|
|
|
|
func TestConnectionManagerCreatesPausedStreamWithBackendGeneratedBearer(t *testing.T) {
|
|
repository := &memoryConnectionRepository{}
|
|
secrets := &memorySecretStore{}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
var transmitter *httptest.Server
|
|
verificationRequested := make(chan struct{}, 1)
|
|
var managementScopeRequested atomic.Bool
|
|
transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/.well-known/ssf-configuration/ssf":
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json",
|
|
"configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status",
|
|
"verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod},
|
|
})
|
|
case "/oidc/token":
|
|
clientID, secret, ok := r.BasicAuth()
|
|
if !ok || clientID != "gateway-machine" || secret != "machine-secret-for-test" {
|
|
t.Fatal("RFC 7662 machine client was not reused")
|
|
}
|
|
_ = r.ParseForm()
|
|
if strings.Contains(r.Form.Get("scope"), "ssf.stream.manage") {
|
|
managementScopeRequested.Store(true)
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"access_token": "machine-token", "token_type": "Bearer"})
|
|
case "/ssf/v1/stream":
|
|
if r.Method == http.MethodGet {
|
|
_, _ = w.Write([]byte(`[]`))
|
|
return
|
|
}
|
|
var request struct {
|
|
Delivery struct {
|
|
AuthorizationHeader string `json:"authorization_header"`
|
|
EndpointURL string `json:"endpoint_url"`
|
|
} `json:"delivery"`
|
|
Description string `json:"description"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&request)
|
|
if len(strings.TrimPrefix(request.Delivery.AuthorizationHeader, "Bearer ")) < 43 {
|
|
t.Fatal("generated Push Bearer has less than 256 bits")
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"stream_id": uuid.NewString(), "iss": transmitter.URL + "/ssf",
|
|
"aud": "urn:easyai:ssf:receiver:" + uuid.NewString(), "events_requested": []string{sessionRevokedEvent},
|
|
"delivery": map[string]any{"method": pushDeliveryMethod, "endpoint_url": request.Delivery.EndpointURL},
|
|
"description": request.Description,
|
|
})
|
|
case "/ssf/v1/verify":
|
|
w.WriteHeader(http.StatusNoContent)
|
|
select {
|
|
case verificationRequested <- struct{}{}:
|
|
default:
|
|
}
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
defer transmitter.Close()
|
|
manager, err := NewConnectionManager(ctx, repository, secrets, ConnectionManagerConfig{
|
|
AppEnv: "test", OIDCEnabled: true, OIDCIssuer: transmitter.URL + "/oidc", OIDCTenantID: uuid.NewString(),
|
|
ManagementClientID: "gateway-machine", ManagementClientSecret: "machine-secret-for-test",
|
|
PublicBaseURL: transmitter.URL, HeartbeatInterval: time.Hour, StaleAfter: 2 * time.Hour, HTTPClient: transmitter.Client(),
|
|
}, &Metrics{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
evaluation, err := manager.Evaluate(ctx, auth.OIDCSecurityEventIdentity{})
|
|
if err != nil || evaluation.Enabled {
|
|
t.Fatalf("unconfigured manager evaluation=%#v err=%v", evaluation, err)
|
|
}
|
|
view, err := manager.Connect(ctx, transmitter.URL+"/ssf", "", nil, "connect-once")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if view.LifecycleStatus != "verifying" || view.StreamID == nil || view.Audience == nil {
|
|
t.Fatalf("connection view=%#v", view)
|
|
}
|
|
encoded, _ := json.Marshal(view)
|
|
if strings.Contains(string(encoded), "credential") || strings.Contains(string(encoded), "machine-secret-for-test") || strings.Contains(string(encoded), "ssf-push-") {
|
|
t.Fatalf("secret metadata escaped in response: %s", encoded)
|
|
}
|
|
if secrets.Len() != 2 {
|
|
t.Fatalf("secret count=%d", secrets.Len())
|
|
}
|
|
restarted := &ConnectionManager{repository: repository, secrets: secrets, config: ConnectionManagerConfig{}}
|
|
restartedClientID, restartedSecret, err := restarted.IntrospectionCredential(ctx)
|
|
if err != nil || restartedClientID != "gateway-machine" || string(restartedSecret) != "machine-secret-for-test" {
|
|
t.Fatalf("persisted management credential was not recoverable after restart: client=%q secret_present=%v err=%v", restartedClientID, len(restartedSecret) > 0, err)
|
|
}
|
|
clear(restartedSecret)
|
|
if !managementScopeRequested.Load() {
|
|
t.Fatal("management scopes were not requested automatically")
|
|
}
|
|
select {
|
|
case <-verificationRequested:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("verification was not started without a restart")
|
|
}
|
|
}
|