diff --git a/apps/api/cmd/migrate/identity_pairing_start_reservation_migration_integration_test.go b/apps/api/cmd/migrate/identity_pairing_start_reservation_migration_integration_test.go new file mode 100644 index 0000000..7f68de3 --- /dev/null +++ b/apps/api/cmd/migrate/identity_pairing_start_reservation_migration_integration_test.go @@ -0,0 +1,254 @@ +package main + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const identityPairingStartReservationUpgradeMigration = "../../migrations/0073_identity_pairing_start_reservation_upgrade.sql" + +func TestIdentityPairingStartReservationUpgradeMigrationDefinesCurrentLifecycle(t *testing.T) { + payload, err := os.ReadFile(identityPairingStartReservationUpgradeMigration) + if err != nil { + t.Fatal(err) + } + content := strings.ToLower(string(payload)) + for _, required := range []string{ + "add column if not exists state text", + "add column if not exists revision_id uuid", + "add column if not exists updated_at timestamptz", + "alter column state set not null", + "gateway_identity_pairing_start_state_check", + "foreign key (revision_id)", + "idx_gateway_identity_pairing_start_revision_unique", + "idx_gateway_identity_pairing_start_expiry", + "canonical_pairing", + "on conflict (singleton) do update", + } { + if !strings.Contains(content, required) { + t.Fatalf("identity pairing start reservation upgrade migration is missing %q", required) + } + } + for _, forbidden := range []string{"exchange_token text", "client_secret", "machine_secret", "secret_value"} { + if strings.Contains(content, forbidden) { + t.Fatalf("identity pairing start reservation upgrade migration stores forbidden value field %q", forbidden) + } + } +} + +func TestIdentityPairingStartReservationUpgradeMigratesLegacyShapeAndIsIdempotent(t *testing.T) { + pool := newIdentityMigrationPostgresTestSchema(t) + ctx := context.Background() + applyIdentityPairingStartReservationPrerequisites(t, ctx, pool) + + revisionID, pairingID, pairingExpiresAt := seedOutstandingIdentityPairingForReservationMigration(t, ctx, pool) + legacyAttemptID := uuid.NewString() + if _, err := pool.Exec(ctx, ` +CREATE TABLE gateway_identity_pairing_start_reservation ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + attempt_id uuid NOT NULL UNIQUE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +)`); err != nil { + t.Fatalf("create legacy pairing start reservation: %v", err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_pairing_start_reservation(singleton,attempt_id,expires_at) +VALUES(true,$1::uuid,now()+interval '2 minutes')`, legacyAttemptID); err != nil { + t.Fatalf("seed legacy pairing start reservation: %v", err) + } + + applyIdentityMigrationTestFile(t, ctx, pool, identityPairingStartReservationUpgradeMigration) + assertCurrentIdentityPairingStartReservation(t, ctx, pool, pairingID, revisionID, pairingExpiresAt) + assertIdentityPairingStartReservationSchema(t, ctx, pool) + + // Migration runners apply each version once, but a repeat execution proves + // that recovery from an interrupted/manual rollout does not duplicate state. + applyIdentityMigrationTestFile(t, ctx, pool, identityPairingStartReservationUpgradeMigration) + assertCurrentIdentityPairingStartReservation(t, ctx, pool, pairingID, revisionID, pairingExpiresAt) + assertIdentityPairingStartReservationSchema(t, ctx, pool) +} + +func TestIdentityPairingStartReservationUpgradePreservesLegacyStartingLease(t *testing.T) { + pool := newIdentityMigrationPostgresTestSchema(t) + ctx := context.Background() + applyIdentityPairingStartReservationPrerequisites(t, ctx, pool) + + attemptID := uuid.NewString() + expiresAt := time.Now().UTC().Add(2 * time.Minute).Truncate(time.Microsecond) + if _, err := pool.Exec(ctx, ` +CREATE TABLE gateway_identity_pairing_start_reservation ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + attempt_id uuid NOT NULL UNIQUE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +)`); err != nil { + t.Fatalf("create legacy pairing start reservation: %v", err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_pairing_start_reservation(singleton,attempt_id,expires_at) +VALUES(true,$1::uuid,$2)`, attemptID, expiresAt); err != nil { + t.Fatalf("seed legacy starting reservation: %v", err) + } + + applyIdentityMigrationTestFile(t, ctx, pool, identityPairingStartReservationUpgradeMigration) + applyIdentityMigrationTestFile(t, ctx, pool, identityPairingStartReservationUpgradeMigration) + + var gotAttemptID, state string + var revisionID *string + var gotExpiresAt, updatedAt time.Time + if err := pool.QueryRow(ctx, ` +SELECT attempt_id::text,state,revision_id::text,expires_at,updated_at +FROM gateway_identity_pairing_start_reservation`). + Scan(&gotAttemptID, &state, &revisionID, &gotExpiresAt, &updatedAt); err != nil { + t.Fatalf("read upgraded legacy starting reservation: %v", err) + } + if gotAttemptID != attemptID || state != "starting" || revisionID != nil || + !gotExpiresAt.Equal(expiresAt) || updatedAt.IsZero() { + t.Fatalf("unexpected upgraded starting reservation attempt=%q state=%q revision=%v expires=%v updated=%v", + gotAttemptID, state, revisionID, gotExpiresAt, updatedAt) + } + + _, err := pool.Exec(ctx, `UPDATE gateway_identity_pairing_start_reservation SET state='paired'`) + requireIdentityPairingStartReservationMigrationCode(t, err, "23514", "paired reservation without revision") +} + +func TestIdentityPairingStartReservationUpgradeAcceptsFresh0071Schema(t *testing.T) { + pool := newIdentityMigrationPostgresTestSchema(t) + ctx := context.Background() + applyIdentityPairingStartReservationPrerequisites(t, ctx, pool) + + revisionID, pairingID, pairingExpiresAt := seedOutstandingIdentityPairingForReservationMigration(t, ctx, pool) + applyIdentityMigrationTestFile(t, ctx, pool, "../../migrations/0071_identity_pairing_start_reservation.sql") + assertCurrentIdentityPairingStartReservation(t, ctx, pool, pairingID, revisionID, pairingExpiresAt) + + applyIdentityMigrationTestFile(t, ctx, pool, identityPairingStartReservationUpgradeMigration) + applyIdentityMigrationTestFile(t, ctx, pool, identityPairingStartReservationUpgradeMigration) + assertCurrentIdentityPairingStartReservation(t, ctx, pool, pairingID, revisionID, pairingExpiresAt) + assertIdentityPairingStartReservationSchema(t, ctx, pool) +} + +func applyIdentityPairingStartReservationPrerequisites(t *testing.T, ctx context.Context, pool *pgxpool.Pool) { + t.Helper() + for _, migration := range []string{ + "../../migrations/0067_identity_configuration_revisions.sql", + "../../migrations/0068_identity_onboarding_exchanges.sql", + "../../migrations/0069_identity_pairing_cancellation.sql", + } { + applyIdentityMigrationTestFile(t, ctx, pool, migration) + } +} + +func seedOutstandingIdentityPairingForReservationMigration( + t *testing.T, + ctx context.Context, + pool *pgxpool.Pool, +) (string, string, time.Time) { + t.Helper() + revisionID := uuid.NewString() + pairingID := uuid.NewString() + expiresAt := time.Now().UTC().Add(30 * time.Minute).Truncate(time.Microsecond) + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url +) VALUES ($1::uuid,'draft','https://auth.test.example','default', + 'https://gateway.test.example','https://gateway-web.test.example')`, revisionID); err != nil { + t.Fatalf("seed identity revision for pairing reservation migration: %v", err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_onboarding_exchanges ( + id,revision_id,remote_exchange_id,exchange_token_ref,status,remote_version,expires_at +) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,'credentials_saved',4,$5)`, + pairingID, revisionID, uuid.NewString(), "identity-exchange-"+pairingID, expiresAt); err != nil { + t.Fatalf("seed identity exchange for pairing reservation migration: %v", err) + } + return revisionID, pairingID, expiresAt +} + +func assertCurrentIdentityPairingStartReservation( + t *testing.T, + ctx context.Context, + pool *pgxpool.Pool, + wantAttemptID string, + wantRevisionID string, + wantExpiresAt time.Time, +) { + t.Helper() + var count int + var attemptID, state, revisionID string + var expiresAt, updatedAt time.Time + if err := pool.QueryRow(ctx, ` +SELECT count(*) OVER (),attempt_id::text,state,revision_id::text,expires_at,updated_at +FROM gateway_identity_pairing_start_reservation`). + Scan(&count, &attemptID, &state, &revisionID, &expiresAt, &updatedAt); err != nil { + t.Fatalf("read upgraded pairing start reservation: %v", err) + } + if count != 1 || attemptID != wantAttemptID || state != "paired" || revisionID != wantRevisionID || + !expiresAt.Equal(wantExpiresAt) || updatedAt.IsZero() { + t.Fatalf("unexpected upgraded pairing reservation count=%d attempt=%q state=%q revision=%q expires=%v updated=%v", + count, attemptID, state, revisionID, expiresAt, updatedAt) + } +} + +func assertIdentityPairingStartReservationSchema(t *testing.T, ctx context.Context, pool *pgxpool.Pool) { + t.Helper() + for _, column := range []struct { + name string + wantDefaultFragment string + }{ + {name: "state", wantDefaultFragment: "starting"}, + {name: "updated_at", wantDefaultFragment: "now()"}, + } { + var nullable, defaultExpression string + if err := pool.QueryRow(ctx, ` +SELECT is_nullable,COALESCE(column_default,'') +FROM information_schema.columns +WHERE table_schema=current_schema() + AND table_name='gateway_identity_pairing_start_reservation' + AND column_name=$1`, column.name).Scan(&nullable, &defaultExpression); err != nil { + t.Fatalf("read upgraded pairing reservation column %s: %v", column.name, err) + } + if nullable != "NO" || !strings.Contains(defaultExpression, column.wantDefaultFragment) { + t.Fatalf("pairing reservation column %s nullable=%q default=%q", column.name, nullable, defaultExpression) + } + } + + var expiryIndex, revisionIndex string + if err := pool.QueryRow(ctx, `SELECT pg_get_indexdef('idx_gateway_identity_pairing_start_expiry'::regclass)`). + Scan(&expiryIndex); err != nil { + t.Fatalf("read pairing reservation expiry index: %v", err) + } + if err := pool.QueryRow(ctx, `SELECT pg_get_indexdef('idx_gateway_identity_pairing_start_revision_unique'::regclass)`). + Scan(&revisionIndex); err != nil { + t.Fatalf("read pairing reservation revision index: %v", err) + } + if !strings.Contains(strings.ToLower(expiryIndex), "(state, expires_at)") || + !strings.Contains(strings.ToLower(revisionIndex), "unique") || + !strings.Contains(strings.ToLower(revisionIndex), "(revision_id)") { + t.Fatalf("unexpected pairing reservation indexes expiry=%q revision=%q", expiryIndex, revisionIndex) + } + + _, err := pool.Exec(ctx, `UPDATE gateway_identity_pairing_start_reservation SET state='starting'`) + requireIdentityPairingStartReservationMigrationCode(t, err, "23514", "starting reservation with revision") + _, err = pool.Exec(ctx, `UPDATE gateway_identity_pairing_start_reservation SET revision_id=$1::uuid`, uuid.NewString()) + requireIdentityPairingStartReservationMigrationCode(t, err, "23503", "reservation with unknown revision") +} + +func requireIdentityPairingStartReservationMigrationCode(t *testing.T, err error, wantCode, operation string) { + t.Helper() + if err == nil { + t.Fatalf("%s unexpectedly satisfied migration constraints", operation) + } + var postgresError *pgconn.PgError + if !errors.As(err, &postgresError) || postgresError.Code != wantCode { + t.Fatalf("%s error=%v, want PostgreSQL code %s", operation, err, wantCode) + } +} diff --git a/apps/api/cmd/migrate/main_test.go b/apps/api/cmd/migrate/main_test.go index 9ccf709..868f967 100644 --- a/apps/api/cmd/migrate/main_test.go +++ b/apps/api/cmd/migrate/main_test.go @@ -1,9 +1,17 @@ package main import ( + "context" + "errors" "os" "strings" "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" ) func TestSecurityEventIdempotencyRepairMigrationExists(t *testing.T) { @@ -85,3 +93,276 @@ func TestIdentityOnboardingExchangeMigrationStoresOnlySecretReferences(t *testin } } } + +func TestIdentityPairingCancellationMigrationSeparatesCleanupLifecycle(t *testing.T) { + payload, err := os.ReadFile("../../migrations/0069_identity_pairing_cancellation.sql") + if err != nil { + t.Fatal(err) + } + content := strings.ToLower(string(payload)) + for _, required := range []string{ + "'cancelled'", + "cleanup_status text not null default 'none'", + "cleanup_completed_at", + "idx_gateway_identity_onboarding_cleanup", + "last_error_category", + "set last_error_category = 'pairing_step_failed'", + "set local lock_timeout = '10s'", + "set local statement_timeout = '60s'", + } { + if !strings.Contains(content, required) { + t.Fatalf("identity pairing cancellation migration is missing %q", required) + } + } + for _, forbidden := range []string{"exchange_token text", "client_secret", "machine_secret"} { + if strings.Contains(content, forbidden) { + t.Fatalf("identity pairing cancellation migration stores forbidden secret field %q", forbidden) + } + } +} + +func TestIdentityPairingErrorCategoryUpgradeMigrationExists(t *testing.T) { + payload, err := os.ReadFile("../../migrations/0074_identity_pairing_error_categories.sql") + if err != nil { + t.Fatal(err) + } + content := strings.ToLower(string(payload)) + for _, required := range []string{ + "drop constraint if exists gateway_identity_onboarding_error_category_check", + "security_event_credential_handoff_unsafe", + "security_event_connection_binding_missing", + "security_event_connection_binding_unavailable", + "security_event_connection_binding_invalid", + "security_event_connection_binding_mismatch", + "cleanup_security_event_credential_handoff_unsafe", + } { + if !strings.Contains(content, required) { + t.Fatalf("identity pairing error category migration is missing %q", required) + } + } +} + +func TestIdentityPairingErrorCategoryUpgradeMigrationExecutesAgainstCurrentSchema(t *testing.T) { + pool := newIdentityMigrationPostgresTestSchema(t) + ctx := context.Background() + for _, migration := range []string{ + "../../migrations/0067_identity_configuration_revisions.sql", + "../../migrations/0068_identity_onboarding_exchanges.sql", + "../../migrations/0069_identity_pairing_cancellation.sql", + "../../migrations/0070_identity_secret_cleanup_queue.sql", + "../../migrations/0071_identity_pairing_start_reservation.sql", + "../../migrations/0072_identity_secret_cleanup_claim_lifecycle.sql", + "../../migrations/0073_identity_pairing_start_reservation_upgrade.sql", + "../../migrations/0074_identity_pairing_error_categories.sql", + } { + applyIdentityMigrationTestFile(t, ctx, pool, migration) + } + + revisionID, pairingID := uuid.NewString(), uuid.NewString() + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url +) VALUES ($1::uuid,'draft','https://auth.test.example','default', + 'https://gateway.test.example','https://gateway-web.test.example')`, revisionID); err != nil { + t.Fatalf("seed identity revision: %v", err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_onboarding_exchanges ( + id,revision_id,remote_exchange_id,exchange_token_ref,status,remote_version,expires_at +) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,'credentials_saved',4,now() + interval '30 minutes')`, + pairingID, revisionID, uuid.NewString(), "identity-exchange-"+pairingID); err != nil { + t.Fatalf("seed onboarding exchange: %v", err) + } + + for _, category := range []string{ + "security_event_credential_handoff_unsafe", + "security_event_connection_binding_missing", + "security_event_connection_binding_unavailable", + "security_event_connection_binding_invalid", + "security_event_connection_binding_mismatch", + "cleanup_security_event_credential_handoff_unsafe", + } { + if _, err := pool.Exec(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET last_error_category=$2 WHERE id=$1::uuid`, pairingID, category); err != nil { + t.Fatalf("persist supported error category %q: %v", category, err) + } + } + _, err := pool.Exec(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET last_error_category='credential_secret_leaked' WHERE id=$1::uuid`, pairingID) + requireIdentityMigrationCheckViolation(t, err, "unknown upgraded identity pairing category") +} + +func TestIdentitySecretCleanupQueueMigrationStoresOnlyReferences(t *testing.T) { + payload, err := os.ReadFile("../../migrations/0070_identity_secret_cleanup_queue.sql") + if err != nil { + t.Fatal(err) + } + content := strings.ToLower(string(payload)) + for _, required := range []string{ + "create table if not exists gateway_identity_secret_cleanup_queue", + "secret_ref text primary key", + "not_before timestamptz not null", + "status text not null default 'pending'", + "claim_token uuid", + "lease_expires_at timestamptz", + "gateway_identity_secret_cleanup_claim_check", + "idx_gateway_identity_secret_cleanup_due", + } { + if !strings.Contains(content, required) { + t.Fatalf("identity Secret cleanup migration is missing %q", required) + } + } + for _, forbidden := range []string{"secret_value", "client_secret", "machine_secret", "token text"} { + if strings.Contains(content, forbidden) { + t.Fatalf("identity Secret cleanup migration stores forbidden value field %q", forbidden) + } + } +} + +func TestIdentityPairingCancellationMigrationExecutesAgainstLegacyData(t *testing.T) { + pool := newIdentityMigrationPostgresTestSchema(t) + ctx := context.Background() + applyIdentityMigrationTestFile(t, ctx, pool, "../../migrations/0067_identity_configuration_revisions.sql") + applyIdentityMigrationTestFile(t, ctx, pool, "../../migrations/0068_identity_onboarding_exchanges.sql") + + revisionID, pairingID := uuid.NewString(), uuid.NewString() + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url +) VALUES ($1::uuid,'draft','https://auth.test.example','default', + 'https://gateway.test.example','https://gateway-web.test.example')`, revisionID); err != nil { + t.Fatalf("seed legacy identity revision: %v", err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO gateway_identity_onboarding_exchanges ( + id,revision_id,remote_exchange_id,exchange_token_ref,status,remote_version,expires_at,last_error_category +) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,'credentials_saved',4,now() + interval '30 minutes','secret_token_abcdef')`, + pairingID, revisionID, uuid.NewString(), "identity-exchange-"+pairingID); err != nil { + t.Fatalf("seed legacy onboarding exchange: %v", err) + } + + applyIdentityMigrationTestFile(t, ctx, pool, "../../migrations/0069_identity_pairing_cancellation.sql") + + var status, cleanupStatus, errorCategory string + var cancelledAt, cleanupCompletedAt *time.Time + if err := pool.QueryRow(ctx, ` +SELECT status,cleanup_status,last_error_category,cancelled_at,cleanup_completed_at +FROM gateway_identity_onboarding_exchanges WHERE id=$1::uuid`, pairingID). + Scan(&status, &cleanupStatus, &errorCategory, &cancelledAt, &cleanupCompletedAt); err != nil { + t.Fatalf("read migrated onboarding exchange: %v", err) + } + if status != "credentials_saved" || cleanupStatus != "none" || errorCategory != "pairing_step_failed" || + cancelledAt != nil || cleanupCompletedAt != nil { + t.Fatalf("unexpected migrated exchange status=%q cleanup=%q error=%q cancelled=%v completed=%v", + status, cleanupStatus, errorCategory, cancelledAt, cleanupCompletedAt) + } + + _, err := pool.Exec(ctx, `UPDATE gateway_identity_onboarding_exchanges SET status='cancelled' WHERE id=$1::uuid`, pairingID) + requireIdentityMigrationCheckViolation(t, err, "cancelled status without cleanup intent") + + if _, err := pool.Exec(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET status='cancelled',cleanup_status='pending',cancelled_at=now() +WHERE id=$1::uuid`, pairingID); err != nil { + t.Fatalf("persist valid pending cleanup lifecycle: %v", err) + } + _, err = pool.Exec(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET cleanup_status='completed' WHERE id=$1::uuid`, pairingID) + requireIdentityMigrationCheckViolation(t, err, "completed cleanup without completion timestamp") + + if _, err := pool.Exec(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET cleanup_status='completed',cleanup_completed_at=now() +WHERE id=$1::uuid`, pairingID); err != nil { + t.Fatalf("persist valid completed cleanup lifecycle: %v", err) + } + _, err = pool.Exec(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET last_error_category='still-invalid' WHERE id=$1::uuid`, pairingID) + requireIdentityMigrationCheckViolation(t, err, "invalid post-migration error category") + + if err := pool.QueryRow(ctx, `SELECT status,cleanup_status FROM gateway_identity_onboarding_exchanges WHERE id=$1::uuid`, pairingID). + Scan(&status, &cleanupStatus); err != nil { + t.Fatalf("read completed cleanup lifecycle: %v", err) + } + if status != "cancelled" || cleanupStatus != "completed" { + t.Fatalf("completed cleanup lifecycle status=%q cleanup=%q", status, cleanupStatus) + } +} + +func applyIdentityMigrationTestFile(t *testing.T, ctx context.Context, pool *pgxpool.Pool, path string) { + t.Helper() + payload, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read migration %s: %v", path, err) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatalf("begin migration %s: %v", path, err) + } + if _, err := tx.Exec(ctx, string(payload)); err != nil { + _ = tx.Rollback(ctx) + t.Fatalf("execute migration %s: %v", path, err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatalf("commit migration %s: %v", path, err) + } +} + +func requireIdentityMigrationCheckViolation(t *testing.T, err error, operation string) { + t.Helper() + if err == nil { + t.Fatalf("%s unexpectedly satisfied migration constraints", operation) + } + var postgresError *pgconn.PgError + if !errors.As(err, &postgresError) || postgresError.Code != "23514" { + t.Fatalf("%s error=%v, want PostgreSQL check violation", operation, err) + } +} + +func newIdentityMigrationPostgresTestSchema(t *testing.T) *pgxpool.Pool { + t.Helper() + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run identity migration PostgreSQL integration tests") + } + ctx := context.Background() + admin, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect identity migration test database: %v", err) + } + var databaseName string + if err := admin.QueryRow(ctx, `SELECT current_database()`).Scan(&databaseName); err != nil { + admin.Close() + t.Fatalf("read identity migration test database name: %v", err) + } + if !strings.Contains(strings.ToLower(databaseName), "test") { + admin.Close() + t.Fatalf("refusing to use non-test database %q", databaseName) + } + + schemaName := "gateway_identity_migration_" + strings.ReplaceAll(uuid.NewString(), "-", "") + schemaIdentifier := pgx.Identifier{schemaName}.Sanitize() + if _, err := admin.Exec(ctx, `CREATE SCHEMA `+schemaIdentifier); err != nil { + admin.Close() + t.Fatalf("create identity migration test schema: %v", err) + } + config, err := pgxpool.ParseConfig(databaseURL) + if err != nil { + _, _ = admin.Exec(ctx, `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`) + admin.Close() + t.Fatalf("parse identity migration test database URL: %v", err) + } + config.ConnConfig.RuntimeParams["search_path"] = schemaName + pool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + _, _ = admin.Exec(ctx, `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`) + admin.Close() + t.Fatalf("connect identity migration test schema: %v", err) + } + t.Cleanup(func() { + pool.Close() + if _, err := admin.Exec(context.Background(), `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`); err != nil { + t.Errorf("drop identity migration test schema: %v", err) + } + admin.Close() + }) + return pool +} diff --git a/apps/api/docs/swagger.json b/apps/api/docs/swagger.json index 22c082c..11d7cd0 100644 --- a/apps/api/docs/swagger.json +++ b/apps/api/docs/swagger.json @@ -2918,7 +2918,7 @@ "BearerAuth": [] } ], - "description": "保留 Superseded Revision 供回滚，清理 BFF Session，并继续允许本地管理登录。", + "description": "将 Revision 转为只读 Superseded 审计历史，清理 BFF Session，并继续允许本地管理登录；重新接入必须使用新的接入码。", "produces": [ "application/json" ], @@ -3116,6 +3116,174 @@ } } }, + "/api/admin/system/identity/pairings/{pairingID}/cancel": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "原子封存未激活 Revision，并异步清理由该 Revision 拥有的临时 Secret 与 SSF 连接。不会撤销已经完成的远端 Exchange；下一次凭据交付会轮换机器凭据。", + "produces": [ + "application/json" + ], + "tags": [ + "identity" + ], + "summary": "放弃本地统一认证配对", + "parameters": [ + { + "type": "string", + "description": "配对 ID", + "name": "pairingID", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "幂等键", + "name": "Idempotency-Key", + "in": "header", + "required": true + }, + { + "type": "string", + "description": "当前 Pairing ETag", + "name": "If-Match", + "in": "header", + "required": true + } + ], + "responses": { + "202": { + "description": "Accepted", + "schema": { + "$ref": "#/definitions/identity.PairingExchange" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "412": { + "description": "Precondition Failed", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "428": { + "description": "Precondition Required", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + } + } + } + }, + "/api/admin/system/identity/pairings/{pairingID}/retire-conflicting-security-event": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "仅当指定 Pairing 仍因 owner 冲突停在 credentials_saved 时，退役不属于该 Revision 的旧连接；陈旧请求不会退役当前 Pairing 已创建的新连接。", + "produces": [ + "application/json" + ], + "tags": [ + "identity" + ], + "summary": "安全退役阻塞配对的旧 SSF 连接", + "parameters": [ + { + "type": "string", + "description": "配对 ID", + "name": "pairingID", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "幂等键", + "name": "Idempotency-Key", + "in": "header", + "required": true + }, + { + "type": "string", + "description": "当前 Pairing ETag", + "name": "If-Match", + "in": "header", + "required": true + } + ], + "responses": { + "202": { + "description": "Accepted", + "schema": { + "$ref": "#/definitions/identity.PairingExchange" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "412": { + "description": "Precondition Failed", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "428": { + "description": "Precondition Required", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + } + } + } + }, "/api/admin/system/identity/revisions/{revisionID}": { "patch": { "security": [ @@ -3290,13 +3458,14 @@ "BearerAuth": [] } ], + "description": "当前远端 OAuth/SSF 资源未版本化，接口会拒绝直接回滚并要求禁用后使用新接入码完成交接。", "produces": [ "application/json" ], "tags": [ "identity" ], - "summary": "回滚统一认证 Revision", + "summary": "请求恢复历史统一认证 Revision", "parameters": [ { "type": "string", @@ -4792,7 +4961,7 @@ }, "/api/v1/auth/login": { "post": { - "description": "使用用户名或邮箱登录本地账号，并返回 24 小时 JWT。", + "description": "使用用户名或邮箱登录本地账号，并返回 24 小时 JWT。非本地身份模式只允许本地应急 Manager 登录。", "consumes": [ "application/json" ], @@ -11492,12 +11661,34 @@ } } }, + "identity.PairingCleanupStatus": { + "type": "string", + "enum": [ + "none", + "pending", + "completed" + ], + "x-enum-varnames": [ + "PairingCleanupNone", + "PairingCleanupPending", + "PairingCleanupCompleted" + ] + }, "identity.PairingExchange": { "type": "object", "properties": { "authCenterAuditId": { "type": "string" }, + "cancelledAt": { + "type": "string" + }, + "cleanupCompletedAt": { + "type": "string" + }, + "cleanupStatus": { + "$ref": "#/definitions/identity.PairingCleanupStatus" + }, "createdAt": { "type": "string" }, @@ -11565,7 +11756,8 @@ "credentials_saved", "completed", "failed", - "expired" + "expired", + "cancelled" ], "x-enum-varnames": [ "PairingMetadataPending", @@ -11574,7 +11766,8 @@ "PairingCredentialsSaved", "PairingCompleted", "PairingFailed", - "PairingExpired" + "PairingExpired", + "PairingCancelled" ] }, "identity.Revision": { diff --git a/apps/api/docs/swagger.yaml b/apps/api/docs/swagger.yaml index 9ba8cf4..9f47de3 100644 --- a/apps/api/docs/swagger.yaml +++ b/apps/api/docs/swagger.yaml @@ -932,10 +932,26 @@ definitions: example: manual recharge type: string type: object + identity.PairingCleanupStatus: + enum: + - none + - pending + - completed + type: string + x-enum-varnames: + - PairingCleanupNone + - PairingCleanupPending + - PairingCleanupCompleted identity.PairingExchange: properties: authCenterAuditId: type: string + cancelledAt: + type: string + cleanupCompletedAt: + type: string + cleanupStatus: + $ref: '#/definitions/identity.PairingCleanupStatus' createdAt: type: string expiresAt: @@ -983,6 +999,7 @@ definitions: - completed - failed - expired + - cancelled type: string x-enum-varnames: - PairingMetadataPending @@ -992,6 +1009,7 @@ definitions: - PairingCompleted - PairingFailed - PairingExpired + - PairingCancelled identity.Revision: properties: activatedAt: @@ -4616,7 +4634,7 @@ paths: - identity /api/admin/system/identity/disable: post: - description: 保留 Superseded Revision 供回滚，清理 BFF Session，并继续允许本地管理登录。 + description: 将 Revision 转为只读 Superseded 审计历史，清理 BFF Session，并继续允许本地管理登录；重新接入必须使用新的接入码。 parameters: - description: 幂等键 in: header @@ -4746,6 +4764,118 @@ paths: summary: 查询统一认证配对状态 tags: - identity + /api/admin/system/identity/pairings/{pairingID}/cancel: + post: + description: 原子封存未激活 Revision，并异步清理由该 Revision 拥有的临时 Secret 与 SSF 连接。不会撤销已经完成的远端 + Exchange；下一次凭据交付会轮换机器凭据。 + parameters: + - description: 配对 ID + in: path + name: pairingID + required: true + type: string + - description: 幂等键 + in: header + name: Idempotency-Key + required: true + type: string + - description: 当前 Pairing ETag + in: header + name: If-Match + required: true + type: string + produces: + - application/json + responses: + "202": + description: Accepted + schema: + $ref: '#/definitions/identity.PairingExchange' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "404": + description: Not Found + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "409": + description: Conflict + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "412": + description: Precondition Failed + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "428": + description: Precondition Required + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + security: + - BearerAuth: [] + summary: 放弃本地统一认证配对 + tags: + - identity + /api/admin/system/identity/pairings/{pairingID}/retire-conflicting-security-event: + post: + description: 仅当指定 Pairing 仍因 owner 冲突停在 credentials_saved 时，退役不属于该 Revision + 的旧连接；陈旧请求不会退役当前 Pairing 已创建的新连接。 + parameters: + - description: 配对 ID + in: path + name: pairingID + required: true + type: string + - description: 幂等键 + in: header + name: Idempotency-Key + required: true + type: string + - description: 当前 Pairing ETag + in: header + name: If-Match + required: true + type: string + produces: + - application/json + responses: + "202": + description: Accepted + schema: + $ref: '#/definitions/identity.PairingExchange' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "404": + description: Not Found + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "409": + description: Conflict + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "412": + description: Precondition Failed + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "428": + description: Precondition Required + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + security: + - BearerAuth: [] + summary: 安全退役阻塞配对的旧 SSF 连接 + tags: + - identity /api/admin/system/identity/revisions/{revisionID}: patch: consumes: @@ -4857,6 +4987,7 @@ paths: - identity /api/admin/system/identity/revisions/{revisionID}/rollback: post: + description: 当前远端 OAuth/SSF 资源未版本化，接口会拒绝直接回滚并要求禁用后使用新接入码完成交接。 parameters: - description: Revision ID in: path @@ -4902,7 +5033,7 @@ paths: $ref: '#/definitions/httpapi.ErrorEnvelope' security: - BearerAuth: [] - summary: 回滚统一认证 Revision + summary: 请求恢复历史统一认证 Revision tags: - identity /api/admin/system/identity/revisions/{revisionID}/validate: @@ -5825,7 +5956,7 @@ paths: post: consumes: - application/json - description: 使用用户名或邮箱登录本地账号，并返回 24 小时 JWT。 + description: 使用用户名或邮箱登录本地账号，并返回 24 小时 JWT。非本地身份模式只允许本地应急 Manager 登录。 parameters: - description: 登录请求，account 可为用户名或邮箱 in: body diff --git a/apps/api/internal/auth/auth.go b/apps/api/internal/auth/auth.go index b401837..72640b5 100644 --- a/apps/api/internal/auth/auth.go +++ b/apps/api/internal/auth/auth.go @@ -22,7 +22,9 @@ import ( type Permission string const ( - OIDCSessionCookieName = "easyai_gateway_oidc_session" + OIDCSessionCookieName = "easyai_gateway_oidc_session" + localBreakGlassTokenPurpose = "local_break_glass_manager" + legacyAccessTokenPurpose = "legacy_access" PermissionPublic Permission = "public" PermissionBasic Permission = "basic" @@ -52,6 +54,7 @@ type User struct { TokenExpiresAt time.Time `json:"-"` TokenIssuedAt time.Time `json:"-"` Issuer string `json:"-"` + TokenPurpose string `json:"-"` } type contextKey string @@ -142,15 +145,24 @@ func (a *Authenticator) Require(permission Permission, next http.Handler) http.H } func (a *Authenticator) Authenticate(r *http.Request) (*User, error) { - token := extractBearer(r.Header.Get("Authorization")) - if token == "" { - token = strings.TrimSpace(r.Header.Get("x-comfy-api-key")) - } - if token == "" { - token = strings.TrimSpace(r.Header.Get("x-goog-api-key")) - } - if token == "" { - token = strings.TrimSpace(r.URL.Query().Get("key")) + var token string + if authorization := strings.TrimSpace(r.Header.Get("Authorization")); authorization != "" { + token = extractBearer(authorization) + if token == "" { + return nil, ErrUnauthorized + } + } else if value := strings.TrimSpace(r.Header.Get("x-comfy-api-key")); value != "" { + token = value + } else if value := strings.TrimSpace(r.Header.Get("x-goog-api-key")); value != "" { + token = value + } else if value := strings.TrimSpace(r.URL.Query().Get("key")); value != "" { + // Query credentials are retained only for API compatibility with + // providers that use `?key=sk-*`. Bearer/OIDC tokens in URLs would leak + // through browser history, reverse-proxy logs, and diagnostics. + if !strings.HasPrefix(value, "sk-") { + return nil, ErrUnauthorized + } + token = value } if token == "" { if cookie, err := r.Cookie(OIDCSessionCookieName); err == nil { @@ -175,10 +187,14 @@ func (a *Authenticator) Authenticate(r *http.Request) (*User, error) { if algorithm == "RS256" || algorithm == "ES256" { return a.AuthenticateOIDCAccessToken(r.Context(), token) } - if !a.legacyJWTEnabled() { - return nil, ErrUnauthorized + user, err := a.verifyJWT(token) + if err != nil { + return nil, err } - return a.verifyJWT(token) + if a.legacyJWTEnabled() || isLocalBreakGlassManager(user) { + return user, nil + } + return nil, ErrUnauthorized } func (a *Authenticator) AuthenticateOIDCAccessToken(ctx context.Context, token string) (*User, error) { @@ -233,6 +249,7 @@ func (a *Authenticator) verifyJWT(tokenString string) (*User, error) { APIKeyName: stringClaim(claims, "apiKeyName"), APIKeyPrefix: stringClaim(claims, "apiKeyPrefix"), APIKeyScopes: stringSliceClaim(claims, "apiKeyScopes"), + TokenPurpose: stringClaim(claims, "tokenPurpose"), } if user.Source == "" { user.Source = "gateway" @@ -248,6 +265,10 @@ func (a *Authenticator) SignJWT(user *User, ttl time.Duration) (string, error) { ttl = time.Hour } now := time.Now() + tokenPurpose := legacyAccessTokenPurpose + if user.Source == "gateway" && hasManagerRole(user.Roles) { + tokenPurpose = localBreakGlassTokenPurpose + } claims := jwt.MapClaims{ "sub": user.ID, "username": user.Username, @@ -264,6 +285,7 @@ func (a *Authenticator) SignJWT(user *User, ttl time.Duration) (string, error) { "apiKeyName": user.APIKeyName, "apiKeyPrefix": user.APIKeyPrefix, "apiKeyScopes": user.APIKeyScopes, + "tokenPurpose": tokenPurpose, "iat": now.Unix(), "exp": now.Add(ttl).Unix(), } @@ -271,6 +293,19 @@ func (a *Authenticator) SignJWT(user *User, ttl time.Duration) (string, error) { return token.SignedString([]byte(a.JWTSecret)) } +func isLocalBreakGlassManager(user *User) bool { + return user != nil && user.Source == "gateway" && user.TokenPurpose == localBreakGlassTokenPurpose && hasManagerRole(user.Roles) +} + +func hasManagerRole(roles []string) bool { + for _, role := range roles { + if role == "manager" || role == "admin" { + return true + } + } + return false +} + func (a *Authenticator) verifyAPIKey(ctx context.Context, apiKey string) (*User, error) { if a.LocalAPIKeyVerifier != nil { user, err := a.LocalAPIKeyVerifier(ctx, apiKey) diff --git a/apps/api/internal/auth/oidc.go b/apps/api/internal/auth/oidc.go index d86cd9c..eaca8c2 100644 --- a/apps/api/internal/auth/oidc.go +++ b/apps/api/internal/auth/oidc.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "math/big" + "net" "net/http" "net/url" "strings" @@ -25,6 +26,7 @@ const maxOIDCResponseBytes = 1 << 20 const defaultOIDCHTTPTimeout = 10 * time.Second type OIDCConfig struct { + AppEnv string Issuer string Audience string TenantID string @@ -90,7 +92,7 @@ func NewOIDCVerifier(config OIDCConfig) (*OIDCVerifier, error) { config.Audience = strings.TrimSpace(config.Audience) config.TenantID = strings.TrimSpace(config.TenantID) config.RolePrefix = strings.TrimSpace(config.RolePrefix) - if err := validatePublicURL(config.Issuer); err != nil || config.Audience == "" || config.TenantID == "" || config.RolePrefix == "" { + if err := validatePublicURL(config.Issuer, config.AppEnv); err != nil || config.Audience == "" || config.TenantID == "" || config.RolePrefix == "" { return nil, errors.New("issuer, audience, tenant and role prefix are required") } if config.IntrospectionEnabled && config.IntrospectionCredentialProvider == nil && @@ -262,10 +264,10 @@ func (v *OIDCVerifier) refresh(ctx context.Context, force bool) error { if err := v.fetchJSON(ctx, discoveryURL, &discovery); err != nil { return err } - if discovery.Issuer != v.config.Issuer || validatePublicURL(discovery.JWKSURI) != nil { + if discovery.Issuer != v.config.Issuer || validatePublicURL(discovery.JWKSURI, v.config.AppEnv) != nil { return errors.New("OIDC discovery metadata is invalid") } - if (v.config.IntrospectionEnabled || v.config.SecurityEventEvaluator != nil) && validatePublicURL(discovery.IntrospectionEndpoint) != nil { + if (v.config.IntrospectionEnabled || v.config.SecurityEventEvaluator != nil) && validatePublicURL(discovery.IntrospectionEndpoint, v.config.AppEnv) != nil { return errors.New("OIDC introspection metadata is invalid") } var set jsonWebKeySet @@ -405,7 +407,7 @@ func decodeBigInt(value string) (*big.Int, error) { return new(big.Int).SetBytes(payload), nil } -func validatePublicURL(raw string) error { +func validatePublicURL(raw, appEnv string) error { parsed, err := url.Parse(raw) if err != nil || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { return errors.New("invalid URL") @@ -413,12 +415,30 @@ func validatePublicURL(raw string) error { if parsed.Scheme == "https" { return nil } - if parsed.Scheme == "http" && (parsed.Hostname() == "127.0.0.1" || parsed.Hostname() == "localhost") { + if parsed.Scheme == "http" && isLocalOIDCEnvironment(appEnv) && isLoopbackOIDCHost(parsed.Hostname()) { return nil } return errors.New("URL must use HTTPS") } +func isLocalOIDCEnvironment(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "development", "dev", "local", "test": + return true + default: + return false + } +} + +func isLoopbackOIDCHost(value string) bool { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "localhost" { + return true + } + ip := net.ParseIP(value) + return ip != nil && ip.IsLoopback() +} + func numericDateClaim(value any) (time.Time, bool) { switch typed := value.(type) { case float64: diff --git a/apps/api/internal/auth/oidc_client.go b/apps/api/internal/auth/oidc_client.go index edd9128..5472adb 100644 --- a/apps/api/internal/auth/oidc_client.go +++ b/apps/api/internal/auth/oidc_client.go @@ -19,6 +19,7 @@ var ErrOIDCInvalidGrant = errors.New("OIDC refresh token is invalid") var errOIDCResponseTooLarge = errors.New("OIDC response exceeds size limit") type OIDCPublicClientConfig struct { + AppEnv string Issuer string ClientID string RedirectURI string @@ -64,7 +65,7 @@ func NewOIDCPublicClient(config OIDCPublicClientConfig) (*OIDCPublicClient, erro return nil, errors.New("offline_access is not allowed for Gateway browser sessions") } } - if validatePublicURL(config.Issuer) != nil || config.ClientID == "" || validatePublicURL(config.RedirectURI) != nil || validatePublicURL(config.PostLogoutRedirectURI) != nil { + if validatePublicURL(config.Issuer, config.AppEnv) != nil || config.ClientID == "" || validatePublicURL(config.RedirectURI, config.AppEnv) != nil || validatePublicURL(config.PostLogoutRedirectURI, config.AppEnv) != nil { return nil, errors.New("issuer, public client id and exact redirect URLs are required") } return &OIDCPublicClient{config: config, client: newOIDCHTTPClient(config.HTTPClient)}, nil @@ -220,9 +221,9 @@ func (c *OIDCPublicClient) configuration(ctx context.Context) (*oauth2.Config, o } var metadata oidcClientDiscovery if err := provider.Claims(&metadata); err != nil || metadata.Issuer != c.config.Issuer || - validatePublicURL(metadata.AuthorizationEndpoint) != nil || validatePublicURL(metadata.TokenEndpoint) != nil || validatePublicURL(metadata.JWKSURI) != nil || - metadata.RevocationEndpoint != "" && validatePublicURL(metadata.RevocationEndpoint) != nil || - metadata.EndSessionEndpoint != "" && validatePublicURL(metadata.EndSessionEndpoint) != nil { + validatePublicURL(metadata.AuthorizationEndpoint, c.config.AppEnv) != nil || validatePublicURL(metadata.TokenEndpoint, c.config.AppEnv) != nil || validatePublicURL(metadata.JWKSURI, c.config.AppEnv) != nil || + metadata.RevocationEndpoint != "" && validatePublicURL(metadata.RevocationEndpoint, c.config.AppEnv) != nil || + metadata.EndSessionEndpoint != "" && validatePublicURL(metadata.EndSessionEndpoint, c.config.AppEnv) != nil { return nil, oidcClientDiscovery{}, errors.New("OIDC discovery metadata is invalid") } endpoint := provider.Endpoint() diff --git a/apps/api/internal/auth/oidc_client_test.go b/apps/api/internal/auth/oidc_client_test.go index 19f16df..01faf9f 100644 --- a/apps/api/internal/auth/oidc_client_test.go +++ b/apps/api/internal/auth/oidc_client_test.go @@ -57,6 +57,7 @@ func TestOIDCPublicClientUsesAuthorizationCodePKCES256WithoutSecret(t *testing.T issuer = server.URL client, err := NewOIDCPublicClient(OIDCPublicClientConfig{ + AppEnv: "test", Issuer: issuer, ClientID: "gateway-public", RedirectURI: "https://gateway.example.com/gateway-api/api/v1/auth/oidc/callback", PostLogoutRedirectURI: "https://gateway.example.com/", Scopes: []string{"openid", "profile", "gateway.access"}, HTTPClient: server.Client(), }) @@ -84,6 +85,7 @@ func TestOIDCPublicClientUsesAuthorizationCodePKCES256WithoutSecret(t *testing.T func TestOIDCPublicClientRejectsOfflineAccess(t *testing.T) { _, err := NewOIDCPublicClient(OIDCPublicClientConfig{ + AppEnv: "production", Issuer: "https://auth.example.com", ClientID: "gateway-public", RedirectURI: "https://gateway.example.com/api/v1/auth/oidc/callback", PostLogoutRedirectURI: "https://gateway.example.com/", @@ -119,6 +121,7 @@ func TestOIDCPublicClientVerifiesIDTokenNonceAndAudience(t *testing.T) { issuer = server.URL client, err := NewOIDCPublicClient(OIDCPublicClientConfig{ + AppEnv: "test", Issuer: issuer, ClientID: "gateway-public-client", RedirectURI: "https://gateway.example.com/callback", PostLogoutRedirectURI: "https://gateway.example.com/", HTTPClient: server.Client(), }) @@ -195,6 +198,7 @@ func TestOIDCPublicClientRefreshAndRevokeNeverSendSecret(t *testing.T) { defer server.Close() issuer = server.URL client, err := NewOIDCPublicClient(OIDCPublicClientConfig{ + AppEnv: "test", Issuer: issuer, ClientID: "gateway-public", RedirectURI: "https://gateway.example.com/callback", PostLogoutRedirectURI: "https://gateway.example.com/", HTTPClient: server.Client(), }) @@ -234,6 +238,7 @@ func TestOIDCPublicClientMapsInvalidGrantWithoutLeakingProviderResponse(t *testi issuer = server.URL client, err := NewOIDCPublicClient(OIDCPublicClientConfig{ + AppEnv: "test", Issuer: issuer, ClientID: "gateway-public", RedirectURI: "https://gateway.example.com/callback", PostLogoutRedirectURI: "https://gateway.example.com/", HTTPClient: server.Client(), }) @@ -245,3 +250,43 @@ func TestOIDCPublicClientMapsInvalidGrantWithoutLeakingProviderResponse(t *testi t.Fatalf("invalid_grant mapping was not stable and redacted: %v", err) } } + +func TestOIDCPublicClientRejectsLoopbackHTTPDiscoveryEndpointsInProduction(t *testing.T) { + var issuer, insecureField string + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path != "/.well-known/openid-configuration" { + http.NotFound(w, r) + return + } + metadata := map[string]any{ + "issuer": issuer, "authorization_endpoint": issuer + "/authorize", + "token_endpoint": issuer + "/token", "jwks_uri": issuer + "/jwks", + "revocation_endpoint": issuer + "/revoke", "end_session_endpoint": issuer + "/logout", + "id_token_signing_alg_values_supported": []string{"RS256", "ES256"}, + } + metadata[insecureField] = "http://127.0.0.1:1/oidc-endpoint" + _ = json.NewEncoder(w).Encode(metadata) + })) + defer server.Close() + issuer = server.URL + + for _, field := range []string{ + "authorization_endpoint", "token_endpoint", "jwks_uri", "revocation_endpoint", "end_session_endpoint", + } { + t.Run(field, func(t *testing.T) { + insecureField = field + client, err := NewOIDCPublicClient(OIDCPublicClientConfig{ + AppEnv: "production", Issuer: issuer, ClientID: "gateway-public", + RedirectURI: "https://gateway.example.com/callback", PostLogoutRedirectURI: "https://gateway.example.com/", + HTTPClient: server.Client(), + }) + if err != nil { + t.Fatal(err) + } + if err := client.ValidateConfiguration(context.Background()); err == nil { + t.Fatalf("production accepted loopback HTTP %s", field) + } + }) + } +} diff --git a/apps/api/internal/auth/oidc_session_cookie_test.go b/apps/api/internal/auth/oidc_session_cookie_test.go index 8c596ac..4230501 100644 --- a/apps/api/internal/auth/oidc_session_cookie_test.go +++ b/apps/api/internal/auth/oidc_session_cookie_test.go @@ -2,10 +2,13 @@ package auth import ( "context" + "errors" "net/http" "net/http/httptest" "testing" "time" + + "github.com/golang-jwt/jwt/v5" ) func TestAuthenticateResolvesOpaqueOIDCSessionCookie(t *testing.T) { @@ -49,6 +52,47 @@ func TestAuthenticateBearerTakesPrecedenceOverOIDCSessionCookie(t *testing.T) { } } +func TestAuthenticateRejectsMalformedExplicitCredentialInsteadOfFallingBackToCookie(t *testing.T) { + authenticator := New("local-jwt-secret", "", "") + var resolved bool + authenticator.OIDCSessionResolver = func(context.Context, string) (*User, error) { + resolved = true + return &User{ID: "cookie-manager", Source: "gateway", Roles: []string{"manager"}}, nil + } + request := httptest.NewRequest(http.MethodPost, "/api/admin/system/identity/disable", nil) + request.Header.Set("Authorization", "not-a-bearer-credential") + request.AddCookie(&http.Cookie{Name: OIDCSessionCookieName, Value: "valid-cookie-session"}) + + if _, err := authenticator.Authenticate(request); !errors.Is(err, ErrUnauthorized) { + t.Fatalf("malformed explicit credential error=%v, want unauthorized", err) + } + if resolved { + t.Fatal("malformed Authorization header fell back to the OIDC session cookie") + } +} + +func TestAuthenticateRejectsManagerJWTInQueryWithoutCookieFallback(t *testing.T) { + authenticator := New("local-jwt-secret", "", "") + managerToken, err := authenticator.SignJWT(&User{ID: "manager", Source: "gateway", Roles: []string{"manager"}}, time.Hour) + if err != nil { + t.Fatal(err) + } + var resolved bool + authenticator.OIDCSessionResolver = func(context.Context, string) (*User, error) { + resolved = true + return &User{ID: "cookie-manager", Source: "gateway", Roles: []string{"manager"}}, nil + } + request := httptest.NewRequest(http.MethodPost, "/api/admin/system/identity/disable?key="+managerToken, nil) + request.AddCookie(&http.Cookie{Name: OIDCSessionCookieName, Value: "valid-cookie-session"}) + + if _, err := authenticator.Authenticate(request); !errors.Is(err, ErrUnauthorized) { + t.Fatalf("manager query JWT error=%v, want unauthorized", err) + } + if resolved { + t.Fatal("rejected query credential fell back to the OIDC session cookie") + } +} + func TestAuthenticateRejectsInvalidOIDCSessionCookie(t *testing.T) { authenticator := New("local-jwt-secret", "", "") request := httptest.NewRequest(http.MethodGet, "/api/v1/me", nil) @@ -81,6 +125,46 @@ func TestAuthenticatorReadsOIDCSessionResolverAndLegacyPolicyDynamically(t *test } } +func TestAuthenticateKeepsOnlySignedBreakGlassManagerWhenLegacyJWTDisabled(t *testing.T) { + authenticator := New("local-jwt-secret", "", "") + authenticator.LegacyJWTEnabledProvider = func() bool { return false } + + managerToken, err := authenticator.SignJWT(&User{ID: "manager", Source: "gateway", Roles: []string{"manager"}}, time.Hour) + if err != nil { + t.Fatal(err) + } + managerRequest := httptest.NewRequest(http.MethodGet, "/api/admin/system/identity/configuration", nil) + managerRequest.Header.Set("Authorization", "Bearer "+managerToken) + manager, err := authenticator.Authenticate(managerRequest) + if err != nil || manager.ID != "manager" { + t.Fatalf("signed break-glass manager was rejected: user=%#v err=%v", manager, err) + } + + userToken, err := authenticator.SignJWT(&User{ID: "user", Source: "gateway", Roles: []string{"user"}}, time.Hour) + if err != nil { + t.Fatal(err) + } + userRequest := httptest.NewRequest(http.MethodGet, "/api/v1/me", nil) + userRequest.Header.Set("Authorization", "Bearer "+userToken) + if _, err := authenticator.Authenticate(userRequest); !errors.Is(err, ErrUnauthorized) { + t.Fatalf("ordinary local JWT error = %v, want unauthorized", err) + } + + legacyManager := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "sub": "legacy-manager", "source": "gateway", "role": []string{"manager"}, + "iat": time.Now().Unix(), "exp": time.Now().Add(time.Hour).Unix(), + }) + legacyManagerToken, err := legacyManager.SignedString([]byte(authenticator.JWTSecret)) + if err != nil { + t.Fatal(err) + } + legacyRequest := httptest.NewRequest(http.MethodGet, "/api/admin/system/identity/configuration", nil) + legacyRequest.Header.Set("Authorization", "Bearer "+legacyManagerToken) + if _, err := authenticator.Authenticate(legacyRequest); !errors.Is(err, ErrUnauthorized) { + t.Fatalf("legacy manager without token purpose error = %v, want unauthorized", err) + } +} + func TestPublicRouteIgnoresExpiredOptionalOIDCSession(t *testing.T) { authenticator := New("local-jwt-secret", "", "") authenticator.OIDCSessionResolver = func(context.Context, string) (*User, error) { diff --git a/apps/api/internal/auth/oidc_test.go b/apps/api/internal/auth/oidc_test.go index 5a0f637..df7683d 100644 --- a/apps/api/internal/auth/oidc_test.go +++ b/apps/api/internal/auth/oidc_test.go @@ -43,6 +43,7 @@ func TestOIDCVerifierAcceptsRS256AndES256StableClaims(t *testing.T) { defer server.Close() issuer = server.URL verifier, err := NewOIDCVerifier(OIDCConfig{ + AppEnv: "test", Issuer: issuer, Audience: "gateway-api", TenantID: "tenant-1", RolePrefix: "gateway.", RequiredScopes: []string{"gateway.access"}, HTTPClient: server.Client(), }) @@ -85,6 +86,7 @@ func TestOIDCVerifierRejectsMissingOrMismatchedSecurityClaims(t *testing.T) { defer server.Close() issuer = server.URL verifier, _ := NewOIDCVerifier(OIDCConfig{ + AppEnv: "test", Issuer: issuer, Audience: "gateway-api", TenantID: "tenant-1", RolePrefix: "gateway.", RequiredScopes: []string{"gateway.access"}, HTTPClient: server.Client(), }) @@ -139,6 +141,7 @@ func TestOIDCVerifierFailsClosedWhenIntrospectionMarksSessionInactive(t *testing defer server.Close() issuer = server.URL verifier, err := NewOIDCVerifier(OIDCConfig{ + AppEnv: "test", Issuer: issuer, Audience: "gateway-api", TenantID: "tenant-1", RolePrefix: "gateway.", RequiredScopes: []string{"gateway.access"}, HTTPClient: server.Client(), IntrospectionEnabled: true, @@ -182,6 +185,7 @@ func TestOIDCSecurityEventEvaluationUsesWatermarkAndFallbackIntrospection(t *tes issuer = server.URL evaluation := OIDCSecurityEventEvaluation{RequireIntrospection: true} verifier, err := NewOIDCVerifier(OIDCConfig{ + AppEnv: "test", Issuer: issuer, Audience: "gateway-api", TenantID: "tenant-1", RolePrefix: "gateway.", RequiredScopes: []string{"gateway.access"}, HTTPClient: server.Client(), IntrospectionEnabled: true, @@ -227,6 +231,72 @@ func TestOIDCSecurityEventEvaluationUsesWatermarkAndFallbackIntrospection(t *tes } } +func TestOIDCVerifierRejectsLoopbackHTTPDiscoveryEndpointsInProduction(t *testing.T) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + var issuer, jwksURI, introspectionEndpoint string + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch request.URL.Path { + case "/.well-known/openid-configuration": + _ = json.NewEncoder(w).Encode(map[string]string{ + "issuer": issuer, "jwks_uri": jwksURI, "introspection_endpoint": introspectionEndpoint, + }) + case "/jwks": + _ = json.NewEncoder(w).Encode(map[string]any{"keys": []any{rsaJWK("rsa-key", &key.PublicKey)}}) + default: + http.NotFound(w, request) + } + })) + defer server.Close() + issuer = server.URL + + for _, test := range []struct { + name string + jwks string + introspection string + }{ + {name: "JWKS", jwks: "http://127.0.0.1:1/jwks"}, + {name: "introspection", jwks: issuer + "/jwks", introspection: "http://localhost:1/introspect"}, + } { + t.Run(test.name, func(t *testing.T) { + jwksURI, introspectionEndpoint = test.jwks, test.introspection + config := OIDCConfig{ + AppEnv: "production", Issuer: issuer, Audience: "gateway-api", TenantID: "tenant-1", RolePrefix: "gateway.", + HTTPClient: server.Client(), + } + if test.introspection != "" { + config.IntrospectionEnabled = true + config.IntrospectionCredentialProvider = func(context.Context) (string, []byte, error) { + return "gateway-machine", []byte("machine-secret-long-enough"), nil + } + } + verifier, createErr := NewOIDCVerifier(config) + if createErr != nil { + t.Fatal(createErr) + } + if validateErr := verifier.ValidateConfiguration(context.Background()); validateErr == nil { + t.Fatalf("production accepted loopback HTTP %s endpoint", test.name) + } + }) + } +} + +func TestOIDCURLPolicyAllowsLoopbackHTTPOnlyInLocalEnvironments(t *testing.T) { + for _, appEnv := range []string{"", "production", "staging"} { + if err := validatePublicURL("http://127.0.0.2:18003/issuer/easyai", appEnv); err == nil { + t.Fatalf("%s accepted loopback HTTP OIDC URL", appEnv) + } + } + for _, appEnv := range []string{"local", "development", "dev", "test"} { + if err := validatePublicURL("http://127.0.0.2:18003/issuer/easyai", appEnv); err != nil { + t.Fatalf("%s rejected loopback HTTP OIDC URL: %v", appEnv, err) + } + } +} + func signedOIDCToken(t *testing.T, issuer, kid string, method jwt.SigningMethod, key any, mutate func(jwt.MapClaims)) string { t.Helper() now := time.Now() diff --git a/apps/api/internal/httpapi/core_flow_integration_test.go b/apps/api/internal/httpapi/core_flow_integration_test.go index a2bea6c..7baeb75 100644 --- a/apps/api/internal/httpapi/core_flow_integration_test.go +++ b/apps/api/internal/httpapi/core_flow_integration_test.go @@ -73,6 +73,12 @@ func TestCoreLocalFlow(t *testing.T) { if registerResponse.AccessToken == "" { t.Fatal("register did not return access token") } + ordinaryUsername := "smoke_user_" + suffixText + doJSON(t, server.URL, http.MethodPost, "/api/v1/auth/register", "", map[string]any{ + "username": ordinaryUsername, + "email": ordinaryUsername + "@example.com", + "password": password, + }, http.StatusCreated, &struct{}{}) var duplicateResponse map[string]any doJSON(t, server.URL, http.MethodPost, "/api/v1/auth/register", "", map[string]any{ @@ -129,6 +135,25 @@ func TestCoreLocalFlow(t *testing.T) { if _, err := testPool.Exec(ctx, `UPDATE gateway_users SET roles = '["admin"]'::jsonb WHERE username = $1`, username); err != nil { t.Fatalf("promote smoke user: %v", err) } + serverMainCtx, cancelServerMain := context.WithCancel(ctx) + serverMain := httptest.NewServer(NewServerWithContext(serverMainCtx, config.Config{ + AppEnv: "test", HTTPAddr: ":0", DatabaseURL: databaseURL, IdentityMode: "server-main", + JWTSecret: "test-secret", CORSAllowedOrigin: "*", + }, db, slog.New(slog.NewTextHandler(io.Discard, nil)))) + var breakGlassLogin struct { + AccessToken string `json:"accessToken"` + } + doJSON(t, serverMain.URL, http.MethodPost, "/api/v1/auth/login", "", map[string]any{ + "account": username, "password": password, + }, http.StatusOK, &breakGlassLogin) + if breakGlassLogin.AccessToken == "" { + t.Fatal("server-main break-glass manager login did not return access token") + } + doJSON(t, serverMain.URL, http.MethodPost, "/api/v1/auth/login", "", map[string]any{ + "account": ordinaryUsername, "password": password, + }, http.StatusForbidden, &map[string]any{}) + serverMain.Close() + cancelServerMain() doJSON(t, server.URL, http.MethodPost, "/api/v1/auth/login", "", map[string]any{ "account": username, "password": password, @@ -1697,6 +1722,9 @@ func TestOriginAllowedSupportsCommaSeparatedOrigins(t *testing.T) { if originAllowed("http://127.0.0.1:5179", allowed) { t.Fatal("unexpected origin should not be allowed") } + if originAllowed("https://evil.example.com", "*") { + t.Fatal("credentialed wildcard CORS origin should not be allowed") + } } func assertRuntimeRecoveryReleasesPendingRateReservations(t *testing.T, ctx context.Context, db *store.Store) { diff --git a/apps/api/internal/httpapi/handlers.go b/apps/api/internal/httpapi/handlers.go index 7c0ee71..52e59ef 100644 --- a/apps/api/internal/httpapi/handlers.go +++ b/apps/api/internal/httpapi/handlers.go @@ -79,7 +79,7 @@ func (s *Server) me(w http.ResponseWriter, r *http.Request) { // @Failure 500 {object} ErrorEnvelope // @Router /api/v1/auth/register [post] func (s *Server) register(w http.ResponseWriter, r *http.Request) { - if !s.localIdentityEnabled() { + if !s.localIdentityEnabled() || !s.ordinaryLocalJWTEnabled() { writeError(w, http.StatusForbidden, "local registration is disabled") return } @@ -111,7 +111,7 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { // login godoc // @Summary 本地登录 -// @Description 使用用户名或邮箱登录本地账号，并返回 24 小时 JWT。 +// @Description 使用用户名或邮箱登录本地账号，并返回 24 小时 JWT。非本地身份模式只允许本地应急 Manager 登录。 // @Tags auth // @Accept json // @Produce json @@ -123,10 +123,6 @@ func (s *Server) register(w http.ResponseWriter, r *http.Request) { // @Failure 500 {object} ErrorEnvelope // @Router /api/v1/auth/login [post] func (s *Server) login(w http.ResponseWriter, r *http.Request) { - if !s.localIdentityEnabled() { - writeError(w, http.StatusForbidden, "local login is disabled") - return - } var input store.LocalLoginInput if err := json.NewDecoder(r.Body).Decode(&input); err != nil { writeError(w, http.StatusBadRequest, "invalid json body") @@ -142,6 +138,10 @@ func (s *Server) login(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "login failed") return } + if !s.localLoginAllowed(user) { + writeError(w, http.StatusForbidden, "local login is disabled except for break-glass managers") + return + } s.writeAuthResponse(w, http.StatusOK, user) } @@ -150,6 +150,19 @@ func (s *Server) localIdentityEnabled() bool { return mode == "" || mode == "standalone" || mode == "hybrid" } +func (s *Server) localLoginAllowed(user store.GatewayUser) bool { + for _, role := range user.Roles { + if role == "manager" || role == "admin" { + return true + } + } + return s.localIdentityEnabled() && s.ordinaryLocalJWTEnabled() +} + +func (s *Server) ordinaryLocalJWTEnabled() bool { + return s.identityRuntime == nil || s.identityRuntime.LegacyJWTEnabled() +} + func (s *Server) writeAuthResponse(w http.ResponseWriter, status int, user store.GatewayUser) { authUser := authUserFromGatewayUser(user) const ttl = 24 * time.Hour diff --git a/apps/api/internal/httpapi/identity_configuration_handlers.go b/apps/api/internal/httpapi/identity_configuration_handlers.go index 06d96ae..8adea1f 100644 --- a/apps/api/internal/httpapi/identity_configuration_handlers.go +++ b/apps/api/internal/httpapi/identity_configuration_handlers.go @@ -1,6 +1,7 @@ package httpapi import ( + "context" "crypto/sha256" "encoding/json" "errors" @@ -68,6 +69,11 @@ type identityWriteOperation struct { once sync.Once } +type identityPairingWorker struct { + cancel context.CancelFunc + done chan struct{} +} + func (operation *identityWriteOperation) close() { if operation != nil { operation.once.Do(operation.release) @@ -163,12 +169,15 @@ func (s *Server) startIdentityPairing(w http.ResponseWriter, r *http.Request) { } defer operation.close() traceID := ensureIdentityTraceID(w, r) + auditID, ok := s.requireIdentityConfigurationAudit(w, r, "pairing.start", "pending", traceID) + if !ok { + return + } pairing, err := s.identityPairing.Start(r.Context(), input, traceID) if err != nil { s.writeIdentityError(w, r, "pairing.start", "", traceID, err) return } - auditID := s.recordIdentityConfigurationAudit(r, "pairing.start", pairing.RevisionID, "accepted", traceID, "") pairing.AuthCenterAuditID = "" s.completeIdentityWrite(w, r, operation, http.StatusAccepted, pairing, pairing.Version, auditID) s.startIdentityPairingWorker(pairing.ID) @@ -196,6 +205,95 @@ func (s *Server) getIdentityPairing(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, pairing) } +// cancelIdentityPairing godoc +// @Summary 放弃本地统一认证配对 +// @Description 原子封存未激活 Revision，并异步清理由该 Revision 拥有的临时 Secret 与 SSF 连接。不会撤销已经完成的远端 Exchange；下一次凭据交付会轮换机器凭据。 +// @Tags identity +// @Produce json +// @Security BearerAuth +// @Param pairingID path string true "配对 ID" +// @Param Idempotency-Key header string true "幂等键" +// @Param If-Match header string true "当前 Pairing ETag" +// @Success 202 {object} identity.PairingExchange +// @Failure 401 {object} ErrorEnvelope +// @Failure 403 {object} ErrorEnvelope +// @Failure 404 {object} ErrorEnvelope +// @Failure 409 {object} ErrorEnvelope +// @Failure 412 {object} ErrorEnvelope +// @Failure 428 {object} ErrorEnvelope +// @Router /api/admin/system/identity/pairings/{pairingID}/cancel [post] +func (s *Server) cancelIdentityPairing(w http.ResponseWriter, r *http.Request) { + expectedVersion, ok := requiredIdentityVersion(w, r) + if !ok { + return + } + pairingID := r.PathValue("pairingID") + operation, ok := s.beginIdentityWriteWithVersion(w, r, "pairing.cancel", expectedVersion, struct { + PairingID string `json:"pairingId"` + }{PairingID: pairingID}) + if !ok { + return + } + defer operation.close() + traceID := ensureIdentityTraceID(w, r) + auditID, ok := s.requireIdentityConfigurationAudit(w, r, "pairing.cancel", pairingID, traceID) + if !ok { + return + } + pairing, err := s.identityPairing.Cancel(r.Context(), pairingID, expectedVersion, traceID, auditID) + if err != nil { + s.writeIdentityError(w, r, "pairing.cancel", pairingID, traceID, err) + return + } + done := s.stopIdentityPairingWorker(pairing.ID) + s.completeIdentityWrite(w, r, operation, http.StatusAccepted, pairing, pairing.Version, auditID) + s.startIdentityPairingCleanupWorker(pairing.ID, done) +} + +// retireIdentityPairingSecurityEventConflict godoc +// @Summary 安全退役阻塞配对的旧 SSF 连接 +// @Description 仅当指定 Pairing 仍因 owner 冲突停在 credentials_saved 时，退役不属于该 Revision 的旧连接；陈旧请求不会退役当前 Pairing 已创建的新连接。 +// @Tags identity +// @Produce json +// @Security BearerAuth +// @Param pairingID path string true "配对 ID" +// @Param Idempotency-Key header string true "幂等键" +// @Param If-Match header string true "当前 Pairing ETag" +// @Success 202 {object} identity.PairingExchange +// @Failure 401 {object} ErrorEnvelope +// @Failure 403 {object} ErrorEnvelope +// @Failure 404 {object} ErrorEnvelope +// @Failure 409 {object} ErrorEnvelope +// @Failure 412 {object} ErrorEnvelope +// @Failure 428 {object} ErrorEnvelope +// @Router /api/admin/system/identity/pairings/{pairingID}/retire-conflicting-security-event [post] +func (s *Server) retireIdentityPairingSecurityEventConflict(w http.ResponseWriter, r *http.Request) { + expectedVersion, ok := requiredIdentityVersion(w, r) + if !ok { + return + } + pairingID := r.PathValue("pairingID") + operation, ok := s.beginIdentityWriteWithVersion(w, r, "pairing.retire_security_event_conflict", expectedVersion, struct { + PairingID string `json:"pairingId"` + }{PairingID: pairingID}) + if !ok { + return + } + defer operation.close() + traceID := ensureIdentityTraceID(w, r) + auditID, ok := s.requireIdentityConfigurationAudit(w, r, "pairing.retire_security_event_conflict", pairingID, traceID) + if !ok { + return + } + pairing, err := s.identityPairing.RetireConflictingSecurityEvents(r.Context(), pairingID, expectedVersion) + if err != nil { + s.writeIdentityError(w, r, "pairing.retire_security_event_conflict", pairingID, traceID, err) + return + } + s.completeIdentityWrite(w, r, operation, http.StatusAccepted, pairing, pairing.Version, auditID) + s.startIdentityPairingWorker(pairing.ID) +} + // updateIdentityDraftPolicy godoc // @Summary 修改统一认证 Draft 策略 // @Tags identity @@ -238,13 +336,16 @@ func (s *Server) updateIdentityDraftPolicy(w http.ResponseWriter, r *http.Reques SessionAbsoluteSeconds: revision.SessionAbsoluteSeconds, SessionRefreshSeconds: revision.SessionRefreshSeconds, } applyIdentityPolicyPatch(&policy, patch) - updated, err := s.store.UpdateIdentityRevisionPolicy(r.Context(), revision.ID, expectedVersion, policy) - if err != nil { - s.writeIdentityError(w, r, "revision.policy", revision.ID, ensureIdentityTraceID(w, r), err) + traceID := ensureIdentityTraceID(w, r) + auditID, ok := s.requireIdentityConfigurationAudit(w, r, "revision.policy", revision.ID, traceID) + if !ok { + return + } + updated, err := s.store.UpdateIdentityRevisionPolicy(r.Context(), revision.ID, expectedVersion, policy) + if err != nil { + s.writeIdentityError(w, r, "revision.policy", revision.ID, traceID, err) return } - traceID := ensureIdentityTraceID(w, r) - auditID := s.recordIdentityConfigurationAudit(r, "revision.policy", revision.ID, "success", traceID, "") s.completeIdentityWrite(w, r, operation, http.StatusOK, updated, updated.Version, auditID) } @@ -292,7 +393,8 @@ func (s *Server) activateIdentityRevision(w http.ResponseWriter, r *http.Request } // rollbackIdentityRevision godoc -// @Summary 回滚统一认证 Revision +// @Summary 请求恢复历史统一认证 Revision +// @Description 当前远端 OAuth/SSF 资源未版本化，接口会拒绝直接回滚并要求禁用后使用新接入码完成交接。 // @Tags identity // @Produce json // @Security BearerAuth @@ -314,7 +416,7 @@ func (s *Server) rollbackIdentityRevision(w http.ResponseWriter, r *http.Request // disableIdentityConfiguration godoc // @Summary 禁用统一认证 -// @Description 保留 Superseded Revision 供回滚，清理 BFF Session，并继续允许本地管理登录。 +// @Description 将 Revision 转为只读 Superseded 审计历史，清理 BFF Session，并继续允许本地管理登录；重新接入必须使用新的接入码。 // @Tags identity // @Produce json // @Security BearerAuth @@ -338,7 +440,10 @@ func (s *Server) disableIdentityConfiguration(w http.ResponseWriter, r *http.Req } defer operation.close() traceID := ensureIdentityTraceID(w, r) - auditID := s.recordIdentityConfigurationAudit(r, "revision.disable", "active", "requested", traceID, "") + auditID, ok := s.requireIdentityConfigurationAudit(w, r, "revision.disable", "active", traceID) + if !ok { + return + } disabled, err := s.identityRuntime.Disable(r.Context(), expectedVersion, traceID, auditID) if err != nil { s.writeIdentityError(w, r, "revision.disable", "active", traceID, err) @@ -358,7 +463,10 @@ func (s *Server) runIdentityRevisionAction(w http.ResponseWriter, r *http.Reques } defer operation.close() traceID := ensureIdentityTraceID(w, r) - auditID := s.recordIdentityConfigurationAudit(r, action, r.PathValue("revisionID"), "requested", traceID, "") + auditID, ok := s.requireIdentityConfigurationAudit(w, r, action, r.PathValue("revisionID"), traceID) + if !ok { + return + } revision, err := run(expectedVersion, traceID, auditID) if err != nil { s.writeIdentityError(w, r, action, r.PathValue("revisionID"), traceID, err) @@ -425,9 +533,7 @@ func (s *Server) beginIdentityWriteWithVersion(w http.ResponseWriter, r *http.Re writeError(w, http.StatusBadRequest, "Idempotency-Key is required", "IDEMPOTENCY_KEY_REQUIRED") return nil, false } - encoded, _ := json.Marshal(request) - digest := sha256.Sum256(append([]byte(fmt.Sprintf("%s\x00%d\x00", operation, version)), encoded...)) - requestHash := fmt.Sprintf("%x", digest[:]) + requestHash := identityRequestHash(operation, version, request) s.identityManagementMu.Lock() write := &identityWriteOperation{operation: operation, key: key, requestHash: requestHash, release: s.identityManagementMu.Unlock} if s.store == nil { @@ -468,6 +574,12 @@ func (s *Server) beginIdentityWriteWithVersion(w http.ResponseWriter, r *http.Re return nil, false } +func identityRequestHash(operation string, version int64, request any) string { + encoded, _ := json.Marshal(request) + digest := sha256.Sum256(append([]byte(fmt.Sprintf("%s\x00%d\x00", operation, version)), encoded...)) + return fmt.Sprintf("%x", digest[:]) +} + func (s *Server) completeIdentityWrite(w http.ResponseWriter, r *http.Request, operation *identityWriteOperation, status int, payload any, version int64, auditID string) { body, _ := json.Marshal(payload) stored, _ := json.Marshal(storedIdentityResponse{Status: status, Body: body, ETag: identityETag(version), AuditID: auditID}) @@ -541,6 +653,11 @@ func (s *Server) writeIdentityError(w http.ResponseWriter, r *http.Request, acti } func identityErrorProjection(err error) (int, string, string) { + var categorized interface{ SafeErrorCategory() string } + safeCategory := "" + if errors.As(err, &categorized) { + safeCategory = categorized.SafeErrorCategory() + } switch { case errors.Is(err, identity.ErrRevisionNotFound): return http.StatusNotFound, "统一认证配置不存在", "IDENTITY_CONFIGURATION_NOT_FOUND" @@ -550,6 +667,20 @@ func identityErrorProjection(err error) (int, string, string) { return http.StatusConflict, "请先保留至少一个可用的本地应急管理员凭据", "BREAK_GLASS_MANAGER_REQUIRED" case errors.Is(err, identity.ErrLocalTenantInvalid): return http.StatusConflict, "本地租户映射无效", "IDENTITY_LOCAL_TENANT_INVALID" + case errors.Is(err, identity.ErrPairingInProgress): + return http.StatusConflict, "请先完成或放弃当前统一认证配对", "IDENTITY_PAIRING_IN_PROGRESS" + case errors.Is(err, identity.ErrActiveConfigurationHandoffRequired): + return http.StatusConflict, "当前仍有 Active 统一认证配置；请先禁用，再使用新接入码配对", "IDENTITY_ACTIVE_CONFIGURATION_HANDOFF_REQUIRED" + case errors.Is(err, identity.ErrRollbackConfigurationHandoffRequired): + return http.StatusConflict, "旧版本关联的远端 OAuth/SSF 资源可能已变化；请禁用后使用新接入码恢复", "IDENTITY_ROLLBACK_CONFIGURATION_HANDOFF_REQUIRED" + case errors.Is(err, identity.ErrSecurityEventRetirementPending): + return http.StatusConflict, "旧安全事件 Stream 尚未完成断开；系统会继续重试，请稍后再次禁用", "IDENTITY_SECURITY_EVENT_RETIREMENT_PENDING" + case errors.Is(err, identity.ErrPairingNotCancellable): + return http.StatusConflict, "当前统一认证配置不能放弃", "IDENTITY_PAIRING_NOT_CANCELLABLE" + case errors.Is(err, identity.ErrPairingConflictNotResolvable): + return http.StatusConflict, "当前配对已不需要退役旧安全事件连接，请刷新状态", "IDENTITY_PAIRING_CONFLICT_NOT_RESOLVABLE" + case safeCategory == "credential_handoff_unsafe": + return http.StatusConflict, "旧安全事件连接与本次认证中心不匹配，系统已拒绝发送凭据；请先在原配置下断开旧连接", "IDENTITY_SECURITY_EVENT_HANDOFF_UNSAFE" case err != nil && (strings.Contains(err.Error(), "invalid") || strings.Contains(err.Error(), "required")): return http.StatusBadRequest, "统一认证配置无效", "IDENTITY_CONFIGURATION_INVALID" default: @@ -583,23 +714,97 @@ func (s *Server) recordIdentityConfigurationAudit(r *http.Request, action, targe return audit.ID } +func (s *Server) requireIdentityConfigurationAudit(w http.ResponseWriter, r *http.Request, action, targetID, traceID string) (string, bool) { + auditID := s.recordIdentityConfigurationAudit(r, action, targetID, "requested", traceID, "") + if auditID == "" { + writeError(w, http.StatusServiceUnavailable, "统一认证审计暂时不可用，操作未执行", "IDENTITY_AUDIT_UNAVAILABLE") + return "", false + } + w.Header().Set("X-Audit-Id", auditID) + return auditID, true +} + func (s *Server) startIdentityPairingWorker(pairingID string) { - if _, loaded := s.identityPairingWorkers.LoadOrStore(pairingID, struct{}{}); loaded { + workerContext, cancel := context.WithCancel(s.ctx) + worker := &identityPairingWorker{cancel: cancel, done: make(chan struct{})} + if _, loaded := s.identityPairingWorkers.LoadOrStore(pairingID, worker); loaded { + cancel() return } go func() { - defer s.identityPairingWorkers.Delete(pairingID) + defer func() { + s.identityPairingWorkers.CompareAndDelete(pairingID, worker) + close(worker.done) + }() delay := time.Second for { - pairing, err := s.identityPairing.Continue(s.ctx, pairingID) + pairing, err := s.identityPairing.Continue(workerContext, pairingID) if err == nil { - if pairing.Status == identity.PairingCompleted || pairing.Status == identity.PairingFailed || pairing.Status == identity.PairingExpired { + if pairing.Status == identity.PairingCompleted || pairing.Status == identity.PairingFailed || pairing.Status == identity.PairingExpired || pairing.Status == identity.PairingCancelled { return } delay = time.Second } else { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } if s.logger != nil { - s.logger.Warn("identity pairing step failed and will retry", "pairing_id", pairingID, "error_category", "pairing_step_failed") + s.logger.Warn("identity pairing step failed and will retry", "pairing_id", pairingID, "error_category", firstNonEmptyText(pairing.LastErrorCategory, "pairing_step_failed")) + } + if delay < 15*time.Second { + delay *= 2 + } + } + select { + case <-workerContext.Done(): + return + case <-time.After(delay): + } + } + }() +} + +func (s *Server) stopIdentityPairingWorker(pairingID string) <-chan struct{} { + if value, ok := s.identityPairingWorkers.Load(pairingID); ok { + worker := value.(*identityPairingWorker) + worker.cancel() + return worker.done + } + done := make(chan struct{}) + close(done) + return done +} + +func (s *Server) startIdentityPairingCleanupWorker(pairingID string, processingDone <-chan struct{}) { + if processingDone == nil { + processingDone = s.stopIdentityPairingWorker(pairingID) + } + if _, loaded := s.identityCleanupWorkers.LoadOrStore(pairingID, struct{}{}); loaded { + return + } + go func() { + defer s.identityCleanupWorkers.Delete(pairingID) + if processingDone != nil { + select { + case <-s.ctx.Done(): + return + case <-processingDone: + } + } + delay := time.Second + for { + pairing, err := s.identityPairing.Cleanup(s.ctx, pairingID) + if err == nil { + if pairing.CleanupStatus == identity.PairingCleanupCompleted { + return + } + delay = time.Second + } else { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, identity.ErrPairingNotCancellable) { + return + } + if s.logger != nil { + s.logger.Warn("identity pairing cleanup failed and will retry", "pairing_id", pairingID, "error_category", firstNonEmptyText(pairing.LastErrorCategory, "pairing_cleanup_failed")) } if delay < 15*time.Second { delay *= 2 diff --git a/apps/api/internal/httpapi/identity_configuration_handlers_test.go b/apps/api/internal/httpapi/identity_configuration_handlers_test.go index 4dc8a23..d616f23 100644 --- a/apps/api/internal/httpapi/identity_configuration_handlers_test.go +++ b/apps/api/internal/httpapi/identity_configuration_handlers_test.go @@ -1,9 +1,12 @@ package httpapi import ( + "errors" "net/http" "net/http/httptest" "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" ) func TestRequiredIdentityWriteHeaders(t *testing.T) { @@ -28,6 +31,46 @@ func TestIdentityErrorProjectionProtectsInternalDetails(t *testing.T) { } } +func TestIdentityPairingRecoveryErrorsUseStableConflictResponses(t *testing.T) { + for _, test := range []struct { + err error + code string + }{ + {err: identity.ErrPairingInProgress, code: "IDENTITY_PAIRING_IN_PROGRESS"}, + {err: identity.ErrPairingNotCancellable, code: "IDENTITY_PAIRING_NOT_CANCELLABLE"}, + {err: identity.ErrPairingConflictNotResolvable, code: "IDENTITY_PAIRING_CONFLICT_NOT_RESOLVABLE"}, + } { + status, message, code := identityErrorProjection(test.err) + if status != http.StatusConflict || code != test.code || message == "" || errors.Is(assertionError(message), test.err) { + t.Fatalf("err=%v status=%d code=%q message=%q", test.err, status, code, message) + } + } +} + +func TestIdentityWriteHashScopesPairingCancellationToTarget(t *testing.T) { + type cancellationRequest struct { + PairingID string `json:"pairingId"` + } + first := identityRequestHash("pairing.cancel", 7, cancellationRequest{PairingID: "pairing-a"}) + second := identityRequestHash("pairing.cancel", 7, cancellationRequest{PairingID: "pairing-b"}) + if first == second { + t.Fatal("pairing cancellation idempotency hash must include the target pairing") + } +} + +func TestIdentityWriteAuditFailsClosedBeforeMutation(t *testing.T) { + server := &Server{} + request := httptest.NewRequest(http.MethodPost, "/api/admin/system/identity/disable", nil) + recorder := httptest.NewRecorder() + + if _, ok := server.requireIdentityConfigurationAudit(recorder, request, "revision.disable", "active", "trace-test"); ok { + t.Fatal("identity write unexpectedly continued without durable audit storage") + } + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("audit failure status=%d, want 503", recorder.Code) + } +} + type assertionError string func (err assertionError) Error() string { return string(err) } diff --git a/apps/api/internal/httpapi/identity_pairing_coordinator.go b/apps/api/internal/httpapi/identity_pairing_coordinator.go new file mode 100644 index 0000000..3dfd16c --- /dev/null +++ b/apps/api/internal/httpapi/identity_pairing_coordinator.go @@ -0,0 +1,126 @@ +package httpapi + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" +) + +const identityPairingReconcileInterval = 5 * time.Second + +type identityPairingReconciliationStore interface { + PendingIdentityPairingExchanges(context.Context) ([]identity.PairingExchange, error) + CompletedIdentityPairingsAwaitingActivation(context.Context) ([]identity.PairingExchange, error) + PendingIdentityPairingCleanups(context.Context) ([]identity.PairingExchange, error) +} + +type identityPairingRestorer interface { + RestoreCompletedSecurityEvents(context.Context, string) error +} + +func (s *Server) reconcileCanonicalIdentityPairing() { + reconcileCanonicalIdentityPairing( + s.ctx, + s.store, + s.identityPairing, + &s.identityRestoredPairings, + s.startIdentityPairingWorker, + func(pairingID string) { s.startIdentityPairingCleanupWorker(pairingID, nil) }, + s.logger, + ) +} + +func (s *Server) runIdentityPairingCoordinator() { + ticker := time.NewTicker(identityPairingReconcileInterval) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + s.reconcileIdentityRuntime() + s.reconcileCanonicalIdentityPairing() + } + } +} + +func reconcileCanonicalIdentityPairing( + ctx context.Context, + repository identityPairingReconciliationStore, + restorer identityPairingRestorer, + restored *sync.Map, + startWorker func(string), + startCleanup func(string), + logger *slog.Logger, +) { + cleanups, cleanupErr := repository.PendingIdentityPairingCleanups(ctx) + if cleanupErr != nil { + if logger != nil { + logger.Warn("pending identity pairing cleanups could not be resumed", "error_category", "pairing_cleanup_resume_failed") + } + } else { + for _, pairing := range cleanups { + startCleanup(pairing.ID) + } + } + + pending, err := repository.PendingIdentityPairingExchanges(ctx) + if err != nil { + if logger != nil { + logger.Warn("pending identity pairings could not be resumed", "error_category", "pairing_resume_failed") + } + return + } + if len(pending) > 0 { + forgetOtherRestoredPairings(restored, "") + startWorker(pending[0].ID) + return + } + + completed, err := repository.CompletedIdentityPairingsAwaitingActivation(ctx) + if err != nil { + if logger != nil { + logger.Warn("completed identity pairings could not be loaded", "error_category", "pairing_receiver_restore_failed") + } + return + } + if len(completed) == 0 { + forgetOtherRestoredPairings(restored, "") + return + } + + pairingID := completed[0].ID + forgetOtherRestoredPairings(restored, pairingID) + if _, loaded := restored.LoadOrStore(pairingID, struct{}{}); loaded { + return + } + if err := restorer.RestoreCompletedSecurityEvents(ctx, pairingID); err != nil { + restored.Delete(pairingID) + if logger != nil { + logger.Warn("completed identity pairing receiver could not be restored", "pairing_id", pairingID, "error_category", "pairing_receiver_restore_failed") + } + } +} + +func (s *Server) reconcileIdentityRuntime() { + if s.identityRuntime == nil || !s.identityRuntime.ReconciliationRequired() { + return + } + ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second) + defer cancel() + if err := s.identityRuntime.ReconcileActive(ctx); err != nil && s.logger != nil { + s.logger.Warn("identity runtime reconciliation failed and will retry", "error_category", "identity_runtime_reconciliation_failed") + } +} + +func forgetOtherRestoredPairings(restored *sync.Map, keepID string) { + restored.Range(func(key, _ any) bool { + if key != keepID { + restored.Delete(key) + } + return true + }) +} diff --git a/apps/api/internal/httpapi/identity_pairing_coordinator_test.go b/apps/api/internal/httpapi/identity_pairing_coordinator_test.go new file mode 100644 index 0000000..052d030 --- /dev/null +++ b/apps/api/internal/httpapi/identity_pairing_coordinator_test.go @@ -0,0 +1,150 @@ +package httpapi + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" +) + +type identityPairingCoordinatorStore struct { + pending []identity.PairingExchange + completed []identity.PairingExchange + cleanups []identity.PairingExchange + pendingErr error + completedErr error + cleanupErr error +} + +func (store identityPairingCoordinatorStore) PendingIdentityPairingExchanges(context.Context) ([]identity.PairingExchange, error) { + return store.pending, store.pendingErr +} + +func (store identityPairingCoordinatorStore) CompletedIdentityPairingsAwaitingActivation(context.Context) ([]identity.PairingExchange, error) { + return store.completed, store.completedErr +} + +func (store identityPairingCoordinatorStore) PendingIdentityPairingCleanups(context.Context) ([]identity.PairingExchange, error) { + return store.cleanups, store.cleanupErr +} + +type identityPairingCoordinatorRestorer struct { + calls int + err error +} + +func (restorer *identityPairingCoordinatorRestorer) RestoreCompletedSecurityEvents(context.Context, string) error { + restorer.calls++ + return restorer.err +} + +func TestReconcileCanonicalIdentityPairingRetriesFailedRestore(t *testing.T) { + repository := identityPairingCoordinatorStore{ + completed: []identity.PairingExchange{{ID: "pairing-1"}}, + } + restorer := &identityPairingCoordinatorRestorer{err: errors.New("SecretStore temporarily unavailable")} + restored := &sync.Map{} + + reconcileCanonicalIdentityPairing(context.Background(), repository, restorer, restored, func(string) {}, func(string) {}, nil) + reconcileCanonicalIdentityPairing(context.Background(), repository, restorer, restored, func(string) {}, func(string) {}, nil) + + if restorer.calls != 2 { + t.Fatalf("restore calls = %d, want 2", restorer.calls) + } +} + +func TestReconcileCanonicalIdentityPairingRestoresSuccessfulPairingOnce(t *testing.T) { + repository := identityPairingCoordinatorStore{ + completed: []identity.PairingExchange{{ID: "pairing-1"}}, + } + restorer := &identityPairingCoordinatorRestorer{} + restored := &sync.Map{} + + reconcileCanonicalIdentityPairing(context.Background(), repository, restorer, restored, func(string) {}, func(string) {}, nil) + reconcileCanonicalIdentityPairing(context.Background(), repository, restorer, restored, func(string) {}, func(string) {}, nil) + + if restorer.calls != 1 { + t.Fatalf("restore calls = %d, want 1", restorer.calls) + } +} + +func TestReconcileCanonicalIdentityPairingPrioritizesPendingWork(t *testing.T) { + repository := identityPairingCoordinatorStore{ + pending: []identity.PairingExchange{{ID: "pending-pairing"}}, + completed: []identity.PairingExchange{{ID: "completed-pairing"}}, + } + restorer := &identityPairingCoordinatorRestorer{} + restored := &sync.Map{} + started := "" + + reconcileCanonicalIdentityPairing(context.Background(), repository, restorer, restored, func(pairingID string) { + started = pairingID + }, func(string) {}, nil) + + if started != "pending-pairing" { + t.Fatalf("started pairing = %q, want pending-pairing", started) + } + if restorer.calls != 0 { + t.Fatalf("restore calls = %d, want 0", restorer.calls) + } +} + +func TestReconcileCanonicalIdentityPairingForgetsTerminalPairing(t *testing.T) { + restored := &sync.Map{} + restored.Store("terminal-pairing", struct{}{}) + + reconcileCanonicalIdentityPairing( + context.Background(), + identityPairingCoordinatorStore{}, + &identityPairingCoordinatorRestorer{}, + restored, + func(string) {}, + func(string) {}, + nil, + ) + + if _, ok := restored.Load("terminal-pairing"); ok { + t.Fatal("terminal pairing restore marker was not removed") + } +} + +func TestReconcileCanonicalIdentityPairingResumesCommittedCleanup(t *testing.T) { + repository := identityPairingCoordinatorStore{ + cleanups: []identity.PairingExchange{{ID: "cancelled-pairing"}}, + } + restorer := &identityPairingCoordinatorRestorer{} + restored := &sync.Map{} + startedCleanup := "" + + reconcileCanonicalIdentityPairing( + context.Background(), + repository, + restorer, + restored, + func(string) {}, + func(pairingID string) { startedCleanup = pairingID }, + nil, + ) + + if startedCleanup != "cancelled-pairing" { + t.Fatalf("started cleanup = %q, want cancelled-pairing", startedCleanup) + } +} + +func TestCleanupCoordinatorCancelsPairingWorkerBeforeCleanup(t *testing.T) { + serverContext, cancelServer := context.WithCancel(context.Background()) + cancelServer() + workerContext, cancelWorker := context.WithCancel(context.Background()) + server := &Server{ctx: serverContext} + server.identityPairingWorkers.Store("pairing-1", &identityPairingWorker{cancel: cancelWorker, done: make(chan struct{})}) + + server.startIdentityPairingCleanupWorker("pairing-1", nil) + + select { + case <-workerContext.Done(): + default: + t.Fatal("coordinator cleanup did not cancel the pairing worker first") + } +} diff --git a/apps/api/internal/httpapi/identity_runtime.go b/apps/api/internal/httpapi/identity_runtime.go index 192a9ef..51fffb4 100644 --- a/apps/api/internal/httpapi/identity_runtime.go +++ b/apps/api/internal/httpapi/identity_runtime.go @@ -79,6 +79,9 @@ func (s *Server) currentIdentityRuntime() *identityRequestRuntime { } func (s *Server) currentSecurityEventManager() *ssfreceiver.ConnectionManager { + if s.identityRuntime != nil { + return s.identityRuntime.SecurityEventManager() + } if runtime := s.currentIdentityRuntime(); runtime != nil { return runtime.SecurityEvents } diff --git a/apps/api/internal/httpapi/local_login_policy_test.go b/apps/api/internal/httpapi/local_login_policy_test.go new file mode 100644 index 0000000..46d0ff7 --- /dev/null +++ b/apps/api/internal/httpapi/local_login_policy_test.go @@ -0,0 +1,26 @@ +package httpapi + +import ( + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestLocalLoginPolicyAlwaysPreservesBreakGlassManager(t *testing.T) { + serverMain := &Server{cfg: config.Config{IdentityMode: "server-main"}} + if !serverMain.localLoginAllowed(store.GatewayUser{Roles: []string{"manager"}}) { + t.Fatal("server-main mode rejected a local break-glass manager") + } + if !serverMain.localLoginAllowed(store.GatewayUser{Roles: []string{"admin"}}) { + t.Fatal("server-main mode rejected a local break-glass admin") + } + if serverMain.localLoginAllowed(store.GatewayUser{Roles: []string{"user"}}) { + t.Fatal("server-main mode accepted an ordinary local user") + } + + hybrid := &Server{cfg: config.Config{IdentityMode: "hybrid"}} + if !hybrid.localLoginAllowed(store.GatewayUser{Roles: []string{"user"}}) { + t.Fatal("hybrid mode rejected an ordinary local user") + } +} diff --git a/apps/api/internal/httpapi/oidc_jit_integration_test.go b/apps/api/internal/httpapi/oidc_jit_integration_test.go index 238d477..d4fd8ad 100644 --- a/apps/api/internal/httpapi/oidc_jit_integration_test.go +++ b/apps/api/internal/httpapi/oidc_jit_integration_test.go @@ -281,7 +281,7 @@ func prepareOIDCJITRevision(t *testing.T, ctx context.Context, db *store.Store, draft, err := identity.NewDraft(identity.PairingInput{ AuthCenterURL: issuer, PublicBaseURL: "http://localhost", WebBaseURL: "http://localhost:5178", LocalTenantKey: localTenantKey, LegacyJWTEnabled: true, - }) + }, "test") if err != nil { t.Fatalf("create OIDC JIT draft: %v", err) } @@ -304,7 +304,7 @@ func prepareOIDCJITRevision(t *testing.T, ctx context.Context, db *store.Store, Capabilities: []string{"oidc_login", "api_access"}, Audience: "gateway-api", Scopes: []string{"gateway.access"}, Clients: identity.ManifestClients{BrowserLogin: &identity.ManifestClient{ClientID: "gateway-public-test"}}, }, - SessionEncryptionKeyRef: sessionReference, TraceID: "oidc-jit-test", AuditID: "oidc-jit-test", + SessionEncryptionKeyRef: sessionReference, TraceID: "oidc-jit-test", AuditID: "oidc-jit-test", AppEnv: "test", }) if err != nil { t.Fatalf("apply OIDC JIT manifest: %v", err) diff --git a/apps/api/internal/httpapi/oidc_session.go b/apps/api/internal/httpapi/oidc_session.go index d442980..dc6ad81 100644 --- a/apps/api/internal/httpapi/oidc_session.go +++ b/apps/api/internal/httpapi/oidc_session.go @@ -357,7 +357,7 @@ func (s *Server) startOIDCSessionCleanup(ctx context.Context) { func (s *Server) protectOIDCSessionCookie(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { runtime := s.currentIdentityRuntime() - if runtime == nil || !runtime.BrowserEnabled || isSafeHTTPMethod(r.Method) || hasExplicitCredential(r) { + if runtime == nil || !runtime.BrowserEnabled || hasExplicitCredential(r) { next.ServeHTTP(w, r) return } @@ -366,7 +366,7 @@ func (s *Server) protectOIDCSessionCookie(next http.Handler) http.Handler { return } origin := strings.TrimSpace(r.Header.Get("Origin")) - if origin == "" || !originMatchesBaseURL(origin, runtime.Revision.WebBaseURL) { + if origin != "" && !originMatchesBaseURL(origin, runtime.Revision.WebBaseURL) || origin == "" && !isSafeHTTPMethod(r.Method) { writeError(w, http.StatusForbidden, "browser session request origin was rejected", errorCodeOIDCSessionCSRF) return } @@ -384,8 +384,16 @@ func isSafeHTTPMethod(method string) bool { } func hasExplicitCredential(r *http.Request) bool { - return strings.TrimSpace(r.Header.Get("Authorization")) != "" || + return extractBearerCredential(r.Header.Get("Authorization")) != "" || strings.TrimSpace(r.Header.Get("x-comfy-api-key")) != "" || strings.TrimSpace(r.Header.Get("x-goog-api-key")) != "" || - strings.TrimSpace(r.URL.Query().Get("key")) != "" + strings.HasPrefix(strings.TrimSpace(r.URL.Query().Get("key")), "sk-") +} + +func extractBearerCredential(value string) string { + fields := strings.Fields(value) + if len(fields) == 2 && strings.EqualFold(fields[0], "bearer") { + return fields[1] + } + return "" } diff --git a/apps/api/internal/httpapi/oidc_session_test.go b/apps/api/internal/httpapi/oidc_session_test.go index 0c67a9a..e7c78fa 100644 --- a/apps/api/internal/httpapi/oidc_session_test.go +++ b/apps/api/internal/httpapi/oidc_session_test.go @@ -161,6 +161,7 @@ func TestOIDCSessionCSRFAcceptsOnlyAllowedOriginForCookieWrites(t *testing.T) { }{ {name: "missing origin", method: http.MethodPost, wantStatus: http.StatusForbidden}, {name: "foreign origin", method: http.MethodDelete, origin: "https://evil.example", wantStatus: http.StatusForbidden}, + {name: "foreign origin cannot read cookie authenticated data", method: http.MethodGet, origin: "https://evil.example", wantStatus: http.StatusForbidden}, {name: "allowed origin", method: http.MethodPatch, origin: "https://gateway.example.com", wantStatus: http.StatusNoContent}, {name: "safe request", method: http.MethodGet, wantStatus: http.StatusNoContent}, {name: "explicit bearer bypasses cookie csrf", method: http.MethodPost, bearer: true, wantStatus: http.StatusNoContent}, @@ -181,6 +182,94 @@ func TestOIDCSessionCSRFAcceptsOnlyAllowedOriginForCookieWrites(t *testing.T) { } } +func TestOIDCSessionCSRFMalformedAuthorizationCannotBypassForeignOrigin(t *testing.T) { + authenticator := auth.New("local-jwt-secret", "", "") + authenticator.OIDCSessionResolver = func(context.Context, string) (*auth.User, error) { + return &auth.User{ID: "manager", Source: "gateway", Roles: []string{"manager"}}, nil + } + server := &Server{ + auth: authenticator, + identityTestRevision: identity.Revision{WebBaseURL: "https://gateway.example.com"}, + identityTestBrowserEnabled: true, + } + called := false + handler := server.protectOIDCSessionCookie(server.requireAdmin(auth.PermissionManager, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + called = true + w.WriteHeader(http.StatusNoContent) + }))) + request := httptest.NewRequest(http.MethodPost, "/api/admin/system/identity/disable", nil) + request.Header.Set("Origin", "https://evil.example.com") + request.Header.Set("Authorization", "malformed") + request.AddCookie(&http.Cookie{Name: auth.OIDCSessionCookieName, Value: "manager-session"}) + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusForbidden || called { + t.Fatalf("malformed Authorization CSRF status=%d handler_called=%t", recorder.Code, called) + } +} + +func TestAdminRouteRejectsManagerJWTInQueryAndAcceptsAuthorizationHeader(t *testing.T) { + authenticator := auth.New("local-jwt-secret", "", "") + managerToken, err := authenticator.SignJWT(&auth.User{ID: "manager", Source: "gateway", Roles: []string{"manager"}}, time.Hour) + if err != nil { + t.Fatal(err) + } + server := &Server{auth: authenticator} + handler := server.requireAdmin(auth.PermissionManager, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + queryRequest := httptest.NewRequest(http.MethodPost, "/api/admin/system/identity/disable?key="+managerToken, nil) + queryRecorder := httptest.NewRecorder() + handler.ServeHTTP(queryRecorder, queryRequest) + if queryRecorder.Code != http.StatusUnauthorized { + t.Fatalf("query manager JWT status=%d, want 401", queryRecorder.Code) + } + + headerRequest := httptest.NewRequest(http.MethodPost, "/api/admin/system/identity/disable", nil) + headerRequest.Header.Set("Authorization", "Bearer "+managerToken) + headerRecorder := httptest.NewRecorder() + handler.ServeHTTP(headerRecorder, headerRequest) + if headerRecorder.Code != http.StatusNoContent { + t.Fatalf("Authorization manager JWT status=%d, want 204", headerRecorder.Code) + } +} + +func TestCORSUsesOnlyCurrentActiveIdentityWebOriginWithoutRestart(t *testing.T) { + server := &Server{ + cfg: config.Config{CORSAllowedOrigin: "https://bootstrap.example.com"}, + identityTestRevision: identity.Revision{ + ID: "active-revision", State: identity.RevisionActive, WebBaseURL: "https://gateway.example.com", + }, + } + handler := server.cors(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })) + request := func(origin string) *httptest.ResponseRecorder { + r := httptest.NewRequest(http.MethodOptions, "/api/admin/system/identity/configuration", nil) + r.Header.Set("Origin", origin) + r.Header.Set("Access-Control-Request-Method", http.MethodGet) + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + return w + } + + active := request("https://gateway.example.com") + if active.Header().Get("Access-Control-Allow-Origin") != "https://gateway.example.com" || active.Header().Get("Access-Control-Allow-Credentials") != "true" { + t.Fatalf("active Web origin headers=%v", active.Header()) + } + if evil := request("https://evil.example.com"); evil.Header().Get("Access-Control-Allow-Origin") != "" { + t.Fatalf("evil origin was allowed: headers=%v", evil.Header()) + } + + server.identityTestRevision = identity.Revision{} + if disabled := request("https://gateway.example.com"); disabled.Header().Get("Access-Control-Allow-Origin") != "" { + t.Fatalf("disabled Revision retained dynamic origin: headers=%v", disabled.Header()) + } + if bootstrap := request("https://bootstrap.example.com"); bootstrap.Header().Get("Access-Control-Allow-Origin") != "https://bootstrap.example.com" { + t.Fatalf("deployment bootstrap origin stopped working: headers=%v", bootstrap.Header()) + } +} + func TestOIDCSessionCSRFIsInactiveWhenOIDCIsDisabled(t *testing.T) { server := &Server{} request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/login", nil) diff --git a/apps/api/internal/httpapi/security_event_connection_handlers.go b/apps/api/internal/httpapi/security_event_connection_handlers.go index b82b890..4cde34c 100644 --- a/apps/api/internal/httpapi/security_event_connection_handlers.go +++ b/apps/api/internal/httpapi/security_event_connection_handlers.go @@ -78,10 +78,28 @@ func (s *Server) putSecurityEventConnection(w http.ResponseWriter, r *http.Reque defer clear(secret) secretDigest := sha256.Sum256(secret) requestHash := securityEventOperationHash("connect", request.TransmitterIssuer+"\x00"+request.ManagementClientID+"\x00"+fmt.Sprintf("%x", secretDigest[:])) + s.securityEventManagementMu.Lock() + defer s.securityEventManagementMu.Unlock() if s.replaySecurityEventOperation(w, r, "connect", idempotencyKey, requestHash) { return } - if current, err := manager.Get(r.Context()); err == nil && !matchConnectionVersion(w, r, current.Version) { + current, currentErr := manager.Get(r.Context()) + switch { + case currentErr == nil: + if !matchConnectionVersion(w, r, current.Version) { + return + } + case errors.Is(currentErr, store.ErrSecurityEventConnectionNotFound): + current = ssfreceiver.ConnectionView{} + if !matchConnectionVersion(w, r, 0) { + return + } + default: + writeError(w, http.StatusServiceUnavailable, "security event connection state is unavailable", "security_event_state_unavailable") + return + } + auditID, ok := s.requireSecurityEventConnectionAudit(w, r, "connect", traceID, current) + if !ok { return } connection, err := manager.Connect(r.Context(), request.TransmitterIssuer, request.ManagementClientID, secret, idempotencyKey) @@ -89,13 +107,15 @@ func (s *Server) putSecurityEventConnection(w http.ResponseWriter, r *http.Reque s.writeSecurityEventConnectionError(w, r, manager, "connect", traceID, err) return } - s.writeSecurityEventOperation(w, r, "connect", idempotencyKey, requestHash, traceID, connection) + s.writeSecurityEventOperation(w, r, "connect", idempotencyKey, requestHash, traceID, auditID, connection) } func (s *Server) verifySecurityEventConnection(w http.ResponseWriter, r *http.Request) { traceID := ensureSecurityEventTraceID(w, r) + s.securityEventManagementMu.Lock() + defer s.securityEventManagementMu.Unlock() manager := s.currentSecurityEventManager() - idempotencyKey, requestHash, ok := s.beginSecurityEventOperation(w, r, manager, "verify", traceID) + idempotencyKey, requestHash, auditID, ok := s.beginSecurityEventOperation(w, r, manager, "verify", traceID) if !ok { return } @@ -104,13 +124,15 @@ func (s *Server) verifySecurityEventConnection(w http.ResponseWriter, r *http.Re s.writeSecurityEventConnectionError(w, r, manager, "verify", traceID, err) return } - s.writeSecurityEventOperation(w, r, "verify", idempotencyKey, requestHash, traceID, connection) + s.writeSecurityEventOperation(w, r, "verify", idempotencyKey, requestHash, traceID, auditID, connection) } func (s *Server) rotateSecurityEventConnectionCredential(w http.ResponseWriter, r *http.Request) { traceID := ensureSecurityEventTraceID(w, r) + s.securityEventManagementMu.Lock() + defer s.securityEventManagementMu.Unlock() manager := s.currentSecurityEventManager() - idempotencyKey, requestHash, ok := s.beginSecurityEventOperation(w, r, manager, "rotate", traceID) + idempotencyKey, requestHash, auditID, ok := s.beginSecurityEventOperation(w, r, manager, "rotate", traceID) if !ok { return } @@ -119,13 +141,15 @@ func (s *Server) rotateSecurityEventConnectionCredential(w http.ResponseWriter, s.writeSecurityEventConnectionError(w, r, manager, "rotate", traceID, err) return } - s.writeSecurityEventOperation(w, r, "rotate", idempotencyKey, requestHash, traceID, connection) + s.writeSecurityEventOperation(w, r, "rotate", idempotencyKey, requestHash, traceID, auditID, connection) } func (s *Server) deleteSecurityEventConnection(w http.ResponseWriter, r *http.Request) { traceID := ensureSecurityEventTraceID(w, r) + s.securityEventManagementMu.Lock() + defer s.securityEventManagementMu.Unlock() manager := s.currentSecurityEventManager() - idempotencyKey, requestHash, ok := s.beginSecurityEventOperation(w, r, manager, "disconnect", traceID) + idempotencyKey, requestHash, auditID, ok := s.beginSecurityEventOperation(w, r, manager, "disconnect", traceID) if !ok { return } @@ -134,7 +158,7 @@ func (s *Server) deleteSecurityEventConnection(w http.ResponseWriter, r *http.Re s.writeSecurityEventConnectionError(w, r, manager, "disconnect", traceID, err) return } - s.writeSecurityEventOperation(w, r, "disconnect", idempotencyKey, requestHash, traceID, connection) + s.writeSecurityEventOperation(w, r, "disconnect", idempotencyKey, requestHash, traceID, auditID, connection) } func (s *Server) securityEventPrerequisites() map[string]any { @@ -164,25 +188,32 @@ func requiredConnectionIdempotencyKey(w http.ResponseWriter, r *http.Request) (s return value, true } -func (s *Server) beginSecurityEventOperation(w http.ResponseWriter, r *http.Request, manager *ssfreceiver.ConnectionManager, operation, traceID string) (string, string, bool) { +func (s *Server) beginSecurityEventOperation(w http.ResponseWriter, r *http.Request, manager *ssfreceiver.ConnectionManager, operation, traceID string) (string, string, string, bool) { if manager == nil { writeError(w, http.StatusNotFound, "security event connection does not exist", "security_event_connection_not_found") - return "", "", false + return "", "", "", false } idempotencyKey, ok := requiredConnectionIdempotencyKey(w, r) if !ok { - return "", "", false + return "", "", "", false } requestHash := securityEventOperationHash(operation, normalizedConnectionETag(r.Header.Get("If-Match"))) if s.replaySecurityEventOperation(w, r, operation, idempotencyKey, requestHash) { - return "", "", false + return "", "", "", false } connection, err := manager.Get(r.Context()) if err != nil { s.writeSecurityEventConnectionError(w, r, manager, operation, traceID, err) - return "", "", false + return "", "", "", false } - return idempotencyKey, requestHash, matchConnectionVersion(w, r, connection.Version) + if !matchConnectionVersion(w, r, connection.Version) { + return "", "", "", false + } + auditID, ok := s.requireSecurityEventConnectionAudit(w, r, operation, traceID, connection) + if !ok { + return "", "", "", false + } + return idempotencyKey, requestHash, auditID, true } func (s *Server) replaySecurityEventOperation(w http.ResponseWriter, r *http.Request, operation, idempotencyKey, requestHash string) bool { @@ -216,8 +247,7 @@ func (s *Server) replaySecurityEventOperation(w http.ResponseWriter, r *http.Req return true } -func (s *Server) writeSecurityEventOperation(w http.ResponseWriter, r *http.Request, operation, idempotencyKey, requestHash, traceID string, connection ssfreceiver.ConnectionView) { - auditID := s.recordSecurityEventConnectionAudit(r, operation, "success", "", traceID, connection) +func (s *Server) writeSecurityEventOperation(w http.ResponseWriter, r *http.Request, operation, idempotencyKey, requestHash, traceID, auditID string, connection ssfreceiver.ConnectionView) { if auditID != "" { w.Header().Set("X-Audit-Id", auditID) } @@ -328,6 +358,16 @@ func (s *Server) recordSecurityEventConnectionAudit(r *http.Request, operation, return audit.ID } +func (s *Server) requireSecurityEventConnectionAudit(w http.ResponseWriter, r *http.Request, operation, traceID string, connection ssfreceiver.ConnectionView) (string, bool) { + auditID := s.recordSecurityEventConnectionAudit(r, operation, "requested", "", traceID, connection) + if auditID == "" { + writeError(w, http.StatusServiceUnavailable, "security event audit is unavailable; operation was not executed", "security_event_audit_unavailable") + return "", false + } + w.Header().Set("X-Audit-Id", auditID) + return auditID, true +} + func securityEventConnectionAuditInput(r *http.Request, actor *auth.User, operation, outcome, errorCategory, traceID string, connection ssfreceiver.ConnectionView) store.AuditLogInput { input := store.AuditLogInput{ Category: "identity", Action: "identity.security_event_connection." + operation, diff --git a/apps/api/internal/httpapi/security_event_connection_handlers_test.go b/apps/api/internal/httpapi/security_event_connection_handlers_test.go index 22e11d9..2a09d38 100644 --- a/apps/api/internal/httpapi/security_event_connection_handlers_test.go +++ b/apps/api/internal/httpapi/security_event_connection_handlers_test.go @@ -43,6 +43,16 @@ func TestSecurityEventConnectionWriteHeaders(t *testing.T) { if _, ok := requiredConnectionIdempotencyKey(recorder, request); ok || recorder.Code != http.StatusBadRequest { t.Fatalf("missing Idempotency-Key status=%d", recorder.Code) } + request = httptest.NewRequest(http.MethodPut, "/connection", nil) + recorder = httptest.NewRecorder() + if matchConnectionVersion(recorder, request, 0) || recorder.Code != http.StatusPreconditionRequired { + t.Fatalf("initial connection without If-Match status=%d", recorder.Code) + } + request.Header.Set("If-Match", `W/"0"`) + recorder = httptest.NewRecorder() + if !matchConnectionVersion(recorder, request, 0) { + t.Fatalf("initial zero version was rejected: status=%d", recorder.Code) + } request = httptest.NewRequest(http.MethodPost, "/connection/verify", nil) request.Header.Set("If-Match", `W/"7"`) diff --git a/apps/api/internal/httpapi/security_events.go b/apps/api/internal/httpapi/security_events.go index addf64e..5845d3d 100644 --- a/apps/api/internal/httpapi/security_events.go +++ b/apps/api/internal/httpapi/security_events.go @@ -18,9 +18,13 @@ import "net/http" // @Failure 503 {object} map[string]string // @Router /api/v1/security-events/ssf [post] func (s *Server) receiveSecurityEvent(w http.ResponseWriter, r *http.Request) { - if s.securityEventReceiver == nil { + receiver := s.securityEventReceiver + if s.identityRuntime != nil { + receiver = s.identityRuntime.SecurityEventReceiver() + } + if receiver == nil { http.NotFound(w, r) return } - s.securityEventReceiver.ServeHTTP(w, r) + receiver.ServeHTTP(w, r) } diff --git a/apps/api/internal/httpapi/security_events_test.go b/apps/api/internal/httpapi/security_events_test.go new file mode 100644 index 0000000..ab0e162 --- /dev/null +++ b/apps/api/internal/httpapi/security_events_test.go @@ -0,0 +1,56 @@ +package httpapi + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identityruntime" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/securityevents" +) + +type preparedReceiverBuilder struct { + receiver http.Handler +} + +func (builder *preparedReceiverBuilder) Build(context.Context, identity.Revision) (*identityruntime.Runtime, error) { + return &identityruntime.Runtime{}, nil +} + +func (builder *preparedReceiverBuilder) PreparedSecurityEventReceiver() http.Handler { + return builder.receiver +} + +func TestReceiveSecurityEventUsesPreparedReceiverBeforeFirstActivation(t *testing.T) { + called := false + builder := &preparedReceiverBuilder{receiver: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + called = true + w.WriteHeader(http.StatusAccepted) + })} + server := &Server{identityRuntime: identityruntime.NewManager(nil, builder)} + request := httptest.NewRequest(http.MethodPost, "/api/v1/security-events/ssf", nil) + response := httptest.NewRecorder() + + server.receiveSecurityEvent(response, request) + + if !called || response.Code != http.StatusAccepted { + t.Fatalf("prepared SSF receiver called=%t status=%d", called, response.Code) + } +} + +func TestSecurityEventWriteAuditFailsClosedBeforeMutation(t *testing.T) { + server := &Server{} + request := httptest.NewRequest(http.MethodPost, "/api/admin/system/security-events/connection/verify", nil) + recorder := httptest.NewRecorder() + + if _, ok := server.requireSecurityEventConnectionAudit( + recorder, request, "verify", "trace-test", securityevents.ConnectionView{}, + ); ok { + t.Fatal("security event write unexpectedly continued without durable audit storage") + } + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("audit failure status=%d, want 503", recorder.Code) + } +} diff --git a/apps/api/internal/httpapi/server.go b/apps/api/internal/httpapi/server.go index b6050f7..d6bf1e7 100644 --- a/apps/api/internal/httpapi/server.go +++ b/apps/api/internal/httpapi/server.go @@ -36,7 +36,10 @@ type Server struct { identityRuntime *identityruntime.Manager identityPairing *identity.PairingService identityManagementMu sync.Mutex + securityEventManagementMu sync.Mutex identityPairingWorkers sync.Map + identityCleanupWorkers sync.Map + identityRestoredPairings sync.Map identityTestRevision identity.Revision identityTestCookieSecure bool identityTestBrowserEnabled bool @@ -79,6 +82,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor if err != nil { panic("invalid identity SecretStore: " + err.Error()) } + go ssfreceiver.RunSecurityEventRetirementWorker(ctx, db) + go ssfreceiver.RunIdentitySecretCleanupWorker(ctx, db, secretStore) runtimeBuilder := identityruntime.NewRuntimeBuilder(ctx, db, secretStore, identityruntime.RuntimeBuilderConfig{ AppEnv: cfg.AppEnv, JWKSCacheTTL: 5 * time.Minute, HeartbeatInterval: time.Duration(cfg.IdentitySecurityEventHeartbeatIntervalSeconds) * time.Second, @@ -90,15 +95,10 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor logger.Error("load active identity runtime failed; local management login remains available", "error_category", "identity_runtime_load_failed") } server.identityPairing = identity.NewPairingService(db, secretStore, func(baseURL string) (identity.OnboardingRemote, error) { - return identity.NewOnboardingClient(baseURL, nil) - }, server.identityRuntime) - if pending, pendingErr := db.PendingIdentityPairingExchanges(ctx); pendingErr == nil { - for _, pairing := range pending { - server.startIdentityPairingWorker(pairing.ID) - } - } else if logger != nil { - logger.Warn("pending identity pairings could not be resumed", "error_category", "pairing_resume_failed") - } + return identity.NewOnboardingClient(baseURL, nil, cfg.AppEnv) + }, server.identityRuntime, cfg.AppEnv) + server.reconcileCanonicalIdentityPairing() + go server.runIdentityPairingCoordinator() server.auth.OIDCVerifierProvider = func() *auth.OIDCVerifier { if runtime := server.identityRuntime.Current(); runtime != nil { return runtime.Verifier @@ -114,8 +114,7 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor return user, oidcSessionRequestError(resolveErr) } server.auth.LegacyJWTEnabledProvider = func() bool { - runtime := server.identityRuntime.Current() - return runtime == nil || runtime.Revision.LegacyJWTEnabled + return server.identityRuntime.LegacyJWTEnabled() } server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey server.runner.StartAsyncQueueWorker(ctx) @@ -210,6 +209,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor mux.Handle("GET /api/admin/system/identity/configuration", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityConfiguration))) mux.Handle("POST /api/admin/system/identity/pairings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.startIdentityPairing))) mux.Handle("GET /api/admin/system/identity/pairings/{pairingID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityPairing))) + mux.Handle("POST /api/admin/system/identity/pairings/{pairingID}/cancel", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.cancelIdentityPairing))) + mux.Handle("POST /api/admin/system/identity/pairings/{pairingID}/retire-conflicting-security-event", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retireIdentityPairingSecurityEventConflict))) mux.Handle("PATCH /api/admin/system/identity/revisions/{revisionID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateIdentityDraftPolicy))) mux.Handle("POST /api/admin/system/identity/revisions/{revisionID}/validate", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.validateIdentityRevision))) mux.Handle("POST /api/admin/system/identity/revisions/{revisionID}/activate", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.activateIdentityRevision))) @@ -349,7 +350,7 @@ func (s *Server) requireAdmin(permission auth.Permission, next http.Handler) htt func (s *Server) cors(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { origin := r.Header.Get("Origin") - if origin != "" && originAllowed(origin, s.cfg.CORSAllowedOrigin) { + if origin != "" && s.corsOriginAllowed(origin) { w.Header().Set("Access-Control-Allow-Origin", origin) w.Header().Set("Vary", "Origin") w.Header().Set("Access-Control-Allow-Credentials", "true") @@ -364,10 +365,21 @@ func (s *Server) cors(next http.Handler) http.Handler { }) } +func (s *Server) corsOriginAllowed(origin string) bool { + if originAllowed(origin, s.cfg.CORSAllowedOrigin) { + return true + } + if s.identityRuntime != nil { + return originMatchesBaseURL(origin, s.identityRuntime.TrustedWebBaseURL()) + } + runtime := s.currentIdentityRuntime() + return runtime != nil && originMatchesBaseURL(origin, runtime.Revision.WebBaseURL) +} + func originAllowed(origin string, allowed string) bool { for _, item := range strings.Split(allowed, ",") { item = strings.TrimSpace(item) - if item == "*" || strings.EqualFold(origin, item) { + if item != "*" && strings.EqualFold(origin, item) { return true } } diff --git a/apps/api/internal/identity/configuration.go b/apps/api/internal/identity/configuration.go index f3eb4ad..5cb41e5 100644 --- a/apps/api/internal/identity/configuration.go +++ b/apps/api/internal/identity/configuration.go @@ -11,10 +11,13 @@ import ( ) var ( - ErrRevisionNotFound = errors.New("identity configuration revision not found") - ErrRevisionConflict = errors.New("identity configuration revision conflicts with current state") - ErrBreakGlassRequired = errors.New("a local break-glass manager credential is required") - ErrLocalTenantInvalid = errors.New("local tenant mapping is invalid") + ErrRevisionNotFound = errors.New("identity configuration revision not found") + ErrRevisionConflict = errors.New("identity configuration revision conflicts with current state") + ErrBreakGlassRequired = errors.New("a local break-glass manager credential is required") + ErrLocalTenantInvalid = errors.New("local tenant mapping is invalid") + ErrActiveConfigurationHandoffRequired = errors.New("active identity configuration requires an explicit remote resource handoff before re-pairing") + ErrRollbackConfigurationHandoffRequired = errors.New("rollback requires a fresh remote resource handoff") + ErrSecurityEventRetirementPending = errors.New("active security event connection must retire before identity can be disabled") ) type RevisionPolicy struct { @@ -111,15 +114,16 @@ type ManifestApplication struct { SessionEncryptionKeyRef string TraceID string AuditID string + AppEnv string } -func NewDraft(input PairingInput) (Revision, error) { - if _, err := input.ConsumerMetadata(false); err != nil { +func NewDraft(input PairingInput, appEnv string) (Revision, error) { + if _, err := input.ConsumerMetadata(false, appEnv); err != nil { return Revision{}, err } - authCenter, _ := exactBaseURL(input.AuthCenterURL) - publicBase, _ := exactBaseURL(input.PublicBaseURL) - webBase, _ := exactBaseURL(input.WebBaseURL) + authCenter, _ := exactBaseURL(input.AuthCenterURL, appEnv) + publicBase, _ := exactBaseURL(input.PublicBaseURL, appEnv) + webBase, _ := exactBaseURL(input.WebBaseURL, appEnv) return Revision{ ID: uuid.NewString(), State: RevisionDraft, SchemaVersion: 1, AuthCenterURL: authCenter, RolePrefix: "gateway.", LocalTenantKey: strings.TrimSpace(input.LocalTenantKey), @@ -133,7 +137,7 @@ func ApplyManifest(revision Revision, input ManifestApplication) (Revision, erro if revision.State != RevisionDraft { return Revision{}, ErrRevisionConflict } - if err := input.Manifest.Validate(); err != nil { + if err := input.Manifest.Validate(input.AppEnv); err != nil { return Revision{}, err } capabilities := make(map[string]bool, len(input.Manifest.Capabilities)) @@ -187,17 +191,17 @@ func CanTransition(from, to RevisionState) bool { } } -func (input PairingInput) ConsumerMetadata(sessionRevocation bool) (ConsumerMetadata, error) { - authCenter, err := exactBaseURL(input.AuthCenterURL) +func (input PairingInput) ConsumerMetadata(sessionRevocation bool, appEnv string) (ConsumerMetadata, error) { + authCenter, err := exactBaseURL(input.AuthCenterURL, appEnv) if err != nil { return ConsumerMetadata{}, errors.New("auth center URL is invalid") } _ = authCenter - publicBase, err := exactBaseURL(input.PublicBaseURL) + publicBase, err := exactBaseURL(input.PublicBaseURL, appEnv) if err != nil { return ConsumerMetadata{}, errors.New("public base URL is invalid") } - webBase, err := exactBaseURL(input.WebBaseURL) + webBase, err := exactBaseURL(input.WebBaseURL, appEnv) if err != nil { return ConsumerMetadata{}, errors.New("web base URL is invalid") } @@ -215,7 +219,7 @@ func (input PairingInput) ConsumerMetadata(sessionRevocation bool) (ConsumerMeta return metadata, nil } -func exactBaseURL(raw string) (string, error) { +func exactBaseURL(raw, appEnv string) (string, error) { parsed, err := url.Parse(strings.TrimSpace(raw)) if err != nil || parsed.Opaque != "" || parsed.User != nil || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" { return "", errors.New("invalid public URL") @@ -225,7 +229,7 @@ func exactBaseURL(raw string) (string, error) { return "", errors.New("base URL must not contain a path") } scheme := strings.ToLower(parsed.Scheme) - if scheme != "https" && !(scheme == "http" && isLoopbackHost(hostname)) { + if scheme != "https" && !(scheme == "http" && isLocalIdentityEnvironment(appEnv) && isLoopbackHost(hostname)) { return "", errors.New("public URL must use HTTPS") } port := parsed.Port() @@ -242,6 +246,49 @@ func exactBaseURL(raw string) (string, error) { return scheme + "://" + host, nil } +func isLocalIdentityEnvironment(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "development", "dev", "local", "test": + return true + default: + return false + } +} + +// ValidateRevisionURLs re-applies the deployment environment URL policy when +// constructing a Runtime. This protects against legacy or directly persisted +// revisions bypassing the pairing boundary. +func ValidateRevisionURLs(revision Revision, appEnv string) error { + urls := []struct { + label string + raw string + base bool + }{ + {label: "auth center", raw: revision.AuthCenterURL, base: true}, + {label: "issuer", raw: revision.Issuer}, + {label: "public base", raw: revision.PublicBaseURL, base: true}, + {label: "web base", raw: revision.WebBaseURL, base: true}, + } + for _, candidate := range urls { + var err error + if candidate.base { + _, err = exactBaseURL(candidate.raw, appEnv) + } else { + err = validatePublicIdentityURL(candidate.raw, appEnv) + } + if err != nil { + return errors.New("identity " + candidate.label + " URL must use HTTPS in this environment") + } + } + if revision.SessionRevocation || revision.SecurityEventIssuer != "" || revision.SecurityEventConfigURL != "" { + if validatePublicIdentityURL(revision.SecurityEventIssuer, appEnv) != nil || + validatePublicIdentityURL(revision.SecurityEventConfigURL, appEnv) != nil { + return errors.New("identity security event URLs must use HTTPS in this environment") + } + } + return nil +} + func isLoopbackHost(host string) bool { if host == "localhost" { return true diff --git a/apps/api/internal/identity/configuration_test.go b/apps/api/internal/identity/configuration_test.go index 1b7b745..f71eede 100644 --- a/apps/api/internal/identity/configuration_test.go +++ b/apps/api/internal/identity/configuration_test.go @@ -48,7 +48,7 @@ func TestValidatePairingInputDerivesExactGatewayURIs(t *testing.T) { AuthCenterURL: "https://auth.example.com", PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default", } - metadata, err := input.ConsumerMetadata(true) + metadata, err := input.ConsumerMetadata(true, "production") if err != nil { t.Fatal(err) } @@ -64,7 +64,7 @@ func TestValidatePairingInputRejectsRemoteHTTPAndURLCredentials(t *testing.T) { {AuthCenterURL: "http://auth.example.com", PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default"}, {AuthCenterURL: "https://user:password@auth.example.com", PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default"}, } { - if _, err := input.ConsumerMetadata(false); err == nil { + if _, err := input.ConsumerMetadata(false, "production"); err == nil { t.Fatalf("unsafe pairing input accepted: %#v", input) } } @@ -75,7 +75,7 @@ func TestNewDraftAppliesSessionDefaultsWithoutPersistingOnboardingCode(t *testin AuthCenterURL: "https://auth.example.com", OnboardingCode: "onb1.must-never-be-persisted", PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default", LegacyJWTEnabled: true, - }) + }, "production") if err != nil { t.Fatal(err) } @@ -87,3 +87,41 @@ func TestNewDraftAppliesSessionDefaultsWithoutPersistingOnboardingCode(t *testin t.Fatalf("draft exposed onboarding code: %s", payload) } } + +func TestNewDraftAllowsLoopbackHTTPOnlyInLocalEnvironments(t *testing.T) { + localInput := PairingInput{ + AuthCenterURL: "http://localhost:18000", PublicBaseURL: "http://127.0.0.1:18089", + WebBaseURL: "http://localhost:5178", LocalTenantKey: "default", + } + secureInput := PairingInput{ + AuthCenterURL: "https://auth.example.com", PublicBaseURL: "https://api.example.com", + WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default", + } + for _, test := range []struct { + name string + mutate func(*PairingInput) + }{ + {name: "auth center", mutate: func(input *PairingInput) { input.AuthCenterURL = localInput.AuthCenterURL }}, + {name: "public base", mutate: func(input *PairingInput) { input.PublicBaseURL = localInput.PublicBaseURL }}, + {name: "web base", mutate: func(input *PairingInput) { input.WebBaseURL = localInput.WebBaseURL }}, + } { + t.Run(test.name, func(t *testing.T) { + input := secureInput + test.mutate(&input) + for _, appEnv := range []string{"", "production", "staging"} { + if _, err := NewDraft(input, appEnv); err == nil { + t.Fatalf("%s accepted loopback HTTP %s URL", appEnv, test.name) + } + } + }) + } + for _, appEnv := range []string{"local", "development", "dev", "test"} { + draft, err := NewDraft(localInput, appEnv) + if err != nil { + t.Fatalf("%s rejected loopback HTTP pairing URLs: %v", appEnv, err) + } + if draft.AuthCenterURL != localInput.AuthCenterURL || draft.PublicBaseURL != localInput.PublicBaseURL || draft.WebBaseURL != localInput.WebBaseURL { + t.Fatalf("%s changed normalized loopback URLs: %#v", appEnv, draft) + } + } +} diff --git a/apps/api/internal/identity/onboarding_client.go b/apps/api/internal/identity/onboarding_client.go index 1b4030b..ec343f7 100644 --- a/apps/api/internal/identity/onboarding_client.go +++ b/apps/api/internal/identity/onboarding_client.go @@ -85,8 +85,8 @@ type CredentialDelivery struct { Version int64 `json:"-"` } -func (manifest ManifestV1) Validate() error { - if manifest.SchemaVersion != 1 || validatePublicIdentityURL(manifest.Issuer) != nil { +func (manifest ManifestV1) Validate(appEnv string) error { + if manifest.SchemaVersion != 1 || validatePublicIdentityURL(manifest.Issuer, appEnv) != nil { return errors.New("application manifest identity metadata is invalid") } if _, err := uuid.Parse(manifest.TenantID); err != nil { @@ -116,9 +116,12 @@ func (manifest ManifestV1) Validate() error { if capabilities["api_access"] && strings.TrimSpace(manifest.Audience) == "" { return errors.New("application manifest audience is missing") } - if capabilities["session_revocation"] { - if manifest.SecurityEvents == nil || validatePublicIdentityURL(manifest.SecurityEvents.TransmitterIssuer) != nil || - validatePublicIdentityURL(manifest.SecurityEvents.ConfigurationEndpoint) != nil || strings.TrimSpace(manifest.SecurityEvents.Audience) == "" { + if capabilities["session_revocation"] && manifest.SecurityEvents == nil { + return errors.New("application manifest security event metadata is invalid") + } + if manifest.SecurityEvents != nil { + if validatePublicIdentityURL(manifest.SecurityEvents.TransmitterIssuer, appEnv) != nil || + validatePublicIdentityURL(manifest.SecurityEvents.ConfigurationEndpoint, appEnv) != nil || strings.TrimSpace(manifest.SecurityEvents.Audience) == "" { return errors.New("application manifest security event metadata is invalid") } } @@ -133,8 +136,8 @@ func (manifest ManifestV1) Validate() error { return nil } -func validatePublicIdentityURL(raw string) error { - _, err := exactBaseURL(raw) +func validatePublicIdentityURL(raw, appEnv string) error { + _, err := exactBaseURL(raw, appEnv) if err == nil { return nil } @@ -145,7 +148,7 @@ func validatePublicIdentityURL(raw string) error { return errors.New("identity URL is invalid") } scheme := strings.ToLower(parsed.URL.Scheme) - if scheme == "https" || scheme == "http" && isLoopbackHost(strings.ToLower(parsed.URL.Hostname())) { + if scheme == "https" || scheme == "http" && isLocalIdentityEnvironment(appEnv) && isLoopbackHost(strings.ToLower(parsed.URL.Hostname())) { return nil } return errors.New("identity URL must use HTTPS") @@ -154,10 +157,11 @@ func validatePublicIdentityURL(raw string) error { type OnboardingClient struct { baseURL string client *http.Client + appEnv string } -func NewOnboardingClient(baseURL string, base *http.Client) (*OnboardingClient, error) { - normalized, err := exactBaseURL(baseURL) +func NewOnboardingClient(baseURL string, base *http.Client, appEnv string) (*OnboardingClient, error) { + normalized, err := exactBaseURL(baseURL, appEnv) if err != nil { return nil, errors.New("auth center URL is invalid") } @@ -169,7 +173,7 @@ func NewOnboardingClient(baseURL string, base *http.Client) (*OnboardingClient, client.Timeout = 10 * time.Second } client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse } - return &OnboardingClient{baseURL: normalized, client: &client}, nil + return &OnboardingClient{baseURL: normalized, client: &client, appEnv: appEnv}, nil } func (client *OnboardingClient) Claim(ctx context.Context, code string) (ClaimedExchange, error) { @@ -208,7 +212,7 @@ func (client *OnboardingClient) DeliverCredential(ctx context.Context, exchange if err != nil || status != http.StatusOK { return CredentialDelivery{}, onboardingProtocolError(err) } - if err := output.Manifest.Validate(); err != nil { + if err := output.Manifest.Validate(client.appEnv); err != nil { return CredentialDelivery{}, err } output.Version, err = parseWeakETag(etag) diff --git a/apps/api/internal/identity/onboarding_client_test.go b/apps/api/internal/identity/onboarding_client_test.go index aadce38..56aecf4 100644 --- a/apps/api/internal/identity/onboarding_client_test.go +++ b/apps/api/internal/identity/onboarding_client_test.go @@ -39,7 +39,7 @@ func TestOnboardingClientKeepsCodeAndExchangeTokenOutOfURLs(t *testing.T) { })) defer server.Close() - client, err := NewOnboardingClient(server.URL, server.Client()) + client, err := NewOnboardingClient(server.URL, server.Client(), "production") if err != nil { t.Fatal(err) } @@ -63,17 +63,17 @@ func TestManifestV1ValidationRequiresStableFieldsAndCapabilityDependencies(t *te Clients: ManifestClients{BrowserLogin: &ManifestClient{ClientID: "browser"}, MachineToMachine: &ManifestClient{ClientID: "service"}}, SecurityEvents: &ManifestSecurityEvents{TransmitterIssuer: "https://auth.example.com/ssf", ConfigurationEndpoint: "https://auth.example.com/.well-known/ssf-configuration/ssf", Audience: "urn:easyai:ssf:receiver:bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"}, } - if err := valid.Validate(); err != nil { + if err := valid.Validate("production"); err != nil { t.Fatal(err) } invalid := valid invalid.Capabilities = []string{"session_revocation"} - if err := invalid.Validate(); err == nil { + if err := invalid.Validate("production"); err == nil { t.Fatal("manifest with missing capability dependencies was accepted") } invalid = valid invalid.SchemaVersion = 2 - if err := invalid.Validate(); err == nil { + if err := invalid.Validate("production"); err == nil { t.Fatal("unsupported manifest schema was accepted") } } @@ -87,7 +87,7 @@ func TestOnboardingClientRejectsRedirects(t *testing.T) { http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect) })) defer redirect.Close() - client, err := NewOnboardingClient(redirect.URL, redirect.Client()) + client, err := NewOnboardingClient(redirect.URL, redirect.Client(), "production") if err != nil { t.Fatal(err) } @@ -95,3 +95,61 @@ func TestOnboardingClientRejectsRedirects(t *testing.T) { t.Fatal("redirecting onboarding endpoint was accepted") } } + +func TestManifestAllowsLoopbackHTTPOnlyInLocalEnvironments(t *testing.T) { + manifest := ManifestV1{ + SchemaVersion: 1, Issuer: "https://auth.example.com/issuer/easyai", + TenantID: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", ApplicationID: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + Capabilities: []string{"machine_to_machine", "token_introspection", "session_revocation"}, Scopes: []string{"gateway.access"}, + Clients: ManifestClients{MachineToMachine: &ManifestClient{ClientID: "service"}}, + SecurityEvents: &ManifestSecurityEvents{ + TransmitterIssuer: "https://auth.example.com/ssf", ConfigurationEndpoint: "https://auth.example.com/.well-known/ssf-configuration/ssf", + Audience: "urn:easyai:ssf:receiver:bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + }, + } + for _, test := range []struct { + name string + mutate func(*ManifestV1) + }{ + {name: "OIDC issuer", mutate: func(value *ManifestV1) { value.Issuer = "http://localhost:18003/issuer/easyai" }}, + {name: "SSF issuer", mutate: func(value *ManifestV1) { value.SecurityEvents.TransmitterIssuer = "http://127.0.0.1:18004/ssf" }}, + {name: "SSF configuration", mutate: func(value *ManifestV1) { + value.SecurityEvents.ConfigurationEndpoint = "http://127.0.0.1:18004/.well-known/ssf-configuration/ssf" + }}, + } { + t.Run(test.name, func(t *testing.T) { + candidate := manifest + securityEvents := *manifest.SecurityEvents + candidate.SecurityEvents = &securityEvents + test.mutate(&candidate) + if err := candidate.Validate("production"); err == nil { + t.Fatalf("production accepted loopback HTTP %s URL", test.name) + } + }) + } + localManifest := manifest + localSecurityEvents := *manifest.SecurityEvents + localManifest.SecurityEvents = &localSecurityEvents + localManifest.Issuer = "http://localhost:18003/issuer/easyai" + localManifest.SecurityEvents.TransmitterIssuer = "http://127.0.0.1:18004/ssf" + localManifest.SecurityEvents.ConfigurationEndpoint = "http://127.0.0.1:18004/.well-known/ssf-configuration/ssf" + if err := localManifest.Validate("test"); err != nil { + t.Fatalf("test environment rejected loopback HTTP manifest URLs: %v", err) + } + localManifest.Capabilities = []string{"machine_to_machine"} + localManifest.Issuer = manifest.Issuer + if err := localManifest.Validate("production"); err == nil { + t.Fatal("production accepted optional loopback HTTP security event metadata") + } +} + +func TestOnboardingClientAllowsLoopbackHTTPOnlyInLocalEnvironments(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + defer server.Close() + if _, err := NewOnboardingClient(server.URL, server.Client(), "production"); err == nil { + t.Fatal("production accepted loopback HTTP Auth Center URL") + } + if _, err := NewOnboardingClient(server.URL, server.Client(), "development"); err != nil { + t.Fatalf("development rejected loopback HTTP Auth Center URL: %v", err) + } +} diff --git a/apps/api/internal/identity/pairing_service.go b/apps/api/internal/identity/pairing_service.go index 7ddc79c..7d4244e 100644 --- a/apps/api/internal/identity/pairing_service.go +++ b/apps/api/internal/identity/pairing_service.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "errors" "strings" + "sync" "time" "github.com/google/uuid" @@ -20,22 +21,45 @@ const ( PairingCompleted PairingStatus = "completed" PairingFailed PairingStatus = "failed" PairingExpired PairingStatus = "expired" + PairingCancelled PairingStatus = "cancelled" +) + +type PairingCleanupStatus string + +const ( + PairingCleanupNone PairingCleanupStatus = "none" + PairingCleanupPending PairingCleanupStatus = "pending" + PairingCleanupCompleted PairingCleanupStatus = "completed" +) + +var ( + ErrPairingInProgress = errors.New("an identity pairing is already in progress") + ErrPairingNotCancellable = errors.New("identity pairing cannot be cancelled") + ErrPairingConflictNotResolvable = errors.New("identity pairing has no resolvable security event conflict") +) + +const ( + identityPairingStartReservationTTL = 2 * time.Minute + identityPairingStartReleaseTimeout = 5 * time.Second ) type PairingExchange struct { - ID string `json:"id"` - RevisionID string `json:"revisionId"` - RemoteExchangeID string `json:"remoteExchangeId"` - ExchangeTokenRef string `json:"-"` - Status PairingStatus `json:"status"` - RemoteVersion int64 `json:"remoteVersion"` - ExpiresAt time.Time `json:"expiresAt"` - Version int64 `json:"version"` - LastErrorCategory string `json:"lastErrorCategory,omitempty"` - AuthCenterAuditID string `json:"authCenterAuditId,omitempty"` - LastTraceID string `json:"lastTraceId,omitempty"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + RevisionID string `json:"revisionId"` + RemoteExchangeID string `json:"remoteExchangeId"` + ExchangeTokenRef string `json:"-"` + Status PairingStatus `json:"status"` + CleanupStatus PairingCleanupStatus `json:"cleanupStatus"` + RemoteVersion int64 `json:"remoteVersion"` + ExpiresAt time.Time `json:"expiresAt"` + Version int64 `json:"version"` + LastErrorCategory string `json:"lastErrorCategory,omitempty"` + AuthCenterAuditID string `json:"authCenterAuditId,omitempty"` + LastTraceID string `json:"lastTraceId,omitempty"` + CancelledAt *time.Time `json:"cancelledAt,omitempty"` + CleanupCompletedAt *time.Time `json:"cleanupCompletedAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } type PairingExchangeUpdate struct { @@ -46,12 +70,20 @@ type PairingExchangeUpdate struct { } type PairingRepository interface { - CreateIdentityConfigurationRevision(context.Context, Revision) (Revision, error) + ReserveIdentityPairingStart(context.Context, string, time.Time) error + ReleaseIdentityPairingStart(context.Context, string) error + CommitIdentityPairingStart(context.Context, Revision, PairingExchange) (PairingExchange, error) + IdentityPairingStartBlocked(context.Context) (bool, error) IdentityConfigurationRevision(context.Context, string) (Revision, error) ApplyIdentityManifest(context.Context, string, int64, ManifestApplication) (Revision, error) - CreateIdentityPairingExchange(context.Context, PairingExchange) (PairingExchange, error) IdentityPairingExchange(context.Context, string) (PairingExchange, error) UpdateIdentityPairingExchange(context.Context, string, int64, PairingExchangeUpdate) (PairingExchange, error) + RecordIdentityPairingRetryFailure(context.Context, string, PairingStatus, string) (PairingExchange, error) + CancelIdentityPairingExchange(context.Context, string, int64, string, string) (PairingExchange, error) + CompleteIdentityPairingCleanup(context.Context, string, int64) (PairingExchange, error) + RecordIdentityPairingCleanupFailure(context.Context, string, string) (PairingExchange, error) + QueueIdentitySecretCleanup(context.Context, string, time.Time) error + RemovePendingIdentitySecretCleanup(context.Context, string) (bool, error) } type PairingSecretStore interface { @@ -70,6 +102,11 @@ type OnboardingRemote interface { type SecurityEventPreparer interface { PrepareSecurityEvents(context.Context, Revision, []byte) error + CleanupPreparedSecurityEvents(context.Context, Revision) error +} + +type SecurityEventConflictResolver interface { + RetireConflictingSecurityEvents(context.Context, Revision) error } type PairingService struct { @@ -77,22 +114,37 @@ type PairingService struct { secrets PairingSecretStore remote func(string) (OnboardingRemote, error) security SecurityEventPreparer + appEnv string + operations sync.Map } -func NewPairingService(repository PairingRepository, secrets PairingSecretStore, remote func(string) (OnboardingRemote, error), security SecurityEventPreparer) *PairingService { - return &PairingService{repository: repository, secrets: secrets, remote: remote, security: security} +func NewPairingService(repository PairingRepository, secrets PairingSecretStore, remote func(string) (OnboardingRemote, error), security SecurityEventPreparer, appEnvs ...string) *PairingService { + appEnv := "production" + if len(appEnvs) > 0 { + appEnv = appEnvs[0] + } + return &PairingService{repository: repository, secrets: secrets, remote: remote, security: security, appEnv: appEnv} } func (service *PairingService) Start(ctx context.Context, input PairingInput, traceID string) (PairingExchange, error) { - draft, err := NewDraft(input) + draft, err := NewDraft(input, service.appEnv) if err != nil { return PairingExchange{}, err } draft.LastTraceID = strings.TrimSpace(traceID) - draft, err = service.repository.CreateIdentityConfigurationRevision(ctx, draft) - if err != nil { + pairingID := uuid.NewString() + tokenReference := "identity-exchange-" + pairingID + if err := service.repository.ReserveIdentityPairingStart(ctx, pairingID, time.Now().UTC().Add(identityPairingStartReservationTTL)); err != nil { return PairingExchange{}, err } + committed := false + defer func() { + if !committed { + releaseContext, cancel := context.WithTimeout(context.Background(), identityPairingStartReleaseTimeout) + defer cancel() + _ = service.repository.ReleaseIdentityPairingStart(releaseContext, pairingID) + } + }() remote, err := service.remote(draft.AuthCenterURL) if err != nil { return PairingExchange{}, err @@ -101,32 +153,211 @@ func (service *PairingService) Start(ctx context.Context, input PairingInput, tr if err != nil { return PairingExchange{}, err } - pairingID := uuid.NewString() - tokenReference := "identity-exchange-" + pairingID - if err := service.secrets.Put(ctx, tokenReference, []byte(claimed.ExchangeToken)); err != nil { + token := []byte(claimed.ExchangeToken) + claimed.ExchangeToken = "" + if err := service.stageIdentitySecret(ctx, tokenReference, token); err != nil { + clear(token) return PairingExchange{}, err } - claimed.ExchangeToken = "" + clear(token) pairing := PairingExchange{ ID: pairingID, RevisionID: draft.ID, RemoteExchangeID: claimed.ExchangeID, ExchangeTokenRef: tokenReference, - Status: PairingMetadataPending, RemoteVersion: claimed.Version, ExpiresAt: claimed.ExpiresAt, + Status: PairingMetadataPending, CleanupStatus: PairingCleanupNone, RemoteVersion: claimed.Version, ExpiresAt: claimed.ExpiresAt, Version: 1, LastTraceID: strings.TrimSpace(traceID), } - pairing, err = service.repository.CreateIdentityPairingExchange(ctx, pairing) + pairing, err = service.repository.CommitIdentityPairingStart(ctx, draft, pairing) if err != nil { - _ = service.secrets.Delete(ctx, tokenReference) + // Commit may have succeeded even when its response was lost. The + // staging row is the source of truth: rollback leaves it queued, while + // a successful commit atomically adopts it. Re-queueing here could + // delete a Secret that the new Pairing already references. return PairingExchange{}, err } + committed = true return pairing, nil } +func (service *PairingService) Cancel(ctx context.Context, pairingID string, expectedVersion int64, traceID, auditID string) (PairingExchange, error) { + return service.repository.CancelIdentityPairingExchange(ctx, pairingID, expectedVersion, strings.TrimSpace(traceID), strings.TrimSpace(auditID)) +} + +func (service *PairingService) RetireConflictingSecurityEvents(ctx context.Context, pairingID string, expectedVersion int64) (PairingExchange, error) { + unlock := service.lockPairingOperation(pairingID) + defer unlock() + pairing, err := service.repository.IdentityPairingExchange(ctx, pairingID) + if err != nil { + return PairingExchange{}, err + } + if pairing.Version != expectedVersion { + return pairing, ErrRevisionConflict + } + if pairing.Status != PairingCredentialsSaved || pairing.CleanupStatus != PairingCleanupNone || + pairing.LastErrorCategory != "security_event_connection_conflict" { + return pairing, ErrPairingConflictNotResolvable + } + revision, err := service.repository.IdentityConfigurationRevision(ctx, pairing.RevisionID) + if err != nil { + return pairing, err + } + if revision.State != RevisionDraft || !revision.SessionRevocation { + return pairing, ErrPairingConflictNotResolvable + } + resolver, ok := service.security.(SecurityEventConflictResolver) + if !ok { + return pairing, errors.New("security event conflict recovery is unavailable") + } + if err := resolver.RetireConflictingSecurityEvents(ctx, revision); err != nil { + return pairing, err + } + return service.repository.RecordIdentityPairingRetryFailure(ctx, pairing.ID, pairing.Status, "security_event_retirement_pending") +} + +// RestoreCompletedSecurityEvents reconstructs the prepared Receiver after a +// process restart while a completed Revision is still awaiting validation or +// activation. The machine Secret is read only from SecretStore and cleared +// immediately after use. +func (service *PairingService) RestoreCompletedSecurityEvents(ctx context.Context, pairingID string) error { + unlock := service.lockPairingOperation(pairingID) + defer unlock() + pairing, err := service.repository.IdentityPairingExchange(ctx, pairingID) + if err != nil { + return err + } + if pairing.Status != PairingCompleted { + return ErrRevisionConflict + } + revision, err := service.repository.IdentityConfigurationRevision(ctx, pairing.RevisionID) + if err != nil { + return err + } + if revision.State == RevisionActive { + return nil + } + if revision.State != RevisionDraft && revision.State != RevisionValidated { + return ErrRevisionConflict + } + if !revision.SessionRevocation { + return nil + } + if service.security == nil || revision.MachineCredentialRef == "" { + return errors.New("completed security event configuration is unavailable") + } + secret, err := service.secrets.Get(ctx, revision.MachineCredentialRef) + if err != nil { + return err + } + defer clear(secret) + if err := service.security.PrepareSecurityEvents(ctx, revision, secret); err != nil { + return err + } + currentRevision, revisionErr := service.repository.IdentityConfigurationRevision(ctx, pairing.RevisionID) + if revisionErr != nil { + return revisionErr + } + if currentRevision.State == RevisionActive { + return nil + } + currentPairing, pairingErr := service.repository.IdentityPairingExchange(ctx, pairingID) + if pairingErr != nil { + return pairingErr + } + if currentPairing.Status == PairingCompleted && + (currentRevision.State == RevisionDraft || currentRevision.State == RevisionValidated) { + return nil + } + if cleanupErr := service.security.CleanupPreparedSecurityEvents(ctx, currentRevision); cleanupErr != nil { + return cleanupErr + } + return ErrRevisionConflict +} + +func (service *PairingService) Cleanup(ctx context.Context, pairingID string) (PairingExchange, error) { + unlock := service.lockPairingOperation(pairingID) + defer unlock() + pairing, err := service.repository.IdentityPairingExchange(ctx, pairingID) + if err != nil { + return PairingExchange{}, err + } + if pairing.Status != PairingCancelled { + return pairing, ErrPairingNotCancellable + } + if pairing.CleanupStatus == PairingCleanupCompleted { + return pairing, nil + } + if pairing.CleanupStatus != PairingCleanupPending { + return pairing, ErrRevisionConflict + } + revision, err := service.repository.IdentityConfigurationRevision(ctx, pairing.RevisionID) + if err != nil { + return service.recordCleanupFailure(ctx, pairing, "cleanup_revision_unavailable", err) + } + if revision.SessionRevocation { + if service.security == nil { + return service.recordCleanupFailure(ctx, pairing, "cleanup_security_event_unavailable", errors.New("security event cleanup is unavailable")) + } + if err := service.security.CleanupPreparedSecurityEvents(ctx, revision); err != nil { + return service.recordCleanupFailure(ctx, pairing, pairingCleanupFailureCategory(err), err) + } + } + for _, reference := range []string{pairing.ExchangeTokenRef, revision.MachineCredentialRef, revision.SessionEncryptionKeyRef} { + if reference == "" { + continue + } + if err := service.secrets.Delete(ctx, reference); err != nil { + return service.recordCleanupFailure(ctx, pairing, "cleanup_secret_store_failed", err) + } + } + completed, err := service.repository.CompleteIdentityPairingCleanup(ctx, pairing.ID, pairing.Version) + if err != nil { + return service.recordCleanupFailure(ctx, pairing, "cleanup_finalize_failed", err) + } + return completed, nil +} + +func (service *PairingService) recordCleanupFailure(ctx context.Context, pairing PairingExchange, category string, cause error) (PairingExchange, error) { + if errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) { + return pairing, cause + } + recorded, err := service.repository.RecordIdentityPairingCleanupFailure(ctx, pairing.ID, category) + if err != nil { + return pairing, cause + } + return recorded, cause +} + func (service *PairingService) Continue(ctx context.Context, pairingID string) (PairingExchange, error) { + unlock := service.lockPairingOperation(pairingID) + defer unlock() + pairing, err := service.continuePairing(ctx, pairingID) + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return pairing, err + } + current, currentErr := service.repository.IdentityPairingExchange(ctx, pairingID) + if currentErr != nil || !isPairingInProgress(current.Status) { + return pairing, err + } + category := pairingRetryFailureCategory(current.Status, err) + recorded, recordErr := service.repository.RecordIdentityPairingRetryFailure(ctx, current.ID, current.Status, category) + if recordErr != nil { + return current, err + } + return recorded, err +} + +func (service *PairingService) lockPairingOperation(pairingID string) func() { + value, _ := service.operations.LoadOrStore(pairingID, &sync.Mutex{}) + mutex := value.(*sync.Mutex) + mutex.Lock() + return mutex.Unlock +} + +func (service *PairingService) continuePairing(ctx context.Context, pairingID string) (PairingExchange, error) { for step := 0; step < 6; step++ { pairing, err := service.repository.IdentityPairingExchange(ctx, pairingID) if err != nil { return PairingExchange{}, err } - if pairing.Status == PairingCompleted || pairing.Status == PairingFailed || pairing.Status == PairingExpired { + if pairing.Status == PairingCompleted || pairing.Status == PairingFailed || pairing.Status == PairingExpired || pairing.Status == PairingCancelled { return pairing, nil } if !pairing.ExpiresAt.After(time.Now()) { @@ -134,7 +365,7 @@ func (service *PairingService) Continue(ctx context.Context, pairingID string) ( Status: PairingExpired, RemoteVersion: pairing.RemoteVersion, LastErrorCategory: "exchange_expired", }) if updateErr == nil { - _ = service.secrets.Delete(ctx, pairing.ExchangeTokenRef) + service.deleteRetiredIdentitySecret(ctx, pairing.ExchangeTokenRef) } return expired, updateErr } @@ -156,7 +387,7 @@ func (service *PairingService) Continue(ctx context.Context, pairingID string) ( metadata, metadataErr := PairingInput{ AuthCenterURL: revision.AuthCenterURL, PublicBaseURL: revision.PublicBaseURL, WebBaseURL: revision.WebBaseURL, LocalTenantKey: revision.LocalTenantKey, - }.ConsumerMetadata(true) + }.ConsumerMetadata(true, service.appEnv) if metadataErr != nil { clear(token) return PairingExchange{}, metadataErr @@ -224,7 +455,7 @@ func (service *PairingService) Continue(ctx context.Context, pairingID string) ( completeErr := remote.Complete(ctx, pairing.RemoteExchangeID, string(token), "gateway-pairing-complete-"+pairing.ID, pairing.RemoteVersion) clear(token) if completeErr != nil { - return PairingExchange{}, completeErr + return PairingExchange{}, pairingStepError{category: "exchange_completion_failed", cause: completeErr} } completed, err := service.repository.UpdateIdentityPairingExchange(ctx, pairing.ID, pairing.Version, PairingExchangeUpdate{ Status: PairingCompleted, RemoteVersion: pairing.RemoteVersion, @@ -232,7 +463,7 @@ func (service *PairingService) Continue(ctx context.Context, pairingID string) ( if err != nil { return PairingExchange{}, err } - _ = service.secrets.Delete(ctx, pairing.ExchangeTokenRef) + service.deleteRetiredIdentitySecret(ctx, pairing.ExchangeTokenRef) return completed, nil default: clear(token) @@ -242,8 +473,71 @@ func (service *PairingService) Continue(ctx context.Context, pairingID string) ( return service.repository.IdentityPairingExchange(ctx, pairingID) } +func isPairingInProgress(status PairingStatus) bool { + switch status { + case PairingMetadataPending, PairingPreparing, PairingReady, PairingCredentialsSaved: + return true + default: + return false + } +} + +type safeErrorCategory interface { + SafeErrorCategory() string +} + +type pairingStepError struct { + category string + cause error +} + +func (err pairingStepError) Error() string { return "identity pairing step failed: " + err.category } +func (err pairingStepError) Unwrap() error { return err.cause } +func (err pairingStepError) SafeErrorCategory() string { return err.category } + +func pairingRetryFailureCategory(status PairingStatus, err error) string { + var categorized safeErrorCategory + if errors.As(err, &categorized) && categorized.SafeErrorCategory() == "exchange_completion_failed" { + return "exchange_completion_failed" + } + switch status { + case PairingMetadataPending: + return "metadata_submission_failed" + case PairingPreparing: + return "exchange_status_unavailable" + case PairingReady: + return "credential_delivery_failed" + case PairingCredentialsSaved: + if errors.As(err, &categorized) { + switch categorized.SafeErrorCategory() { + case "configuration_invalid", "connection_conflict", "discovery_failed", "management_token_failed", + "stream_create_failed", "stream_response_invalid", "receiver_activation_failed", "preparation_failed", "retirement_pending", + "credential_handoff_unsafe", "connection_binding_missing", "connection_binding_unavailable", + "connection_binding_invalid", "connection_binding_mismatch": + return "security_event_" + categorized.SafeErrorCategory() + } + } + return "security_event_preparation_failed" + default: + return "pairing_step_failed" + } +} + +func pairingCleanupFailureCategory(err error) string { + var categorized safeErrorCategory + if errors.As(err, &categorized) { + switch categorized.SafeErrorCategory() { + case "configuration_invalid", "retirement_pending", "connection_conflict", "connection_cleanup_failed", "secret_cleanup_failed", + "discovery_failed", "management_token_failed", "stream_create_failed", "stream_response_invalid", + "receiver_activation_failed", "preparation_failed", "credential_handoff_unsafe": + return "cleanup_security_event_" + categorized.SafeErrorCategory() + } + } + return "cleanup_security_event_failed" +} + func (service *PairingService) saveDelivery(ctx context.Context, revision Revision, pairing PairingExchange, delivery CredentialDelivery) (Revision, error) { - if err := delivery.Manifest.Validate(); err != nil { + if err := delivery.Manifest.Validate(service.appEnv); err != nil { return Revision{}, err } machineReference := "" @@ -251,10 +545,10 @@ func (service *PairingService) saveDelivery(ctx context.Context, revision Revisi if delivery.MachineCredential == nil || delivery.MachineCredential.ClientID != delivery.Manifest.Clients.MachineToMachine.ClientID { return Revision{}, errors.New("onboarding machine credential is invalid") } - machineReference = "identity-machine-" + revision.ID + machineReference = "identity-machine-" + uuid.NewString() secret := []byte(delivery.MachineCredential.ClientSecret) delivery.MachineCredential.ClientSecret = "" - if err := service.secrets.Put(ctx, machineReference, secret); err != nil { + if err := service.stageIdentitySecret(ctx, machineReference, secret); err != nil { clear(secret) return Revision{}, err } @@ -262,49 +556,77 @@ func (service *PairingService) saveDelivery(ctx context.Context, revision Revisi } sessionReference := "" if delivery.Manifest.Clients.BrowserLogin != nil { - sessionReference = "identity-session-" + revision.ID + sessionReference = "identity-session-" + uuid.NewString() key := make([]byte, 32) if _, err := rand.Read(key); err != nil { - _ = service.secrets.Delete(ctx, machineReference) + _ = service.retireIdentitySecret(ctx, machineReference) return Revision{}, err } - if err := service.secrets.Put(ctx, sessionReference, key); err != nil { + if err := service.stageIdentitySecret(ctx, sessionReference, key); err != nil { clear(key) - _ = service.secrets.Delete(ctx, machineReference) + _ = service.retireIdentitySecret(ctx, machineReference) return Revision{}, err } clear(key) } updated, err := service.repository.ApplyIdentityManifest(ctx, revision.ID, revision.Version, ManifestApplication{ Manifest: delivery.Manifest, MachineCredentialRef: machineReference, SessionEncryptionKeyRef: sessionReference, - TraceID: pairing.LastTraceID, AuditID: pairing.AuthCenterAuditID, + TraceID: pairing.LastTraceID, AuditID: pairing.AuthCenterAuditID, AppEnv: service.appEnv, }) if err != nil { - if machineReference != "" { - _ = service.secrets.Delete(ctx, machineReference) - } - if sessionReference != "" { - _ = service.secrets.Delete(ctx, sessionReference) - } + // ApplyIdentityManifest adopts the staged references in the same DB + // transaction as the Revision update. On rollback the staging rows + // remain; on an ambiguous successful commit they are gone. Do not + // re-queue here or a valid active credential could be destroyed. return Revision{}, err } return updated, nil } +func (service *PairingService) stageIdentitySecret(ctx context.Context, reference string, value []byte) error { + if err := service.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC().Add(10*time.Minute)); err != nil { + return err + } + if err := service.secrets.Put(ctx, reference, value); err != nil { + deleteErr := service.secrets.Delete(ctx, reference) + if deleteErr == nil { + _, _ = service.repository.RemovePendingIdentitySecretCleanup(ctx, reference) + } + return err + } + return nil +} + +func (service *PairingService) retireIdentitySecret(ctx context.Context, reference string) error { + if reference == "" { + return nil + } + return service.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC()) +} + +func (service *PairingService) deleteRetiredIdentitySecret(ctx context.Context, reference string) { + if err := service.secrets.Delete(ctx, reference); err == nil { + _, _ = service.repository.RemovePendingIdentitySecretCleanup(ctx, reference) + } +} + func (service *PairingService) updateFromRemote(ctx context.Context, pairing PairingExchange, remote Exchange) (PairingExchange, error) { status := PairingPreparing + category := "" switch remote.Status { case ExchangeReady, ExchangeCredentialDelivered: status = PairingReady case ExchangeCompleted: status = PairingFailed - remote.LastErrorCategory = "remote_state_invalid" + category = "remote_state_invalid" case ExchangeFailed: status = PairingFailed + category = "remote_exchange_failed" case ExchangeExpired: status = PairingExpired + category = "exchange_expired" } return service.repository.UpdateIdentityPairingExchange(ctx, pairing.ID, pairing.Version, PairingExchangeUpdate{ - Status: status, RemoteVersion: remote.Version, LastErrorCategory: remote.LastErrorCategory, AuthCenterAuditID: remote.AuditID, + Status: status, RemoteVersion: remote.Version, LastErrorCategory: category, AuthCenterAuditID: remote.AuditID, }) } diff --git a/apps/api/internal/identity/pairing_service_test.go b/apps/api/internal/identity/pairing_service_test.go index dd73124..c6869e6 100644 --- a/apps/api/internal/identity/pairing_service_test.go +++ b/apps/api/internal/identity/pairing_service_test.go @@ -2,14 +2,68 @@ package identity import ( "context" + "encoding/json" "errors" + "strings" + "sync" "testing" "time" ) type pairingRepositoryFake struct { - revision Revision - exchange PairingExchange + revision Revision + exchange PairingExchange + blocked bool + startReservationErr error + startReservationID string + startReservationState string + startReservationCalls int + startReservationReleases int + pendingSecretCleanups map[string]time.Time + commitStartErr error + commitStartAmbiguous bool + applyManifestAmbiguousOnce bool + failCredentialsSavedOnce bool +} + +func (f *pairingRepositoryFake) ReserveIdentityPairingStart(_ context.Context, attemptID string, _ time.Time) error { + f.startReservationCalls++ + if f.startReservationErr != nil { + return f.startReservationErr + } + if f.blocked || f.startReservationID != "" { + return ErrPairingInProgress + } + f.startReservationID = attemptID + f.startReservationState = "starting" + return nil +} +func (f *pairingRepositoryFake) ReleaseIdentityPairingStart(_ context.Context, attemptID string) error { + if f.startReservationID == attemptID && f.startReservationState == "starting" { + f.startReservationID = "" + f.startReservationState = "" + f.startReservationReleases++ + } + return nil +} +func (f *pairingRepositoryFake) CommitIdentityPairingStart(_ context.Context, revision Revision, exchange PairingExchange) (PairingExchange, error) { + if f.commitStartErr != nil && !f.commitStartAmbiguous { + return PairingExchange{}, f.commitStartErr + } + if f.startReservationID != exchange.ID || f.startReservationState != "starting" { + return PairingExchange{}, ErrPairingInProgress + } + if _, staged := f.pendingSecretCleanups[exchange.ExchangeTokenRef]; !staged { + return PairingExchange{}, errors.New("exchange token was not staged") + } + delete(f.pendingSecretCleanups, exchange.ExchangeTokenRef) + f.revision = revision + f.exchange = exchange + f.startReservationState = "paired" + if f.commitStartAmbiguous { + return PairingExchange{}, f.commitStartErr + } + return exchange, nil } func (f *pairingRepositoryFake) CreateIdentityConfigurationRevision(_ context.Context, revision Revision) (Revision, error) { @@ -20,12 +74,33 @@ func (f *pairingRepositoryFake) IdentityConfigurationRevision(context.Context, s return f.revision, nil } func (f *pairingRepositoryFake) ApplyIdentityManifest(_ context.Context, _ string, _ int64, applied ManifestApplication) (Revision, error) { + for _, reference := range []string{applied.MachineCredentialRef, applied.SessionEncryptionKeyRef} { + if reference == "" { + continue + } + if _, staged := f.pendingSecretCleanups[reference]; !staged { + return Revision{}, errors.New("identity secret was not staged") + } + } + oldMachineReference, oldSessionReference := f.revision.MachineCredentialRef, f.revision.SessionEncryptionKeyRef revision, err := ApplyManifest(f.revision, applied) if err != nil { return Revision{}, err } revision.Version++ f.revision = revision + for _, reference := range []string{applied.MachineCredentialRef, applied.SessionEncryptionKeyRef} { + delete(f.pendingSecretCleanups, reference) + } + for _, reference := range []string{oldMachineReference, oldSessionReference} { + if reference != "" && reference != applied.MachineCredentialRef && reference != applied.SessionEncryptionKeyRef { + f.pendingSecretCleanups[reference] = time.Now() + } + } + if f.applyManifestAmbiguousOnce { + f.applyManifestAmbiguousOnce = false + return Revision{}, errors.New("manifest commit response lost") + } return revision, nil } func (f *pairingRepositoryFake) CreateIdentityPairingExchange(_ context.Context, exchange PairingExchange) (PairingExchange, error) { @@ -39,9 +114,97 @@ func (f *pairingRepositoryFake) UpdateIdentityPairingExchange(_ context.Context, if f.exchange.ID != id || f.exchange.Version != expected { return PairingExchange{}, ErrRevisionConflict } + if update.Status == PairingCredentialsSaved && f.failCredentialsSavedOnce { + f.failCredentialsSavedOnce = false + return PairingExchange{}, errors.New("pairing status temporarily unavailable") + } f.exchange.Status, f.exchange.RemoteVersion = update.Status, update.RemoteVersion f.exchange.LastErrorCategory, f.exchange.AuthCenterAuditID = update.LastErrorCategory, update.AuthCenterAuditID f.exchange.Version++ + if update.Status == PairingCompleted || update.Status == PairingFailed || update.Status == PairingExpired { + if f.pendingSecretCleanups == nil { + f.pendingSecretCleanups = map[string]time.Time{} + } + f.pendingSecretCleanups[f.exchange.ExchangeTokenRef] = time.Now() + } + return f.exchange, nil +} +func (f *pairingRepositoryFake) RecordIdentityPairingRetryFailure(_ context.Context, id string, status PairingStatus, category string) (PairingExchange, error) { + if f.exchange.ID != id || f.exchange.Status != status { + return PairingExchange{}, ErrRevisionConflict + } + f.exchange.LastErrorCategory = category + return f.exchange, nil +} +func (f *pairingRepositoryFake) IdentityPairingStartBlocked(context.Context) (bool, error) { + return f.blocked, nil +} +func (f *pairingRepositoryFake) QueueIdentitySecretCleanup(_ context.Context, reference string, notBefore time.Time) error { + if f.pendingSecretCleanups == nil { + f.pendingSecretCleanups = map[string]time.Time{} + } + current, exists := f.pendingSecretCleanups[reference] + if !exists || notBefore.Before(current) { + f.pendingSecretCleanups[reference] = notBefore + } + return nil +} +func (f *pairingRepositoryFake) RemovePendingIdentitySecretCleanup(_ context.Context, reference string) (bool, error) { + if _, exists := f.pendingSecretCleanups[reference]; !exists { + return false, nil + } + delete(f.pendingSecretCleanups, reference) + return true, nil +} +func (f *pairingRepositoryFake) CancelIdentityPairingExchange(_ context.Context, id string, expected int64, traceID, auditID string) (PairingExchange, error) { + if f.exchange.ID != id || f.revision.State == RevisionActive || f.revision.State == RevisionSuperseded { + return PairingExchange{}, ErrRevisionConflict + } + if f.exchange.Status == PairingCancelled { + return f.exchange, nil + } + if f.exchange.Version != expected { + return PairingExchange{}, ErrRevisionConflict + } + f.exchange.Status = PairingCancelled + f.exchange.CleanupStatus = PairingCleanupPending + f.exchange.Version++ + now := time.Now().UTC() + f.exchange.CancelledAt = &now + f.revision.State = RevisionFailed + f.revision.LastErrorCategory = "pairing_cancelled" + f.revision.LastTraceID = traceID + f.revision.LastAuditID = auditID + f.revision.Version++ + if f.exchange.ExchangeTokenRef != "" { + if f.pendingSecretCleanups == nil { + f.pendingSecretCleanups = map[string]time.Time{} + } + f.pendingSecretCleanups[f.exchange.ExchangeTokenRef] = time.Now() + } + return f.exchange, nil +} +func (f *pairingRepositoryFake) CompleteIdentityPairingCleanup(_ context.Context, id string, expected int64) (PairingExchange, error) { + if f.exchange.ID != id || f.exchange.Version != expected || f.exchange.Status != PairingCancelled || f.exchange.CleanupStatus != PairingCleanupPending { + return PairingExchange{}, ErrRevisionConflict + } + f.exchange.CleanupStatus = PairingCleanupCompleted + f.exchange.Version++ + now := time.Now().UTC() + f.exchange.CleanupCompletedAt = &now + f.revision.MachineCredentialRef = "" + f.revision.SessionEncryptionKeyRef = "" + if f.startReservationState == "paired" && f.startReservationID == f.exchange.ID { + f.startReservationID = "" + f.startReservationState = "" + } + return f.exchange, nil +} +func (f *pairingRepositoryFake) RecordIdentityPairingCleanupFailure(_ context.Context, id string, category string) (PairingExchange, error) { + if f.exchange.ID != id || f.exchange.Status != PairingCancelled || f.exchange.CleanupStatus != PairingCleanupPending { + return PairingExchange{}, ErrRevisionConflict + } + f.exchange.LastErrorCategory = category return f.exchange, nil } @@ -74,14 +237,18 @@ func (f *secretStoreFake) Delete(_ context.Context, reference string) error { type onboardingRemoteFake struct { claimed ClaimedExchange + claimErr error + claimCalls int view Exchange delivery CredentialDelivery deliveryCalls int completed bool + completeErr error } func (f *onboardingRemoteFake) Claim(context.Context, string) (ClaimedExchange, error) { - return f.claimed, nil + f.claimCalls++ + return f.claimed, f.claimErr } func (f *onboardingRemoteFake) SubmitMetadata(context.Context, ClaimedExchange, ConsumerMetadata, string) (Exchange, error) { f.view.Status, f.view.Version = ExchangePreparing, 2 @@ -99,14 +266,159 @@ func (f *onboardingRemoteFake) DeliverCredential(context.Context, Exchange, stri } func (f *onboardingRemoteFake) Complete(context.Context, string, string, string, int64) error { f.completed = true + return f.completeErr +} + +type securityEventPreparerFake struct { + called bool + err error + cleanupCalled bool + cleanupErr error + conflictRetireCalled bool + conflictRetireErr error + conflictRevisionID string +} + +type blockingSecurityEventPreparer struct { + prepareStarted chan struct{} + releasePrepare chan struct{} + cleanupStarted chan struct{} + cleanupOnce sync.Once +} + +func (preparer *blockingSecurityEventPreparer) PrepareSecurityEvents(context.Context, Revision, []byte) error { + close(preparer.prepareStarted) + <-preparer.releasePrepare return nil } -type securityEventPreparerFake struct{ called bool } +func (preparer *blockingSecurityEventPreparer) CleanupPreparedSecurityEvents(context.Context, Revision) error { + preparer.cleanupOnce.Do(func() { close(preparer.cleanupStarted) }) + return nil +} func (f *securityEventPreparerFake) PrepareSecurityEvents(context.Context, Revision, []byte) error { f.called = true - return nil + return f.err +} +func (f *securityEventPreparerFake) CleanupPreparedSecurityEvents(context.Context, Revision) error { + f.cleanupCalled = true + return f.cleanupErr +} +func (f *securityEventPreparerFake) RetireConflictingSecurityEvents(_ context.Context, revision Revision) error { + f.conflictRetireCalled = true + f.conflictRevisionID = revision.ID + return f.conflictRetireErr +} + +type safeCategoryTestError struct { + category string + message string +} + +func (err safeCategoryTestError) Error() string { return err.message } +func (err safeCategoryTestError) SafeErrorCategory() string { return err.category } + +func TestPairingStartReservationSerializesAndRecoversFailedClaims(t *testing.T) { + input := PairingInput{ + AuthCenterURL: "https://auth.example.com", OnboardingCode: "one-time-code-value", + PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default", + } + + t.Run("existing pairing blocks before remote claim", func(t *testing.T) { + repository := &pairingRepositoryFake{blocked: true} + remote := &onboardingRemoteFake{} + service := NewPairingService(repository, &secretStoreFake{}, func(string) (OnboardingRemote, error) { return remote, nil }, nil) + + if _, err := service.Start(context.Background(), input, "trace-blocked"); !errors.Is(err, ErrPairingInProgress) { + t.Fatalf("blocked Start error=%v", err) + } + if repository.startReservationCalls != 1 || repository.startReservationReleases != 0 || repository.startReservationID != "" || remote.claimCalls != 0 { + t.Fatalf("reservation/claim lifecycle calls=%d releases=%d id=%q claims=%d", + repository.startReservationCalls, repository.startReservationReleases, repository.startReservationID, remote.claimCalls) + } + }) + + t.Run("active machine credential blocks destructive rotation before remote claim", func(t *testing.T) { + repository := &pairingRepositoryFake{startReservationErr: ErrActiveConfigurationHandoffRequired} + remote := &onboardingRemoteFake{} + service := NewPairingService(repository, &secretStoreFake{}, func(string) (OnboardingRemote, error) { return remote, nil }, nil) + + if _, err := service.Start(context.Background(), input, "trace-active"); !errors.Is(err, ErrActiveConfigurationHandoffRequired) { + t.Fatalf("active credential guard error=%v", err) + } + if remote.claimCalls != 0 { + t.Fatal("active credential guard ran after the one-time onboarding code was claimed") + } + }) + + t.Run("remote failure releases database guard", func(t *testing.T) { + repository := &pairingRepositoryFake{} + remote := &onboardingRemoteFake{claimErr: errors.New("remote unavailable")} + service := NewPairingService(repository, &secretStoreFake{}, func(string) (OnboardingRemote, error) { return remote, nil }, nil) + + if _, err := service.Start(context.Background(), input, "trace-failed"); err == nil { + t.Fatal("remote claim failure was ignored") + } + if repository.startReservationCalls != 1 || repository.startReservationReleases != 1 || repository.startReservationID != "" || remote.claimCalls != 1 { + t.Fatalf("reservation was not released after remote failure: %#v remote=%#v", repository, remote) + } + }) +} + +func TestPairingStartLeavesUncommittedExchangeTokenInDurableCleanupQueue(t *testing.T) { + repository := &pairingRepositoryFake{commitStartErr: errors.New("database commit unavailable")} + secrets := &secretStoreFake{values: map[string][]byte{}} + remote := &onboardingRemoteFake{claimed: ClaimedExchange{ + ExchangeID: "11111111-1111-1111-1111-111111111111", ExchangeToken: "exchange-token-value", + Version: 1, ExpiresAt: time.Now().Add(time.Hour), + }} + service := NewPairingService(repository, secrets, func(string) (OnboardingRemote, error) { return remote, nil }, nil) + + if _, err := service.Start(context.Background(), PairingInput{ + AuthCenterURL: "https://auth.example.com", OnboardingCode: "one-time-code-value", + PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default", + }, "trace-uncommitted"); err == nil { + t.Fatal("failed pairing commit was ignored") + } + if len(secrets.values) != 1 || len(repository.pendingSecretCleanups) != 1 || repository.startReservationID != "" || repository.startReservationReleases != 1 { + t.Fatalf("uncommitted token was not left recoverable: secrets=%d cleanup=%d reservation=%q releases=%d", + len(secrets.values), len(repository.pendingSecretCleanups), repository.startReservationID, repository.startReservationReleases) + } + for reference := range secrets.values { + if _, queued := repository.pendingSecretCleanups[reference]; !queued { + t.Fatalf("uncommitted Secret reference %q is not queued for cleanup", reference) + } + } +} + +func TestPairingStartAmbiguousCommitNeverRequeuesAdoptedExchangeToken(t *testing.T) { + repository := &pairingRepositoryFake{ + commitStartErr: errors.New("commit response lost"), commitStartAmbiguous: true, + } + secrets := &secretStoreFake{values: map[string][]byte{}} + remote := &onboardingRemoteFake{claimed: ClaimedExchange{ + ExchangeID: "11111111-1111-1111-1111-111111111111", ExchangeToken: "exchange-token-value", + Version: 1, ExpiresAt: time.Now().Add(time.Hour), + }} + service := NewPairingService(repository, secrets, func(string) (OnboardingRemote, error) { return remote, nil }, nil) + + if _, err := service.Start(context.Background(), PairingInput{ + AuthCenterURL: "https://auth.example.com", OnboardingCode: "one-time-code-value", + PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default", + }, "trace-ambiguous"); err == nil { + t.Fatal("ambiguous pairing commit was not reported") + } + if repository.exchange.ID == "" || repository.revision.ID == "" { + t.Fatal("test did not simulate a committed Pairing") + } + if len(secrets.values) != 1 || len(repository.pendingSecretCleanups) != 0 { + t.Fatalf("adopted exchange token was re-queued: secrets=%d cleanup=%d", len(secrets.values), len(repository.pendingSecretCleanups)) + } + if repository.startReservationID != repository.exchange.ID || repository.startReservationState != "paired" || repository.startReservationReleases != 0 { + t.Fatalf("committed reservation was not retained: id=%q state=%q releases=%d", + repository.startReservationID, repository.startReservationState, repository.startReservationReleases) + } } func TestPairingRetriesWithRotatedCredentialAfterSecretStoreFailure(t *testing.T) { @@ -155,6 +467,91 @@ func TestPairingRetriesWithRotatedCredentialAfterSecretStoreFailure(t *testing.T } } +func TestPairingRetryNeverOverwritesOrDeletesAdoptedIdentitySecrets(t *testing.T) { + repository := &pairingRepositoryFake{applyManifestAmbiguousOnce: true, failCredentialsSavedOnce: true} + secrets := &secretStoreFake{values: map[string][]byte{}} + remote := &onboardingRemoteFake{ + claimed: ClaimedExchange{ExchangeID: "11111111-1111-1111-1111-111111111111", ExchangeToken: "exchange-token-value", Version: 1, ExpiresAt: time.Now().Add(time.Hour)}, + view: Exchange{ExchangeID: "11111111-1111-1111-1111-111111111111", ApplicationID: "22222222-2222-2222-2222-222222222222", Version: 1, ExpiresAt: time.Now().Add(time.Hour)}, + delivery: CredentialDelivery{Manifest: ManifestV1{ + SchemaVersion: 1, Issuer: "https://auth.example.com/issuer/shared", TenantID: "33333333-3333-3333-3333-333333333333", + ApplicationID: "22222222-2222-2222-2222-222222222222", Audience: "urn:easyai:resource:22222222-2222-2222-2222-222222222222", + Capabilities: []string{"oidc_login", "api_access", "machine_to_machine", "token_introspection", "session_revocation"}, Scopes: []string{"openid", "gateway.access"}, + Clients: ManifestClients{BrowserLogin: &ManifestClient{ClientID: "browser"}, MachineToMachine: &ManifestClient{ClientID: "service"}}, + SecurityEvents: &ManifestSecurityEvents{TransmitterIssuer: "https://auth.example.com/ssf", ConfigurationEndpoint: "https://auth.example.com/.well-known/ssf-configuration/ssf", Audience: "urn:easyai:ssf:receiver:22222222-2222-2222-2222-222222222222"}, + }, MachineCredential: &MachineCredential{ClientID: "service", ClientSecret: "placeholder", IssuedAt: time.Now()}}, + } + preparer := &securityEventPreparerFake{} + service := NewPairingService(repository, secrets, func(string) (OnboardingRemote, error) { return remote, nil }, preparer) + pairing, err := service.Start(context.Background(), PairingInput{ + AuthCenterURL: "https://auth.example.com", OnboardingCode: "one-time-code-value", + PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", LocalTenantKey: "default", + }, "trace-retry") + if err != nil { + t.Fatal(err) + } + + if _, err := service.Continue(context.Background(), pairing.ID); err == nil { + t.Fatal("simulated pairing status persistence failure was ignored") + } + firstMachine := repository.revision.MachineCredentialRef + firstSession := repository.revision.SessionEncryptionKeyRef + if firstMachine == "" || firstSession == "" { + t.Fatalf("first delivery was not adopted: %#v", repository.revision) + } + if _, ok := secrets.values[firstMachine]; !ok { + t.Fatal("first adopted machine Secret was deleted") + } + if _, ok := secrets.values[firstSession]; !ok { + t.Fatal("first adopted session Secret was deleted") + } + for _, reference := range []string{firstMachine, firstSession} { + if _, queued := repository.pendingSecretCleanups[reference]; queued { + t.Fatalf("ambiguously adopted identity Secret %q was re-queued", reference) + } + } + + if _, err := service.Continue(context.Background(), pairing.ID); err == nil { + t.Fatal("simulated pairing status persistence failure was ignored") + } + secondMachine := repository.revision.MachineCredentialRef + secondSession := repository.revision.SessionEncryptionKeyRef + if secondMachine == firstMachine || secondSession == firstSession { + t.Fatalf("identity Secret references were overwritten in place: first=(%s,%s) second=(%s,%s)", firstMachine, firstSession, secondMachine, secondSession) + } + for _, reference := range []string{firstMachine, firstSession} { + if _, queued := repository.pendingSecretCleanups[reference]; !queued { + t.Fatalf("first superseded identity Secret %q was not queued for cleanup", reference) + } + } + + completed, err := service.Continue(context.Background(), pairing.ID) + if err != nil { + t.Fatal(err) + } + if completed.Status != PairingCompleted { + t.Fatalf("retry did not complete: %#v", completed) + } + thirdMachine := repository.revision.MachineCredentialRef + thirdSession := repository.revision.SessionEncryptionKeyRef + if thirdMachine == secondMachine || thirdSession == secondSession { + t.Fatalf("second retry overwrote identity Secret references in place: second=(%s,%s) third=(%s,%s)", secondMachine, secondSession, thirdMachine, thirdSession) + } + for _, reference := range []string{thirdMachine, thirdSession} { + if _, ok := secrets.values[reference]; !ok { + t.Fatalf("active identity Secret %q was deleted", reference) + } + if _, queued := repository.pendingSecretCleanups[reference]; queued { + t.Fatalf("active identity Secret %q remained in cleanup queue", reference) + } + } + for _, reference := range []string{firstMachine, firstSession, secondMachine, secondSession} { + if _, queued := repository.pendingSecretCleanups[reference]; !queued { + t.Fatalf("superseded identity Secret %q was not queued for cleanup", reference) + } + } +} + func TestPairingExpirationIsPersistedAndExchangeTokenIsDestroyed(t *testing.T) { repository := &pairingRepositoryFake{exchange: PairingExchange{ ID: "pairing", RevisionID: "revision", ExchangeTokenRef: "identity-exchange-pairing", @@ -177,3 +574,307 @@ func TestPairingExpirationIsPersistedAndExchangeTokenIsDestroyed(t *testing.T) { t.Fatal("expired exchange token remained in SecretStore") } } + +func TestPairingPersistsOnlySafeSecurityEventFailureCategory(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ + ID: "revision", State: RevisionDraft, AuthCenterURL: "https://auth.example.com", + SessionRevocation: true, MachineCredentialRef: "identity-machine-revision", + }, + exchange: PairingExchange{ + ID: "pairing", RevisionID: "revision", RemoteExchangeID: "11111111-1111-1111-1111-111111111111", + ExchangeTokenRef: "identity-exchange-pairing", Status: PairingCredentialsSaved, + RemoteVersion: 4, ExpiresAt: time.Now().Add(time.Hour), Version: 7, + }, + } + secrets := &secretStoreFake{values: map[string][]byte{ + "identity-exchange-pairing": []byte("temporary-exchange-token-value-000000"), + "identity-machine-revision": []byte("temporary-machine-secret-value-0000000"), + }} + remote := &onboardingRemoteFake{} + preparer := &securityEventPreparerFake{err: safeCategoryTestError{ + category: "discovery_failed", + message: "sensitive-marker https://internal.example/ssf token-value", + }} + service := NewPairingService(repository, secrets, func(string) (OnboardingRemote, error) { return remote, nil }, preparer) + + failed, err := service.Continue(context.Background(), "pairing") + if err == nil { + t.Fatal("security event discovery failure was ignored") + } + if failed.Status != PairingCredentialsSaved || failed.LastErrorCategory != "security_event_discovery_failed" { + t.Fatalf("unsafe or missing pairing failure state: %#v", failed) + } + encoded, marshalErr := json.Marshal(failed) + if marshalErr != nil { + t.Fatal(marshalErr) + } + if strings.Contains(string(encoded), "sensitive-marker") || strings.Contains(string(encoded), "internal.example") || strings.Contains(string(encoded), "token-value") { + t.Fatalf("raw security event error escaped through pairing JSON: %s", encoded) + } + + preparer.err = nil + completed, err := service.Continue(context.Background(), "pairing") + if err != nil { + t.Fatal(err) + } + if completed.Status != PairingCompleted || completed.LastErrorCategory != "" { + t.Fatalf("successful retry did not clear the diagnostic category: %#v", completed) + } +} + +func TestPairingCancellationPersistsIntentBeforeCleaningTemporaryResources(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ + ID: "revision", State: RevisionDraft, Version: 3, SessionRevocation: true, + MachineCredentialRef: "identity-machine-revision", SessionEncryptionKeyRef: "identity-session-revision", + }, + exchange: PairingExchange{ + ID: "pairing", RevisionID: "revision", ExchangeTokenRef: "identity-exchange-pairing", + Status: PairingCredentialsSaved, CleanupStatus: PairingCleanupNone, RemoteVersion: 4, + ExpiresAt: time.Now().Add(time.Hour), Version: 7, + }, + } + secrets := &secretStoreFake{values: map[string][]byte{ + "identity-exchange-pairing": []byte("temporary-exchange-token-value-000000"), + "identity-machine-revision": []byte("temporary-machine-secret-value-0000000"), + "identity-session-revision": []byte("temporary-session-secret-value-0000000"), + }} + preparer := &securityEventPreparerFake{} + service := NewPairingService(repository, secrets, func(string) (OnboardingRemote, error) { + t.Fatal("cancellation must not call the onboarding remote") + return nil, nil + }, preparer) + + cancelled, err := service.Cancel(context.Background(), "pairing", 7, "trace-cancel", "audit-cancel") + if err != nil { + t.Fatal(err) + } + if cancelled.Status != PairingCancelled || cancelled.CleanupStatus != PairingCleanupPending || cancelled.CancelledAt == nil { + t.Fatalf("cancellation intent was not persisted: %#v", cancelled) + } + if repository.revision.State != RevisionFailed || repository.revision.LastErrorCategory != "pairing_cancelled" { + t.Fatalf("draft was not atomically retired: %#v", repository.revision) + } + if len(secrets.values) != 3 { + t.Fatal("synchronous cancellation removed resources before the cleanup worker could resume them") + } + replayed, err := service.Cancel(context.Background(), "pairing", 7, "trace-replay", "audit-replay") + if err != nil || replayed.Version != cancelled.Version || replayed.Status != PairingCancelled { + t.Fatalf("lost cancellation response could not be replayed safely: %#v err=%v", replayed, err) + } + + cleaned, err := service.Cleanup(context.Background(), "pairing") + if err != nil { + t.Fatal(err) + } + if cleaned.CleanupStatus != PairingCleanupCompleted || cleaned.CleanupCompletedAt == nil || !preparer.cleanupCalled { + t.Fatalf("pairing cleanup was not completed: %#v", cleaned) + } + if len(secrets.values) != 0 || repository.revision.MachineCredentialRef != "" || repository.revision.SessionEncryptionKeyRef != "" { + t.Fatalf("temporary secret references survived cleanup: secrets=%d revision=%#v", len(secrets.values), repository.revision) + } +} + +func TestPairingCleanupRetriesWithSafeCategory(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ID: "revision", State: RevisionFailed, Version: 4, SessionRevocation: true}, + exchange: PairingExchange{ + ID: "pairing", RevisionID: "revision", Status: PairingCancelled, CleanupStatus: PairingCleanupPending, + RemoteVersion: 4, ExpiresAt: time.Now().Add(time.Hour), Version: 8, + }, + } + preparer := &securityEventPreparerFake{cleanupErr: safeCategoryTestError{ + category: "retirement_pending", message: "sensitive cleanup marker token-value", + }} + service := NewPairingService(repository, &secretStoreFake{values: map[string][]byte{}}, nil, preparer) + + pending, err := service.Cleanup(context.Background(), "pairing") + if err == nil { + t.Fatal("pending security event retirement was treated as completed") + } + if pending.CleanupStatus != PairingCleanupPending || pending.LastErrorCategory != "cleanup_security_event_retirement_pending" || strings.Contains(pending.LastErrorCategory, "sensitive") { + t.Fatalf("cleanup failure was not safely persisted: %#v", pending) + } + + preparer.cleanupErr = nil + completed, err := service.Cleanup(context.Background(), "pairing") + if err != nil { + t.Fatal(err) + } + if completed.CleanupStatus != PairingCleanupCompleted { + t.Fatalf("cleanup retry did not complete: %#v", completed) + } +} + +func TestPairingMapsUntrustedRemoteErrorCategoryToStableValue(t *testing.T) { + repository := &pairingRepositoryFake{exchange: PairingExchange{ID: "pairing", Version: 2}} + service := NewPairingService(repository, &secretStoreFake{}, nil, nil) + + updated, err := service.updateFromRemote(context.Background(), repository.exchange, Exchange{ + Status: ExchangeFailed, Version: 3, LastErrorCategory: "secret_token_abcdef", + }) + if err != nil { + t.Fatal(err) + } + if updated.Status != PairingFailed || updated.LastErrorCategory != "remote_exchange_failed" { + t.Fatalf("untrusted remote category was persisted: %#v", updated) + } +} + +func TestPairingConflictRetirementIsScopedToCurrentBlockedPairing(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ID: "revision", State: RevisionDraft, Version: 3, SessionRevocation: true}, + exchange: PairingExchange{ + ID: "pairing", RevisionID: "revision", Status: PairingCredentialsSaved, + CleanupStatus: PairingCleanupNone, Version: 7, LastErrorCategory: "security_event_connection_conflict", + }, + } + preparer := &securityEventPreparerFake{} + service := NewPairingService(repository, &secretStoreFake{}, nil, preparer) + + resolved, err := service.RetireConflictingSecurityEvents(context.Background(), "pairing", 7) + if err != nil { + t.Fatal(err) + } + if resolved.ID != "pairing" || resolved.LastErrorCategory != "security_event_retirement_pending" || !preparer.conflictRetireCalled || preparer.conflictRevisionID != "revision" { + t.Fatalf("conflict retirement escaped pairing scope: pairing=%#v preparer=%#v", resolved, preparer) + } + + preparer.conflictRetireCalled = false + if _, err := service.RetireConflictingSecurityEvents(context.Background(), "pairing", 6); !errors.Is(err, ErrRevisionConflict) { + t.Fatalf("stale pairing version error=%v", err) + } + repository.exchange.LastErrorCategory = "" + if _, err := service.RetireConflictingSecurityEvents(context.Background(), "pairing", 7); !errors.Is(err, ErrPairingConflictNotResolvable) { + t.Fatalf("resolved pairing could retire another connection: %v", err) + } + if preparer.conflictRetireCalled { + t.Fatal("stale conflict action reached the security event manager") + } +} + +func TestCompletedPairingRestoresPreparedReceiverAfterRestart(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ + ID: "revision", State: RevisionValidated, SessionRevocation: true, + MachineCredentialRef: "identity-machine-revision", + }, + exchange: PairingExchange{ID: "pairing", RevisionID: "revision", Status: PairingCompleted, Version: 8}, + } + secrets := &secretStoreFake{values: map[string][]byte{ + "identity-machine-revision": []byte("machine-secret-value-restored-from-store"), + }} + preparer := &securityEventPreparerFake{} + service := NewPairingService(repository, secrets, nil, preparer) + + if err := service.RestoreCompletedSecurityEvents(context.Background(), "pairing"); err != nil { + t.Fatal(err) + } + if !preparer.called { + t.Fatal("completed pairing did not reconstruct its prepared security event receiver") + } +} + +func TestCompletedPairingDoesNotRestoreReceiverForSupersededRevision(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ + ID: "revision", State: RevisionSuperseded, Version: 5, SessionRevocation: true, + MachineCredentialRef: "identity-machine-revision", + }, + exchange: PairingExchange{ID: "pairing", RevisionID: "revision", Status: PairingCompleted, Version: 8}, + } + preparer := &securityEventPreparerFake{} + service := NewPairingService(repository, &secretStoreFake{values: map[string][]byte{ + "identity-machine-revision": []byte("must-not-be-read"), + }}, nil, preparer) + + if err := service.RestoreCompletedSecurityEvents(context.Background(), "pairing"); !errors.Is(err, ErrRevisionConflict) { + t.Fatalf("restore error=%v, want revision conflict", err) + } + if preparer.called { + t.Fatal("superseded Revision reconstructed an orphan prepared Receiver") + } +} + +func TestCancelledPairingCleanupWaitsForCompletedReceiverRestore(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ + ID: "revision", State: RevisionValidated, Version: 3, SessionRevocation: true, + MachineCredentialRef: "identity-machine-revision", + }, + exchange: PairingExchange{ + ID: "pairing", RevisionID: "revision", ExchangeTokenRef: "identity-exchange-pairing", + Status: PairingCompleted, CleanupStatus: PairingCleanupNone, Version: 8, + }, + } + secrets := &secretStoreFake{values: map[string][]byte{ + "identity-exchange-pairing": []byte("exchange-token"), + "identity-machine-revision": []byte("machine-secret"), + }} + preparer := &blockingSecurityEventPreparer{ + prepareStarted: make(chan struct{}), + releasePrepare: make(chan struct{}), + cleanupStarted: make(chan struct{}), + } + service := NewPairingService(repository, secrets, nil, preparer) + restoreResult := make(chan error, 1) + go func() { restoreResult <- service.RestoreCompletedSecurityEvents(context.Background(), "pairing") }() + <-preparer.prepareStarted + + cancelled, err := service.Cancel(context.Background(), "pairing", 8, "trace", "audit") + if err != nil { + t.Fatal(err) + } + cleanupResult := make(chan PairingExchange, 1) + cleanupError := make(chan error, 1) + go func() { + cleaned, cleanupErr := service.Cleanup(context.Background(), "pairing") + cleanupResult <- cleaned + cleanupError <- cleanupErr + }() + + select { + case <-preparer.cleanupStarted: + t.Fatal("cleanup overtook an in-flight completed receiver restore") + case <-time.After(100 * time.Millisecond): + } + close(preparer.releasePrepare) + if err := <-restoreResult; !errors.Is(err, ErrRevisionConflict) { + t.Fatalf("restore error = %v, want revision conflict after cancellation", err) + } + if err := <-cleanupError; err != nil { + t.Fatal(err) + } + cleaned := <-cleanupResult + if cancelled.Status != PairingCancelled || cleaned.CleanupStatus != PairingCleanupCompleted { + t.Fatalf("cancel/cleanup did not converge: cancelled=%#v cleaned=%#v", cancelled, cleaned) + } + if len(secrets.values) != 0 { + t.Fatalf("cancelled restore left temporary secrets: %d", len(secrets.values)) + } +} + +func TestPairingClassifiesExchangeCompletionFailureSeparatelyFromSSF(t *testing.T) { + repository := &pairingRepositoryFake{ + revision: Revision{ID: "revision", State: RevisionDraft, AuthCenterURL: "https://auth.example.com"}, + exchange: PairingExchange{ + ID: "pairing", RevisionID: "revision", RemoteExchangeID: "11111111-1111-1111-1111-111111111111", + ExchangeTokenRef: "identity-exchange-pairing", Status: PairingCredentialsSaved, + RemoteVersion: 4, ExpiresAt: time.Now().Add(time.Hour), Version: 7, + }, + } + secrets := &secretStoreFake{values: map[string][]byte{ + "identity-exchange-pairing": []byte("temporary-exchange-token-value-000000"), + }} + remote := &onboardingRemoteFake{completeErr: errors.New("upstream body contains sensitive marker")} + service := NewPairingService(repository, secrets, func(string) (OnboardingRemote, error) { return remote, nil }, nil) + + failed, err := service.Continue(context.Background(), "pairing") + if err == nil { + t.Fatal("exchange completion failure was ignored") + } + if failed.LastErrorCategory != "exchange_completion_failed" { + t.Fatalf("completion failure was misclassified: %#v", failed) + } +} diff --git a/apps/api/internal/identityruntime/builder.go b/apps/api/internal/identityruntime/builder.go index 71374d9..5c3b569 100644 --- a/apps/api/internal/identityruntime/builder.go +++ b/apps/api/internal/identityruntime/builder.go @@ -3,8 +3,10 @@ package identityruntime import ( "context" "errors" + "net/http" "net/url" "slices" + "strings" "sync" "time" @@ -24,10 +26,23 @@ type RuntimeBuilderConfig struct { } type preparedSecurityRuntime struct { - manager *securityevents.ConnectionManager - cancel context.CancelFunc + manager *securityevents.ConnectionManager + cancel context.CancelFunc + receiverReady bool + blockedCategory string } +type safeRuntimeError struct { + category string + cause error +} + +func (err safeRuntimeError) Error() string { + return "identity runtime operation failed: " + err.category +} +func (err safeRuntimeError) Unwrap() error { return err.cause } +func (err safeRuntimeError) SafeErrorCategory() string { return err.category } + type RuntimeBuilder struct { ctx context.Context store *store.Store @@ -54,7 +69,36 @@ func NewRuntimeBuilder(ctx context.Context, data *store.Store, secrets PairingSe return &RuntimeBuilder{ctx: ctx, store: data, secrets: secrets, config: config, metrics: metrics, prepared: map[string]preparedSecurityRuntime{}} } +func (builder *RuntimeBuilder) PreparedSecurityEventReceiver() http.Handler { + builder.mutex.Lock() + defer builder.mutex.Unlock() + if len(builder.prepared) != 1 { + return nil + } + for _, prepared := range builder.prepared { + if prepared.receiverReady { + return prepared.manager + } + } + return nil +} + +func (builder *RuntimeBuilder) PreparedSecurityEventManager() *securityevents.ConnectionManager { + builder.mutex.Lock() + defer builder.mutex.Unlock() + if len(builder.prepared) != 1 { + return nil + } + for _, prepared := range builder.prepared { + return prepared.manager + } + return nil +} + func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revision) (*Runtime, error) { + if err := identity.ValidateRevisionURLs(revision, builder.config.AppEnv); err != nil { + return nil, err + } if builder.store == nil || builder.secrets == nil || revision.Issuer == "" || revision.TenantID == "" || revision.Audience == "" || revision.RolePrefix == "" { return nil, errors.New("identity runtime configuration is incomplete") @@ -69,23 +113,28 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi var securityManager *securityevents.ConnectionManager if revision.SessionRevocation { - prepared := builder.takePreparedSecurityRuntime(revision.ID) + prepared := builder.preparedSecurityRuntime(revision.ID) securityManager = prepared.manager - if prepared.cancel != nil { - runtime.close = func() { + if securityManager != nil { + if err := securityManager.ValidateConfiguredConnectionBinding(ctx); err != nil { cancel() - prepared.cancel() + return nil, err + } + if !prepared.receiverReady { + cancel() + return nil, safeRuntimeError{category: "security_event_not_ready", cause: store.ErrSecurityEventConnectionConflict} } } if securityManager == nil { var err error - securityManager, err = builder.newSecurityEventManager(runtimeCtx, revision) + securityManager, err = builder.newSecurityEventManager(runtimeCtx, revision, true) if err != nil { cancel() return nil, err } } runtime.SecurityEvents = securityManager + runtime.securityEventDisconnector = securityManager } var evaluator func(context.Context, auth.OIDCSecurityEventIdentity) (auth.OIDCSecurityEventEvaluation, error) @@ -100,6 +149,7 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi return revision.MachineClientID, secret, err } verifier, err := auth.NewOIDCVerifier(auth.OIDCConfig{ + AppEnv: builder.config.AppEnv, Issuer: revision.Issuer, Audience: revision.Audience, TenantID: revision.TenantID, RolePrefix: revision.RolePrefix, RequiredScopes: append([]string(nil), revision.Scopes...), JWKSCacheTTL: builder.config.JWKSCacheTTL, IntrospectionEnabled: revision.TokenIntrospection, @@ -134,6 +184,7 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi return nil, err } client, err := auth.NewOIDCPublicClient(auth.OIDCPublicClientConfig{ + AppEnv: builder.config.AppEnv, Issuer: revision.Issuer, ClientID: revision.BrowserClientID, RedirectURI: revision.PublicBaseURL + "/api/v1/auth/oidc/callback", PostLogoutRedirectURI: revision.WebBaseURL + "/", Scopes: append([]string{"openid", "profile"}, revision.Scopes...), @@ -161,46 +212,205 @@ func (builder *RuntimeBuilder) Build(ctx context.Context, revision identity.Revi } func (builder *RuntimeBuilder) PrepareSecurityEvents(ctx context.Context, revision identity.Revision, managementSecret []byte) error { - if !revision.SessionRevocation || revision.SecurityEventIssuer == "" || revision.MachineClientID == "" { + if err := identity.ValidateRevisionURLs(revision, builder.config.AppEnv); err != nil { + return safeRuntimeError{category: "configuration_invalid", cause: err} + } + if !revision.SessionRevocation || revision.SecurityEventIssuer == "" || revision.SecurityEventAudience == "" || revision.MachineClientID == "" { return errors.New("security event configuration is incomplete") } + if prepared := builder.preparedSecurityRuntime(revision.ID); prepared.manager != nil && prepared.blockedCategory != "" { + view, err := prepared.manager.Get(ctx) + if err == nil { + category := prepared.blockedCategory + if view.LifecycleStatus == "retiring" || view.LifecycleStatus == "disconnect_pending" { + category = "retirement_pending" + } + return safeRuntimeError{category: category, cause: store.ErrSecurityEventConnectionConflict} + } + if !errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return safeSecurityEventRuntimeError(err) + } + builder.removePreparedSecurityRuntime(revision.ID, prepared) + } runtimeCtx, cancel := context.WithCancel(builder.ctx) - manager, err := builder.newSecurityEventManager(runtimeCtx, revision) + manager, err := builder.newSecurityEventManager(runtimeCtx, revision, false) if err != nil { cancel() - return err + return safeRuntimeError{category: "configuration_invalid", cause: err} } secretCopy := append([]byte(nil), managementSecret...) _, err = manager.Connect(ctx, revision.SecurityEventIssuer, revision.MachineClientID, secretCopy, "identity-pairing-ssf-"+revision.ID) clear(secretCopy) if err != nil { + // A conflicting persisted connection is itself the resource an + // administrator must safely retire. Keep this manager reachable by the + // recovery API even when there is no Active identity Runtime yet. + if errors.Is(err, store.ErrSecurityEventConnectionConflict) { + builder.replacePreparedSecurityRuntime(revision.ID, preparedSecurityRuntime{ + manager: manager, cancel: cancel, blockedCategory: safeSecurityEventCategory(err, "connection_conflict"), + }) + return safeSecurityEventRuntimeError(err) + } cancel() - return err + return safeSecurityEventRuntimeError(err) } + if err := manager.ValidateConfiguredConnectionBinding(ctx); err != nil { + builder.replacePreparedSecurityRuntime(revision.ID, preparedSecurityRuntime{ + manager: manager, cancel: cancel, blockedCategory: safeSecurityEventCategory(err, "connection_binding_mismatch"), + }) + return safeSecurityEventRuntimeError(err) + } + builder.replacePreparedSecurityRuntime(revision.ID, preparedSecurityRuntime{manager: manager, cancel: cancel, receiverReady: true}) + return nil +} + +func (builder *RuntimeBuilder) removePreparedSecurityRuntime(revisionID string, prepared preparedSecurityRuntime) { builder.mutex.Lock() - previous := builder.prepared[revision.ID] - builder.prepared[revision.ID] = preparedSecurityRuntime{manager: manager, cancel: cancel} + current := builder.prepared[revisionID] + if current.manager == prepared.manager { + delete(builder.prepared, revisionID) + } + builder.mutex.Unlock() + if prepared.cancel != nil { + prepared.cancel() + } +} + +func (builder *RuntimeBuilder) replacePreparedSecurityRuntime(revisionID string, replacement preparedSecurityRuntime) { + builder.mutex.Lock() + previous := builder.prepared[revisionID] + builder.prepared[revisionID] = replacement builder.mutex.Unlock() if previous.cancel != nil { previous.cancel() } +} + +func (builder *RuntimeBuilder) CleanupPreparedSecurityEvents(ctx context.Context, revision identity.Revision) error { + builder.mutex.Lock() + prepared, exists := builder.prepared[revision.ID] + builder.mutex.Unlock() + if !exists { + runtimeCtx, cancel := context.WithCancel(builder.ctx) + manager, err := builder.newSecurityEventManager(runtimeCtx, revision, false) + if err != nil { + cancel() + return safeRuntimeError{category: "configuration_invalid", cause: err} + } + prepared = preparedSecurityRuntime{manager: manager, cancel: cancel} + } + err := prepared.manager.DiscardPreparedConnection(ctx, "identity-pairing-ssf-"+revision.ID) + if err != nil { + builder.mutex.Lock() + if _, alreadyStored := builder.prepared[revision.ID]; !alreadyStored { + builder.prepared[revision.ID] = prepared + } + builder.mutex.Unlock() + return safeSecurityEventRuntimeError(err) + } + builder.mutex.Lock() + current := builder.prepared[revision.ID] + if current.manager == prepared.manager { + delete(builder.prepared, revision.ID) + } + builder.mutex.Unlock() + if prepared.cancel != nil { + prepared.cancel() + } return nil } -func (builder *RuntimeBuilder) newSecurityEventManager(ctx context.Context, revision identity.Revision) (*securityevents.ConnectionManager, error) { - return securityevents.NewConnectionManager(ctx, builder.store, builder.secrets, securityevents.ConnectionManagerConfig{ - AppEnv: builder.config.AppEnv, OIDCEnabled: true, OIDCIssuer: revision.Issuer, OIDCTenantID: revision.TenantID, - ManagementClientID: revision.MachineClientID, PublicBaseURL: revision.PublicBaseURL, - HeartbeatInterval: builder.config.HeartbeatInterval, StaleAfter: builder.config.StaleAfter, ClockSkew: builder.config.ClockSkew, - }, builder.metrics) +func (builder *RuntimeBuilder) RetireConflictingSecurityEvents(ctx context.Context, revision identity.Revision) error { + prepared := builder.preparedSecurityRuntime(revision.ID) + if prepared.manager == nil { + runtimeCtx, cancel := context.WithCancel(builder.ctx) + manager, err := builder.newSecurityEventManager(runtimeCtx, revision, false) + if err != nil { + cancel() + return safeRuntimeError{category: "configuration_invalid", cause: err} + } + prepared = preparedSecurityRuntime{manager: manager, cancel: cancel} + builder.replacePreparedSecurityRuntime(revision.ID, prepared) + } + if err := prepared.manager.RetireConflictingConnection(ctx, "identity-pairing-ssf-"+revision.ID); err != nil { + return safeSecurityEventRuntimeError(err) + } + return nil } -func (builder *RuntimeBuilder) takePreparedSecurityRuntime(revisionID string) preparedSecurityRuntime { +// RecoverSecurityEventDisconnector reconstructs only the SSF management +// surface needed to disable a fail-closed Active Revision. It deliberately +// avoids OIDC discovery, JIT, BFF Session and tenant Runtime construction. +func (builder *RuntimeBuilder) RecoverSecurityEventDisconnector(_ context.Context, revision identity.Revision) (SecurityEventDisconnector, error) { + prepared := builder.preparedSecurityRuntime(revision.ID) + if prepared.manager != nil { + if err := prepared.manager.ValidateConfiguredConnectionBinding(builder.ctx); err != nil { + return nil, err + } + return prepared.manager, nil + } + manager, err := builder.newSecurityEventManager(builder.ctx, revision, true) + if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return manager, nil +} + +// AdoptPreparedSecurityEvents transfers the prepared manager lifetime to an +// activated Runtime. Validation only peeks at prepared state, so Verification +// callbacks remain available between validation and activation. +func (builder *RuntimeBuilder) AdoptPreparedSecurityEvents(revisionID string) context.CancelFunc { builder.mutex.Lock() defer builder.mutex.Unlock() prepared := builder.prepared[revisionID] delete(builder.prepared, revisionID) - return prepared + return prepared.cancel +} + +func safeSecurityEventRuntimeError(err error) error { + var categorized interface{ SafeErrorCategory() string } + if errors.As(err, &categorized) { + return err + } + if errors.Is(err, store.ErrSecurityEventConnectionConflict) { + return safeRuntimeError{category: "connection_conflict", cause: err} + } + return safeRuntimeError{category: "preparation_failed", cause: err} +} + +func safeSecurityEventCategory(err error, fallback string) string { + var categorized interface{ SafeErrorCategory() string } + if errors.As(err, &categorized) && categorized.SafeErrorCategory() != "" { + return categorized.SafeErrorCategory() + } + return fallback +} + +func (builder *RuntimeBuilder) newSecurityEventManager(ctx context.Context, revision identity.Revision, strictRevisionBinding bool) (*securityevents.ConnectionManager, error) { + return securityevents.NewConnectionManager(ctx, builder.store, builder.secrets, builder.securityEventManagerConfig(revision, strictRevisionBinding), builder.metrics) +} + +func (builder *RuntimeBuilder) securityEventManagerConfig(revision identity.Revision, strictRevisionBinding bool) securityevents.ConnectionManagerConfig { + return securityevents.ConnectionManagerConfig{ + AppEnv: builder.config.AppEnv, OIDCEnabled: true, OIDCIssuer: revision.Issuer, OIDCTenantID: revision.TenantID, + ManagementClientID: revision.MachineClientID, PublicBaseURL: revision.PublicBaseURL, + ExpectedTransmitterIssuer: strings.TrimRight(strings.TrimSpace(revision.SecurityEventIssuer), "/"), + ExpectedAudience: revision.SecurityEventAudience, + ExpectedOwnerKey: "identity-pairing-ssf-" + revision.ID, + StrictRevisionBinding: strictRevisionBinding, + HeartbeatInterval: builder.config.HeartbeatInterval, + StaleAfter: builder.config.StaleAfter, + ClockSkew: builder.config.ClockSkew, + } +} + +func (builder *RuntimeBuilder) preparedSecurityRuntime(revisionID string) preparedSecurityRuntime { + builder.mutex.Lock() + defer builder.mutex.Unlock() + return builder.prepared[revisionID] } func secureCookieFor(baseURL string) bool { diff --git a/apps/api/internal/identityruntime/builder_security_events_test.go b/apps/api/internal/identityruntime/builder_security_events_test.go new file mode 100644 index 0000000..b108e54 --- /dev/null +++ b/apps/api/internal/identityruntime/builder_security_events_test.go @@ -0,0 +1,81 @@ +package identityruntime + +import ( + "context" + "strings" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" +) + +func TestRuntimeBuilderRejectsLoopbackHTTPOutsideLocalEnvironments(t *testing.T) { + revision := identity.Revision{ + AuthCenterURL: "https://auth.example.com", Issuer: "https://auth.example.com/issuer/easyai", + PublicBaseURL: "https://api.example.com", WebBaseURL: "https://gateway.example.com", + SecurityEventIssuer: "https://auth.example.com/ssf", SecurityEventConfigURL: "https://auth.example.com/.well-known/ssf-configuration/ssf", + SessionRevocation: true, + } + production := &RuntimeBuilder{config: RuntimeBuilderConfig{AppEnv: "production"}} + for _, test := range []struct { + name string + mutate func(*identity.Revision) + }{ + {name: "auth center", mutate: func(value *identity.Revision) { value.AuthCenterURL = "http://localhost:18000" }}, + {name: "OIDC issuer", mutate: func(value *identity.Revision) { value.Issuer = "http://localhost:18003/issuer/easyai" }}, + {name: "public base", mutate: func(value *identity.Revision) { value.PublicBaseURL = "http://127.0.0.1:18089" }}, + {name: "web base", mutate: func(value *identity.Revision) { value.WebBaseURL = "http://localhost:5178" }}, + {name: "SSF issuer", mutate: func(value *identity.Revision) { value.SecurityEventIssuer = "http://localhost:18004/ssf" }}, + {name: "SSF configuration", mutate: func(value *identity.Revision) { + value.SecurityEventConfigURL = "http://localhost:18004/.well-known/ssf-configuration/ssf" + }}, + } { + t.Run(test.name, func(t *testing.T) { + candidate := revision + test.mutate(&candidate) + if _, err := production.Build(context.Background(), candidate); err == nil || !strings.Contains(err.Error(), "HTTPS") { + t.Fatalf("production runtime did not reject loopback HTTP %s URL: %v", test.name, err) + } + }) + } + localRevision := revision + localRevision.AuthCenterURL = "http://localhost:18000" + localRevision.Issuer = "http://localhost:18003/issuer/easyai" + localRevision.PublicBaseURL = "http://127.0.0.1:18089" + localRevision.WebBaseURL = "http://localhost:5178" + localRevision.SecurityEventIssuer = "http://localhost:18004/ssf" + localRevision.SecurityEventConfigURL = "http://localhost:18004/.well-known/ssf-configuration/ssf" + development := &RuntimeBuilder{config: RuntimeBuilderConfig{AppEnv: "development"}} + if _, err := development.Build(context.Background(), localRevision); err == nil || strings.Contains(err.Error(), "HTTPS") { + t.Fatalf("development runtime did not pass URL policy before completeness checks: %v", err) + } +} + +func TestSecurityEventManagerConfigBindsRuntimeToTargetRevision(t *testing.T) { + builder := &RuntimeBuilder{config: RuntimeBuilderConfig{AppEnv: "test"}} + revision := identity.Revision{ + ID: "revision-target", + Issuer: "https://auth.example/issuer", + TenantID: "tenant-public-id", + MachineClientID: "gateway-machine", + PublicBaseURL: "https://gateway.example", + SecurityEventIssuer: "https://auth.example/ssf/", + SecurityEventAudience: "urn:easyai:ssf:receiver:target", + } + + runtimeConfig := builder.securityEventManagerConfig(revision, true) + if !runtimeConfig.StrictRevisionBinding || + runtimeConfig.ExpectedTransmitterIssuer != "https://auth.example/ssf" || + runtimeConfig.ExpectedAudience != revision.SecurityEventAudience || + runtimeConfig.ManagementClientID != revision.MachineClientID || + runtimeConfig.ExpectedOwnerKey != "identity-pairing-ssf-revision-target" { + t.Fatalf("runtime SSF binding config=%#v", runtimeConfig) + } + + recoveryConfig := builder.securityEventManagerConfig(revision, false) + if recoveryConfig.StrictRevisionBinding { + t.Fatal("conflict recovery was incorrectly blocked by strict Revision binding") + } + if recoveryConfig.ExpectedOwnerKey != runtimeConfig.ExpectedOwnerKey { + t.Fatal("recovery manager lost the target owner needed for explicit validation") + } +} diff --git a/apps/api/internal/identityruntime/manager.go b/apps/api/internal/identityruntime/manager.go index e43e10a..9060bad 100644 --- a/apps/api/internal/identityruntime/manager.go +++ b/apps/api/internal/identityruntime/manager.go @@ -3,6 +3,8 @@ package identityruntime import ( "context" "errors" + "net/http" + "strings" "sync" "sync/atomic" "time" @@ -27,15 +29,20 @@ type Builder interface { Build(context.Context, identity.Revision) (*Runtime, error) } +type SecurityEventDisconnector interface { + Disconnect(context.Context) (securityevents.ConnectionView, error) +} + type Runtime struct { - Revision identity.Revision - Verifier *auth.OIDCVerifier - PublicClient *auth.OIDCPublicClient - Sessions *oidcsession.Service - SessionCipher *oidcsession.Cipher - SecurityEvents *securityevents.ConnectionManager - CookieSecure bool - close func() + Revision identity.Revision + Verifier *auth.OIDCVerifier + PublicClient *auth.OIDCPublicClient + Sessions *oidcsession.Service + SessionCipher *oidcsession.Cipher + SecurityEvents *securityevents.ConnectionManager + CookieSecure bool + close func() + securityEventDisconnector SecurityEventDisconnector } func (runtime *Runtime) Close() { @@ -45,36 +52,126 @@ func (runtime *Runtime) Close() { } type Manager struct { - repository Repository - builder Builder - operation sync.Mutex - current atomic.Pointer[Runtime] + repository Repository + builder Builder + operation sync.Mutex + current atomic.Pointer[Runtime] + reconciliationRequired atomic.Bool + legacyJWTAllowed atomic.Bool + trustedWebBaseURL atomic.Pointer[string] } +const identityRuntimeReconciliationTimeout = 5 * time.Second + func NewManager(repository Repository, builder Builder) *Manager { - return &Manager{repository: repository, builder: builder} + manager := &Manager{repository: repository, builder: builder} + manager.legacyJWTAllowed.Store(true) + return manager } func (manager *Manager) Current() *Runtime { return manager.current.Load() } +func (manager *Manager) ReconciliationRequired() bool { + return manager.reconciliationRequired.Load() +} + +func (manager *Manager) LegacyJWTEnabled() bool { + return manager.legacyJWTAllowed.Load() +} + +// TrustedWebBaseURL is the persisted Active Revision's exact browser origin. +// It intentionally survives a fail-closed Runtime build so the local +// break-glass manager can still repair or disable a broken Active Revision. +func (manager *Manager) TrustedWebBaseURL() string { + value := manager.trustedWebBaseURL.Load() + if value == nil { + return "" + } + return *value +} + +// SecurityEventReceiver resolves the request-time receiver. During first +// onboarding there is no Active Runtime yet, so the builder-owned prepared +// receiver must remain reachable for SSF Verification callbacks. +func (manager *Manager) SecurityEventReceiver() http.Handler { + provider, ok := manager.builder.(interface{ PreparedSecurityEventReceiver() http.Handler }) + if ok { + if prepared := provider.PreparedSecurityEventReceiver(); prepared != nil { + return prepared + } + } + if runtime := manager.Current(); runtime != nil && runtime.SecurityEvents != nil { + return runtime.SecurityEvents + } + return manager.SecurityEventManager() +} + +// SecurityEventManager resolves the manager used by administrative recovery +// operations. A prepared manager may represent an older persisted connection +// that must be retired before the first identity Runtime can be activated. +func (manager *Manager) SecurityEventManager() *securityevents.ConnectionManager { + if runtime := manager.Current(); runtime != nil && runtime.SecurityEvents != nil { + return runtime.SecurityEvents + } + managerProvider, ok := manager.builder.(interface { + PreparedSecurityEventManager() *securityevents.ConnectionManager + }) + if ok { + return managerProvider.PreparedSecurityEventManager() + } + return nil +} + func (manager *Manager) LoadActive(ctx context.Context) error { manager.operation.Lock() defer manager.operation.Unlock() revision, err := manager.repository.ActiveIdentityConfigurationRevision(ctx) if errors.Is(err, identity.ErrRevisionNotFound) { + manager.publishDisabledRuntime() return nil } if err != nil { + manager.failClosedRuntime() return err } + manager.rememberTrustedWebBaseURL(revision.WebBaseURL) + runtime, err := manager.builder.Build(ctx, revision) + if err != nil { + manager.failClosedRuntime() + return err + } + manager.publishRuntime(runtime, revision) + return nil +} + +// ReconcileActive restores a fail-closed Runtime after an uncertain mutation +// outcome. It is safe to call periodically: a matching Active Runtime is left +// untouched, while a changed or missing Active Revision is published atomically. +func (manager *Manager) ReconcileActive(ctx context.Context) error { + manager.operation.Lock() + defer manager.operation.Unlock() + revision, err := manager.repository.ActiveIdentityConfigurationRevision(ctx) + if errors.Is(err, identity.ErrRevisionNotFound) { + manager.publishDisabledRuntime() + return nil + } + if err != nil { + manager.reconciliationRequired.Store(true) + return err + } + manager.rememberTrustedWebBaseURL(revision.WebBaseURL) + if current := manager.current.Load(); current != nil && current.Revision.ID == revision.ID && current.Revision.Version == revision.Version { + manager.reconciliationRequired.Store(false) + return nil + } runtime, err := manager.builder.Build(ctx, revision) if err != nil { + manager.reconciliationRequired.Store(true) return err } - runtime.Revision = revision - manager.current.Store(runtime) + manager.publishRuntime(runtime, revision) return nil } @@ -85,7 +182,7 @@ func (manager *Manager) Validate(ctx context.Context, id string, expectedVersion if err != nil { return identity.Revision{}, err } - if revision.Version != expectedVersion || revision.State != identity.RevisionDraft && revision.State != identity.RevisionSuperseded && revision.State != identity.RevisionActive { + if revision.Version != expectedVersion || revision.State != identity.RevisionDraft && revision.State != identity.RevisionActive { return identity.Revision{}, identity.ErrRevisionConflict } candidate, buildErr := manager.builder.Build(ctx, revision) @@ -101,9 +198,7 @@ func (manager *Manager) Validate(ctx context.Context, id string, expectedVersion candidate.Close() return identity.Revision{}, err } - candidate.Revision = revalidated - old := manager.current.Swap(candidate) - retireRuntime(old) + manager.publishRuntime(candidate, revalidated) return revalidated, nil } candidate.Close() @@ -126,12 +221,9 @@ func (manager *Manager) Activate(ctx context.Context, id string, expectedVersion } activated, _, err := manager.repository.ActivateIdentityRevision(ctx, id, expectedVersion, traceID, auditID) if err != nil { - candidate.Close() - return identity.Revision{}, err + return manager.reconcileActivationMutation(ctx, id, candidate, err) } - candidate.Revision = activated - old := manager.current.Swap(candidate) - retireRuntime(old) + manager.publishRuntime(candidate, activated) return activated, nil } @@ -145,46 +237,235 @@ func (manager *Manager) Rollback(ctx context.Context, id string, expectedVersion if revision.Version != expectedVersion || revision.State != identity.RevisionSuperseded { return identity.Revision{}, identity.ErrRevisionConflict } - candidate, err := manager.builder.Build(ctx, revision) - if err != nil { - return identity.Revision{}, err - } - validated, err := manager.repository.MarkIdentityRevisionValidated(ctx, id, expectedVersion, traceID, auditID) - if err != nil { - candidate.Close() - return identity.Revision{}, err - } - activated, _, err := manager.repository.ActivateIdentityRevision(ctx, id, validated.Version, traceID, auditID) - if err != nil { - candidate.Close() - return identity.Revision{}, err - } - candidate.Revision = activated - old := manager.current.Swap(candidate) - retireRuntime(old) - return activated, nil + // Auth Center currently reuses and mutates OAuth/SSF resources. An older + // local Revision therefore cannot prove that its remote redirect URIs, + // scopes, audiences, or credentials still match. Require a fresh onboarding + // exchange until a versioned remote-resource handoff exists. + return identity.Revision{}, identity.ErrRollbackConfigurationHandoffRequired } func (manager *Manager) Disable(ctx context.Context, expectedVersion int64, traceID, auditID string) (identity.Revision, error) { manager.operation.Lock() defer manager.operation.Unlock() - disabled, err := manager.repository.DisableActiveIdentityRevision(ctx, expectedVersion, traceID, auditID) - if err != nil { + if err := manager.retireActiveSecurityEventsBeforeDisable(ctx, expectedVersion); err != nil { return identity.Revision{}, err } - old := manager.current.Swap(nil) - retireRuntime(old) + disabled, err := manager.repository.DisableActiveIdentityRevision(ctx, expectedVersion, traceID, auditID) + if err != nil { + return manager.reconcileDisableMutation(ctx, expectedVersion, err) + } + manager.publishDisabledRuntime() return disabled, nil } +func (manager *Manager) retireActiveSecurityEventsBeforeDisable(ctx context.Context, expectedVersion int64) error { + active, err := manager.repository.ActiveIdentityConfigurationRevision(ctx) + if err != nil { + return err + } + if active.Version != expectedVersion { + return identity.ErrRevisionConflict + } + if !active.SessionRevocation { + return nil + } + runtime := manager.current.Load() + var disconnector SecurityEventDisconnector + if runtime != nil && runtime.Revision.ID == active.ID { + disconnector = runtime.securityEventDisconnector + if disconnector == nil && runtime.SecurityEvents != nil { + disconnector = runtime.SecurityEvents + } + } + if disconnector == nil { + recoverer, ok := manager.builder.(interface { + RecoverSecurityEventDisconnector(context.Context, identity.Revision) (SecurityEventDisconnector, error) + }) + if !ok { + return identity.ErrSecurityEventRetirementPending + } + var recoverErr error + disconnector, recoverErr = recoverer.RecoverSecurityEventDisconnector(ctx, active) + if recoverErr != nil { + return recoverErr + } + // A missing local connection means there is no bound local Stream or + // credential left to retire. The already-required audit record makes + // this fail-closed recovery decision traceable. + if disconnector == nil { + return nil + } + } + connection, err := disconnector.Disconnect(ctx) + if err != nil { + return err + } + if connection.LifecycleStatus != "retiring" { + return identity.ErrSecurityEventRetirementPending + } + return nil +} + +func (manager *Manager) reconcileActivationMutation(ctx context.Context, targetID string, candidate *Runtime, mutationErr error) (identity.Revision, error) { + reconcileCtx, cancel := identityRuntimeReconciliationContext(ctx) + defer cancel() + active, err := manager.repository.ActiveIdentityConfigurationRevision(reconcileCtx) + if err == nil && active.ID == targetID && active.State == identity.RevisionActive { + manager.publishRuntime(candidate, active) + return active, nil + } + candidate.Close() + if errors.Is(err, identity.ErrRevisionNotFound) { + manager.publishDisabledRuntime() + return identity.Revision{}, mutationErr + } + if err != nil { + manager.failClosedRuntime() + return identity.Revision{}, mutationErr + } + current := manager.current.Load() + if current == nil || current.Revision.ID != active.ID { + manager.failClosedRuntime() + } + return identity.Revision{}, mutationErr +} + +func (manager *Manager) reconcileRollbackValidation(ctx context.Context, targetID string, expectedVersion int64, mutationErr error) (identity.Revision, error) { + reconcileCtx, cancel := identityRuntimeReconciliationContext(ctx) + defer cancel() + revision, err := manager.repository.IdentityConfigurationRevision(reconcileCtx, targetID) + if err == nil && revision.State == identity.RevisionValidated && revision.Version == expectedVersion+1 { + return revision, nil + } + return identity.Revision{}, mutationErr +} + +func (manager *Manager) reconcileDisableMutation(ctx context.Context, expectedVersion int64, mutationErr error) (identity.Revision, error) { + previous := manager.current.Load() + reconcileCtx, cancel := identityRuntimeReconciliationContext(ctx) + defer cancel() + active, err := manager.repository.ActiveIdentityConfigurationRevision(reconcileCtx) + if err == nil { + if previous == nil || active.ID != previous.Revision.ID { + manager.failClosedRuntime() + } + return identity.Revision{}, mutationErr + } + if !errors.Is(err, identity.ErrRevisionNotFound) { + manager.failClosedRuntime() + return identity.Revision{}, mutationErr + } + manager.publishDisabledRuntime() + if previous == nil || previous.Revision.ID == "" { + return identity.Revision{}, mutationErr + } + disabled, lookupErr := manager.repository.IdentityConfigurationRevision(reconcileCtx, previous.Revision.ID) + if lookupErr == nil && disabled.State == identity.RevisionSuperseded && disabled.Version == expectedVersion+1 { + return disabled, nil + } + return identity.Revision{}, mutationErr +} + +func identityRuntimeReconciliationContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), identityRuntimeReconciliationTimeout) +} + +func (manager *Manager) publishRuntime(candidate *Runtime, revision identity.Revision) { + candidate.Revision = revision + manager.rememberTrustedWebBaseURL(revision.WebBaseURL) + manager.legacyJWTAllowed.Store(revision.LegacyJWTEnabled) + old := manager.current.Swap(candidate) + manager.adoptPreparedSecurityEvents(candidate, revision.ID) + manager.reconciliationRequired.Store(false) + retireRuntime(old) +} + +func (manager *Manager) publishDisabledRuntime() { + old := manager.current.Swap(nil) + manager.trustedWebBaseURL.Store(nil) + manager.legacyJWTAllowed.Store(true) + manager.reconciliationRequired.Store(false) + retireRuntime(old) +} + +func (manager *Manager) failClosedRuntime() { + manager.legacyJWTAllowed.Store(false) + manager.reconciliationRequired.Store(true) + old := manager.current.Swap(nil) + retireRuntime(old) +} + +func (manager *Manager) rememberTrustedWebBaseURL(value string) { + normalized := strings.TrimSpace(value) + if normalized == "" { + manager.trustedWebBaseURL.Store(nil) + return + } + manager.trustedWebBaseURL.Store(&normalized) +} + func (manager *Manager) PrepareSecurityEvents(ctx context.Context, revision identity.Revision, secret []byte) error { + manager.operation.Lock() + defer manager.operation.Unlock() + current, err := manager.repository.IdentityConfigurationRevision(ctx, revision.ID) + if err != nil { + return err + } + if current.State == identity.RevisionActive { + // Activation already crossed the prepared-manager adoption point. A + // stale restart restore must not insert a second Receiver afterwards. + return nil + } + if current.Version != revision.Version || current.State != identity.RevisionDraft && current.State != identity.RevisionValidated { + return identity.ErrRevisionConflict + } preparer, ok := manager.builder.(interface { PrepareSecurityEvents(context.Context, identity.Revision, []byte) error }) if !ok { return errors.New("security event runtime preparation is unavailable") } - return preparer.PrepareSecurityEvents(ctx, revision, secret) + return preparer.PrepareSecurityEvents(ctx, current, secret) +} + +func (manager *Manager) CleanupPreparedSecurityEvents(ctx context.Context, revision identity.Revision) error { + cleaner, ok := manager.builder.(interface { + CleanupPreparedSecurityEvents(context.Context, identity.Revision) error + }) + if !ok { + return errors.New("security event runtime cleanup is unavailable") + } + return cleaner.CleanupPreparedSecurityEvents(ctx, revision) +} + +func (manager *Manager) RetireConflictingSecurityEvents(ctx context.Context, revision identity.Revision) error { + resolver, ok := manager.builder.(interface { + RetireConflictingSecurityEvents(context.Context, identity.Revision) error + }) + if !ok { + return errors.New("security event conflict recovery is unavailable") + } + return resolver.RetireConflictingSecurityEvents(ctx, revision) +} + +func (manager *Manager) adoptPreparedSecurityEvents(runtime *Runtime, revisionID string) { + adopter, ok := manager.builder.(interface { + AdoptPreparedSecurityEvents(string) context.CancelFunc + }) + if !ok { + return + } + cancel := adopter.AdoptPreparedSecurityEvents(revisionID) + if cancel == nil { + return + } + closeRuntime := runtime.close + runtime.close = func() { + if closeRuntime != nil { + closeRuntime() + } + cancel() + } } func retireRuntime(runtime *Runtime) { diff --git a/apps/api/internal/identityruntime/manager_test.go b/apps/api/internal/identityruntime/manager_test.go index 2d03e99..6ef7dac 100644 --- a/apps/api/internal/identityruntime/manager_test.go +++ b/apps/api/internal/identityruntime/manager_test.go @@ -3,15 +3,25 @@ package identityruntime import ( "context" "errors" + "net/http" "testing" + "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/securityevents" ) type runtimeRepositoryFake struct { - revisions map[string]identity.Revision - active identity.Revision - activateCalled bool + revisions map[string]identity.Revision + active identity.Revision + activateCalled bool + activeLookupErr error + markValidatedErr error + markValidatedErrAfterApply bool + activateErr error + activateErrAfterApply bool + disableErr error + disableErrAfterApply bool } func (f *runtimeRepositoryFake) IdentityConfigurationRevision(_ context.Context, id string) (identity.Revision, error) { @@ -22,18 +32,27 @@ func (f *runtimeRepositoryFake) IdentityConfigurationRevision(_ context.Context, return revision, nil } func (f *runtimeRepositoryFake) ActiveIdentityConfigurationRevision(context.Context) (identity.Revision, error) { + if f.activeLookupErr != nil { + return identity.Revision{}, f.activeLookupErr + } if f.active.ID == "" { return identity.Revision{}, identity.ErrRevisionNotFound } return f.active, nil } func (f *runtimeRepositoryFake) MarkIdentityRevisionValidated(_ context.Context, id string, expected int64, _, _ string) (identity.Revision, error) { + if f.markValidatedErr != nil && !f.markValidatedErrAfterApply { + return identity.Revision{}, f.markValidatedErr + } revision := f.revisions[id] if revision.Version != expected { return identity.Revision{}, identity.ErrRevisionConflict } revision.State, revision.Version = identity.RevisionValidated, revision.Version+1 f.revisions[id] = revision + if f.markValidatedErr != nil { + return identity.Revision{}, f.markValidatedErr + } return revision, nil } func (f *runtimeRepositoryFake) RevalidateActiveIdentityRevision(_ context.Context, id string, expected int64, traceID, auditID string) (identity.Revision, error) { @@ -56,6 +75,9 @@ func (f *runtimeRepositoryFake) MarkIdentityRevisionFailed(_ context.Context, id } func (f *runtimeRepositoryFake) ActivateIdentityRevision(_ context.Context, id string, expected int64, traceID, auditID string) (identity.Revision, bool, error) { f.activateCalled = true + if f.activateErr != nil && !f.activateErrAfterApply { + return identity.Revision{}, false, f.activateErr + } revision := f.revisions[id] if revision.Version != expected || revision.State != identity.RevisionValidated { return identity.Revision{}, false, identity.ErrRevisionConflict @@ -67,9 +89,15 @@ func (f *runtimeRepositoryFake) ActivateIdentityRevision(_ context.Context, id s } revision.State, revision.Version, revision.LastTraceID, revision.LastAuditID = identity.RevisionActive, revision.Version+1, traceID, auditID f.active, f.revisions[id] = revision, revision + if f.activateErr != nil { + return identity.Revision{}, false, f.activateErr + } return revision, true, nil } func (f *runtimeRepositoryFake) DisableActiveIdentityRevision(_ context.Context, expected int64, traceID, auditID string) (identity.Revision, error) { + if f.disableErr != nil && !f.disableErrAfterApply { + return identity.Revision{}, f.disableErr + } if f.active.Version != expected { return identity.Revision{}, identity.ErrRevisionConflict } @@ -77,22 +105,87 @@ func (f *runtimeRepositoryFake) DisableActiveIdentityRevision(_ context.Context, disabled.State, disabled.Version, disabled.LastTraceID, disabled.LastAuditID = identity.RevisionSuperseded, disabled.Version+1, traceID, auditID f.revisions[disabled.ID] = disabled f.active = identity.Revision{} + if f.disableErr != nil { + return identity.Revision{}, f.disableErr + } return disabled, nil } type runtimeBuilderFake struct { - err error - builtIDs []string + err error + builtIDs []string + buildStarted chan struct{} + buildRelease chan struct{} + prepareStarted chan struct{} + prepareRelease chan struct{} + prepareCalls int + cleanupCalledFor string + adoptedRevision string + preparedCancel context.CancelFunc + preparedManager *securityevents.ConnectionManager + preparedReceiver http.Handler + recoveredSecurityEventDisconnector SecurityEventDisconnector + recoverSecurityEventErr error +} + +type runtimeSecurityEventDisconnector struct { + lifecycle string + err error + calls int +} + +func (f *runtimeSecurityEventDisconnector) Disconnect(context.Context) (securityevents.ConnectionView, error) { + f.calls++ + return securityevents.ConnectionView{LifecycleStatus: f.lifecycle}, f.err } func (f *runtimeBuilderFake) Build(_ context.Context, revision identity.Revision) (*Runtime, error) { f.builtIDs = append(f.builtIDs, revision.ID) + if f.buildStarted != nil { + close(f.buildStarted) + } + if f.buildRelease != nil { + <-f.buildRelease + } if f.err != nil { return nil, f.err } return &Runtime{Revision: revision}, nil } +func (f *runtimeBuilderFake) PrepareSecurityEvents(_ context.Context, _ identity.Revision, _ []byte) error { + f.prepareCalls++ + if f.prepareStarted != nil { + close(f.prepareStarted) + } + if f.prepareRelease != nil { + <-f.prepareRelease + } + return f.err +} + +func (f *runtimeBuilderFake) CleanupPreparedSecurityEvents(_ context.Context, revision identity.Revision) error { + f.cleanupCalledFor = revision.ID + return f.err +} + +func (f *runtimeBuilderFake) AdoptPreparedSecurityEvents(revisionID string) context.CancelFunc { + f.adoptedRevision = revisionID + return f.preparedCancel +} + +func (f *runtimeBuilderFake) PreparedSecurityEventManager() *securityevents.ConnectionManager { + return f.preparedManager +} + +func (f *runtimeBuilderFake) PreparedSecurityEventReceiver() http.Handler { + return f.preparedReceiver +} + +func (f *runtimeBuilderFake) RecoverSecurityEventDisconnector(context.Context, identity.Revision) (SecurityEventDisconnector, error) { + return f.recoveredSecurityEventDisconnector, f.recoverSecurityEventErr +} + func TestValidationFailureKeepsCurrentRuntimeAndDoesNotActivate(t *testing.T) { active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4} draft := identity.Revision{ID: "draft", State: identity.RevisionDraft, Version: 1} @@ -112,6 +205,39 @@ func TestValidationFailureKeepsCurrentRuntimeAndDoesNotActivate(t *testing.T) { } } +func TestLoadActiveMarksTransientFailureForAutomaticReconciliation(t *testing.T) { + active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4} + repository := &runtimeRepositoryFake{ + revisions: map[string]identity.Revision{"active": active}, + active: active, + activeLookupErr: errors.New("database temporarily unavailable"), + } + builder := &runtimeBuilderFake{} + manager := NewManager(repository, builder) + + if err := manager.LoadActive(context.Background()); err == nil { + t.Fatal("transient active lookup failure was ignored") + } + if !manager.ReconciliationRequired() || manager.Current() != nil { + t.Fatal("failed startup load was not left fail-closed and retryable") + } + repository.activeLookupErr = nil + builder.err = errors.New("SecretStore temporarily unavailable") + if err := manager.ReconcileActive(context.Background()); err == nil { + t.Fatal("transient runtime build failure was ignored") + } + if !manager.ReconciliationRequired() { + t.Fatal("failed runtime build cleared automatic reconciliation") + } + builder.err = nil + if err := manager.ReconcileActive(context.Background()); err != nil { + t.Fatal(err) + } + if manager.ReconciliationRequired() || manager.Current() == nil || manager.Current().Revision.ID != active.ID { + t.Fatalf("startup runtime was not recovered: current=%#v required=%v", manager.Current(), manager.ReconciliationRequired()) + } +} + func TestActivationSwapsRuntimeOnlyAfterRepositoryActivation(t *testing.T) { active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2} candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3} @@ -128,21 +254,209 @@ func TestActivationSwapsRuntimeOnlyAfterRepositoryActivation(t *testing.T) { } } -func TestRollbackValidatesAndAtomicallyReplacesActiveRuntime(t *testing.T) { +func TestActivationReconcilesCommitAppliedThenResponseLost(t *testing.T) { + active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2} + candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3} + repository := &runtimeRepositoryFake{ + revisions: map[string]identity.Revision{"old": active, "new": candidate}, + active: active, + activateErr: errors.New("commit response lost"), + activateErrAfterApply: true, + } + manager := NewManager(repository, &runtimeBuilderFake{}) + manager.current.Store(&Runtime{Revision: active}) + + activated, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit") + if err != nil { + t.Fatal(err) + } + if activated.State != identity.RevisionActive || manager.Current() == nil || manager.Current().Revision.ID != candidate.ID { + t.Fatalf("committed activation was not reconciled: activated=%#v current=%#v", activated, manager.Current()) + } +} + +func TestActivationKeepsCurrentRuntimeWhenMutationWasNotApplied(t *testing.T) { + active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2} + candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3} + mutationErr := errors.New("activation rejected before commit") + repository := &runtimeRepositoryFake{ + revisions: map[string]identity.Revision{"old": active, "new": candidate}, + active: active, + activateErr: mutationErr, + } + manager := NewManager(repository, &runtimeBuilderFake{}) + original := &Runtime{Revision: active} + manager.current.Store(original) + + if _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit"); !errors.Is(err, mutationErr) { + t.Fatalf("activation error = %v, want %v", err, mutationErr) + } + if manager.Current() != original { + t.Fatal("definitely uncommitted activation replaced the current runtime") + } +} + +func TestActivationFailsClosedWhenCommitOutcomeCannotBeReconciled(t *testing.T) { + active := identity.Revision{ID: "old", State: identity.RevisionActive, Version: 2} + candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3} + repository := &runtimeRepositoryFake{ + revisions: map[string]identity.Revision{"old": active, "new": candidate}, + active: active, + activateErr: errors.New("commit outcome unknown"), + activeLookupErr: errors.New("database unavailable during reconciliation"), + } + builder := &runtimeBuilderFake{} + manager := NewManager(repository, builder) + manager.current.Store(&Runtime{Revision: active}) + + if _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit"); err == nil { + t.Fatal("unknown activation outcome was reported as successful") + } + if manager.Current() != nil { + t.Fatal("unknown activation outcome did not fail OIDC closed") + } + if !manager.ReconciliationRequired() { + t.Fatal("unknown activation outcome did not request background reconciliation") + } + + repository.activeLookupErr = nil + builder.err = errors.New("runtime dependency temporarily unavailable") + if err := manager.ReconcileActive(context.Background()); err == nil { + t.Fatal("runtime reconciliation unexpectedly ignored build failure") + } + if !manager.ReconciliationRequired() || manager.Current() != nil { + t.Fatal("failed runtime rebuild cleared fail-closed reconciliation state") + } + builder.err = nil + if err := manager.ReconcileActive(context.Background()); err != nil { + t.Fatal(err) + } + if manager.ReconciliationRequired() || manager.Current() == nil || manager.Current().Revision.ID != active.ID { + t.Fatalf("runtime did not recover after reconciliation: current=%#v required=%v", manager.Current(), manager.ReconciliationRequired()) + } +} + +func TestRollbackRejectsOIDCOnlyRevisionBeforeRuntimeBuild(t *testing.T) { active := identity.Revision{ID: "current", State: identity.RevisionActive, Version: 5} previous := identity.Revision{ID: "previous", State: identity.RevisionSuperseded, Version: 3} repository := &runtimeRepositoryFake{ revisions: map[string]identity.Revision{"current": active, "previous": previous}, active: active, } - manager := NewManager(repository, &runtimeBuilderFake{}) + builder := &runtimeBuilderFake{} + manager := NewManager(repository, builder) manager.current.Store(&Runtime{Revision: active}) - rolledBack, err := manager.Rollback(context.Background(), previous.ID, previous.Version, "trace", "audit") + if _, err := manager.Rollback(context.Background(), previous.ID, previous.Version, "trace", "audit"); !errors.Is(err, identity.ErrRollbackConfigurationHandoffRequired) { + t.Fatalf("rollback error=%v, want remote resource handoff required", err) + } + if len(builder.builtIDs) != 0 || repository.activateCalled || manager.Current() == nil || manager.Current().Revision.ID != active.ID { + t.Fatalf("unsafe rollback changed runtime: built=%v activate=%t current=%#v", builder.builtIDs, repository.activateCalled, manager.Current()) + } +} + +func TestRollbackRejectsSupersededMachineCredentialBeforeRuntimeBuild(t *testing.T) { + active := identity.Revision{ID: "current", State: identity.RevisionActive, Version: 5} + previous := identity.Revision{ + ID: "previous", State: identity.RevisionSuperseded, Version: 3, + MachineCredentialRef: "identity-machine-previous", TokenIntrospection: true, + } + repository := &runtimeRepositoryFake{ + revisions: map[string]identity.Revision{"current": active, "previous": previous}, active: active, + } + builder := &runtimeBuilderFake{} + manager := NewManager(repository, builder) + manager.current.Store(&Runtime{Revision: active}) + + if _, err := manager.Rollback(context.Background(), previous.ID, previous.Version, "trace", "audit"); !errors.Is(err, identity.ErrRollbackConfigurationHandoffRequired) { + t.Fatalf("rollback error=%v, want credential handoff required", err) + } + if len(builder.builtIDs) != 0 || repository.activateCalled { + t.Fatal("unsafe rollback reached runtime build or activation") + } +} + +func TestValidateRejectsSupersededRevisionBeforeRuntimeBuild(t *testing.T) { + previous := identity.Revision{ID: "previous", State: identity.RevisionSuperseded, Version: 3} + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"previous": previous}} + builder := &runtimeBuilderFake{} + manager := NewManager(repository, builder) + + if _, err := manager.Validate(context.Background(), previous.ID, previous.Version, "trace", "audit"); !errors.Is(err, identity.ErrRevisionConflict) { + t.Fatalf("validate error=%v, want revision conflict", err) + } + if len(builder.builtIDs) != 0 || repository.revisions[previous.ID].State != identity.RevisionSuperseded { + t.Fatalf("superseded revision reached runtime validation: built=%v revision=%#v", builder.builtIDs, repository.revisions[previous.ID]) + } +} + +func TestDisableKeepsActiveRevisionUntilSecurityEventStreamCanRetire(t *testing.T) { + active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4, SessionRevocation: true} + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active} + disconnector := &runtimeSecurityEventDisconnector{lifecycle: "disconnect_pending"} + manager := NewManager(repository, &runtimeBuilderFake{}) + original := &Runtime{Revision: active, securityEventDisconnector: disconnector} + manager.current.Store(original) + + if _, err := manager.Disable(context.Background(), active.Version, "trace", "audit"); !errors.Is(err, identity.ErrSecurityEventRetirementPending) { + t.Fatalf("disable error=%v, want security event retirement pending", err) + } + if disconnector.calls != 1 || repository.active.ID != active.ID || manager.Current() != original { + t.Fatalf("pending retirement changed Active state: calls=%d repository=%#v current=%#v", disconnector.calls, repository.active, manager.Current()) + } +} + +func TestDisableRecoversSSFWithoutBuildingBrokenFullIdentityRuntime(t *testing.T) { + active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4, SessionRevocation: true} + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active} + disconnector := &runtimeSecurityEventDisconnector{lifecycle: "retiring"} + builder := &runtimeBuilderFake{ + err: errors.New("OIDC discovery unavailable"), + recoveredSecurityEventDisconnector: disconnector, + } + manager := NewManager(repository, builder) + + disabled, err := manager.Disable(context.Background(), active.Version, "trace", "audit") if err != nil { t.Fatal(err) } - if rolledBack.State != identity.RevisionActive || manager.Current().Revision.ID != previous.ID || repository.active.ID != previous.ID { - t.Fatalf("rollback did not replace active runtime: revision=%#v current=%#v", rolledBack, manager.Current()) + if disabled.State != identity.RevisionSuperseded || disconnector.calls != 1 || len(builder.builtIDs) != 0 || manager.Current() != nil { + t.Fatalf("SSF-only recovery did not disable safely: disabled=%#v calls=%d built=%v current=%#v", disabled, disconnector.calls, builder.builtIDs, manager.Current()) + } +} + +func TestDisableRetiresSecurityEventStreamBeforeSupersedingActiveRevision(t *testing.T) { + active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4, SessionRevocation: true} + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active} + disconnector := &runtimeSecurityEventDisconnector{lifecycle: "retiring"} + manager := NewManager(repository, &runtimeBuilderFake{}) + manager.current.Store(&Runtime{Revision: active, securityEventDisconnector: disconnector}) + + disabled, err := manager.Disable(context.Background(), active.Version, "trace", "audit") + if err != nil { + t.Fatal(err) + } + if disconnector.calls != 1 || disabled.State != identity.RevisionSuperseded || repository.active.ID != "" || manager.Current() != nil { + t.Fatalf("safe disable state: calls=%d disabled=%#v repository=%#v current=%#v", disconnector.calls, disabled, repository.active, manager.Current()) + } +} + +func TestDisableReconcilesCommitAppliedThenResponseLost(t *testing.T) { + active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4} + repository := &runtimeRepositoryFake{ + revisions: map[string]identity.Revision{"active": active}, + active: active, + disableErr: errors.New("commit response lost"), + disableErrAfterApply: true, + } + manager := NewManager(repository, &runtimeBuilderFake{}) + manager.current.Store(&Runtime{Revision: active}) + + disabled, err := manager.Disable(context.Background(), active.Version, "trace", "audit") + if err != nil { + t.Fatal(err) + } + if disabled.State != identity.RevisionSuperseded || manager.Current() != nil { + t.Fatalf("committed disable was not reconciled: disabled=%#v current=%#v", disabled, manager.Current()) } } @@ -162,3 +476,192 @@ func TestActiveRevalidationSwapsOnlyAfterSuccessfulValidation(t *testing.T) { t.Fatalf("active runtime was not safely revalidated: %#v", revalidated) } } + +func TestLoadFailureKeepsPersistedActiveWebOriginForBreakGlassRecovery(t *testing.T) { + active := identity.Revision{ + ID: "active", State: identity.RevisionActive, Version: 4, + WebBaseURL: "https://gateway.example.com", + } + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"active": active}, active: active} + manager := NewManager(repository, &runtimeBuilderFake{err: errors.New("OIDC discovery unavailable")}) + + if err := manager.LoadActive(context.Background()); err == nil { + t.Fatal("active runtime load unexpectedly succeeded") + } + if manager.Current() != nil || manager.TrustedWebBaseURL() != active.WebBaseURL { + t.Fatalf("failed Active load lost recovery origin: current=%#v origin=%q", manager.Current(), manager.TrustedWebBaseURL()) + } + + repository.active = identity.Revision{} + if err := manager.ReconcileActive(context.Background()); err != nil { + t.Fatal(err) + } + if manager.TrustedWebBaseURL() != "" { + t.Fatalf("confirmed disable retained stale recovery origin %q", manager.TrustedWebBaseURL()) + } +} + +func TestActivationFinishingFirstPreventsStalePreparedSecurityEventRestore(t *testing.T) { + candidate := identity.Revision{ID: "candidate", State: identity.RevisionValidated, Version: 3} + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"candidate": candidate}} + builder := &runtimeBuilderFake{buildStarted: make(chan struct{}), buildRelease: make(chan struct{})} + manager := NewManager(repository, builder) + + activationDone := make(chan error, 1) + go func() { + _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit") + activationDone <- err + }() + <-builder.buildStarted + restoreDone := make(chan error, 1) + go func() { + restoreDone <- manager.PrepareSecurityEvents(context.Background(), candidate, []byte("secret")) + }() + close(builder.buildRelease) + if err := <-activationDone; err != nil { + t.Fatal(err) + } + if err := <-restoreDone; err != nil { + t.Fatal(err) + } + if builder.prepareCalls != 0 || manager.Current() == nil || manager.Current().Revision.ID != candidate.ID { + t.Fatalf("stale restore survived activation: prepare_calls=%d current=%#v", builder.prepareCalls, manager.Current()) + } +} + +func TestPreparedSecurityEventRestoreFinishingFirstIsAdoptedByActivation(t *testing.T) { + candidate := identity.Revision{ID: "candidate", State: identity.RevisionValidated, Version: 3} + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"candidate": candidate}} + builder := &runtimeBuilderFake{ + buildStarted: make(chan struct{}), buildRelease: make(chan struct{}), + prepareStarted: make(chan struct{}), prepareRelease: make(chan struct{}), + } + manager := NewManager(repository, builder) + + restoreDone := make(chan error, 1) + go func() { + restoreDone <- manager.PrepareSecurityEvents(context.Background(), candidate, []byte("secret")) + }() + <-builder.prepareStarted + activationDone := make(chan error, 1) + go func() { + _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit") + activationDone <- err + }() + select { + case <-builder.buildStarted: + t.Fatal("activation did not wait for prepared Receiver restoration") + case <-time.After(50 * time.Millisecond): + } + close(builder.prepareRelease) + if err := <-restoreDone; err != nil { + t.Fatal(err) + } + <-builder.buildStarted + close(builder.buildRelease) + if err := <-activationDone; err != nil { + t.Fatal(err) + } + if builder.prepareCalls != 1 || builder.adoptedRevision != candidate.ID { + t.Fatalf("prepared restore was not adopted: prepare_calls=%d adopted=%q", builder.prepareCalls, builder.adoptedRevision) + } +} + +func TestManagerDelegatesPreparedSecurityEventCleanupWithoutChangingActiveRuntime(t *testing.T) { + active := identity.Revision{ID: "active", State: identity.RevisionActive, Version: 4} + draft := identity.Revision{ID: "draft", State: identity.RevisionFailed, Version: 2} + builder := &runtimeBuilderFake{} + manager := NewManager(&runtimeRepositoryFake{}, builder) + manager.current.Store(&Runtime{Revision: active}) + + if err := manager.CleanupPreparedSecurityEvents(context.Background(), draft); err != nil { + t.Fatal(err) + } + if builder.cleanupCalledFor != draft.ID || manager.Current().Revision.ID != active.ID { + t.Fatalf("cleanup changed active runtime or skipped the draft: called=%q current=%#v", builder.cleanupCalledFor, manager.Current()) + } +} + +func TestActivationAdoptsPreparedSecurityEventLifetime(t *testing.T) { + candidate := identity.Revision{ID: "new", State: identity.RevisionValidated, Version: 3} + repository := &runtimeRepositoryFake{revisions: map[string]identity.Revision{"new": candidate}} + preparedContext, preparedCancel := context.WithCancel(context.Background()) + builder := &runtimeBuilderFake{preparedCancel: preparedCancel} + manager := NewManager(repository, builder) + + if _, err := manager.Activate(context.Background(), candidate.ID, candidate.Version, "trace", "audit"); err != nil { + t.Fatal(err) + } + if builder.adoptedRevision != candidate.ID { + t.Fatalf("prepared security events were not adopted: %q", builder.adoptedRevision) + } + manager.Current().Close() + select { + case <-preparedContext.Done(): + default: + t.Fatal("closing the active runtime did not release its prepared security event manager") + } +} + +func TestPreparedSecurityEventReceiverRemainsAvailableUntilAdoption(t *testing.T) { + cancelled := false + builder := &RuntimeBuilder{prepared: map[string]preparedSecurityRuntime{ + "draft": {manager: &securityevents.ConnectionManager{}, cancel: func() { cancelled = true }, receiverReady: true}, + }} + if builder.PreparedSecurityEventReceiver() == nil { + t.Fatal("prepared receiver was unavailable before activation") + } + cancel := builder.AdoptPreparedSecurityEvents("draft") + if cancel == nil || builder.PreparedSecurityEventReceiver() != nil { + t.Fatal("prepared receiver ownership was not transferred exactly once") + } + cancel() + if !cancelled { + t.Fatal("adopted receiver lifetime could not be released") + } +} + +func TestSecurityEventManagerExposesPreparedRecoveryManagerWithoutActiveRuntime(t *testing.T) { + prepared := &securityevents.ConnectionManager{} + manager := NewManager(&runtimeRepositoryFake{}, &runtimeBuilderFake{preparedManager: prepared}) + + if manager.SecurityEventManager() != prepared || manager.SecurityEventReceiver() != prepared { + t.Fatal("prepared security event manager was unavailable to recovery handlers") + } +} + +func TestSecurityEventManagerPrefersActiveRuntime(t *testing.T) { + active := &securityevents.ConnectionManager{} + prepared := &securityevents.ConnectionManager{} + manager := NewManager(&runtimeRepositoryFake{}, &runtimeBuilderFake{preparedManager: prepared}) + manager.current.Store(&Runtime{SecurityEvents: active}) + + if manager.SecurityEventManager() != active { + t.Fatal("prepared recovery manager replaced the active runtime manager") + } +} + +func TestSecurityEventReceiverPrefersReadyDraftDuringRepairing(t *testing.T) { + active := &securityevents.ConnectionManager{} + prepared := &securityevents.ConnectionManager{} + manager := NewManager(&runtimeRepositoryFake{}, &runtimeBuilderFake{preparedManager: prepared, preparedReceiver: prepared}) + manager.current.Store(&Runtime{SecurityEvents: active}) + + if manager.SecurityEventReceiver() != prepared { + t.Fatal("verification callback was not routed to the ready draft receiver") + } +} + +func TestConflictingPreparedManagerIsRecoverableButNotUsedAsReceiver(t *testing.T) { + prepared := &securityevents.ConnectionManager{} + builder := &RuntimeBuilder{prepared: map[string]preparedSecurityRuntime{ + "draft": {manager: prepared}, + }} + + if builder.PreparedSecurityEventManager() != prepared { + t.Fatal("conflicting connection manager was unavailable for retirement") + } + if builder.PreparedSecurityEventReceiver() != nil { + t.Fatal("unprepared conflict manager was exposed as a verification receiver") + } +} diff --git a/apps/api/internal/securityevents/connection_manager.go b/apps/api/internal/securityevents/connection_manager.go index 6d572d2..99eee3a 100644 --- a/apps/api/internal/securityevents/connection_manager.go +++ b/apps/api/internal/securityevents/connection_manager.go @@ -34,12 +34,15 @@ type ConnectionRepository interface { CreateSecurityEventConnection(context.Context, store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error) SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) BindSecurityEventConnection(context.Context, string, string, string, string, int64) (store.SecurityEventConnection, error) - SetSecurityEventManagementCredential(context.Context, string, string, string) (store.SecurityEventConnection, error) - UpdateSecurityEventConnectionLifecycle(context.Context, string, string, *string) (store.SecurityEventConnection, error) - SetSecurityEventNextCredential(context.Context, string, string) (store.SecurityEventConnection, error) + SetSecurityEventManagementCredential(context.Context, string, string, string, int64) (store.SecurityEventConnection, error) + TransitionSecurityEventConnectionLifecycle(context.Context, string, string, int64, string, *string) (store.SecurityEventConnection, error) + SetSecurityEventNextCredential(context.Context, string, string, int64) (store.SecurityEventConnection, error) PromoteSecurityEventCredential(context.Context, string, string) (store.SecurityEventConnection, error) - DeleteSecurityEventConnection(context.Context, string) error + DiscardPreparedSecurityEventConnection(context.Context, string, string, int64) error + FinalizeRetiringSecurityEventConnection(context.Context, string, int64) error SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error) + QueueIdentitySecretCleanup(context.Context, string, time.Time) error + RemovePendingIdentitySecretCleanup(context.Context, string) (bool, error) } type ConnectionManagerConfig struct { @@ -49,6 +52,10 @@ type ConnectionManagerConfig struct { OIDCTenantID string ManagementClientID string ManagementClientSecret string + ExpectedTransmitterIssuer string + ExpectedAudience string + ExpectedOwnerKey string + StrictRevisionBinding bool PublicBaseURL string HeartbeatInterval time.Duration StaleAfter time.Duration @@ -79,15 +86,23 @@ type connectionRuntime struct { cancel context.CancelFunc } +type connectionRetirementAuthorization struct { + connectionID string + credentialRef string + ownerKey string +} + type ConnectionManager struct { - ctx context.Context - repository ConnectionRepository - secrets SecretStore - config ConnectionManagerConfig - metrics *Metrics - mutex sync.RWMutex - operation sync.Mutex - runtime *connectionRuntime + ctx context.Context + repository ConnectionRepository + secrets SecretStore + config ConnectionManagerConfig + metrics *Metrics + mutex sync.RWMutex + operation sync.Mutex + runtime *connectionRuntime + retirement *connectionRetirementAuthorization + bindingMismatch bool } type transmitterConfiguration struct { @@ -118,6 +133,17 @@ func (e remoteHTTPError) Error() string { return fmt.Sprintf("SSF endpoint returned HTTP %d", e.status) } +type safeConnectionError struct { + category string + cause error +} + +func (err safeConnectionError) Error() string { + return "security event operation failed: " + err.category +} +func (err safeConnectionError) Unwrap() error { return err.cause } +func (err safeConnectionError) SafeErrorCategory() string { return err.category } + func NewConnectionManager(ctx context.Context, repository ConnectionRepository, secrets SecretStore, config ConnectionManagerConfig, metrics *Metrics) (*ConnectionManager, error) { if repository == nil || secrets == nil || !config.OIDCEnabled || config.OIDCIssuer == "" || config.OIDCTenantID == "" { return nil, errors.New("security event connection prerequisites are incomplete") @@ -138,16 +164,35 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository, config.CredentialOverlapDuration = 3 * time.Minute } if config.RetirementDuration == 0 { - config.RetirementDuration = 6 * time.Minute + config.RetirementDuration = defaultSecurityEventRetirementDuration } manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: config, metrics: metrics} connection, err := repository.SecurityEventConnection(ctx) if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + if config.StrictRevisionBinding { + return nil, safeConnectionError{category: "connection_binding_missing", cause: err} + } return manager, nil } if err != nil { return nil, fmt.Errorf("load security event connection: %w", err) } + if config.StrictRevisionBinding { + if err := manager.validateConnectionBinding(connection); err != nil { + return nil, err + } + } else if hasRevisionBindingExpectation(config) { + if err := manager.validateConnectionBinding(connection); err != nil { + // A recovery manager may inspect a foreign singleton connection, but + // it must not activate it: activate() would combine the old persisted + // Secret with the new Revision's token endpoint. + if connection.LifecycleStatus == "retiring" { + go manager.finishRetirement(connection) + } + manager.bindingMismatch = true + return manager, nil + } + } if connection.LifecycleStatus == "retiring" { if connection.StreamID != nil && connection.Audience != nil { _ = manager.activate(ctx, connection) @@ -165,7 +210,9 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository, if connection.StreamID != nil && connection.Audience != nil { if err := manager.activate(ctx, connection); err != nil { category := "credential_unavailable" - _, _ = repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "degraded", &category) + _, _ = repository.TransitionSecurityEventConnectionLifecycle( + ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "degraded", &category, + ) } } if connection.LifecycleStatus == "bootstrap" { @@ -174,6 +221,42 @@ func NewConnectionManager(ctx context.Context, repository ConnectionRepository, return manager, nil } +func hasRevisionBindingExpectation(config ConnectionManagerConfig) bool { + return strings.TrimSpace(config.ExpectedTransmitterIssuer) != "" || + strings.TrimSpace(config.ExpectedAudience) != "" || + strings.TrimSpace(config.ExpectedOwnerKey) != "" +} + +// ValidateConfiguredConnectionBinding verifies that the singleton persisted +// SSF connection belongs to the identity Revision represented by this manager. +// Recovery managers call this explicitly only when they are about to become an +// Active Runtime; cleanup and retirement remain able to inspect old bindings. +func (m *ConnectionManager) ValidateConfiguredConnectionBinding(ctx context.Context) error { + connection, err := m.repository.SecurityEventConnection(ctx) + if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return safeConnectionError{category: "connection_binding_missing", cause: err} + } + if err != nil { + return safeConnectionError{category: "connection_binding_unavailable", cause: err} + } + return m.validateConnectionBinding(connection) +} + +func (m *ConnectionManager) validateConnectionBinding(connection store.SecurityEventConnection) error { + expectedIssuer := strings.TrimRight(strings.TrimSpace(m.config.ExpectedTransmitterIssuer), "/") + expectedAudience := strings.TrimSpace(m.config.ExpectedAudience) + expectedClientID := strings.TrimSpace(m.config.ManagementClientID) + expectedOwner := strings.TrimSpace(m.config.ExpectedOwnerKey) + if expectedIssuer == "" || expectedAudience == "" || expectedClientID == "" || expectedOwner == "" { + return safeConnectionError{category: "connection_binding_invalid", cause: errors.New("security event Revision binding is incomplete")} + } + if connection.TransmitterIssuer != expectedIssuer || connection.Audience == nil || *connection.Audience != expectedAudience || + connection.ManagementClientID != expectedClientID || connection.IdempotencyKey != expectedOwner { + return safeConnectionError{category: "connection_binding_mismatch", cause: store.ErrSecurityEventConnectionConflict} + } + return nil +} + func (m *ConnectionManager) Get(ctx context.Context) (ConnectionView, error) { connection, err := m.repository.SecurityEventConnection(ctx) if err != nil { @@ -183,6 +266,9 @@ func (m *ConnectionManager) Get(ctx context.Context) (ConnectionView, error) { } func (m *ConnectionManager) IntrospectionCredential(ctx context.Context) (string, []byte, error) { + if err := m.requireOperationalBinding(); err != nil { + return "", nil, err + } connection, err := m.repository.SecurityEventConnection(ctx) if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { if m.config.ManagementClientID != "" && m.config.ManagementClientSecret != "" { @@ -212,6 +298,10 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien if (managementClientID == "") != (len(managementSecret) == 0) { return ConnectionView{}, errors.New("machine client id and secret must be provided together") } + if err == nil && (existing.LifecycleStatus == "retiring" || + existing.LifecycleStatus == "disconnect_pending" && existing.IdempotencyKey == idempotencyKey) { + return ConnectionView{}, safeConnectionError{category: "retirement_pending", cause: store.ErrSecurityEventConnectionConflict} + } if !credentialsProvided { if err == nil { managementClientID, managementSecret, _ = m.managementCredential(ctx, existing) @@ -225,10 +315,46 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien return ConnectionView{}, err } if err == nil { + if m.bindingMismatch && existing.IdempotencyKey == idempotencyKey { + return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict} + } if existing.TransmitterIssuer != issuer { - return ConnectionView{}, store.ErrSecurityEventConnectionConflict + return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict} } if credentialsProvided { + if existing.IdempotencyKey != idempotencyKey { + if existing.ManagementClientID == "" || existing.ManagementClientID != managementClientID { + return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict} + } + if existing.StreamID == nil { + m.retirement = &connectionRetirementAuthorization{ + connectionID: existing.ConnectionID, credentialRef: valueOrEmpty(existing.ManagementCredentialRef), ownerKey: idempotencyKey, + } + return ConnectionView{}, safeConnectionError{category: "connection_conflict", cause: store.ErrSecurityEventConnectionConflict} + } + if m.retirement == nil || m.retirement.connectionID != existing.ConnectionID || + m.retirement.credentialRef != valueOrEmpty(existing.ManagementCredentialRef) || + m.retirement.ownerKey != idempotencyKey { + if err := m.proveManagementCredentialHandoff(ctx, existing, managementClientID, managementSecret); err != nil { + return ConnectionView{}, safeConnectionError{category: "credential_handoff_unsafe", cause: err} + } + existing, err = m.persistManagementCredential(ctx, existing, managementClientID, managementSecret) + if err != nil { + return ConnectionView{}, err + } + m.retirement = &connectionRetirementAuthorization{ + connectionID: existing.ConnectionID, credentialRef: valueOrEmpty(existing.ManagementCredentialRef), ownerKey: idempotencyKey, + } + } + if existing.LifecycleStatus == "disconnect_pending" { + go m.retryDisconnect(existing.ConnectionID) + return ConnectionView{}, safeConnectionError{category: "retirement_pending", cause: store.ErrSecurityEventConnectionConflict} + } + return ConnectionView{}, safeConnectionError{category: "connection_conflict", cause: store.ErrSecurityEventConnectionConflict} + } + if existing.ManagementClientID != "" && existing.ManagementClientID != managementClientID { + return ConnectionView{}, store.ErrSecurityEventConnectionConflict + } existing, err = m.persistManagementCredential(ctx, existing, managementClientID, managementSecret) if err != nil { return ConnectionView{}, err @@ -247,17 +373,17 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien connectionID := uuid.NewString() reference := "ssf-push-" + connectionID managementReference := "ssf-management-" + connectionID - if err := m.secrets.Put(ctx, managementReference, managementSecret); err != nil { + if err := m.stageIdentitySecret(ctx, managementReference, managementSecret); err != nil { return ConnectionView{}, err } secret, err := randomPushBearer() if err != nil { - _ = m.secrets.Delete(ctx, managementReference) + _ = m.retireIdentitySecret(ctx, managementReference) return ConnectionView{}, err } defer clear(secret) - if err := m.secrets.Put(ctx, reference, secret); err != nil { - _ = m.secrets.Delete(ctx, managementReference) + if err := m.stageIdentitySecret(ctx, reference, secret); err != nil { + _ = m.retireIdentitySecret(ctx, managementReference) return ConnectionView{}, err } endpoint := strings.TrimRight(m.config.PublicBaseURL, "/") + "/api/v1/security-events/ssf" @@ -267,8 +393,9 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien ManagementCredentialRef: managementReference, IdempotencyKey: idempotencyKey, }) if err != nil { - _ = m.secrets.Delete(ctx, reference) - _ = m.secrets.Delete(ctx, managementReference) + // Commit outcome can be ambiguous: rollback keeps the staging rows, + // while commit atomically removes them. Requeueing here could delete + // credentials already referenced by the committed connection. return ConnectionView{}, err } return m.resumeConnecting(ctx, connection) @@ -276,17 +403,14 @@ func (m *ConnectionManager) Connect(ctx context.Context, issuer, managementClien func (m *ConnectionManager) persistManagementCredential(ctx context.Context, connection store.SecurityEventConnection, clientID string, secret []byte) (store.SecurityEventConnection, error) { reference := "ssf-management-" + connection.ConnectionID + "-" + uuid.NewString() - if err := m.secrets.Put(ctx, reference, secret); err != nil { + if err := m.stageIdentitySecret(ctx, reference, secret); err != nil { return store.SecurityEventConnection{}, err } - updated, err := m.repository.SetSecurityEventManagementCredential(ctx, connection.ConnectionID, clientID, reference) + updated, err := m.repository.SetSecurityEventManagementCredential(ctx, connection.ConnectionID, clientID, reference, connection.Version) if err != nil { - _ = m.secrets.Delete(ctx, reference) + // The staging row is the source of truth when commit outcome is unknown. return store.SecurityEventConnection{}, err } - if connection.ManagementCredentialRef != nil { - _ = m.secrets.Delete(ctx, *connection.ManagementCredentialRef) - } return updated, nil } @@ -337,6 +461,9 @@ func (m *ConnectionManager) resumeConnecting(ctx context.Context, connection sto } func (m *ConnectionManager) Verify(ctx context.Context) (ConnectionView, error) { + if err := m.requireOperationalBinding(); err != nil { + return ConnectionView{}, err + } connection, err := m.repository.SecurityEventConnection(ctx) if err != nil { return ConnectionView{}, err @@ -366,6 +493,9 @@ func (m *ConnectionManager) Verify(ctx context.Context) (ConnectionView, error) func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionView, error) { m.operation.Lock() defer m.operation.Unlock() + if err := m.requireOperationalBinding(); err != nil { + return ConnectionView{}, err + } connection, err := m.repository.SecurityEventConnection(ctx) if err != nil { return ConnectionView{}, err @@ -381,14 +511,14 @@ func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionVie if err != nil { return ConnectionView{}, err } - if err := m.secrets.Put(ctx, nextReference, next); err != nil { + if err := m.stageIdentitySecret(ctx, nextReference, next); err != nil { clear(next) return ConnectionView{}, err } - connection, err = m.repository.SetSecurityEventNextCredential(ctx, connection.ConnectionID, nextReference) + connection, err = m.repository.SetSecurityEventNextCredential(ctx, connection.ConnectionID, nextReference, connection.Version) if err != nil { clear(next) - _ = m.secrets.Delete(ctx, nextReference) + // Never recreate cleanup intent after an ambiguous adoption commit. return ConnectionView{}, err } } else { @@ -424,10 +554,56 @@ func (m *ConnectionManager) RotateCredential(ctx context.Context) (ConnectionVie func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, error) { m.operation.Lock() defer m.operation.Unlock() + if err := m.requireOperationalBinding(); err != nil { + return ConnectionView{}, err + } connection, err := m.repository.SecurityEventConnection(ctx) if err != nil { return ConnectionView{}, err } + return m.disconnect(ctx, connection) +} + +// RetireConflictingConnection is a pairing-scoped recovery operation. It may +// retire only a connection owned by another workflow; a stale request becomes +// a no-op once the current pairing owns the singleton connection. +func (m *ConnectionManager) RetireConflictingConnection(ctx context.Context, currentOwnerKey string) error { + m.operation.Lock() + defer m.operation.Unlock() + connection, err := m.repository.SecurityEventConnection(ctx) + if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return nil + } + if err != nil { + return err + } + if strings.TrimSpace(currentOwnerKey) == "" { + return errors.New("current security event owner is required") + } + if connection.IdempotencyKey == currentOwnerKey { + return nil + } + if connection.LifecycleStatus == "retiring" { + go m.finishRetirement(connection) + return nil + } + if m.retirement == nil || m.retirement.connectionID != connection.ConnectionID || + m.retirement.credentialRef != valueOrEmpty(connection.ManagementCredentialRef) || + m.retirement.ownerKey != currentOwnerKey { + return safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict} + } + if connection.LifecycleStatus == "disconnect_pending" { + go m.retryDisconnect(connection.ConnectionID) + return nil + } + _, err = m.disconnect(ctx, connection) + return err +} + +func (m *ConnectionManager) disconnect(ctx context.Context, connection store.SecurityEventConnection) (ConnectionView, error) { + if connection.LifecycleStatus == "retiring" || connection.LifecycleStatus == "disconnect_pending" { + return m.view(connection), nil + } if connection.StreamID != nil { configuration, client, discoverErr := m.discover(ctx, connection.TransmitterIssuer) if discoverErr == nil { @@ -441,12 +617,20 @@ func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, err } if discoverErr != nil { category := "remote_disconnect_failed" - connection, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "disconnect_pending", &category) + transitioned, transitionErr := m.repository.TransitionSecurityEventConnectionLifecycle( + ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "disconnect_pending", &category, + ) + if transitionErr != nil { + return ConnectionView{}, transitionErr + } + connection = transitioned go m.retryDisconnect(connection.ConnectionID) return m.view(connection), nil } } - connection, err = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "retiring", nil) + connection, err := m.repository.TransitionSecurityEventConnectionLifecycle( + ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "retiring", nil, + ) if err != nil { return ConnectionView{}, err } @@ -454,6 +638,51 @@ func (m *ConnectionManager) Disconnect(ctx context.Context) (ConnectionView, err return m.view(connection), nil } +// DiscardPreparedConnection removes only the connection created by one identity +// pairing. Bound streams continue through the normal retirement window so +// revocation watermarks and introspection fallback are never bypassed. +func (m *ConnectionManager) DiscardPreparedConnection(ctx context.Context, ownerKey string) error { + m.operation.Lock() + defer m.operation.Unlock() + connection, err := m.repository.SecurityEventConnection(ctx) + if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return nil + } + if err != nil { + return safeConnectionError{category: "connection_cleanup_failed", cause: err} + } + if strings.TrimSpace(ownerKey) == "" || connection.IdempotencyKey != ownerKey { + return nil + } + if connection.StreamID != nil { + if connection.LifecycleStatus != "retiring" && connection.LifecycleStatus != "disconnect_pending" { + if _, err := m.disconnect(ctx, connection); err != nil { + return safeConnectionError{category: "connection_cleanup_failed", cause: err} + } + } + return safeConnectionError{category: "retirement_pending", cause: errors.New("security event retirement is pending")} + } + err = m.repository.DiscardPreparedSecurityEventConnection( + ctx, connection.ConnectionID, ownerKey, connection.Version, + ) + if err != nil && !errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return safeConnectionError{category: "connection_cleanup_failed", cause: err} + } + m.stopRuntime() + return nil +} + +func connectionSecretReferences(connection store.SecurityEventConnection) []string { + references := []string{connection.CredentialRef} + if connection.ManagementCredentialRef != nil { + references = append(references, *connection.ManagementCredentialRef) + } + if connection.NextCredentialRef != nil { + references = append(references, *connection.NextCredentialRef) + } + return references +} + func (m *ConnectionManager) retryDisconnect(connectionID string) { delay := time.Second for { @@ -478,7 +707,9 @@ func (m *ConnectionManager) retryDisconnect(connectionID string) { } } if err == nil { - connection, err = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "retiring", nil) + connection, err = m.repository.TransitionSecurityEventConnectionLifecycle( + m.ctx, connectionID, "disconnect_pending", connection.Version, "retiring", nil, + ) if err == nil { go m.finishRetirement(connection) } @@ -625,49 +856,57 @@ func (m *ConnectionManager) finishAutomaticVerification(connectionID string, con defer cancel() ticker := time.NewTicker(250 * time.Millisecond) defer ticker.Stop() + expectedLifecycle := "verifying" + if rotating { + expectedLifecycle = "rotating" + } for { connection, err := m.repository.SecurityEventConnection(ctx) - if err != nil || connection.ConnectionID != connectionID || connection.StreamID == nil { + if err != nil || connection.ConnectionID != connectionID || connection.StreamID == nil || connection.LifecycleStatus != expectedLifecycle { return } _, verifiedAt, _ := m.repository.SecurityEventMetrics(ctx, connection.TransmitterIssuer, valueOrEmpty(connection.Audience)) if verifiedAt != nil && !verifiedAt.Before(started) { if err := m.setStreamStatus(ctx, client, configuration, token, *connection.StreamID, "enabled", "verification_succeeded"); err != nil { category := "stream_enable_failed" - lifecycle := "verifying" - if rotating { - lifecycle = "rotating" - } - _, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connectionID, lifecycle, &category) + _, _ = m.repository.TransitionSecurityEventConnectionLifecycle( + ctx, connectionID, expectedLifecycle, connection.Version, expectedLifecycle, &category, + ) return } if rotating && connection.NextCredentialRef != nil { - oldReference, nextReference := connection.CredentialRef, *connection.NextCredentialRef + nextReference := *connection.NextCredentialRef select { case <-m.ctx.Done(): return case <-time.After(m.config.CredentialOverlapDuration): } + current, currentErr := m.repository.SecurityEventConnection(m.ctx) + if currentErr != nil || current.ConnectionID != connectionID || current.LifecycleStatus != "rotating" || + current.NextCredentialRef == nil || *current.NextCredentialRef != nextReference { + return + } connection, err = m.repository.PromoteSecurityEventCredential(m.ctx, connectionID, nextReference) if err == nil { - _ = m.secrets.Delete(m.ctx, oldReference) _ = m.activate(m.ctx, connection) go m.finishBootstrap(connectionID) } return } - connection, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connectionID, "bootstrap", nil) - go m.finishBootstrap(connectionID) + connection, err = m.repository.TransitionSecurityEventConnectionLifecycle( + ctx, connectionID, expectedLifecycle, connection.Version, "bootstrap", nil, + ) + if err == nil { + go m.finishBootstrap(connectionID) + } return } select { case <-ctx.Done(): category := "verification_timeout" - lifecycle := "verifying" - if rotating { - lifecycle = "rotating" - } - _, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, lifecycle, &category) + _, _ = m.repository.TransitionSecurityEventConnectionLifecycle( + m.ctx, connectionID, expectedLifecycle, connection.Version, expectedLifecycle, &category, + ) return case <-ticker.C: } @@ -691,46 +930,89 @@ func (m *ConnectionManager) finishBootstrap(connectionID string) { if err == nil && connection.ConnectionID == connectionID && connection.LifecycleStatus == "bootstrap" { mode, _, metricsErr := m.repository.SecurityEventMetrics(m.ctx, connection.TransmitterIssuer, valueOrEmpty(connection.Audience)) if metricsErr == nil && mode == "push_healthy" { - _, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "enabled", nil) + _, _ = m.repository.TransitionSecurityEventConnectionLifecycle( + m.ctx, connectionID, "bootstrap", connection.Version, "enabled", nil, + ) return } category := "bootstrap_verification_stale" - _, _ = m.repository.UpdateSecurityEventConnectionLifecycle(m.ctx, connectionID, "degraded", &category) + _, _ = m.repository.TransitionSecurityEventConnectionLifecycle( + m.ctx, connectionID, "bootstrap", connection.Version, "degraded", &category, + ) } } -func (m *ConnectionManager) finishRetirement(connection store.SecurityEventConnection) { - elapsed := time.Since(connection.UpdatedAt) - delay := m.config.RetirementDuration - elapsed - if delay > 0 { - select { - case <-m.ctx.Done(): - return - case <-time.After(delay): - } - } - delay = time.Second +func (m *ConnectionManager) finishRetirement(retirement store.SecurityEventConnection) { + retryDelay := time.Second for { - if err := m.repository.DeleteSecurityEventConnection(m.ctx, connection.ConnectionID); err == nil || errors.Is(err, store.ErrSecurityEventConnectionNotFound) { - break - } - select { - case <-m.ctx.Done(): + connection, err := m.repository.SecurityEventConnection(m.ctx) + if errors.Is(err, store.ErrSecurityEventConnectionNotFound) { return - case <-time.After(delay): } - if delay < time.Minute { - delay *= 2 + if err != nil { + if !waitForConnectionRetry(m.ctx, retryDelay) { + return + } + retryDelay = nextConnectionRetryDelay(retryDelay) + continue } + if connection.ConnectionID != retirement.ConnectionID || connection.LifecycleStatus != "retiring" { + return + } + if remaining := m.config.RetirementDuration - time.Since(connection.UpdatedAt); remaining > 0 { + select { + case <-m.ctx.Done(): + return + case <-time.After(remaining): + } + continue + } + m.stopRuntime() + err = m.repository.FinalizeRetiringSecurityEventConnection(m.ctx, connection.ConnectionID, connection.Version) + if err == nil || errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return + } + if !waitForConnectionRetry(m.ctx, retryDelay) { + return + } + retryDelay = nextConnectionRetryDelay(retryDelay) } - m.stopRuntime() - _ = m.secrets.Delete(m.ctx, connection.CredentialRef) - if connection.ManagementCredentialRef != nil { - _ = m.secrets.Delete(m.ctx, *connection.ManagementCredentialRef) +} + +func waitForConnectionRetry(ctx context.Context, delay time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(delay): + return true } - if connection.NextCredentialRef != nil { - _ = m.secrets.Delete(m.ctx, *connection.NextCredentialRef) +} + +func nextConnectionRetryDelay(delay time.Duration) time.Duration { + if delay < time.Minute { + return delay * 2 } + return delay +} + +const identitySecretStagingTTL = 10 * time.Minute + +func (m *ConnectionManager) stageIdentitySecret(ctx context.Context, reference string, value []byte) error { + if err := m.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC().Add(identitySecretStagingTTL)); err != nil { + return err + } + if err := m.secrets.Put(ctx, reference, value); err != nil { + deleteErr := m.secrets.Delete(ctx, reference) + if deleteErr == nil || errors.Is(deleteErr, ErrSecretNotFound) { + _, _ = m.repository.RemovePendingIdentitySecretCleanup(ctx, reference) + } + return err + } + return nil +} + +func (m *ConnectionManager) retireIdentitySecret(ctx context.Context, reference string) error { + return m.repository.QueueIdentitySecretCleanup(ctx, reference, time.Now().UTC()) } func (m *ConnectionManager) validatePrerequisites(issuer, managementClientID string, managementSecret []byte, requirePublicBaseURL bool) error { @@ -794,6 +1076,10 @@ func (m *ConnectionManager) managementToken(ctx context.Context, client *http.Cl return "", err } defer clear(managementSecret) + return m.requestManagementToken(ctx, client, scopes, managementClientID, managementSecret) +} + +func (m *ConnectionManager) requestManagementToken(ctx context.Context, client *http.Client, scopes, managementClientID string, managementSecret []byte) (string, error) { form := url.Values{"grant_type": {"client_credentials"}, "scope": {scopes}} request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(m.config.OIDCIssuer, "/")+"/token", strings.NewReader(form.Encode())) if err != nil { @@ -820,6 +1106,44 @@ func (m *ConnectionManager) managementToken(ctx context.Context, client *http.Cl return payload.AccessToken, nil } +func (m *ConnectionManager) proveManagementCredentialHandoff(ctx context.Context, connection store.SecurityEventConnection, managementClientID string, managementSecret []byte) error { + if connection.StreamID == nil || connection.Audience == nil { + return store.ErrSecurityEventConnectionConflict + } + configuration, client, err := m.discover(ctx, connection.TransmitterIssuer) + if err != nil { + return err + } + token, err := m.requestManagementToken(ctx, client, "ssf.stream.read ssf.stream.manage", managementClientID, managementSecret) + if err != nil { + return err + } + var streams []streamConfiguration + if err := requestJSON(ctx, client, http.MethodGet, configuration.ConfigurationEndpoint, token, nil, &streams); err != nil { + return err + } + for _, stream := range streams { + if stream.StreamID != *connection.StreamID { + continue + } + if err := validateCreatedStream(stream, connection.TransmitterIssuer, connection.EndpointURL); err != nil || stream.Audience != *connection.Audience { + return store.ErrSecurityEventConnectionConflict + } + return nil + } + // An authenticated empty list proves that the new credential belongs to a + // token issuer trusted by the old transmitter. Disconnect treats the absent + // Stream as an idempotent remote deletion. + return nil +} + +func (m *ConnectionManager) requireOperationalBinding() error { + if m.bindingMismatch { + return safeConnectionError{category: "credential_handoff_unsafe", cause: store.ErrSecurityEventConnectionConflict} + } + return nil +} + func (m *ConnectionManager) managementCredential(ctx context.Context, connection store.SecurityEventConnection) (string, []byte, error) { if connection.ManagementCredentialRef != nil && connection.ManagementClientID != "" { secret, err := m.secrets.Get(ctx, *connection.ManagementCredentialRef) @@ -882,8 +1206,10 @@ func (m *ConnectionManager) deleteStream(ctx context.Context, client *http.Clien } func (m *ConnectionManager) connectionError(ctx context.Context, connection store.SecurityEventConnection, category string, cause error) error { - _, _ = m.repository.UpdateSecurityEventConnectionLifecycle(ctx, connection.ConnectionID, "error", &category) - return cause + _, _ = m.repository.TransitionSecurityEventConnectionLifecycle( + ctx, connection.ConnectionID, connection.LifecycleStatus, connection.Version, "error", &category, + ) + return safeConnectionError{category: category, cause: cause} } func (m *ConnectionManager) view(connection store.SecurityEventConnection) ConnectionView { @@ -1054,7 +1380,12 @@ var blockedNetworkPrefixes = []netip.Prefix{ } func isLocalHostname(value string) bool { - return value == "localhost" || value == "127.0.0.1" || value == "::1" + value = strings.ToLower(strings.TrimSpace(value)) + if value == "localhost" { + return true + } + ip := net.ParseIP(value) + return ip != nil && ip.IsLoopback() } func isLocalEnvironment(value string) bool { diff --git a/apps/api/internal/securityevents/connection_manager_test.go b/apps/api/internal/securityevents/connection_manager_test.go index 6b1fbef..be67479 100644 --- a/apps/api/internal/securityevents/connection_manager_test.go +++ b/apps/api/internal/securityevents/connection_manager_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net" "net/http" "net/http/httptest" @@ -20,11 +21,81 @@ import ( ) type memoryConnectionRepository struct { - mutex sync.Mutex - connection *store.SecurityEventConnection - mode string - verifiedAt *time.Time - evaluation store.SecurityEventEvaluation + mutex sync.Mutex + connection *store.SecurityEventConnection + beforeDiscard func(*store.SecurityEventConnection) + beforeFinalize func(*store.SecurityEventConnection) + finalizeCalls int + cleanupDue map[string]time.Time + cleanupClaims map[string]string + cleanupClaimed chan struct{} + mode string + verifiedAt *time.Time + evaluation store.SecurityEventEvaluation +} + +func (r *memoryConnectionRepository) QueueIdentitySecretCleanup(_ context.Context, reference string, notBefore time.Time) error { + r.mutex.Lock() + defer r.mutex.Unlock() + if r.cleanupDue == nil { + r.cleanupDue = map[string]time.Time{} + } + if r.cleanupClaims[reference] != "" { + return store.ErrIdentitySecretCleanupConflict + } + current, exists := r.cleanupDue[reference] + if !exists || notBefore.Before(current) { + r.cleanupDue[reference] = notBefore + } + return nil +} +func (r *memoryConnectionRepository) RemovePendingIdentitySecretCleanup(_ context.Context, reference string) (bool, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + if r.cleanupClaims[reference] != "" { + return false, nil + } + _, exists := r.cleanupDue[reference] + delete(r.cleanupDue, reference) + return exists, nil +} +func (r *memoryConnectionRepository) ClaimIdentitySecretCleanups(_ context.Context, limit int, lease time.Duration) ([]store.IdentitySecretCleanupClaim, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + if r.cleanupClaims == nil { + r.cleanupClaims = map[string]string{} + } + if r.cleanupClaimed != nil { + select { + case r.cleanupClaimed <- struct{}{}: + default: + } + } + now := time.Now().UTC() + claims := make([]store.IdentitySecretCleanupClaim, 0) + for reference, notBefore := range r.cleanupDue { + if !notBefore.After(now) && r.cleanupClaims[reference] == "" { + token := uuid.NewString() + r.cleanupClaims[reference] = token + claims = append(claims, store.IdentitySecretCleanupClaim{ + Reference: reference, ClaimToken: token, LeaseExpiresAt: now.Add(lease), + }) + if len(claims) == limit { + break + } + } + } + return claims, nil +} +func (r *memoryConnectionRepository) CompleteIdentitySecretCleanup(_ context.Context, reference, claimToken string) (bool, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + if r.cleanupClaims[reference] != claimToken { + return false, nil + } + delete(r.cleanupClaims, reference) + delete(r.cleanupDue, reference) + return true, nil } func (*memoryConnectionRepository) ApplySessionRevoked(context.Context, store.ApplySessionRevokedInput) (store.ApplySecurityEventResult, error) { @@ -70,15 +141,29 @@ func (r *memoryConnectionRepository) CreateSecurityEventConnection(_ context.Con ManagementCredentialRef: &input.ManagementCredentialRef, LifecycleStatus: "connecting", Version: 1, IdempotencyKey: input.IdempotencyKey, CreatedAt: now, UpdatedAt: now, HealthMode: "disabled"} r.connection = &value + delete(r.cleanupDue, input.CredentialRef) + delete(r.cleanupDue, input.ManagementCredentialRef) return value, nil } -func (r *memoryConnectionRepository) SetSecurityEventManagementCredential(_ context.Context, id, clientID, reference string) (store.SecurityEventConnection, error) { +func (r *memoryConnectionRepository) SetSecurityEventManagementCredential(_ context.Context, id, clientID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() if r.connection == nil || r.connection.ConnectionID != id { return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound } + if r.connection.Version != expectedVersion || r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending" { + return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict + } + oldReference := r.connection.ManagementCredentialRef r.connection.ManagementClientID, r.connection.ManagementCredentialRef = clientID, &reference + r.connection.Version++ + delete(r.cleanupDue, reference) + if oldReference != nil && *oldReference != reference { + if r.cleanupDue == nil { + r.cleanupDue = map[string]time.Time{} + } + r.cleanupDue[*oldReference] = time.Now().UTC() + } return *r.connection, nil } func (r *memoryConnectionRepository) SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) { @@ -101,19 +186,35 @@ func (r *memoryConnectionRepository) BindSecurityEventConnection(_ context.Conte r.connection.LifecycleStatus, r.connection.Version = lifecycle, r.connection.Version+1 return *r.connection, nil } -func (r *memoryConnectionRepository) UpdateSecurityEventConnectionLifecycle(_ context.Context, id, lifecycle string, category *string) (store.SecurityEventConnection, error) { +func (r *memoryConnectionRepository) TransitionSecurityEventConnectionLifecycle(_ context.Context, id, expectedLifecycle string, expectedVersion int64, lifecycle string, category *string) (store.SecurityEventConnection, error) { r.mutex.Lock() defer r.mutex.Unlock() - if r.connection == nil || r.connection.ConnectionID != id { - return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound + if r.connection == nil || r.connection.ConnectionID != id || r.connection.LifecycleStatus != expectedLifecycle || r.connection.Version != expectedVersion { + return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict + } + if (r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending") && + lifecycle != "retiring" && lifecycle != "disconnect_pending" { + return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict } r.connection.LifecycleStatus, r.connection.LastErrorCategory = lifecycle, category r.connection.Version++ r.connection.UpdatedAt = time.Now().UTC() return *r.connection, nil } -func (r *memoryConnectionRepository) SetSecurityEventNextCredential(context.Context, string, string) (store.SecurityEventConnection, error) { - return store.SecurityEventConnection{}, errors.New("not used") +func (r *memoryConnectionRepository) SetSecurityEventNextCredential(_ context.Context, id, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + if r.connection == nil || r.connection.ConnectionID != id { + return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionNotFound + } + if r.connection.Version != expectedVersion || r.connection.NextCredentialRef != nil || r.connection.LifecycleStatus == "retiring" || r.connection.LifecycleStatus == "disconnect_pending" { + return store.SecurityEventConnection{}, store.ErrSecurityEventConnectionConflict + } + r.connection.NextCredentialRef = &reference + r.connection.LifecycleStatus = "rotating" + r.connection.Version++ + delete(r.cleanupDue, reference) + return *r.connection, nil } func TestConnectionManagerBootstrapOnlyBecomesEnabledWhenPushIsHealthy(t *testing.T) { @@ -164,12 +265,510 @@ func TestConnectionManagerResumesBootstrapAfterRestart(t *testing.T) { t.Fatal("bootstrap lifecycle did not resume after process restart") } +func TestNewConnectionManagerStrictRevisionBindingRejectsDifferentPersistedConnection(t *testing.T) { + audience := "urn:easyai:ssf:receiver:target" + streamID := uuid.NewString() + base := store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), + TransmitterIssuer: "https://auth.example/ssf", + Audience: &audience, + StreamID: &streamID, + CredentialRef: "missing-credential-must-not-be-read", + ManagementClientID: "gateway-machine", + LifecycleStatus: "enabled", + IdempotencyKey: "identity-pairing-ssf-target-revision", + Version: 1, + } + config := ConnectionManagerConfig{ + OIDCEnabled: true, + OIDCIssuer: "https://auth.example/issuer", + OIDCTenantID: uuid.NewString(), + ManagementClientID: "gateway-machine", + ExpectedTransmitterIssuer: "https://auth.example/ssf", + ExpectedAudience: audience, + ExpectedOwnerKey: "identity-pairing-ssf-target-revision", + StrictRevisionBinding: true, + } + + tests := []struct { + name string + mutate func(*store.SecurityEventConnection) + }{ + {name: "transmitter issuer", mutate: func(connection *store.SecurityEventConnection) { + connection.TransmitterIssuer = "https://old-auth.example/ssf" + }}, + {name: "audience", mutate: func(connection *store.SecurityEventConnection) { + other := "urn:easyai:ssf:receiver:old" + connection.Audience = &other + }}, + {name: "missing audience", mutate: func(connection *store.SecurityEventConnection) { + connection.Audience = nil + }}, + {name: "management client", mutate: func(connection *store.SecurityEventConnection) { + connection.ManagementClientID = "old-machine" + }}, + {name: "owner", mutate: func(connection *store.SecurityEventConnection) { + connection.IdempotencyKey = "identity-pairing-ssf-old-revision" + }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + connection := base + test.mutate(&connection) + repository := &memoryConnectionRepository{connection: &connection} + + manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, config, &Metrics{}) + if manager != nil { + t.Fatal("strict runtime construction returned a manager for a different Revision binding") + } + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_mismatch" { + t.Fatalf("error=%v category=%v", err, categorized) + } + }) + } +} + +func TestNewConnectionManagerStrictRevisionBindingRequiresPersistedConnection(t *testing.T) { + manager, err := NewConnectionManager(context.Background(), &memoryConnectionRepository{}, &memorySecretStore{}, ConnectionManagerConfig{ + OIDCEnabled: true, + OIDCIssuer: "https://auth.example/issuer", + OIDCTenantID: uuid.NewString(), + ManagementClientID: "gateway-machine", + ExpectedTransmitterIssuer: "https://auth.example/ssf", + ExpectedAudience: "urn:easyai:ssf:receiver:target", + ExpectedOwnerKey: "identity-pairing-ssf-target-revision", + StrictRevisionBinding: true, + }, &Metrics{}) + if manager != nil { + t.Fatal("strict runtime construction succeeded without a persisted SSF connection") + } + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_missing" { + t.Fatalf("error=%v category=%v", err, categorized) + } +} + +func TestNewConnectionManagerStrictRevisionBindingAcceptsExactPersistedConnection(t *testing.T) { + audience := "urn:easyai:ssf:receiver:target" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), + TransmitterIssuer: "https://auth.example/ssf", + Audience: &audience, + ManagementClientID: "gateway-machine", + LifecycleStatus: "enabled", + IdempotencyKey: "identity-pairing-ssf-target-revision", + Version: 1, + }} + + manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, ConnectionManagerConfig{ + OIDCEnabled: true, + OIDCIssuer: "https://auth.example/issuer", + OIDCTenantID: uuid.NewString(), + ManagementClientID: "gateway-machine", + ExpectedTransmitterIssuer: "https://auth.example/ssf/", + ExpectedAudience: audience, + ExpectedOwnerKey: "identity-pairing-ssf-target-revision", + StrictRevisionBinding: true, + }, &Metrics{}) + if err != nil || manager == nil { + t.Fatalf("exact Revision binding was rejected: manager=%v err=%v", manager != nil, err) + } +} + +func TestConnectionManagerRecoveryCanLoadDifferentPersistedConnection(t *testing.T) { + oldAudience := "urn:easyai:ssf:receiver:old" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), + TransmitterIssuer: "https://old-auth.example/ssf", + Audience: &oldAudience, + ManagementClientID: "old-machine", + LifecycleStatus: "error", + IdempotencyKey: "identity-pairing-ssf-old-revision", + Version: 1, + }} + config := ConnectionManagerConfig{ + OIDCEnabled: true, + OIDCIssuer: "https://auth.example/issuer", + OIDCTenantID: uuid.NewString(), + ManagementClientID: "gateway-machine", + ExpectedTransmitterIssuer: "https://auth.example/ssf", + ExpectedAudience: "urn:easyai:ssf:receiver:target", + ExpectedOwnerKey: "identity-pairing-ssf-target-revision", + } + + manager, err := NewConnectionManager(context.Background(), repository, &memorySecretStore{}, config, &Metrics{}) + if err != nil || manager == nil { + t.Fatalf("recovery manager could not load the conflicting connection: manager=%v err=%v", manager != nil, err) + } + err = manager.ValidateConfiguredConnectionBinding(context.Background()) + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_binding_mismatch" { + t.Fatalf("explicit runtime binding validation did not reject recovery connection: err=%v category=%v", err, categorized) + } +} + +func TestForeignRevisionRecoveryNeverUsesPersistedCredentialOnGenericOperations(t *testing.T) { + var requests atomic.Int32 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + defer endpoint.Close() + + audience, streamID := "urn:easyai:ssf:receiver:old", uuid.NewString() + managementReference := "ssf-management-old" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: endpoint.URL + "/old-ssf", EndpointURL: endpoint.URL + "/receiver", + Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-old", ManagementClientID: "shared-machine", + ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 3, + IdempotencyKey: "identity-pairing-ssf-old-revision", + }} + secrets := &memorySecretStore{values: map[string][]byte{ + managementReference: []byte("old-machine-secret-must-never-leave"), + "ssf-push-old": []byte("old-push-secret-value-000000000000"), + }} + manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: endpoint.URL + "/new-oidc", OIDCTenantID: uuid.NewString(), + ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: endpoint.URL + "/new-ssf", + ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision", + PublicBaseURL: endpoint.URL, HTTPClient: endpoint.Client(), + }, &Metrics{}) + if err != nil { + t.Fatal(err) + } + + assertUnsafe := func(name string, err error) { + t.Helper() + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "credential_handoff_unsafe" { + t.Fatalf("%s error=%v", name, err) + } + } + _, _, credentialErr := manager.IntrospectionCredential(context.Background()) + assertUnsafe("introspection credential", credentialErr) + _, verifyErr := manager.Verify(context.Background()) + assertUnsafe("verify", verifyErr) + _, rotateErr := manager.RotateCredential(context.Background()) + assertUnsafe("rotate", rotateErr) + _, disconnectErr := manager.Disconnect(context.Background()) + assertUnsafe("disconnect", disconnectErr) + _, connectErr := manager.Connect(context.Background(), endpoint.URL+"/new-ssf", "shared-machine", []byte("new-machine-secret-value-000000000"), "identity-pairing-ssf-new-revision") + assertUnsafe("cross-issuer connect", connectErr) + assertUnsafe("retire without proved handoff", manager.RetireConflictingConnection(context.Background(), "identity-pairing-ssf-new-revision")) + + if requests.Load() != 0 { + t.Fatalf("foreign recovery sent %d authenticated or discovery requests", requests.Load()) + } + connection, err := repository.SecurityEventConnection(context.Background()) + if err != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference { + t.Fatalf("foreign recovery changed persisted credential: connection=%#v err=%v", connection, err) + } + repository.mutex.Lock() + queued := len(repository.cleanupDue) + repository.mutex.Unlock() + if queued != 0 || !secrets.Has(managementReference) { + t.Fatalf("foreign recovery queued=%d old_secret_present=%t", queued, secrets.Has(managementReference)) + } +} + +func TestPairingProvesRotatedCredentialBeforeRetiringLegacyStream(t *testing.T) { + const oldSecret = "old-machine-secret-value-0000000000" + const newSecret = "new-machine-secret-value-0000000000" + audience, streamID := "urn:easyai:ssf:receiver:legacy", uuid.NewString() + managementReference := "ssf-management-legacy" + var oldSecretSeen atomic.Bool + var newSecretRequests atomic.Int32 + var transmitter *httptest.Server + transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/.well-known/ssf-configuration/ssf": + _ = json.NewEncoder(w).Encode(map[string]any{ + "spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json", + "configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status", + "verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod}, + }) + case "/oidc/token": + _, secret, _ := r.BasicAuth() + if secret == oldSecret { + oldSecretSeen.Store(true) + } + if secret != newSecret { + http.Error(w, "invalid client", http.StatusUnauthorized) + return + } + newSecretRequests.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"access_token": "proved-management-token", "token_type": "Bearer"}) + case "/ssf/v1/stream": + if r.Header.Get("Authorization") != "Bearer proved-management-token" { + http.Error(w, "invalid token", http.StatusUnauthorized) + return + } + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + return + } + description := "easyai-gateway:legacy" + _ = json.NewEncoder(w).Encode([]map[string]any{{ + "stream_id": streamID, "iss": transmitter.URL + "/ssf", "aud": audience, + "events_requested": []string{sessionRevokedEvent}, + "delivery": map[string]any{"method": pushDeliveryMethod, "endpoint_url": transmitter.URL + "/receiver"}, + "description": description, + }}) + default: + http.NotFound(w, r) + } + })) + defer transmitter.Close() + + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: "legacy", TransmitterIssuer: transmitter.URL + "/ssf", EndpointURL: transmitter.URL + "/receiver", + Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-legacy", ManagementClientID: "shared-machine", + ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 2, IdempotencyKey: "manual-legacy-owner", + }} + secrets := &memorySecretStore{values: map[string][]byte{ + managementReference: []byte(oldSecret), "ssf-push-legacy": []byte("legacy-push-secret-value-000000000"), + }} + manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: transmitter.URL + "/oidc", OIDCTenantID: uuid.NewString(), + ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: transmitter.URL + "/ssf", + ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision", + PublicBaseURL: transmitter.URL, HTTPClient: transmitter.Client(), RetirementDuration: time.Hour, + }, &Metrics{}) + if err != nil { + t.Fatal(err) + } + + _, err = manager.Connect(context.Background(), transmitter.URL+"/ssf", "shared-machine", []byte(newSecret), "identity-pairing-ssf-new-revision") + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_conflict" { + t.Fatalf("proved handoff error=%v", err) + } + connection, err := repository.SecurityEventConnection(context.Background()) + if err != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef == managementReference { + t.Fatalf("rotated credential was not adopted: connection=%#v err=%v", connection, err) + } + rotatedSecret, err := secrets.Get(context.Background(), *connection.ManagementCredentialRef) + if err != nil || string(rotatedSecret) != newSecret { + t.Fatalf("adopted credential invalid: present=%t err=%v", len(rotatedSecret) > 0, err) + } + clear(rotatedSecret) + if err := manager.RetireConflictingConnection(context.Background(), "identity-pairing-ssf-new-revision"); err != nil { + t.Fatal(err) + } + connection, err = repository.SecurityEventConnection(context.Background()) + if err != nil || connection.LifecycleStatus != "retiring" { + t.Fatalf("legacy connection was not retired: connection=%#v err=%v", connection, err) + } + if oldSecretSeen.Load() || newSecretRequests.Load() < 2 { + t.Fatalf("old_secret_sent=%t new_secret_requests=%d", oldSecretSeen.Load(), newSecretRequests.Load()) + } +} + +func TestFailedCredentialProofDoesNotReplaceLegacyCredential(t *testing.T) { + const oldSecret = "old-machine-secret-value-0000000000" + const newSecret = "new-machine-secret-value-0000000000" + var oldSecretSeen atomic.Bool + var tokenRequests atomic.Int32 + evilIssuer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, secret, _ := r.BasicAuth() + if secret == oldSecret { + oldSecretSeen.Store(true) + } + if secret == newSecret { + tokenRequests.Add(1) + } + _ = json.NewEncoder(w).Encode(map[string]any{"access_token": "untrusted-token", "token_type": "Bearer"}) + })) + defer evilIssuer.Close() + audience, streamID := "urn:easyai:ssf:receiver:legacy", uuid.NewString() + managementReference := "ssf-management-legacy" + var transmitter *httptest.Server + transmitter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/.well-known/ssf-configuration/ssf": + _ = json.NewEncoder(w).Encode(map[string]any{ + "spec_version": "1_0", "issuer": transmitter.URL + "/ssf", "jwks_uri": transmitter.URL + "/ssf/jwks.json", + "configuration_endpoint": transmitter.URL + "/ssf/v1/stream", "status_endpoint": transmitter.URL + "/ssf/v1/status", + "verification_endpoint": transmitter.URL + "/ssf/v1/verify", "delivery_methods_supported": []string{pushDeliveryMethod}, + }) + case "/ssf/v1/stream": + http.Error(w, "untrusted token issuer", http.StatusUnauthorized) + default: + http.NotFound(w, r) + } + })) + defer transmitter.Close() + + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: transmitter.URL + "/ssf", EndpointURL: transmitter.URL + "/receiver", + Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-legacy", ManagementClientID: "shared-machine", + ManagementCredentialRef: &managementReference, LifecycleStatus: "enabled", Version: 2, IdempotencyKey: "manual-legacy-owner", + }} + secrets := &memorySecretStore{values: map[string][]byte{managementReference: []byte(oldSecret)}} + manager, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: evilIssuer.URL, OIDCTenantID: uuid.NewString(), + ManagementClientID: "shared-machine", ExpectedTransmitterIssuer: transmitter.URL + "/ssf", + ExpectedAudience: "urn:easyai:ssf:receiver:new", ExpectedOwnerKey: "identity-pairing-ssf-new-revision", + PublicBaseURL: transmitter.URL, HTTPClient: &http.Client{}, + }, &Metrics{}) + if err != nil { + t.Fatal(err) + } + _, err = manager.Connect(context.Background(), transmitter.URL+"/ssf", "shared-machine", []byte(newSecret), "identity-pairing-ssf-new-revision") + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "credential_handoff_unsafe" { + t.Fatalf("unproved handoff error=%v", err) + } + connection, getErr := repository.SecurityEventConnection(context.Background()) + if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference { + t.Fatalf("failed proof replaced credential: connection=%#v err=%v", connection, getErr) + } + repository.mutex.Lock() + queued := len(repository.cleanupDue) + repository.mutex.Unlock() + if queued != 0 || !secrets.Has(managementReference) || oldSecretSeen.Load() || tokenRequests.Load() != 1 { + t.Fatalf("queued=%d old_present=%t old_sent=%t new_requests=%d", queued, secrets.Has(managementReference), oldSecretSeen.Load(), tokenRequests.Load()) + } +} + func TestRemoteNotFoundIsRecognizedForIdempotentDisconnect(t *testing.T) { if !isRemoteHTTPStatus(remoteHTTPError{status: http.StatusNotFound}, http.StatusNotFound) { t.Fatal("remote 404 was not recognized as an already-deleted Stream") } } +func TestDiscardPreparedConnectionOnlyDeletesOwnedUnboundResources(t *testing.T) { + for _, test := range []struct { + name string + owner string + wantGone bool + }{ + {name: "owned", owner: "identity-pairing-ssf-revision", wantGone: true}, + {name: "different owner", owner: "identity-pairing-ssf-other", wantGone: false}, + } { + t.Run(test.name, func(t *testing.T) { + managementReference := "ssf-management-connection" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + CredentialRef: "ssf-push-connection", ManagementCredentialRef: &managementReference, + LifecycleStatus: "error", LastErrorCategory: pointerTo("discovery_failed"), + IdempotencyKey: "identity-pairing-ssf-revision", Version: 2, + }} + secrets := &memorySecretStore{values: map[string][]byte{ + "ssf-push-connection": []byte("push-secret-value-for-cleanup-00000000"), + "ssf-management-connection": []byte("machine-secret-value-for-cleanup-00000"), + }} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets} + + if err := manager.DiscardPreparedConnection(context.Background(), test.owner); err != nil { + t.Fatal(err) + } + _, getErr := repository.SecurityEventConnection(context.Background()) + gone := errors.Is(getErr, store.ErrSecurityEventConnectionNotFound) + if gone != test.wantGone { + t.Fatalf("connection gone=%t want=%t", gone, test.wantGone) + } + if test.wantGone && secrets.Len() != 2 { + t.Fatalf("owned connection Secrets were deleted before durable cleanup: %d", secrets.Len()) + } + if !test.wantGone && secrets.Len() != 2 { + t.Fatal("a different pairing deleted shared security event resources") + } + repository.mutex.Lock() + queued := len(repository.cleanupDue) + repository.mutex.Unlock() + if test.wantGone && queued != 2 { + t.Fatalf("owned connection queued %d Secrets for cleanup, want 2", queued) + } + if !test.wantGone && queued != 0 { + t.Fatal("a different pairing queued shared security event Secrets for cleanup") + } + }) + } +} + +func TestDiscardPreparedConnectionKeepsBoundRetirementResumable(t *testing.T) { + audience, streamID := "urn:easyai:ssf:receiver:"+uuid.NewString(), uuid.NewString() + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + Audience: &audience, StreamID: &streamID, CredentialRef: "ssf-push-connection", + LifecycleStatus: "retiring", IdempotencyKey: "identity-pairing-ssf-revision", Version: 3, + }} + secrets := &memorySecretStore{values: map[string][]byte{"ssf-push-connection": []byte("push-secret-value-for-retirement-00000")}} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets} + + err := manager.DiscardPreparedConnection(context.Background(), "identity-pairing-ssf-revision") + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" { + t.Fatalf("retirement did not return a safe resumable category: %v", err) + } + if _, getErr := repository.SecurityEventConnection(context.Background()); getErr != nil || secrets.Len() != 1 { + t.Fatal("retirement cleanup deleted local state before the safety window completed") + } +} + +func TestDiscardPreparedConnectionCannotDeleteConnectionBoundAfterSnapshot(t *testing.T) { + managementReference := "ssf-management-connection" + connection := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + CredentialRef: "ssf-push-connection", ManagementCredentialRef: &managementReference, + LifecycleStatus: "error", LastErrorCategory: pointerTo("discovery_failed"), + IdempotencyKey: "identity-pairing-ssf-revision", Version: 2, + } + repository := &memoryConnectionRepository{connection: connection} + repository.beforeDiscard = func(current *store.SecurityEventConnection) { + audience := "urn:easyai:ssf:receiver:" + uuid.NewString() + streamID := uuid.NewString() + current.Audience = &audience + current.StreamID = &streamID + current.LifecycleStatus = "verifying" + current.Version++ + } + secrets := &memorySecretStore{values: map[string][]byte{ + "ssf-push-connection": []byte("push-secret-value-for-cleanup-00000000"), + "ssf-management-connection": []byte("machine-secret-value-for-cleanup-00000"), + }} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets} + + err := manager.DiscardPreparedConnection(context.Background(), "identity-pairing-ssf-revision") + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "connection_cleanup_failed" { + t.Fatalf("stale cleanup error=%v, want safe connection_cleanup_failed", err) + } + persisted, getErr := repository.SecurityEventConnection(context.Background()) + if getErr != nil || persisted.StreamID == nil || persisted.Version != 3 { + t.Fatalf("stale cleanup removed the newly bound connection: connection=%+v err=%v", persisted, getErr) + } + if secrets.Len() != 2 { + t.Fatalf("stale cleanup removed Secrets used by the newly bound connection: %d", secrets.Len()) + } + repository.mutex.Lock() + queued := len(repository.cleanupDue) + repository.mutex.Unlock() + if queued != 0 { + t.Fatalf("stale cleanup queued %d Secrets used by the newly bound connection", queued) + } +} + +func TestConnectionFailureReturnsOnlySafeCategory(t *testing.T) { + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", LifecycleStatus: "connecting", Version: 1, + }} + manager := &ConnectionManager{repository: repository} + err := manager.connectionError(context.Background(), *repository.connection, "discovery_failed", errors.New("sensitive-marker token-value https://internal.example")) + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "discovery_failed" { + t.Fatalf("connection failure category=%v", err) + } + if strings.Contains(err.Error(), "sensitive-marker") || strings.Contains(err.Error(), "internal.example") || strings.Contains(err.Error(), "token-value") { + t.Fatalf("connection failure exposed its cause: %q", err.Error()) + } +} + +func pointerTo(value string) *string { return &value } + func TestConnectionLifecycleCanResumeOnlyIncompleteOrFailedConnections(t *testing.T) { for _, test := range []struct { status string @@ -207,6 +806,198 @@ func TestExistingConnectionCredentialRepairDoesNotRequirePublicBaseURL(t *testin } } +func TestTransmitterIssuerAllowsLoopbackHTTPOnlyInLocalEnvironments(t *testing.T) { + for _, appEnv := range []string{"", "production", "staging"} { + if _, err := validatedIssuerURL("http://127.0.0.2:18004/ssf", appEnv); err == nil { + t.Fatalf("%s accepted loopback HTTP transmitter issuer", appEnv) + } + } + for _, appEnv := range []string{"local", "development", "dev", "test"} { + if _, err := validatedIssuerURL("http://127.0.0.2:18004/ssf", appEnv); err != nil { + t.Fatalf("%s rejected loopback HTTP transmitter issuer: %v", appEnv, err) + } + } +} + +func TestDraftConnectionCannotReplaceCredentialOwnedByAnotherRevision(t *testing.T) { + managementReference := "ssf-management-active" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + CredentialRef: "ssf-push-active", ManagementClientID: "active-machine", + ManagementCredentialRef: &managementReference, LifecycleStatus: "error", Version: 2, + IdempotencyKey: "identity-pairing-ssf-active", + }} + secrets := &memorySecretStore{values: map[string][]byte{ + "ssf-push-active": []byte("push-secret-value-for-active-000000000"), + "ssf-management-active": []byte("active-machine-secret-value-000000000"), + }} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets, config: ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), + }} + newSecret := []byte("draft-machine-secret-value-0000000000") + defer clear(newSecret) + + _, err := manager.Connect(context.Background(), "https://auth.example/ssf", "draft-machine", newSecret, "identity-pairing-ssf-draft") + if !errors.Is(err, store.ErrSecurityEventConnectionConflict) { + t.Fatalf("draft credential replacement error=%v", err) + } + connection, getErr := repository.SecurityEventConnection(context.Background()) + if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference || connection.ManagementClientID != "active-machine" { + t.Fatalf("active connection credential metadata changed: %#v err=%v", connection, getErr) + } + if !secrets.Has(managementReference) || secrets.Len() != 2 { + t.Fatal("active connection credential was deleted or a draft credential was retained") + } +} + +func TestDraftConnectionReportsExistingRetirementAsResumable(t *testing.T) { + managementReference := "ssf-management-active" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + CredentialRef: "ssf-push-active", ManagementClientID: "active-machine", + ManagementCredentialRef: &managementReference, LifecycleStatus: "retiring", Version: 3, + IdempotencyKey: "identity-pairing-ssf-active", + }} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), + }} + secret := []byte("draft-machine-secret-value-0000000000") + defer clear(secret) + + _, err := manager.Connect(context.Background(), "https://other-auth.example/ssf", "draft-machine", secret, "identity-pairing-ssf-draft") + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" { + t.Fatalf("retirement conflict was not classified as resumable: %v", err) + } +} + +func TestRetiringConnectionRejectsSameOwnerReconnectBeforeReadingOrReplacingSecrets(t *testing.T) { + for _, status := range []string{"retiring", "disconnect_pending"} { + for _, credentialsProvided := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/credentials=%t", status, credentialsProvided), func(t *testing.T) { + managementReference := "ssf-management-retiring" + owner := "identity-pairing-ssf-current" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + CredentialRef: "ssf-push-retiring", ManagementClientID: "machine-client", + ManagementCredentialRef: &managementReference, LifecycleStatus: status, Version: 3, + IdempotencyKey: owner, + }} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), + }} + var clientID string + var secret []byte + if credentialsProvided { + clientID = "machine-client" + secret = []byte("replacement-machine-secret-value-000000") + defer clear(secret) + } + + _, err := manager.Connect(context.Background(), "https://auth.example/ssf", clientID, secret, owner) + var categorized interface{ SafeErrorCategory() string } + if !errors.As(err, &categorized) || categorized.SafeErrorCategory() != "retirement_pending" { + t.Fatalf("same-owner reconnect during retirement error=%v", err) + } + connection, getErr := repository.SecurityEventConnection(context.Background()) + if getErr != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference { + t.Fatalf("retiring connection credential metadata changed: %v", getErr) + } + repository.mutex.Lock() + queued := len(repository.cleanupDue) + repository.mutex.Unlock() + if queued != 0 { + t.Fatal("retiring reconnect staged or retired a Secret") + } + }) + } + } +} + +func TestConflictingConnectionRetirementNeverRetiresCurrentPairingOwner(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + oldOwner := "identity-pairing-ssf-active" + currentOwner := "identity-pairing-ssf-draft" + repository := &memoryConnectionRepository{connection: &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + CredentialRef: "ssf-push-active", ManagementClientID: "machine-client", LifecycleStatus: "enabled", Version: 3, + IdempotencyKey: oldOwner, UpdatedAt: time.Now().UTC(), + }} + manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), + RetirementDuration: time.Hour, + }} + secret := []byte("replacement-machine-secret-value-000000") + defer clear(secret) + _, connectErr := manager.Connect(ctx, "https://auth.example/ssf", "machine-client", secret, currentOwner) + var categorized interface{ SafeErrorCategory() string } + if !errors.As(connectErr, &categorized) || categorized.SafeErrorCategory() != "connection_conflict" { + t.Fatalf("local-only retirement handoff error=%v", connectErr) + } + + if err := manager.RetireConflictingConnection(ctx, currentOwner); err != nil { + t.Fatal(err) + } + connection, err := repository.SecurityEventConnection(ctx) + if err != nil || connection.LifecycleStatus != "retiring" { + t.Fatalf("old owner was not safely retired: connection=%#v err=%v", connection, err) + } + + repository.connection.IdempotencyKey = currentOwner + repository.connection.LifecycleStatus = "enabled" + if err := manager.RetireConflictingConnection(ctx, currentOwner); err != nil { + t.Fatal(err) + } + connection, err = repository.SecurityEventConnection(ctx) + if err != nil || connection.LifecycleStatus != "enabled" { + t.Fatalf("current pairing connection was retired by a stale recovery action: connection=%#v err=%v", connection, err) + } +} + +func TestDisconnectDoesNotReturnSuccessWhenLifecycleCASFails(t *testing.T) { + streamID := uuid.NewString() + current := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "://invalid", StreamID: &streamID, + CredentialRef: "ssf-push", LifecycleStatus: "verifying", Version: 4, + } + repository := &memoryConnectionRepository{connection: current} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}, config: ConnectionManagerConfig{AppEnv: "test"}} + stale := *current + stale.LifecycleStatus = "enabled" + + if _, err := manager.disconnect(context.Background(), stale); !errors.Is(err, store.ErrSecurityEventConnectionConflict) { + t.Fatalf("disconnect CAS conflict was reported as success: %v", err) + } + connection, err := repository.SecurityEventConnection(context.Background()) + if err != nil || connection.LifecycleStatus != "verifying" { + t.Fatalf("disconnect CAS conflict changed lifecycle: connection=%#v err=%v", connection, err) + } +} + +func TestDisconnectTreatsRetirementStatesAsIdempotent(t *testing.T) { + for _, lifecycle := range []string{"disconnect_pending", "retiring"} { + t.Run(lifecycle, func(t *testing.T) { + connection := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-disconnect-idempotent", + LifecycleStatus: lifecycle, Version: 9, + } + repository := &memoryConnectionRepository{connection: connection} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}} + + view, err := manager.disconnect(context.Background(), *connection) + if err != nil { + t.Fatal(err) + } + current, getErr := repository.SecurityEventConnection(context.Background()) + if getErr != nil || current.LifecycleStatus != lifecycle || current.Version != 9 || view.Version != 9 { + t.Fatalf("idempotent disconnect changed retirement generation: lifecycle=%q version=%d view=%d err=%v", + current.LifecycleStatus, current.Version, view.Version, getErr) + } + }) + } +} + func TestTransmitterSSRFProtectionBlocksReservedNetworks(t *testing.T) { for _, address := range []string{ "127.0.0.1", "10.0.0.1", "169.254.169.254", "100.64.0.1", "192.0.2.1", "198.51.100.1", "203.0.113.1", "2001:db8::1", @@ -268,10 +1059,445 @@ func TestRetiringConnectionStillAppliesRevocationWatermark(t *testing.T) { t.Fatalf("retiring evaluation=%#v err=%v", evaluation, err) } } + +func TestRetirementDurablyQueuesReferencesBeforeRemovingConnection(t *testing.T) { + ctx := context.Background() + connection := store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-retiring", LifecycleStatus: "retiring", + Version: 1, UpdatedAt: time.Now().Add(-time.Minute), + } + repository := &memoryConnectionRepository{connection: &connection} + secrets := &memorySecretStore{values: map[string][]byte{ + connection.CredentialRef: []byte("push-secret-value-for-retirement-00000"), + }} + manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: ConnectionManagerConfig{RetirementDuration: time.Millisecond}} + + manager.finishRetirement(connection) + + if _, err := repository.SecurityEventConnection(context.Background()); !errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + t.Fatalf("finalized retiring connection still exists: %v", err) + } + repository.mutex.Lock() + _, queued := repository.cleanupDue[connection.CredentialRef] + repository.mutex.Unlock() + if !queued { + t.Fatal("retirement removed the connection before durably queueing its Secret reference") + } + if !secrets.Has(connection.CredentialRef) { + t.Fatal("retirement bypassed the durable Secret cleanup worker") + } +} + +type retireDuringSecretReadStore struct { + once sync.Once + onRetire func() +} + +func (*retireDuringSecretReadStore) Put(context.Context, string, []byte) error { return nil } +func (s *retireDuringSecretReadStore) Get(context.Context, string) ([]byte, error) { + s.once.Do(s.onRetire) + return nil, ErrSecretNotFound +} +func (*retireDuringSecretReadStore) Delete(context.Context, string) error { return nil } + +func retireMemoryConnection(repository *memoryConnectionRepository, connectionID string) { + repository.mutex.Lock() + defer repository.mutex.Unlock() + if repository.connection == nil || repository.connection.ConnectionID != connectionID { + return + } + repository.connection.LifecycleStatus = "retiring" + repository.connection.Version++ + repository.connection.UpdatedAt = time.Now().UTC() +} + +func TestConnectionManagerConstructionFailureCannotReviveConcurrentRetirement(t *testing.T) { + audience := "urn:easyai:ssf:receiver:" + uuid.NewString() + streamID := uuid.NewString() + connection := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + EndpointURL: "https://gateway.example/api/v1/security-events/ssf", Audience: &audience, StreamID: &streamID, + CredentialRef: "ssf-push-constructor-race", LifecycleStatus: "enabled", Version: 7, + } + initialVersion := connection.Version + repository := &memoryConnectionRepository{connection: connection} + secrets := &retireDuringSecretReadStore{onRetire: func() { + retireMemoryConnection(repository, connection.ConnectionID) + }} + + if _, err := NewConnectionManager(context.Background(), repository, secrets, ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), + }, &Metrics{}); err != nil { + t.Fatal(err) + } + current, err := repository.SecurityEventConnection(context.Background()) + if err != nil || current.LifecycleStatus != "retiring" || current.Version != initialVersion+1 { + t.Fatalf("constructor failure revived concurrent retirement: lifecycle=%q version=%d err=%v", + current.LifecycleStatus, current.Version, err) + } +} + +func TestConnectionOperationFailureCannotReviveConcurrentRetirement(t *testing.T) { + connection := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-operation-race", + LifecycleStatus: "verifying", Version: 11, + } + repository := &memoryConnectionRepository{connection: connection} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}} + staleOperationSnapshot := *connection + retireMemoryConnection(repository, connection.ConnectionID) + + _ = manager.connectionError(context.Background(), staleOperationSnapshot, "discovery_failed", errors.New("remote operation failed")) + current, err := repository.SecurityEventConnection(context.Background()) + if err != nil || current.LifecycleStatus != "retiring" || current.Version != staleOperationSnapshot.Version+1 { + t.Fatalf("operation failure revived concurrent retirement: lifecycle=%q version=%d err=%v", + current.LifecycleStatus, current.Version, err) + } +} + +func TestConnectionErrorCannotMoveCurrentRetirementGenerationBackToError(t *testing.T) { + connection := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-current-retirement", + LifecycleStatus: "retiring", Version: 5, + } + repository := &memoryConnectionRepository{connection: connection} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: &memorySecretStore{}} + + _ = manager.connectionError(context.Background(), *connection, "discovery_failed", errors.New("late remote operation failed")) + current, err := repository.SecurityEventConnection(context.Background()) + if err != nil || current.LifecycleStatus != "retiring" || current.Version != connection.Version { + t.Fatalf("late operation failure revived current retirement generation: lifecycle=%q version=%d err=%v", + current.LifecycleStatus, current.Version, err) + } +} + +func TestFinishRetirementDoesNotDeleteAConnectionRevivedAfterItsSnapshot(t *testing.T) { + connection := store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-retirement-generation", + LifecycleStatus: "retiring", Version: 3, UpdatedAt: time.Now().Add(-time.Minute), + } + staleRetirementSnapshot := connection + connection.LifecycleStatus = "enabled" + connection.Version++ + repository := &memoryConnectionRepository{connection: &connection} + secrets := &memorySecretStore{values: map[string][]byte{ + connection.CredentialRef: []byte("push-secret-value-for-retirement-generation"), + }} + manager := &ConnectionManager{ctx: context.Background(), repository: repository, secrets: secrets, + config: ConnectionManagerConfig{RetirementDuration: time.Millisecond}} + + manager.finishRetirement(staleRetirementSnapshot) + + current, err := repository.SecurityEventConnection(context.Background()) + if err != nil || current.LifecycleStatus != "enabled" || current.Version != connection.Version { + t.Fatalf("stale retirement deleted or changed the current connection: lifecycle=%q version=%d err=%v", + current.LifecycleStatus, current.Version, err) + } + if !secrets.Has(connection.CredentialRef) { + t.Fatal("stale retirement deleted a Secret referenced by the current connection") + } +} + +func TestIdentitySecretCleanupWorkerResumesWithoutConnectionManager(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + reference := "ssf-management-staged-restart" + repository := &memoryConnectionRepository{cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}} + secrets := &memorySecretStore{values: map[string][]byte{ + reference: []byte("staged-machine-secret-value-for-restart"), + }} + go RunIdentitySecretCleanupWorker(ctx, repository, secrets) + + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + repository.mutex.Lock() + _, queued := repository.cleanupDue[reference] + repository.mutex.Unlock() + secretExists := secrets.Has(reference) + if !queued && !secretExists { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("queued identity Secret cleanup did not resume after restart") +} + +func TestIdentitySecretCleanupWorkerKeepsClaimWhenDeleteIsUncertain(t *testing.T) { + reference := "ssf-management-worker-delete-uncertain" + repository := &memoryConnectionRepository{cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}} + secrets := &failedStagingSecretStore{deleteErr: errors.New("SecretStore delete result is uncertain")} + + cleanupClaimedIdentitySecrets(context.Background(), repository, secrets) + + repository.mutex.Lock() + _, pending := repository.cleanupDue[reference] + claimToken := repository.cleanupClaims[reference] + repository.mutex.Unlock() + if !pending || claimToken == "" { + t.Fatal("uncertain worker deletion released its durable claim") + } +} + +func TestConnectionManagerDoesNotStartIdentitySecretCleanupWorker(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + reference := "ssf-management-manager-must-not-clean" + claimed := make(chan struct{}, 1) + repository := &memoryConnectionRepository{ + cleanupDue: map[string]time.Time{reference: time.Now().Add(-time.Minute)}, + cleanupClaimed: claimed, + } + if _, err := NewConnectionManager(ctx, repository, &memorySecretStore{}, ConnectionManagerConfig{ + OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), + }, &Metrics{}); err != nil { + t.Fatal(err) + } + select { + case <-claimed: + t.Fatal("ConnectionManager started an identity Secret cleanup worker") + case <-time.After(50 * time.Millisecond): + } +} + +type failedStagingSecretStore struct { + deleteErr error + deletes atomic.Int64 +} + +func (*failedStagingSecretStore) Put(context.Context, string, []byte) error { + return errors.New("SecretStore write result is uncertain") +} +func (*failedStagingSecretStore) Get(context.Context, string) ([]byte, error) { + return nil, ErrSecretNotFound +} +func (s *failedStagingSecretStore) Delete(context.Context, string) error { + s.deletes.Add(1) + return s.deleteErr +} + +func TestStageIdentitySecretKeepsPendingCleanupWhenPutAndDeleteAreUncertain(t *testing.T) { + reference := "ssf-management-uncertain-put" + repository := &memoryConnectionRepository{} + secrets := &failedStagingSecretStore{deleteErr: errors.New("SecretStore delete result is uncertain")} + manager := &ConnectionManager{repository: repository, secrets: secrets} + + if err := manager.stageIdentitySecret(context.Background(), reference, []byte("machine-secret-value-for-uncertain-put")); err == nil { + t.Fatal("uncertain SecretStore Put unexpectedly succeeded") + } + if secrets.deletes.Load() != 1 { + t.Fatalf("cleanup delete attempts=%d want=1", secrets.deletes.Load()) + } + repository.mutex.Lock() + _, pending := repository.cleanupDue[reference] + repository.mutex.Unlock() + if !pending { + t.Fatal("uncertain SecretStore result lost the durable cleanup intent") + } +} + +func TestStageIdentitySecretRemovesPendingCleanupAfterConfirmedDelete(t *testing.T) { + reference := "ssf-management-confirmed-delete" + repository := &memoryConnectionRepository{} + secrets := &failedStagingSecretStore{} + manager := &ConnectionManager{repository: repository, secrets: secrets} + + if err := manager.stageIdentitySecret(context.Background(), reference, []byte("machine-secret-value-for-confirmed-delete")); err == nil { + t.Fatal("failed SecretStore Put unexpectedly succeeded") + } + repository.mutex.Lock() + _, pending := repository.cleanupDue[reference] + repository.mutex.Unlock() + if pending { + t.Fatal("confirmed Secret deletion retained a stale cleanup intent") + } +} + +type ambiguousAdoptionRepository struct { + *memoryConnectionRepository + failCreate bool + failManagement bool + failNext bool +} + +func (r *ambiguousAdoptionRepository) CreateSecurityEventConnection(ctx context.Context, input store.CreateSecurityEventConnectionInput) (store.SecurityEventConnection, error) { + connection, err := r.memoryConnectionRepository.CreateSecurityEventConnection(ctx, input) + if err == nil && r.failCreate { + return store.SecurityEventConnection{}, errors.New("database commit result is uncertain") + } + return connection, err +} + +func (r *ambiguousAdoptionRepository) SetSecurityEventManagementCredential(ctx context.Context, connectionID, clientID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { + connection, err := r.memoryConnectionRepository.SetSecurityEventManagementCredential(ctx, connectionID, clientID, reference, expectedVersion) + if err == nil && r.failManagement { + return store.SecurityEventConnection{}, errors.New("database commit result is uncertain") + } + return connection, err +} + +func (r *ambiguousAdoptionRepository) SetSecurityEventNextCredential(ctx context.Context, connectionID, reference string, expectedVersion int64) (store.SecurityEventConnection, error) { + connection, err := r.memoryConnectionRepository.SetSecurityEventNextCredential(ctx, connectionID, reference, expectedVersion) + if err == nil && r.failNext { + return store.SecurityEventConnection{}, errors.New("database commit result is uncertain") + } + return connection, err +} + +func TestConnectDoesNotRequeueSecretsAfterAmbiguousAdoptionCommit(t *testing.T) { + ctx := context.Background() + base := &memoryConnectionRepository{} + repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failCreate: true} + secrets := &memorySecretStore{} + manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets, config: ConnectionManagerConfig{ + AppEnv: "test", OIDCEnabled: true, OIDCIssuer: "https://auth.example/issuer", OIDCTenantID: uuid.NewString(), + PublicBaseURL: "http://localhost:18089", + }} + managementSecret := []byte("machine-secret-value-for-ambiguous-create") + defer clear(managementSecret) + + if _, err := manager.Connect(ctx, "https://auth.example/ssf", "machine-client", managementSecret, "ambiguous-create"); err == nil { + t.Fatal("ambiguous connection commit unexpectedly reported success") + } + connection, err := base.SecurityEventConnection(ctx) + if err != nil || connection.ManagementCredentialRef == nil { + t.Fatalf("simulated committed connection was not retained: %v", err) + } + base.mutex.Lock() + _, pushQueued := base.cleanupDue[connection.CredentialRef] + _, managementQueued := base.cleanupDue[*connection.ManagementCredentialRef] + base.mutex.Unlock() + if pushQueued || managementQueued { + t.Fatal("ambiguous committed connection credentials were requeued for deletion") + } + if !secrets.Has(connection.CredentialRef) || !secrets.Has(*connection.ManagementCredentialRef) { + t.Fatal("ambiguous committed connection lost an active Secret") + } +} + +func TestManagementCredentialDoesNotRequeueSecretAfterAmbiguousAdoptionCommit(t *testing.T) { + ctx := context.Background() + oldReference := "ssf-management-before-ambiguous-commit" + connection := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), CredentialRef: "ssf-push-active", ManagementCredentialRef: &oldReference, + ManagementClientID: "machine-client", LifecycleStatus: "enabled", + } + base := &memoryConnectionRepository{connection: connection} + repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failManagement: true} + secrets := &memorySecretStore{values: map[string][]byte{oldReference: []byte("old-machine-secret-value-before-commit")}} + manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets} + newSecret := []byte("new-machine-secret-value-after-commit") + defer clear(newSecret) + + if _, err := manager.persistManagementCredential(ctx, *connection, "machine-client", newSecret); err == nil { + t.Fatal("ambiguous management credential commit unexpectedly reported success") + } + committed, err := base.SecurityEventConnection(ctx) + if err != nil || committed.ManagementCredentialRef == nil || *committed.ManagementCredentialRef == oldReference { + t.Fatalf("simulated committed management credential was not retained: %v", err) + } + newReference := *committed.ManagementCredentialRef + base.mutex.Lock() + _, newQueued := base.cleanupDue[newReference] + _, oldQueued := base.cleanupDue[oldReference] + base.mutex.Unlock() + if newQueued || !oldQueued { + t.Fatal("ambiguous commit queued the active credential or lost retirement of the old credential") + } + if !secrets.Has(newReference) { + t.Fatal("ambiguous committed management credential lost its Secret") + } +} + +func TestNextCredentialDoesNotRequeueSecretAfterAmbiguousAdoptionCommit(t *testing.T) { + ctx := context.Background() + audience := "urn:easyai:ssf:receiver:" + uuid.NewString() + streamID := uuid.NewString() + connection := &store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), TransmitterIssuer: "https://auth.example/ssf", + EndpointURL: "https://gateway.example/api/v1/security-events/ssf", Audience: &audience, StreamID: &streamID, + CredentialRef: "ssf-push-active", LifecycleStatus: "enabled", + } + base := &memoryConnectionRepository{connection: connection} + repository := &ambiguousAdoptionRepository{memoryConnectionRepository: base, failNext: true} + secrets := &memorySecretStore{} + manager := &ConnectionManager{ctx: ctx, repository: repository, secrets: secrets} + + if _, err := manager.RotateCredential(ctx); err == nil { + t.Fatal("ambiguous next credential commit unexpectedly reported success") + } + committed, err := base.SecurityEventConnection(ctx) + if err != nil || committed.NextCredentialRef == nil { + t.Fatalf("simulated committed next credential was not retained: %v", err) + } + nextReference := *committed.NextCredentialRef + base.mutex.Lock() + _, queued := base.cleanupDue[nextReference] + base.mutex.Unlock() + if queued { + t.Fatal("ambiguous commit requeued the active next credential") + } + if !secrets.Has(nextReference) { + t.Fatal("ambiguous commit deleted the active next credential") + } +} + func (r *memoryConnectionRepository) PromoteSecurityEventCredential(context.Context, string, string) (store.SecurityEventConnection, error) { return store.SecurityEventConnection{}, errors.New("not used") } -func (r *memoryConnectionRepository) DeleteSecurityEventConnection(context.Context, string) error { +func (r *memoryConnectionRepository) DiscardPreparedSecurityEventConnection(_ context.Context, id, ownerKey string, expectedVersion int64) error { + r.mutex.Lock() + defer r.mutex.Unlock() + if r.connection == nil { + return store.ErrSecurityEventConnectionConflict + } + if r.beforeDiscard != nil { + r.beforeDiscard(r.connection) + } + if r.connection.ConnectionID != id || r.connection.IdempotencyKey != ownerKey || + r.connection.Version != expectedVersion || r.connection.StreamID != nil { + return store.ErrSecurityEventConnectionConflict + } + if r.cleanupDue == nil { + r.cleanupDue = map[string]time.Time{} + } + references := connectionSecretReferences(*r.connection) + for _, reference := range references { + if r.cleanupClaims[reference] != "" { + return store.ErrIdentitySecretCleanupConflict + } + } + now := time.Now().UTC() + for _, reference := range references { + r.cleanupDue[reference] = now + } + r.connection = nil + return nil +} +func (r *memoryConnectionRepository) FinalizeRetiringSecurityEventConnection(_ context.Context, id string, expectedVersion int64) error { + r.mutex.Lock() + defer r.mutex.Unlock() + if r.connection == nil { + return store.ErrSecurityEventConnectionNotFound + } + if r.beforeFinalize != nil { + r.beforeFinalize(r.connection) + } + if r.connection.ConnectionID != id || r.connection.LifecycleStatus != "retiring" || r.connection.Version != expectedVersion { + return store.ErrSecurityEventConnectionConflict + } + if r.cleanupDue == nil { + r.cleanupDue = map[string]time.Time{} + } + for _, reference := range connectionSecretReferences(*r.connection) { + if r.cleanupClaims[reference] != "" { + return store.ErrIdentitySecretCleanupConflict + } + } + now := time.Now().UTC() + for _, reference := range connectionSecretReferences(*r.connection) { + r.cleanupDue[reference] = now + } + r.finalizeCalls++ r.connection = nil return nil } @@ -280,10 +1506,13 @@ func (r *memoryConnectionRepository) SecurityEventMetrics(context.Context, strin } type memorySecretStore struct { + mutex sync.Mutex values map[string][]byte } func (s *memorySecretStore) Put(_ context.Context, reference string, value []byte) error { + s.mutex.Lock() + defer s.mutex.Unlock() if s.values == nil { s.values = map[string][]byte{} } @@ -291,6 +1520,8 @@ func (s *memorySecretStore) Put(_ context.Context, reference string, value []byt return nil } func (s *memorySecretStore) Get(_ context.Context, reference string) ([]byte, error) { + s.mutex.Lock() + defer s.mutex.Unlock() value, ok := s.values[reference] if !ok { return nil, ErrSecretNotFound @@ -298,10 +1529,25 @@ func (s *memorySecretStore) Get(_ context.Context, reference string) ([]byte, er return append([]byte(nil), value...), nil } func (s *memorySecretStore) Delete(_ context.Context, reference string) error { + s.mutex.Lock() + defer s.mutex.Unlock() delete(s.values, reference) return nil } +func (s *memorySecretStore) Has(reference string) bool { + s.mutex.Lock() + defer s.mutex.Unlock() + _, ok := s.values[reference] + return ok +} + +func (s *memorySecretStore) Len() int { + s.mutex.Lock() + defer s.mutex.Unlock() + return len(s.values) +} + func TestConnectionManagerCreatesPausedStreamWithBackendGeneratedBearer(t *testing.T) { repository := &memoryConnectionRepository{} secrets := &memorySecretStore{} @@ -384,8 +1630,8 @@ func TestConnectionManagerCreatesPausedStreamWithBackendGeneratedBearer(t *testi if strings.Contains(string(encoded), "credential") || strings.Contains(string(encoded), "machine-secret-for-test") || strings.Contains(string(encoded), "ssf-push-") { t.Fatalf("secret metadata escaped in response: %s", encoded) } - if len(secrets.values) != 2 { - t.Fatalf("secret count=%d", len(secrets.values)) + if secrets.Len() != 2 { + t.Fatalf("secret count=%d", secrets.Len()) } restarted := &ConnectionManager{repository: repository, secrets: secrets, config: ConnectionManagerConfig{}} restartedClientID, restartedSecret, err := restarted.IntrospectionCredential(ctx) diff --git a/apps/api/internal/securityevents/identity_secret_cleanup_worker.go b/apps/api/internal/securityevents/identity_secret_cleanup_worker.go new file mode 100644 index 0000000..a67fa94 --- /dev/null +++ b/apps/api/internal/securityevents/identity_secret_cleanup_worker.go @@ -0,0 +1,52 @@ +package securityevents + +import ( + "context" + "errors" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +const ( + identitySecretCleanupInterval = time.Minute + identitySecretCleanupLease = 5 * time.Minute + identitySecretCleanupBatch = 100 +) + +type IdentitySecretCleanupRepository interface { + ClaimIdentitySecretCleanups(context.Context, int, time.Duration) ([]store.IdentitySecretCleanupClaim, error) + CompleteIdentitySecretCleanup(context.Context, string, string) (bool, error) +} + +// RunIdentitySecretCleanupWorker owns cleanup at the server lifecycle rather +// than at any individual Active or Prepared identity Runtime. PostgreSQL claim +// tokens make running one worker per server replica safe. +func RunIdentitySecretCleanupWorker(ctx context.Context, repository IdentitySecretCleanupRepository, secrets SecretStore) { + if repository == nil || secrets == nil { + return + } + ticker := time.NewTicker(identitySecretCleanupInterval) + defer ticker.Stop() + for { + cleanupClaimedIdentitySecrets(ctx, repository, secrets) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func cleanupClaimedIdentitySecrets(ctx context.Context, repository IdentitySecretCleanupRepository, secrets SecretStore) { + claims, err := repository.ClaimIdentitySecretCleanups(ctx, identitySecretCleanupBatch, identitySecretCleanupLease) + if err != nil { + return + } + for _, claim := range claims { + if err := secrets.Delete(ctx, claim.Reference); err != nil && !errors.Is(err, ErrSecretNotFound) { + continue + } + _, _ = repository.CompleteIdentitySecretCleanup(ctx, claim.Reference, claim.ClaimToken) + } +} diff --git a/apps/api/internal/securityevents/retirement_worker.go b/apps/api/internal/securityevents/retirement_worker.go new file mode 100644 index 0000000..fe142af --- /dev/null +++ b/apps/api/internal/securityevents/retirement_worker.go @@ -0,0 +1,79 @@ +package securityevents + +import ( + "context" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +const ( + defaultSecurityEventRetirementDuration = 6 * time.Minute + securityEventRetirementWorkerInterval = time.Minute +) + +type SecurityEventRetirementRepository interface { + SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) + FinalizeRetiringSecurityEventConnection(context.Context, string, int64) error +} + +// RunSecurityEventRetirementWorker owns the local end of SSF retirement at the +// server lifecycle. It intentionally does not share an Active identity +// Runtime's context, because that Runtime is closed before the RFC 8935 overlap +// period expires. Store-level generation CAS makes one worker per replica safe. +func RunSecurityEventRetirementWorker(ctx context.Context, repository SecurityEventRetirementRepository) { + runSecurityEventRetirementWorker( + ctx, + repository, + defaultSecurityEventRetirementDuration, + securityEventRetirementWorkerInterval, + ) +} + +func runSecurityEventRetirementWorker( + ctx context.Context, + repository SecurityEventRetirementRepository, + retirementDuration time.Duration, + interval time.Duration, +) { + if repository == nil { + return + } + if retirementDuration <= 0 { + retirementDuration = defaultSecurityEventRetirementDuration + } + if interval <= 0 { + interval = securityEventRetirementWorkerInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + finalizeDueSecurityEventRetirement(ctx, repository, retirementDuration) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func finalizeDueSecurityEventRetirement( + ctx context.Context, + repository SecurityEventRetirementRepository, + retirementDuration time.Duration, +) { + connection, err := repository.SecurityEventConnection(ctx) + if err != nil || connection.LifecycleStatus != "retiring" { + return + } + if retirementDuration <= 0 { + retirementDuration = defaultSecurityEventRetirementDuration + } + if time.Since(connection.UpdatedAt) < retirementDuration { + return + } + // Finalization atomically queues every Secret reference and removes only the + // exact retiring generation. A concurrent worker or changed generation is a + // benign CAS miss and will be observed on the next loop. + _ = repository.FinalizeRetiringSecurityEventConnection(ctx, connection.ConnectionID, connection.Version) +} diff --git a/apps/api/internal/securityevents/retirement_worker_test.go b/apps/api/internal/securityevents/retirement_worker_test.go new file mode 100644 index 0000000..df39085 --- /dev/null +++ b/apps/api/internal/securityevents/retirement_worker_test.go @@ -0,0 +1,178 @@ +package securityevents + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" + "github.com/google/uuid" +) + +func TestSecurityEventRetirementWorkerOutlivesCanceledRuntime(t *testing.T) { + connection := retiringWorkerTestConnection(time.Now().UTC()) + repository := &memoryConnectionRepository{connection: &connection} + secrets := &memorySecretStore{values: retirementWorkerTestSecrets(connection)} + + runtimeCtx, cancelRuntime := context.WithCancel(context.Background()) + manager := &ConnectionManager{ + ctx: runtimeCtx, repository: repository, secrets: secrets, + config: ConnectionManagerConfig{RetirementDuration: 40 * time.Millisecond}, + } + go manager.finishRetirement(connection) + cancelRuntime() + time.Sleep(10 * time.Millisecond) + if _, err := repository.SecurityEventConnection(context.Background()); err != nil { + t.Fatalf("runtime cancellation unexpectedly finalized retirement: %v", err) + } + + serverCtx, cancelServer := context.WithCancel(context.Background()) + defer cancelServer() + go runSecurityEventRetirementWorker(serverCtx, repository, 40*time.Millisecond, 5*time.Millisecond) + waitForRetirementFinalization(t, repository) + + for reference := range retirementWorkerTestSecrets(connection) { + repository.mutex.Lock() + _, queued := repository.cleanupDue[reference] + repository.mutex.Unlock() + if !queued { + t.Fatalf("retired Secret %q was not durably queued", reference) + } + if !secrets.Has(reference) { + t.Fatalf("retirement worker directly deleted Secret %q before the durable cleanup worker claimed it", reference) + } + } +} + +func TestSecurityEventRetirementWorkerResumesAfterServerRestart(t *testing.T) { + connection := retiringWorkerTestConnection(time.Now().UTC()) + repository := &memoryConnectionRepository{connection: &connection} + + firstCtx, stopFirst := context.WithCancel(context.Background()) + go runSecurityEventRetirementWorker(firstCtx, repository, 80*time.Millisecond, 5*time.Millisecond) + time.Sleep(15 * time.Millisecond) + stopFirst() + if _, err := repository.SecurityEventConnection(context.Background()); err != nil { + t.Fatalf("not-yet-due retirement disappeared before restart: %v", err) + } + + time.Sleep(75 * time.Millisecond) + secondCtx, stopSecond := context.WithCancel(context.Background()) + defer stopSecond() + go runSecurityEventRetirementWorker(secondCtx, repository, 80*time.Millisecond, 5*time.Millisecond) + waitForRetirementFinalization(t, repository) +} + +func TestConcurrentSecurityEventRetirementWorkersFinalizeOneGenerationOnce(t *testing.T) { + connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute)) + repository := &memoryConnectionRepository{connection: &connection} + + start := make(chan struct{}) + done := make(chan struct{}, 2) + for range 2 { + go func() { + <-start + finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond) + done <- struct{}{} + }() + } + close(start) + <-done + <-done + + if _, err := repository.SecurityEventConnection(context.Background()); !errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + t.Fatalf("current retiring generation still exists: %v", err) + } + repository.mutex.Lock() + finalizeCalls := repository.finalizeCalls + queued := len(repository.cleanupDue) + repository.mutex.Unlock() + if finalizeCalls != 1 { + t.Fatalf("retirement generation finalized %d times, want 1", finalizeCalls) + } + if queued != len(retirementWorkerTestSecrets(connection)) { + t.Fatalf("queued Secret references=%d want=%d", queued, len(retirementWorkerTestSecrets(connection))) + } +} + +func TestSecurityEventRetirementWorkerPreservesChangedGeneration(t *testing.T) { + connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute)) + initialVersion := connection.Version + repository := &memoryConnectionRepository{connection: &connection} + repository.beforeFinalize = func(current *store.SecurityEventConnection) { + current.LifecycleStatus = "enabled" + current.Version++ + } + + finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond) + + current, err := repository.SecurityEventConnection(context.Background()) + if err != nil || current.LifecycleStatus != "enabled" || current.Version != initialVersion+1 { + t.Fatalf("changed generation was not preserved: lifecycle=%q version=%d err=%v", current.LifecycleStatus, current.Version, err) + } + repository.mutex.Lock() + queued := len(repository.cleanupDue) + repository.mutex.Unlock() + if queued != 0 { + t.Fatalf("changed generation queued %d active Secret references", queued) + } +} + +func TestSecurityEventRetirementWorkerKeepsConnectionWhenCleanupQueueCannotAdoptReferences(t *testing.T) { + connection := retiringWorkerTestConnection(time.Now().UTC().Add(-time.Minute)) + repository := &memoryConnectionRepository{ + connection: &connection, + cleanupClaims: map[string]string{connection.CredentialRef: uuid.NewString()}, + } + + finalizeDueSecurityEventRetirement(context.Background(), repository, time.Millisecond) + + current, err := repository.SecurityEventConnection(context.Background()) + if err != nil || current.ConnectionID != connection.ConnectionID || current.LifecycleStatus != "retiring" { + t.Fatalf("failed durable handoff lost retiring connection: lifecycle=%q err=%v", current.LifecycleStatus, err) + } + repository.mutex.Lock() + finalizeCalls := repository.finalizeCalls + repository.mutex.Unlock() + if finalizeCalls != 0 { + t.Fatalf("failed durable handoff finalized generation %d times", finalizeCalls) + } +} + +func retiringWorkerTestConnection(updatedAt time.Time) store.SecurityEventConnection { + managementReference := "ssf-management-retirement-worker-" + uuid.NewString() + nextReference := "ssf-push-next-retirement-worker-" + uuid.NewString() + return store.SecurityEventConnection{ + ConnectionID: uuid.NewString(), + CredentialRef: "ssf-push-retirement-worker-" + uuid.NewString(), + NextCredentialRef: &nextReference, + ManagementCredentialRef: &managementReference, + LifecycleStatus: "retiring", + Version: 9, + UpdatedAt: updatedAt, + } +} + +func retirementWorkerTestSecrets(connection store.SecurityEventConnection) map[string][]byte { + secrets := map[string][]byte{connection.CredentialRef: []byte("retired-push-secret-value")} + if connection.NextCredentialRef != nil { + secrets[*connection.NextCredentialRef] = []byte("retired-next-push-secret-value") + } + if connection.ManagementCredentialRef != nil { + secrets[*connection.ManagementCredentialRef] = []byte("retired-management-secret-value") + } + return secrets +} + +func waitForRetirementFinalization(t *testing.T, repository *memoryConnectionRepository) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if _, err := repository.SecurityEventConnection(context.Background()); errors.Is(err, store.ErrSecurityEventConnectionNotFound) { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("retiring connection was not finalized by the server-lifecycle worker") +} diff --git a/apps/api/internal/store/identity_configuration_secret_cleanup_integration_test.go b/apps/api/internal/store/identity_configuration_secret_cleanup_integration_test.go new file mode 100644 index 0000000..aafb44e --- /dev/null +++ b/apps/api/internal/store/identity_configuration_secret_cleanup_integration_test.go @@ -0,0 +1,141 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" + "github.com/google/uuid" +) + +func TestDisableActiveIdentityRevisionQueuesSecretsAfterRuntimeRetirementAndClearsReferences(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + seedIdentityActivationPrerequisites(t, ctx, db, "default") + revisionID := seedIdentityLifecycleRevision(t, ctx, db, identity.RevisionActive, "default") + machineReference := "identity-machine-disable-" + uuid.NewString() + sessionReference := "identity-session-disable-" + uuid.NewString() + if _, err := db.pool.Exec(ctx, `UPDATE gateway_identity_configuration_revisions +SET machine_credential_ref=$2,session_encryption_key_ref=$3 WHERE id=$1::uuid`, + revisionID, machineReference, sessionReference); err != nil { + t.Fatalf("seed active identity Secret references: %v", err) + } + if err := db.QueueIdentitySecretCleanup(ctx, machineReference, time.Now().Add(-time.Minute)); err != nil { + t.Fatalf("seed prematurely due machine Secret cleanup: %v", err) + } + + disableStartedAt := time.Now().UTC() + disabled, err := db.DisableActiveIdentityRevision(ctx, 1, "trace-disable-secrets", "audit-disable-secrets") + if err != nil { + t.Fatalf("disable active identity revision: %v", err) + } + if disabled.State != identity.RevisionSuperseded || disabled.Version != 2 || + disabled.MachineCredentialRef != "" || disabled.SessionEncryptionKeyRef != "" { + t.Fatalf("disabled revision retained Secret references: %#v", disabled) + } + + persisted, err := db.IdentityConfigurationRevision(ctx, revisionID) + if err != nil { + t.Fatalf("read disabled identity revision: %v", err) + } + if persisted.State != identity.RevisionSuperseded || persisted.MachineCredentialRef != "" || + persisted.SessionEncryptionKeyRef != "" { + t.Fatalf("persisted disabled revision retained Secret references: %#v", persisted) + } + + type queuedSecret struct { + reference string + status string + notBefore time.Time + } + rows, err := db.pool.Query(ctx, `SELECT secret_ref,status,not_before +FROM gateway_identity_secret_cleanup_queue ORDER BY secret_ref`) + if err != nil { + t.Fatalf("read disabled identity Secret cleanup queue: %v", err) + } + defer rows.Close() + queued := make(map[string]queuedSecret, 2) + for rows.Next() { + var secret queuedSecret + if err := rows.Scan(&secret.reference, &secret.status, &secret.notBefore); err != nil { + t.Fatalf("scan disabled identity Secret cleanup: %v", err) + } + queued[secret.reference] = secret + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate disabled identity Secret cleanup queue: %v", err) + } + if len(queued) != 2 { + t.Fatalf("disabled identity cleanup queue entries=%v, want both Secret references", queued) + } + minimumSafeCleanup := disableStartedAt.Add(45 * time.Second) + for _, reference := range []string{machineReference, sessionReference} { + secret, ok := queued[reference] + if !ok || secret.status != "pending" || secret.notBefore.Before(minimumSafeCleanup) { + t.Fatalf("queued Secret %q=%#v present=%t, want pending cleanup no earlier than %s", + reference, secret, ok, minimumSafeCleanup) + } + } + claims, err := db.ClaimIdentitySecretCleanups(ctx, 10, time.Minute) + if err != nil { + t.Fatalf("claim not-yet-due disabled identity Secrets: %v", err) + } + if len(claims) != 0 { + t.Fatalf("disabled identity Secrets became claimable before Runtime retirement: %#v", claims) + } +} + +func TestDisableActiveIdentityRevisionCleanupConflictRollsBackStateReferencesAndQueue(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + seedIdentityActivationPrerequisites(t, ctx, db, "default") + revisionID := seedIdentityLifecycleRevision(t, ctx, db, identity.RevisionActive, "default") + machineReference := "identity-machine-disable-conflict-" + uuid.NewString() + sessionReference := "identity-session-disable-conflict-" + uuid.NewString() + if _, err := db.pool.Exec(ctx, `UPDATE gateway_identity_configuration_revisions +SET machine_credential_ref=$2,session_encryption_key_ref=$3 WHERE id=$1::uuid`, + revisionID, machineReference, sessionReference); err != nil { + t.Fatalf("seed conflicting active identity Secret references: %v", err) + } + if err := db.QueueIdentitySecretCleanup(ctx, machineReference, time.Now().Add(-time.Minute)); err != nil { + t.Fatalf("queue machine Secret before conflict claim: %v", err) + } + claims, err := db.ClaimIdentitySecretCleanups(ctx, 1, time.Minute) + if err != nil || len(claims) != 1 || claims[0].Reference != machineReference { + t.Fatalf("claim machine Secret before disable: claims=%#v error=%v", claims, err) + } + + _, err = db.DisableActiveIdentityRevision(ctx, 1, "trace-disable-conflict", "audit-disable-conflict") + if !errors.Is(err, ErrIdentitySecretCleanupConflict) { + t.Fatalf("disable with claimed Secret cleanup error=%v, want cleanup conflict", err) + } + persisted, err := db.IdentityConfigurationRevision(ctx, revisionID) + if err != nil { + t.Fatalf("read active identity revision after cleanup conflict: %v", err) + } + if persisted.State != identity.RevisionActive || persisted.Version != 1 || + persisted.MachineCredentialRef != machineReference || persisted.SessionEncryptionKeyRef != sessionReference { + t.Fatalf("cleanup conflict partially disabled identity revision: %#v", persisted) + } + + var status, claimToken string + if err := db.pool.QueryRow(ctx, `SELECT status,claim_token::text FROM gateway_identity_secret_cleanup_queue +WHERE secret_ref=$1`, machineReference).Scan(&status, &claimToken); err != nil { + t.Fatalf("read claimed machine Secret after disable rollback: %v", err) + } + if status != "claimed" || claimToken != claims[0].ClaimToken { + t.Fatalf("disable rollback changed claimed cleanup status=%q token=%q", status, claimToken) + } + var sessionQueueEntries int + if err := db.pool.QueryRow(ctx, `SELECT count(*) FROM gateway_identity_secret_cleanup_queue +WHERE secret_ref=$1`, sessionReference).Scan(&sessionQueueEntries); err != nil { + t.Fatalf("count session Secret cleanup after disable rollback: %v", err) + } + if sessionQueueEntries != 0 { + t.Fatalf("disable cleanup conflict left %d partial session cleanup entries", sessionQueueEntries) + } +} diff --git a/apps/api/internal/store/identity_configurations.go b/apps/api/internal/store/identity_configurations.go index ec52884..2ef31ad 100644 --- a/apps/api/internal/store/identity_configurations.go +++ b/apps/api/internal/store/identity_configurations.go @@ -29,6 +29,55 @@ COALESCE(machine_credential_ref,''),COALESCE(session_encryption_key_ref,''),sess session_refresh_seconds,version,COALESCE(last_error_category,''),COALESCE(last_trace_id,''),COALESCE(last_audit_id,''), validated_at,activated_at,superseded_at,created_at,updated_at` +// identityConfigurationLifecycleLockID serializes the database-side boundary +// between onboarding reservations and Active Revision mutations. A transaction +// must hold this lock before deciding that no Active configuration or foreign +// pairing reservation exists. +const identityConfigurationLifecycleLockID int64 = 0x4541494944454e54 + +// identityDisabledSecretCleanupDelay is intentionally longer than the +// IdentityRuntimeManager's 30-second old-Runtime retirement window. Starting +// cleanup one minute after the database handoff leaves an additional 30 +// seconds for commit, Runtime publication, timer scheduling, and worker jitter. +const identityDisabledSecretCleanupDelay = time.Minute + +func lockIdentityConfigurationLifecycle(ctx context.Context, tx pgx.Tx) error { + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, identityConfigurationLifecycleLockID) + return err +} + +// beginIdentityConfigurationLifecycleTx deliberately uses READ COMMITTED. +// PostgreSQL fixes the transaction snapshot when a SERIALIZABLE transaction +// executes the advisory-lock SELECT, before that statement finishes waiting. +// A later state check can therefore miss the preceding lock holder's commit. +// READ COMMITTED gives every statement after lock acquisition a fresh snapshot, +// while the transaction-scoped advisory lock serializes all lifecycle decisions. +func (s *Store) beginIdentityConfigurationLifecycleTx(ctx context.Context) (pgx.Tx, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return nil, err + } + if err := lockIdentityConfigurationLifecycle(ctx, tx); err != nil { + _ = tx.Rollback(ctx) + return nil, err + } + return tx, nil +} + +func queueDisabledIdentitySecretCleanupTx(ctx context.Context, tx pgx.Tx, reference string, notBefore time.Time) error { + tag, err := tx.Exec(ctx, `INSERT INTO gateway_identity_secret_cleanup_queue(secret_ref,not_before) +VALUES($1,$2) ON CONFLICT(secret_ref) DO UPDATE +SET not_before=GREATEST(gateway_identity_secret_cleanup_queue.not_before,EXCLUDED.not_before),updated_at=now() +WHERE gateway_identity_secret_cleanup_queue.status='pending'`, reference, notBefore) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrIdentitySecretCleanupConflict + } + return nil +} + func (s *Store) CreateIdentityConfigurationRevision(ctx context.Context, revision identity.Revision) (identity.Revision, error) { scopes, _ := json.Marshal(revision.Scopes) capabilities, _ := json.Marshal(revision.Capabilities) @@ -58,7 +107,16 @@ FROM gateway_identity_configuration_revisions WHERE state='active'`)) func (s *Store) LatestInactiveIdentityConfigurationRevision(ctx context.Context) (identity.Revision, error) { revision, err := scanIdentityRevision(s.pool.QueryRow(ctx, `SELECT `+identityRevisionColumns+` -FROM gateway_identity_configuration_revisions WHERE state IN ('draft','validated','failed') ORDER BY created_at DESC LIMIT 1`)) +FROM gateway_identity_configuration_revisions +WHERE id=( + SELECT revision.id + FROM gateway_identity_configuration_revisions revision + JOIN gateway_identity_onboarding_exchanges exchange ON exchange.revision_id=revision.id + WHERE revision.state IN ('draft','validated','failed') + ORDER BY NOT (exchange.status='cancelled' AND exchange.cleanup_status='completed') DESC, + revision.created_at DESC,exchange.created_at DESC + LIMIT 1 +)`)) return revision, normalizeIdentityRevisionError(err) } @@ -108,10 +166,16 @@ VALUES($1,$2,$3,$4::jsonb) ON CONFLICT(operation,idempotency_key) DO NOTHING`, } func (s *Store) ApplyIdentityManifest(ctx context.Context, id string, expectedVersion int64, applied identity.ManifestApplication) (identity.Revision, error) { - current, err := s.IdentityConfigurationRevision(ctx, id) + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) if err != nil { return identity.Revision{}, err } + defer tx.Rollback(ctx) + current, err := scanIdentityRevision(tx.QueryRow(ctx, `SELECT `+identityRevisionColumns+` +FROM gateway_identity_configuration_revisions WHERE id=$1::uuid FOR UPDATE`, id)) + if err != nil { + return identity.Revision{}, normalizeIdentityRevisionError(err) + } if current.Version != expectedVersion { return identity.Revision{}, identity.ErrRevisionConflict } @@ -119,9 +183,20 @@ func (s *Store) ApplyIdentityManifest(ctx context.Context, id string, expectedVe if err != nil { return identity.Revision{}, err } + newReferences := make([]string, 0, 2) + for _, reference := range []string{updated.MachineCredentialRef, updated.SessionEncryptionKeyRef} { + if reference != "" { + newReferences = append(newReferences, reference) + } + } + if len(newReferences) > 0 { + if err := adoptPendingIdentitySecrets(ctx, tx, newReferences...); err != nil { + return identity.Revision{}, err + } + } scopes, _ := json.Marshal(updated.Scopes) capabilities, _ := json.Marshal(updated.Capabilities) - revision, err := scanIdentityRevision(s.pool.QueryRow(ctx, ` + revision, err := scanIdentityRevision(tx.QueryRow(ctx, ` UPDATE gateway_identity_configuration_revisions SET issuer=$3,tenant_id=$4,application_id=$5,audience=NULLIF($6,''),browser_client_id=NULLIF($7,''),machine_client_id=NULLIF($8,''), scopes=$9::jsonb,capabilities=$10::jsonb,token_introspection=$11,session_revocation=$12, @@ -138,14 +213,35 @@ RETURNING `+identityRevisionColumns, if errors.Is(err, pgx.ErrNoRows) { return identity.Revision{}, identity.ErrRevisionConflict } - return revision, err + if err != nil { + return identity.Revision{}, err + } + activeReferences := map[string]struct{}{} + for _, reference := range newReferences { + activeReferences[reference] = struct{}{} + } + for _, reference := range []string{current.MachineCredentialRef, current.SessionEncryptionKeyRef} { + if reference == "" { + continue + } + if _, stillActive := activeReferences[reference]; stillActive { + continue + } + if err := queueIdentitySecretCleanupTx(ctx, tx, reference, time.Now().UTC()); err != nil { + return identity.Revision{}, err + } + } + if err := tx.Commit(ctx); err != nil { + return identity.Revision{}, err + } + return revision, nil } func (s *Store) MarkIdentityRevisionValidated(ctx context.Context, id string, expectedVersion int64, traceID, auditID string) (identity.Revision, error) { revision, err := scanIdentityRevision(s.pool.QueryRow(ctx, ` UPDATE gateway_identity_configuration_revisions SET state='validated',validated_at=now(),last_error_category=NULL, last_trace_id=NULLIF($3,''),last_audit_id=NULLIF($4,''),version=version+1,updated_at=now() -WHERE id=$1::uuid AND version=$2 AND state IN ('draft','superseded') +WHERE id=$1::uuid AND version=$2 AND state='draft' RETURNING `+identityRevisionColumns, id, expectedVersion, traceID, auditID)) if errors.Is(err, pgx.ErrNoRows) { return identity.Revision{}, identity.ErrRevisionConflict @@ -178,11 +274,22 @@ RETURNING `+identityRevisionColumns, id, expectedVersion, category, traceID, aud } func (s *Store) ActivateIdentityRevision(ctx context.Context, id string, expectedVersion int64, traceID, auditID string) (identity.Revision, bool, error) { - tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + tx, err := s.beginIdentityConfigurationLifecycleTx(ctx) if err != nil { return identity.Revision{}, false, err } defer tx.Rollback(ctx) + var foreignPairingReservation bool + if err := tx.QueryRow(ctx, `SELECT EXISTS( + SELECT 1 FROM gateway_identity_pairing_start_reservation + WHERE (state='starting' AND expires_at > now()) + OR (state='paired' AND revision_id IS DISTINCT FROM $1::uuid) +)`, id).Scan(&foreignPairingReservation); err != nil { + return identity.Revision{}, false, err + } + if foreignPairingReservation { + return identity.Revision{}, false, identity.ErrPairingInProgress + } if ok, err := hasBreakGlassManager(ctx, tx); err != nil { return identity.Revision{}, false, err } else if !ok { @@ -197,9 +304,16 @@ SELECT local_tenant_key FROM gateway_identity_configuration_revisions WHERE id=$ return identity.Revision{}, false, identity.ErrLocalTenantInvalid } var previousID, previousIssuer, previousTenant, previousAudience, previousClient, previousSessionRef string - _ = tx.QueryRow(ctx, `SELECT id::text,COALESCE(issuer,''),COALESCE(tenant_id,''),COALESCE(audience,''), + if err := tx.QueryRow(ctx, `SELECT id::text,COALESCE(issuer,''),COALESCE(tenant_id,''),COALESCE(audience,''), COALESCE(browser_client_id,''),COALESCE(session_encryption_key_ref,'') FROM gateway_identity_configuration_revisions -WHERE state='active' FOR UPDATE`).Scan(&previousID, &previousIssuer, &previousTenant, &previousAudience, &previousClient, &previousSessionRef) +WHERE state='active' FOR UPDATE`).Scan( + &previousID, &previousIssuer, &previousTenant, &previousAudience, &previousClient, &previousSessionRef, + ); err != nil && !errors.Is(err, pgx.ErrNoRows) { + return identity.Revision{}, false, err + } + if previousID != "" && previousID != id { + return identity.Revision{}, false, identity.ErrActiveConfigurationHandoffRequired + } var nextIssuer, nextTenant, nextAudience, nextClient, nextSessionRef string if err := tx.QueryRow(ctx, `SELECT COALESCE(issuer,''),COALESCE(tenant_id,''),COALESCE(audience,''), COALESCE(browser_client_id,''),COALESCE(session_encryption_key_ref,'') FROM gateway_identity_configuration_revisions @@ -228,6 +342,9 @@ WHERE id=$1::uuid AND version=$2 AND state='validated' RETURNING `+identityRevis return identity.Revision{}, false, err } } + if _, err := tx.Exec(ctx, `DELETE FROM gateway_identity_pairing_start_reservation WHERE revision_id=$1::uuid`, id); err != nil { + return identity.Revision{}, false, err + } if err := tx.Commit(ctx); err != nil { return identity.Revision{}, false, err } @@ -235,7 +352,7 @@ WHERE id=$1::uuid AND version=$2 AND state='validated' RETURNING `+identityRevis } func (s *Store) DisableActiveIdentityRevision(ctx context.Context, expectedVersion int64, traceID, auditID string) (identity.Revision, error) { - tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + tx, err := s.beginIdentityConfigurationLifecycleTx(ctx) if err != nil { return identity.Revision{}, err } @@ -245,9 +362,33 @@ func (s *Store) DisableActiveIdentityRevision(ctx context.Context, expectedVersi } else if !ok { return identity.Revision{}, identity.ErrBreakGlassRequired } + var activeID, machineReference, sessionReference string + if err := tx.QueryRow(ctx, `SELECT id::text,COALESCE(machine_credential_ref,''),COALESCE(session_encryption_key_ref,'') +FROM gateway_identity_configuration_revisions +WHERE state='active' AND version=$1 FOR UPDATE`, expectedVersion).Scan( + &activeID, &machineReference, &sessionReference, + ); errors.Is(err, pgx.ErrNoRows) { + return identity.Revision{}, identity.ErrRevisionConflict + } else if err != nil { + return identity.Revision{}, err + } + cleanupNotBefore := time.Now().UTC().Add(identityDisabledSecretCleanupDelay) + references := make(map[string]struct{}, 2) + for _, reference := range []string{machineReference, sessionReference} { + if reference != "" { + references[reference] = struct{}{} + } + } + for reference := range references { + if err := queueDisabledIdentitySecretCleanupTx(ctx, tx, reference, cleanupNotBefore); err != nil { + return identity.Revision{}, err + } + } revision, err := scanIdentityRevision(tx.QueryRow(ctx, `UPDATE gateway_identity_configuration_revisions SET state='superseded', -superseded_at=now(),last_trace_id=NULLIF($2,''),last_audit_id=NULLIF($3,''),version=version+1,updated_at=now() -WHERE state='active' AND version=$1 RETURNING `+identityRevisionColumns, expectedVersion, traceID, auditID)) +superseded_at=now(),machine_credential_ref=NULL,session_encryption_key_ref=NULL, +last_trace_id=NULLIF($3,''),last_audit_id=NULLIF($4,''),version=version+1,updated_at=now() +WHERE id=$1::uuid AND state='active' AND version=$2 +RETURNING `+identityRevisionColumns, activeID, expectedVersion, traceID, auditID)) if errors.Is(err, pgx.ErrNoRows) { return identity.Revision{}, identity.ErrRevisionConflict } diff --git a/apps/api/internal/store/identity_pairing.go b/apps/api/internal/store/identity_pairing.go index 6930ebc..d5de66d 100644 --- a/apps/api/internal/store/identity_pairing.go +++ b/apps/api/internal/store/identity_pairing.go @@ -2,24 +2,156 @@ package store import ( "context" + "encoding/json" "errors" + "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" "github.com/jackc/pgx/v5" ) const identityPairingColumns = ` -id::text,revision_id::text,remote_exchange_id::text,exchange_token_ref,status,remote_version,expires_at,version, -COALESCE(last_error_category,''),COALESCE(auth_center_audit_id,''),COALESCE(last_trace_id,''),created_at,updated_at` +id::text,revision_id::text,remote_exchange_id::text,exchange_token_ref,status,cleanup_status,remote_version,expires_at,version, +COALESCE(last_error_category,''),COALESCE(auth_center_audit_id,''),COALESCE(last_trace_id,''),cancelled_at,cleanup_completed_at, +created_at,updated_at` + +func (s *Store) ReserveIdentityPairingStart(ctx context.Context, attemptID string, expiresAt time.Time) error { + tx, err := s.beginIdentityConfigurationLifecycleTx(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, `DELETE FROM gateway_identity_pairing_start_reservation +WHERE state='starting' AND expires_at <= now()`); err != nil { + return err + } + var activeConfigurationExists bool + if err := tx.QueryRow(ctx, `SELECT EXISTS( + SELECT 1 FROM gateway_identity_configuration_revisions + WHERE state='active' +)`).Scan(&activeConfigurationExists); err != nil { + return err + } + if activeConfigurationExists { + return identity.ErrActiveConfigurationHandoffRequired + } + var blocked bool + if err := tx.QueryRow(ctx, `SELECT EXISTS( + SELECT 1 FROM gateway_identity_configuration_revisions revision + JOIN gateway_identity_onboarding_exchanges exchange ON exchange.revision_id=revision.id + WHERE revision.state IN ('draft','validated','failed') + AND NOT (exchange.status='cancelled' AND exchange.cleanup_status='completed') +)`).Scan(&blocked); err != nil { + return err + } + if blocked { + return identity.ErrPairingInProgress + } + if _, err := tx.Exec(ctx, `INSERT INTO gateway_identity_pairing_start_reservation(singleton,attempt_id,state,expires_at) +VALUES(true,$1::uuid,'starting',$2)`, attemptID, expiresAt); err != nil { + if isUniqueViolation(err) { + return identity.ErrPairingInProgress + } + return err + } + return tx.Commit(ctx) +} + +func (s *Store) ReleaseIdentityPairingStart(ctx context.Context, attemptID string) error { + _, err := s.pool.Exec(ctx, `DELETE FROM gateway_identity_pairing_start_reservation +WHERE attempt_id=$1::uuid AND state='starting'`, attemptID) + return err +} + +func (s *Store) CommitIdentityPairingStart(ctx context.Context, revision identity.Revision, exchange identity.PairingExchange) (identity.PairingExchange, error) { + tx, err := s.beginIdentityConfigurationLifecycleTx(ctx) + if err != nil { + return identity.PairingExchange{}, err + } + defer tx.Rollback(ctx) + var reserved bool + if err := tx.QueryRow(ctx, `SELECT true FROM gateway_identity_pairing_start_reservation +WHERE singleton=true AND attempt_id=$1::uuid AND state='starting' FOR UPDATE`, exchange.ID).Scan(&reserved); errors.Is(err, pgx.ErrNoRows) { + return identity.PairingExchange{}, identity.ErrPairingInProgress + } else if err != nil { + return identity.PairingExchange{}, err + } + if !reserved { + return identity.PairingExchange{}, identity.ErrPairingInProgress + } + var activeConfigurationExists bool + if err := tx.QueryRow(ctx, `SELECT EXISTS( + SELECT 1 FROM gateway_identity_configuration_revisions WHERE state='active' +)`).Scan(&activeConfigurationExists); err != nil { + return identity.PairingExchange{}, err + } + if activeConfigurationExists { + return identity.PairingExchange{}, identity.ErrActiveConfigurationHandoffRequired + } + if err := adoptPendingIdentitySecrets(ctx, tx, exchange.ExchangeTokenRef); err != nil { + return identity.PairingExchange{}, err + } + scopes, _ := json.Marshal(revision.Scopes) + capabilities, _ := json.Marshal(revision.Capabilities) + if _, err := tx.Exec(ctx, `INSERT INTO gateway_identity_configuration_revisions ( + id,state,schema_version,auth_center_url,role_prefix,local_tenant_key,public_base_url,web_base_url, + jit_enabled,legacy_jwt_enabled,scopes,capabilities,session_idle_seconds,session_absolute_seconds,session_refresh_seconds,last_trace_id +) VALUES ($1::uuid,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12::jsonb,$13,$14,$15,NULLIF($16,''))`, + revision.ID, revision.State, revision.SchemaVersion, revision.AuthCenterURL, revision.RolePrefix, + revision.LocalTenantKey, revision.PublicBaseURL, revision.WebBaseURL, revision.JITEnabled, revision.LegacyJWTEnabled, + string(scopes), string(capabilities), revision.SessionIdleSeconds, revision.SessionAbsoluteSeconds, + revision.SessionRefreshSeconds, revision.LastTraceID, + ); err != nil { + return identity.PairingExchange{}, err + } + created, err := scanIdentityPairing(tx.QueryRow(ctx, `INSERT INTO gateway_identity_onboarding_exchanges ( + id,revision_id,remote_exchange_id,exchange_token_ref,status,cleanup_status,remote_version,expires_at,last_trace_id +) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,NULLIF($9,'')) +RETURNING `+identityPairingColumns, + exchange.ID, revision.ID, exchange.RemoteExchangeID, exchange.ExchangeTokenRef, + exchange.Status, exchange.CleanupStatus, exchange.RemoteVersion, exchange.ExpiresAt, exchange.LastTraceID, + )) + if err != nil { + return identity.PairingExchange{}, err + } + tag, err := tx.Exec(ctx, `UPDATE gateway_identity_pairing_start_reservation +SET state='paired',revision_id=$2::uuid,expires_at=$3,updated_at=now() +WHERE attempt_id=$1::uuid AND state='starting'`, exchange.ID, revision.ID, exchange.ExpiresAt) + if err != nil { + return identity.PairingExchange{}, err + } + if tag.RowsAffected() != 1 { + return identity.PairingExchange{}, identity.ErrPairingInProgress + } + if err := tx.Commit(ctx); err != nil { + return identity.PairingExchange{}, err + } + return created, nil +} + +func (s *Store) IdentityPairingStartBlocked(ctx context.Context) (bool, error) { + var blocked bool + err := s.pool.QueryRow(ctx, ` +SELECT EXISTS( + SELECT 1 FROM gateway_identity_configuration_revisions revision + JOIN gateway_identity_onboarding_exchanges exchange ON exchange.revision_id=revision.id + WHERE revision.state IN ('draft','validated','failed') + AND NOT (exchange.status='cancelled' AND exchange.cleanup_status='completed') + ) OR EXISTS ( + SELECT 1 FROM gateway_identity_pairing_start_reservation + WHERE state='paired' OR expires_at > now() +)`).Scan(&blocked) + return blocked, err +} func (s *Store) CreateIdentityPairingExchange(ctx context.Context, exchange identity.PairingExchange) (identity.PairingExchange, error) { return scanIdentityPairing(s.pool.QueryRow(ctx, ` INSERT INTO gateway_identity_onboarding_exchanges ( - id,revision_id,remote_exchange_id,exchange_token_ref,status,remote_version,expires_at,last_trace_id -) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,NULLIF($8,'')) + id,revision_id,remote_exchange_id,exchange_token_ref,status,cleanup_status,remote_version,expires_at,last_trace_id +) VALUES ($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,NULLIF($9,'')) RETURNING `+identityPairingColumns, exchange.ID, exchange.RevisionID, exchange.RemoteExchangeID, exchange.ExchangeTokenRef, - exchange.Status, exchange.RemoteVersion, exchange.ExpiresAt, exchange.LastTraceID, + exchange.Status, exchange.CleanupStatus, exchange.RemoteVersion, exchange.ExpiresAt, exchange.LastTraceID, )) } @@ -43,8 +175,67 @@ FROM gateway_identity_onboarding_exchanges WHERE revision_id=$1::uuid`, revision func (s *Store) PendingIdentityPairingExchanges(ctx context.Context) ([]identity.PairingExchange, error) { rows, err := s.pool.Query(ctx, `SELECT `+identityPairingColumns+` +FROM gateway_identity_onboarding_exchanges exchange +WHERE exchange.id=( + SELECT candidate_exchange.id + FROM gateway_identity_onboarding_exchanges candidate_exchange + JOIN gateway_identity_configuration_revisions revision ON revision.id=candidate_exchange.revision_id + WHERE revision.state IN ('draft','validated','failed') + ORDER BY NOT (candidate_exchange.status='cancelled' AND candidate_exchange.cleanup_status='completed') DESC, + revision.created_at DESC,candidate_exchange.created_at DESC + LIMIT 1 +) +AND exchange.status IN ('metadata_pending','preparing','ready','credentials_saved')`) + if err != nil { + return nil, err + } + defer rows.Close() + exchanges := make([]identity.PairingExchange, 0) + for rows.Next() { + exchange, scanErr := scanIdentityPairing(rows) + if scanErr != nil { + return nil, scanErr + } + exchanges = append(exchanges, exchange) + } + return exchanges, rows.Err() +} + +func (s *Store) PendingIdentityPairingCleanups(ctx context.Context) ([]identity.PairingExchange, error) { + rows, err := s.pool.Query(ctx, `SELECT `+identityPairingColumns+` FROM gateway_identity_onboarding_exchanges -WHERE status IN ('metadata_pending','preparing','ready','credentials_saved') ORDER BY created_at`) +WHERE status='cancelled' AND cleanup_status='pending' ORDER BY updated_at`) + if err != nil { + return nil, err + } + defer rows.Close() + exchanges := make([]identity.PairingExchange, 0) + for rows.Next() { + exchange, scanErr := scanIdentityPairing(rows) + if scanErr != nil { + return nil, scanErr + } + exchanges = append(exchanges, exchange) + } + return exchanges, rows.Err() +} + +func (s *Store) CompletedIdentityPairingsAwaitingActivation(ctx context.Context) ([]identity.PairingExchange, error) { + rows, err := s.pool.Query(ctx, `SELECT `+identityPairingColumns+` +FROM gateway_identity_onboarding_exchanges exchange +WHERE exchange.id=( + SELECT candidate_exchange.id + FROM gateway_identity_onboarding_exchanges candidate_exchange + JOIN gateway_identity_configuration_revisions revision ON revision.id=candidate_exchange.revision_id + WHERE revision.state IN ('draft','validated','failed') + ORDER BY NOT (candidate_exchange.status='cancelled' AND candidate_exchange.cleanup_status='completed') DESC, + revision.created_at DESC,candidate_exchange.created_at DESC + LIMIT 1 +) +AND exchange.status='completed' AND EXISTS ( + SELECT 1 FROM gateway_identity_configuration_revisions revision + WHERE revision.id=exchange.revision_id AND revision.state IN ('draft','validated') +)`) if err != nil { return nil, err } @@ -61,13 +252,139 @@ WHERE status IN ('metadata_pending','preparing','ready','credentials_saved') ORD } func (s *Store) UpdateIdentityPairingExchange(ctx context.Context, id string, expectedVersion int64, update identity.PairingExchangeUpdate) (identity.PairingExchange, error) { - exchange, err := scanIdentityPairing(s.pool.QueryRow(ctx, ` + tx, err := s.pool.Begin(ctx) + if err != nil { + return identity.PairingExchange{}, err + } + defer tx.Rollback(ctx) + exchange, err := scanIdentityPairing(tx.QueryRow(ctx, ` UPDATE gateway_identity_onboarding_exchanges SET status=$3,remote_version=$4,last_error_category=NULLIF($5,''), auth_center_audit_id=COALESCE(NULLIF($6,''),auth_center_audit_id),version=version+1,updated_at=now() WHERE id=$1::uuid AND version=$2 RETURNING `+identityPairingColumns, id, expectedVersion, update.Status, update.RemoteVersion, update.LastErrorCategory, update.AuthCenterAuditID, )) + if errors.Is(err, pgx.ErrNoRows) { + return identity.PairingExchange{}, identity.ErrRevisionConflict + } + if err != nil { + return identity.PairingExchange{}, err + } + if update.Status == identity.PairingCompleted || update.Status == identity.PairingFailed || update.Status == identity.PairingExpired { + if err := queueIdentitySecretCleanupTx(ctx, tx, exchange.ExchangeTokenRef, time.Now().UTC()); err != nil { + return identity.PairingExchange{}, err + } + } + if err := tx.Commit(ctx); err != nil { + return identity.PairingExchange{}, err + } + return exchange, nil +} + +func (s *Store) RecordIdentityPairingRetryFailure(ctx context.Context, id string, status identity.PairingStatus, category string) (identity.PairingExchange, error) { + exchange, err := scanIdentityPairing(s.pool.QueryRow(ctx, ` +UPDATE gateway_identity_onboarding_exchanges SET last_error_category=$3,updated_at=now() +WHERE id=$1::uuid AND status=$2 AND cleanup_status='none' +RETURNING `+identityPairingColumns, id, status, category)) + if errors.Is(err, pgx.ErrNoRows) { + return identity.PairingExchange{}, identity.ErrRevisionConflict + } + return exchange, err +} + +func (s *Store) CancelIdentityPairingExchange(ctx context.Context, id string, expectedVersion int64, traceID, auditID string) (identity.PairingExchange, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + if err != nil { + return identity.PairingExchange{}, err + } + defer tx.Rollback(ctx) + exchange, err := scanIdentityPairing(tx.QueryRow(ctx, `SELECT `+identityPairingColumns+` +FROM gateway_identity_onboarding_exchanges WHERE id=$1::uuid FOR UPDATE`, id)) + if errors.Is(err, pgx.ErrNoRows) { + return identity.PairingExchange{}, identity.ErrRevisionNotFound + } + if err != nil { + return identity.PairingExchange{}, err + } + if exchange.Status == identity.PairingCancelled { + return exchange, tx.Commit(ctx) + } + if exchange.Version != expectedVersion { + return identity.PairingExchange{}, identity.ErrRevisionConflict + } + revision, err := scanIdentityRevision(tx.QueryRow(ctx, `SELECT `+identityRevisionColumns+` +FROM gateway_identity_configuration_revisions WHERE id=$1::uuid FOR UPDATE`, exchange.RevisionID)) + if err != nil { + return identity.PairingExchange{}, normalizeIdentityRevisionError(err) + } + if revision.State != identity.RevisionDraft && revision.State != identity.RevisionValidated && revision.State != identity.RevisionFailed { + return identity.PairingExchange{}, identity.ErrPairingNotCancellable + } + if _, err := tx.Exec(ctx, `UPDATE gateway_identity_configuration_revisions SET state='failed', +last_error_category='pairing_cancelled',last_trace_id=NULLIF($2,''),last_audit_id=NULLIF($3,''), +version=version+1,updated_at=now() WHERE id=$1::uuid`, exchange.RevisionID, traceID, auditID); err != nil { + return identity.PairingExchange{}, err + } + exchange, err = scanIdentityPairing(tx.QueryRow(ctx, ` +UPDATE gateway_identity_onboarding_exchanges SET status='cancelled',cleanup_status='pending',cancelled_at=now(), +cleanup_completed_at=NULL,version=version+1,updated_at=now() WHERE id=$1::uuid AND version=$2 +RETURNING `+identityPairingColumns, id, expectedVersion)) + if errors.Is(err, pgx.ErrNoRows) { + return identity.PairingExchange{}, identity.ErrRevisionConflict + } + if err != nil { + return identity.PairingExchange{}, err + } + if err := queueIdentitySecretCleanupTx(ctx, tx, exchange.ExchangeTokenRef, time.Now().UTC()); err != nil { + return identity.PairingExchange{}, err + } + if err := tx.Commit(ctx); err != nil { + return identity.PairingExchange{}, err + } + return exchange, nil +} + +func (s *Store) CompleteIdentityPairingCleanup(ctx context.Context, id string, expectedVersion int64) (identity.PairingExchange, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable}) + if err != nil { + return identity.PairingExchange{}, err + } + defer tx.Rollback(ctx) + var revisionID string + if err := tx.QueryRow(ctx, `SELECT revision_id::text FROM gateway_identity_onboarding_exchanges +WHERE id=$1::uuid AND version=$2 AND status='cancelled' AND cleanup_status='pending' FOR UPDATE`, id, expectedVersion).Scan(&revisionID); errors.Is(err, pgx.ErrNoRows) { + return identity.PairingExchange{}, identity.ErrRevisionConflict + } else if err != nil { + return identity.PairingExchange{}, err + } + tag, err := tx.Exec(ctx, `UPDATE gateway_identity_configuration_revisions SET machine_credential_ref=NULL, + session_encryption_key_ref=NULL,version=version+1,updated_at=now() WHERE id=$1::uuid AND state='failed'`, revisionID) + if err != nil { + return identity.PairingExchange{}, err + } + if tag.RowsAffected() != 1 { + return identity.PairingExchange{}, identity.ErrRevisionConflict + } + exchange, err := scanIdentityPairing(tx.QueryRow(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET cleanup_status='completed',cleanup_completed_at=now(),version=version+1,updated_at=now() +WHERE id=$1::uuid AND version=$2 AND status='cancelled' AND cleanup_status='pending' +RETURNING `+identityPairingColumns, id, expectedVersion)) + if err != nil { + return identity.PairingExchange{}, err + } + if _, err := tx.Exec(ctx, `DELETE FROM gateway_identity_pairing_start_reservation WHERE revision_id=$1::uuid`, revisionID); err != nil { + return identity.PairingExchange{}, err + } + if err := tx.Commit(ctx); err != nil { + return identity.PairingExchange{}, err + } + return exchange, nil +} + +func (s *Store) RecordIdentityPairingCleanupFailure(ctx context.Context, id, category string) (identity.PairingExchange, error) { + exchange, err := scanIdentityPairing(s.pool.QueryRow(ctx, `UPDATE gateway_identity_onboarding_exchanges +SET last_error_category=$2,updated_at=now() WHERE id=$1::uuid AND status='cancelled' AND cleanup_status='pending' +RETURNING `+identityPairingColumns, id, category)) if errors.Is(err, pgx.ErrNoRows) { return identity.PairingExchange{}, identity.ErrRevisionConflict } @@ -78,9 +395,10 @@ func scanIdentityPairing(row scanner) (identity.PairingExchange, error) { var exchange identity.PairingExchange var status string if err := row.Scan( - &exchange.ID, &exchange.RevisionID, &exchange.RemoteExchangeID, &exchange.ExchangeTokenRef, &status, + &exchange.ID, &exchange.RevisionID, &exchange.RemoteExchangeID, &exchange.ExchangeTokenRef, &status, &exchange.CleanupStatus, &exchange.RemoteVersion, &exchange.ExpiresAt, &exchange.Version, &exchange.LastErrorCategory, - &exchange.AuthCenterAuditID, &exchange.LastTraceID, &exchange.CreatedAt, &exchange.UpdatedAt, + &exchange.AuthCenterAuditID, &exchange.LastTraceID, &exchange.CancelledAt, &exchange.CleanupCompletedAt, + &exchange.CreatedAt, &exchange.UpdatedAt, ); err != nil { return identity.PairingExchange{}, err } diff --git a/apps/api/internal/store/identity_pairing_integration_test.go b/apps/api/internal/store/identity_pairing_integration_test.go new file mode 100644 index 0000000..976cfb2 --- /dev/null +++ b/apps/api/internal/store/identity_pairing_integration_test.go @@ -0,0 +1,715 @@ +package store + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/identity" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestIdentityPairingCancellationKeepsOlderInactivePairingBlockedUntilEveryCleanupCompletes(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx := context.Background() + + type pairingFixture struct { + revisionID string + pairingID string + machineRef string + sessionRef string + createdAt time.Time + displayName string + } + now := time.Now().UTC() + fixtures := []pairingFixture{ + { + revisionID: uuid.NewString(), pairingID: uuid.NewString(), + machineRef: "identity-machine-older", sessionRef: "identity-session-older", + createdAt: now.Add(-time.Minute), displayName: "older", + }, + { + revisionID: uuid.NewString(), pairingID: uuid.NewString(), + machineRef: "identity-machine-latest", sessionRef: "identity-session-latest", + createdAt: now, displayName: "latest", + }, + } + + for _, fixture := range fixtures { + if _, err := db.pool.Exec(ctx, ` +INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url, + machine_credential_ref,session_encryption_key_ref,created_at,updated_at +) VALUES ($1::uuid,'draft','https://auth.test.example','default','https://gateway.test.example', + 'https://gateway-web.test.example',$2,$3,$4,$4)`, + fixture.revisionID, fixture.machineRef, fixture.sessionRef, fixture.createdAt); err != nil { + t.Fatalf("seed %s inactive revision: %v", fixture.displayName, err) + } + if _, err := db.CreateIdentityPairingExchange(ctx, identity.PairingExchange{ + ID: fixture.pairingID, RevisionID: fixture.revisionID, RemoteExchangeID: uuid.NewString(), + ExchangeTokenRef: "identity-exchange-" + fixture.pairingID, + Status: identity.PairingCredentialsSaved, CleanupStatus: identity.PairingCleanupNone, + RemoteVersion: 4, ExpiresAt: now.Add(time.Hour), LastTraceID: "trace-start-" + fixture.displayName, + }); err != nil { + t.Fatalf("seed %s pairing: %v", fixture.displayName, err) + } + } + orphanRevisionID := uuid.NewString() + if _, err := db.pool.Exec(ctx, ` +INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url,created_at,updated_at +) VALUES ($1::uuid,'draft','https://auth.test.example','default','https://orphan-api.test.example', + 'https://orphan-web.test.example',$2,$2)`, orphanRevisionID, now.Add(time.Minute)); err != nil { + t.Fatalf("seed orphan revision: %v", err) + } + canonical, err := db.LatestInactiveIdentityConfigurationRevision(ctx) + if err != nil || canonical.ID != fixtures[1].revisionID { + t.Fatalf("canonical inactive revision=%#v error=%v, want latest exchange-backed revision", canonical, err) + } + + blocked, err := db.IdentityPairingStartBlocked(ctx) + if err != nil || !blocked { + t.Fatalf("two inactive pairings blocked=%t error=%v, want blocked", blocked, err) + } + + latest := fixtures[1] + latestCancelled, err := db.CancelIdentityPairingExchange(ctx, latest.pairingID, 1, "trace-cancel-latest", "audit-cancel-latest") + if err != nil { + t.Fatalf("cancel latest pairing: %v", err) + } + latestReplay, err := db.CancelIdentityPairingExchange(ctx, latest.pairingID, 1, "trace-replayed", "audit-replayed") + if err != nil { + t.Fatalf("replay latest cancellation with original version: %v", err) + } + if latestReplay.Status != identity.PairingCancelled || latestReplay.CleanupStatus != identity.PairingCleanupPending || + latestReplay.Version != latestCancelled.Version || latestCancelled.CancelledAt == nil || latestReplay.CancelledAt == nil || + !latestReplay.CancelledAt.Equal(*latestCancelled.CancelledAt) { + t.Fatalf("cancellation replay changed persisted intent: first=%#v replay=%#v", latestCancelled, latestReplay) + } + latestCleaned, err := db.CompleteIdentityPairingCleanup(ctx, latest.pairingID, latestCancelled.Version) + if err != nil { + t.Fatalf("complete latest cleanup: %v", err) + } + if latestCleaned.CleanupStatus != identity.PairingCleanupCompleted || latestCleaned.CleanupCompletedAt == nil { + t.Fatalf("latest cleanup state=%#v", latestCleaned) + } + assertIdentityRevisionSecretRefs(t, ctx, db, latest.revisionID, "", "") + assertIdentityRevisionSecretRefs(t, ctx, db, fixtures[0].revisionID, fixtures[0].machineRef, fixtures[0].sessionRef) + + blocked, err = db.IdentityPairingStartBlocked(ctx) + if err != nil || !blocked { + t.Fatalf("older pairing was ignored after latest cleanup: blocked=%t error=%v", blocked, err) + } + canonical, err = db.LatestInactiveIdentityConfigurationRevision(ctx) + if err != nil || canonical.ID != fixtures[0].revisionID { + t.Fatalf("canonical inactive revision after latest cleanup=%#v error=%v, want older blocker", canonical, err) + } + + older := fixtures[0] + olderCancelled, err := db.CancelIdentityPairingExchange(ctx, older.pairingID, 1, "trace-cancel-older", "audit-cancel-older") + if err != nil { + t.Fatalf("cancel older pairing: %v", err) + } + olderReplay, err := db.CancelIdentityPairingExchange(ctx, older.pairingID, 1, "trace-replayed", "audit-replayed") + if err != nil || olderReplay.Version != olderCancelled.Version { + t.Fatalf("replay older cancellation: replay=%#v error=%v", olderReplay, err) + } + if _, err := db.CompleteIdentityPairingCleanup(ctx, older.pairingID, olderCancelled.Version); err != nil { + t.Fatalf("complete older cleanup: %v", err) + } + assertIdentityRevisionSecretRefs(t, ctx, db, older.revisionID, "", "") + + blocked, err = db.IdentityPairingStartBlocked(ctx) + if err != nil || blocked { + t.Fatalf("all inactive pairing cleanup blocked=%t error=%v, want unblocked", blocked, err) + } + canonical, err = db.LatestInactiveIdentityConfigurationRevision(ctx) + if err != nil || canonical.ID != fixtures[1].revisionID || canonical.ID == orphanRevisionID { + t.Fatalf("canonical cleaned revision=%#v error=%v, orphan draft must stay hidden", canonical, err) + } +} + +func TestIdentityPairingStartReservationSerializesAndRejectsStaleCommit(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx := context.Background() + firstAttempt := uuid.NewString() + secondAttempt := uuid.NewString() + + if err := db.ReserveIdentityPairingStart(ctx, firstAttempt, time.Now().Add(time.Minute)); err != nil { + t.Fatalf("reserve first pairing start: %v", err) + } + if err := db.ReserveIdentityPairingStart(ctx, secondAttempt, time.Now().Add(time.Minute)); !errors.Is(err, identity.ErrPairingInProgress) { + t.Fatalf("second pairing reservation error=%v, want in-progress conflict", err) + } + if _, err := db.pool.Exec(ctx, `UPDATE gateway_identity_pairing_start_reservation SET expires_at=now()-interval '1 second' +WHERE attempt_id=$1::uuid`, firstAttempt); err != nil { + t.Fatalf("expire first pairing reservation: %v", err) + } + if err := db.ReserveIdentityPairingStart(ctx, secondAttempt, time.Now().Add(time.Minute)); err != nil { + t.Fatalf("take over expired pairing reservation: %v", err) + } + staleRevision := identity.Revision{ + ID: uuid.NewString(), State: identity.RevisionDraft, SchemaVersion: 1, AuthCenterURL: "https://auth.test.example", + RolePrefix: "gateway.", LocalTenantKey: "default", PublicBaseURL: "https://gateway.test.example", + WebBaseURL: "https://gateway-web.test.example", Scopes: []string{}, Capabilities: []string{}, + SessionIdleSeconds: 1800, SessionAbsoluteSeconds: 28800, SessionRefreshSeconds: 60, + } + if _, err := db.CommitIdentityPairingStart(ctx, staleRevision, identity.PairingExchange{ + ID: firstAttempt, RevisionID: staleRevision.ID, RemoteExchangeID: uuid.NewString(), + ExchangeTokenRef: "identity-exchange-" + firstAttempt, Status: identity.PairingMetadataPending, + CleanupStatus: identity.PairingCleanupNone, RemoteVersion: 1, ExpiresAt: time.Now().Add(time.Minute), + }); !errors.Is(err, identity.ErrPairingInProgress) { + t.Fatalf("stale start commit error=%v, want in-progress conflict", err) + } + validRevision := staleRevision + validRevision.ID = uuid.NewString() + tokenReference := "identity-exchange-" + secondAttempt + if err := db.QueueIdentitySecretCleanup(ctx, tokenReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatalf("stage replacement exchange token reference: %v", err) + } + committed, err := db.CommitIdentityPairingStart(ctx, validRevision, identity.PairingExchange{ + ID: secondAttempt, RevisionID: validRevision.ID, RemoteExchangeID: uuid.NewString(), + ExchangeTokenRef: tokenReference, Status: identity.PairingMetadataPending, + CleanupStatus: identity.PairingCleanupNone, RemoteVersion: 1, ExpiresAt: time.Now().Add(time.Minute), + }) + if err != nil { + t.Fatalf("commit replacement pairing: %v", err) + } + if err := db.ReleaseIdentityPairingStart(ctx, secondAttempt); err != nil { + t.Fatalf("release paired reservation: %v", err) + } + var reservationState string + if err := db.pool.QueryRow(ctx, `SELECT state FROM gateway_identity_pairing_start_reservation +WHERE attempt_id=$1::uuid`, secondAttempt).Scan(&reservationState); err != nil || reservationState != "paired" { + t.Fatalf("paired reservation state=%q error=%v", reservationState, err) + } + blocked, err := db.IdentityPairingStartBlocked(ctx) + if err != nil || !blocked { + t.Fatalf("committed pairing did not retain singleton reservation: blocked=%t error=%v", blocked, err) + } + cancelled, err := db.CancelIdentityPairingExchange(ctx, committed.ID, committed.Version, "trace-cancel", "audit-cancel") + if err != nil { + t.Fatalf("cancel replacement pairing: %v", err) + } + if _, err := db.CompleteIdentityPairingCleanup(ctx, committed.ID, cancelled.Version); err != nil { + t.Fatalf("complete replacement pairing cleanup: %v", err) + } + blocked, err = db.IdentityPairingStartBlocked(ctx) + if err != nil || blocked { + t.Fatalf("completed cleanup retained singleton reservation: blocked=%t error=%v", blocked, err) + } +} + +func TestIdentityPairingStartRejectsAnyActiveConfigurationBeforeReservation(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx := context.Background() + if _, err := db.pool.Exec(ctx, `INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url +) VALUES ($1::uuid,'active','https://auth.test.example','default','https://gateway.test.example', + 'https://gateway-web.test.example')`, uuid.NewString()); err != nil { + t.Fatalf("seed active identity revision: %v", err) + } + + err := db.ReserveIdentityPairingStart(ctx, uuid.NewString(), time.Now().Add(time.Minute)) + if !errors.Is(err, identity.ErrActiveConfigurationHandoffRequired) { + t.Fatalf("active identity configuration reservation error=%v", err) + } + var reservations int + if err := db.pool.QueryRow(ctx, `SELECT count(*) FROM gateway_identity_pairing_start_reservation`).Scan(&reservations); err != nil { + t.Fatal(err) + } + if reservations != 0 { + t.Fatalf("active credential guard left %d start reservations", reservations) + } +} + +func TestConcurrentIdentityPairingStartReservationsHaveSingleWinner(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + attempts := []string{uuid.NewString(), uuid.NewString()} + ready := make(chan struct{}, len(attempts)) + start := make(chan struct{}) + type reservationResult struct { + attemptID string + err error + } + results := make(chan reservationResult, len(attempts)) + for _, attemptID := range attempts { + go func() { + ready <- struct{}{} + <-start + results <- reservationResult{ + attemptID: attemptID, + err: db.ReserveIdentityPairingStart(ctx, attemptID, time.Now().Add(5*time.Minute)), + } + }() + } + for range attempts { + <-ready + } + close(start) + + var winner string + conflicts := 0 + for range attempts { + result := <-results + switch { + case result.err == nil: + if winner != "" { + t.Fatalf("multiple concurrent reservations succeeded: %s and %s", winner, result.attemptID) + } + winner = result.attemptID + case errors.Is(result.err, identity.ErrPairingInProgress): + conflicts++ + default: + t.Fatalf("reserve concurrent pairing start %s: %v", result.attemptID, result.err) + } + } + if winner == "" || conflicts != 1 { + t.Fatalf("concurrent reservation winner=%q conflicts=%d, want one winner and one conflict", winner, conflicts) + } + + var persistedAttemptID, state string + var reservations int + if err := db.pool.QueryRow(ctx, `SELECT count(*),COALESCE(max(attempt_id::text),''),COALESCE(max(state),'') +FROM gateway_identity_pairing_start_reservation`).Scan(&reservations, &persistedAttemptID, &state); err != nil { + t.Fatalf("read concurrent reservation result: %v", err) + } + if reservations != 1 || persistedAttemptID != winner || state != "starting" { + t.Fatalf("persisted reservations=%d attempt=%q state=%q, want winner %q in starting state", + reservations, persistedAttemptID, state, winner) + } +} + +func TestConcurrentIdentityActivationAndPairingReservationRemainMutuallyExclusive(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + seedIdentityActivationPrerequisites(t, ctx, db, "default") + revisionID := seedIdentityLifecycleRevision(t, ctx, db, identity.RevisionValidated, "default") + attemptID := uuid.NewString() + + gate, err := db.pool.Begin(ctx) + if err != nil { + t.Fatalf("begin identity lifecycle gate: %v", err) + } + defer gate.Rollback(context.Background()) + if err := lockIdentityConfigurationLifecycle(ctx, gate); err != nil { + t.Fatalf("lock identity lifecycle gate: %v", err) + } + + reserveResult := make(chan error, 1) + go func() { + reserveResult <- db.ReserveIdentityPairingStart(ctx, attemptID, time.Now().Add(5*time.Minute)) + }() + waitForIdentityLifecycleLockWaiters(t, ctx, db, 1) + + type activationResult struct { + revision identity.Revision + err error + } + activateResult := make(chan activationResult, 1) + go func() { + revision, _, activateErr := db.ActivateIdentityRevision(ctx, revisionID, 1, "trace-activate", "audit-activate") + activateResult <- activationResult{revision: revision, err: activateErr} + }() + waitForIdentityLifecycleLockWaiters(t, ctx, db, 2) + if err := gate.Commit(ctx); err != nil { + t.Fatalf("release identity lifecycle gate: %v", err) + } + + reserveErr := <-reserveResult + activated := <-activateResult + if reserveErr != nil { + t.Fatalf("first queued pairing reservation failed: %v", reserveErr) + } + + var activeRevisions, startingReservations int + if err := db.pool.QueryRow(ctx, `SELECT + (SELECT count(*) FROM gateway_identity_configuration_revisions WHERE state='active'), + (SELECT count(*) FROM gateway_identity_pairing_start_reservation WHERE state='starting')`).Scan( + &activeRevisions, &startingReservations, + ); err != nil { + t.Fatalf("read activation and reservation invariant: %v", err) + } + if !errors.Is(activated.err, identity.ErrPairingInProgress) || activeRevisions != 0 || startingReservations != 1 { + t.Fatalf("racing activation error=%v revision_state=%q left active=%d starting reservations=%d; "+ + "want pairing-in-progress, active=0, reservation=1", + activated.err, activated.revision.State, activeRevisions, startingReservations) + } +} + +func TestConcurrentIdentityPairingCommitRejectsActiveRevisionCommittedAheadOfLifecycleLock(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + attemptID := uuid.NewString() + if err := db.ReserveIdentityPairingStart(ctx, attemptID, time.Now().Add(5*time.Minute)); err != nil { + t.Fatalf("reserve pairing start before commit race: %v", err) + } + tokenReference := "identity-exchange-" + attemptID + if err := db.QueueIdentitySecretCleanup(ctx, tokenReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatalf("stage exchange token reference before commit race: %v", err) + } + draft := identity.Revision{ + ID: uuid.NewString(), State: identity.RevisionDraft, SchemaVersion: 1, AuthCenterURL: "https://auth.test.example", + RolePrefix: "gateway.", LocalTenantKey: "default", PublicBaseURL: "https://gateway.test.example", + WebBaseURL: "https://gateway-web.test.example", Scopes: []string{}, Capabilities: []string{}, + SessionIdleSeconds: 1800, SessionAbsoluteSeconds: 28800, SessionRefreshSeconds: 60, + } + exchange := identity.PairingExchange{ + ID: attemptID, RevisionID: draft.ID, RemoteExchangeID: uuid.NewString(), ExchangeTokenRef: tokenReference, + Status: identity.PairingMetadataPending, CleanupStatus: identity.PairingCleanupNone, + RemoteVersion: 1, ExpiresAt: time.Now().Add(5 * time.Minute), + } + + gate, err := db.pool.Begin(ctx) + if err != nil { + t.Fatalf("begin identity lifecycle gate: %v", err) + } + defer gate.Rollback(context.Background()) + if err := lockIdentityConfigurationLifecycle(ctx, gate); err != nil { + t.Fatalf("lock identity lifecycle gate: %v", err) + } + + commitResult := make(chan error, 1) + go func() { + _, commitErr := db.CommitIdentityPairingStart(ctx, draft, exchange) + commitResult <- commitErr + }() + waitForIdentityLifecycleLockWaiters(t, ctx, db, 1) + activeID := uuid.NewString() + if _, err := gate.Exec(ctx, `INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url +) VALUES ($1::uuid,'active','https://auth.test.example','default','https://active-gateway.test.example', + 'https://active-gateway-web.test.example')`, activeID); err != nil { + t.Fatalf("commit active revision ahead of pairing commit: %v", err) + } + if err := gate.Commit(ctx); err != nil { + t.Fatalf("release identity lifecycle gate with active revision: %v", err) + } + + commitErr := <-commitResult + if !errors.Is(commitErr, identity.ErrActiveConfigurationHandoffRequired) { + t.Fatalf("pairing commit after active revision error=%v, want active handoff required", commitErr) + } + var activeRevisions, draftRevisions, exchanges, startingReservations int + if err := db.pool.QueryRow(ctx, `SELECT + (SELECT count(*) FROM gateway_identity_configuration_revisions WHERE state='active'), + (SELECT count(*) FROM gateway_identity_configuration_revisions WHERE id=$1::uuid), + (SELECT count(*) FROM gateway_identity_onboarding_exchanges WHERE id=$2::uuid), + (SELECT count(*) FROM gateway_identity_pairing_start_reservation WHERE attempt_id=$2::uuid AND state='starting')`, + draft.ID, attemptID).Scan(&activeRevisions, &draftRevisions, &exchanges, &startingReservations); err != nil { + t.Fatalf("read pairing commit lifecycle invariant: %v", err) + } + if activeRevisions != 1 || draftRevisions != 0 || exchanges != 0 || startingReservations != 1 { + t.Fatalf("pairing commit race left active=%d draft=%d exchanges=%d starting reservations=%d; "+ + "want active=1 draft=0 exchanges=0 reservation=1", + activeRevisions, draftRevisions, exchanges, startingReservations) + } +} + +func TestConcurrentIdentityDisableAndActivationPreserveSingleActiveRevision(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + seedIdentityActivationPrerequisites(t, ctx, db, "default") + previousID := seedIdentityLifecycleRevision(t, ctx, db, identity.RevisionActive, "default") + nextID := seedIdentityLifecycleRevision(t, ctx, db, identity.RevisionValidated, "default") + + gate, err := db.pool.Begin(ctx) + if err != nil { + t.Fatalf("begin identity lifecycle gate: %v", err) + } + defer gate.Rollback(context.Background()) + if err := lockIdentityConfigurationLifecycle(ctx, gate); err != nil { + t.Fatalf("lock identity lifecycle gate: %v", err) + } + + disableResult := make(chan error, 1) + go func() { + _, disableErr := db.DisableActiveIdentityRevision(ctx, 1, "trace-disable", "audit-disable") + disableResult <- disableErr + }() + waitForIdentityLifecycleLockWaiters(t, ctx, db, 1) + + activateResult := make(chan error, 1) + go func() { + _, _, activateErr := db.ActivateIdentityRevision(ctx, nextID, 1, "trace-activate", "audit-activate") + activateResult <- activateErr + }() + waitForIdentityLifecycleLockWaiters(t, ctx, db, 2) + if err := gate.Commit(ctx); err != nil { + t.Fatalf("release identity lifecycle gate: %v", err) + } + + if err := <-disableResult; err != nil { + t.Fatalf("disable first queued active revision: %v", err) + } + activateErr := <-activateResult + if activateErr != nil { + t.Fatalf("activate revision after first queued disable: %v", activateErr) + } + + var activeIDs []string + rows, err := db.pool.Query(ctx, `SELECT id::text FROM gateway_identity_configuration_revisions +WHERE state='active' ORDER BY id`) + if err != nil { + t.Fatalf("read active revisions after disable/activate race: %v", err) + } + defer rows.Close() + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan active revision after disable/activate race: %v", err) + } + activeIDs = append(activeIDs, id) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate active revisions after disable/activate race: %v", err) + } + if len(activeIDs) != 1 || activeIDs[0] != nextID { + t.Fatalf("disable/activate race left active revisions=%v; previous=%q next=%q", + activeIDs, previousID, nextID) + } +} + +func TestCompletedIdentityPairingRestoreUsesCanonicalVisibleRevision(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx := context.Background() + now := time.Now().UTC() + var latestRevisionID string + + for index, createdAt := range []time.Time{now.Add(-time.Minute), now} { + revisionID, pairingID := uuid.NewString(), uuid.NewString() + if _, err := db.pool.Exec(ctx, ` +INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url,created_at,updated_at +) VALUES ($1::uuid,'validated','https://auth.test.example','default','https://gateway.test.example', + 'https://gateway-web.test.example',$2,$2)`, revisionID, createdAt); err != nil { + t.Fatalf("seed completed revision %d: %v", index, err) + } + if _, err := db.CreateIdentityPairingExchange(ctx, identity.PairingExchange{ + ID: pairingID, RevisionID: revisionID, RemoteExchangeID: uuid.NewString(), + ExchangeTokenRef: "identity-exchange-" + pairingID, Status: identity.PairingCompleted, + CleanupStatus: identity.PairingCleanupNone, RemoteVersion: 4, ExpiresAt: now.Add(time.Hour), + }); err != nil { + t.Fatalf("seed completed pairing %d: %v", index, err) + } + latestRevisionID = revisionID + } + + completed, err := db.CompletedIdentityPairingsAwaitingActivation(ctx) + if err != nil { + t.Fatal(err) + } + if len(completed) != 1 || completed[0].RevisionID != latestRevisionID { + t.Fatalf("restored completed pairings=%#v, want only canonical latest revision %s", completed, latestRevisionID) + } + visible, err := db.LatestInactiveIdentityConfigurationRevision(ctx) + if err != nil || visible.ID != latestRevisionID { + t.Fatalf("visible inactive revision=%#v error=%v, want same canonical revision", visible, err) + } +} + +func TestIdentityPairingPersistsBindingErrorCategoryAllowedByLatestMigration(t *testing.T) { + db := newIdentityPairingPostgresTestStore(t) + ctx := context.Background() + revisionID := uuid.NewString() + if _, err := db.pool.Exec(ctx, `INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,local_tenant_key,public_base_url,web_base_url +) VALUES ($1::uuid,'draft','https://auth.test.example','default','https://gateway.test.example', + 'https://gateway-web.test.example')`, revisionID); err != nil { + t.Fatalf("seed revision for binding error category: %v", err) + } + pairingID := uuid.NewString() + created, err := db.CreateIdentityPairingExchange(ctx, identity.PairingExchange{ + ID: pairingID, RevisionID: revisionID, RemoteExchangeID: uuid.NewString(), + ExchangeTokenRef: "identity-exchange-" + pairingID, Status: identity.PairingCredentialsSaved, + CleanupStatus: identity.PairingCleanupNone, RemoteVersion: 4, ExpiresAt: time.Now().Add(time.Hour), + }) + if err != nil { + t.Fatalf("create pairing for binding error category: %v", err) + } + + const category = "security_event_connection_binding_mismatch" + updated, err := db.RecordIdentityPairingRetryFailure(ctx, created.ID, created.Status, category) + if err != nil { + t.Fatalf("persist binding error category: %v", err) + } + if updated.LastErrorCategory != category { + t.Fatalf("binding error category=%q, want %q", updated.LastErrorCategory, category) + } + var persisted string + if err := db.pool.QueryRow(ctx, `SELECT last_error_category FROM gateway_identity_onboarding_exchanges +WHERE id=$1::uuid`, created.ID).Scan(&persisted); err != nil { + t.Fatalf("read persisted binding error category: %v", err) + } + if persisted != category { + t.Fatalf("persisted binding error category=%q, want %q", persisted, category) + } +} + +func assertIdentityRevisionSecretRefs(t *testing.T, ctx context.Context, db *Store, revisionID, wantMachine, wantSession string) { + t.Helper() + revision, err := db.IdentityConfigurationRevision(ctx, revisionID) + if err != nil { + t.Fatalf("read revision %s: %v", revisionID, err) + } + if revision.MachineCredentialRef != wantMachine || revision.SessionEncryptionKeyRef != wantSession { + t.Fatalf("revision %s secret refs machine=%q session=%q, want machine=%q session=%q", + revisionID, revision.MachineCredentialRef, revision.SessionEncryptionKeyRef, wantMachine, wantSession) + } +} + +func seedIdentityActivationPrerequisites(t *testing.T, ctx context.Context, db *Store, tenantKey string) { + t.Helper() + if _, err := db.pool.Exec(ctx, `INSERT INTO gateway_tenants(tenant_key,name,status) +VALUES($1,$2,'active')`, tenantKey, "Identity lifecycle test tenant"); err != nil { + t.Fatalf("seed identity lifecycle tenant: %v", err) + } + if _, err := db.pool.Exec(ctx, `INSERT INTO gateway_users(user_key,username,password_hash,roles,source,status) +VALUES($1,$2,$3,'["manager"]'::jsonb,'gateway','active')`, + "identity-lifecycle-manager", "identity-lifecycle-manager", "test-only-password-hash"); err != nil { + t.Fatalf("seed identity lifecycle break-glass manager: %v", err) + } +} + +func seedIdentityLifecycleRevision( + t *testing.T, + ctx context.Context, + db *Store, + state identity.RevisionState, + tenantKey string, +) string { + t.Helper() + revisionID := uuid.NewString() + if _, err := db.pool.Exec(ctx, `INSERT INTO gateway_identity_configuration_revisions ( + id,state,auth_center_url,issuer,tenant_id,application_id,audience,browser_client_id, + local_tenant_key,public_base_url,web_base_url,validated_at,activated_at +) VALUES ( + $1::uuid,$2,'https://auth.test.example','https://issuer.test.example',$3,$4,$5,$6, + $7,'https://gateway.test.example','https://gateway-web.test.example', + CASE WHEN $2 IN ('validated','active') THEN now() ELSE NULL END, + CASE WHEN $2='active' THEN now() ELSE NULL END +)`, revisionID, state, uuid.NewString(), uuid.NewString(), "urn:easyai:resource:"+revisionID, + "browser-"+revisionID, tenantKey); err != nil { + t.Fatalf("seed %s identity lifecycle revision: %v", state, err) + } + return revisionID +} + +func waitForIdentityLifecycleLockWaiters(t *testing.T, ctx context.Context, db *Store, want int) { + t.Helper() + classID := int64(uint64(identityConfigurationLifecycleLockID) >> 32) + objectID := int64(uint64(identityConfigurationLifecycleLockID) & 0xffffffff) + deadline := time.Now().Add(5 * time.Second) + for { + var waiters int + if err := db.pool.QueryRow(ctx, `SELECT count(*) FROM pg_locks +WHERE locktype='advisory' AND database=(SELECT oid FROM pg_database WHERE datname=current_database()) + AND classid::bigint=$1 AND objid::bigint=$2 AND objsubid=1 AND NOT granted`, classID, objectID).Scan(&waiters); err != nil { + t.Fatalf("read identity lifecycle lock waiters: %v", err) + } + if waiters >= want { + return + } + if time.Now().After(deadline) { + t.Fatalf("identity lifecycle lock waiters=%d, want at least %d", waiters, want) + } + time.Sleep(10 * time.Millisecond) + } +} + +func newIdentityPairingPostgresTestStore(t *testing.T) *Store { + t.Helper() + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run identity pairing PostgreSQL integration tests") + } + ctx := context.Background() + admin, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect identity pairing test database: %v", err) + } + var databaseName string + if err := admin.QueryRow(ctx, `SELECT current_database()`).Scan(&databaseName); err != nil { + admin.Close() + t.Fatalf("read identity pairing test database name: %v", err) + } + if !strings.Contains(strings.ToLower(databaseName), "test") { + admin.Close() + t.Fatalf("refusing to use non-test database %q", databaseName) + } + + schemaName := "gateway_identity_pairing_" + strings.ReplaceAll(uuid.NewString(), "-", "") + schemaIdentifier := pgx.Identifier{schemaName}.Sanitize() + if _, err := admin.Exec(ctx, `CREATE SCHEMA `+schemaIdentifier); err != nil { + admin.Close() + t.Fatalf("create identity pairing test schema: %v", err) + } + config, err := pgxpool.ParseConfig(databaseURL) + if err != nil { + _, _ = admin.Exec(ctx, `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`) + admin.Close() + t.Fatalf("parse identity pairing test database URL: %v", err) + } + config.ConnConfig.RuntimeParams["search_path"] = schemaName + pool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + _, _ = admin.Exec(ctx, `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`) + admin.Close() + t.Fatalf("connect identity pairing test schema: %v", err) + } + t.Cleanup(func() { + pool.Close() + if _, err := admin.Exec(context.Background(), `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`); err != nil { + t.Errorf("drop identity pairing test schema: %v", err) + } + admin.Close() + }) + + _, filename, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("locate identity pairing integration test") + } + migrationDirectory := filepath.Join(filepath.Dir(filename), "..", "..", "migrations") + for _, migrationName := range []string{ + "0001_init.sql", + "0061_oidc_server_sessions.sql", + "0067_identity_configuration_revisions.sql", + "0068_identity_onboarding_exchanges.sql", + "0069_identity_pairing_cancellation.sql", + "0070_identity_secret_cleanup_queue.sql", + "0071_identity_pairing_start_reservation.sql", + "0072_identity_secret_cleanup_claim_lifecycle.sql", + "0073_identity_pairing_start_reservation_upgrade.sql", + "0074_identity_pairing_error_categories.sql", + } { + migration, err := os.ReadFile(filepath.Join(migrationDirectory, migrationName)) + if err != nil { + t.Fatalf("read %s: %v", migrationName, err) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatalf("begin %s: %v", migrationName, err) + } + if _, err := tx.Exec(ctx, string(migration)); err != nil { + _ = tx.Rollback(ctx) + t.Fatalf("execute %s: %v", migrationName, err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatalf("commit %s: %v", migrationName, err) + } + } + return &Store{pool: pool} +} diff --git a/apps/api/internal/store/identity_secret_cleanup_migration_integration_test.go b/apps/api/internal/store/identity_secret_cleanup_migration_integration_test.go new file mode 100644 index 0000000..c750df2 --- /dev/null +++ b/apps/api/internal/store/identity_secret_cleanup_migration_integration_test.go @@ -0,0 +1,206 @@ +package store + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestIdentitySecretCleanupClaimLifecycleUpgradeMigrationExists(t *testing.T) { + payload, err := os.ReadFile(identitySecretCleanupMigrationPath(t, "0072_identity_secret_cleanup_claim_lifecycle.sql")) + if err != nil { + t.Fatal(err) + } + content := strings.ToLower(string(payload)) + for _, required := range []string{ + "add column if not exists status text", + "add column if not exists claim_token uuid", + "add column if not exists lease_expires_at timestamptz", + "alter column status set default 'pending'", + "alter column status set not null", + "gateway_identity_secret_cleanup_status_check", + "gateway_identity_secret_cleanup_claim_check", + "drop index if exists idx_gateway_identity_secret_cleanup_due", + "create index idx_gateway_identity_secret_cleanup_due", + } { + if !strings.Contains(content, required) { + t.Fatalf("identity Secret cleanup lifecycle migration is missing %q", required) + } + } + for _, forbidden := range []string{"secret_value", "client_secret", "machine_secret", "token text"} { + if strings.Contains(content, forbidden) { + t.Fatalf("identity Secret cleanup lifecycle migration stores forbidden value field %q", forbidden) + } + } +} + +func TestIdentitySecretCleanupClaimLifecycleUpgradeMigratesLegacyQueue(t *testing.T) { + db := newIdentitySecretCleanupMigrationPostgresStore(t) + ctx := context.Background() + reference := "identity-cleanup-legacy-" + uuid.NewString() + + if _, err := db.pool.Exec(ctx, ` +CREATE TABLE gateway_identity_secret_cleanup_queue ( + secret_ref text PRIMARY KEY CHECK (secret_ref ~ '^[a-z0-9][a-z0-9-]{0,127}$'), + not_before timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +);`); err != nil { + t.Fatalf("create legacy identity Secret cleanup queue: %v", err) + } + if _, err := db.pool.Exec(ctx, `CREATE INDEX idx_gateway_identity_secret_cleanup_due + ON gateway_identity_secret_cleanup_queue(not_before,updated_at)`); err != nil { + t.Fatalf("create legacy identity Secret cleanup due index: %v", err) + } + if _, err := db.pool.Exec(ctx, `INSERT INTO gateway_identity_secret_cleanup_queue(secret_ref,not_before) +VALUES($1,now()-interval '1 minute')`, reference); err != nil { + t.Fatalf("seed legacy identity Secret cleanup row: %v", err) + } + + applyIdentitySecretCleanupMigration(t, ctx, db.pool, "0072_identity_secret_cleanup_claim_lifecycle.sql") + + var status string + var claimToken, leaseExpiresAt *string + if err := db.pool.QueryRow(ctx, ` +SELECT status,claim_token::text,lease_expires_at::text +FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=$1`, reference). + Scan(&status, &claimToken, &leaseExpiresAt); err != nil { + t.Fatalf("read migrated cleanup row: %v", err) + } + if status != "pending" || claimToken != nil || leaseExpiresAt != nil { + t.Fatalf("legacy cleanup row was not backfilled to an unclaimed pending lifecycle") + } + + claims, err := db.ClaimIdentitySecretCleanups(ctx, 1, time.Minute) + if err != nil { + t.Fatalf("claim migrated cleanup row: %v", err) + } + if len(claims) != 1 || claims[0].Reference != reference || claims[0].ClaimToken == "" { + t.Fatalf("claim migrated cleanup row returned unexpected claim metadata") + } + completed, err := db.CompleteIdentitySecretCleanup(ctx, reference, claims[0].ClaimToken) + if err != nil || !completed { + t.Fatalf("complete migrated cleanup claim: completed=%t err=%v", completed, err) + } + + invalidReference := "identity-cleanup-invalid-" + uuid.NewString() + _, err = db.pool.Exec(ctx, ` +INSERT INTO gateway_identity_secret_cleanup_queue( + secret_ref,not_before,status,claim_token,lease_expires_at +) VALUES($1,now(),'claimed',NULL,NULL)`, invalidReference) + requireIdentitySecretCleanupMigrationCheckViolation(t, err) + + var indexDefinition string + if err := db.pool.QueryRow(ctx, ` +SELECT pg_get_indexdef(indexrelid) +FROM pg_index +WHERE indexrelid='idx_gateway_identity_secret_cleanup_due'::regclass`).Scan(&indexDefinition); err != nil { + t.Fatalf("read upgraded cleanup due index: %v", err) + } + if !strings.Contains(strings.ToLower(indexDefinition), "(status, not_before, lease_expires_at, updated_at)") { + t.Fatalf("cleanup due index does not cover the claim lifecycle") + } + + // The production migration runner applies each version once. Reapplying here + // proves that the upgrade also accepts the already-current 0070 schema shape. + applyIdentitySecretCleanupMigration(t, ctx, db.pool, "0072_identity_secret_cleanup_claim_lifecycle.sql") +} + +func newIdentitySecretCleanupMigrationPostgresStore(t *testing.T) *Store { + t.Helper() + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run identity Secret cleanup migration integration tests") + } + ctx := context.Background() + admin, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect identity Secret cleanup migration test database: %v", err) + } + var databaseName string + if err := admin.QueryRow(ctx, `SELECT current_database()`).Scan(&databaseName); err != nil { + admin.Close() + t.Fatalf("read identity Secret cleanup migration test database name: %v", err) + } + if !strings.Contains(strings.ToLower(databaseName), "test") { + admin.Close() + t.Fatalf("refusing to use non-test database %q", databaseName) + } + + schemaName := "gateway_identity_cleanup_migration_" + strings.ReplaceAll(uuid.NewString(), "-", "") + schemaIdentifier := pgx.Identifier{schemaName}.Sanitize() + if _, err := admin.Exec(ctx, `CREATE SCHEMA `+schemaIdentifier); err != nil { + admin.Close() + t.Fatalf("create identity Secret cleanup migration test schema: %v", err) + } + config, err := pgxpool.ParseConfig(databaseURL) + if err != nil { + _, _ = admin.Exec(ctx, `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`) + admin.Close() + t.Fatalf("parse identity Secret cleanup migration test database URL: %v", err) + } + config.ConnConfig.RuntimeParams["search_path"] = schemaName + pool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + _, _ = admin.Exec(ctx, `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`) + admin.Close() + t.Fatalf("connect identity Secret cleanup migration test schema: %v", err) + } + t.Cleanup(func() { + pool.Close() + if _, err := admin.Exec(context.Background(), `DROP SCHEMA IF EXISTS `+schemaIdentifier+` CASCADE`); err != nil { + t.Errorf("drop identity Secret cleanup migration test schema: %v", err) + } + admin.Close() + }) + return &Store{pool: pool} +} + +func applyIdentitySecretCleanupMigration(t *testing.T, ctx context.Context, pool *pgxpool.Pool, name string) { + t.Helper() + payload, err := os.ReadFile(identitySecretCleanupMigrationPath(t, name)) + if err != nil { + t.Fatalf("read identity Secret cleanup migration: %v", err) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatalf("begin identity Secret cleanup migration: %v", err) + } + if _, err := tx.Exec(ctx, string(payload)); err != nil { + _ = tx.Rollback(ctx) + t.Fatalf("execute identity Secret cleanup migration: %v", err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatalf("commit identity Secret cleanup migration: %v", err) + } +} + +func identitySecretCleanupMigrationPath(t *testing.T, name string) string { + t.Helper() + _, filename, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("locate identity Secret cleanup migration test") + } + return filepath.Join(filepath.Dir(filename), "..", "..", "migrations", name) +} + +func requireIdentitySecretCleanupMigrationCheckViolation(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("invalid cleanup claim lifecycle unexpectedly satisfied migration constraints") + } + var postgresError *pgconn.PgError + if !errors.As(err, &postgresError) || postgresError.Code != "23514" { + t.Fatalf("invalid cleanup claim lifecycle error=%v, want PostgreSQL check violation", err) + } +} diff --git a/apps/api/internal/store/security_event_connections.go b/apps/api/internal/store/security_event_connections.go index e53851e..668cf10 100644 --- a/apps/api/internal/store/security_event_connections.go +++ b/apps/api/internal/store/security_event_connections.go @@ -2,6 +2,7 @@ package store import ( "context" + "database/sql" "encoding/json" "errors" "time" @@ -13,6 +14,7 @@ import ( var ( ErrSecurityEventConnectionNotFound = errors.New("security event connection not found") ErrSecurityEventConnectionConflict = errors.New("security event connection conflicts with current state") + ErrIdentitySecretCleanupConflict = errors.New("identity secret cleanup conflicts with current state") ) type SecurityEventConnection struct { @@ -44,6 +46,121 @@ type SecurityEventConnectionIdempotency struct { Response json.RawMessage } +type IdentitySecretCleanupClaim struct { + Reference string + ClaimToken string + LeaseExpiresAt time.Time +} + +func (s *Store) QueueIdentitySecretCleanup(ctx context.Context, reference string, notBefore time.Time) error { + tag, err := s.pool.Exec(ctx, `INSERT INTO gateway_identity_secret_cleanup_queue(secret_ref,not_before) +VALUES($1,$2) ON CONFLICT(secret_ref) DO UPDATE +SET not_before=LEAST(gateway_identity_secret_cleanup_queue.not_before,EXCLUDED.not_before),updated_at=now() +WHERE gateway_identity_secret_cleanup_queue.status='pending'`, reference, notBefore) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrIdentitySecretCleanupConflict + } + return nil +} + +func (s *Store) RemovePendingIdentitySecretCleanup(ctx context.Context, reference string) (bool, error) { + tag, err := s.pool.Exec(ctx, `DELETE FROM gateway_identity_secret_cleanup_queue +WHERE secret_ref=$1 AND status='pending'`, reference) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func (s *Store) ClaimIdentitySecretCleanups(ctx context.Context, limit int, lease time.Duration) ([]IdentitySecretCleanupClaim, error) { + if limit <= 0 || limit > 1000 { + limit = 100 + } + if lease <= 0 || lease > time.Hour { + lease = 2 * time.Minute + } + leaseSeconds := int64((lease + time.Second - 1) / time.Second) + rows, err := s.pool.Query(ctx, `WITH candidates AS MATERIALIZED ( + SELECT secret_ref FROM gateway_identity_secret_cleanup_queue + WHERE (status='pending' AND not_before <= now()) + OR (status='claimed' AND lease_expires_at <= now()) + ORDER BY COALESCE(lease_expires_at,not_before),created_at + FOR UPDATE SKIP LOCKED + LIMIT $1 +) +UPDATE gateway_identity_secret_cleanup_queue queued +SET status='claimed',claim_token=gen_random_uuid(), + lease_expires_at=now()+make_interval(secs => $2::int),updated_at=now() +FROM candidates +WHERE queued.secret_ref=candidates.secret_ref +RETURNING queued.secret_ref,queued.claim_token::text,queued.lease_expires_at`, limit, leaseSeconds) + if err != nil { + return nil, err + } + defer rows.Close() + claims := make([]IdentitySecretCleanupClaim, 0) + for rows.Next() { + var claim IdentitySecretCleanupClaim + if err := rows.Scan(&claim.Reference, &claim.ClaimToken, &claim.LeaseExpiresAt); err != nil { + return nil, err + } + claims = append(claims, claim) + } + return claims, rows.Err() +} + +func (s *Store) CompleteIdentitySecretCleanup(ctx context.Context, reference, claimToken string) (bool, error) { + if _, err := uuid.Parse(claimToken); err != nil { + return false, ErrIdentitySecretCleanupConflict + } + tag, err := s.pool.Exec(ctx, `DELETE FROM gateway_identity_secret_cleanup_queue +WHERE secret_ref=$1 AND status='claimed' AND claim_token=$2::uuid`, reference, claimToken) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func adoptPendingIdentitySecrets(ctx context.Context, tx pgx.Tx, references ...string) error { + unique := make(map[string]struct{}, len(references)) + for _, reference := range references { + if reference == "" { + return ErrIdentitySecretCleanupConflict + } + unique[reference] = struct{}{} + } + values := make([]string, 0, len(unique)) + for reference := range unique { + values = append(values, reference) + } + tag, err := tx.Exec(ctx, `DELETE FROM gateway_identity_secret_cleanup_queue +WHERE secret_ref=ANY($1) AND status='pending'`, values) + if err != nil { + return err + } + if tag.RowsAffected() != int64(len(values)) { + return ErrIdentitySecretCleanupConflict + } + return nil +} + +func queueIdentitySecretCleanupTx(ctx context.Context, tx pgx.Tx, reference string, notBefore time.Time) error { + tag, err := tx.Exec(ctx, `INSERT INTO gateway_identity_secret_cleanup_queue(secret_ref,not_before) +VALUES($1,$2) ON CONFLICT(secret_ref) DO UPDATE +SET not_before=LEAST(gateway_identity_secret_cleanup_queue.not_before,EXCLUDED.not_before),updated_at=now() +WHERE gateway_identity_secret_cleanup_queue.status='pending'`, reference, notBefore) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrIdentitySecretCleanupConflict + } + return nil +} + func (s *Store) SecurityEventConnectionIdempotency(ctx context.Context, operation, key string) (SecurityEventConnectionIdempotency, error) { var value SecurityEventConnectionIdempotency err := s.pool.QueryRow(ctx, `SELECT request_hash,response FROM gateway_security_event_connection_idempotency @@ -64,20 +181,22 @@ VALUES($1,$2,$3,$4::jsonb) ON CONFLICT(operation,idempotency_key) DO NOTHING`, o } func (s *Store) CreateSecurityEventConnection(ctx context.Context, input CreateSecurityEventConnectionInput) (SecurityEventConnection, error) { - var value SecurityEventConnection - err := s.pool.QueryRow(ctx, ` + tx, err := s.pool.Begin(ctx) + if err != nil { + return SecurityEventConnection{}, err + } + defer tx.Rollback(ctx) + _, err = tx.Exec(ctx, ` INSERT INTO gateway_security_event_connections( connection_id,transmitter_issuer,endpoint_url,credential_ref,management_client_id,management_credential_ref,lifecycle_status,idempotency_key ) VALUES($1::uuid,$2,$3,$4,$5,$6,'connecting',$7) -RETURNING connection_id::text,transmitter_issuer,endpoint_url,audience,stream_id::text,credential_ref, - next_credential_ref,management_client_id,management_credential_ref,lifecycle_status,version,last_error_category,idempotency_key,created_at,updated_at`, + `, input.ConnectionID, input.TransmitterIssuer, input.EndpointURL, input.CredentialRef, input.ManagementClientID, input.ManagementCredentialRef, input.IdempotencyKey, - ).Scan(&value.ConnectionID, &value.TransmitterIssuer, &value.EndpointURL, &value.Audience, &value.StreamID, - &value.CredentialRef, &value.NextCredentialRef, &value.ManagementClientID, &value.ManagementCredentialRef, &value.LifecycleStatus, &value.Version, - &value.LastErrorCategory, &value.IdempotencyKey, &value.CreatedAt, &value.UpdatedAt) + ) if err != nil { if isUniqueViolation(err) { + _ = tx.Rollback(ctx) existing, getErr := s.SecurityEventConnection(ctx) if getErr == nil && existing.IdempotencyKey == input.IdempotencyKey && existing.TransmitterIssuer == input.TransmitterIssuer { return existing, nil @@ -86,7 +205,13 @@ RETURNING connection_id::text,transmitter_issuer,endpoint_url,audience,stream_id } return SecurityEventConnection{}, err } - return value, nil + if err := adoptPendingIdentitySecrets(ctx, tx, input.CredentialRef, input.ManagementCredentialRef); err != nil { + return SecurityEventConnection{}, err + } + if err := tx.Commit(ctx); err != nil { + return SecurityEventConnection{}, err + } + return s.SecurityEventConnection(ctx) } func (s *Store) SecurityEventConnection(ctx context.Context) (SecurityEventConnection, error) { @@ -126,49 +251,56 @@ WHERE connection_id=$1::uuid AND version=$5`, connectionID, audience, streamID, return s.SecurityEventConnection(ctx) } -func (s *Store) SetSecurityEventManagementCredential(ctx context.Context, connectionID, clientID, reference string) (SecurityEventConnection, error) { - tag, err := s.pool.Exec(ctx, `UPDATE gateway_security_event_connections -SET management_client_id=$2,management_credential_ref=$3,last_error_category=NULL,updated_at=now() -WHERE connection_id=$1::uuid`, connectionID, clientID, reference) +func (s *Store) SetSecurityEventManagementCredential(ctx context.Context, connectionID, clientID, reference string, expectedVersion int64) (SecurityEventConnection, error) { + tx, err := s.pool.Begin(ctx) if err != nil { return SecurityEventConnection{}, err } - if tag.RowsAffected() == 0 { + defer tx.Rollback(ctx) + var oldReference sql.NullString + var lifecycle string + var version int64 + if err := tx.QueryRow(ctx, `SELECT management_credential_ref,lifecycle_status,version FROM gateway_security_event_connections +WHERE connection_id=$1::uuid FOR UPDATE`, connectionID).Scan(&oldReference, &lifecycle, &version); errors.Is(err, pgx.ErrNoRows) { return SecurityEventConnection{}, ErrSecurityEventConnectionNotFound - } - return s.SecurityEventConnection(ctx) -} - -func (s *Store) UpdateSecurityEventConnectionLifecycle(ctx context.Context, connectionID, lifecycle string, errorCategory *string) (SecurityEventConnection, error) { - tag, err := s.pool.Exec(ctx, `UPDATE gateway_security_event_connections -SET lifecycle_status=$2,last_error_category=$3,version=version+1,updated_at=now() WHERE connection_id=$1::uuid`, connectionID, lifecycle, errorCategory) - if err != nil { + } else if err != nil { return SecurityEventConnection{}, err } - if tag.RowsAffected() == 0 { - return SecurityEventConnection{}, ErrSecurityEventConnectionNotFound + if version != expectedVersion || lifecycle == "retiring" || lifecycle == "disconnect_pending" { + return SecurityEventConnection{}, ErrSecurityEventConnectionConflict } - return s.SecurityEventConnection(ctx) -} - -func (s *Store) SetSecurityEventNextCredential(ctx context.Context, connectionID, reference string) (SecurityEventConnection, error) { - tag, err := s.pool.Exec(ctx, `UPDATE gateway_security_event_connections -SET next_credential_ref=$2,lifecycle_status='rotating',last_error_category=NULL,version=version+1,updated_at=now() -WHERE connection_id=$1::uuid`, connectionID, reference) - if err != nil { - return SecurityEventConnection{}, err - } - if tag.RowsAffected() == 0 { - return SecurityEventConnection{}, ErrSecurityEventConnectionNotFound - } - return s.SecurityEventConnection(ctx) -} - -func (s *Store) PromoteSecurityEventCredential(ctx context.Context, connectionID, nextReference string) (SecurityEventConnection, error) { - tag, err := s.pool.Exec(ctx, `UPDATE gateway_security_event_connections -SET credential_ref=$2,next_credential_ref=NULL,lifecycle_status='bootstrap',last_error_category=NULL, + tag, err := tx.Exec(ctx, `UPDATE gateway_security_event_connections +SET management_client_id=$2,management_credential_ref=$3,last_error_category=NULL, version=version+1,updated_at=now() -WHERE connection_id=$1::uuid AND next_credential_ref=$2`, connectionID, nextReference) +WHERE connection_id=$1::uuid AND version=$4 + AND lifecycle_status NOT IN ('retiring','disconnect_pending')`, connectionID, clientID, reference, expectedVersion) + if err != nil { + return SecurityEventConnection{}, err + } + if tag.RowsAffected() == 0 { + return SecurityEventConnection{}, ErrSecurityEventConnectionConflict + } + if err := adoptPendingIdentitySecrets(ctx, tx, reference); err != nil { + return SecurityEventConnection{}, err + } + if oldReference.Valid && oldReference.String != reference { + if err := queueIdentitySecretCleanupTx(ctx, tx, oldReference.String, time.Now().UTC()); err != nil { + return SecurityEventConnection{}, err + } + } + if err := tx.Commit(ctx); err != nil { + return SecurityEventConnection{}, err + } + return s.SecurityEventConnection(ctx) +} + +func (s *Store) TransitionSecurityEventConnectionLifecycle(ctx context.Context, connectionID, expectedLifecycle string, expectedVersion int64, lifecycle string, errorCategory *string) (SecurityEventConnection, error) { + tag, err := s.pool.Exec(ctx, `UPDATE gateway_security_event_connections +SET lifecycle_status=$4,last_error_category=$5,version=version+1,updated_at=now() +WHERE connection_id=$1::uuid AND lifecycle_status=$2 AND version=$3 + AND (lifecycle_status NOT IN ('retiring','disconnect_pending') + OR $4 IN ('retiring','disconnect_pending'))`, + connectionID, expectedLifecycle, expectedVersion, lifecycle, errorCategory) if err != nil { return SecurityEventConnection{}, err } @@ -178,16 +310,168 @@ WHERE connection_id=$1::uuid AND next_credential_ref=$2`, connectionID, nextRefe return s.SecurityEventConnection(ctx) } -func (s *Store) DeleteSecurityEventConnection(ctx context.Context, connectionID string) error { - if _, err := uuid.Parse(connectionID); err != nil { - return ErrSecurityEventConnectionNotFound +func (s *Store) SetSecurityEventNextCredential(ctx context.Context, connectionID, reference string, expectedVersion int64) (SecurityEventConnection, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return SecurityEventConnection{}, err } - tag, err := s.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + defer tx.Rollback(ctx) + tag, err := tx.Exec(ctx, `UPDATE gateway_security_event_connections +SET next_credential_ref=$2,lifecycle_status='rotating',last_error_category=NULL,version=version+1,updated_at=now() +WHERE connection_id=$1::uuid AND version=$3 AND next_credential_ref IS NULL + AND lifecycle_status IN ('connecting','verifying','bootstrap','enabled','degraded','error')`, connectionID, reference, expectedVersion) + if err != nil { + return SecurityEventConnection{}, err + } + if tag.RowsAffected() == 0 { + return SecurityEventConnection{}, ErrSecurityEventConnectionConflict + } + if err := adoptPendingIdentitySecrets(ctx, tx, reference); err != nil { + return SecurityEventConnection{}, err + } + if err := tx.Commit(ctx); err != nil { + return SecurityEventConnection{}, err + } + return s.SecurityEventConnection(ctx) +} + +func (s *Store) PromoteSecurityEventCredential(ctx context.Context, connectionID, nextReference string) (SecurityEventConnection, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return SecurityEventConnection{}, err + } + defer tx.Rollback(ctx) + var oldReference string + if err := tx.QueryRow(ctx, `SELECT credential_ref FROM gateway_security_event_connections +WHERE connection_id=$1::uuid AND next_credential_ref=$2 AND lifecycle_status='rotating' FOR UPDATE`, connectionID, nextReference).Scan(&oldReference); errors.Is(err, pgx.ErrNoRows) { + return SecurityEventConnection{}, ErrSecurityEventConnectionConflict + } else if err != nil { + return SecurityEventConnection{}, err + } + tag, err := tx.Exec(ctx, `UPDATE gateway_security_event_connections +SET credential_ref=$2,next_credential_ref=NULL,lifecycle_status='bootstrap',last_error_category=NULL, + version=version+1,updated_at=now() +WHERE connection_id=$1::uuid AND next_credential_ref=$2 AND lifecycle_status='rotating'`, connectionID, nextReference) + if err != nil { + return SecurityEventConnection{}, err + } + if tag.RowsAffected() == 0 { + return SecurityEventConnection{}, ErrSecurityEventConnectionConflict + } + if oldReference != nextReference { + if err := queueIdentitySecretCleanupTx(ctx, tx, oldReference, time.Now().UTC()); err != nil { + return SecurityEventConnection{}, err + } + } + if err := tx.Commit(ctx); err != nil { + return SecurityEventConnection{}, err + } + return s.SecurityEventConnection(ctx) +} + +// DiscardPreparedSecurityEventConnection atomically retires the local Secrets +// and removes only the exact, still-unbound connection generation observed by +// the caller. A concurrent Bind or credential update changes the version and +// therefore preserves both the connection and its Secrets. +func (s *Store) DiscardPreparedSecurityEventConnection(ctx context.Context, connectionID, ownerKey string, expectedVersion int64) error { + if _, err := uuid.Parse(connectionID); err != nil || ownerKey == "" || expectedVersion <= 0 { + return ErrSecurityEventConnectionConflict + } + tx, err := s.pool.Begin(ctx) if err != nil { return err } - if tag.RowsAffected() == 0 { - return ErrSecurityEventConnectionNotFound + defer tx.Rollback(ctx) + + var credentialReference string + var nextReference, managementReference sql.NullString + err = tx.QueryRow(ctx, `SELECT credential_ref,next_credential_ref,management_credential_ref +FROM gateway_security_event_connections +WHERE connection_id=$1::uuid AND idempotency_key=$2 AND version=$3 AND stream_id IS NULL +FOR UPDATE`, connectionID, ownerKey, expectedVersion).Scan( + &credentialReference, &nextReference, &managementReference, + ) + if errors.Is(err, pgx.ErrNoRows) { + return ErrSecurityEventConnectionConflict } - return nil + if err != nil { + return err + } + + references := map[string]struct{}{credentialReference: {}} + if nextReference.Valid { + references[nextReference.String] = struct{}{} + } + if managementReference.Valid { + references[managementReference.String] = struct{}{} + } + now := time.Now().UTC() + for reference := range references { + if err := queueIdentitySecretCleanupTx(ctx, tx, reference, now); err != nil { + return err + } + } + tag, err := tx.Exec(ctx, `DELETE FROM gateway_security_event_connections +WHERE connection_id=$1::uuid AND idempotency_key=$2 AND version=$3 AND stream_id IS NULL`, + connectionID, ownerKey, expectedVersion) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrSecurityEventConnectionConflict + } + return tx.Commit(ctx) +} + +// FinalizeRetiringSecurityEventConnection atomically transfers every Secret +// reference owned by the exact retiring generation to the durable cleanup +// queue before removing that generation. The SecretStore worker can therefore +// resume after a crash without the database losing its last cleanup intent. +func (s *Store) FinalizeRetiringSecurityEventConnection(ctx context.Context, connectionID string, expectedVersion int64) error { + if _, err := uuid.Parse(connectionID); err != nil || expectedVersion <= 0 { + return ErrSecurityEventConnectionConflict + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + var credentialReference string + var nextReference, managementReference sql.NullString + err = tx.QueryRow(ctx, `SELECT credential_ref,next_credential_ref,management_credential_ref +FROM gateway_security_event_connections +WHERE connection_id=$1::uuid AND lifecycle_status='retiring' AND version=$2 +FOR UPDATE`, connectionID, expectedVersion).Scan( + &credentialReference, &nextReference, &managementReference, + ) + if errors.Is(err, pgx.ErrNoRows) { + return ErrSecurityEventConnectionConflict + } + if err != nil { + return err + } + + references := map[string]struct{}{credentialReference: {}} + if nextReference.Valid { + references[nextReference.String] = struct{}{} + } + if managementReference.Valid { + references[managementReference.String] = struct{}{} + } + now := time.Now().UTC() + for reference := range references { + if err := queueIdentitySecretCleanupTx(ctx, tx, reference, now); err != nil { + return err + } + } + tag, err := tx.Exec(ctx, `DELETE FROM gateway_security_event_connections +WHERE connection_id=$1::uuid AND lifecycle_status='retiring' AND version=$2`, connectionID, expectedVersion) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrSecurityEventConnectionConflict + } + return tx.Commit(ctx) } diff --git a/apps/api/internal/store/security_event_secret_cleanup_integration_test.go b/apps/api/internal/store/security_event_secret_cleanup_integration_test.go new file mode 100644 index 0000000..f7278d6 --- /dev/null +++ b/apps/api/internal/store/security_event_secret_cleanup_integration_test.go @@ -0,0 +1,665 @@ +package store + +import ( + "context" + "errors" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestSecurityEventCredentialReferenceSwitchesRemainCrashRecoverable(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + + connectionID := uuid.NewString() + pushReference := "ssf-push-" + connectionID + managementReference := "ssf-management-" + connectionID + nextReference := "ssf-push-next-" + uuid.NewString() + newManagementReference := "ssf-management-next-" + uuid.NewString() + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, + []string{pushReference, managementReference, nextReference, newManagementReference}) + }) + if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections`); err != nil { + t.Fatal(err) + } + + for _, reference := range []string{pushReference, managementReference} { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + connection, err := db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: "identity-pairing-ssf-" + uuid.NewString(), + }) + if err != nil { + t.Fatal(err) + } + assertIdentitySecretQueueState(t, ctx, db, pushReference, false) + assertIdentitySecretQueueState(t, ctx, db, managementReference, false) + + if err := db.QueueIdentitySecretCleanup(ctx, newManagementReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + if _, err := db.SetSecurityEventManagementCredential(ctx, connectionID, "machine-client", newManagementReference, connection.Version); err != nil { + t.Fatal(err) + } + assertIdentitySecretQueueState(t, ctx, db, newManagementReference, false) + assertIdentitySecretQueueState(t, ctx, db, managementReference, true) + + if err := db.QueueIdentitySecretCleanup(ctx, nextReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + connection, err = db.SecurityEventConnection(ctx) + if err != nil { + t.Fatal(err) + } + connection, err = db.SetSecurityEventNextCredential(ctx, connectionID, nextReference, connection.Version) + if err != nil { + t.Fatal(err) + } + assertIdentitySecretQueueState(t, ctx, db, nextReference, false) + if _, err := db.PromoteSecurityEventCredential(ctx, connection.ConnectionID, nextReference); err != nil { + t.Fatal(err) + } + assertIdentitySecretQueueState(t, ctx, db, pushReference, true) +} + +func TestDiscardPreparedSecurityEventConnectionQueuesAllSecretsAtomically(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + connectionID := uuid.NewString() + ownerKey := "identity-pairing-ssf-" + uuid.NewString() + pushReference := "ssf-push-" + connectionID + managementReference := "ssf-management-" + connectionID + nextReference := "ssf-push-next-" + uuid.NewString() + references := []string{pushReference, managementReference, nextReference} + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, references) + }) + if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections`); err != nil { + t.Fatal(err) + } + for _, reference := range []string{pushReference, managementReference} { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + connection, err := db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: ownerKey, + }) + if err != nil { + t.Fatal(err) + } + if err := db.QueueIdentitySecretCleanup(ctx, nextReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + connection, err = db.SetSecurityEventNextCredential(ctx, connectionID, nextReference, connection.Version) + if err != nil { + t.Fatal(err) + } + + if err := db.DiscardPreparedSecurityEventConnection(ctx, connectionID, ownerKey, connection.Version); err != nil { + t.Fatal(err) + } + if _, err := db.SecurityEventConnection(ctx); !errors.Is(err, ErrSecurityEventConnectionNotFound) { + t.Fatalf("discarded prepared connection still exists: %v", err) + } + for _, reference := range references { + assertIdentitySecretQueueStatus(t, ctx, db, reference, "pending") + } +} + +func TestConcurrentPreparedDiscardAndBindPreserveTheWinningGeneration(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + connectionID := uuid.NewString() + ownerKey := "identity-pairing-ssf-" + uuid.NewString() + pushReference := "ssf-push-" + connectionID + managementReference := "ssf-management-" + connectionID + references := []string{pushReference, managementReference} + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, references) + }) + if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections`); err != nil { + t.Fatal(err) + } + for _, reference := range references { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + connection, err := db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: ownerKey, + }) + if err != nil { + t.Fatal(err) + } + + type result struct { + operation string + err error + } + start := make(chan struct{}) + results := make(chan result, 2) + audience := "urn:easyai:ssf:receiver:" + uuid.NewString() + streamID := uuid.NewString() + go func() { + <-start + _, err := db.BindSecurityEventConnection(ctx, connectionID, audience, streamID, "verifying", connection.Version) + results <- result{operation: "bind", err: err} + }() + go func() { + <-start + err := db.DiscardPreparedSecurityEventConnection(ctx, connectionID, ownerKey, connection.Version) + results <- result{operation: "discard", err: err} + }() + close(start) + + outcomes := map[string]error{} + for range 2 { + outcome := <-results + outcomes[outcome.operation] = outcome.err + } + bindWon := outcomes["bind"] == nil + discardWon := outcomes["discard"] == nil + if bindWon == discardWon { + t.Fatalf("concurrent outcomes bind=%v discard=%v, want exactly one winner", outcomes["bind"], outcomes["discard"]) + } + loser := outcomes["bind"] + if bindWon { + loser = outcomes["discard"] + } + if !errors.Is(loser, ErrSecurityEventConnectionConflict) { + t.Fatalf("concurrent loser error=%v, want connection conflict", loser) + } + if bindWon { + persisted, err := db.SecurityEventConnection(ctx) + if err != nil || persisted.StreamID == nil || *persisted.StreamID != streamID { + t.Fatalf("winning bound generation was not preserved: connection=%+v err=%v", persisted, err) + } + for _, reference := range references { + assertIdentitySecretQueueState(t, ctx, db, reference, false) + } + return + } + if _, err := db.SecurityEventConnection(ctx); !errors.Is(err, ErrSecurityEventConnectionNotFound) { + t.Fatalf("winning discard left a connection behind: %v", err) + } + for _, reference := range references { + assertIdentitySecretQueueStatus(t, ctx, db, reference, "pending") + } +} + +func TestConcurrentSecurityEventNextCredentialsAdoptOnlyOneStagedSecret(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + connectionID := uuid.NewString() + pushReference := "ssf-push-" + connectionID + managementReference := "ssf-management-" + connectionID + candidates := []string{ + "ssf-push-next-" + uuid.NewString(), + "ssf-push-next-" + uuid.NewString(), + } + allReferences := append([]string{pushReference, managementReference}, candidates...) + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, allReferences) + }) + if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections`); err != nil { + t.Fatal(err) + } + for _, reference := range []string{pushReference, managementReference} { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + connection, err := db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: "identity-pairing-ssf-" + uuid.NewString(), + }) + if err != nil { + t.Fatal(err) + } + for _, reference := range candidates { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + + type result struct { + reference string + err error + } + start := make(chan struct{}) + results := make(chan result, len(candidates)) + var workers sync.WaitGroup + for _, reference := range candidates { + workers.Add(1) + go func(reference string) { + defer workers.Done() + <-start + _, err := db.SetSecurityEventNextCredential(ctx, connectionID, reference, connection.Version) + results <- result{reference: reference, err: err} + }(reference) + } + close(start) + workers.Wait() + close(results) + + var adopted, pending string + for result := range results { + switch { + case result.err == nil: + if adopted != "" { + t.Fatal("more than one concurrent next credential was adopted") + } + adopted = result.reference + case errors.Is(result.err, ErrSecurityEventConnectionConflict): + pending = result.reference + default: + t.Fatalf("concurrent next credential returned unexpected error: %v", result.err) + } + } + if adopted == "" || pending == "" { + t.Fatalf("concurrent next credential outcomes adopted=%t pending=%t", adopted != "", pending != "") + } + connection, err = db.SecurityEventConnection(ctx) + if err != nil || connection.NextCredentialRef == nil || *connection.NextCredentialRef != adopted { + t.Fatalf("connection did not retain the sole adopted next credential: %v", err) + } + assertIdentitySecretQueueState(t, ctx, db, adopted, false) + assertIdentitySecretQueueStatus(t, ctx, db, pending, "pending") +} + +func TestRetiringSecurityEventConnectionCannotAdoptManagementSecret(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + connectionID := uuid.NewString() + pushReference := "ssf-push-" + connectionID + managementReference := "ssf-management-" + connectionID + newManagementReference := "ssf-management-next-" + uuid.NewString() + allReferences := []string{pushReference, managementReference, newManagementReference} + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, allReferences) + }) + if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections`); err != nil { + t.Fatal(err) + } + for _, reference := range []string{pushReference, managementReference} { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + connection, err := db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: "identity-pairing-ssf-" + uuid.NewString(), + }) + if err != nil { + t.Fatal(err) + } + connection, err = db.TransitionSecurityEventConnectionLifecycle( + ctx, connectionID, connection.LifecycleStatus, connection.Version, "retiring", nil, + ) + if err != nil { + t.Fatal(err) + } + if err := db.QueueIdentitySecretCleanup(ctx, newManagementReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + + if _, err := db.SetSecurityEventManagementCredential(ctx, connectionID, "machine-client", newManagementReference, connection.Version); !errors.Is(err, ErrSecurityEventConnectionConflict) { + t.Fatalf("retiring connection management adoption error=%v, want conflict", err) + } + connection, err = db.SecurityEventConnection(ctx) + if err != nil || connection.ManagementCredentialRef == nil || *connection.ManagementCredentialRef != managementReference { + t.Fatalf("retiring connection changed its management credential: %v", err) + } + assertIdentitySecretQueueStatus(t, ctx, db, newManagementReference, "pending") +} + +func TestSecurityEventRetirementRequiresTheCurrentConnectionGeneration(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + connectionID := uuid.NewString() + pushReference := "ssf-push-" + connectionID + managementReference := "ssf-management-" + connectionID + references := []string{pushReference, managementReference} + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, references) + }) + if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections`); err != nil { + t.Fatal(err) + } + for _, reference := range references { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + connection, err := db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: "identity-pairing-ssf-" + uuid.NewString(), + }) + if err != nil { + t.Fatal(err) + } + + if _, err := db.TransitionSecurityEventConnectionLifecycle( + ctx, connectionID, "connecting", connection.Version-1, "retiring", nil, + ); !errors.Is(err, ErrSecurityEventConnectionConflict) { + t.Fatalf("stale retirement transition error=%v, want conflict", err) + } + current, err := db.SecurityEventConnection(ctx) + if err != nil || current.LifecycleStatus != "connecting" || current.Version != connection.Version { + t.Fatalf("stale transition changed connection generation: lifecycle=%q version=%d err=%v", + current.LifecycleStatus, current.Version, err) + } + current, err = db.TransitionSecurityEventConnectionLifecycle( + ctx, connectionID, "connecting", connection.Version, "retiring", nil, + ) + if err != nil { + t.Fatal(err) + } + if _, err := db.TransitionSecurityEventConnectionLifecycle( + ctx, connectionID, "retiring", current.Version, "error", nil, + ); !errors.Is(err, ErrSecurityEventConnectionConflict) { + t.Fatalf("retiring generation revival error=%v, want conflict", err) + } + if err := db.FinalizeRetiringSecurityEventConnection(ctx, connectionID, connection.Version); !errors.Is(err, ErrSecurityEventConnectionConflict) { + t.Fatalf("stale retirement deletion error=%v, want conflict", err) + } + if persisted, err := db.SecurityEventConnection(ctx); err != nil || persisted.LifecycleStatus != "retiring" || persisted.Version != current.Version { + t.Fatalf("stale retirement deletion removed current generation: lifecycle=%q version=%d err=%v", + persisted.LifecycleStatus, persisted.Version, err) + } + if err := db.FinalizeRetiringSecurityEventConnection(ctx, connectionID, current.Version); err != nil { + t.Fatal(err) + } + if _, err := db.SecurityEventConnection(ctx); !errors.Is(err, ErrSecurityEventConnectionNotFound) { + t.Fatalf("current retiring generation was not deleted: %v", err) + } + for _, reference := range references { + assertIdentitySecretQueueStatus(t, ctx, db, reference, "pending") + } +} + +func TestConcurrentSecurityEventRetirementFinalizationQueuesOneGenerationAtomically(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + connectionID := uuid.NewString() + pushReference := "ssf-push-retire-worker-" + connectionID + managementReference := "ssf-management-retire-worker-" + connectionID + nextReference := "ssf-push-next-retire-worker-" + uuid.NewString() + references := []string{pushReference, managementReference, nextReference} + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, references) + }) + if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_security_event_connections`); err != nil { + t.Fatal(err) + } + for _, reference := range []string{pushReference, managementReference} { + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + } + connection, err := db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: "identity-pairing-ssf-" + uuid.NewString(), + }) + if err != nil { + t.Fatal(err) + } + if err := db.QueueIdentitySecretCleanup(ctx, nextReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + connection, err = db.SetSecurityEventNextCredential(ctx, connectionID, nextReference, connection.Version) + if err != nil { + t.Fatal(err) + } + connection, err = db.TransitionSecurityEventConnectionLifecycle( + ctx, connectionID, connection.LifecycleStatus, connection.Version, "retiring", nil, + ) + if err != nil { + t.Fatal(err) + } + + start := make(chan struct{}) + results := make(chan error, 2) + var workers sync.WaitGroup + for range 2 { + workers.Add(1) + go func() { + defer workers.Done() + <-start + results <- db.FinalizeRetiringSecurityEventConnection(ctx, connectionID, connection.Version) + }() + } + close(start) + workers.Wait() + close(results) + succeeded, conflicted := 0, 0 + for err := range results { + switch { + case err == nil: + succeeded++ + case errors.Is(err, ErrSecurityEventConnectionConflict): + conflicted++ + default: + t.Fatalf("unexpected concurrent finalization error: %v", err) + } + } + if succeeded != 1 || conflicted != 1 { + t.Fatalf("concurrent finalization succeeded=%d conflicted=%d, want 1/1", succeeded, conflicted) + } + if _, err := db.SecurityEventConnection(ctx); !errors.Is(err, ErrSecurityEventConnectionNotFound) { + t.Fatalf("finalized connection still exists: %v", err) + } + for _, reference := range references { + assertIdentitySecretQueueStatus(t, ctx, db, reference, "pending") + } +} + +func TestClaimedStagingSecretsCannotBeAdoptedByAConnection(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + connectionID := uuid.NewString() + pushReference := "ssf-push-claimed-" + uuid.NewString() + managementReference := "ssf-management-claimed-" + uuid.NewString() + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_security_event_connections WHERE connection_id=$1::uuid`, connectionID) + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, + []string{pushReference, managementReference}) + }) + + if err := db.QueueIdentitySecretCleanup(ctx, pushReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatal(err) + } + if err := db.QueueIdentitySecretCleanup(ctx, managementReference, time.Now().Add(-time.Minute)); err != nil { + t.Fatal(err) + } + claims, err := db.ClaimIdentitySecretCleanups(ctx, 1, time.Minute) + if err != nil || len(claims) != 1 || claims[0].Reference != managementReference { + t.Fatalf("claim staging management Secret: count=%d reference_matches=%t err=%v", + len(claims), len(claims) == 1 && claims[0].Reference == managementReference, err) + } + + _, err = db.CreateSecurityEventConnection(ctx, CreateSecurityEventConnectionInput{ + ConnectionID: connectionID, TransmitterIssuer: "https://auth.test.example/ssf", + EndpointURL: "https://gateway.test.example/api/v1/security-events/ssf", CredentialRef: pushReference, + ManagementClientID: "machine-client", ManagementCredentialRef: managementReference, + IdempotencyKey: "identity-pairing-ssf-" + uuid.NewString(), + }) + if !errors.Is(err, ErrIdentitySecretCleanupConflict) { + t.Fatalf("claimed Secret adoption error=%v, want cleanup conflict", err) + } + if _, err := db.SecurityEventConnection(ctx); !errors.Is(err, ErrSecurityEventConnectionNotFound) { + t.Fatalf("failed adoption committed connection: %v", err) + } + assertIdentitySecretQueueStatus(t, ctx, db, pushReference, "pending") + assertIdentitySecretQueueStatus(t, ctx, db, managementReference, "claimed") +} + +func TestIdentitySecretCleanupCompletionRejectsExpiredClaimABA(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + reference := "ssf-management-aba-" + uuid.NewString() + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=$1`, reference) + }) + if err := db.QueueIdentitySecretCleanup(ctx, reference, time.Now().Add(-time.Minute)); err != nil { + t.Fatal(err) + } + first, err := db.ClaimIdentitySecretCleanups(ctx, 1, time.Minute) + if err != nil || len(first) != 1 { + t.Fatalf("first claim count=%d err=%v", len(first), err) + } + if _, err := db.pool.Exec(ctx, `UPDATE gateway_identity_secret_cleanup_queue SET lease_expires_at=now()-interval '1 second' WHERE secret_ref=$1`, reference); err != nil { + t.Fatal(err) + } + second, err := db.ClaimIdentitySecretCleanups(ctx, 1, time.Minute) + if err != nil || len(second) != 1 || second[0].ClaimToken == first[0].ClaimToken { + t.Fatalf("second claim count=%d token_rotated=%t err=%v", + len(second), len(second) == 1 && second[0].ClaimToken != first[0].ClaimToken, err) + } + completed, err := db.CompleteIdentitySecretCleanup(ctx, reference, first[0].ClaimToken) + if err != nil || completed { + t.Fatalf("stale completion completed=%t err=%v", completed, err) + } + assertIdentitySecretQueueStatus(t, ctx, db, reference, "claimed") + completed, err = db.CompleteIdentitySecretCleanup(ctx, reference, second[0].ClaimToken) + if err != nil || !completed { + t.Fatalf("current completion completed=%t err=%v", completed, err) + } +} + +func TestConcurrentIdentitySecretCleanupClaimsAreDisjoint(t *testing.T) { + db := newSecurityEventSecretCleanupPostgresStore(t) + ctx := context.Background() + references := make([]string, 20) + for index := range references { + references[index] = "ssf-cleanup-concurrent-" + uuid.NewString() + if err := db.QueueIdentitySecretCleanup(ctx, references[index], time.Now().Add(-time.Minute)); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=ANY($1)`, references) + }) + + start := make(chan struct{}) + results := make(chan []IdentitySecretCleanupClaim, 2) + errorsCh := make(chan error, 2) + var workers sync.WaitGroup + for range 2 { + workers.Add(1) + go func() { + defer workers.Done() + <-start + claims, err := db.ClaimIdentitySecretCleanups(ctx, len(references), time.Minute) + results <- claims + errorsCh <- err + }() + } + close(start) + workers.Wait() + close(results) + close(errorsCh) + for err := range errorsCh { + if err != nil { + t.Fatal(err) + } + } + claimed := map[string]string{} + for claims := range results { + for _, claim := range claims { + if previous := claimed[claim.Reference]; previous != "" { + t.Fatalf("Secret %q was claimed by more than one worker", claim.Reference) + } + claimed[claim.Reference] = claim.ClaimToken + } + } + if len(claimed) != len(references) { + t.Fatalf("claimed %d Secrets, want %d", len(claimed), len(references)) + } +} + +func newSecurityEventSecretCleanupPostgresStore(t *testing.T) *Store { + t.Helper() + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run identity Secret cleanup PostgreSQL integration tests") + } + ctx := context.Background() + probe, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + var databaseName string + if err := probe.QueryRow(ctx, `SELECT current_database()`).Scan(&databaseName); err != nil { + probe.Close() + t.Fatal(err) + } + probe.Close() + if !strings.Contains(strings.ToLower(databaseName), "test") { + t.Fatalf("refusing to migrate non-test database %q", databaseName) + } + applyOIDCJITTestMigrations(t, ctx, databaseURL) + db, err := Connect(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(db.Close) + return db +} + +func assertIdentitySecretQueueState(t *testing.T, ctx context.Context, db *Store, reference string, expected bool) { + t.Helper() + var exists bool + if err := db.pool.QueryRow(ctx, `SELECT EXISTS( +SELECT 1 FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=$1)`, reference).Scan(&exists); err != nil { + t.Fatal(err) + } + if exists != expected { + t.Fatalf("Secret cleanup reference %q exists=%t want=%t", reference, exists, expected) + } +} + +func assertIdentitySecretQueueStatus(t *testing.T, ctx context.Context, db *Store, reference, expected string) { + t.Helper() + var status string + if err := db.pool.QueryRow(ctx, `SELECT status FROM gateway_identity_secret_cleanup_queue WHERE secret_ref=$1`, reference).Scan(&status); err != nil { + t.Fatal(err) + } + if status != expected { + t.Fatalf("Secret cleanup reference %q status=%q want=%q", reference, status, expected) + } +} diff --git a/apps/api/migrations/0069_identity_pairing_cancellation.sql b/apps/api/migrations/0069_identity_pairing_cancellation.sql new file mode 100644 index 0000000..dcd691c --- /dev/null +++ b/apps/api/migrations/0069_identity_pairing_cancellation.sql @@ -0,0 +1,69 @@ +SET LOCAL lock_timeout = '10s'; +SET LOCAL statement_timeout = '60s'; + +ALTER TABLE gateway_identity_onboarding_exchanges + DROP CONSTRAINT gateway_identity_onboarding_exchanges_status_check; + +ALTER TABLE gateway_identity_onboarding_exchanges + ADD CONSTRAINT gateway_identity_onboarding_exchanges_status_check + CHECK (status IN ('metadata_pending','preparing','ready','credentials_saved','completed','failed','expired','cancelled')), + ADD COLUMN cleanup_status text NOT NULL DEFAULT 'none', + ADD COLUMN cancelled_at timestamptz, + ADD COLUMN cleanup_completed_at timestamptz; + +ALTER TABLE gateway_identity_onboarding_exchanges + ADD CONSTRAINT gateway_identity_onboarding_cleanup_status_check + CHECK (cleanup_status IN ('none','pending','completed')), + ADD CONSTRAINT gateway_identity_onboarding_cleanup_lifecycle_check + CHECK ( + (status <> 'cancelled' AND cleanup_status = 'none' AND cancelled_at IS NULL AND cleanup_completed_at IS NULL) OR + (status = 'cancelled' AND cancelled_at IS NOT NULL AND ( + (cleanup_status = 'pending' AND cleanup_completed_at IS NULL) OR + (cleanup_status = 'completed' AND cleanup_completed_at IS NOT NULL) + )) + ); + +UPDATE gateway_identity_onboarding_exchanges +SET last_error_category = 'pairing_step_failed' +WHERE last_error_category IS NOT NULL + AND last_error_category NOT IN ( + 'metadata_submission_failed','exchange_status_unavailable','credential_delivery_failed', + 'exchange_completion_failed','exchange_expired','remote_exchange_failed','remote_state_invalid', + 'security_event_configuration_invalid','security_event_connection_conflict', + 'security_event_discovery_failed','security_event_management_token_failed', + 'security_event_stream_create_failed','security_event_stream_response_invalid', + 'security_event_receiver_activation_failed','security_event_preparation_failed', + 'security_event_retirement_pending','pairing_step_failed', + 'cleanup_revision_unavailable','cleanup_security_event_unavailable', + 'cleanup_security_event_configuration_invalid','cleanup_security_event_connection_conflict', + 'cleanup_security_event_discovery_failed','cleanup_security_event_management_token_failed', + 'cleanup_security_event_stream_create_failed','cleanup_security_event_stream_response_invalid', + 'cleanup_security_event_receiver_activation_failed','cleanup_security_event_preparation_failed', + 'cleanup_security_event_retirement_pending','cleanup_security_event_connection_cleanup_failed', + 'cleanup_security_event_secret_cleanup_failed','cleanup_security_event_failed', + 'cleanup_secret_store_failed','cleanup_finalize_failed' + ); + +ALTER TABLE gateway_identity_onboarding_exchanges + ADD CONSTRAINT gateway_identity_onboarding_error_category_check + CHECK (last_error_category IS NULL OR last_error_category IN ( + 'metadata_submission_failed','exchange_status_unavailable','credential_delivery_failed', + 'exchange_completion_failed','exchange_expired','remote_exchange_failed','remote_state_invalid', + 'security_event_configuration_invalid','security_event_connection_conflict', + 'security_event_discovery_failed','security_event_management_token_failed', + 'security_event_stream_create_failed','security_event_stream_response_invalid', + 'security_event_receiver_activation_failed','security_event_preparation_failed', + 'security_event_retirement_pending','pairing_step_failed', + 'cleanup_revision_unavailable','cleanup_security_event_unavailable', + 'cleanup_security_event_configuration_invalid','cleanup_security_event_connection_conflict', + 'cleanup_security_event_discovery_failed','cleanup_security_event_management_token_failed', + 'cleanup_security_event_stream_create_failed','cleanup_security_event_stream_response_invalid', + 'cleanup_security_event_receiver_activation_failed','cleanup_security_event_preparation_failed', + 'cleanup_security_event_retirement_pending','cleanup_security_event_connection_cleanup_failed', + 'cleanup_security_event_secret_cleanup_failed','cleanup_security_event_failed', + 'cleanup_secret_store_failed','cleanup_finalize_failed' + )); + +CREATE INDEX IF NOT EXISTS idx_gateway_identity_onboarding_cleanup + ON gateway_identity_onboarding_exchanges(cleanup_status, updated_at) + WHERE cleanup_status = 'pending'; diff --git a/apps/api/migrations/0070_identity_secret_cleanup_queue.sql b/apps/api/migrations/0070_identity_secret_cleanup_queue.sql new file mode 100644 index 0000000..b1a8b9b --- /dev/null +++ b/apps/api/migrations/0070_identity_secret_cleanup_queue.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS gateway_identity_secret_cleanup_queue ( + secret_ref text PRIMARY KEY CHECK (secret_ref ~ '^[a-z0-9][a-z0-9-]{0,127}$'), + not_before timestamptz NOT NULL, + status text NOT NULL DEFAULT 'pending', + claim_token uuid, + lease_expires_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT gateway_identity_secret_cleanup_status_check + CHECK (status IN ('pending','claimed')), + CONSTRAINT gateway_identity_secret_cleanup_claim_check + CHECK ( + (status = 'pending' AND claim_token IS NULL AND lease_expires_at IS NULL) OR + (status = 'claimed' AND claim_token IS NOT NULL AND lease_expires_at IS NOT NULL) + ) +); + +CREATE INDEX IF NOT EXISTS idx_gateway_identity_secret_cleanup_due + ON gateway_identity_secret_cleanup_queue(status, not_before, lease_expires_at, updated_at); diff --git a/apps/api/migrations/0071_identity_pairing_start_reservation.sql b/apps/api/migrations/0071_identity_pairing_start_reservation.sql new file mode 100644 index 0000000..c7901f9 --- /dev/null +++ b/apps/api/migrations/0071_identity_pairing_start_reservation.sql @@ -0,0 +1,29 @@ +SET LOCAL lock_timeout = '10s'; +SET LOCAL statement_timeout = '60s'; + +CREATE TABLE IF NOT EXISTS gateway_identity_pairing_start_reservation ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + attempt_id uuid NOT NULL UNIQUE, + state text NOT NULL DEFAULT 'starting' CHECK (state IN ('starting','paired')), + revision_id uuid UNIQUE REFERENCES gateway_identity_configuration_revisions(id) ON DELETE RESTRICT, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT gateway_identity_pairing_start_state_check CHECK ( + (state='starting' AND revision_id IS NULL) OR + (state='paired' AND revision_id IS NOT NULL) + ) +); + +CREATE INDEX IF NOT EXISTS idx_gateway_identity_pairing_start_expiry + ON gateway_identity_pairing_start_reservation(state,expires_at); + +INSERT INTO gateway_identity_pairing_start_reservation(singleton,attempt_id,state,revision_id,expires_at) +SELECT true,exchange.id,'paired',revision.id,exchange.expires_at +FROM gateway_identity_configuration_revisions revision +JOIN gateway_identity_onboarding_exchanges exchange ON exchange.revision_id=revision.id +WHERE revision.state IN ('draft','validated','failed') + AND NOT (exchange.status='cancelled' AND exchange.cleanup_status='completed') +ORDER BY revision.created_at DESC,exchange.created_at DESC +LIMIT 1 +ON CONFLICT DO NOTHING; diff --git a/apps/api/migrations/0072_identity_secret_cleanup_claim_lifecycle.sql b/apps/api/migrations/0072_identity_secret_cleanup_claim_lifecycle.sql new file mode 100644 index 0000000..3d27f8c --- /dev/null +++ b/apps/api/migrations/0072_identity_secret_cleanup_claim_lifecycle.sql @@ -0,0 +1,34 @@ +SET LOCAL lock_timeout = '10s'; +SET LOCAL statement_timeout = '60s'; + +ALTER TABLE gateway_identity_secret_cleanup_queue + ADD COLUMN IF NOT EXISTS status text, + ADD COLUMN IF NOT EXISTS claim_token uuid, + ADD COLUMN IF NOT EXISTS lease_expires_at timestamptz; + +ALTER TABLE gateway_identity_secret_cleanup_queue + DROP CONSTRAINT IF EXISTS gateway_identity_secret_cleanup_status_check, + DROP CONSTRAINT IF EXISTS gateway_identity_secret_cleanup_claim_check; + +UPDATE gateway_identity_secret_cleanup_queue +SET status='pending',claim_token=NULL,lease_expires_at=NULL,updated_at=now() +WHERE status IS NULL + OR status NOT IN ('pending','claimed') + OR (status='pending' AND (claim_token IS NOT NULL OR lease_expires_at IS NOT NULL)) + OR (status='claimed' AND (claim_token IS NULL OR lease_expires_at IS NULL)); + +ALTER TABLE gateway_identity_secret_cleanup_queue + ALTER COLUMN status SET DEFAULT 'pending', + ALTER COLUMN status SET NOT NULL, + ADD CONSTRAINT gateway_identity_secret_cleanup_status_check + CHECK (status IN ('pending','claimed')), + ADD CONSTRAINT gateway_identity_secret_cleanup_claim_check + CHECK ( + (status='pending' AND claim_token IS NULL AND lease_expires_at IS NULL) OR + (status='claimed' AND claim_token IS NOT NULL AND lease_expires_at IS NOT NULL) + ); + +DROP INDEX IF EXISTS idx_gateway_identity_secret_cleanup_due; + +CREATE INDEX idx_gateway_identity_secret_cleanup_due + ON gateway_identity_secret_cleanup_queue(status,not_before,lease_expires_at,updated_at); diff --git a/apps/api/migrations/0073_identity_pairing_start_reservation_upgrade.sql b/apps/api/migrations/0073_identity_pairing_start_reservation_upgrade.sql new file mode 100644 index 0000000..7903e9d --- /dev/null +++ b/apps/api/migrations/0073_identity_pairing_start_reservation_upgrade.sql @@ -0,0 +1,89 @@ +SET LOCAL lock_timeout = '10s'; +SET LOCAL statement_timeout = '60s'; + +-- 0071 was briefly released with only the singleton reservation lease fields. +-- Add the durable lifecycle columns without rewriting that already-applied +-- migration, then reconcile the singleton with the canonical outstanding +-- pairing (when one exists). +ALTER TABLE gateway_identity_pairing_start_reservation + ADD COLUMN IF NOT EXISTS state text, + ADD COLUMN IF NOT EXISTS revision_id uuid, + ADD COLUMN IF NOT EXISTS updated_at timestamptz; + +UPDATE gateway_identity_pairing_start_reservation +SET state=CASE WHEN revision_id IS NULL THEN 'starting' ELSE 'paired' END, + updated_at=COALESCE(updated_at,created_at,now()) +WHERE state IS NULL + OR state NOT IN ('starting','paired') + OR (state='starting' AND revision_id IS NOT NULL) + OR (state='paired' AND revision_id IS NULL) + OR updated_at IS NULL; + +WITH canonical_pairing AS ( + SELECT exchange.id AS attempt_id, + revision.id AS revision_id, + exchange.expires_at + FROM gateway_identity_configuration_revisions revision + JOIN gateway_identity_onboarding_exchanges exchange + ON exchange.revision_id=revision.id + WHERE revision.state IN ('draft','validated','failed') + AND NOT (exchange.status='cancelled' AND exchange.cleanup_status='completed') + ORDER BY revision.created_at DESC,exchange.created_at DESC + LIMIT 1 +) +INSERT INTO gateway_identity_pairing_start_reservation( + singleton,attempt_id,state,revision_id,expires_at,updated_at +) +SELECT true,attempt_id,'paired',revision_id,expires_at,now() +FROM canonical_pairing +ON CONFLICT (singleton) DO UPDATE +SET attempt_id=EXCLUDED.attempt_id, + state=EXCLUDED.state, + revision_id=EXCLUDED.revision_id, + expires_at=EXCLUDED.expires_at, + updated_at=now(); + +ALTER TABLE gateway_identity_pairing_start_reservation + ALTER COLUMN state SET DEFAULT 'starting', + ALTER COLUMN state SET NOT NULL, + ALTER COLUMN updated_at SET DEFAULT now(), + ALTER COLUMN updated_at SET NOT NULL; + +ALTER TABLE gateway_identity_pairing_start_reservation + DROP CONSTRAINT IF EXISTS gateway_identity_pairing_start_state_value_check, + DROP CONSTRAINT IF EXISTS gateway_identity_pairing_start_state_check; + +ALTER TABLE gateway_identity_pairing_start_reservation + ADD CONSTRAINT gateway_identity_pairing_start_state_value_check + CHECK (state IN ('starting','paired')), + ADD CONSTRAINT gateway_identity_pairing_start_state_check CHECK ( + (state='starting' AND revision_id IS NULL) OR + (state='paired' AND revision_id IS NOT NULL) + ); + +DO $migration$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid='gateway_identity_pairing_start_reservation'::regclass + AND contype='f' + AND pg_get_constraintdef(oid) LIKE 'FOREIGN KEY (revision_id)%' + ) THEN + ALTER TABLE gateway_identity_pairing_start_reservation + ADD CONSTRAINT gateway_identity_pairing_start_revision_fk + FOREIGN KEY (revision_id) + REFERENCES gateway_identity_configuration_revisions(id) + ON DELETE RESTRICT; + END IF; +END +$migration$; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_gateway_identity_pairing_start_revision_unique + ON gateway_identity_pairing_start_reservation(revision_id) + WHERE revision_id IS NOT NULL; + +DROP INDEX IF EXISTS idx_gateway_identity_pairing_start_expiry; + +CREATE INDEX idx_gateway_identity_pairing_start_expiry + ON gateway_identity_pairing_start_reservation(state,expires_at); diff --git a/apps/api/migrations/0074_identity_pairing_error_categories.sql b/apps/api/migrations/0074_identity_pairing_error_categories.sql new file mode 100644 index 0000000..5935c99 --- /dev/null +++ b/apps/api/migrations/0074_identity_pairing_error_categories.sql @@ -0,0 +1,32 @@ +SET LOCAL lock_timeout = '10s'; +SET LOCAL statement_timeout = '60s'; + +-- Pairing recovery added explicit, redacted error categories after 0069 was +-- released. Keep the database allow-list in lockstep so PostgreSQL can persist +-- the reason an administrator must resolve instead of hiding it behind a CHECK +-- violation. +ALTER TABLE gateway_identity_onboarding_exchanges + DROP CONSTRAINT IF EXISTS gateway_identity_onboarding_error_category_check; + +ALTER TABLE gateway_identity_onboarding_exchanges + ADD CONSTRAINT gateway_identity_onboarding_error_category_check + CHECK (last_error_category IS NULL OR last_error_category IN ( + 'metadata_submission_failed','exchange_status_unavailable','credential_delivery_failed', + 'exchange_completion_failed','exchange_expired','remote_exchange_failed','remote_state_invalid', + 'security_event_configuration_invalid','security_event_connection_conflict', + 'security_event_discovery_failed','security_event_management_token_failed', + 'security_event_stream_create_failed','security_event_stream_response_invalid', + 'security_event_receiver_activation_failed','security_event_preparation_failed', + 'security_event_retirement_pending','security_event_credential_handoff_unsafe', + 'security_event_connection_binding_missing','security_event_connection_binding_unavailable', + 'security_event_connection_binding_invalid','security_event_connection_binding_mismatch', + 'pairing_step_failed', + 'cleanup_revision_unavailable','cleanup_security_event_unavailable', + 'cleanup_security_event_configuration_invalid','cleanup_security_event_connection_conflict', + 'cleanup_security_event_discovery_failed','cleanup_security_event_management_token_failed', + 'cleanup_security_event_stream_create_failed','cleanup_security_event_stream_response_invalid', + 'cleanup_security_event_receiver_activation_failed','cleanup_security_event_preparation_failed', + 'cleanup_security_event_retirement_pending','cleanup_security_event_connection_cleanup_failed', + 'cleanup_security_event_secret_cleanup_failed','cleanup_security_event_credential_handoff_unsafe', + 'cleanup_security_event_failed','cleanup_secret_store_failed','cleanup_finalize_failed' + )); diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index fbf66db..d4616a0 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1035,19 +1035,25 @@ export type IdentityPairingStatus = | 'credentials_saved' | 'completed' | 'failed' - | 'expired'; + | 'expired' + | 'cancelled'; + +export type IdentityPairingCleanupStatus = 'none' | 'pending' | 'completed'; export interface IdentityPairingExchange { id: string; revisionId: string; remoteExchangeId: string; status: IdentityPairingStatus; + cleanupStatus: IdentityPairingCleanupStatus; remoteVersion: number; expiresAt: string; version: number; lastErrorCategory?: string; authCenterAuditId?: string; lastTraceId?: string; + cancelledAt?: string; + cleanupCompletedAt?: string; createdAt: string; updatedAt: string; }