wangbo
447e7ed701
将准入登记、派发、释放、恢复和验收清理中的 PostgreSQL NOTIFY 从同步复制业务事务移到提交后的独立异步提交事务，避免数据库对象锁串行阻塞跨城事务。通知继续作为有轮询兜底的限时提示，不改变任务状态、租约和账务的同步提交语义。 验证：Go 全量测试、go vet、迁移安全检查、gofmt、diff check 通过；新增集成回归确认 NOTIFY 对象锁存在时准入状态仍先提交可见。
723 lines
24 KiB
Go
723 lines
24 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/subtle"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
const SystemSettingGatewayTrafficMode = "gateway_traffic_mode"
|
|
|
|
var (
|
|
acceptanceReleaseSHAPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
|
acceptanceDigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
|
)
|
|
|
|
var (
|
|
ErrProductionTrafficPaused = errors.New("production traffic is paused for validation")
|
|
ErrAcceptanceNotAuthorized = errors.New("acceptance request is not authorized")
|
|
ErrAcceptanceRunNotActive = errors.New("acceptance run is not active")
|
|
ErrAcceptanceStateConflict = errors.New("acceptance state changed")
|
|
ErrAcceptancePromotionGates = errors.New("acceptance promotion gates have not passed")
|
|
)
|
|
|
|
type GatewayTrafficMode struct {
|
|
Mode string `json:"mode"`
|
|
RunID string `json:"runId,omitempty"`
|
|
Revision int64 `json:"revision"`
|
|
ReleaseSHA string `json:"releaseSha,omitempty"`
|
|
APIImageDigest string `json:"apiImageDigest,omitempty"`
|
|
WorkerImageDigest string `json:"workerImageDigest,omitempty"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
}
|
|
|
|
type AcceptanceRun struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
ReleaseSHA string `json:"releaseSha"`
|
|
APIImageDigest string `json:"apiImageDigest"`
|
|
WorkerImageDigest string `json:"workerImageDigest"`
|
|
APIKeyID string `json:"apiKeyId"`
|
|
UserID string `json:"userId"`
|
|
EmulatorBaseURL string `json:"emulatorBaseUrl"`
|
|
CallbackURL string `json:"callbackUrl,omitempty"`
|
|
CapacityProfile string `json:"capacityProfile"`
|
|
Config map[string]any `json:"config"`
|
|
Report map[string]any `json:"report"`
|
|
FailureReason string `json:"failureReason,omitempty"`
|
|
StartedAt string `json:"startedAt,omitempty"`
|
|
FinishedAt string `json:"finishedAt,omitempty"`
|
|
PromotedAt string `json:"promotedAt,omitempty"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
}
|
|
|
|
type CreateAcceptanceRunInput struct {
|
|
ReleaseSHA string `json:"releaseSha"`
|
|
APIImageDigest string `json:"apiImageDigest"`
|
|
WorkerImageDigest string `json:"workerImageDigest"`
|
|
APIKeyID string `json:"apiKeyId"`
|
|
UserID string `json:"userId"`
|
|
Token string `json:"token"`
|
|
EmulatorBaseURL string `json:"emulatorBaseUrl"`
|
|
CallbackURL string `json:"callbackUrl,omitempty"`
|
|
CapacityProfile string `json:"capacityProfile"`
|
|
Config map[string]any `json:"config,omitempty"`
|
|
}
|
|
|
|
type FinishAcceptanceRunInput struct {
|
|
RunID string `json:"-"`
|
|
Passed bool `json:"passed"`
|
|
Report map[string]any `json:"report"`
|
|
FailureReason string `json:"failureReason,omitempty"`
|
|
}
|
|
|
|
type PromoteAcceptanceRunInput struct {
|
|
RunID string `json:"-"`
|
|
Revision int64 `json:"revision"`
|
|
ReleaseSHA string `json:"releaseSha"`
|
|
APIImageDigest string `json:"apiImageDigest"`
|
|
WorkerImageDigest string `json:"workerImageDigest"`
|
|
}
|
|
|
|
func DefaultGatewayTrafficMode() GatewayTrafficMode {
|
|
return GatewayTrafficMode{Mode: "live"}
|
|
}
|
|
|
|
func (s *Store) GetGatewayTrafficMode(ctx context.Context) (GatewayTrafficMode, error) {
|
|
var value []byte
|
|
var updatedAt time.Time
|
|
err := s.pool.QueryRow(ctx, `
|
|
SELECT value, updated_at
|
|
FROM system_settings
|
|
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode).Scan(&value, &updatedAt)
|
|
if err != nil {
|
|
if IsNotFound(err) || IsUndefinedDatabaseObject(err) {
|
|
return DefaultGatewayTrafficMode(), nil
|
|
}
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
var mode GatewayTrafficMode
|
|
if err := json.Unmarshal(value, &mode); err != nil {
|
|
return GatewayTrafficMode{}, fmt.Errorf("decode gateway traffic mode: %w", err)
|
|
}
|
|
mode.Mode = normalizeTrafficMode(mode.Mode)
|
|
mode.UpdatedAt = updatedAt
|
|
return mode, nil
|
|
}
|
|
|
|
func (s *Store) CreateAcceptanceRun(ctx context.Context, input CreateAcceptanceRunInput) (AcceptanceRun, error) {
|
|
input.ReleaseSHA = strings.TrimSpace(input.ReleaseSHA)
|
|
input.APIImageDigest = strings.TrimSpace(input.APIImageDigest)
|
|
input.WorkerImageDigest = strings.TrimSpace(input.WorkerImageDigest)
|
|
input.APIKeyID = strings.TrimSpace(input.APIKeyID)
|
|
input.UserID = strings.TrimSpace(input.UserID)
|
|
input.Token = strings.TrimSpace(input.Token)
|
|
input.EmulatorBaseURL = strings.TrimRight(strings.TrimSpace(input.EmulatorBaseURL), "/")
|
|
input.CallbackURL = strings.TrimSpace(input.CallbackURL)
|
|
input.CapacityProfile = strings.ToUpper(strings.TrimSpace(input.CapacityProfile))
|
|
if input.CapacityProfile == "" {
|
|
input.CapacityProfile = "P24"
|
|
}
|
|
if input.ReleaseSHA == "" || input.APIImageDigest == "" || input.WorkerImageDigest == "" ||
|
|
input.APIKeyID == "" || input.UserID == "" || input.Token == "" || input.EmulatorBaseURL == "" ||
|
|
input.CallbackURL == "" {
|
|
return AcceptanceRun{}, errors.New("release SHA, image digests, API key, user, token, emulator URL, and callback URL are required")
|
|
}
|
|
if !acceptanceReleaseSHAPattern.MatchString(input.ReleaseSHA) ||
|
|
!acceptanceDigestPattern.MatchString(input.APIImageDigest) ||
|
|
!acceptanceDigestPattern.MatchString(input.WorkerImageDigest) {
|
|
return AcceptanceRun{}, errors.New("release SHA and image digests must use full immutable identifiers")
|
|
}
|
|
if len(input.Token) < 32 {
|
|
return AcceptanceRun{}, errors.New("acceptance token must contain at least 32 characters")
|
|
}
|
|
if !validAcceptanceURL(input.EmulatorBaseURL) || !validAcceptanceURL(input.CallbackURL) {
|
|
return AcceptanceRun{}, errors.New("emulator and callback URLs must be absolute HTTP URLs without user information")
|
|
}
|
|
switch input.CapacityProfile {
|
|
case "P24", "P28", "P32":
|
|
default:
|
|
return AcceptanceRun{}, errors.New("capacity profile must be P24, P28, or P32")
|
|
}
|
|
config, _ := json.Marshal(sanitizeAcceptanceMetadata(sanitizeJSONForStorage(input.Config)))
|
|
tokenHash := acceptanceTokenSHA256(input.Token)
|
|
if err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
|
|
_, err := cancelSafeAcceptanceTasksTx(ctx, tx, "")
|
|
return err
|
|
}); err != nil {
|
|
return AcceptanceRun{}, err
|
|
}
|
|
return scanAcceptanceRun(s.pool.QueryRow(ctx, `
|
|
INSERT INTO gateway_acceptance_runs (
|
|
release_sha, api_image_digest, worker_image_digest, api_key_id, user_id,
|
|
token_sha256, emulator_base_url, callback_url, capacity_profile, config
|
|
)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, NULLIF($8, ''), $9, $10::jsonb)
|
|
RETURNING `+acceptanceRunColumns,
|
|
input.ReleaseSHA, input.APIImageDigest, input.WorkerImageDigest, input.APIKeyID, input.UserID,
|
|
tokenHash, input.EmulatorBaseURL, input.CallbackURL, input.CapacityProfile, config,
|
|
))
|
|
}
|
|
|
|
func (s *Store) GetAcceptanceRun(ctx context.Context, runID string) (AcceptanceRun, error) {
|
|
return scanAcceptanceRun(s.pool.QueryRow(ctx, `
|
|
SELECT `+acceptanceRunColumns+`
|
|
FROM gateway_acceptance_runs
|
|
WHERE id = $1::uuid`, strings.TrimSpace(runID)))
|
|
}
|
|
|
|
func (s *Store) ActivateAcceptanceRun(ctx context.Context, runID string) (GatewayTrafficMode, error) {
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
defer rollbackTransaction(tx)
|
|
|
|
var currentValue []byte
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT value
|
|
FROM system_settings
|
|
WHERE setting_key = $1
|
|
FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(¤tValue); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
var current GatewayTrafficMode
|
|
if err := json.Unmarshal(currentValue, ¤t); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if normalizeTrafficMode(current.Mode) != "live" {
|
|
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
|
}
|
|
run, err := scanAcceptanceRun(tx.QueryRow(ctx, `
|
|
SELECT `+acceptanceRunColumns+`
|
|
FROM gateway_acceptance_runs
|
|
WHERE id = $1::uuid
|
|
FOR UPDATE`, strings.TrimSpace(runID)))
|
|
if err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if run.Status != "pending" && run.Status != "failed" {
|
|
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
|
}
|
|
next := GatewayTrafficMode{
|
|
Mode: "validation",
|
|
RunID: run.ID,
|
|
Revision: current.Revision + 1,
|
|
ReleaseSHA: run.ReleaseSHA,
|
|
APIImageDigest: run.APIImageDigest,
|
|
WorkerImageDigest: run.WorkerImageDigest,
|
|
}
|
|
value, _ := json.Marshal(next)
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE system_settings
|
|
SET value = $2::jsonb, updated_at = now()
|
|
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode, value); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE gateway_acceptance_runs
|
|
SET status = 'running', started_at = COALESCE(started_at, now()),
|
|
finished_at = NULL, failure_reason = NULL, updated_at = now()
|
|
WHERE id = $1::uuid`, run.ID); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
next.UpdatedAt = time.Now()
|
|
return next, nil
|
|
}
|
|
|
|
func (s *Store) FinishAcceptanceRun(ctx context.Context, input FinishAcceptanceRunInput) (AcceptanceRun, error) {
|
|
status := "failed"
|
|
if input.Passed {
|
|
status = "passed"
|
|
}
|
|
report, _ := json.Marshal(sanitizeAcceptanceMetadata(sanitizeJSONForStorage(input.Report)))
|
|
return scanAcceptanceRun(s.pool.QueryRow(ctx, `
|
|
UPDATE gateway_acceptance_runs
|
|
SET status = $2, report = $3::jsonb, failure_reason = NULLIF($4, ''),
|
|
finished_at = now(), updated_at = now()
|
|
WHERE id = $1::uuid AND status = 'running'
|
|
RETURNING `+acceptanceRunColumns,
|
|
strings.TrimSpace(input.RunID), status, report, strings.TrimSpace(input.FailureReason),
|
|
))
|
|
}
|
|
|
|
func (s *Store) RetryAcceptanceRun(ctx context.Context, runID string) (AcceptanceRun, error) {
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return AcceptanceRun{}, err
|
|
}
|
|
defer rollbackTransaction(tx)
|
|
var value []byte
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT value
|
|
FROM system_settings
|
|
WHERE setting_key = $1
|
|
FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil {
|
|
return AcceptanceRun{}, err
|
|
}
|
|
var mode GatewayTrafficMode
|
|
if err := json.Unmarshal(value, &mode); err != nil {
|
|
return AcceptanceRun{}, err
|
|
}
|
|
if normalizeTrafficMode(mode.Mode) != "validation" || mode.RunID != strings.TrimSpace(runID) {
|
|
return AcceptanceRun{}, ErrAcceptanceStateConflict
|
|
}
|
|
run, err := scanAcceptanceRun(tx.QueryRow(ctx, `
|
|
UPDATE gateway_acceptance_runs
|
|
SET status = 'running', report = '{}'::jsonb, failure_reason = NULL,
|
|
finished_at = NULL, updated_at = now()
|
|
WHERE id = $1::uuid AND status = 'failed'
|
|
RETURNING `+acceptanceRunColumns, strings.TrimSpace(runID)))
|
|
if err != nil {
|
|
return AcceptanceRun{}, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return AcceptanceRun{}, err
|
|
}
|
|
return run, nil
|
|
}
|
|
|
|
func (s *Store) PromoteAcceptanceRun(ctx context.Context, input PromoteAcceptanceRunInput) (GatewayTrafficMode, error) {
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
defer rollbackTransaction(tx)
|
|
var value []byte
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT value
|
|
FROM system_settings
|
|
WHERE setting_key = $1
|
|
FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
var current GatewayTrafficMode
|
|
if err := json.Unmarshal(value, ¤t); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if normalizeTrafficMode(current.Mode) != "validation" ||
|
|
current.RunID != strings.TrimSpace(input.RunID) ||
|
|
current.Revision != input.Revision ||
|
|
current.ReleaseSHA != strings.TrimSpace(input.ReleaseSHA) ||
|
|
current.APIImageDigest != strings.TrimSpace(input.APIImageDigest) ||
|
|
current.WorkerImageDigest != strings.TrimSpace(input.WorkerImageDigest) {
|
|
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
|
}
|
|
var status string
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT status
|
|
FROM gateway_acceptance_runs
|
|
WHERE id = $1::uuid
|
|
FOR UPDATE`, current.RunID).Scan(&status); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if status != "passed" {
|
|
return GatewayTrafficMode{}, ErrAcceptancePromotionGates
|
|
}
|
|
next := GatewayTrafficMode{Mode: "live", Revision: current.Revision + 1}
|
|
nextValue, _ := json.Marshal(next)
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE system_settings
|
|
SET value = $2::jsonb, updated_at = now()
|
|
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode, nextValue); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE gateway_acceptance_runs
|
|
SET status = 'promoted', promoted_at = now(), updated_at = now()
|
|
WHERE id = $1::uuid`, current.RunID); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
next.UpdatedAt = time.Now()
|
|
return next, nil
|
|
}
|
|
|
|
func (s *Store) AbortAcceptanceRun(ctx context.Context, input PromoteAcceptanceRunInput) (GatewayTrafficMode, error) {
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
defer rollbackTransaction(tx)
|
|
var value []byte
|
|
if err := tx.QueryRow(ctx, `
|
|
SELECT value
|
|
FROM system_settings
|
|
WHERE setting_key = $1
|
|
FOR UPDATE`, SystemSettingGatewayTrafficMode).Scan(&value); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
var current GatewayTrafficMode
|
|
if err := json.Unmarshal(value, ¤t); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if normalizeTrafficMode(current.Mode) != "validation" ||
|
|
current.RunID != strings.TrimSpace(input.RunID) ||
|
|
current.Revision != input.Revision ||
|
|
current.ReleaseSHA != strings.TrimSpace(input.ReleaseSHA) ||
|
|
current.APIImageDigest != strings.TrimSpace(input.APIImageDigest) ||
|
|
current.WorkerImageDigest != strings.TrimSpace(input.WorkerImageDigest) {
|
|
return GatewayTrafficMode{}, ErrAcceptanceStateConflict
|
|
}
|
|
cancelled, err := cancelSafeAcceptanceTasksTx(ctx, tx, current.RunID)
|
|
if err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
next := GatewayTrafficMode{Mode: "live", Revision: current.Revision + 1}
|
|
nextValue, _ := json.Marshal(next)
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE system_settings
|
|
SET value = $2::jsonb, updated_at = now()
|
|
WHERE setting_key = $1`, SystemSettingGatewayTrafficMode, nextValue); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE gateway_acceptance_runs
|
|
SET status = 'aborted', finished_at = COALESCE(finished_at, now()), updated_at = now()
|
|
WHERE id = $1::uuid AND status IN ('running', 'failed', 'passed')`, current.RunID); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return GatewayTrafficMode{}, err
|
|
}
|
|
if cancelled > 0 {
|
|
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
|
}
|
|
next.UpdatedAt = time.Now()
|
|
return next, nil
|
|
}
|
|
|
|
func cancelSafeAcceptanceTasksTx(ctx context.Context, tx pgx.Tx, runID string) (int64, error) {
|
|
runID = strings.TrimSpace(runID)
|
|
tag, err := tx.Exec(ctx, `
|
|
UPDATE gateway_tasks task
|
|
SET status = 'cancelled',
|
|
error = NULL,
|
|
error_code = 'acceptance_run_aborted',
|
|
error_message = 'acceptance run ended before upstream submission',
|
|
billing_status = CASE
|
|
WHEN gateway_user_id IS NULL THEN 'not_required'
|
|
WHEN reservation_amount > 0 THEN 'pending'
|
|
ELSE 'released'
|
|
END,
|
|
billing_updated_at = now(),
|
|
remote_task_payload = '{}'::jsonb,
|
|
locked_by = NULL,
|
|
locked_at = NULL,
|
|
heartbeat_at = NULL,
|
|
execution_token = NULL,
|
|
execution_lease_expires_at = NULL,
|
|
finished_at = now(),
|
|
updated_at = now()
|
|
WHERE task.run_mode = 'acceptance'
|
|
AND task.status IN ('queued', 'running')
|
|
AND COALESCE(task.remote_task_id, '') = ''
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM gateway_task_attempts attempt
|
|
WHERE attempt.task_id = task.id
|
|
AND attempt.upstream_submission_status <> 'not_submitted'
|
|
)
|
|
AND (
|
|
task.acceptance_run_id = NULLIF($1, '')::uuid
|
|
OR (
|
|
NULLIF($1, '') IS NULL
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM gateway_acceptance_runs run
|
|
WHERE run.id = task.acceptance_run_id
|
|
AND run.status IN ('failed', 'aborted')
|
|
)
|
|
)
|
|
)`, runID)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
DELETE FROM gateway_task_param_preprocessing_logs log
|
|
USING gateway_task_attempts attempt, gateway_tasks task
|
|
WHERE log.attempt_id = attempt.id
|
|
AND attempt.task_id = task.id
|
|
AND attempt.upstream_submission_status = 'not_submitted'
|
|
AND task.error_code = 'acceptance_run_aborted'
|
|
AND (
|
|
task.acceptance_run_id = NULLIF($1, '')::uuid
|
|
OR (
|
|
NULLIF($1, '') IS NULL
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM gateway_acceptance_runs run
|
|
WHERE run.id = task.acceptance_run_id
|
|
AND run.status IN ('failed', 'aborted')
|
|
)
|
|
)
|
|
)`, runID); err != nil {
|
|
return 0, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
DELETE FROM gateway_task_attempts attempt
|
|
USING gateway_tasks task
|
|
WHERE attempt.task_id = task.id
|
|
AND attempt.upstream_submission_status = 'not_submitted'
|
|
AND task.error_code = 'acceptance_run_aborted'
|
|
AND (
|
|
task.acceptance_run_id = NULLIF($1, '')::uuid
|
|
OR (
|
|
NULLIF($1, '') IS NULL
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM gateway_acceptance_runs run
|
|
WHERE run.id = task.acceptance_run_id
|
|
AND run.status IN ('failed', 'aborted')
|
|
)
|
|
)
|
|
)`, runID); err != nil {
|
|
return 0, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
INSERT INTO settlement_outbox (
|
|
task_id, event_type, action, amount, currency, pricing_snapshot, payload,
|
|
status, next_attempt_at
|
|
)
|
|
SELECT task.id, 'task.billing.release', 'release', task.reservation_amount,
|
|
task.billing_currency, task.pricing_snapshot,
|
|
jsonb_build_object('taskId', task.id, 'reason', 'acceptance_run_aborted'),
|
|
'pending', now()
|
|
FROM gateway_tasks task
|
|
WHERE task.error_code = 'acceptance_run_aborted'
|
|
AND task.billing_status = 'pending'
|
|
AND task.reservation_amount > 0
|
|
AND (
|
|
task.acceptance_run_id = NULLIF($1, '')::uuid
|
|
OR (
|
|
NULLIF($1, '') IS NULL
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM gateway_acceptance_runs run
|
|
WHERE run.id = task.acceptance_run_id
|
|
AND run.status IN ('failed', 'aborted')
|
|
)
|
|
)
|
|
)
|
|
ON CONFLICT (task_id, event_type) DO NOTHING`, runID); err != nil {
|
|
return 0, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
UPDATE gateway_concurrency_leases lease
|
|
SET released_at = now()
|
|
FROM gateway_tasks task
|
|
WHERE lease.task_id = task.id
|
|
AND lease.released_at IS NULL
|
|
AND task.error_code = 'acceptance_run_aborted'
|
|
AND (
|
|
task.acceptance_run_id = NULLIF($1, '')::uuid
|
|
OR (
|
|
NULLIF($1, '') IS NULL
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM gateway_acceptance_runs run
|
|
WHERE run.id = task.acceptance_run_id
|
|
AND run.status IN ('failed', 'aborted')
|
|
)
|
|
)
|
|
)`, runID); err != nil {
|
|
return 0, err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
DELETE FROM gateway_task_admissions admission
|
|
USING gateway_tasks task
|
|
WHERE admission.task_id = task.id
|
|
AND task.error_code = 'acceptance_run_aborted'
|
|
AND (
|
|
task.acceptance_run_id = NULLIF($1, '')::uuid
|
|
OR (
|
|
NULLIF($1, '') IS NULL
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM gateway_acceptance_runs run
|
|
WHERE run.id = task.acceptance_run_id
|
|
AND run.status IN ('failed', 'aborted')
|
|
)
|
|
)
|
|
)`, runID); err != nil {
|
|
return 0, err
|
|
}
|
|
return tag.RowsAffected(), nil
|
|
}
|
|
|
|
func (s *Store) AuthorizeAcceptanceTask(ctx context.Context, runID string, token string, user *auth.User) (string, error) {
|
|
mode, err := s.GetGatewayTrafficMode(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
runID = strings.TrimSpace(runID)
|
|
token = strings.TrimSpace(token)
|
|
if mode.Mode == "live" {
|
|
if runID != "" || token != "" {
|
|
return "", ErrAcceptanceRunNotActive
|
|
}
|
|
return "", nil
|
|
}
|
|
if runID == "" || token == "" || runID != mode.RunID {
|
|
return "", ErrProductionTrafficPaused
|
|
}
|
|
if user == nil || strings.TrimSpace(user.APIKeyID) == "" {
|
|
return "", ErrAcceptanceNotAuthorized
|
|
}
|
|
var activeRunID, apiKeyID, userID, tokenHash, status string
|
|
var participantAuthorized bool
|
|
err = s.pool.QueryRow(ctx, `
|
|
SELECT id::text, api_key_id, user_id, token_sha256, status,
|
|
(
|
|
(api_key_id = $2 AND user_id = $3)
|
|
OR EXISTS (
|
|
SELECT 1
|
|
FROM jsonb_array_elements(
|
|
CASE
|
|
WHEN jsonb_typeof(config->'participants') = 'array'
|
|
THEN config->'participants'
|
|
ELSE '[]'::jsonb
|
|
END
|
|
) participant
|
|
WHERE participant->>'apiKeyId' = $2
|
|
AND participant->>'userId' = $3
|
|
)
|
|
)
|
|
FROM gateway_acceptance_runs
|
|
WHERE id = $1::uuid`, runID, strings.TrimSpace(user.APIKeyID), strings.TrimSpace(user.ID)).Scan(
|
|
&activeRunID, &apiKeyID, &userID, &tokenHash, &status, &participantAuthorized,
|
|
)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return "", ErrAcceptanceNotAuthorized
|
|
}
|
|
return "", err
|
|
}
|
|
if status != "running" || !participantAuthorized ||
|
|
subtle.ConstantTimeCompare([]byte(tokenHash), []byte(acceptanceTokenSHA256(token))) != 1 {
|
|
return "", ErrAcceptanceNotAuthorized
|
|
}
|
|
return activeRunID, nil
|
|
}
|
|
|
|
func (s *Store) AcceptanceCandidateOverride(ctx context.Context, runID string) (string, string, error) {
|
|
var baseURL, status string
|
|
err := s.pool.QueryRow(ctx, `
|
|
SELECT emulator_base_url, status
|
|
FROM gateway_acceptance_runs
|
|
WHERE id = $1::uuid`, strings.TrimSpace(runID)).Scan(&baseURL, &status)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if status != "running" && status != "passed" {
|
|
return "", "", ErrAcceptanceRunNotActive
|
|
}
|
|
return strings.TrimRight(strings.TrimSpace(baseURL), "/"), "acceptance-protocol-emulator", nil
|
|
}
|
|
|
|
func (s *Store) AcceptanceCallbackURL(ctx context.Context, taskID string) (string, bool, error) {
|
|
var callbackURL string
|
|
err := s.pool.QueryRow(ctx, `
|
|
SELECT COALESCE(run.callback_url, '')
|
|
FROM gateway_tasks task
|
|
JOIN gateway_acceptance_runs run ON run.id = task.acceptance_run_id
|
|
WHERE task.id = $1::uuid`, strings.TrimSpace(taskID)).Scan(&callbackURL)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return "", false, nil
|
|
}
|
|
return "", false, err
|
|
}
|
|
return strings.TrimSpace(callbackURL), true, nil
|
|
}
|
|
|
|
func normalizeTrafficMode(value string) string {
|
|
if strings.EqualFold(strings.TrimSpace(value), "validation") {
|
|
return "validation"
|
|
}
|
|
return "live"
|
|
}
|
|
|
|
func acceptanceTokenSHA256(token string) string {
|
|
sum := sha256.Sum256([]byte(strings.TrimSpace(token)))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func validAcceptanceURL(raw string) bool {
|
|
parsed, err := url.Parse(strings.TrimSpace(raw))
|
|
if err != nil || parsed.User != nil || parsed.Host == "" {
|
|
return false
|
|
}
|
|
return parsed.Scheme == "http" || parsed.Scheme == "https"
|
|
}
|
|
|
|
func sanitizeAcceptanceMetadata(value any) any {
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
next := make(map[string]any, len(typed))
|
|
for key, item := range typed {
|
|
normalized := strings.ToLower(strings.NewReplacer("_", "", "-", "", ".", "").Replace(key))
|
|
if strings.Contains(normalized, "password") ||
|
|
strings.Contains(normalized, "secret") ||
|
|
strings.Contains(normalized, "credential") ||
|
|
strings.Contains(normalized, "connectionstring") ||
|
|
strings.Contains(normalized, "databaseurl") ||
|
|
(strings.Contains(normalized, "token") && !strings.HasSuffix(normalized, "count")) {
|
|
next[key] = "[REDACTED]"
|
|
continue
|
|
}
|
|
next[key] = sanitizeAcceptanceMetadata(item)
|
|
}
|
|
return next
|
|
case []any:
|
|
next := make([]any, len(typed))
|
|
for index, item := range typed {
|
|
next[index] = sanitizeAcceptanceMetadata(item)
|
|
}
|
|
return next
|
|
default:
|
|
return value
|
|
}
|
|
}
|
|
|
|
const acceptanceRunColumns = `
|
|
id::text, status, release_sha, api_image_digest, worker_image_digest,
|
|
api_key_id, user_id, emulator_base_url, COALESCE(callback_url, ''),
|
|
capacity_profile, config, report, COALESCE(failure_reason, ''),
|
|
COALESCE(started_at::text, ''), COALESCE(finished_at::text, ''),
|
|
COALESCE(promoted_at::text, ''), created_at, updated_at`
|
|
|
|
func scanAcceptanceRun(scanner taskScanner) (AcceptanceRun, error) {
|
|
var run AcceptanceRun
|
|
var config, report []byte
|
|
err := scanner.Scan(
|
|
&run.ID, &run.Status, &run.ReleaseSHA, &run.APIImageDigest, &run.WorkerImageDigest,
|
|
&run.APIKeyID, &run.UserID, &run.EmulatorBaseURL, &run.CallbackURL,
|
|
&run.CapacityProfile, &config, &report, &run.FailureReason,
|
|
&run.StartedAt, &run.FinishedAt, &run.PromotedAt, &run.CreatedAt, &run.UpdatedAt,
|
|
)
|
|
if err != nil {
|
|
return AcceptanceRun{}, err
|
|
}
|
|
run.Config = decodeObject(config)
|
|
run.Report = decodeObject(report)
|
|
return run, nil
|
|
}
|