feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
@@ -19,7 +19,11 @@ type RuntimeRecoveryResult struct {
|
||||
CleanedTaskAdmissions int64 `json:"cleanedTaskAdmissions"`
|
||||
}
|
||||
|
||||
const runtimeRecoveryBatchSize = 100
|
||||
const (
|
||||
runtimeRecoveryBatchSize = 50
|
||||
runtimeWorkerStaleAfter = 60 * time.Second
|
||||
runtimeCounterRepairBatch = 100
|
||||
)
|
||||
|
||||
var ErrConcurrencyLeaseLost = errors.New("concurrency lease lost")
|
||||
|
||||
@@ -364,11 +368,24 @@ func (s *Store) ReleaseConcurrencyLeases(ctx context.Context, leases []Concurren
|
||||
if len(leaseIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
// A lost release after database failover is conservative: the lease
|
||||
// remains unavailable until its TTL or task-admission cleanup. It cannot
|
||||
// create duplicate execution, so avoid a dedicated synchronous wait.
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL synchronous_commit = off`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE id = ANY($1::uuid[]) AND released_at IS NULL`, leaseIDs)
|
||||
return err
|
||||
WHERE id = ANY($1::uuid[]) AND released_at IS NULL`, leaseIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *Store) RenewConcurrencyLeases(ctx context.Context, leases []ConcurrencyLease) error {
|
||||
@@ -417,6 +434,12 @@ func (s *Store) AttachRateLimitResultToAttempt(ctx context.Context, attemptID st
|
||||
return err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
// The subsequent durable `submitting` transition is the safety barrier for
|
||||
// these task-scoped associations. If execution stops before that point,
|
||||
// the task admission owns and reaps the leases by task_id.
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL synchronous_commit = off`); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, reservation := range result.Reservations {
|
||||
if reservation.ReservationID == "" {
|
||||
@@ -456,106 +479,121 @@ func concurrencyLeaseIDs(leases []ConcurrencyLease) []string {
|
||||
}
|
||||
|
||||
func (s *Store) RecoverInterruptedRuntimeState(ctx context.Context) (RuntimeRecoveryResult, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
var result RuntimeRecoveryResult
|
||||
if err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
|
||||
recovered, err := recoverExpiredRuntimeTasksTx(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result = recovered
|
||||
return nil
|
||||
}); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-runtime-recovery', 0))`); err != nil {
|
||||
if err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
|
||||
cleaned, err := cleanupTerminalRuntimeStateTx(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases += cleaned.ReleasedConcurrencyLeases
|
||||
result.ReleasedRateReservations += cleaned.ReleasedRateReservations
|
||||
result.CleanedTaskAdmissions += cleaned.CleanedTaskAdmissions
|
||||
return nil
|
||||
}); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
if err := s.repairRuntimeClientCounters(ctx); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
if result.RequeuedAsyncTasks > 0 || result.FailedTasks > 0 || result.CleanedTaskAdmissions > 0 {
|
||||
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
result := RuntimeRecoveryResult{}
|
||||
type runtimeRecoveryTask struct {
|
||||
ID string
|
||||
Async bool
|
||||
Production bool
|
||||
HasGatewayUser bool
|
||||
SubmissionUnsafe bool
|
||||
}
|
||||
|
||||
func recoverExpiredRuntimeTasksTx(ctx context.Context, tx pgx.Tx) (RuntimeRecoveryResult, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
UPDATE gateway_rate_limit_reservations reservation
|
||||
SET status = 'released',
|
||||
reason = 'execution_lease_expired',
|
||||
finalized_at = now(),
|
||||
updated_at = now()
|
||||
SELECT task.id::text,
|
||||
task.async_mode,
|
||||
task.run_mode IN ('production', 'acceptance', 'acceptance_canary'),
|
||||
task.gateway_user_id IS NOT NULL,
|
||||
COALESCE(task.remote_task_id, '') = ''
|
||||
AND COALESCE((
|
||||
SELECT (
|
||||
(attempt.status = 'running' AND attempt.upstream_submission_status IN ('submitting', 'response_received'))
|
||||
OR (attempt.status = 'failed' AND attempt.upstream_submission_status = 'submitting')
|
||||
OR (
|
||||
attempt.status = 'failed'
|
||||
AND attempt.upstream_submission_status = 'response_received'
|
||||
AND attempt.error_code = 'execution_lease_expired'
|
||||
)
|
||||
)
|
||||
FROM gateway_task_attempts attempt
|
||||
WHERE attempt.task_id = task.id
|
||||
ORDER BY attempt.attempt_no DESC, attempt.started_at DESC
|
||||
LIMIT 1
|
||||
), false)
|
||||
FROM gateway_tasks task
|
||||
WHERE reservation.task_id = task.id
|
||||
AND reservation.status = 'reserved'
|
||||
WHERE task.status = 'running'
|
||||
AND task.execution_lease_expires_at IS NOT NULL
|
||||
AND task.execution_lease_expires_at <= now()
|
||||
AND (
|
||||
task.status IN ('succeeded', 'failed', 'cancelled')
|
||||
OR (
|
||||
task.status IN ('queued', 'running')
|
||||
AND task.execution_lease_expires_at IS NOT NULL
|
||||
AND task.execution_lease_expires_at <= now()
|
||||
task.locked_by IS NULL
|
||||
OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_worker_instances worker
|
||||
WHERE worker.instance_id = task.locked_by
|
||||
AND worker.status = 'active'
|
||||
AND worker.heartbeat_at > now() - $1::interval
|
||||
)
|
||||
)
|
||||
RETURNING scope_type, scope_key, metric, window_start, reserved_amount::float8`)
|
||||
ORDER BY task.execution_lease_expires_at ASC, task.id ASC
|
||||
LIMIT $2
|
||||
FOR UPDATE OF task SKIP LOCKED`, runtimeWorkerStaleAfter.String(), runtimeRecoveryBatchSize)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
reservations := make([]RateLimitReservation, 0)
|
||||
tasks := make([]runtimeRecoveryTask, 0, runtimeRecoveryBatchSize)
|
||||
for rows.Next() {
|
||||
var reservation RateLimitReservation
|
||||
if err := rows.Scan(&reservation.ScopeType, &reservation.ScopeKey, &reservation.Metric, &reservation.WindowStart, &reservation.Amount); err != nil {
|
||||
var task runtimeRecoveryTask
|
||||
if err := rows.Scan(&task.ID, &task.Async, &task.Production, &task.HasGatewayUser, &task.SubmissionUnsafe); err != nil {
|
||||
rows.Close()
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
reservations = append(reservations, reservation)
|
||||
tasks = append(tasks, task)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
rows.Close()
|
||||
for _, reservation := range reservations {
|
||||
if err := releaseCounterReservation(ctx, tx, reservation.ScopeType, reservation.ScopeKey, reservation.Metric, reservation.WindowStart, reservation.Amount); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return RuntimeRecoveryResult{}, nil
|
||||
}
|
||||
result.ReleasedRateReservations = int64(len(reservations))
|
||||
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE released_at IS NULL
|
||||
AND expires_at <= now()`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases = tag.RowsAffected()
|
||||
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_task_attempts
|
||||
SET status = 'failed',
|
||||
retryable = true,
|
||||
error_code = 'execution_lease_expired',
|
||||
error_message = 'attempt execution lease expired',
|
||||
finished_at = now()
|
||||
WHERE status = 'running'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_tasks task
|
||||
WHERE task.id = gateway_task_attempts.task_id
|
||||
AND (
|
||||
task.status IN ('succeeded', 'failed', 'cancelled')
|
||||
OR (
|
||||
task.execution_lease_expires_at IS NOT NULL
|
||||
AND task.execution_lease_expires_at <= now()
|
||||
)
|
||||
)
|
||||
)`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.FailedAttempts = tag.RowsAffected()
|
||||
|
||||
asyncTaskRows, err := tx.Query(ctx, `
|
||||
WITH recoverable_async_tasks AS MATERIALIZED (
|
||||
SELECT id
|
||||
FROM gateway_tasks
|
||||
WHERE async_mode = true
|
||||
AND status = 'running'
|
||||
AND execution_lease_expires_at IS NOT NULL
|
||||
AND execution_lease_expires_at <= now()
|
||||
ORDER BY execution_lease_expires_at ASC, id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_tasks task
|
||||
result := RuntimeRecoveryResult{}
|
||||
taskIDs := make([]string, 0, len(tasks))
|
||||
releaseBillingTaskIDs := make([]string, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
taskIDs = append(taskIDs, task.ID)
|
||||
eventType := "task.failed"
|
||||
eventStatus := "failed"
|
||||
if task.Production && task.SubmissionUnsafe {
|
||||
if err := markTaskExecutionManualReviewTx(ctx, tx, task.ID, task.HasGatewayUser); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.FailedTasks++
|
||||
} else if task.Async {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'queued',
|
||||
error = NULL,
|
||||
error_code = NULL,
|
||||
@@ -568,156 +606,254 @@ SET status = 'queued',
|
||||
next_run_at = now(),
|
||||
finished_at = NULL,
|
||||
updated_at = now()
|
||||
FROM recoverable_async_tasks recoverable
|
||||
WHERE task.id = recoverable.id
|
||||
RETURNING task.id::text`, runtimeRecoveryBatchSize)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
asyncTaskIDs := make([]string, 0)
|
||||
for asyncTaskRows.Next() {
|
||||
var taskID string
|
||||
if err := asyncTaskRows.Scan(&taskID); err != nil {
|
||||
asyncTaskRows.Close()
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
asyncTaskIDs = append(asyncTaskIDs, taskID)
|
||||
}
|
||||
if err := asyncTaskRows.Err(); err != nil {
|
||||
asyncTaskRows.Close()
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
asyncTaskRows.Close()
|
||||
for _, taskID := range asyncTaskIDs {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
|
||||
SELECT
|
||||
task.id,
|
||||
COALESCE((SELECT MAX(seq) + 1 FROM gateway_task_events WHERE task_id = $1::uuid), 1),
|
||||
'task.queued',
|
||||
'queued',
|
||||
NULL,
|
||||
0,
|
||||
NULL,
|
||||
'{}'::jsonb,
|
||||
false
|
||||
FROM gateway_tasks task
|
||||
WHERE task.id = $1::uuid
|
||||
AND (
|
||||
SELECT count(*)
|
||||
FROM gateway_task_events event
|
||||
WHERE event.task_id = task.id
|
||||
AND event.event_type NOT IN (
|
||||
'task.completed', 'task.failed', 'task.cancelled',
|
||||
'task.billing.settled', 'task.billing.released', 'task.billing.review'
|
||||
)
|
||||
) < 16
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_task_events event
|
||||
WHERE event.task_id = task.id
|
||||
AND event.seq = (SELECT MAX(last_event.seq) FROM gateway_task_events last_event WHERE last_event.task_id = task.id)
|
||||
AND event.event_type = 'task.queued'
|
||||
AND COALESCE(event.status, '') = 'queued'
|
||||
AND event.platform_id IS NULL
|
||||
AND event.simulated = false
|
||||
)`, taskID); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
}
|
||||
result.RequeuedAsyncTasks = int64(len(asyncTaskIDs))
|
||||
if len(asyncTaskIDs) > 0 {
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, asyncTaskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases += tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
DELETE FROM gateway_task_admissions
|
||||
WHERE task_id = ANY($1::uuid[])`, asyncTaskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.CleanedTaskAdmissions += tag.RowsAffected()
|
||||
}
|
||||
|
||||
taskRows, err := tx.Query(ctx, `
|
||||
WHERE id = $1::uuid`, task.ID); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
eventType = "task.queued"
|
||||
eventStatus = "queued"
|
||||
result.RequeuedAsyncTasks++
|
||||
} else {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'failed',
|
||||
billing_status = CASE
|
||||
WHEN gateway_user_id IS NULL THEN 'not_required'
|
||||
WHEN reservation_amount > 0 THEN 'pending'
|
||||
ELSE 'released'
|
||||
END,
|
||||
billing_updated_at = now(),
|
||||
error = NULL,
|
||||
error_code = 'server_restarted',
|
||||
error_message = 'task interrupted by service restart',
|
||||
error_message = 'task interrupted after its owner and execution lease expired',
|
||||
remote_task_payload = '{}'::jsonb,
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE async_mode = false
|
||||
AND status = 'running'
|
||||
AND execution_lease_expires_at IS NOT NULL
|
||||
AND execution_lease_expires_at <= now()
|
||||
RETURNING id::text`)
|
||||
WHERE id = $1::uuid`, task.ID); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
if task.Production && task.HasGatewayUser {
|
||||
releaseBillingTaskIDs = append(releaseBillingTaskIDs, task.ID)
|
||||
}
|
||||
result.FailedTasks++
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
|
||||
SELECT $1::uuid,
|
||||
COALESCE(MAX(event.seq), 0) + 1,
|
||||
$2,
|
||||
$3,
|
||||
'runtime_recovery',
|
||||
0,
|
||||
NULL,
|
||||
'{}'::jsonb,
|
||||
false
|
||||
FROM gateway_task_events event
|
||||
WHERE event.task_id = $1::uuid
|
||||
HAVING NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_task_events latest
|
||||
WHERE latest.task_id = $1::uuid
|
||||
AND latest.seq = (SELECT MAX(last_event.seq) FROM gateway_task_events last_event WHERE last_event.task_id = $1::uuid)
|
||||
AND latest.event_type = $2
|
||||
AND latest.status = $3
|
||||
)`, task.ID, eventType, eventStatus); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
}
|
||||
if len(releaseBillingTaskIDs) > 0 {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
task_id, event_type, action, amount, currency, pricing_snapshot, payload,
|
||||
status, next_attempt_at
|
||||
)
|
||||
SELECT task.id, 'task.billing.release', 'release', task.reservation_amount,
|
||||
task.billing_currency, task.pricing_snapshot,
|
||||
jsonb_build_object('taskId', task.id, 'reason', 'execution_lease_expired'),
|
||||
'pending', now()
|
||||
FROM gateway_tasks task
|
||||
WHERE task.id = ANY($1::uuid[])
|
||||
AND task.reservation_amount > 0
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, releaseBillingTaskIDs); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_task_attempts attempt
|
||||
SET status = 'failed',
|
||||
retryable = task.error_code IS DISTINCT FROM 'upstream_submission_unknown',
|
||||
error_code = CASE
|
||||
WHEN task.error_code = 'upstream_submission_unknown' THEN 'upstream_submission_unknown'
|
||||
ELSE 'execution_lease_expired'
|
||||
END,
|
||||
error_message = CASE
|
||||
WHEN task.error_code = 'upstream_submission_unknown' THEN 'upstream submission result is unknown'
|
||||
ELSE 'attempt execution lease expired after worker heartbeat became stale'
|
||||
END,
|
||||
finished_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE attempt.task_id = task.id
|
||||
AND attempt.task_id = ANY($1::uuid[])
|
||||
AND attempt.status = 'running'`, taskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
taskIDs := make([]string, 0)
|
||||
for taskRows.Next() {
|
||||
result.FailedAttempts = tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, taskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases = tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
DELETE FROM gateway_task_admissions
|
||||
WHERE task_id = ANY($1::uuid[])`, taskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.CleanedTaskAdmissions = tag.RowsAffected()
|
||||
released, err := releaseRuntimeRateReservationsTx(ctx, tx, taskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedRateReservations = released
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func cleanupTerminalRuntimeStateTx(ctx context.Context, tx pgx.Tx) (RuntimeRecoveryResult, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT task.id::text
|
||||
FROM gateway_tasks task
|
||||
WHERE task.status IN ('succeeded', 'failed', 'cancelled')
|
||||
AND (
|
||||
EXISTS (
|
||||
SELECT 1 FROM gateway_concurrency_leases lease
|
||||
WHERE lease.task_id = task.id AND lease.released_at IS NULL
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM gateway_task_admissions admission
|
||||
WHERE admission.task_id = task.id
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM gateway_rate_limit_reservations reservation
|
||||
WHERE reservation.task_id = task.id AND reservation.status = 'reserved'
|
||||
)
|
||||
)
|
||||
ORDER BY task.updated_at ASC, task.id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE OF task SKIP LOCKED`, runtimeRecoveryBatchSize)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
taskIDs := make([]string, 0, runtimeRecoveryBatchSize)
|
||||
for rows.Next() {
|
||||
var taskID string
|
||||
if err := taskRows.Scan(&taskID); err != nil {
|
||||
taskRows.Close()
|
||||
if err := rows.Scan(&taskID); err != nil {
|
||||
rows.Close()
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
taskIDs = append(taskIDs, taskID)
|
||||
}
|
||||
if err := taskRows.Err(); err != nil {
|
||||
taskRows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
taskRows.Close()
|
||||
rows.Close()
|
||||
if len(taskIDs) == 0 {
|
||||
return RuntimeRecoveryResult{}, nil
|
||||
}
|
||||
var result RuntimeRecoveryResult
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE task_id = ANY($1::uuid[])
|
||||
AND released_at IS NULL`, taskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases = tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
DELETE FROM gateway_task_admissions
|
||||
WHERE task_id = ANY($1::uuid[])`, taskIDs)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.CleanedTaskAdmissions = tag.RowsAffected()
|
||||
result.ReleasedRateReservations, err = releaseRuntimeRateReservationsTx(ctx, tx, taskIDs)
|
||||
return result, err
|
||||
}
|
||||
|
||||
for _, taskID := range taskIDs {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
|
||||
VALUES (
|
||||
$1::uuid,
|
||||
COALESCE((SELECT MAX(seq) + 1 FROM gateway_task_events WHERE task_id = $1::uuid), 1),
|
||||
'task.failed',
|
||||
'failed',
|
||||
NULL,
|
||||
0,
|
||||
NULL,
|
||||
'{}'::jsonb,
|
||||
false
|
||||
)`, taskID); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
func releaseRuntimeRateReservationsTx(ctx context.Context, tx pgx.Tx, taskIDs []string) (int64, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
UPDATE gateway_rate_limit_reservations reservation
|
||||
SET status = 'released',
|
||||
reason = 'runtime_state_recovered',
|
||||
finalized_at = now(),
|
||||
updated_at = now()
|
||||
WHERE reservation.task_id = ANY($1::uuid[])
|
||||
AND reservation.status = 'reserved'
|
||||
RETURNING scope_type, scope_key, metric, window_start, reserved_amount::float8`, taskIDs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
reservations := make([]RateLimitReservation, 0, len(taskIDs))
|
||||
for rows.Next() {
|
||||
var reservation RateLimitReservation
|
||||
if err := rows.Scan(
|
||||
&reservation.ScopeType,
|
||||
&reservation.ScopeKey,
|
||||
&reservation.Metric,
|
||||
&reservation.WindowStart,
|
||||
&reservation.Amount,
|
||||
); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
reservations = append(reservations, reservation)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
rows.Close()
|
||||
for _, reservation := range reservations {
|
||||
if err := releaseCounterReservation(
|
||||
ctx,
|
||||
tx,
|
||||
reservation.ScopeType,
|
||||
reservation.ScopeKey,
|
||||
reservation.Metric,
|
||||
reservation.WindowStart,
|
||||
reservation.Amount,
|
||||
); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
result.FailedTasks = int64(len(taskIDs))
|
||||
tag, err = tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases lease
|
||||
SET released_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE lease.task_id = task.id
|
||||
AND lease.released_at IS NULL
|
||||
AND task.status IN ('succeeded', 'failed', 'cancelled')`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.ReleasedConcurrencyLeases += tag.RowsAffected()
|
||||
tag, err = tx.Exec(ctx, `
|
||||
DELETE FROM gateway_task_admissions admission
|
||||
USING gateway_tasks task
|
||||
WHERE admission.task_id = task.id
|
||||
AND task.status IN ('succeeded', 'failed', 'cancelled')`)
|
||||
if err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
result.CleanedTaskAdmissions += tag.RowsAffected()
|
||||
if _, err := tx.Exec(ctx, `
|
||||
return int64(len(reservations)), nil
|
||||
}
|
||||
|
||||
func (s *Store) repairRuntimeClientCounters(ctx context.Context) error {
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
WITH candidates AS MATERIALIZED (
|
||||
SELECT state.client_id
|
||||
FROM runtime_client_states state
|
||||
WHERE state.running_count IS DISTINCT FROM (
|
||||
SELECT count(*)
|
||||
FROM gateway_task_attempts attempt
|
||||
WHERE attempt.client_id = state.client_id
|
||||
AND attempt.status = 'running'
|
||||
)
|
||||
ORDER BY state.updated_at ASC, state.client_id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE runtime_client_states state
|
||||
SET running_count = (
|
||||
SELECT count(*)
|
||||
@@ -726,16 +862,9 @@ SET running_count = (
|
||||
AND attempt.status = 'running'
|
||||
),
|
||||
updated_at = now()
|
||||
WHERE state.running_count IS DISTINCT FROM (
|
||||
SELECT count(*)
|
||||
FROM gateway_task_attempts attempt
|
||||
WHERE attempt.client_id = state.client_id
|
||||
AND attempt.status = 'running'
|
||||
)`); err != nil {
|
||||
return RuntimeRecoveryResult{}, err
|
||||
}
|
||||
|
||||
return result, tx.Commit(ctx)
|
||||
FROM candidates
|
||||
WHERE state.client_id = candidates.client_id`, runtimeCounterRepairBatch)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) finishRateLimitReservations(ctx context.Context, reservations []RateLimitReservation, actualByMetric map[string]float64, status string, reason string) error {
|
||||
|
||||
Reference in New Issue
Block a user