fix(queue): 隔离异步准入故障并持久化退避
将 X-Async 调度改为逐任务阻塞协议,区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障,避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈,并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证:API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过;pnpm audit 命中未改动的 Nx 工具链既有漏洞。
This commit is contained in:
@@ -65,21 +65,34 @@ type TaskAdmissionInput struct {
|
||||
}
|
||||
|
||||
type TaskAdmission struct {
|
||||
TaskID string
|
||||
PlatformID string
|
||||
PlatformModelID string
|
||||
UserGroupID string
|
||||
QueueKey string
|
||||
Mode string
|
||||
Status string
|
||||
Priority int
|
||||
EnqueuedAt time.Time
|
||||
WaitDeadlineAt time.Time
|
||||
AdmittedAt time.Time
|
||||
WaiterID string
|
||||
WaiterLeaseExpires time.Time
|
||||
Scopes []AdmissionScope
|
||||
ReselectRequestedAt time.Time
|
||||
TaskID string
|
||||
PlatformID string
|
||||
PlatformModelID string
|
||||
UserGroupID string
|
||||
QueueKey string
|
||||
Mode string
|
||||
Status string
|
||||
Priority int
|
||||
EnqueuedAt time.Time
|
||||
WaitDeadlineAt time.Time
|
||||
AdmittedAt time.Time
|
||||
WaiterID string
|
||||
WaiterLeaseExpires time.Time
|
||||
Scopes []AdmissionScope
|
||||
ReselectRequestedAt time.Time
|
||||
DispatchNextAt time.Time
|
||||
DispatchFailureCount int
|
||||
DispatchLastOutcome string
|
||||
DispatchLastErrorCode string
|
||||
DispatchLastErrorAt time.Time
|
||||
SnapshotInvalid bool
|
||||
}
|
||||
|
||||
type TaskAdmissionBlocker struct {
|
||||
Reason string
|
||||
ScopeType string
|
||||
ScopeKey string
|
||||
RetryAt time.Time
|
||||
}
|
||||
|
||||
type TaskAdmissionResult struct {
|
||||
@@ -87,6 +100,7 @@ type TaskAdmissionResult struct {
|
||||
Admitted bool
|
||||
NewlyAdmitted bool
|
||||
Leases []ConcurrencyLease
|
||||
Blockers []TaskAdmissionBlocker
|
||||
}
|
||||
|
||||
type TaskAdmissionBatchOutcome struct {
|
||||
@@ -96,12 +110,15 @@ type TaskAdmissionBatchOutcome struct {
|
||||
}
|
||||
|
||||
type TaskAdmissionMetricsSnapshot struct {
|
||||
QueueDepth int
|
||||
WaitingSync int
|
||||
WaitingAsync int
|
||||
OldestWaitSeconds float64
|
||||
ExpiredWaiterBacklog int
|
||||
ExpiredDeadlineBacklog int
|
||||
QueueDepth int
|
||||
WaitingSync int
|
||||
WaitingAsync int
|
||||
DeferredAsync int
|
||||
PendingReselect int
|
||||
OldestWaitSeconds float64
|
||||
OldestDispatchableWaitSeconds float64
|
||||
ExpiredWaiterBacklog int
|
||||
ExpiredDeadlineBacklog int
|
||||
}
|
||||
|
||||
type TaskAdmissionReapResult struct {
|
||||
@@ -347,7 +364,7 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
}
|
||||
return taskAdmissionTxOutcome{Result: result}, nil
|
||||
}
|
||||
if bindingChanged {
|
||||
if bindingChanged || activeLeases == 0 {
|
||||
targetStates, stateErr := admissionScopeStatesTx(ctx, tx, scopes)
|
||||
if stateErr != nil {
|
||||
return taskAdmissionTxOutcome{}, stateErr
|
||||
@@ -392,22 +409,31 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
}
|
||||
}
|
||||
|
||||
head, err := isTaskAdmissionHeadTx(ctx, tx, admission, scopes)
|
||||
head, headBlockers, err := isTaskAdmissionHeadTx(ctx, tx, admission, scopes)
|
||||
if err != nil {
|
||||
return taskAdmissionTxOutcome{}, err
|
||||
}
|
||||
if !head {
|
||||
return taskAdmissionTxOutcome{
|
||||
Result: TaskAdmissionResult{Admission: admission},
|
||||
Result: TaskAdmissionResult{Admission: admission, Blockers: headBlockers},
|
||||
}, nil
|
||||
}
|
||||
blockers := make([]TaskAdmissionBlocker, 0, len(scopeStates))
|
||||
for _, state := range scopeStates {
|
||||
if state.Saturated {
|
||||
return taskAdmissionTxOutcome{
|
||||
Result: TaskAdmissionResult{Admission: admission},
|
||||
}, nil
|
||||
blockers = append(blockers, TaskAdmissionBlocker{
|
||||
Reason: "saturated",
|
||||
ScopeType: state.Scope.ScopeType,
|
||||
ScopeKey: state.Scope.ScopeKey,
|
||||
RetryAt: state.NextLeaseExpiration,
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(blockers) > 0 {
|
||||
return taskAdmissionTxOutcome{
|
||||
Result: TaskAdmissionResult{Admission: admission, Blockers: blockers},
|
||||
}, nil
|
||||
}
|
||||
|
||||
leases := make([]ConcurrencyLease, 0, len(scopes))
|
||||
for _, scope := range scopes {
|
||||
@@ -498,9 +524,6 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
if admissionErr != nil {
|
||||
return outcomes, admissionErr
|
||||
}
|
||||
if !result.Admitted {
|
||||
break
|
||||
}
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
@@ -585,9 +608,6 @@ func (s *Store) tryTaskAdmissionAtomicBatchOnce(
|
||||
if outcome.NotifyTaskID != "" {
|
||||
notify = true
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return outcomes, err
|
||||
@@ -713,6 +733,11 @@ SET platform_id = $2::uuid,
|
||||
wait_deadline_at = LEAST(wait_deadline_at, $6),
|
||||
scope_snapshot = $7::jsonb,
|
||||
reselect_requested_at = NULL,
|
||||
dispatch_next_at = now(),
|
||||
dispatch_failure_count = 0,
|
||||
dispatch_last_outcome = 'candidate_migrated',
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE task_id = $1::uuid
|
||||
AND status = 'waiting'
|
||||
@@ -720,7 +745,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
input.TaskID,
|
||||
input.PlatformID,
|
||||
input.PlatformModelID,
|
||||
@@ -779,13 +806,20 @@ SET status = 'waiting',
|
||||
waiter_id = NULLIF($7, ''),
|
||||
waiter_lease_expires_at = $8,
|
||||
scope_snapshot = $9::jsonb,
|
||||
dispatch_next_at = now(),
|
||||
dispatch_failure_count = 0,
|
||||
dispatch_last_outcome = NULL,
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE task_id = $1::uuid
|
||||
RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
input.TaskID,
|
||||
input.PlatformID,
|
||||
input.PlatformModelID,
|
||||
@@ -915,6 +949,7 @@ LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||
WHERE admission.mode = 'async'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.dispatch_next_at <= now()
|
||||
AND (
|
||||
admission.status = 'waiting'
|
||||
OR (
|
||||
@@ -964,13 +999,16 @@ SELECT admission.task_id::text, admission.platform_id::text, admission.platform_
|
||||
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
|
||||
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
|
||||
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
|
||||
admission.scope_snapshot, admission.reselect_requested_at
|
||||
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
|
||||
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
|
||||
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at
|
||||
FROM gateway_task_admissions admission
|
||||
JOIN gateway_tasks task ON task.id = admission.task_id
|
||||
LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||
WHERE admission.mode = 'async'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.dispatch_next_at <= now()
|
||||
AND (
|
||||
admission.status = 'waiting'
|
||||
OR (
|
||||
@@ -1006,27 +1044,105 @@ LIMIT $1`, limit)
|
||||
return admissions, rows.Err()
|
||||
}
|
||||
|
||||
// RequestWaitingTaskAdmissionReselect marks every queued task bound to a
|
||||
// saturated platform model so the dispatcher can route the next batch to a
|
||||
// different eligible candidate. The durable marker lets multiple dispatchers
|
||||
// observe the same decision without assigning tasks to a specific Worker.
|
||||
func (s *Store) RequestWaitingTaskAdmissionReselect(ctx context.Context, platformModelID string) (int64, error) {
|
||||
result, err := s.pool.Exec(ctx, `
|
||||
// RequestTaskAdmissionReselect marks one waiting task for candidate
|
||||
// reselection. The expected platform-model binding makes the operation a CAS:
|
||||
// a concurrent dispatcher that already migrated or admitted the task wins.
|
||||
func (s *Store) RequestTaskAdmissionReselect(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
expectedPlatformModelID string,
|
||||
) (TaskAdmission, bool, error) {
|
||||
row := s.pool.QueryRow(ctx, `
|
||||
UPDATE gateway_task_admissions admission
|
||||
SET reselect_requested_at = now(),
|
||||
dispatch_next_at = now(),
|
||||
dispatch_last_outcome = 'reselect',
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE admission.task_id = task.id
|
||||
AND admission.platform_model_id = $1::uuid
|
||||
WHERE admission.task_id = $1::uuid
|
||||
AND admission.task_id = task.id
|
||||
AND admission.platform_model_id = $2::uuid
|
||||
AND admission.mode = 'async'
|
||||
AND admission.status = 'waiting'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.reselect_requested_at IS NULL`, platformModelID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
AND admission.reselect_requested_at IS NULL
|
||||
RETURNING admission.task_id::text, admission.platform_id::text, admission.platform_model_id::text,
|
||||
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
|
||||
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
|
||||
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
|
||||
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
|
||||
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
|
||||
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at`,
|
||||
taskID,
|
||||
expectedPlatformModelID,
|
||||
)
|
||||
admission, err := scanTaskAdmission(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return TaskAdmission{}, false, nil
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
if err != nil {
|
||||
return TaskAdmission{}, false, err
|
||||
}
|
||||
s.notifyTaskAdmissionBestEffort(ctx, taskID)
|
||||
return admission, true, nil
|
||||
}
|
||||
|
||||
// DeferWaitingTaskAdmission quarantines one task-specific dispatcher failure
|
||||
// until retryAt. Deferred tasks are omitted from FIFO head selection so a
|
||||
// corrupt or temporarily unroutable row cannot strand healthy followers.
|
||||
func (s *Store) DeferWaitingTaskAdmission(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
expectedPlatformModelID string,
|
||||
retryAt time.Time,
|
||||
outcome string,
|
||||
errorCode string,
|
||||
) (TaskAdmission, bool, error) {
|
||||
if retryAt.Before(time.Now().Add(time.Second)) {
|
||||
retryAt = time.Now().Add(time.Second)
|
||||
}
|
||||
row := s.pool.QueryRow(ctx, `
|
||||
UPDATE gateway_task_admissions admission
|
||||
SET dispatch_next_at = $3,
|
||||
dispatch_failure_count = dispatch_failure_count + 1,
|
||||
dispatch_last_outcome = NULLIF($4, ''),
|
||||
dispatch_last_error_code = NULLIF($5, ''),
|
||||
dispatch_last_error_at = now(),
|
||||
updated_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE admission.task_id = $1::uuid
|
||||
AND admission.task_id = task.id
|
||||
AND (NULLIF($2, '') IS NULL OR admission.platform_model_id = NULLIF($2, '')::uuid)
|
||||
AND admission.mode = 'async'
|
||||
AND admission.status = 'waiting'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.dispatch_next_at <= now()
|
||||
RETURNING admission.task_id::text, admission.platform_id::text, admission.platform_model_id::text,
|
||||
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
|
||||
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
|
||||
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
|
||||
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
|
||||
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
|
||||
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at`,
|
||||
taskID,
|
||||
strings.TrimSpace(expectedPlatformModelID),
|
||||
retryAt,
|
||||
strings.TrimSpace(outcome),
|
||||
strings.TrimSpace(errorCode),
|
||||
)
|
||||
admission, err := scanTaskAdmission(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return TaskAdmission{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return TaskAdmission{}, false, err
|
||||
}
|
||||
s.notifyTaskAdmissionBestEffort(ctx, taskID)
|
||||
return admission, true, nil
|
||||
}
|
||||
|
||||
// ListWaitingTaskAdmissionIDs returns one FIFO leader for every independent
|
||||
@@ -1094,7 +1210,16 @@ func (s *Store) TaskAdmissionMetrics(ctx context.Context) (TaskAdmissionMetricsS
|
||||
SELECT COUNT(*) FILTER (WHERE status = 'waiting')::int,
|
||||
COUNT(*) FILTER (WHERE status = 'waiting' AND mode = 'sync')::int,
|
||||
COUNT(*) FILTER (WHERE status = 'waiting' AND mode = 'async')::int,
|
||||
COUNT(*) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'async' AND dispatch_next_at > now()
|
||||
)::int,
|
||||
COUNT(*) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'async' AND reselect_requested_at IS NOT NULL
|
||||
)::int,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - MIN(enqueued_at) FILTER (WHERE status = 'waiting')), 0)::float8,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - MIN(enqueued_at) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'async' AND dispatch_next_at <= now()
|
||||
)), 0)::float8,
|
||||
COUNT(*) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'sync' AND waiter_lease_expires_at <= now()
|
||||
)::int,
|
||||
@@ -1105,17 +1230,28 @@ FROM gateway_task_admissions`).Scan(
|
||||
&snapshot.QueueDepth,
|
||||
&snapshot.WaitingSync,
|
||||
&snapshot.WaitingAsync,
|
||||
&snapshot.DeferredAsync,
|
||||
&snapshot.PendingReselect,
|
||||
&snapshot.OldestWaitSeconds,
|
||||
&snapshot.OldestDispatchableWaitSeconds,
|
||||
&snapshot.ExpiredWaiterBacklog,
|
||||
&snapshot.ExpiredDeadlineBacklog,
|
||||
)
|
||||
return snapshot, err
|
||||
}
|
||||
|
||||
func (s *Store) ReapExpiredTaskAdmissions(ctx context.Context, limit int) (TaskAdmissionReapResult, error) {
|
||||
func (s *Store) ReapExpiredTaskAdmissions(
|
||||
ctx context.Context,
|
||||
limit int,
|
||||
defaultCallbackURLs ...string,
|
||||
) (TaskAdmissionReapResult, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 500
|
||||
}
|
||||
defaultCallbackURL := ""
|
||||
if len(defaultCallbackURLs) > 0 {
|
||||
defaultCallbackURL = strings.TrimSpace(defaultCallbackURLs[0])
|
||||
}
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
@@ -1155,6 +1291,7 @@ LIMIT $1`, limit)
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
var deadlineExpired bool
|
||||
var dispatchFailureCount int
|
||||
err := tx.QueryRow(ctx, `
|
||||
DELETE FROM gateway_task_admissions
|
||||
WHERE task_id = $1::uuid
|
||||
@@ -1163,7 +1300,7 @@ WHERE task_id = $1::uuid
|
||||
wait_deadline_at <= now()
|
||||
OR (mode = 'sync' AND waiter_lease_expires_at <= now())
|
||||
)
|
||||
RETURNING wait_deadline_at <= now()`, taskID).Scan(&deadlineExpired)
|
||||
RETURNING wait_deadline_at <= now(), dispatch_failure_count`, taskID).Scan(&deadlineExpired, &dispatchFailureCount)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
@@ -1177,22 +1314,112 @@ RETURNING wait_deadline_at <= now()`, taskID).Scan(&deadlineExpired)
|
||||
code = "queue_timeout"
|
||||
message = ErrQueueTimeout.Error()
|
||||
status = "failed"
|
||||
if dispatchFailureCount > 0 {
|
||||
code = "admission_dispatch_failed"
|
||||
message = "task admission dispatcher could not prepare the task before its queue deadline"
|
||||
}
|
||||
result.ExpiredDeadlines++
|
||||
} else {
|
||||
result.ExpiredWaiters++
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = $2,
|
||||
error = $3,
|
||||
error = NULL,
|
||||
error_code = $4,
|
||||
error_message = $3,
|
||||
public_error = $5::jsonb,
|
||||
billing_status = CASE
|
||||
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
|
||||
WHEN reservation_amount > 0 THEN 'pending'
|
||||
ELSE 'released'
|
||||
END,
|
||||
billing_updated_at = now(),
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'queued'`, taskID, status, message, code); err != nil {
|
||||
AND status = 'queued'`, taskID, status, message, code, string(publicErrorJSON))
|
||||
if err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
if _, err := releaseRuntimeRateReservationsTx(ctx, tx, []string{taskID}); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
payloadJSON, _ := json.Marshal(map[string]any{"taskId": taskID, "reason": code})
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
|
||||
)
|
||||
SELECT id, 'task.billing.release', 'release', reservation_amount, billing_currency,
|
||||
pricing_snapshot, $2::jsonb, 'pending', now()
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
AND run_mode IN ('production', 'acceptance', 'acceptance_canary')
|
||||
AND gateway_user_id IS NOT NULL
|
||||
AND reservation_amount > 0
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
eventType := "task.cancelled"
|
||||
if status == "failed" {
|
||||
eventType = "task.failed"
|
||||
}
|
||||
var eventID string
|
||||
var eventSeq int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH next_seq AS (
|
||||
SELECT COALESCE(MAX(seq), 0) + 1 AS seq
|
||||
FROM gateway_task_events
|
||||
WHERE task_id = $1::uuid
|
||||
)
|
||||
INSERT INTO gateway_task_events (
|
||||
task_id, seq, event_type, status, phase, progress, message, payload, simulated
|
||||
)
|
||||
SELECT $1::uuid, next_seq.seq, $2, $3, 'admission', 0, $4, '{}'::jsonb,
|
||||
task.run_mode = 'simulation'
|
||||
FROM next_seq
|
||||
JOIN gateway_tasks task ON task.id = $1::uuid
|
||||
RETURNING id::text, seq`, taskID, eventType, status, message).Scan(&eventID, &eventSeq); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
var acceptanceCallbackURL string
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE((
|
||||
SELECT run.callback_url
|
||||
FROM gateway_tasks task
|
||||
JOIN gateway_acceptance_runs run ON run.id = task.acceptance_run_id
|
||||
WHERE task.id = $1::uuid
|
||||
), '')`, taskID).Scan(&acceptanceCallbackURL); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
callbackURL := defaultCallbackURL
|
||||
if strings.TrimSpace(acceptanceCallbackURL) != "" {
|
||||
callbackURL = strings.TrimSpace(acceptanceCallbackURL)
|
||||
}
|
||||
if callbackURL != "" {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
|
||||
ON CONFLICT (task_id, seq, callback_url) DO NOTHING`, taskID, eventID, eventSeq, callbackURL); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
}
|
||||
reaped++
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
@@ -1448,7 +1675,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
input.TaskID, input.PlatformID, input.PlatformModelID, input.UserGroupID,
|
||||
input.QueueKey, input.Mode, input.Priority, deadline, waiterID, waiterLease, scopeSnapshot)
|
||||
return scanTaskAdmission(row)
|
||||
@@ -1467,7 +1696,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
taskID, waiterID, admissionWaiterLeaseTTL.String())
|
||||
return scanTaskAdmission(row)
|
||||
}
|
||||
@@ -1478,13 +1709,19 @@ UPDATE gateway_task_admissions
|
||||
SET status = 'admitted',
|
||||
admitted_at = now(),
|
||||
waiter_lease_expires_at = NULL,
|
||||
dispatch_failure_count = 0,
|
||||
dispatch_last_outcome = 'admitted',
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE task_id = $1::uuid AND status = 'waiting'
|
||||
RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
taskID)
|
||||
return scanTaskAdmission(row)
|
||||
}
|
||||
@@ -1503,7 +1740,9 @@ SELECT task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at
|
||||
FROM gateway_task_admissions
|
||||
WHERE task_id = $1::uuid`, taskID)
|
||||
admission, err := scanTaskAdmission(row)
|
||||
@@ -1518,6 +1757,7 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
|
||||
var admittedAt sql.NullTime
|
||||
var waiterLeaseExpires sql.NullTime
|
||||
var reselectRequestedAt sql.NullTime
|
||||
var dispatchLastErrorAt sql.NullTime
|
||||
var scopeSnapshot []byte
|
||||
err := row.Scan(
|
||||
&admission.TaskID,
|
||||
@@ -1535,6 +1775,11 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
|
||||
&waiterLeaseExpires,
|
||||
&scopeSnapshot,
|
||||
&reselectRequestedAt,
|
||||
&admission.DispatchNextAt,
|
||||
&admission.DispatchFailureCount,
|
||||
&admission.DispatchLastOutcome,
|
||||
&admission.DispatchLastErrorCode,
|
||||
&dispatchLastErrorAt,
|
||||
)
|
||||
if admittedAt.Valid {
|
||||
admission.AdmittedAt = admittedAt.Time
|
||||
@@ -1545,15 +1790,18 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
|
||||
if reselectRequestedAt.Valid {
|
||||
admission.ReselectRequestedAt = reselectRequestedAt.Time
|
||||
}
|
||||
if dispatchLastErrorAt.Valid {
|
||||
admission.DispatchLastErrorAt = dispatchLastErrorAt.Time
|
||||
}
|
||||
if err == nil && len(scopeSnapshot) > 0 {
|
||||
if unmarshalErr := json.Unmarshal(scopeSnapshot, &admission.Scopes); unmarshalErr != nil {
|
||||
return TaskAdmission{}, fmt.Errorf("unmarshal task admission scopes: %w", unmarshalErr)
|
||||
admission.SnapshotInvalid = true
|
||||
}
|
||||
}
|
||||
return admission, err
|
||||
}
|
||||
|
||||
func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmission, scopes []AdmissionScope) (bool, error) {
|
||||
func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmission, scopes []AdmissionScope) (bool, []TaskAdmissionBlocker, error) {
|
||||
for _, scope := range scopes {
|
||||
var head string
|
||||
switch scope.ScopeType {
|
||||
@@ -1561,10 +1809,12 @@ func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmissi
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT task_id::text
|
||||
FROM gateway_task_admissions
|
||||
WHERE platform_model_id = $1::uuid AND status = 'waiting'
|
||||
WHERE platform_model_id = $1::uuid
|
||||
AND status = 'waiting'
|
||||
AND (mode <> 'async' OR dispatch_next_at <= statement_timestamp())
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
LIMIT 1`, admission.PlatformModelID).Scan(&head); err != nil {
|
||||
return false, err
|
||||
return false, nil, err
|
||||
}
|
||||
case "user_group":
|
||||
if admission.UserGroupID == "" {
|
||||
@@ -1573,19 +1823,25 @@ LIMIT 1`, admission.PlatformModelID).Scan(&head); err != nil {
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT task_id::text
|
||||
FROM gateway_task_admissions
|
||||
WHERE user_group_id = $1::uuid AND status = 'waiting'
|
||||
WHERE user_group_id = $1::uuid
|
||||
AND status = 'waiting'
|
||||
AND (mode <> 'async' OR dispatch_next_at <= statement_timestamp())
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
LIMIT 1`, admission.UserGroupID).Scan(&head); err != nil {
|
||||
return false, err
|
||||
return false, nil, err
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if head != admission.TaskID {
|
||||
return false, nil
|
||||
return false, []TaskAdmissionBlocker{{
|
||||
Reason: "not_queue_head",
|
||||
ScopeType: scope.ScopeType,
|
||||
ScopeKey: scope.ScopeKey,
|
||||
}}, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
return true, nil, nil
|
||||
}
|
||||
|
||||
func expireTaskAdmissionTx(ctx context.Context, tx pgx.Tx, taskID string) error {
|
||||
|
||||
Reference in New Issue
Block a user