fix(queue): 隔离异步准入故障并持久化退避

将 X-Async 调度改为逐任务阻塞协议,区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障,避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈,并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证:API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过;pnpm audit 命中未改动的 Nx 工具链既有漏洞。
This commit is contained in:
2026-08-04 18:23:35 +08:00
parent 44d5cf2b9d
commit f4214dd489
12 changed files with 1654 additions and 133 deletions
+321 -65
View File
@@ -65,21 +65,34 @@ type TaskAdmissionInput struct {
}
type TaskAdmission struct {
TaskID string
PlatformID string
PlatformModelID string
UserGroupID string
QueueKey string
Mode string
Status string
Priority int
EnqueuedAt time.Time
WaitDeadlineAt time.Time
AdmittedAt time.Time
WaiterID string
WaiterLeaseExpires time.Time
Scopes []AdmissionScope
ReselectRequestedAt time.Time
TaskID string
PlatformID string
PlatformModelID string
UserGroupID string
QueueKey string
Mode string
Status string
Priority int
EnqueuedAt time.Time
WaitDeadlineAt time.Time
AdmittedAt time.Time
WaiterID string
WaiterLeaseExpires time.Time
Scopes []AdmissionScope
ReselectRequestedAt time.Time
DispatchNextAt time.Time
DispatchFailureCount int
DispatchLastOutcome string
DispatchLastErrorCode string
DispatchLastErrorAt time.Time
SnapshotInvalid bool
}
type TaskAdmissionBlocker struct {
Reason string
ScopeType string
ScopeKey string
RetryAt time.Time
}
type TaskAdmissionResult struct {
@@ -87,6 +100,7 @@ type TaskAdmissionResult struct {
Admitted bool
NewlyAdmitted bool
Leases []ConcurrencyLease
Blockers []TaskAdmissionBlocker
}
type TaskAdmissionBatchOutcome struct {
@@ -96,12 +110,15 @@ type TaskAdmissionBatchOutcome struct {
}
type TaskAdmissionMetricsSnapshot struct {
QueueDepth int
WaitingSync int
WaitingAsync int
OldestWaitSeconds float64
ExpiredWaiterBacklog int
ExpiredDeadlineBacklog int
QueueDepth int
WaitingSync int
WaitingAsync int
DeferredAsync int
PendingReselect int
OldestWaitSeconds float64
OldestDispatchableWaitSeconds float64
ExpiredWaiterBacklog int
ExpiredDeadlineBacklog int
}
type TaskAdmissionReapResult struct {
@@ -347,7 +364,7 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
}
return taskAdmissionTxOutcome{Result: result}, nil
}
if bindingChanged {
if bindingChanged || activeLeases == 0 {
targetStates, stateErr := admissionScopeStatesTx(ctx, tx, scopes)
if stateErr != nil {
return taskAdmissionTxOutcome{}, stateErr
@@ -392,22 +409,31 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
}
}
head, err := isTaskAdmissionHeadTx(ctx, tx, admission, scopes)
head, headBlockers, err := isTaskAdmissionHeadTx(ctx, tx, admission, scopes)
if err != nil {
return taskAdmissionTxOutcome{}, err
}
if !head {
return taskAdmissionTxOutcome{
Result: TaskAdmissionResult{Admission: admission},
Result: TaskAdmissionResult{Admission: admission, Blockers: headBlockers},
}, nil
}
blockers := make([]TaskAdmissionBlocker, 0, len(scopeStates))
for _, state := range scopeStates {
if state.Saturated {
return taskAdmissionTxOutcome{
Result: TaskAdmissionResult{Admission: admission},
}, nil
blockers = append(blockers, TaskAdmissionBlocker{
Reason: "saturated",
ScopeType: state.Scope.ScopeType,
ScopeKey: state.Scope.ScopeKey,
RetryAt: state.NextLeaseExpiration,
})
}
}
if len(blockers) > 0 {
return taskAdmissionTxOutcome{
Result: TaskAdmissionResult{Admission: admission, Blockers: blockers},
}, nil
}
leases := make([]ConcurrencyLease, 0, len(scopes))
for _, scope := range scopes {
@@ -498,9 +524,6 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
if admissionErr != nil {
return outcomes, admissionErr
}
if !result.Admitted {
break
}
}
return outcomes, nil
}
@@ -585,9 +608,6 @@ func (s *Store) tryTaskAdmissionAtomicBatchOnce(
if outcome.NotifyTaskID != "" {
notify = true
}
if !outcome.Result.Admitted {
break
}
}
if err := tx.Commit(ctx); err != nil {
return outcomes, err
@@ -713,6 +733,11 @@ SET platform_id = $2::uuid,
wait_deadline_at = LEAST(wait_deadline_at, $6),
scope_snapshot = $7::jsonb,
reselect_requested_at = NULL,
dispatch_next_at = now(),
dispatch_failure_count = 0,
dispatch_last_outcome = 'candidate_migrated',
dispatch_last_error_code = NULL,
dispatch_last_error_at = NULL,
updated_at = now()
WHERE task_id = $1::uuid
AND status = 'waiting'
@@ -720,7 +745,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
enqueued_at, wait_deadline_at, admitted_at,
COALESCE(waiter_id, ''), waiter_lease_expires_at,
scope_snapshot, reselect_requested_at`,
scope_snapshot, reselect_requested_at, dispatch_next_at,
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
input.TaskID,
input.PlatformID,
input.PlatformModelID,
@@ -779,13 +806,20 @@ SET status = 'waiting',
waiter_id = NULLIF($7, ''),
waiter_lease_expires_at = $8,
scope_snapshot = $9::jsonb,
dispatch_next_at = now(),
dispatch_failure_count = 0,
dispatch_last_outcome = NULL,
dispatch_last_error_code = NULL,
dispatch_last_error_at = NULL,
updated_at = now()
WHERE task_id = $1::uuid
RETURNING task_id::text, platform_id::text, platform_model_id::text,
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
enqueued_at, wait_deadline_at, admitted_at,
COALESCE(waiter_id, ''), waiter_lease_expires_at,
scope_snapshot, reselect_requested_at`,
scope_snapshot, reselect_requested_at, dispatch_next_at,
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
input.TaskID,
input.PlatformID,
input.PlatformModelID,
@@ -915,6 +949,7 @@ LEFT JOIN river_job job ON job.id = task.river_job_id
WHERE admission.mode = 'async'
AND task.status = 'queued'
AND task.next_run_at <= now()
AND admission.dispatch_next_at <= now()
AND (
admission.status = 'waiting'
OR (
@@ -964,13 +999,16 @@ SELECT admission.task_id::text, admission.platform_id::text, admission.platform_
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
admission.scope_snapshot, admission.reselect_requested_at
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at
FROM gateway_task_admissions admission
JOIN gateway_tasks task ON task.id = admission.task_id
LEFT JOIN river_job job ON job.id = task.river_job_id
WHERE admission.mode = 'async'
AND task.status = 'queued'
AND task.next_run_at <= now()
AND admission.dispatch_next_at <= now()
AND (
admission.status = 'waiting'
OR (
@@ -1006,27 +1044,105 @@ LIMIT $1`, limit)
return admissions, rows.Err()
}
// RequestWaitingTaskAdmissionReselect marks every queued task bound to a
// saturated platform model so the dispatcher can route the next batch to a
// different eligible candidate. The durable marker lets multiple dispatchers
// observe the same decision without assigning tasks to a specific Worker.
func (s *Store) RequestWaitingTaskAdmissionReselect(ctx context.Context, platformModelID string) (int64, error) {
result, err := s.pool.Exec(ctx, `
// RequestTaskAdmissionReselect marks one waiting task for candidate
// reselection. The expected platform-model binding makes the operation a CAS:
// a concurrent dispatcher that already migrated or admitted the task wins.
func (s *Store) RequestTaskAdmissionReselect(
ctx context.Context,
taskID string,
expectedPlatformModelID string,
) (TaskAdmission, bool, error) {
row := s.pool.QueryRow(ctx, `
UPDATE gateway_task_admissions admission
SET reselect_requested_at = now(),
dispatch_next_at = now(),
dispatch_last_outcome = 'reselect',
dispatch_last_error_code = NULL,
dispatch_last_error_at = NULL,
updated_at = now()
FROM gateway_tasks task
WHERE admission.task_id = task.id
AND admission.platform_model_id = $1::uuid
WHERE admission.task_id = $1::uuid
AND admission.task_id = task.id
AND admission.platform_model_id = $2::uuid
AND admission.mode = 'async'
AND admission.status = 'waiting'
AND task.status = 'queued'
AND task.next_run_at <= now()
AND admission.reselect_requested_at IS NULL`, platformModelID)
if err != nil {
return 0, err
AND admission.reselect_requested_at IS NULL
RETURNING admission.task_id::text, admission.platform_id::text, admission.platform_model_id::text,
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at`,
taskID,
expectedPlatformModelID,
)
admission, err := scanTaskAdmission(row)
if errors.Is(err, pgx.ErrNoRows) {
return TaskAdmission{}, false, nil
}
return result.RowsAffected(), nil
if err != nil {
return TaskAdmission{}, false, err
}
s.notifyTaskAdmissionBestEffort(ctx, taskID)
return admission, true, nil
}
// DeferWaitingTaskAdmission quarantines one task-specific dispatcher failure
// until retryAt. Deferred tasks are omitted from FIFO head selection so a
// corrupt or temporarily unroutable row cannot strand healthy followers.
func (s *Store) DeferWaitingTaskAdmission(
ctx context.Context,
taskID string,
expectedPlatformModelID string,
retryAt time.Time,
outcome string,
errorCode string,
) (TaskAdmission, bool, error) {
if retryAt.Before(time.Now().Add(time.Second)) {
retryAt = time.Now().Add(time.Second)
}
row := s.pool.QueryRow(ctx, `
UPDATE gateway_task_admissions admission
SET dispatch_next_at = $3,
dispatch_failure_count = dispatch_failure_count + 1,
dispatch_last_outcome = NULLIF($4, ''),
dispatch_last_error_code = NULLIF($5, ''),
dispatch_last_error_at = now(),
updated_at = now()
FROM gateway_tasks task
WHERE admission.task_id = $1::uuid
AND admission.task_id = task.id
AND (NULLIF($2, '') IS NULL OR admission.platform_model_id = NULLIF($2, '')::uuid)
AND admission.mode = 'async'
AND admission.status = 'waiting'
AND task.status = 'queued'
AND task.next_run_at <= now()
AND admission.dispatch_next_at <= now()
RETURNING admission.task_id::text, admission.platform_id::text, admission.platform_model_id::text,
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at`,
taskID,
strings.TrimSpace(expectedPlatformModelID),
retryAt,
strings.TrimSpace(outcome),
strings.TrimSpace(errorCode),
)
admission, err := scanTaskAdmission(row)
if errors.Is(err, pgx.ErrNoRows) {
return TaskAdmission{}, false, nil
}
if err != nil {
return TaskAdmission{}, false, err
}
s.notifyTaskAdmissionBestEffort(ctx, taskID)
return admission, true, nil
}
// ListWaitingTaskAdmissionIDs returns one FIFO leader for every independent
@@ -1094,7 +1210,16 @@ func (s *Store) TaskAdmissionMetrics(ctx context.Context) (TaskAdmissionMetricsS
SELECT COUNT(*) FILTER (WHERE status = 'waiting')::int,
COUNT(*) FILTER (WHERE status = 'waiting' AND mode = 'sync')::int,
COUNT(*) FILTER (WHERE status = 'waiting' AND mode = 'async')::int,
COUNT(*) FILTER (
WHERE status = 'waiting' AND mode = 'async' AND dispatch_next_at > now()
)::int,
COUNT(*) FILTER (
WHERE status = 'waiting' AND mode = 'async' AND reselect_requested_at IS NOT NULL
)::int,
COALESCE(EXTRACT(EPOCH FROM now() - MIN(enqueued_at) FILTER (WHERE status = 'waiting')), 0)::float8,
COALESCE(EXTRACT(EPOCH FROM now() - MIN(enqueued_at) FILTER (
WHERE status = 'waiting' AND mode = 'async' AND dispatch_next_at <= now()
)), 0)::float8,
COUNT(*) FILTER (
WHERE status = 'waiting' AND mode = 'sync' AND waiter_lease_expires_at <= now()
)::int,
@@ -1105,17 +1230,28 @@ FROM gateway_task_admissions`).Scan(
&snapshot.QueueDepth,
&snapshot.WaitingSync,
&snapshot.WaitingAsync,
&snapshot.DeferredAsync,
&snapshot.PendingReselect,
&snapshot.OldestWaitSeconds,
&snapshot.OldestDispatchableWaitSeconds,
&snapshot.ExpiredWaiterBacklog,
&snapshot.ExpiredDeadlineBacklog,
)
return snapshot, err
}
func (s *Store) ReapExpiredTaskAdmissions(ctx context.Context, limit int) (TaskAdmissionReapResult, error) {
func (s *Store) ReapExpiredTaskAdmissions(
ctx context.Context,
limit int,
defaultCallbackURLs ...string,
) (TaskAdmissionReapResult, error) {
if limit <= 0 || limit > 1000 {
limit = 500
}
defaultCallbackURL := ""
if len(defaultCallbackURLs) > 0 {
defaultCallbackURL = strings.TrimSpace(defaultCallbackURLs[0])
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return TaskAdmissionReapResult{}, err
@@ -1155,6 +1291,7 @@ LIMIT $1`, limit)
return TaskAdmissionReapResult{}, err
}
var deadlineExpired bool
var dispatchFailureCount int
err := tx.QueryRow(ctx, `
DELETE FROM gateway_task_admissions
WHERE task_id = $1::uuid
@@ -1163,7 +1300,7 @@ WHERE task_id = $1::uuid
wait_deadline_at <= now()
OR (mode = 'sync' AND waiter_lease_expires_at <= now())
)
RETURNING wait_deadline_at <= now()`, taskID).Scan(&deadlineExpired)
RETURNING wait_deadline_at <= now(), dispatch_failure_count`, taskID).Scan(&deadlineExpired, &dispatchFailureCount)
if errors.Is(err, pgx.ErrNoRows) {
continue
}
@@ -1177,22 +1314,112 @@ RETURNING wait_deadline_at <= now()`, taskID).Scan(&deadlineExpired)
code = "queue_timeout"
message = ErrQueueTimeout.Error()
status = "failed"
if dispatchFailureCount > 0 {
code = "admission_dispatch_failed"
message = "task admission dispatcher could not prepare the task before its queue deadline"
}
result.ExpiredDeadlines++
} else {
result.ExpiredWaiters++
}
if _, err := tx.Exec(ctx, `
publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET status = $2,
error = $3,
error = NULL,
error_code = $4,
error_message = $3,
public_error = $5::jsonb,
billing_status = CASE
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
WHEN reservation_amount > 0 THEN 'pending'
ELSE 'released'
END,
billing_updated_at = now(),
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
finished_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND status = 'queued'`, taskID, status, message, code); err != nil {
AND status = 'queued'`, taskID, status, message, code, string(publicErrorJSON))
if err != nil {
return TaskAdmissionReapResult{}, err
}
if tag.RowsAffected() == 0 {
continue
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_concurrency_leases
SET released_at = statement_timestamp()
WHERE task_id = $1::uuid
AND released_at IS NULL`, taskID); err != nil {
return TaskAdmissionReapResult{}, err
}
if _, err := releaseRuntimeRateReservationsTx(ctx, tx, []string{taskID}); err != nil {
return TaskAdmissionReapResult{}, err
}
payloadJSON, _ := json.Marshal(map[string]any{"taskId": taskID, "reason": code})
if _, err := tx.Exec(ctx, `
INSERT INTO settlement_outbox (
task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
)
SELECT id, 'task.billing.release', 'release', reservation_amount, billing_currency,
pricing_snapshot, $2::jsonb, 'pending', now()
FROM gateway_tasks
WHERE id = $1::uuid
AND run_mode IN ('production', 'acceptance', 'acceptance_canary')
AND gateway_user_id IS NOT NULL
AND reservation_amount > 0
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
return TaskAdmissionReapResult{}, err
}
eventType := "task.cancelled"
if status == "failed" {
eventType = "task.failed"
}
var eventID string
var eventSeq int64
if err := tx.QueryRow(ctx, `
WITH next_seq AS (
SELECT COALESCE(MAX(seq), 0) + 1 AS seq
FROM gateway_task_events
WHERE task_id = $1::uuid
)
INSERT INTO gateway_task_events (
task_id, seq, event_type, status, phase, progress, message, payload, simulated
)
SELECT $1::uuid, next_seq.seq, $2, $3, 'admission', 0, $4, '{}'::jsonb,
task.run_mode = 'simulation'
FROM next_seq
JOIN gateway_tasks task ON task.id = $1::uuid
RETURNING id::text, seq`, taskID, eventType, status, message).Scan(&eventID, &eventSeq); err != nil {
return TaskAdmissionReapResult{}, err
}
var acceptanceCallbackURL string
if err := tx.QueryRow(ctx, `
SELECT COALESCE((
SELECT run.callback_url
FROM gateway_tasks task
JOIN gateway_acceptance_runs run ON run.id = task.acceptance_run_id
WHERE task.id = $1::uuid
), '')`, taskID).Scan(&acceptanceCallbackURL); err != nil {
return TaskAdmissionReapResult{}, err
}
callbackURL := defaultCallbackURL
if strings.TrimSpace(acceptanceCallbackURL) != "" {
callbackURL = strings.TrimSpace(acceptanceCallbackURL)
}
if callbackURL != "" {
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
ON CONFLICT (task_id, seq, callback_url) DO NOTHING`, taskID, eventID, eventSeq, callbackURL); err != nil {
return TaskAdmissionReapResult{}, err
}
}
reaped++
}
if err := tx.Commit(ctx); err != nil {
@@ -1448,7 +1675,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
enqueued_at, wait_deadline_at, admitted_at,
COALESCE(waiter_id, ''), waiter_lease_expires_at,
scope_snapshot, reselect_requested_at`,
scope_snapshot, reselect_requested_at, dispatch_next_at,
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
input.TaskID, input.PlatformID, input.PlatformModelID, input.UserGroupID,
input.QueueKey, input.Mode, input.Priority, deadline, waiterID, waiterLease, scopeSnapshot)
return scanTaskAdmission(row)
@@ -1467,7 +1696,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
enqueued_at, wait_deadline_at, admitted_at,
COALESCE(waiter_id, ''), waiter_lease_expires_at,
scope_snapshot, reselect_requested_at`,
scope_snapshot, reselect_requested_at, dispatch_next_at,
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
taskID, waiterID, admissionWaiterLeaseTTL.String())
return scanTaskAdmission(row)
}
@@ -1478,13 +1709,19 @@ UPDATE gateway_task_admissions
SET status = 'admitted',
admitted_at = now(),
waiter_lease_expires_at = NULL,
dispatch_failure_count = 0,
dispatch_last_outcome = 'admitted',
dispatch_last_error_code = NULL,
dispatch_last_error_at = NULL,
updated_at = now()
WHERE task_id = $1::uuid AND status = 'waiting'
RETURNING task_id::text, platform_id::text, platform_model_id::text,
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
enqueued_at, wait_deadline_at, admitted_at,
COALESCE(waiter_id, ''), waiter_lease_expires_at,
scope_snapshot, reselect_requested_at`,
scope_snapshot, reselect_requested_at, dispatch_next_at,
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
taskID)
return scanTaskAdmission(row)
}
@@ -1503,7 +1740,9 @@ SELECT task_id::text, platform_id::text, platform_model_id::text,
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
enqueued_at, wait_deadline_at, admitted_at,
COALESCE(waiter_id, ''), waiter_lease_expires_at,
scope_snapshot, reselect_requested_at
scope_snapshot, reselect_requested_at, dispatch_next_at,
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at
FROM gateway_task_admissions
WHERE task_id = $1::uuid`, taskID)
admission, err := scanTaskAdmission(row)
@@ -1518,6 +1757,7 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
var admittedAt sql.NullTime
var waiterLeaseExpires sql.NullTime
var reselectRequestedAt sql.NullTime
var dispatchLastErrorAt sql.NullTime
var scopeSnapshot []byte
err := row.Scan(
&admission.TaskID,
@@ -1535,6 +1775,11 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
&waiterLeaseExpires,
&scopeSnapshot,
&reselectRequestedAt,
&admission.DispatchNextAt,
&admission.DispatchFailureCount,
&admission.DispatchLastOutcome,
&admission.DispatchLastErrorCode,
&dispatchLastErrorAt,
)
if admittedAt.Valid {
admission.AdmittedAt = admittedAt.Time
@@ -1545,15 +1790,18 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
if reselectRequestedAt.Valid {
admission.ReselectRequestedAt = reselectRequestedAt.Time
}
if dispatchLastErrorAt.Valid {
admission.DispatchLastErrorAt = dispatchLastErrorAt.Time
}
if err == nil && len(scopeSnapshot) > 0 {
if unmarshalErr := json.Unmarshal(scopeSnapshot, &admission.Scopes); unmarshalErr != nil {
return TaskAdmission{}, fmt.Errorf("unmarshal task admission scopes: %w", unmarshalErr)
admission.SnapshotInvalid = true
}
}
return admission, err
}
func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmission, scopes []AdmissionScope) (bool, error) {
func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmission, scopes []AdmissionScope) (bool, []TaskAdmissionBlocker, error) {
for _, scope := range scopes {
var head string
switch scope.ScopeType {
@@ -1561,10 +1809,12 @@ func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmissi
if err := tx.QueryRow(ctx, `
SELECT task_id::text
FROM gateway_task_admissions
WHERE platform_model_id = $1::uuid AND status = 'waiting'
WHERE platform_model_id = $1::uuid
AND status = 'waiting'
AND (mode <> 'async' OR dispatch_next_at <= statement_timestamp())
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
LIMIT 1`, admission.PlatformModelID).Scan(&head); err != nil {
return false, err
return false, nil, err
}
case "user_group":
if admission.UserGroupID == "" {
@@ -1573,19 +1823,25 @@ LIMIT 1`, admission.PlatformModelID).Scan(&head); err != nil {
if err := tx.QueryRow(ctx, `
SELECT task_id::text
FROM gateway_task_admissions
WHERE user_group_id = $1::uuid AND status = 'waiting'
WHERE user_group_id = $1::uuid
AND status = 'waiting'
AND (mode <> 'async' OR dispatch_next_at <= statement_timestamp())
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
LIMIT 1`, admission.UserGroupID).Scan(&head); err != nil {
return false, err
return false, nil, err
}
default:
continue
}
if head != admission.TaskID {
return false, nil
return false, []TaskAdmissionBlocker{{
Reason: "not_queue_head",
ScopeType: scope.ScopeType,
ScopeKey: scope.ScopeKey,
}}, nil
}
}
return true, nil
return true, nil, nil
}
func expireTaskAdmissionTx(ctx context.Context, tx pgx.Tx, taskID string) error {