fix(queue): 隔离异步准入故障并持久化退避
将 X-Async 调度改为逐任务阻塞协议，区分全局容量、平台容量、用户组 FIFO、任务级异常和系统级故障，避免单个历史毒任务触发整批回滚。\n\n新增持久化退避、单任务 CAS 重选、幂等终态事务、固定标签指标和损坏快照自愈，并修复 Worker 从 preparing 直接进入 finalizing 时的自等待死锁。\n\n验证：API 全量测试、PostgreSQL 集成场景、race、go vet、govulncheck、pnpm lint/test/build、Compose 与发布脚本测试通过；pnpm audit 命中未改动的 Nx 工具链既有漏洞。
This commit is contained in:
@@ -65,21 +65,34 @@ type TaskAdmissionInput struct {
|
||||
}
|
||||
|
||||
type TaskAdmission struct {
|
||||
TaskID string
|
||||
PlatformID string
|
||||
PlatformModelID string
|
||||
UserGroupID string
|
||||
QueueKey string
|
||||
Mode string
|
||||
Status string
|
||||
Priority int
|
||||
EnqueuedAt time.Time
|
||||
WaitDeadlineAt time.Time
|
||||
AdmittedAt time.Time
|
||||
WaiterID string
|
||||
WaiterLeaseExpires time.Time
|
||||
Scopes []AdmissionScope
|
||||
ReselectRequestedAt time.Time
|
||||
TaskID string
|
||||
PlatformID string
|
||||
PlatformModelID string
|
||||
UserGroupID string
|
||||
QueueKey string
|
||||
Mode string
|
||||
Status string
|
||||
Priority int
|
||||
EnqueuedAt time.Time
|
||||
WaitDeadlineAt time.Time
|
||||
AdmittedAt time.Time
|
||||
WaiterID string
|
||||
WaiterLeaseExpires time.Time
|
||||
Scopes []AdmissionScope
|
||||
ReselectRequestedAt time.Time
|
||||
DispatchNextAt time.Time
|
||||
DispatchFailureCount int
|
||||
DispatchLastOutcome string
|
||||
DispatchLastErrorCode string
|
||||
DispatchLastErrorAt time.Time
|
||||
SnapshotInvalid bool
|
||||
}
|
||||
|
||||
type TaskAdmissionBlocker struct {
|
||||
Reason string
|
||||
ScopeType string
|
||||
ScopeKey string
|
||||
RetryAt time.Time
|
||||
}
|
||||
|
||||
type TaskAdmissionResult struct {
|
||||
@@ -87,6 +100,7 @@ type TaskAdmissionResult struct {
|
||||
Admitted bool
|
||||
NewlyAdmitted bool
|
||||
Leases []ConcurrencyLease
|
||||
Blockers []TaskAdmissionBlocker
|
||||
}
|
||||
|
||||
type TaskAdmissionBatchOutcome struct {
|
||||
@@ -96,12 +110,15 @@ type TaskAdmissionBatchOutcome struct {
|
||||
}
|
||||
|
||||
type TaskAdmissionMetricsSnapshot struct {
|
||||
QueueDepth int
|
||||
WaitingSync int
|
||||
WaitingAsync int
|
||||
OldestWaitSeconds float64
|
||||
ExpiredWaiterBacklog int
|
||||
ExpiredDeadlineBacklog int
|
||||
QueueDepth int
|
||||
WaitingSync int
|
||||
WaitingAsync int
|
||||
DeferredAsync int
|
||||
PendingReselect int
|
||||
OldestWaitSeconds float64
|
||||
OldestDispatchableWaitSeconds float64
|
||||
ExpiredWaiterBacklog int
|
||||
ExpiredDeadlineBacklog int
|
||||
}
|
||||
|
||||
type TaskAdmissionReapResult struct {
|
||||
@@ -347,7 +364,7 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
}
|
||||
return taskAdmissionTxOutcome{Result: result}, nil
|
||||
}
|
||||
if bindingChanged {
|
||||
if bindingChanged || activeLeases == 0 {
|
||||
targetStates, stateErr := admissionScopeStatesTx(ctx, tx, scopes)
|
||||
if stateErr != nil {
|
||||
return taskAdmissionTxOutcome{}, stateErr
|
||||
@@ -392,22 +409,31 @@ WHERE id = $1::uuid`, input.TaskID).Scan(&taskActive); err != nil {
|
||||
}
|
||||
}
|
||||
|
||||
head, err := isTaskAdmissionHeadTx(ctx, tx, admission, scopes)
|
||||
head, headBlockers, err := isTaskAdmissionHeadTx(ctx, tx, admission, scopes)
|
||||
if err != nil {
|
||||
return taskAdmissionTxOutcome{}, err
|
||||
}
|
||||
if !head {
|
||||
return taskAdmissionTxOutcome{
|
||||
Result: TaskAdmissionResult{Admission: admission},
|
||||
Result: TaskAdmissionResult{Admission: admission, Blockers: headBlockers},
|
||||
}, nil
|
||||
}
|
||||
blockers := make([]TaskAdmissionBlocker, 0, len(scopeStates))
|
||||
for _, state := range scopeStates {
|
||||
if state.Saturated {
|
||||
return taskAdmissionTxOutcome{
|
||||
Result: TaskAdmissionResult{Admission: admission},
|
||||
}, nil
|
||||
blockers = append(blockers, TaskAdmissionBlocker{
|
||||
Reason: "saturated",
|
||||
ScopeType: state.Scope.ScopeType,
|
||||
ScopeKey: state.Scope.ScopeKey,
|
||||
RetryAt: state.NextLeaseExpiration,
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(blockers) > 0 {
|
||||
return taskAdmissionTxOutcome{
|
||||
Result: TaskAdmissionResult{Admission: admission, Blockers: blockers},
|
||||
}, nil
|
||||
}
|
||||
|
||||
leases := make([]ConcurrencyLease, 0, len(scopes))
|
||||
for _, scope := range scopes {
|
||||
@@ -498,9 +524,6 @@ func (s *Store) TryTaskAdmissionBatchWithAdmittedHook(
|
||||
if admissionErr != nil {
|
||||
return outcomes, admissionErr
|
||||
}
|
||||
if !result.Admitted {
|
||||
break
|
||||
}
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
@@ -585,9 +608,6 @@ func (s *Store) tryTaskAdmissionAtomicBatchOnce(
|
||||
if outcome.NotifyTaskID != "" {
|
||||
notify = true
|
||||
}
|
||||
if !outcome.Result.Admitted {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return outcomes, err
|
||||
@@ -713,6 +733,11 @@ SET platform_id = $2::uuid,
|
||||
wait_deadline_at = LEAST(wait_deadline_at, $6),
|
||||
scope_snapshot = $7::jsonb,
|
||||
reselect_requested_at = NULL,
|
||||
dispatch_next_at = now(),
|
||||
dispatch_failure_count = 0,
|
||||
dispatch_last_outcome = 'candidate_migrated',
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE task_id = $1::uuid
|
||||
AND status = 'waiting'
|
||||
@@ -720,7 +745,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
input.TaskID,
|
||||
input.PlatformID,
|
||||
input.PlatformModelID,
|
||||
@@ -779,13 +806,20 @@ SET status = 'waiting',
|
||||
waiter_id = NULLIF($7, ''),
|
||||
waiter_lease_expires_at = $8,
|
||||
scope_snapshot = $9::jsonb,
|
||||
dispatch_next_at = now(),
|
||||
dispatch_failure_count = 0,
|
||||
dispatch_last_outcome = NULL,
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE task_id = $1::uuid
|
||||
RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
input.TaskID,
|
||||
input.PlatformID,
|
||||
input.PlatformModelID,
|
||||
@@ -915,6 +949,7 @@ LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||
WHERE admission.mode = 'async'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.dispatch_next_at <= now()
|
||||
AND (
|
||||
admission.status = 'waiting'
|
||||
OR (
|
||||
@@ -964,13 +999,16 @@ SELECT admission.task_id::text, admission.platform_id::text, admission.platform_
|
||||
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
|
||||
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
|
||||
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
|
||||
admission.scope_snapshot, admission.reselect_requested_at
|
||||
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
|
||||
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
|
||||
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at
|
||||
FROM gateway_task_admissions admission
|
||||
JOIN gateway_tasks task ON task.id = admission.task_id
|
||||
LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||
WHERE admission.mode = 'async'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.dispatch_next_at <= now()
|
||||
AND (
|
||||
admission.status = 'waiting'
|
||||
OR (
|
||||
@@ -1006,27 +1044,105 @@ LIMIT $1`, limit)
|
||||
return admissions, rows.Err()
|
||||
}
|
||||
|
||||
// RequestWaitingTaskAdmissionReselect marks every queued task bound to a
|
||||
// saturated platform model so the dispatcher can route the next batch to a
|
||||
// different eligible candidate. The durable marker lets multiple dispatchers
|
||||
// observe the same decision without assigning tasks to a specific Worker.
|
||||
func (s *Store) RequestWaitingTaskAdmissionReselect(ctx context.Context, platformModelID string) (int64, error) {
|
||||
result, err := s.pool.Exec(ctx, `
|
||||
// RequestTaskAdmissionReselect marks one waiting task for candidate
|
||||
// reselection. The expected platform-model binding makes the operation a CAS:
|
||||
// a concurrent dispatcher that already migrated or admitted the task wins.
|
||||
func (s *Store) RequestTaskAdmissionReselect(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
expectedPlatformModelID string,
|
||||
) (TaskAdmission, bool, error) {
|
||||
row := s.pool.QueryRow(ctx, `
|
||||
UPDATE gateway_task_admissions admission
|
||||
SET reselect_requested_at = now(),
|
||||
dispatch_next_at = now(),
|
||||
dispatch_last_outcome = 'reselect',
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE admission.task_id = task.id
|
||||
AND admission.platform_model_id = $1::uuid
|
||||
WHERE admission.task_id = $1::uuid
|
||||
AND admission.task_id = task.id
|
||||
AND admission.platform_model_id = $2::uuid
|
||||
AND admission.mode = 'async'
|
||||
AND admission.status = 'waiting'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.reselect_requested_at IS NULL`, platformModelID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
AND admission.reselect_requested_at IS NULL
|
||||
RETURNING admission.task_id::text, admission.platform_id::text, admission.platform_model_id::text,
|
||||
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
|
||||
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
|
||||
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
|
||||
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
|
||||
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
|
||||
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at`,
|
||||
taskID,
|
||||
expectedPlatformModelID,
|
||||
)
|
||||
admission, err := scanTaskAdmission(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return TaskAdmission{}, false, nil
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
if err != nil {
|
||||
return TaskAdmission{}, false, err
|
||||
}
|
||||
s.notifyTaskAdmissionBestEffort(ctx, taskID)
|
||||
return admission, true, nil
|
||||
}
|
||||
|
||||
// DeferWaitingTaskAdmission quarantines one task-specific dispatcher failure
|
||||
// until retryAt. Deferred tasks are omitted from FIFO head selection so a
|
||||
// corrupt or temporarily unroutable row cannot strand healthy followers.
|
||||
func (s *Store) DeferWaitingTaskAdmission(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
expectedPlatformModelID string,
|
||||
retryAt time.Time,
|
||||
outcome string,
|
||||
errorCode string,
|
||||
) (TaskAdmission, bool, error) {
|
||||
if retryAt.Before(time.Now().Add(time.Second)) {
|
||||
retryAt = time.Now().Add(time.Second)
|
||||
}
|
||||
row := s.pool.QueryRow(ctx, `
|
||||
UPDATE gateway_task_admissions admission
|
||||
SET dispatch_next_at = $3,
|
||||
dispatch_failure_count = dispatch_failure_count + 1,
|
||||
dispatch_last_outcome = NULLIF($4, ''),
|
||||
dispatch_last_error_code = NULLIF($5, ''),
|
||||
dispatch_last_error_at = now(),
|
||||
updated_at = now()
|
||||
FROM gateway_tasks task
|
||||
WHERE admission.task_id = $1::uuid
|
||||
AND admission.task_id = task.id
|
||||
AND (NULLIF($2, '') IS NULL OR admission.platform_model_id = NULLIF($2, '')::uuid)
|
||||
AND admission.mode = 'async'
|
||||
AND admission.status = 'waiting'
|
||||
AND task.status = 'queued'
|
||||
AND task.next_run_at <= now()
|
||||
AND admission.dispatch_next_at <= now()
|
||||
RETURNING admission.task_id::text, admission.platform_id::text, admission.platform_model_id::text,
|
||||
COALESCE(admission.user_group_id::text, ''), admission.queue_key, admission.mode,
|
||||
admission.status, admission.priority, admission.enqueued_at, admission.wait_deadline_at,
|
||||
admission.admitted_at, COALESCE(admission.waiter_id, ''), admission.waiter_lease_expires_at,
|
||||
admission.scope_snapshot, admission.reselect_requested_at, admission.dispatch_next_at,
|
||||
admission.dispatch_failure_count, COALESCE(admission.dispatch_last_outcome, ''),
|
||||
COALESCE(admission.dispatch_last_error_code, ''), admission.dispatch_last_error_at`,
|
||||
taskID,
|
||||
strings.TrimSpace(expectedPlatformModelID),
|
||||
retryAt,
|
||||
strings.TrimSpace(outcome),
|
||||
strings.TrimSpace(errorCode),
|
||||
)
|
||||
admission, err := scanTaskAdmission(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return TaskAdmission{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return TaskAdmission{}, false, err
|
||||
}
|
||||
s.notifyTaskAdmissionBestEffort(ctx, taskID)
|
||||
return admission, true, nil
|
||||
}
|
||||
|
||||
// ListWaitingTaskAdmissionIDs returns one FIFO leader for every independent
|
||||
@@ -1094,7 +1210,16 @@ func (s *Store) TaskAdmissionMetrics(ctx context.Context) (TaskAdmissionMetricsS
|
||||
SELECT COUNT(*) FILTER (WHERE status = 'waiting')::int,
|
||||
COUNT(*) FILTER (WHERE status = 'waiting' AND mode = 'sync')::int,
|
||||
COUNT(*) FILTER (WHERE status = 'waiting' AND mode = 'async')::int,
|
||||
COUNT(*) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'async' AND dispatch_next_at > now()
|
||||
)::int,
|
||||
COUNT(*) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'async' AND reselect_requested_at IS NOT NULL
|
||||
)::int,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - MIN(enqueued_at) FILTER (WHERE status = 'waiting')), 0)::float8,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - MIN(enqueued_at) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'async' AND dispatch_next_at <= now()
|
||||
)), 0)::float8,
|
||||
COUNT(*) FILTER (
|
||||
WHERE status = 'waiting' AND mode = 'sync' AND waiter_lease_expires_at <= now()
|
||||
)::int,
|
||||
@@ -1105,17 +1230,28 @@ FROM gateway_task_admissions`).Scan(
|
||||
&snapshot.QueueDepth,
|
||||
&snapshot.WaitingSync,
|
||||
&snapshot.WaitingAsync,
|
||||
&snapshot.DeferredAsync,
|
||||
&snapshot.PendingReselect,
|
||||
&snapshot.OldestWaitSeconds,
|
||||
&snapshot.OldestDispatchableWaitSeconds,
|
||||
&snapshot.ExpiredWaiterBacklog,
|
||||
&snapshot.ExpiredDeadlineBacklog,
|
||||
)
|
||||
return snapshot, err
|
||||
}
|
||||
|
||||
func (s *Store) ReapExpiredTaskAdmissions(ctx context.Context, limit int) (TaskAdmissionReapResult, error) {
|
||||
func (s *Store) ReapExpiredTaskAdmissions(
|
||||
ctx context.Context,
|
||||
limit int,
|
||||
defaultCallbackURLs ...string,
|
||||
) (TaskAdmissionReapResult, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 500
|
||||
}
|
||||
defaultCallbackURL := ""
|
||||
if len(defaultCallbackURLs) > 0 {
|
||||
defaultCallbackURL = strings.TrimSpace(defaultCallbackURLs[0])
|
||||
}
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
@@ -1155,6 +1291,7 @@ LIMIT $1`, limit)
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
var deadlineExpired bool
|
||||
var dispatchFailureCount int
|
||||
err := tx.QueryRow(ctx, `
|
||||
DELETE FROM gateway_task_admissions
|
||||
WHERE task_id = $1::uuid
|
||||
@@ -1163,7 +1300,7 @@ WHERE task_id = $1::uuid
|
||||
wait_deadline_at <= now()
|
||||
OR (mode = 'sync' AND waiter_lease_expires_at <= now())
|
||||
)
|
||||
RETURNING wait_deadline_at <= now()`, taskID).Scan(&deadlineExpired)
|
||||
RETURNING wait_deadline_at <= now(), dispatch_failure_count`, taskID).Scan(&deadlineExpired, &dispatchFailureCount)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
@@ -1177,22 +1314,112 @@ RETURNING wait_deadline_at <= now()`, taskID).Scan(&deadlineExpired)
|
||||
code = "queue_timeout"
|
||||
message = ErrQueueTimeout.Error()
|
||||
status = "failed"
|
||||
if dispatchFailureCount > 0 {
|
||||
code = "admission_dispatch_failed"
|
||||
message = "task admission dispatcher could not prepare the task before its queue deadline"
|
||||
}
|
||||
result.ExpiredDeadlines++
|
||||
} else {
|
||||
result.ExpiredWaiters++
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = $2,
|
||||
error = $3,
|
||||
error = NULL,
|
||||
error_code = $4,
|
||||
error_message = $3,
|
||||
public_error = $5::jsonb,
|
||||
billing_status = CASE
|
||||
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
|
||||
WHEN reservation_amount > 0 THEN 'pending'
|
||||
ELSE 'released'
|
||||
END,
|
||||
billing_updated_at = now(),
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'queued'`, taskID, status, message, code); err != nil {
|
||||
AND status = 'queued'`, taskID, status, message, code, string(publicErrorJSON))
|
||||
if err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
if _, err := releaseRuntimeRateReservationsTx(ctx, tx, []string{taskID}); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
payloadJSON, _ := json.Marshal(map[string]any{"taskId": taskID, "reason": code})
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
|
||||
)
|
||||
SELECT id, 'task.billing.release', 'release', reservation_amount, billing_currency,
|
||||
pricing_snapshot, $2::jsonb, 'pending', now()
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
AND run_mode IN ('production', 'acceptance', 'acceptance_canary')
|
||||
AND gateway_user_id IS NOT NULL
|
||||
AND reservation_amount > 0
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
eventType := "task.cancelled"
|
||||
if status == "failed" {
|
||||
eventType = "task.failed"
|
||||
}
|
||||
var eventID string
|
||||
var eventSeq int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH next_seq AS (
|
||||
SELECT COALESCE(MAX(seq), 0) + 1 AS seq
|
||||
FROM gateway_task_events
|
||||
WHERE task_id = $1::uuid
|
||||
)
|
||||
INSERT INTO gateway_task_events (
|
||||
task_id, seq, event_type, status, phase, progress, message, payload, simulated
|
||||
)
|
||||
SELECT $1::uuid, next_seq.seq, $2, $3, 'admission', 0, $4, '{}'::jsonb,
|
||||
task.run_mode = 'simulation'
|
||||
FROM next_seq
|
||||
JOIN gateway_tasks task ON task.id = $1::uuid
|
||||
RETURNING id::text, seq`, taskID, eventType, status, message).Scan(&eventID, &eventSeq); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
var acceptanceCallbackURL string
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE((
|
||||
SELECT run.callback_url
|
||||
FROM gateway_tasks task
|
||||
JOIN gateway_acceptance_runs run ON run.id = task.acceptance_run_id
|
||||
WHERE task.id = $1::uuid
|
||||
), '')`, taskID).Scan(&acceptanceCallbackURL); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
callbackURL := defaultCallbackURL
|
||||
if strings.TrimSpace(acceptanceCallbackURL) != "" {
|
||||
callbackURL = strings.TrimSpace(acceptanceCallbackURL)
|
||||
}
|
||||
if callbackURL != "" {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
|
||||
ON CONFLICT (task_id, seq, callback_url) DO NOTHING`, taskID, eventID, eventSeq, callbackURL); err != nil {
|
||||
return TaskAdmissionReapResult{}, err
|
||||
}
|
||||
}
|
||||
reaped++
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
@@ -1448,7 +1675,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
input.TaskID, input.PlatformID, input.PlatformModelID, input.UserGroupID,
|
||||
input.QueueKey, input.Mode, input.Priority, deadline, waiterID, waiterLease, scopeSnapshot)
|
||||
return scanTaskAdmission(row)
|
||||
@@ -1467,7 +1696,9 @@ RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
taskID, waiterID, admissionWaiterLeaseTTL.String())
|
||||
return scanTaskAdmission(row)
|
||||
}
|
||||
@@ -1478,13 +1709,19 @@ UPDATE gateway_task_admissions
|
||||
SET status = 'admitted',
|
||||
admitted_at = now(),
|
||||
waiter_lease_expires_at = NULL,
|
||||
dispatch_failure_count = 0,
|
||||
dispatch_last_outcome = 'admitted',
|
||||
dispatch_last_error_code = NULL,
|
||||
dispatch_last_error_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE task_id = $1::uuid AND status = 'waiting'
|
||||
RETURNING task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at`,
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at`,
|
||||
taskID)
|
||||
return scanTaskAdmission(row)
|
||||
}
|
||||
@@ -1503,7 +1740,9 @@ SELECT task_id::text, platform_id::text, platform_model_id::text,
|
||||
COALESCE(user_group_id::text, ''), queue_key, mode, status, priority,
|
||||
enqueued_at, wait_deadline_at, admitted_at,
|
||||
COALESCE(waiter_id, ''), waiter_lease_expires_at,
|
||||
scope_snapshot, reselect_requested_at
|
||||
scope_snapshot, reselect_requested_at, dispatch_next_at,
|
||||
dispatch_failure_count, COALESCE(dispatch_last_outcome, ''),
|
||||
COALESCE(dispatch_last_error_code, ''), dispatch_last_error_at
|
||||
FROM gateway_task_admissions
|
||||
WHERE task_id = $1::uuid`, taskID)
|
||||
admission, err := scanTaskAdmission(row)
|
||||
@@ -1518,6 +1757,7 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
|
||||
var admittedAt sql.NullTime
|
||||
var waiterLeaseExpires sql.NullTime
|
||||
var reselectRequestedAt sql.NullTime
|
||||
var dispatchLastErrorAt sql.NullTime
|
||||
var scopeSnapshot []byte
|
||||
err := row.Scan(
|
||||
&admission.TaskID,
|
||||
@@ -1535,6 +1775,11 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
|
||||
&waiterLeaseExpires,
|
||||
&scopeSnapshot,
|
||||
&reselectRequestedAt,
|
||||
&admission.DispatchNextAt,
|
||||
&admission.DispatchFailureCount,
|
||||
&admission.DispatchLastOutcome,
|
||||
&admission.DispatchLastErrorCode,
|
||||
&dispatchLastErrorAt,
|
||||
)
|
||||
if admittedAt.Valid {
|
||||
admission.AdmittedAt = admittedAt.Time
|
||||
@@ -1545,15 +1790,18 @@ func scanTaskAdmission(row pgx.Row) (TaskAdmission, error) {
|
||||
if reselectRequestedAt.Valid {
|
||||
admission.ReselectRequestedAt = reselectRequestedAt.Time
|
||||
}
|
||||
if dispatchLastErrorAt.Valid {
|
||||
admission.DispatchLastErrorAt = dispatchLastErrorAt.Time
|
||||
}
|
||||
if err == nil && len(scopeSnapshot) > 0 {
|
||||
if unmarshalErr := json.Unmarshal(scopeSnapshot, &admission.Scopes); unmarshalErr != nil {
|
||||
return TaskAdmission{}, fmt.Errorf("unmarshal task admission scopes: %w", unmarshalErr)
|
||||
admission.SnapshotInvalid = true
|
||||
}
|
||||
}
|
||||
return admission, err
|
||||
}
|
||||
|
||||
func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmission, scopes []AdmissionScope) (bool, error) {
|
||||
func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmission, scopes []AdmissionScope) (bool, []TaskAdmissionBlocker, error) {
|
||||
for _, scope := range scopes {
|
||||
var head string
|
||||
switch scope.ScopeType {
|
||||
@@ -1561,10 +1809,12 @@ func isTaskAdmissionHeadTx(ctx context.Context, tx pgx.Tx, admission TaskAdmissi
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT task_id::text
|
||||
FROM gateway_task_admissions
|
||||
WHERE platform_model_id = $1::uuid AND status = 'waiting'
|
||||
WHERE platform_model_id = $1::uuid
|
||||
AND status = 'waiting'
|
||||
AND (mode <> 'async' OR dispatch_next_at <= statement_timestamp())
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
LIMIT 1`, admission.PlatformModelID).Scan(&head); err != nil {
|
||||
return false, err
|
||||
return false, nil, err
|
||||
}
|
||||
case "user_group":
|
||||
if admission.UserGroupID == "" {
|
||||
@@ -1573,19 +1823,25 @@ LIMIT 1`, admission.PlatformModelID).Scan(&head); err != nil {
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT task_id::text
|
||||
FROM gateway_task_admissions
|
||||
WHERE user_group_id = $1::uuid AND status = 'waiting'
|
||||
WHERE user_group_id = $1::uuid
|
||||
AND status = 'waiting'
|
||||
AND (mode <> 'async' OR dispatch_next_at <= statement_timestamp())
|
||||
ORDER BY priority ASC, enqueued_at ASC, task_id ASC
|
||||
LIMIT 1`, admission.UserGroupID).Scan(&head); err != nil {
|
||||
return false, err
|
||||
return false, nil, err
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if head != admission.TaskID {
|
||||
return false, nil
|
||||
return false, []TaskAdmissionBlocker{{
|
||||
Reason: "not_queue_head",
|
||||
ScopeType: scope.ScopeType,
|
||||
ScopeKey: scope.ScopeKey,
|
||||
}}, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
return true, nil, nil
|
||||
}
|
||||
|
||||
func expireTaskAdmissionTx(ctx context.Context, tx pgx.Tx, taskID string) error {
|
||||
|
||||
@@ -131,6 +131,9 @@ RETURNING id::text`, platform.ID, "queue-model-"+suffix).Scan(&platformModelID);
|
||||
if err != nil || result.Admitted {
|
||||
t.Fatalf("second task should wait: result=%+v err=%v", result, err)
|
||||
}
|
||||
if len(result.Blockers) != 1 || result.Blockers[0].Reason != "saturated" || result.Blockers[0].ScopeType != "platform_model" {
|
||||
t.Fatalf("second task blockers=%+v, want saturated platform_model", result.Blockers)
|
||||
}
|
||||
higherPriorityAsync := createTask(true)
|
||||
result, err = first.TryTaskAdmission(ctx, inputFor(higherPriorityAsync, 10, ""))
|
||||
if err != nil || result.Admitted {
|
||||
@@ -176,6 +179,9 @@ WHERE task_id = ANY($1::uuid[])`, taskIDs).Scan(&attempts); err != nil {
|
||||
if result.Admitted {
|
||||
t.Fatal("lower-priority task bypassed higher-priority asynchronous waiter")
|
||||
}
|
||||
if len(result.Blockers) != 1 || result.Blockers[0].Reason != "not_queue_head" || result.Blockers[0].ScopeType != "platform_model" {
|
||||
t.Fatalf("lower-priority blockers=%+v, want platform FIFO head blocker", result.Blockers)
|
||||
}
|
||||
result, err = first.TryTaskAdmission(ctx, inputFor(higherPriorityAsync, 10, ""))
|
||||
if err != nil || !result.Admitted {
|
||||
t.Fatalf("higher-priority async task was not admitted first: result=%+v err=%v", result, err)
|
||||
@@ -451,12 +457,23 @@ WHERE id = $1::uuid`, queuedAtomicTask.ID, queuedSyntheticRiverJobID)
|
||||
if listedSnapshot == nil || len(listedSnapshot.Scopes) != len(queuedAdmission.Scopes) {
|
||||
t.Fatalf("listed admission snapshot=%+v, want %d scopes", listedSnapshot, len(queuedAdmission.Scopes))
|
||||
}
|
||||
markedForReselect, err := first.RequestWaitingTaskAdmissionReselect(ctx, platformModelID)
|
||||
_, markedForReselect, err := first.RequestTaskAdmissionReselect(ctx, queuedAtomicTask.ID, platformModelID)
|
||||
if err != nil {
|
||||
t.Fatalf("request waiting admission reselection: %v", err)
|
||||
}
|
||||
if markedForReselect < 1 {
|
||||
t.Fatalf("marked admissions=%d, want at least the queued task", markedForReselect)
|
||||
if !markedForReselect {
|
||||
t.Fatal("queued task was not marked for reselection")
|
||||
}
|
||||
var platformReselectCount int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT count(*)
|
||||
FROM gateway_task_admissions
|
||||
WHERE platform_model_id = $1::uuid
|
||||
AND reselect_requested_at IS NOT NULL`, platformModelID).Scan(&platformReselectCount); err != nil {
|
||||
t.Fatalf("count task-scoped reselections: %v", err)
|
||||
}
|
||||
if platformReselectCount != 1 {
|
||||
t.Fatalf("platform reselection fanout=%d, want exactly one task", platformReselectCount)
|
||||
}
|
||||
reselectAdmission, err := first.GetTaskAdmission(ctx, queuedAtomicTask.ID)
|
||||
if err != nil {
|
||||
@@ -1070,6 +1087,276 @@ WHERE id = $1::uuid`, ambiguousTask.ID).Scan(&ambiguousStatus); err != nil {
|
||||
t.Fatalf("ambiguous submission task status = %s, want running", ambiguousStatus)
|
||||
}
|
||||
|
||||
globalHolder := createTask(true)
|
||||
globalWaiter := createTask(true)
|
||||
globalScope := AdmissionScope{
|
||||
ScopeType: "worker_capacity",
|
||||
ScopeKey: "global-blocker-" + suffix,
|
||||
ScopeName: "global blocker",
|
||||
ConcurrentLimit: 1,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 10,
|
||||
MaxWaitSeconds: 600,
|
||||
}
|
||||
globalHolderInput := inputFor(globalHolder, 100, "")
|
||||
globalHolderInput.Scopes = []AdmissionScope{globalScope}
|
||||
globalWaiterInput := inputFor(globalWaiter, 100, "")
|
||||
globalWaiterInput.Scopes = []AdmissionScope{globalScope}
|
||||
if result, err := first.TryTaskAdmission(ctx, globalHolderInput); err != nil || !result.Admitted {
|
||||
t.Fatalf("admit global capacity holder: result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := first.QueueTaskAdmissionWithHook(ctx, globalWaiterInput, nil); err != nil {
|
||||
t.Fatalf("queue global capacity waiter: %v", err)
|
||||
}
|
||||
globalWaitResult, err := second.TryTaskAdmission(ctx, globalWaiterInput)
|
||||
if err != nil || globalWaitResult.Admitted || len(globalWaitResult.Blockers) != 1 ||
|
||||
globalWaitResult.Blockers[0].Reason != "saturated" ||
|
||||
globalWaitResult.Blockers[0].ScopeType != "worker_capacity" {
|
||||
t.Fatalf("global capacity waiter result=%+v err=%v", globalWaitResult, err)
|
||||
}
|
||||
if !globalWaitResult.Admission.ReselectRequestedAt.IsZero() {
|
||||
t.Fatal("global capacity wait unexpectedly requested candidate reselection")
|
||||
}
|
||||
if err := first.DeleteTaskAdmission(ctx, globalHolder.ID); err != nil {
|
||||
t.Fatalf("release global capacity holder: %v", err)
|
||||
}
|
||||
if err := first.DeleteTaskAdmission(ctx, globalWaiter.ID); err != nil {
|
||||
t.Fatalf("delete global capacity waiter: %v", err)
|
||||
}
|
||||
|
||||
orphanedAdmittedTask := createTask(true)
|
||||
orphanedAdmittedInput := inputFor(orphanedAdmittedTask, 100, "")
|
||||
orphanedAdmittedInput.Scopes = []AdmissionScope{{
|
||||
ScopeType: "worker_capacity",
|
||||
ScopeKey: "orphaned-admitted-" + suffix,
|
||||
ConcurrentLimit: 1,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 10,
|
||||
MaxWaitSeconds: 600,
|
||||
}}
|
||||
if result, err := first.TryTaskAdmission(ctx, orphanedAdmittedInput); err != nil || !result.Admitted {
|
||||
t.Fatalf("admit orphan candidate: result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := first.pool.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = now()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, orphanedAdmittedTask.ID); err != nil {
|
||||
t.Fatalf("orphan admitted task leases: %v", err)
|
||||
}
|
||||
if result, err := second.TryTaskAdmission(ctx, orphanedAdmittedInput); err != nil || !result.Admitted || !result.NewlyAdmitted {
|
||||
t.Fatalf("recover admitted row without lease: result=%+v err=%v", result, err)
|
||||
}
|
||||
if err := first.DeleteTaskAdmission(ctx, orphanedAdmittedTask.ID); err != nil {
|
||||
t.Fatalf("delete recovered admitted row: %v", err)
|
||||
}
|
||||
|
||||
resilienceTask := createTask(true)
|
||||
resilienceInput := inputFor(resilienceTask, 100, "")
|
||||
resilienceInput.Scopes = []AdmissionScope{{
|
||||
ScopeType: "worker_capacity",
|
||||
ScopeKey: "resilience-" + suffix,
|
||||
ScopeName: "resilience capacity",
|
||||
ConcurrentLimit: 5,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 10,
|
||||
MaxWaitSeconds: 600,
|
||||
}}
|
||||
if _, err := first.QueueTaskAdmissionWithHook(ctx, resilienceInput, nil); err != nil {
|
||||
t.Fatalf("queue resilience task: %v", err)
|
||||
}
|
||||
if _, err := first.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_admissions
|
||||
SET scope_snapshot = '[{"ScopeType":123}]'::jsonb
|
||||
WHERE task_id = $1::uuid`, resilienceTask.ID); err != nil {
|
||||
t.Fatalf("corrupt resilience scope snapshot: %v", err)
|
||||
}
|
||||
listedAdmissions, err := first.ListWaitingAsyncAdmissions(ctx, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("list admissions with one corrupt snapshot: %v", err)
|
||||
}
|
||||
corruptFound := false
|
||||
for _, listed := range listedAdmissions {
|
||||
if listed.TaskID == resilienceTask.ID {
|
||||
corruptFound = listed.SnapshotInvalid
|
||||
break
|
||||
}
|
||||
}
|
||||
if !corruptFound {
|
||||
t.Fatal("corrupt admission snapshot was not isolated for rebuilding")
|
||||
}
|
||||
repairedAdmission, err := first.RebindWaitingTaskAdmission(ctx, resilienceInput)
|
||||
if err != nil || repairedAdmission.SnapshotInvalid || len(repairedAdmission.Scopes) != 1 {
|
||||
t.Fatalf("repair corrupt admission snapshot=%+v err=%v", repairedAdmission, err)
|
||||
}
|
||||
deferred, changed, err := first.DeferWaitingTaskAdmission(
|
||||
ctx,
|
||||
resilienceTask.ID,
|
||||
platformModelID,
|
||||
time.Now().Add(time.Minute),
|
||||
"deferred",
|
||||
"snapshot_invalid",
|
||||
)
|
||||
if err != nil || !changed || deferred.DispatchFailureCount != 1 || deferred.DispatchLastErrorCode != "snapshot_invalid" {
|
||||
t.Fatalf("defer corrupt admission=%+v changed=%v err=%v", deferred, changed, err)
|
||||
}
|
||||
listedAdmissions, err = first.ListWaitingAsyncAdmissions(ctx, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("list admissions after defer: %v", err)
|
||||
}
|
||||
for _, listed := range listedAdmissions {
|
||||
if listed.TaskID == resilienceTask.ID {
|
||||
t.Fatal("deferred admission remained dispatchable")
|
||||
}
|
||||
}
|
||||
if _, err := first.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_admissions
|
||||
SET dispatch_next_at = now(), scope_snapshot = '[]'::jsonb
|
||||
WHERE task_id = $1::uuid`, resilienceTask.ID); err != nil {
|
||||
t.Fatalf("make resilience admission dispatchable: %v", err)
|
||||
}
|
||||
reselected, changed, err := first.RequestTaskAdmissionReselect(ctx, resilienceTask.ID, platformModelID)
|
||||
if err != nil || !changed || reselected.ReselectRequestedAt.IsZero() {
|
||||
t.Fatalf("task-scoped reselection=%+v changed=%v err=%v", reselected, changed, err)
|
||||
}
|
||||
if _, changed, err := second.RequestTaskAdmissionReselect(ctx, resilienceTask.ID, platformModelID); err != nil || changed {
|
||||
t.Fatalf("duplicate task-scoped reselection changed=%v err=%v", changed, err)
|
||||
}
|
||||
if _, err := first.FailQueuedTaskWithCallback(
|
||||
ctx,
|
||||
resilienceTask.ID,
|
||||
"no_model_candidate",
|
||||
"no enabled platform model matches request",
|
||||
"https://callback.invalid/admission-resilience",
|
||||
false,
|
||||
); err != nil {
|
||||
t.Fatalf("terminally fail resilience task: %v", err)
|
||||
}
|
||||
if _, err := second.FailQueuedTaskWithCallback(
|
||||
ctx,
|
||||
resilienceTask.ID,
|
||||
"no_model_candidate",
|
||||
"duplicate terminalization",
|
||||
"https://callback.invalid/admission-resilience",
|
||||
false,
|
||||
); !errors.Is(err, ErrTaskExecutionFinished) {
|
||||
t.Fatalf("duplicate terminalization error=%v, want task finished", err)
|
||||
}
|
||||
var resilienceStatus, resilienceCode string
|
||||
var resilienceAdmissions, resilienceLeases, resilienceFailedEvents, resilienceCallbacks int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT task.status, COALESCE(task.error_code, ''),
|
||||
(SELECT count(*) FROM gateway_task_admissions admission WHERE admission.task_id = task.id),
|
||||
(SELECT count(*) FROM gateway_concurrency_leases lease WHERE lease.task_id = task.id AND lease.released_at IS NULL),
|
||||
(SELECT count(*) FROM gateway_task_events event WHERE event.task_id = task.id AND event.event_type = 'task.failed'),
|
||||
(SELECT count(*) FROM gateway_task_callback_outbox callback WHERE callback.task_id = task.id)
|
||||
FROM gateway_tasks task
|
||||
WHERE task.id = $1::uuid`, resilienceTask.ID).Scan(
|
||||
&resilienceStatus,
|
||||
&resilienceCode,
|
||||
&resilienceAdmissions,
|
||||
&resilienceLeases,
|
||||
&resilienceFailedEvents,
|
||||
&resilienceCallbacks,
|
||||
); err != nil {
|
||||
t.Fatalf("read resilience terminal state: %v", err)
|
||||
}
|
||||
if resilienceStatus != "failed" || resilienceCode != "no_model_candidate" ||
|
||||
resilienceAdmissions != 0 || resilienceLeases != 0 ||
|
||||
resilienceFailedEvents != 1 || resilienceCallbacks != 1 {
|
||||
t.Fatalf(
|
||||
"resilience terminal state=%s/%s admissions=%d leases=%d events=%d callbacks=%d",
|
||||
resilienceStatus,
|
||||
resilienceCode,
|
||||
resilienceAdmissions,
|
||||
resilienceLeases,
|
||||
resilienceFailedEvents,
|
||||
resilienceCallbacks,
|
||||
)
|
||||
}
|
||||
retryExpiredTask := createTask(true)
|
||||
retryExpiredInput := resilienceInput
|
||||
retryExpiredInput.TaskID = retryExpiredTask.ID
|
||||
if _, err := first.QueueTaskAdmissionWithHook(ctx, retryExpiredInput, nil); err != nil {
|
||||
t.Fatalf("queue retry-expired task: %v", err)
|
||||
}
|
||||
if _, err := first.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_admissions
|
||||
SET enqueued_at = now() - interval '2 seconds',
|
||||
wait_deadline_at = now() - interval '1 second',
|
||||
dispatch_failure_count = 3,
|
||||
dispatch_last_outcome = 'deferred',
|
||||
dispatch_last_error_code = 'client_error',
|
||||
dispatch_last_error_at = now() - interval '1 minute'
|
||||
WHERE task_id = $1::uuid`, retryExpiredTask.ID); err != nil {
|
||||
t.Fatalf("expire deferred task: %v", err)
|
||||
}
|
||||
reaped, err = first.ReapExpiredTaskAdmissions(ctx, 10, "https://callback.invalid/admission-timeout")
|
||||
if err != nil || reaped.ExpiredDeadlines < 1 {
|
||||
t.Fatalf("reap retry-expired task=%+v err=%v", reaped, err)
|
||||
}
|
||||
var retryExpiredCode string
|
||||
var retryExpiredEvents, retryExpiredCallbacks int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(task.error_code, ''),
|
||||
(SELECT count(*) FROM gateway_task_events event WHERE event.task_id = task.id AND event.event_type = 'task.failed'),
|
||||
(SELECT count(*) FROM gateway_task_callback_outbox callback WHERE callback.task_id = task.id)
|
||||
FROM gateway_tasks task
|
||||
WHERE task.id = $1::uuid`, retryExpiredTask.ID).Scan(
|
||||
&retryExpiredCode,
|
||||
&retryExpiredEvents,
|
||||
&retryExpiredCallbacks,
|
||||
); err != nil {
|
||||
t.Fatalf("read retry-expired task: %v", err)
|
||||
}
|
||||
if retryExpiredCode != "admission_dispatch_failed" || retryExpiredEvents != 1 || retryExpiredCallbacks != 1 {
|
||||
t.Fatalf(
|
||||
"retry-expired task code=%s events=%d callbacks=%d, want admission_dispatch_failed/1/1",
|
||||
retryExpiredCode,
|
||||
retryExpiredEvents,
|
||||
retryExpiredCallbacks,
|
||||
)
|
||||
}
|
||||
|
||||
disconnectedTask := createTask(true)
|
||||
disconnectedInput := resilienceInput
|
||||
disconnectedInput.TaskID = disconnectedTask.ID
|
||||
if _, err := first.QueueTaskAdmissionWithHook(ctx, disconnectedInput, nil); err != nil {
|
||||
t.Fatalf("queue disconnected task: %v", err)
|
||||
}
|
||||
if _, changed, err := first.CancelQueuedTaskWithCallback(
|
||||
ctx,
|
||||
disconnectedTask.ID,
|
||||
"client disconnected before upstream submission",
|
||||
"https://callback.invalid/admission-disconnect",
|
||||
); err != nil || !changed {
|
||||
t.Fatalf("cancel disconnected task changed=%v err=%v", changed, err)
|
||||
}
|
||||
var disconnectedEvents, disconnectedCallbacks, disconnectedAdmissions int
|
||||
if err := first.pool.QueryRow(ctx, `
|
||||
SELECT (SELECT count(*) FROM gateway_task_events event WHERE event.task_id = task.id AND event.event_type = 'task.cancelled'),
|
||||
(SELECT count(*) FROM gateway_task_callback_outbox callback WHERE callback.task_id = task.id),
|
||||
(SELECT count(*) FROM gateway_task_admissions admission WHERE admission.task_id = task.id)
|
||||
FROM gateway_tasks task
|
||||
WHERE task.id = $1::uuid`, disconnectedTask.ID).Scan(
|
||||
&disconnectedEvents,
|
||||
&disconnectedCallbacks,
|
||||
&disconnectedAdmissions,
|
||||
); err != nil {
|
||||
t.Fatalf("read disconnected task cleanup: %v", err)
|
||||
}
|
||||
if disconnectedEvents != 1 || disconnectedCallbacks != 1 || disconnectedAdmissions != 0 {
|
||||
t.Fatalf(
|
||||
"disconnected task events=%d callbacks=%d admissions=%d, want 1/1/0",
|
||||
disconnectedEvents,
|
||||
disconnectedCallbacks,
|
||||
disconnectedAdmissions,
|
||||
)
|
||||
}
|
||||
|
||||
terminalResidue := createTask(true)
|
||||
result, err = first.TryTaskAdmission(ctx, inputFor(terminalResidue, 100, ""))
|
||||
if err != nil || !result.Admitted {
|
||||
@@ -1247,6 +1534,150 @@ WHERE task_id = ANY($1::uuid[])`, taskIDs); err != nil {
|
||||
if !leaderSet[tasks[0].ID] || leaderSet[tasks[2].ID] {
|
||||
t.Fatalf("grouped candidate leaders = %v, want only group head %s", leaders, tasks[0].ID)
|
||||
}
|
||||
if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = ANY($1::uuid[])`, taskIDs); err != nil {
|
||||
t.Fatalf("clear grouped admission leaders: %v", err)
|
||||
}
|
||||
|
||||
createAsyncTask := func(modelIndex int) GatewayTask {
|
||||
t.Helper()
|
||||
task, createErr := db.CreateTask(ctx, CreateTaskInput{
|
||||
Kind: "images.generate",
|
||||
Model: fmt.Sprintf("leader-model-%d-%s", modelIndex, suffix),
|
||||
RunMode: "production",
|
||||
Async: true,
|
||||
Request: map[string]any{"prompt": "async admission independence test"},
|
||||
}, &auth.User{ID: "leader-test-user-" + suffix, Source: "gateway"})
|
||||
if createErr != nil {
|
||||
t.Fatalf("create async task: %v", createErr)
|
||||
}
|
||||
taskIDs = append(taskIDs, task.ID)
|
||||
return task
|
||||
}
|
||||
platformScope := func(modelID string) AdmissionScope {
|
||||
return AdmissionScope{
|
||||
ScopeType: "platform_model",
|
||||
ScopeKey: modelID,
|
||||
ConcurrentLimit: 1,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 10,
|
||||
MaxWaitSeconds: 600,
|
||||
}
|
||||
}
|
||||
groupScope := AdmissionScope{
|
||||
ScopeType: "user_group",
|
||||
ScopeKey: group.ID,
|
||||
ConcurrentLimit: 1,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 10,
|
||||
MaxWaitSeconds: 600,
|
||||
}
|
||||
holder := createAsyncTask(0)
|
||||
blocked := createAsyncTask(0)
|
||||
independent := createAsyncTask(1)
|
||||
holderInput := TaskAdmissionInput{
|
||||
TaskID: holder.ID, PlatformID: platform.ID, PlatformModelID: modelIDs[0],
|
||||
UserGroupID: group.ID, QueueKey: "leader-test:" + modelIDs[0], Mode: "async", Priority: 100,
|
||||
Scopes: []AdmissionScope{platformScope(modelIDs[0]), groupScope},
|
||||
}
|
||||
blockedInput := holderInput
|
||||
blockedInput.TaskID = blocked.ID
|
||||
independentInput := TaskAdmissionInput{
|
||||
TaskID: independent.ID, PlatformID: platform.ID, PlatformModelID: modelIDs[1],
|
||||
QueueKey: "leader-test:" + modelIDs[1], Mode: "async", Priority: 100,
|
||||
Scopes: []AdmissionScope{platformScope(modelIDs[1])},
|
||||
}
|
||||
if result, err := db.TryTaskAdmission(ctx, holderInput); err != nil || !result.Admitted {
|
||||
t.Fatalf("admit local blocker holder: result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := db.QueueTaskAdmissionWithHook(ctx, blockedInput, nil); err != nil {
|
||||
t.Fatalf("queue locally blocked task: %v", err)
|
||||
}
|
||||
if _, err := db.QueueTaskAdmissionWithHook(ctx, independentInput, nil); err != nil {
|
||||
t.Fatalf("queue independent task: %v", err)
|
||||
}
|
||||
outcomes, err := db.TryTaskAdmissionAtomicBatchWithAdmittedHook(
|
||||
ctx,
|
||||
[]TaskAdmissionInput{blockedInput, independentInput},
|
||||
nil,
|
||||
)
|
||||
if err != nil || len(outcomes) != 2 {
|
||||
t.Fatalf("independent atomic outcomes=%+v err=%v", outcomes, err)
|
||||
}
|
||||
if outcomes[0].Result.Admitted || len(outcomes[0].Result.Blockers) == 0 {
|
||||
t.Fatalf("locally blocked outcome=%+v, want explicit blockers", outcomes[0])
|
||||
}
|
||||
if !outcomes[1].Result.Admitted {
|
||||
t.Fatalf("independent candidate was cross-blocked: %+v", outcomes[1])
|
||||
}
|
||||
for _, taskID := range []string{holder.ID, blocked.ID, independent.ID} {
|
||||
if err := db.DeleteTaskAdmission(ctx, taskID); err != nil {
|
||||
t.Fatalf("clear independent atomic admission %s: %v", taskID, err)
|
||||
}
|
||||
}
|
||||
|
||||
hookTasks := []GatewayTask{createAsyncTask(0), createAsyncTask(0), createAsyncTask(1)}
|
||||
hookInputs := make([]TaskAdmissionInput, 0, len(hookTasks))
|
||||
for index, task := range hookTasks {
|
||||
input := TaskAdmissionInput{
|
||||
TaskID: task.ID, PlatformID: platform.ID, PlatformModelID: modelIDs[index%len(modelIDs)],
|
||||
QueueKey: "hook-isolation:" + task.ID, Mode: "async", Priority: 100,
|
||||
Scopes: []AdmissionScope{{
|
||||
ScopeType: "worker_capacity",
|
||||
ScopeKey: "hook-isolation-" + task.ID,
|
||||
ConcurrentLimit: 1,
|
||||
Amount: 1,
|
||||
LeaseTTLSeconds: 120,
|
||||
QueueLimit: 10,
|
||||
MaxWaitSeconds: 600,
|
||||
}},
|
||||
}
|
||||
if _, err := db.QueueTaskAdmissionWithHook(ctx, input, nil); err != nil {
|
||||
t.Fatalf("queue hook isolation task %d: %v", index, err)
|
||||
}
|
||||
hookInputs = append(hookInputs, input)
|
||||
}
|
||||
hookFailure := errors.New("synthetic per-task hook failure")
|
||||
outcomes, err = db.TryTaskAdmissionAtomicBatchWithAdmittedHook(
|
||||
ctx,
|
||||
hookInputs,
|
||||
func(_ pgx.Tx, input TaskAdmissionInput) error {
|
||||
if input.TaskID == hookTasks[1].ID {
|
||||
return hookFailure
|
||||
}
|
||||
return nil
|
||||
},
|
||||
)
|
||||
if !errors.Is(err, hookFailure) || len(outcomes) != 2 || outcomes[1].TaskID != hookTasks[1].ID {
|
||||
t.Fatalf("hook failure outcomes=%+v err=%v", outcomes, err)
|
||||
}
|
||||
var hookAdmitted, hookLeases int
|
||||
if err := db.pool.QueryRow(ctx, `
|
||||
SELECT (SELECT count(*) FROM gateway_task_admissions WHERE task_id = ANY($1::uuid[]) AND status = 'admitted'),
|
||||
(SELECT count(*) FROM gateway_concurrency_leases WHERE task_id = ANY($1::uuid[]) AND released_at IS NULL)`,
|
||||
[]string{hookTasks[0].ID, hookTasks[1].ID, hookTasks[2].ID},
|
||||
).Scan(&hookAdmitted, &hookLeases); err != nil {
|
||||
t.Fatalf("read rolled back hook batch: %v", err)
|
||||
}
|
||||
if hookAdmitted != 0 || hookLeases != 0 {
|
||||
t.Fatalf("failed hook batch left admitted=%d leases=%d", hookAdmitted, hookLeases)
|
||||
}
|
||||
if _, changed, err := db.DeferWaitingTaskAdmission(
|
||||
ctx,
|
||||
hookTasks[1].ID,
|
||||
hookInputs[1].PlatformModelID,
|
||||
time.Now().Add(time.Minute),
|
||||
"deferred",
|
||||
"synthetic_hook_failure",
|
||||
); err != nil || !changed {
|
||||
t.Fatalf("isolate hook failure changed=%v err=%v", changed, err)
|
||||
}
|
||||
for _, index := range []int{0, 2} {
|
||||
if result, err := db.TryTaskAdmission(ctx, hookInputs[index]); err != nil || !result.Admitted {
|
||||
t.Fatalf("admit hook survivor %d result=%+v err=%v", index, result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerCapacityAllocationAndFailover(t *testing.T) {
|
||||
|
||||
@@ -545,17 +545,40 @@ WHERE id = $1::uuid
|
||||
}
|
||||
|
||||
func (s *Store) FailQueuedTask(ctx context.Context, taskID string, code string, message string) (GatewayTask, error) {
|
||||
return s.FailQueuedTaskWithCallback(ctx, taskID, code, message, "", false)
|
||||
}
|
||||
|
||||
// FailQueuedTaskWithCallback terminally fails a task before upstream
|
||||
// submission and atomically releases every durable resource owned by its
|
||||
// admission. The task advisory lock makes competing dispatchers idempotent.
|
||||
func (s *Store) FailQueuedTaskWithCallback(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
code string,
|
||||
message string,
|
||||
callbackURL string,
|
||||
simulated bool,
|
||||
) (GatewayTask, error) {
|
||||
code = strings.TrimSpace(code)
|
||||
message = truncateUTF8Bytes(message, 2048)
|
||||
publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
var task GatewayTask
|
||||
err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, "task-admission:"+taskID); err != nil {
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
task, err = scanGatewayTask(tx.QueryRow(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'failed',
|
||||
error = NULL,
|
||||
error_code = NULLIF($2, ''),
|
||||
error_message = NULLIF($3, ''),
|
||||
public_error = $4::jsonb,
|
||||
public_error = $4::jsonb,
|
||||
billing_status = CASE
|
||||
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
|
||||
ELSE 'released'
|
||||
WHEN reservation_amount > 0 THEN 'pending'
|
||||
ELSE 'released'
|
||||
END,
|
||||
billing_updated_at = now(),
|
||||
locked_by = NULL,
|
||||
@@ -566,14 +589,76 @@ SET status = 'failed',
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048), string(publicErrorJSON))
|
||||
AND status = 'queued'
|
||||
AND COALESCE(remote_task_id, '') = ''
|
||||
RETURNING `+gatewayTaskColumns, taskID, code, message, string(publicErrorJSON)))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrTaskExecutionFinished
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_concurrency_leases
|
||||
SET released_at = statement_timestamp()
|
||||
WHERE task_id = $1::uuid
|
||||
AND released_at IS NULL`, taskID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := releaseRuntimeRateReservationsTx(ctx, tx, []string{taskID}); err != nil {
|
||||
return err
|
||||
}
|
||||
payloadJSON, _ := json.Marshal(map[string]any{"taskId": taskID, "reason": code})
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
|
||||
)
|
||||
SELECT id, 'task.billing.release', 'release', reservation_amount, billing_currency,
|
||||
pricing_snapshot, $2::jsonb, 'pending', now()
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
AND run_mode IN ('production', 'acceptance', 'acceptance_canary')
|
||||
AND gateway_user_id IS NOT NULL
|
||||
AND reservation_amount > 0
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
|
||||
return err
|
||||
}
|
||||
var eventID string
|
||||
var eventSeq int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH next_seq AS (
|
||||
SELECT COALESCE(MAX(seq), 0) + 1 AS seq
|
||||
FROM gateway_task_events
|
||||
WHERE task_id = $1::uuid
|
||||
)
|
||||
INSERT INTO gateway_task_events (
|
||||
task_id, seq, event_type, status, phase, progress, message, payload, simulated
|
||||
)
|
||||
SELECT $1::uuid, next_seq.seq, 'task.failed', 'failed', 'admission', 0,
|
||||
NULLIF($2, ''), '{}'::jsonb, $3
|
||||
FROM next_seq
|
||||
RETURNING id::text, seq`, taskID, message, simulated).Scan(&eventID, &eventSeq); err != nil {
|
||||
return err
|
||||
}
|
||||
callbackURL = strings.TrimSpace(callbackURL)
|
||||
if callbackURL != "" {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
|
||||
ON CONFLICT (task_id, seq, callback_url) DO NOTHING`, taskID, eventID, eventSeq, callbackURL); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return GatewayTask{}, ErrTaskExecutionFinished
|
||||
}
|
||||
return s.GetTask(ctx, taskID)
|
||||
s.notifyTaskAdmissionBestEffort(ctx, "*")
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func (s *Store) RenewTaskExecutionLease(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) error {
|
||||
@@ -888,6 +973,25 @@ WHERE attempt.id = $1::uuid
|
||||
}
|
||||
|
||||
func (s *Store) CancelQueuedTask(ctx context.Context, taskID string, message string) (GatewayTask, bool, error) {
|
||||
return s.cancelQueuedTask(ctx, taskID, message, "", false)
|
||||
}
|
||||
|
||||
func (s *Store) CancelQueuedTaskWithCallback(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
message string,
|
||||
callbackURL string,
|
||||
) (GatewayTask, bool, error) {
|
||||
return s.cancelQueuedTask(ctx, taskID, message, callbackURL, true)
|
||||
}
|
||||
|
||||
func (s *Store) cancelQueuedTask(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
message string,
|
||||
callbackURL string,
|
||||
emitTerminalEvent bool,
|
||||
) (GatewayTask, bool, error) {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
message = "任务已取消"
|
||||
@@ -941,8 +1045,11 @@ WHERE task_id = $1::uuid
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM gateway_task_admissions WHERE task_id = $1::uuid`, taskID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := releaseRuntimeRateReservationsTx(ctx, tx, []string{taskID}); err != nil {
|
||||
return err
|
||||
}
|
||||
payloadJSON, _ := json.Marshal(map[string]any{"taskId": taskID, "reason": "queued_cancelled"})
|
||||
_, err = tx.Exec(ctx, `
|
||||
if _, err = tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
|
||||
)
|
||||
@@ -953,7 +1060,37 @@ WHERE id = $1::uuid
|
||||
AND run_mode IN ('production', 'acceptance', 'acceptance_canary')
|
||||
AND gateway_user_id IS NOT NULL
|
||||
AND reservation_amount > 0
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON))
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
|
||||
return err
|
||||
}
|
||||
if !emitTerminalEvent {
|
||||
return nil
|
||||
}
|
||||
var eventID string
|
||||
var eventSeq int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH next_seq AS (
|
||||
SELECT COALESCE(MAX(seq), 0) + 1 AS seq
|
||||
FROM gateway_task_events
|
||||
WHERE task_id = $1::uuid
|
||||
)
|
||||
INSERT INTO gateway_task_events (
|
||||
task_id, seq, event_type, status, phase, progress, message, payload, simulated
|
||||
)
|
||||
SELECT $1::uuid, next_seq.seq, 'task.cancelled', 'cancelled', 'admission', 0,
|
||||
NULLIF($2, ''), '{}'::jsonb, $3
|
||||
FROM next_seq
|
||||
RETURNING id::text, seq`, taskID, message, task.RunMode == "simulation").Scan(&eventID, &eventSeq); err != nil {
|
||||
return err
|
||||
}
|
||||
callbackURL = strings.TrimSpace(callbackURL)
|
||||
if callbackURL == "" {
|
||||
return nil
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
|
||||
ON CONFLICT (task_id, seq, callback_url) DO NOTHING`, taskID, eventID, eventSeq, callbackURL)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user