fix(queue): 回收失效任务并阻止不确定提交重放
Worker 进程心跳不能证明单个 River job goroutine 仍存活，改以任务执行租约作为回收所有权栅栏。\n\n排队任务若上一次 attempt 在租约中断时处于 submitting 或 response_received，则转入人工复核并生成 release 记录，不再重新提交上游；明确被上游拒绝的响应仍允许重试。\n\n验证：go vet ./...；env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1；真实 PostgreSQL 下相关 store 与 River 集成测试通过。
This commit is contained in:
@@ -304,58 +304,15 @@ FOR UPDATE`, taskID).Scan(&queuedReady, &runningExpired, &production, &hasGatewa
|
||||
if !queuedReady && !runningExpired {
|
||||
return ErrTaskExecutionLeaseUnavailable
|
||||
}
|
||||
if runningExpired && production {
|
||||
var submissionAmbiguous bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE((
|
||||
SELECT (
|
||||
(status = 'running' AND upstream_submission_status IN ('submitting', 'response_received'))
|
||||
OR (status = 'failed' AND upstream_submission_status = 'submitting')
|
||||
)
|
||||
FROM gateway_task_attempts
|
||||
WHERE task_id = $1::uuid
|
||||
ORDER BY attempt_no DESC, started_at DESC
|
||||
LIMIT 1
|
||||
), false)`, taskID).Scan(&submissionAmbiguous); err != nil {
|
||||
if (queuedReady || runningExpired) && production {
|
||||
submissionAmbiguous, err := taskExecutionRequiresManualReviewTx(ctx, tx, taskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if submissionAmbiguous {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'failed',
|
||||
billing_status = CASE WHEN $2 THEN 'manual_review' ELSE 'not_required' END,
|
||||
billing_updated_at = now(),
|
||||
error = NULL,
|
||||
error_code = 'upstream_submission_unknown',
|
||||
error_message = 'upstream submission result is unknown',
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
remote_task_payload = '{}'::jsonb,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil {
|
||||
if err := markTaskExecutionManualReviewTx(ctx, tx, taskID, hasGatewayUser); err != nil {
|
||||
return err
|
||||
}
|
||||
if hasGatewayUser {
|
||||
payloadJSON, _ := json.Marshal(map[string]any{
|
||||
"taskId": taskID, "classification": "upstream_submission_unknown",
|
||||
})
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
task_id, event_type, action, amount, currency, pricing_snapshot, payload,
|
||||
status, next_attempt_at, manual_review_reason
|
||||
)
|
||||
SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency,
|
||||
pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_submission_unknown'
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
manualReview = true
|
||||
return nil
|
||||
}
|
||||
@@ -389,11 +346,116 @@ RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func taskExecutionRequiresManualReviewTx(ctx context.Context, tx pgx.Tx, taskID string) (bool, error) {
|
||||
var submissionAmbiguous bool
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE((
|
||||
SELECT (
|
||||
(status = 'running' AND upstream_submission_status IN ('submitting', 'response_received'))
|
||||
OR (status = 'failed' AND upstream_submission_status = 'submitting')
|
||||
OR (
|
||||
status = 'failed'
|
||||
AND upstream_submission_status = 'response_received'
|
||||
AND error_code = 'execution_lease_expired'
|
||||
)
|
||||
)
|
||||
FROM gateway_task_attempts
|
||||
WHERE task_id = $1::uuid
|
||||
ORDER BY attempt_no DESC, started_at DESC
|
||||
LIMIT 1
|
||||
), false)`, taskID).Scan(&submissionAmbiguous)
|
||||
return submissionAmbiguous, err
|
||||
}
|
||||
|
||||
func markTaskExecutionManualReviewTx(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
taskID string,
|
||||
hasGatewayUser bool,
|
||||
) error {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'failed',
|
||||
billing_status = CASE WHEN $2 THEN 'manual_review' ELSE 'not_required' END,
|
||||
billing_updated_at = now(),
|
||||
error = NULL,
|
||||
error_code = 'upstream_submission_unknown',
|
||||
error_message = 'upstream submission result is unknown',
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
remote_task_payload = '{}'::jsonb,
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil {
|
||||
return err
|
||||
}
|
||||
if !hasGatewayUser {
|
||||
return nil
|
||||
}
|
||||
payloadJSON, _ := json.Marshal(map[string]any{
|
||||
"taskId": taskID, "classification": "upstream_submission_unknown",
|
||||
})
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
task_id, event_type, action, amount, currency, pricing_snapshot, payload,
|
||||
status, next_attempt_at, manual_review_reason
|
||||
)
|
||||
SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency,
|
||||
pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_submission_unknown'
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ClaimTaskPreparation(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) (GatewayTask, error) {
|
||||
if leaseTTL <= 0 {
|
||||
leaseTTL = 5 * time.Minute
|
||||
}
|
||||
task, err := scanGatewayTask(s.pool.QueryRow(ctx, `
|
||||
var task GatewayTask
|
||||
manualReview := false
|
||||
err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
|
||||
var queuedReady bool
|
||||
var production bool
|
||||
var hasGatewayUser bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT status = 'queued'
|
||||
AND next_run_at <= now()
|
||||
AND (
|
||||
execution_token IS NULL
|
||||
OR execution_lease_expires_at IS NULL
|
||||
OR execution_lease_expires_at <= now()
|
||||
),
|
||||
run_mode = 'production',
|
||||
gateway_user_id IS NOT NULL
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
FOR UPDATE`, taskID).Scan(&queuedReady, &production, &hasGatewayUser); err != nil {
|
||||
return err
|
||||
}
|
||||
if !queuedReady {
|
||||
return ErrTaskExecutionLeaseUnavailable
|
||||
}
|
||||
if production {
|
||||
submissionAmbiguous, err := taskExecutionRequiresManualReviewTx(ctx, tx, taskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if submissionAmbiguous {
|
||||
if err := markTaskExecutionManualReviewTx(ctx, tx, taskID, hasGatewayUser); err != nil {
|
||||
return err
|
||||
}
|
||||
manualReview = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
var err error
|
||||
task, err = scanGatewayTask(tx.QueryRow(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET execution_token = $2::uuid,
|
||||
execution_lease_expires_at = now() + ($3::int * interval '1 second'),
|
||||
@@ -409,9 +471,17 @@ WHERE id = $1::uuid
|
||||
OR execution_lease_expires_at <= now()
|
||||
)
|
||||
RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second)))
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return GatewayTask{}, ErrTaskExecutionLeaseUnavailable
|
||||
}
|
||||
if err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
if manualReview {
|
||||
return GatewayTask{}, ErrTaskExecutionManualReview
|
||||
}
|
||||
return task, err
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user