fix(queue): 回收失效任务并阻止不确定提交重放

Worker 进程心跳不能证明单个 River job goroutine 仍存活，改以任务执行租约作为回收所有权栅栏。\n\n排队任务若上一次 attempt 在租约中断时处于 submitting 或 response_received，则转入人工复核并生成 release 记录，不再重新提交上游；明确被上游拒绝的响应仍允许重试。\n\n验证：go vet ./...；env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1；真实 PostgreSQL 下相关 store 与 River 集成测试通过。
2026-07-30 19:15:19 +08:00
parent 9e06ed6162
commit f33d6d64e0
4 changed files with 256 additions and 65 deletions
@@ -168,6 +168,127 @@ WHERE task_id=$1::uuid AND event_type='task.billing.review'`, created.ID).Scan(&
 	}
 }

+func TestQueuedPreparationDoesNotReplayInterruptedUpstreamSubmission(t *testing.T) {
+	db := billingV2IntegrationStore(t)
+	ctx := context.Background()
+	tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db)
+
+	for _, submissionStatus := range []string{"submitting", "response_received"} {
+		t.Run(submissionStatus, func(t *testing.T) {
+			user := &auth.User{
+				ID:              "billing-queued-review-" + uuid.NewString(),
+				GatewayUserID:   gatewayUserID,
+				GatewayTenantID: tenantID,
+			}
+			created, err := db.CreateTask(ctx, CreateTaskInput{
+				Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
+				Request: map[string]any{"model": "billing-v2-model"},
+			}, user)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() {
+				_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
+			})
+
+			token := uuid.NewString()
+			if _, err := db.ClaimTaskExecution(ctx, created.ID, token, 5*time.Minute); err != nil {
+				t.Fatal(err)
+			}
+			attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
+				TaskID: created.ID, AttemptNo: 1, Status: "running",
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, submissionStatus); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := db.pool.Exec(ctx, `
+UPDATE gateway_task_attempts
+SET status='failed',
+    retryable=true,
+    error_code='execution_lease_expired',
+    error_message='attempt execution lease expired',
+    finished_at=now()
+WHERE id=$1::uuid`, attemptID); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := db.pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET status='queued',
+    execution_token=NULL,
+    execution_lease_expires_at=NULL,
+    heartbeat_at=NULL,
+    locked_at=NULL,
+    updated_at=now()
+WHERE id=$1::uuid`, created.ID); err != nil {
+				t.Fatal(err)
+			}
+
+			if _, err := db.ClaimTaskPreparation(ctx, created.ID, uuid.NewString(), 5*time.Minute); !errors.Is(err, ErrTaskExecutionManualReview) {
+				t.Fatalf("queued ambiguous preparation error=%v", err)
+			}
+			review, err := db.GetTask(ctx, created.ID)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
+				t.Fatalf("manual review task=%+v", review)
+			}
+		})
+	}
+}
+
+func TestQueuedPreparationCanResumeAfterKnownRejectedResponse(t *testing.T) {
+	db := billingV2IntegrationStore(t)
+	ctx := context.Background()
+	_, gatewayUserID := seedWalletReservationUser(t, ctx, db)
+	user := &auth.User{ID: "billing-queued-known-response-" + uuid.NewString(), GatewayUserID: gatewayUserID}
+	created, err := db.CreateTask(ctx, CreateTaskInput{
+		Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
+		Request: map[string]any{"model": "billing-v2-model"},
+	}, user)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
+	})
+
+	if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil {
+		t.Fatal(err)
+	}
+	attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
+		TaskID: created.ID, AttemptNo: 1, Status: "running",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "response_received"); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.FinishTaskAttempt(ctx, FinishTaskAttemptInput{
+		AttemptID: attemptID, Status: "failed", ErrorCode: "upstream_rejected",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.pool.Exec(ctx, `
+UPDATE gateway_tasks
+SET status='queued',
+    execution_token=NULL,
+    execution_lease_expires_at=NULL,
+    heartbeat_at=NULL,
+    locked_at=NULL,
+    updated_at=now()
+WHERE id=$1::uuid`, created.ID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.ClaimTaskPreparation(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil {
+		t.Fatalf("known rejected response should remain retryable: %v", err)
+	}
+}
+
 func TestExpiredExecutionLeaseCanResumeAfterKnownRejectedResponse(t *testing.T) {
 	db := billingV2IntegrationStore(t)
 	ctx := context.Background()