fix(acceptance): 清理中止验收的未提交任务

验收失败后,排队及尚未提交上游的任务会继续占用队列与 API 数据库连接池,导致下一轮容量测试被历史负载污染。

中止验收时仅取消确认未提交上游的任务,删除其未提交 attempt 与准入记录,释放租约并生成幂等退款事件;提交中或已收到上游响应的任务继续自然收敛。新建验收前也会清理历史失败 Run 的安全遗留任务,并等待其余任务终态。

验证:Go 全量测试通过;临时 PostgreSQL 集成测试覆盖排队、运行未提交、提交中与退款事件;gofmt、bash -n、ShellCheck、git diff --check 通过。
This commit is contained in:
2026-07-31 03:16:38 +08:00
parent 1bd9e618ca
commit 2606cc8223
3 changed files with 297 additions and 0 deletions
@@ -114,6 +114,68 @@ WHERE setting_key = $1`, SystemSettingGatewayTrafficMode)
if err != nil {
t.Fatalf("activate retry acceptance run: %v", err)
}
createAcceptanceTask := func(label string) GatewayTask {
t.Helper()
task, createErr := db.CreateTask(ctx, CreateTaskInput{
Kind: "images.edits", Model: "acceptance-cleanup-" + label,
RunMode: "acceptance", AcceptanceRunID: failedRun.ID,
Request: map[string]any{"prompt": "acceptance cleanup integration test"},
}, &auth.User{ID: "acceptance-user", Source: "gateway"})
if createErr != nil {
t.Fatalf("create acceptance cleanup task: %v", createErr)
}
return task
}
queuedTask := createAcceptanceTask("queued")
runningTask := createAcceptanceTask("running")
notSubmittedTask := createAcceptanceTask("not-submitted")
submittingTask := createAcceptanceTask("submitting")
var billingUserID string
if err := db.pool.QueryRow(ctx, `
INSERT INTO gateway_users (user_key, username)
VALUES ($1, $1)
RETURNING id::text`, "acceptance-cleanup-"+time.Now().Format("150405.000000000")).Scan(&billingUserID); err != nil {
t.Fatalf("create acceptance cleanup billing user: %v", err)
}
if _, err := db.pool.Exec(ctx, `
UPDATE gateway_tasks
SET gateway_user_id = $2::uuid,
reservation_amount = 1,
billing_status = 'not_started'
WHERE id = $1::uuid`, queuedTask.ID, billingUserID); err != nil {
t.Fatalf("reserve acceptance cleanup task billing: %v", err)
}
if _, err := db.ClaimTaskExecution(ctx, runningTask.ID, "10000000-0000-4000-8000-000000000001", time.Minute); err != nil {
t.Fatalf("claim safe running acceptance task: %v", err)
}
notSubmittedClaim, err := db.ClaimTaskExecution(ctx, notSubmittedTask.ID, "10000000-0000-4000-8000-000000000002", time.Minute)
if err != nil {
t.Fatalf("claim not-submitted acceptance task: %v", err)
}
notSubmittedAttemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
TaskID: notSubmittedTask.ID, ExecutionToken: notSubmittedClaim.ExecutionToken,
AttemptNo: 1, Status: "running", Simulated: true,
})
if err != nil {
t.Fatalf("create not-submitted acceptance task attempt: %v", err)
}
submittingClaim, err := db.ClaimTaskExecution(ctx, submittingTask.ID, "10000000-0000-4000-8000-000000000003", time.Minute)
if err != nil {
t.Fatalf("claim submitting acceptance task: %v", err)
}
submittingAttemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
TaskID: submittingTask.ID, ExecutionToken: submittingClaim.ExecutionToken,
AttemptNo: 1, Status: "running", Simulated: true,
})
if err != nil {
t.Fatalf("create submitting acceptance task attempt: %v", err)
}
if _, err := db.pool.Exec(ctx, `
UPDATE gateway_task_attempts
SET upstream_submission_status = 'submitting', upstream_submission_updated_at = now()
WHERE id = $1::uuid`, submittingAttemptID); err != nil {
t.Fatalf("mark acceptance task attempt submitting: %v", err)
}
if _, err := db.FinishAcceptanceRun(ctx, FinishAcceptanceRunInput{
RunID: failedRun.ID, Passed: false, FailureReason: "load failed",
}); err != nil {
@@ -133,4 +195,46 @@ WHERE setting_key = $1`, SystemSettingGatewayTrafficMode)
if aborted.Mode != "live" || aborted.Revision != failedMode.Revision+1 {
t.Fatalf("unexpected aborted mode: %+v", aborted)
}
var safeCancelled, submittingRunning, notSubmittedAttempts, releaseEvents int
if err := db.pool.QueryRow(ctx, `
SELECT
count(*) FILTER (
WHERE id = ANY($1::uuid[])
AND status = 'cancelled'
AND error_code = 'acceptance_run_aborted'
),
count(*) FILTER (
WHERE id = $2::uuid
AND status = 'running'
),
(SELECT count(*) FROM gateway_task_attempts WHERE id = $3::uuid),
(SELECT count(*)
FROM settlement_outbox
WHERE task_id = $4::uuid
AND action = 'release'
AND status = 'pending')
FROM gateway_tasks`,
[]string{queuedTask.ID, runningTask.ID, notSubmittedTask.ID},
submittingTask.ID,
notSubmittedAttemptID,
queuedTask.ID,
).Scan(&safeCancelled, &submittingRunning, &notSubmittedAttempts, &releaseEvents); err != nil {
t.Fatalf("read aborted acceptance task states: %v", err)
}
if safeCancelled != 3 || submittingRunning != 1 || notSubmittedAttempts != 0 || releaseEvents != 1 {
t.Fatalf(
"abort cleanup safe_cancelled=%d submitting_running=%d not_submitted_attempts=%d release_events=%d",
safeCancelled,
submittingRunning,
notSubmittedAttempts,
releaseEvents,
)
}
if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_tasks WHERE id = ANY($1::uuid[])`,
[]string{queuedTask.ID, runningTask.ID, notSubmittedTask.ID, submittingTask.ID}); err != nil {
t.Fatalf("delete acceptance cleanup tasks: %v", err)
}
if _, err := db.pool.Exec(ctx, `DELETE FROM gateway_users WHERE id = $1::uuid`, billingUserID); err != nil {
t.Fatalf("delete acceptance cleanup billing user: %v", err)
}
}