fix(queue): 防止过期执行占位阻塞与重复提交

队列恢复后,仍在运行标记中的 River job 可能保留 waiting admission,形成全局 FIFO 队头阻塞;同时旧执行在租约被接管后仍可能创建 attempt 或切换到 submitting。

新增 stale admission 自动让出机制,并用当前 execution token 对 attempt 创建和上游提交状态切换做 fencing。任务、River job 和结算状态均不在让出流程中改写。

验证:go vet ./...;env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1;真实 PostgreSQL 集成测试覆盖 stale admission 让出与旧 token 提交拦截。
This commit is contained in:
2026-07-30 18:25:00 +08:00
parent fa818e9ebf
commit 38f87b6970
7 changed files with 306 additions and 8 deletions
@@ -48,6 +48,15 @@ func TestTaskIdempotencyAndExecutionLease(t *testing.T) {
if err := db.RenewTaskExecutionLease(ctx, created.Task.ID, firstToken, 5*time.Minute); err != nil {
t.Fatalf("renew first lease: %v", err)
}
firstAttemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
TaskID: created.Task.ID,
ExecutionToken: firstToken,
AttemptNo: 1,
Status: "running",
})
if err != nil {
t.Fatalf("create first owned attempt: %v", err)
}
if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.Task.ID); err != nil {
t.Fatal(err)
}
@@ -55,6 +64,41 @@ func TestTaskIdempotencyAndExecutionLease(t *testing.T) {
if _, err := db.ClaimTaskExecution(ctx, created.Task.ID, secondToken, 5*time.Minute); err != nil {
t.Fatalf("take over expired lease: %v", err)
}
if err := db.SetAttemptUpstreamSubmissionStatusForExecution(
ctx,
firstAttemptID,
created.Task.ID,
firstToken,
"submitting",
); !errors.Is(err, ErrTaskExecutionLeaseLost) {
t.Fatalf("stale attempt submission fence error=%v", err)
}
if _, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
TaskID: created.Task.ID,
ExecutionToken: firstToken,
AttemptNo: 2,
Status: "running",
}); !errors.Is(err, ErrTaskExecutionLeaseLost) {
t.Fatalf("stale attempt creation fence error=%v", err)
}
secondAttemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
TaskID: created.Task.ID,
ExecutionToken: secondToken,
AttemptNo: 2,
Status: "running",
})
if err != nil {
t.Fatalf("create current owned attempt: %v", err)
}
if err := db.SetAttemptUpstreamSubmissionStatusForExecution(
ctx,
secondAttemptID,
created.Task.ID,
secondToken,
"submitting",
); err != nil {
t.Fatalf("current attempt submission fence: %v", err)
}
if _, err := db.FinishTaskFailure(ctx, FinishTaskFailureInput{TaskID: created.Task.ID, ExecutionToken: firstToken, Code: "old_worker", Message: "old"}); !errors.Is(err, ErrTaskExecutionLeaseLost) {
t.Fatalf("old worker terminal error=%v", err)
}