fix(queue): 防止过期执行占位阻塞与重复提交
队列恢复后,仍在运行标记中的 River job 可能保留 waiting admission,形成全局 FIFO 队头阻塞;同时旧执行在租约被接管后仍可能创建 attempt 或切换到 submitting。 新增 stale admission 自动让出机制,并用当前 execution token 对 attempt 创建和上游提交状态切换做 fencing。任务、River job 和结算状态均不在让出流程中改写。 验证:go vet ./...;env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1;真实 PostgreSQL 集成测试覆盖 stale admission 让出与旧 token 提交拦截。
This commit is contained in:
@@ -48,6 +48,15 @@ func TestTaskIdempotencyAndExecutionLease(t *testing.T) {
|
||||
if err := db.RenewTaskExecutionLease(ctx, created.Task.ID, firstToken, 5*time.Minute); err != nil {
|
||||
t.Fatalf("renew first lease: %v", err)
|
||||
}
|
||||
firstAttemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
|
||||
TaskID: created.Task.ID,
|
||||
ExecutionToken: firstToken,
|
||||
AttemptNo: 1,
|
||||
Status: "running",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create first owned attempt: %v", err)
|
||||
}
|
||||
if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.Task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -55,6 +64,41 @@ func TestTaskIdempotencyAndExecutionLease(t *testing.T) {
|
||||
if _, err := db.ClaimTaskExecution(ctx, created.Task.ID, secondToken, 5*time.Minute); err != nil {
|
||||
t.Fatalf("take over expired lease: %v", err)
|
||||
}
|
||||
if err := db.SetAttemptUpstreamSubmissionStatusForExecution(
|
||||
ctx,
|
||||
firstAttemptID,
|
||||
created.Task.ID,
|
||||
firstToken,
|
||||
"submitting",
|
||||
); !errors.Is(err, ErrTaskExecutionLeaseLost) {
|
||||
t.Fatalf("stale attempt submission fence error=%v", err)
|
||||
}
|
||||
if _, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
|
||||
TaskID: created.Task.ID,
|
||||
ExecutionToken: firstToken,
|
||||
AttemptNo: 2,
|
||||
Status: "running",
|
||||
}); !errors.Is(err, ErrTaskExecutionLeaseLost) {
|
||||
t.Fatalf("stale attempt creation fence error=%v", err)
|
||||
}
|
||||
secondAttemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
|
||||
TaskID: created.Task.ID,
|
||||
ExecutionToken: secondToken,
|
||||
AttemptNo: 2,
|
||||
Status: "running",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create current owned attempt: %v", err)
|
||||
}
|
||||
if err := db.SetAttemptUpstreamSubmissionStatusForExecution(
|
||||
ctx,
|
||||
secondAttemptID,
|
||||
created.Task.ID,
|
||||
secondToken,
|
||||
"submitting",
|
||||
); err != nil {
|
||||
t.Fatalf("current attempt submission fence: %v", err)
|
||||
}
|
||||
if _, err := db.FinishTaskFailure(ctx, FinishTaskFailureInput{TaskID: created.Task.ID, ExecutionToken: firstToken, Code: "old_worker", Message: "old"}); !errors.Is(err, ErrTaskExecutionLeaseLost) {
|
||||
t.Fatalf("old worker terminal error=%v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user