fix(queue): 防止过期执行占位阻塞与重复提交
队列恢复后,仍在运行标记中的 River job 可能保留 waiting admission,形成全局 FIFO 队头阻塞;同时旧执行在租约被接管后仍可能创建 attempt 或切换到 submitting。 新增 stale admission 自动让出机制,并用当前 execution token 对 attempt 创建和上游提交状态切换做 fencing。任务、River job 和结算状态均不在让出流程中改写。 验证:go vet ./...;env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1;真实 PostgreSQL 集成测试覆盖 stale admission 让出与旧 token 提交拦截。
This commit is contained in:
@@ -1206,6 +1206,7 @@ func (s *Service) runCandidate(
|
||||
|
||||
attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{
|
||||
TaskID: task.ID,
|
||||
ExecutionToken: task.ExecutionToken,
|
||||
AttemptNo: attemptNo,
|
||||
PlatformID: candidate.PlatformID,
|
||||
PlatformModelID: candidate.PlatformModelID,
|
||||
@@ -1334,7 +1335,13 @@ func (s *Service) runCandidate(
|
||||
if strings.TrimSpace(task.RemoteTaskID) != "" {
|
||||
submissionStatus = "response_received"
|
||||
markUpstreamSubmissionStarted(ctx)
|
||||
if err := s.store.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, submissionStatus); err != nil {
|
||||
if err := s.store.SetAttemptUpstreamSubmissionStatusForExecution(
|
||||
ctx,
|
||||
attemptID,
|
||||
task.ID,
|
||||
task.ExecutionToken,
|
||||
submissionStatus,
|
||||
); err != nil {
|
||||
return clients.Response{}, fmt.Errorf("restore upstream submission status: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -1345,7 +1352,13 @@ func (s *Service) runCandidate(
|
||||
if submissionStatus == status {
|
||||
return nil
|
||||
}
|
||||
if err := s.store.SetAttemptUpstreamSubmissionStatus(context.WithoutCancel(ctx), attemptID, status); err != nil {
|
||||
if err := s.store.SetAttemptUpstreamSubmissionStatusForExecution(
|
||||
context.WithoutCancel(ctx),
|
||||
attemptID,
|
||||
task.ID,
|
||||
task.ExecutionToken,
|
||||
status,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
submissionStatus = status
|
||||
@@ -1883,6 +1896,7 @@ func (s *Service) recordFailedAttempt(ctx context.Context, input failedAttemptRe
|
||||
metrics := mergeMetrics(append([]map[string]any{baseMetrics, failure}, input.ExtraMetrics...)...)
|
||||
attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{
|
||||
TaskID: input.Task.ID,
|
||||
ExecutionToken: input.Task.ExecutionToken,
|
||||
AttemptNo: attemptNo,
|
||||
PlatformID: platformID,
|
||||
PlatformModelID: platformModelID,
|
||||
|
||||
Reference in New Issue
Block a user