feat(routing): 引入多执行池智能调度
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
@@ -151,7 +151,7 @@ func TestExpiredExecutionLeaseDoesNotReplayAmbiguousUpstreamSubmission(t *testin
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
|
||||
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_timeout" {
|
||||
t.Fatalf("manual review task=%+v", review)
|
||||
}
|
||||
var attemptStatus string
|
||||
@@ -162,7 +162,7 @@ FROM gateway_task_attempts
|
||||
WHERE id=$1::uuid`, attemptID).Scan(&attemptStatus, &attemptErrorCode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if attemptStatus != "failed" || attemptErrorCode != "upstream_submission_unknown" {
|
||||
if attemptStatus != "failed" || attemptErrorCode != "upstream_timeout" {
|
||||
t.Fatalf("manual review attempt status=%s error=%s", attemptStatus, attemptErrorCode)
|
||||
}
|
||||
var outboxStatus string
|
||||
@@ -174,11 +174,84 @@ FROM settlement_outbox
|
||||
WHERE task_id=$1::uuid AND event_type='task.billing.review'`, created.ID).Scan(&outboxStatus, &outboxAction, &reviewReason); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outboxStatus != "manual_review" || outboxAction != "release" || reviewReason != "upstream_submission_unknown" {
|
||||
if outboxStatus != "manual_review" || outboxAction != "release" || reviewReason != "upstream_timeout" {
|
||||
t.Fatalf("review outbox status=%s action=%s reason=%s", outboxStatus, outboxAction, reviewReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveInterruptedTaskExecutionPreservesSubmissionFence(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx := context.Background()
|
||||
user := &auth.User{ID: "interrupted-worker-" + uuid.NewString(), Source: "gateway"}
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
submissionStatus string
|
||||
remoteTaskID string
|
||||
wantManualReview bool
|
||||
}
|
||||
for _, tc := range []testCase{
|
||||
{name: "not submitted requeues", submissionStatus: "not_submitted"},
|
||||
{name: "ambiguous submission stops", submissionStatus: "submitting", wantManualReview: true},
|
||||
{name: "known remote task requeues for polling", submissionStatus: "response_received", remoteTaskID: "remote-" + uuid.NewString()},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
created, err := db.CreateTask(ctx, CreateTaskInput{
|
||||
Kind: "images.generations", Model: "interrupted-worker-model", RunMode: "production", Async: true,
|
||||
Request: map[string]any{"model": "interrupted-worker-model"},
|
||||
}, user)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
|
||||
})
|
||||
|
||||
executionToken := uuid.NewString()
|
||||
if _, err := db.ClaimTaskExecution(ctx, created.ID, executionToken, 5*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
|
||||
TaskID: created.ID, ExecutionToken: executionToken, AttemptNo: 1, Status: "running",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.SetAttemptUpstreamSubmissionStatusForExecution(
|
||||
ctx, attemptID, created.ID, executionToken, tc.submissionStatus,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.remoteTaskID != "" {
|
||||
if err := db.SetTaskRemoteTask(ctx, created.ID, executionToken, attemptID, tc.remoteTaskID, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
resolved, err := db.ResolveInterruptedTaskExecution(ctx, created.ID, executionToken)
|
||||
if tc.wantManualReview {
|
||||
if !errors.Is(err, ErrTaskExecutionManualReview) {
|
||||
t.Fatalf("resolve error=%v, want ErrTaskExecutionManualReview", err)
|
||||
}
|
||||
failed, getErr := db.GetTask(ctx, created.ID)
|
||||
if getErr != nil {
|
||||
t.Fatal(getErr)
|
||||
}
|
||||
if failed.Status != "failed" || failed.ErrorCode != "upstream_timeout" || failed.SubmissionState != "submission_confirmation_pending" {
|
||||
t.Fatalf("ambiguous task=%+v", failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolved.Status != "queued" || resolved.ExecutionToken != "" || resolved.RemoteTaskID != tc.remoteTaskID {
|
||||
t.Fatalf("resolved task=%+v", resolved)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedPreparationDoesNotReplayInterruptedUpstreamSubmission(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx := context.Background()
|
||||
@@ -244,7 +317,7 @@ WHERE id=$1::uuid`, created.ID); err != nil {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
|
||||
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_timeout" {
|
||||
t.Fatalf("manual review task=%+v", review)
|
||||
}
|
||||
})
|
||||
@@ -365,7 +438,7 @@ func TestFinishTaskManualReviewCreatesVisibleBillingRecord(t *testing.T) {
|
||||
}
|
||||
if _, err := db.FinishTaskManualReview(ctx, FinishTaskManualReviewInput{
|
||||
TaskID: created.ID, ExecutionToken: token, AttemptID: attemptID, TaskStatus: "failed",
|
||||
Code: "upstream_submission_unknown", Message: "upstream submission result is unknown",
|
||||
Code: "upstream_timeout", Message: "upstream submission confirmation timed out",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -374,7 +447,7 @@ func TestFinishTaskManualReviewCreatesVisibleBillingRecord(t *testing.T) {
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM settlement_outbox
|
||||
WHERE task_id=$1::uuid AND status='manual_review'
|
||||
AND action='release' AND manual_review_reason='upstream_submission_unknown'
|
||||
AND action='release' AND manual_review_reason='upstream_timeout'
|
||||
)`, created.ID).Scan(&visible); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user