feat(routing): 引入多执行池智能调度

将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。

实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。

新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
2026-08-05 22:25:37 +08:00
parent 03c0873649
commit 7786692d32
58 changed files with 4510 additions and 348 deletions
@@ -151,7 +151,7 @@ func TestExpiredExecutionLeaseDoesNotReplayAmbiguousUpstreamSubmission(t *testin
if err != nil {
t.Fatal(err)
}
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_timeout" {
t.Fatalf("manual review task=%+v", review)
}
var attemptStatus string
@@ -162,7 +162,7 @@ FROM gateway_task_attempts
WHERE id=$1::uuid`, attemptID).Scan(&attemptStatus, &attemptErrorCode); err != nil {
t.Fatal(err)
}
if attemptStatus != "failed" || attemptErrorCode != "upstream_submission_unknown" {
if attemptStatus != "failed" || attemptErrorCode != "upstream_timeout" {
t.Fatalf("manual review attempt status=%s error=%s", attemptStatus, attemptErrorCode)
}
var outboxStatus string
@@ -174,11 +174,84 @@ FROM settlement_outbox
WHERE task_id=$1::uuid AND event_type='task.billing.review'`, created.ID).Scan(&outboxStatus, &outboxAction, &reviewReason); err != nil {
t.Fatal(err)
}
if outboxStatus != "manual_review" || outboxAction != "release" || reviewReason != "upstream_submission_unknown" {
if outboxStatus != "manual_review" || outboxAction != "release" || reviewReason != "upstream_timeout" {
t.Fatalf("review outbox status=%s action=%s reason=%s", outboxStatus, outboxAction, reviewReason)
}
}
func TestResolveInterruptedTaskExecutionPreservesSubmissionFence(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
user := &auth.User{ID: "interrupted-worker-" + uuid.NewString(), Source: "gateway"}
type testCase struct {
name string
submissionStatus string
remoteTaskID string
wantManualReview bool
}
for _, tc := range []testCase{
{name: "not submitted requeues", submissionStatus: "not_submitted"},
{name: "ambiguous submission stops", submissionStatus: "submitting", wantManualReview: true},
{name: "known remote task requeues for polling", submissionStatus: "response_received", remoteTaskID: "remote-" + uuid.NewString()},
} {
t.Run(tc.name, func(t *testing.T) {
created, err := db.CreateTask(ctx, CreateTaskInput{
Kind: "images.generations", Model: "interrupted-worker-model", RunMode: "production", Async: true,
Request: map[string]any{"model": "interrupted-worker-model"},
}, user)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
})
executionToken := uuid.NewString()
if _, err := db.ClaimTaskExecution(ctx, created.ID, executionToken, 5*time.Minute); err != nil {
t.Fatal(err)
}
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
TaskID: created.ID, ExecutionToken: executionToken, AttemptNo: 1, Status: "running",
})
if err != nil {
t.Fatal(err)
}
if err := db.SetAttemptUpstreamSubmissionStatusForExecution(
ctx, attemptID, created.ID, executionToken, tc.submissionStatus,
); err != nil {
t.Fatal(err)
}
if tc.remoteTaskID != "" {
if err := db.SetTaskRemoteTask(ctx, created.ID, executionToken, attemptID, tc.remoteTaskID, nil); err != nil {
t.Fatal(err)
}
}
resolved, err := db.ResolveInterruptedTaskExecution(ctx, created.ID, executionToken)
if tc.wantManualReview {
if !errors.Is(err, ErrTaskExecutionManualReview) {
t.Fatalf("resolve error=%v, want ErrTaskExecutionManualReview", err)
}
failed, getErr := db.GetTask(ctx, created.ID)
if getErr != nil {
t.Fatal(getErr)
}
if failed.Status != "failed" || failed.ErrorCode != "upstream_timeout" || failed.SubmissionState != "submission_confirmation_pending" {
t.Fatalf("ambiguous task=%+v", failed)
}
return
}
if err != nil {
t.Fatal(err)
}
if resolved.Status != "queued" || resolved.ExecutionToken != "" || resolved.RemoteTaskID != tc.remoteTaskID {
t.Fatalf("resolved task=%+v", resolved)
}
})
}
}
func TestQueuedPreparationDoesNotReplayInterruptedUpstreamSubmission(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
@@ -244,7 +317,7 @@ WHERE id=$1::uuid`, created.ID); err != nil {
if err != nil {
t.Fatal(err)
}
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_timeout" {
t.Fatalf("manual review task=%+v", review)
}
})
@@ -365,7 +438,7 @@ func TestFinishTaskManualReviewCreatesVisibleBillingRecord(t *testing.T) {
}
if _, err := db.FinishTaskManualReview(ctx, FinishTaskManualReviewInput{
TaskID: created.ID, ExecutionToken: token, AttemptID: attemptID, TaskStatus: "failed",
Code: "upstream_submission_unknown", Message: "upstream submission result is unknown",
Code: "upstream_timeout", Message: "upstream submission confirmation timed out",
}); err != nil {
t.Fatal(err)
}
@@ -374,7 +447,7 @@ func TestFinishTaskManualReviewCreatesVisibleBillingRecord(t *testing.T) {
SELECT EXISTS (
SELECT 1 FROM settlement_outbox
WHERE task_id=$1::uuid AND status='manual_review'
AND action='release' AND manual_review_reason='upstream_submission_unknown'
AND action='release' AND manual_review_reason='upstream_timeout'
)`, created.ID).Scan(&visible); err != nil {
t.Fatal(err)
}