feat(routing): 引入多执行池智能调度

将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。

实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。

新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
2026-08-05 22:25:37 +08:00
parent 03c0873649
commit 7786692d32
58 changed files with 4510 additions and 348 deletions
+76 -6
View File
@@ -392,8 +392,8 @@ func markTaskExecutionManualReviewTx(
UPDATE gateway_task_attempts
SET status = 'failed',
retryable = false,
error_code = 'upstream_submission_unknown',
error_message = 'upstream submission result is unknown',
error_code = 'upstream_timeout',
error_message = 'upstream submission confirmation timed out',
finished_at = COALESCE(finished_at, now()),
upstream_submission_updated_at = now()
WHERE task_id = $1::uuid
@@ -407,14 +407,15 @@ SET status = 'failed',
billing_status = CASE WHEN $2 THEN 'manual_review' ELSE 'not_required' END,
billing_updated_at = now(),
error = NULL,
error_code = 'upstream_submission_unknown',
error_message = 'upstream submission result is unknown',
error_code = 'upstream_timeout',
error_message = 'upstream submission confirmation timed out',
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
remote_task_payload = '{}'::jsonb,
submission_state = 'submission_confirmation_pending',
finished_at = now(),
updated_at = now()
WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil {
@@ -424,7 +425,7 @@ WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil {
return nil
}
payloadJSON, _ := json.Marshal(map[string]any{
"taskId": taskID, "classification": "upstream_submission_unknown",
"taskId": taskID, "classification": "upstream_timeout",
})
if _, err := tx.Exec(ctx, `
INSERT INTO settlement_outbox (
@@ -432,7 +433,7 @@ INSERT INTO settlement_outbox (
status, next_attempt_at, manual_review_reason
)
SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency,
pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_submission_unknown'
pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_timeout'
FROM gateway_tasks
WHERE id = $1::uuid
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
@@ -866,6 +867,75 @@ WHERE task_id = $1::uuid
return task, changed, nil
}
// ResolveInterruptedTaskExecution atomically preserves the no-duplicate-submit
// invariant when a Worker disappears or its execution context is cancelled.
// A known remote task is safe to requeue for polling takeover. An ambiguous
// submission without a remote ID is moved to manual review instead of replayed.
func (s *Store) ResolveInterruptedTaskExecution(
ctx context.Context,
taskID string,
executionToken string,
) (GatewayTask, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return GatewayTask{}, err
}
defer rollbackTransaction(tx)
var remoteTaskID string
var hasGatewayUser bool
if err := tx.QueryRow(ctx, `
SELECT COALESCE(remote_task_id, ''), gateway_user_id IS NOT NULL
FROM gateway_tasks
WHERE id = $1::uuid
AND status = 'running'
AND execution_token = $2::uuid
FOR UPDATE`, taskID, executionToken).Scan(&remoteTaskID, &hasGatewayUser); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return GatewayTask{}, ErrTaskExecutionLeaseLost
}
return GatewayTask{}, err
}
ambiguous, err := taskExecutionRequiresManualReviewTx(ctx, tx, taskID)
if err != nil {
return GatewayTask{}, err
}
if remoteTaskID == "" && ambiguous {
if err := markTaskExecutionManualReviewTx(ctx, tx, taskID, hasGatewayUser); err != nil {
return GatewayTask{}, err
}
if err := tx.Commit(ctx); err != nil {
return GatewayTask{}, err
}
return GatewayTask{}, ErrTaskExecutionManualReview
}
nextRunAt := time.Now().Add(time.Second)
queued, err := scanGatewayTask(tx.QueryRow(ctx, `
UPDATE gateway_tasks
SET status = 'queued',
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
next_run_at = $3,
error = NULL,
error_code = NULL,
error_message = NULL,
public_error = NULL,
updated_at = now()
WHERE id = $1::uuid
AND status = 'running'
AND execution_token = $2::uuid
RETURNING `+gatewayTaskColumns, taskID, executionToken, nextRunAt))
if err != nil {
return GatewayTask{}, err
}
if err := tx.Commit(ctx); err != nil {
return GatewayTask{}, err
}
return queued, nil
}
func (s *Store) SetTaskRiverJobID(ctx context.Context, taskID string, riverJobID int64) error {
if riverJobID <= 0 {
return nil