feat(routing): 引入多执行池智能调度
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口,新增 Kubernetes 和静态容量适配器,并以 shadow 模式接入生产配置。 实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌,以及提交状态不明时禁止重复分配的安全语义。 新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
@@ -392,8 +392,8 @@ func markTaskExecutionManualReviewTx(
|
||||
UPDATE gateway_task_attempts
|
||||
SET status = 'failed',
|
||||
retryable = false,
|
||||
error_code = 'upstream_submission_unknown',
|
||||
error_message = 'upstream submission result is unknown',
|
||||
error_code = 'upstream_timeout',
|
||||
error_message = 'upstream submission confirmation timed out',
|
||||
finished_at = COALESCE(finished_at, now()),
|
||||
upstream_submission_updated_at = now()
|
||||
WHERE task_id = $1::uuid
|
||||
@@ -407,14 +407,15 @@ SET status = 'failed',
|
||||
billing_status = CASE WHEN $2 THEN 'manual_review' ELSE 'not_required' END,
|
||||
billing_updated_at = now(),
|
||||
error = NULL,
|
||||
error_code = 'upstream_submission_unknown',
|
||||
error_message = 'upstream submission result is unknown',
|
||||
error_code = 'upstream_timeout',
|
||||
error_message = 'upstream submission confirmation timed out',
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
remote_task_payload = '{}'::jsonb,
|
||||
submission_state = 'submission_confirmation_pending',
|
||||
finished_at = now(),
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil {
|
||||
@@ -424,7 +425,7 @@ WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil {
|
||||
return nil
|
||||
}
|
||||
payloadJSON, _ := json.Marshal(map[string]any{
|
||||
"taskId": taskID, "classification": "upstream_submission_unknown",
|
||||
"taskId": taskID, "classification": "upstream_timeout",
|
||||
})
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO settlement_outbox (
|
||||
@@ -432,7 +433,7 @@ INSERT INTO settlement_outbox (
|
||||
status, next_attempt_at, manual_review_reason
|
||||
)
|
||||
SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency,
|
||||
pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_submission_unknown'
|
||||
pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_timeout'
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
|
||||
@@ -866,6 +867,75 @@ WHERE task_id = $1::uuid
|
||||
return task, changed, nil
|
||||
}
|
||||
|
||||
// ResolveInterruptedTaskExecution atomically preserves the no-duplicate-submit
|
||||
// invariant when a Worker disappears or its execution context is cancelled.
|
||||
// A known remote task is safe to requeue for polling takeover. An ambiguous
|
||||
// submission without a remote ID is moved to manual review instead of replayed.
|
||||
func (s *Store) ResolveInterruptedTaskExecution(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
executionToken string,
|
||||
) (GatewayTask, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
var remoteTaskID string
|
||||
var hasGatewayUser bool
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(remote_task_id, ''), gateway_user_id IS NOT NULL
|
||||
FROM gateway_tasks
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'running'
|
||||
AND execution_token = $2::uuid
|
||||
FOR UPDATE`, taskID, executionToken).Scan(&remoteTaskID, &hasGatewayUser); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return GatewayTask{}, ErrTaskExecutionLeaseLost
|
||||
}
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
ambiguous, err := taskExecutionRequiresManualReviewTx(ctx, tx, taskID)
|
||||
if err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
if remoteTaskID == "" && ambiguous {
|
||||
if err := markTaskExecutionManualReviewTx(ctx, tx, taskID, hasGatewayUser); err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
return GatewayTask{}, ErrTaskExecutionManualReview
|
||||
}
|
||||
nextRunAt := time.Now().Add(time.Second)
|
||||
queued, err := scanGatewayTask(tx.QueryRow(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET status = 'queued',
|
||||
locked_by = NULL,
|
||||
locked_at = NULL,
|
||||
heartbeat_at = NULL,
|
||||
execution_token = NULL,
|
||||
execution_lease_expires_at = NULL,
|
||||
next_run_at = $3,
|
||||
error = NULL,
|
||||
error_code = NULL,
|
||||
error_message = NULL,
|
||||
public_error = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'running'
|
||||
AND execution_token = $2::uuid
|
||||
RETURNING `+gatewayTaskColumns, taskID, executionToken, nextRunAt))
|
||||
if err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return GatewayTask{}, err
|
||||
}
|
||||
return queued, nil
|
||||
}
|
||||
|
||||
func (s *Store) SetTaskRiverJobID(ctx context.Context, taskID string, riverJobID int64) error {
|
||||
if riverJobID <= 0 {
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user