feat(acceptance): 建立同构验收与弹性容量体系

实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
@@ -281,6 +281,16 @@ func nullableTaskListTime(value *time.Time) any {
 }

 func (s *Store) ClaimTaskExecution(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) (GatewayTask, error) {
+	return s.ClaimTaskExecutionForOwner(ctx, taskID, executionToken, leaseTTL, "")
+}
+
+func (s *Store) ClaimTaskExecutionForOwner(
+	ctx context.Context,
+	taskID string,
+	executionToken string,
+	leaseTTL time.Duration,
+	ownerInstanceID string,
+) (GatewayTask, error) {
 	if leaseTTL <= 0 {
 		leaseTTL = 5 * time.Minute
 	}
@@ -323,6 +333,7 @@ UPDATE gateway_tasks
 SET status = 'running',
    execution_token = $2::uuid,
    execution_lease_expires_at = now() + ($3::int * interval '1 second'),
+    locked_by = NULLIF($4::text, ''),
    locked_at = now(),
    heartbeat_at = now(),
    updated_at = now()
@@ -331,7 +342,7 @@ WHERE id = $1::uuid
    (status = 'queued' AND next_run_at <= now())
    OR (status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now()))
  )
-RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second)))
+RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second), strings.TrimSpace(ownerInstanceID)))
 		return err
 	})
 	if err != nil {
@@ -349,7 +360,8 @@ RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second
 func taskExecutionRequiresManualReviewTx(ctx context.Context, tx pgx.Tx, taskID string) (bool, error) {
 	var submissionAmbiguous bool
 	err := tx.QueryRow(ctx, `
-SELECT COALESCE((
+SELECT COALESCE(task.remote_task_id, '') = ''
+       AND COALESCE((
  SELECT (
    (status = 'running' AND upstream_submission_status IN ('submitting', 'response_received'))
    OR (status = 'failed' AND upstream_submission_status = 'submitting')
@@ -363,7 +375,9 @@ SELECT COALESCE((
  WHERE task_id = $1::uuid
  ORDER BY attempt_no DESC, started_at DESC
  LIMIT 1
-), false)`, taskID).Scan(&submissionAmbiguous)
+), false)
+FROM gateway_tasks task
+WHERE task.id = $1::uuid`, taskID).Scan(&submissionAmbiguous)
 	return submissionAmbiguous, err
 }

@@ -414,6 +428,16 @@ ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err
 }

 func (s *Store) ClaimTaskPreparation(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) (GatewayTask, error) {
+	return s.ClaimTaskPreparationForOwner(ctx, taskID, executionToken, leaseTTL, "")
+}
+
+func (s *Store) ClaimTaskPreparationForOwner(
+	ctx context.Context,
+	taskID string,
+	executionToken string,
+	leaseTTL time.Duration,
+	ownerInstanceID string,
+) (GatewayTask, error) {
 	if leaseTTL <= 0 {
 		leaseTTL = 5 * time.Minute
 	}
@@ -459,6 +483,7 @@ FOR UPDATE`, taskID).Scan(&queuedReady, &production, &hasGatewayUser); err != ni
 UPDATE gateway_tasks
 SET execution_token = $2::uuid,
    execution_lease_expires_at = now() + ($3::int * interval '1 second'),
+    locked_by = NULLIF($4::text, ''),
    locked_at = now(),
    heartbeat_at = now(),
    updated_at = now()
@@ -470,7 +495,7 @@ WHERE id = $1::uuid
    OR execution_lease_expires_at IS NULL
    OR execution_lease_expires_at <= now()
  )
-RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second)))
+RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second), strings.TrimSpace(ownerInstanceID)))
 		return err
 	})
 	if errors.Is(err, pgx.ErrNoRows) {
@@ -565,7 +590,19 @@ SELECT COALESCE((SELECT status IN ('queued', 'running') FROM gateway_tasks WHERE
 }

 func (s *Store) MarkTaskRunning(ctx context.Context, taskID string, executionToken string, modelType string, _ map[string]any) error {
-	tag, err := s.pool.Exec(ctx, `
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return err
+	}
+	defer rollbackTransaction(tx)
+	// The execution claim is the durable ownership boundary. This derived
+	// status update is always followed by either the durable upstream
+	// submission barrier or a terminal task transition, so it must be locally
+	// visible immediately but does not need its own synchronous-replica wait.
+	if _, err := tx.Exec(ctx, `SET LOCAL synchronous_commit = off`); err != nil {
+		return err
+	}
+	tag, err := tx.Exec(ctx, `
 UPDATE gateway_tasks
 SET status = 'running',
 	    model_type = NULLIF($3::text, ''),
@@ -582,7 +619,7 @@ WHERE id = $1::uuid
 	if tag.RowsAffected() != 1 {
 		return ErrTaskExecutionLeaseLost
 	}
-	return nil
+	return tx.Commit(ctx)
 }

 func (s *Store) ClaimAsyncQueuedTask(ctx context.Context, workerID string) (GatewayTask, error) {
@@ -1107,6 +1144,13 @@ func (s *Store) CreateTaskAttempt(ctx context.Context, input CreateTaskAttemptIn
 		return "", err
 	}
 	defer rollbackTransaction(tx)
+	// OnUpstreamSubmissionStarted synchronously persists `submitting` before
+	// any provider request leaves the process. PostgreSQL WAL ordering makes
+	// that later barrier cover this attempt row as well, avoiding a redundant
+	// cross-region wait without weakening duplicate-submission protection.
+	if _, err := tx.Exec(ctx, `SET LOCAL synchronous_commit = off`); err != nil {
+		return "", err
+	}

 	if strings.TrimSpace(input.ExecutionToken) != "" {
 		var ownsExecution bool
@@ -1447,6 +1491,11 @@ func taskAttemptMetricInt(metrics map[string]any, key string) int {
 }

 func (s *Store) FinishTaskAttempt(ctx context.Context, input FinishTaskAttemptInput) error {
+	if input.UpstreamSubmissionStatus != "" {
+		if err := validateAttemptUpstreamSubmissionStatus(input.UpstreamSubmissionStatus); err != nil {
+			return err
+		}
+	}
 	statusCode := input.StatusCode
 	if statusCode == 0 {
 		statusCode = taskAttemptMetricInt(input.Metrics, "statusCode")
@@ -1469,6 +1518,11 @@ SET status = $2::text,
    response_duration_ms = $8,
    error_code = NULLIF($9::text, ''),
    error_message = NULLIF(left($10::text, 2048), ''),
+    upstream_submission_status = COALESCE(NULLIF($13::text, ''), upstream_submission_status),
+    upstream_submission_updated_at = CASE
+      WHEN NULLIF($13::text, '') IS NULL THEN upstream_submission_updated_at
+      ELSE now()
+    END,
    finished_at = now()
 WHERE id = $1::uuid`,
 		input.AttemptID,
@@ -1483,6 +1537,7 @@ WHERE id = $1::uuid`,
 		truncateUTF8Bytes(input.ErrorMessage, 2048),
 		usageJSON,
 		metricsJSON,
+		input.UpstreamSubmissionStatus,
 	)
 	return err
 }
@@ -1542,6 +1597,7 @@ func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessIn
 		finalChargeAmount = strconv.FormatFloat(input.FinalChargeAmount, 'f', 9, 64)
 	}
 	currency := normalizeWalletCurrency(input.BillingCurrency)
+	finalizedAdmission := false
 	err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
 		if strings.TrimSpace(input.AttemptID) != "" {
 			if _, err := tx.Exec(ctx, `
@@ -1571,7 +1627,8 @@ WHERE id = $1::uuid`,
 				return err
 			}
 		}
-		tag, err := tx.Exec(ctx, `
+		var billingStatus string
+		err := tx.QueryRow(ctx, `
 UPDATE gateway_tasks
 SET status = 'succeeded',
    result = $2::jsonb,
@@ -1607,7 +1664,8 @@ SET status = 'succeeded',
    updated_at = now()
 WHERE id = $1::uuid
  AND status = 'running'
-  AND execution_token = $16::uuid`,
+  AND execution_token = $16::uuid
+RETURNING billing_status`,
 			input.TaskID,
 			string(resultJSON),
 			string(billingsJSON),
@@ -1624,13 +1682,13 @@ WHERE id = $1::uuid
 			nullableTime(input.ResponseFinishedAt),
 			input.ResponseDurationMS,
 			input.ExecutionToken,
-		)
+		).Scan(&billingStatus)
+		if errors.Is(err, pgx.ErrNoRows) {
+			return ErrTaskExecutionLeaseLost
+		}
 		if err != nil {
 			return err
 		}
-		if tag.RowsAffected() != 1 {
-			return ErrTaskExecutionLeaseLost
-		}
 		payloadJSON, _ := json.Marshal(map[string]any{"taskId": input.TaskID, "pricingVersion": "effective-pricing-v2"})
 		_, err = tx.Exec(ctx, `
 INSERT INTO settlement_outbox (
@@ -1643,11 +1701,66 @@ WHERE id = $1::uuid
  AND gateway_user_id IS NOT NULL
 ON CONFLICT (task_id, event_type) DO NOTHING`,
 			input.TaskID, finalChargeAmount, currency, string(pricingSnapshotJSON), string(payloadJSON))
-		return err
+		if err != nil {
+			return err
+		}
+		if input.FinalizeAdmission {
+			if _, err := tx.Exec(ctx, `
+UPDATE gateway_concurrency_leases
+SET released_at = now()
+WHERE task_id = $1::uuid
+  AND released_at IS NULL`, input.TaskID); err != nil {
+				return err
+			}
+			if _, err := tx.Exec(ctx, `
+DELETE FROM gateway_task_admissions
+WHERE task_id = $1::uuid`, input.TaskID); err != nil {
+				return err
+			}
+			finalizedAdmission = true
+		}
+		var eventID string
+		var eventSeq int64
+		if err := tx.QueryRow(ctx, `
+WITH next_seq AS (
+  SELECT COALESCE(MAX(seq), 0) + 1 AS seq
+  FROM gateway_task_events
+  WHERE task_id = $1::uuid
+)
+INSERT INTO gateway_task_events (
+  task_id, seq, event_type, status, phase, progress, message, payload, simulated
+)
+SELECT $1::uuid, next_seq.seq, 'task.completed', 'succeeded', NULL, 0, NULL, '{}'::jsonb, $2
+FROM next_seq
+RETURNING id::text, seq`,
+			input.TaskID,
+			input.Simulated,
+		).Scan(&eventID, &eventSeq); err != nil {
+			return err
+		}
+		callbackURL := strings.TrimSpace(input.CompletionCallbackURL)
+		if callbackURL != "" {
+			if _, err := tx.Exec(ctx, `
+INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
+VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
+ON CONFLICT (task_id, seq, callback_url) DO NOTHING`,
+				input.TaskID,
+				eventID,
+				eventSeq,
+				callbackURL,
+			); err != nil {
+				return err
+			}
+		}
+		_ = billingStatus
+		return nil
 	})
 	if err != nil {
 		return GatewayTask{}, err
 	}
+	if finalizedAdmission {
+		s.notifyTaskAdmissionBestEffort(ctx, "*")
+	}
 	return s.GetTask(ctx, input.TaskID)
 }

@@ -1903,6 +2016,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
 	metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
 	resultJSON, _ := json.Marshal(minimalTaskResult(nil))
 	message := truncateUTF8Bytes(input.Message, 2048)
+	finalizedAdmission := false
 	err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
 		tag, err := tx.Exec(ctx, `
 	UPDATE gateway_tasks
@@ -1963,11 +2077,65 @@ WHERE id = $1::uuid
  AND gateway_user_id IS NOT NULL
  AND reservation_amount > 0
 ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON))
-		return err
+		if err != nil {
+			return err
+		}
+		if input.FinalizeAdmission {
+			if _, err := tx.Exec(ctx, `
+UPDATE gateway_concurrency_leases
+SET released_at = now()
+WHERE task_id = $1::uuid
+  AND released_at IS NULL`, input.TaskID); err != nil {
+				return err
+			}
+			if _, err := tx.Exec(ctx, `
+DELETE FROM gateway_task_admissions
+WHERE task_id = $1::uuid`, input.TaskID); err != nil {
+				return err
+			}
+			finalizedAdmission = true
+		}
+		var eventID string
+		var eventSeq int64
+		if err := tx.QueryRow(ctx, `
+WITH next_seq AS (
+  SELECT COALESCE(MAX(seq), 0) + 1 AS seq
+  FROM gateway_task_events
+  WHERE task_id = $1::uuid
+)
+INSERT INTO gateway_task_events (
+  task_id, seq, event_type, status, phase, progress, message, payload, simulated
+)
+SELECT $1::uuid, next_seq.seq, 'task.failed', 'failed', NULL, 0, NULL, '{}'::jsonb, $2
+FROM next_seq
+RETURNING id::text, seq`,
+			input.TaskID,
+			input.Simulated,
+		).Scan(&eventID, &eventSeq); err != nil {
+			return err
+		}
+		callbackURL := strings.TrimSpace(input.CompletionCallbackURL)
+		if callbackURL != "" {
+			if _, err := tx.Exec(ctx, `
+INSERT INTO gateway_task_callback_outbox (task_id, event_id, seq, callback_url, payload)
+VALUES ($1::uuid, $2::uuid, $3, $4, '{}'::jsonb)
+ON CONFLICT (task_id, seq, callback_url) DO NOTHING`,
+				input.TaskID,
+				eventID,
+				eventSeq,
+				callbackURL,
+			); err != nil {
+				return err
+			}
+		}
+		return nil
 	})
 	if err != nil {
 		return GatewayTask{}, err
 	}
+	if finalizedAdmission {
+		s.notifyTaskAdmissionBestEffort(ctx, "*")
+	}
 	return s.GetTask(ctx, input.TaskID)
 }

@@ -2093,6 +2261,15 @@ func (s *Store) addTaskEvent(
 		return TaskEvent{}, err
 	}
 	defer rollbackTransaction(tx)
+	if !taskEventTerminal(eventType) {
+		// Progress/history events are recoverable hints. A later synchronous
+		// upstream or terminal task transition is their durability barrier, so
+		// waiting for the cross-region replica on every event only amplifies
+		// queue latency under media bursts.
+		if _, err := tx.Exec(ctx, `SET LOCAL synchronous_commit = off`); err != nil {
+			return TaskEvent{}, err
+		}
+	}
 	if _, err := tx.Exec(ctx, `SELECT 1 FROM gateway_tasks WHERE id = $1::uuid FOR UPDATE`, taskID); err != nil {
 		return TaskEvent{}, err
 	}