feat(billing): 完成异步结算与请求执行闭环

统一任务成功、Attempt 与结算 Outbox 的事务边界，增加多实例安全结算、人工复核、请求幂等与执行租约。钱包决策使用九位精确金额，并通过审计保护约束保留流水事实；同时补充管理接口、指标与 PostgreSQL 集成测试。
2026-07-21 00:25:53 +08:00
parent 7cea21f765
commit 5b2b94b1bd
33 changed files with 2884 additions and 418 deletions
@@ -0,0 +1,96 @@
+package runner
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+	"github.com/google/uuid"
+)
+
+const billingSettlementPollInterval = time.Second
+
+func billingSettlementRetryDelay(attempt int) time.Duration {
+	if attempt < 1 {
+		attempt = 1
+	}
+	delay := time.Second
+	for current := 1; current < attempt && delay < 15*time.Minute; current++ {
+		delay *= 2
+	}
+	if delay > 15*time.Minute {
+		return 15 * time.Minute
+	}
+	return delay
+}
+
+func billingSettlementErrorCode(err error) string {
+	if errors.Is(err, store.ErrInsufficientWalletBalance) {
+		return "insufficient_balance"
+	}
+	return "settlement_failed"
+}
+
+func billingSettlementErrorMessage(code string) string {
+	if code == "insufficient_balance" {
+		return "wallet balance is insufficient for settlement"
+	}
+	return "billing settlement processing failed"
+}
+
+func (s *Service) StartBillingSettlementWorker(ctx context.Context) {
+	workerID := "billing-" + uuid.NewString()
+	go s.runBillingSettlementWorker(ctx, workerID)
+}
+
+func (s *Service) runBillingSettlementWorker(ctx context.Context, workerID string) {
+	ticker := time.NewTicker(billingSettlementPollInterval)
+	defer ticker.Stop()
+
+	for {
+		s.processBillingSettlementBatch(ctx, workerID)
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (s *Service) processBillingSettlementBatch(ctx context.Context, workerID string) {
+	items, err := s.store.ClaimBillingSettlements(ctx, workerID, store.BillingSettlementBatchSize, store.BillingSettlementLockTimeout)
+	if err != nil {
+		if ctx.Err() == nil {
+			s.logger.Error("claim billing settlements failed", "error_category", "billing_settlement_claim_failed")
+		}
+		return
+	}
+	for _, item := range items {
+		if ctx.Err() != nil {
+			return
+		}
+		if err := s.store.ProcessBillingSettlement(ctx, item); err != nil {
+			code := billingSettlementErrorCode(err)
+			markErr := s.store.MarkBillingSettlementFailed(
+				context.WithoutCancel(ctx),
+				item,
+				code,
+				billingSettlementErrorMessage(code),
+				billingSettlementRetryDelay(item.Attempts),
+			)
+			if markErr != nil {
+				s.logger.Error("mark billing settlement failed", "settlementID", item.ID, "taskID", item.TaskID, "error_category", "billing_settlement_state_failed")
+				continue
+			}
+			s.observeBillingEvent("settlement_retry")
+			if item.Attempts >= store.BillingSettlementMaxAttempts {
+				s.observeBillingEvent("manual_review")
+			}
+			s.logger.Warn("billing settlement scheduled for retry", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action, "error_category", code, "attempt", item.Attempts)
+			continue
+		}
+		s.observeBillingEvent("settlement_completed")
+		s.logger.Debug("billing settlement completed", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action)
+	}
+}