package runner import ( "context" "errors" "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" "github.com/google/uuid" ) const billingSettlementPollInterval = time.Second func billingSettlementRetryDelay(attempt int) time.Duration { if attempt < 1 { attempt = 1 } delay := time.Second for current := 1; current < attempt && delay < 15*time.Minute; current++ { delay *= 2 } if delay > 15*time.Minute { return 15 * time.Minute } return delay } func billingSettlementErrorCode(err error) string { if errors.Is(err, store.ErrInsufficientWalletBalance) { return "insufficient_balance" } return "settlement_failed" } func billingSettlementErrorMessage(code string) string { if code == "insufficient_balance" { return "wallet balance is insufficient for settlement" } return "billing settlement processing failed" } func (s *Service) StartBillingSettlementWorker(ctx context.Context) { workerID := "billing-" + uuid.NewString() go s.runBillingSettlementWorker(ctx, workerID) } func (s *Service) runBillingSettlementWorker(ctx context.Context, workerID string) { ticker := time.NewTicker(billingSettlementPollInterval) defer ticker.Stop() for { s.processBillingSettlementBatch(ctx, workerID) select { case <-ctx.Done(): return case <-ticker.C: } } } func (s *Service) processBillingSettlementBatch(ctx context.Context, workerID string) { items, err := s.store.ClaimBillingSettlements(ctx, workerID, store.BillingSettlementBatchSize, store.BillingSettlementLockTimeout) if err != nil { if ctx.Err() == nil { s.logger.Error("claim billing settlements failed", "error_category", "billing_settlement_claim_failed") } return } for _, item := range items { if ctx.Err() != nil { return } if err := s.store.ProcessBillingSettlement(ctx, item); err != nil { code := billingSettlementErrorCode(err) markErr := s.store.MarkBillingSettlementFailed( context.WithoutCancel(ctx), item, code, billingSettlementErrorMessage(code), billingSettlementRetryDelay(item.Attempts), ) if markErr != nil { s.logger.Error("mark billing settlement failed", "settlementID", item.ID, "taskID", item.TaskID, "error_category", "billing_settlement_state_failed") continue } s.observeBillingEvent("settlement_retry") if item.Attempts >= store.BillingSettlementMaxAttempts { s.observeBillingEvent("manual_review") } s.logger.Warn("billing settlement scheduled for retry", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action, "error_category", code, "attempt", item.Attempts) continue } s.observeBillingEvent("settlement_completed") s.logger.Debug("billing settlement completed", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action) } }