feat(billing): 完成异步结算与请求执行闭环

统一任务成功、Attempt 与结算 Outbox 的事务边界,增加多实例安全结算、人工复核、请求幂等与执行租约。钱包决策使用九位精确金额,并通过审计保护约束保留流水事实;同时补充管理接口、指标与 PostgreSQL 集成测试。
This commit is contained in:
2026-07-21 00:25:53 +08:00
parent 7cea21f765
commit 5b2b94b1bd
33 changed files with 2884 additions and 418 deletions
+191 -112
View File
@@ -47,16 +47,21 @@ func normalizeAPIKeyScopes(scopes []string) []string {
}
var (
ErrInvalidCredentials = errors.New("invalid account or password")
ErrInvalidInvitation = errors.New("invalid or expired invitation code")
ErrInvalidAPIKeyScopes = errors.New("api key scopes must not be empty")
ErrAccessRuleResourceDenied = errors.New("access rule resource is not available")
ErrInsufficientWalletBalance = errors.New("insufficient wallet balance")
ErrLocalUserRequired = errors.New("local gateway user is required")
ErrWalletBalanceUnchanged = errors.New("wallet balance unchanged")
ErrProtectedDefault = errors.New("protected default resource cannot be deleted")
ErrUserAlreadyExists = errors.New("user already exists")
ErrWeakPassword = errors.New("password must be at least 8 characters")
ErrInvalidCredentials = errors.New("invalid account or password")
ErrInvalidInvitation = errors.New("invalid or expired invitation code")
ErrInvalidAPIKeyScopes = errors.New("api key scopes must not be empty")
ErrAccessRuleResourceDenied = errors.New("access rule resource is not available")
ErrInsufficientWalletBalance = errors.New("insufficient wallet balance")
ErrLocalUserRequired = errors.New("local gateway user is required")
ErrWalletBalanceUnchanged = errors.New("wallet balance unchanged")
ErrBalanceBelowFrozen = errors.New("wallet balance cannot be below frozen balance")
ErrInvalidWalletAmount = errors.New("wallet amount must be a decimal with at most nine fractional digits")
ErrIdempotencyKeyReused = errors.New("idempotency key was reused for a different request")
ErrTaskExecutionLeaseUnavailable = errors.New("task execution lease is unavailable")
ErrTaskExecutionLeaseLost = errors.New("task execution lease was lost")
ErrProtectedDefault = errors.New("protected default resource cannot be deleted")
ErrUserAlreadyExists = errors.New("user already exists")
ErrWeakPassword = errors.New("password must be at least 8 characters")
)
func Connect(ctx context.Context, databaseURL string) (*Store, error) {
@@ -414,64 +419,81 @@ type RateLimitWindow struct {
}
type CreateTaskInput struct {
Kind string `json:"kind"`
Model string `json:"model"`
RunMode string `json:"runMode"`
Async bool `json:"async"`
Request map[string]any `json:"request"`
ConversationID string `json:"conversationId"`
NewMessageCount int `json:"newMessageCount"`
MessageRefs []TaskMessageRefInput `json:"messageRefs"`
Kind string `json:"kind"`
Model string `json:"model"`
RunMode string `json:"runMode"`
Async bool `json:"async"`
Request map[string]any `json:"request"`
ConversationID string `json:"conversationId"`
NewMessageCount int `json:"newMessageCount"`
MessageRefs []TaskMessageRefInput `json:"messageRefs"`
IdempotencyKeyHash string `json:"-"`
IdempotencyRequestHash string `json:"-"`
}
type CreateTaskResult struct {
Task GatewayTask
Replayed bool
}
type GatewayTask struct {
ID string `json:"id"`
Kind string `json:"kind"`
RunMode string `json:"runMode"`
UserID string `json:"userId"`
GatewayUserID string `json:"gatewayUserId,omitempty"`
UserSource string `json:"userSource,omitempty"`
GatewayTenantID string `json:"gatewayTenantId,omitempty"`
TenantID string `json:"tenantId,omitempty"`
TenantKey string `json:"tenantKey,omitempty"`
APIKeyID string `json:"apiKeyId,omitempty"`
APIKeyName string `json:"apiKeyName,omitempty"`
APIKeyPrefix string `json:"apiKeyPrefix,omitempty"`
UserGroupID string `json:"userGroupId,omitempty"`
UserGroupKey string `json:"userGroupKey,omitempty"`
Model string `json:"model"`
ModelType string `json:"modelType,omitempty"`
RequestedModel string `json:"requestedModel,omitempty"`
ResolvedModel string `json:"resolvedModel,omitempty"`
RequestID string `json:"requestId,omitempty"`
ConversationID string `json:"conversationId,omitempty"`
NewMessageCount int `json:"newMessageCount,omitempty"`
Request map[string]any `json:"request,omitempty"`
AsyncMode bool `json:"asyncMode"`
RiverJobID int64 `json:"riverJobId,omitempty"`
Status string `json:"status"`
Cancellable *bool `json:"cancellable,omitempty"`
Submitted *bool `json:"submitted,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attemptCount"`
RemoteTaskID string `json:"remoteTaskId,omitempty"`
RemoteTaskPayload map[string]any `json:"remoteTaskPayload,omitempty"`
Result map[string]any `json:"result,omitempty"`
Billings []any `json:"billings,omitempty"`
Usage map[string]any `json:"usage"`
Metrics map[string]any `json:"metrics"`
BillingSummary map[string]any `json:"billingSummary"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
FinishedAt string `json:"finishedAt,omitempty"`
Error string `json:"error,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
Attempts []TaskAttempt `json:"attempts,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
Kind string `json:"kind"`
RunMode string `json:"runMode"`
UserID string `json:"userId"`
GatewayUserID string `json:"gatewayUserId,omitempty"`
UserSource string `json:"userSource,omitempty"`
GatewayTenantID string `json:"gatewayTenantId,omitempty"`
TenantID string `json:"tenantId,omitempty"`
TenantKey string `json:"tenantKey,omitempty"`
APIKeyID string `json:"apiKeyId,omitempty"`
APIKeyName string `json:"apiKeyName,omitempty"`
APIKeyPrefix string `json:"apiKeyPrefix,omitempty"`
UserGroupID string `json:"userGroupId,omitempty"`
UserGroupKey string `json:"userGroupKey,omitempty"`
Model string `json:"model"`
ModelType string `json:"modelType,omitempty"`
RequestedModel string `json:"requestedModel,omitempty"`
ResolvedModel string `json:"resolvedModel,omitempty"`
RequestID string `json:"requestId,omitempty"`
ConversationID string `json:"conversationId,omitempty"`
NewMessageCount int `json:"newMessageCount,omitempty"`
Request map[string]any `json:"request,omitempty"`
AsyncMode bool `json:"asyncMode"`
RiverJobID int64 `json:"riverJobId,omitempty"`
Status string `json:"status"`
Cancellable *bool `json:"cancellable,omitempty"`
Submitted *bool `json:"submitted,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attemptCount"`
RemoteTaskID string `json:"remoteTaskId,omitempty"`
RemoteTaskPayload map[string]any `json:"remoteTaskPayload,omitempty"`
Result map[string]any `json:"result,omitempty"`
Billings []any `json:"billings,omitempty"`
Usage map[string]any `json:"usage"`
Metrics map[string]any `json:"metrics"`
BillingSummary map[string]any `json:"billingSummary"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
BillingVersion string `json:"billingVersion"`
BillingStatus string `json:"billingStatus"`
BillingCurrency string `json:"billingCurrency"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
ReservationAmount float64 `json:"reservationAmount"`
ExecutionToken string `json:"-"`
ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"`
BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"`
BillingSettledAt string `json:"billingSettledAt,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
FinishedAt string `json:"finishedAt,omitempty"`
Error string `json:"error,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
Attempts []TaskAttempt `json:"attempts,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
const gatewayTaskColumns = `
@@ -485,7 +507,11 @@ request, COALESCE(async_mode, false), COALESCE(river_job_id, 0), status, COALESC
COALESCE(remote_task_id, ''), COALESCE(remote_task_payload, '{}'::jsonb),
COALESCE(result, '{}'::jsonb), COALESCE(billings, '[]'::jsonb),
COALESCE(usage, '{}'::jsonb), COALESCE(metrics, '{}'::jsonb), COALESCE(billing_summary, '{}'::jsonb),
COALESCE(final_charge_amount, 0)::float8, COALESCE(response_started_at::text, ''),
COALESCE(final_charge_amount, 0)::float8, billing_version, billing_status, billing_currency,
COALESCE(pricing_snapshot, '{}'::jsonb), COALESCE(request_fingerprint, ''),
COALESCE(reservation_amount, 0)::float8, COALESCE(execution_token::text, ''),
COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''),
COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''),
COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''),
COALESCE(error_code, ''), COALESCE(error_message, ''),
created_at, updated_at, COALESCE(finished_at::text, '')`
@@ -505,35 +531,39 @@ type TaskEvent struct {
}
type TaskAttempt struct {
ID string `json:"id"`
TaskID string `json:"taskId"`
AttemptNo int `json:"attemptNo"`
PlatformID string `json:"platformId,omitempty"`
PlatformName string `json:"platformName,omitempty"`
Provider string `json:"provider,omitempty"`
PlatformModelID string `json:"platformModelId,omitempty"`
ModelName string `json:"modelName,omitempty"`
ProviderModelName string `json:"providerModelName,omitempty"`
ModelAlias string `json:"modelAlias,omitempty"`
ModelType string `json:"modelType,omitempty"`
ClientID string `json:"clientId,omitempty"`
QueueKey string `json:"queueKey"`
Status string `json:"status"`
Retryable bool `json:"retryable"`
Simulated bool `json:"simulated"`
RequestID string `json:"requestId,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Usage map[string]any `json:"usage,omitempty"`
Metrics map[string]any `json:"metrics,omitempty"`
RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"`
ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
StartedAt time.Time `json:"startedAt"`
FinishedAt string `json:"finishedAt,omitempty"`
ID string `json:"id"`
TaskID string `json:"taskId"`
AttemptNo int `json:"attemptNo"`
PlatformID string `json:"platformId,omitempty"`
PlatformName string `json:"platformName,omitempty"`
Provider string `json:"provider,omitempty"`
PlatformModelID string `json:"platformModelId,omitempty"`
ModelName string `json:"modelName,omitempty"`
ProviderModelName string `json:"providerModelName,omitempty"`
ModelAlias string `json:"modelAlias,omitempty"`
ModelType string `json:"modelType,omitempty"`
ClientID string `json:"clientId,omitempty"`
QueueKey string `json:"queueKey"`
Status string `json:"status"`
Retryable bool `json:"retryable"`
Simulated bool `json:"simulated"`
RequestID string `json:"requestId,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Usage map[string]any `json:"usage,omitempty"`
Metrics map[string]any `json:"metrics,omitempty"`
RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"`
ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"`
UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
StartedAt time.Time `json:"startedAt"`
FinishedAt string `json:"finishedAt,omitempty"`
}
type TaskParamPreprocessingLog struct {
@@ -1716,6 +1746,9 @@ ON CONFLICT (gateway_user_id, currency) DO NOTHING`,
); err != nil {
return GatewayUser{}, err
}
if err := ensureWalletAccountAuditGuard(ctx, tx, user.ID, "resource"); err != nil {
return GatewayUser{}, err
}
if err := tx.Commit(ctx); err != nil {
return GatewayUser{}, err
}
@@ -1826,6 +1859,11 @@ ORDER BY window_start DESC, scope_type ASC, scope_key ASC, metric ASC`)
}
func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *auth.User) (GatewayTask, error) {
result, err := s.CreateTaskIdempotent(ctx, input, user)
return result.Task, err
}
func (s *Store) CreateTaskIdempotent(ctx context.Context, input CreateTaskInput, user *auth.User) (CreateTaskResult, error) {
requestBody, _ := json.Marshal(input.Request)
runMode := normalizeRunMode(input.RunMode, input.Request)
status := "queued"
@@ -1834,7 +1872,7 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut
tx, err := s.pool.Begin(ctx)
if err != nil {
return GatewayTask{}, err
return CreateTaskResult{}, err
}
defer tx.Rollback(ctx)
@@ -1842,33 +1880,62 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut
INSERT INTO gateway_tasks (
kind, run_mode, user_id, gateway_user_id, user_source, gateway_tenant_id, tenant_id, tenant_key,
api_key_id, api_key_name, api_key_prefix, user_group_id, user_group_key,
model, requested_model, request, async_mode, status, result, billings, conversation_id, new_message_count, finished_at
model, requested_model, request, async_mode, status, result, billings, conversation_id, new_message_count,
idempotency_key_hash, idempotency_request_hash, finished_at
)
VALUES ($1, $2, $3, NULLIF($4, '')::uuid, COALESCE(NULLIF($5, ''), 'gateway'), NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''), NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, '')::uuid, NULLIF($13, ''), $14, $14, $15, $16, $17, $18::jsonb, $19::jsonb, NULLIF($20, '')::uuid, $21, CASE WHEN $22 THEN now() ELSE NULL END)
VALUES ($1, $2, $3, NULLIF($4, '')::uuid, COALESCE(NULLIF($5, ''), 'gateway'), NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''), NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, '')::uuid, NULLIF($13, ''), $14, $14, $15, $16, $17, $18::jsonb, $19::jsonb, NULLIF($20, '')::uuid, $21, NULLIF($22, ''), NULLIF($23, ''), NULL)
ON CONFLICT (user_id, idempotency_key_hash) WHERE idempotency_key_hash IS NOT NULL DO NOTHING
RETURNING `+gatewayTaskColumns,
input.Kind, runMode, user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, input.NewMessageCount, false,
input.Kind, runMode, user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, input.NewMessageCount, strings.TrimSpace(input.IdempotencyKeyHash), strings.TrimSpace(input.IdempotencyRequestHash),
))
replayed := false
if errors.Is(err, pgx.ErrNoRows) && strings.TrimSpace(input.IdempotencyKeyHash) != "" {
var existingRequestHash string
if err := tx.QueryRow(ctx, `
SELECT COALESCE(idempotency_request_hash, '')
FROM gateway_tasks
WHERE user_id = $1 AND idempotency_key_hash = $2
FOR UPDATE`, user.ID, strings.TrimSpace(input.IdempotencyKeyHash)).Scan(&existingRequestHash); err != nil {
return CreateTaskResult{}, err
}
if existingRequestHash != strings.TrimSpace(input.IdempotencyRequestHash) {
return CreateTaskResult{}, ErrIdempotencyKeyReused
}
task, err = scanGatewayTask(tx.QueryRow(ctx, `
SELECT `+gatewayTaskColumns+`
FROM gateway_tasks
WHERE user_id = $1 AND idempotency_key_hash = $2`, user.ID, strings.TrimSpace(input.IdempotencyKeyHash)))
replayed = true
}
if err != nil {
return GatewayTask{}, err
return CreateTaskResult{}, err
}
if err := insertTaskMessageRefs(ctx, tx, task.ID, input.MessageRefs); err != nil {
return GatewayTask{}, err
}
events := taskEventsForCreate(task.ID, runMode, status, nil)
for _, event := range events {
payload, _ := json.Marshal(event.Payload)
if _, err := tx.Exec(ctx, `
if !replayed {
if err := insertTaskMessageRefs(ctx, tx, task.ID, input.MessageRefs); err != nil {
return CreateTaskResult{}, err
}
events := taskEventsForCreate(task.ID, runMode, status, nil)
for _, event := range events {
payload, _ := json.Marshal(event.Payload)
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
VALUES ($1::uuid, $2, $3::text, NULLIF($4::text, ''), NULLIF($5::text, ''), $6, NULLIF($7::text, ''), $8::jsonb, $9)`,
task.ID, event.Seq, event.EventType, event.Status, event.Phase, event.Progress, event.Message, string(payload), event.Simulated,
); err != nil {
return GatewayTask{}, err
task.ID, event.Seq, event.EventType, event.Status, event.Phase, event.Progress, event.Message, string(payload), event.Simulated,
); err != nil {
return CreateTaskResult{}, err
}
}
}
if err := tx.Commit(ctx); err != nil {
return GatewayTask{}, err
return CreateTaskResult{}, err
}
return task, nil
if replayed {
task, err = s.GetTask(ctx, task.ID)
if err != nil {
return CreateTaskResult{}, err
}
}
return CreateTaskResult{Task: task, Replayed: replayed}, nil
}
func (s *Store) GetTask(ctx context.Context, taskID string) (GatewayTask, error) {
@@ -1900,6 +1967,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
var usageBytes []byte
var metricsBytes []byte
var billingSummaryBytes []byte
var pricingSnapshotBytes []byte
var remoteTaskPayloadBytes []byte
if err := scanner.Scan(
&task.ID,
@@ -1936,6 +2004,16 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
&metricsBytes,
&billingSummaryBytes,
&task.FinalChargeAmount,
&task.BillingVersion,
&task.BillingStatus,
&task.BillingCurrency,
&pricingSnapshotBytes,
&task.RequestFingerprint,
&task.ReservationAmount,
&task.ExecutionToken,
&task.ExecutionLeaseUntil,
&task.BillingUpdatedAt,
&task.BillingSettledAt,
&task.ResponseStartedAt,
&task.ResponseFinishedAt,
&task.ResponseDurationMS,
@@ -1955,6 +2033,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
task.Usage = decodeObject(usageBytes)
task.Metrics = decodeObject(metricsBytes)
task.BillingSummary = decodeObject(billingSummaryBytes)
task.PricingSnapshot = decodeObject(pricingSnapshotBytes)
return task, nil
}