fix(worker): 防止事务泄漏并恢复滞留队列

为 PostgreSQL 连接增加可配置的事务空闲与锁等待超时，并在请求取消或 Worker 退出后使用独立有界上下文回滚事务。\n\n恢复过期任务时按批次使用 SKIP LOCKED，周期重建缺失或终态 River Job；锁等待超时只在确认尚未提交上游时安全重排，避免重复执行和重复结算。\n\n验证：完整 Go 测试、go vet、生产 Kustomize 渲染、gofmt 与 git diff --check 均通过。
2026-07-30 17:27:11 +08:00
parent bc44af751e
commit 6bab0f0749
32 changed files with 545 additions and 106 deletions
@@ -59,6 +59,8 @@ type Config struct {
 	BillingEngineMode                             string
 	ProcessRole                                   string
 	DatabaseMaxConns                              int
+	DatabaseIdleInTransactionTimeoutSeconds       int
+	DatabaseLockTimeoutSeconds                    int
 	MediaRequestConcurrency                       int
 	MediaMaterializationConcurrency               int
 	AsyncQueueWorkerEnabled                       bool
@@ -106,20 +108,27 @@ func Load() Config {
 		TaskProgressCallbackURL: env("TASK_PROGRESS_CALLBACK_URL",
 			strings.TrimRight(env("SERVER_MAIN_BASE_URL", "http://localhost:3000"), "/")+"/internal/platform/task-progress-callbacks",
 		),
-		TaskProgressCallbackTimeoutMS:     envIntValidated("TASK_PROGRESS_CALLBACK_TIMEOUT_MS", 5000),
-		TaskProgressCallbackMaxAttempts:   envIntValidated("TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS", 10),
-		TaskCleanupEnabled:                env("AI_GATEWAY_TASK_CLEANUP_ENABLED", "false") == "true",
-		TaskRetentionDays:                 envIntValidated("AI_GATEWAY_TASK_RETENTION_DAYS", 30),
-		TaskAnalysisRetentionDays:         envIntValidated("AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS", 7),
-		TaskCleanupIntervalSeconds:        envIntValidated("AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS", 300),
-		TaskCleanupBatchSize:              envIntValidated("AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE", 1000),
-		CORSAllowedOrigin:                 env("CORS_ALLOWED_ORIGIN", "http://localhost:5178,http://127.0.0.1:5178"),
-		GlobalHTTPProxy:                   globalProxy.HTTPProxy,
-		GlobalHTTPProxySource:             globalProxy.Source,
-		LogLevel:                          logLevel(env("LOG_LEVEL", "info")),
-		BillingEngineMode:                 strings.ToLower(env("BILLING_ENGINE_MODE", "observe")),
-		ProcessRole:                       strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_PROCESS_ROLE", "all"))),
-		DatabaseMaxConns:                  envInt("AI_GATEWAY_DATABASE_MAX_CONNS", 0),
+		TaskProgressCallbackTimeoutMS:   envIntValidated("TASK_PROGRESS_CALLBACK_TIMEOUT_MS", 5000),
+		TaskProgressCallbackMaxAttempts: envIntValidated("TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS", 10),
+		TaskCleanupEnabled:              env("AI_GATEWAY_TASK_CLEANUP_ENABLED", "false") == "true",
+		TaskRetentionDays:               envIntValidated("AI_GATEWAY_TASK_RETENTION_DAYS", 30),
+		TaskAnalysisRetentionDays:       envIntValidated("AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS", 7),
+		TaskCleanupIntervalSeconds:      envIntValidated("AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS", 300),
+		TaskCleanupBatchSize:            envIntValidated("AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE", 1000),
+		CORSAllowedOrigin:               env("CORS_ALLOWED_ORIGIN", "http://localhost:5178,http://127.0.0.1:5178"),
+		GlobalHTTPProxy:                 globalProxy.HTTPProxy,
+		GlobalHTTPProxySource:           globalProxy.Source,
+		LogLevel:                        logLevel(env("LOG_LEVEL", "info")),
+		BillingEngineMode:               strings.ToLower(env("BILLING_ENGINE_MODE", "observe")),
+		ProcessRole:                     strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_PROCESS_ROLE", "all"))),
+		DatabaseMaxConns:                envInt("AI_GATEWAY_DATABASE_MAX_CONNS", 0),
+
+		DatabaseIdleInTransactionTimeoutSeconds: envOptionalIntValidated(
+			"AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS",
+			60,
+		),
+		DatabaseLockTimeoutSeconds: envOptionalIntValidated("AI_GATEWAY_DATABASE_LOCK_TIMEOUT_SECONDS", 30),
+
 		MediaRequestConcurrency:           envIntValidated("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY", 16),
 		MediaMaterializationConcurrency:   envIntValidated("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY", 8),
 		AsyncQueueWorkerEnabled:           env("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "true") == "true",
@@ -138,6 +147,12 @@ func (c Config) Validate() error {
 	if c.DatabaseMaxConns < 0 || c.DatabaseMaxConns > 1000 {
 		return errors.New("AI_GATEWAY_DATABASE_MAX_CONNS must be between 1 and 1000 when configured")
 	}
+	if c.DatabaseIdleInTransactionTimeoutSeconds < 0 || c.DatabaseIdleInTransactionTimeoutSeconds > 3600 {
+		return errors.New("AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS must be between 0 and 3600")
+	}
+	if c.DatabaseLockTimeoutSeconds < 0 || c.DatabaseLockTimeoutSeconds > 3600 {
+		return errors.New("AI_GATEWAY_DATABASE_LOCK_TIMEOUT_SECONDS must be between 0 and 3600")
+	}
 	if c.MediaMaterializationConcurrency != 0 && (c.MediaMaterializationConcurrency < 1 || c.MediaMaterializationConcurrency > 256) {
 		return errors.New("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY must be between 1 and 256")
 	}
@@ -336,6 +351,18 @@ func envIntValidated(key string, fallback int) int {
 	return parsed
 }

+func envOptionalIntValidated(key string, fallback int) int {
+	value := envValue(key)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.Atoi(value)
+	if err != nil {
+		return -1
+	}
+	return parsed
+}
+
 func envInt64Validated(key string, fallback int64) int64 {
 	value := envValue(key)
 	if value == "" {