perf(storage): 极简化任务历史并增加保留治理
停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON,并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。 增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭,待兼容协议和异步恢复在线验证后单独启用。 验证:Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
@@ -41,8 +41,13 @@ type Config struct {
|
||||
LocalTempAssetTTLHours int
|
||||
TaskProgressCallbackEnabled bool
|
||||
TaskProgressCallbackURL string
|
||||
TaskProgressCallbackTimeoutMS string
|
||||
TaskProgressCallbackMaxAttempts string
|
||||
TaskProgressCallbackTimeoutMS int
|
||||
TaskProgressCallbackMaxAttempts int
|
||||
TaskCleanupEnabled bool
|
||||
TaskRetentionDays int
|
||||
TaskAnalysisRetentionDays int
|
||||
TaskCleanupIntervalSeconds int
|
||||
TaskCleanupBatchSize int
|
||||
CORSAllowedOrigin string
|
||||
GlobalHTTPProxy string
|
||||
GlobalHTTPProxySource string
|
||||
@@ -87,8 +92,13 @@ func Load() Config {
|
||||
TaskProgressCallbackURL: env("TASK_PROGRESS_CALLBACK_URL",
|
||||
strings.TrimRight(env("SERVER_MAIN_BASE_URL", "http://localhost:3000"), "/")+"/internal/platform/task-progress-callbacks",
|
||||
),
|
||||
TaskProgressCallbackTimeoutMS: env("TASK_PROGRESS_CALLBACK_TIMEOUT_MS", "5000"),
|
||||
TaskProgressCallbackMaxAttempts: env("TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS", "10"),
|
||||
TaskProgressCallbackTimeoutMS: envIntValidated("TASK_PROGRESS_CALLBACK_TIMEOUT_MS", 5000),
|
||||
TaskProgressCallbackMaxAttempts: envIntValidated("TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS", 10),
|
||||
TaskCleanupEnabled: env("AI_GATEWAY_TASK_CLEANUP_ENABLED", "false") == "true",
|
||||
TaskRetentionDays: envIntValidated("AI_GATEWAY_TASK_RETENTION_DAYS", 30),
|
||||
TaskAnalysisRetentionDays: envIntValidated("AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS", 7),
|
||||
TaskCleanupIntervalSeconds: envIntValidated("AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS", 300),
|
||||
TaskCleanupBatchSize: envIntValidated("AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE", 1000),
|
||||
CORSAllowedOrigin: env("CORS_ALLOWED_ORIGIN", "http://localhost:5178,http://127.0.0.1:5178"),
|
||||
GlobalHTTPProxy: globalProxy.HTTPProxy,
|
||||
GlobalHTTPProxySource: globalProxy.Source,
|
||||
@@ -111,6 +121,24 @@ func (c Config) Validate() error {
|
||||
if c.AsyncWorkerRefreshIntervalSeconds < 1 {
|
||||
return errors.New("AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS must be positive")
|
||||
}
|
||||
if c.TaskProgressCallbackTimeoutMS != 0 && (c.TaskProgressCallbackTimeoutMS < 100 || c.TaskProgressCallbackTimeoutMS > 60000) {
|
||||
return errors.New("TASK_PROGRESS_CALLBACK_TIMEOUT_MS must be between 100 and 60000")
|
||||
}
|
||||
if c.TaskProgressCallbackMaxAttempts != 0 && (c.TaskProgressCallbackMaxAttempts < 1 || c.TaskProgressCallbackMaxAttempts > 100) {
|
||||
return errors.New("TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS must be between 1 and 100")
|
||||
}
|
||||
if c.TaskRetentionDays != 0 && (c.TaskRetentionDays < 1 || c.TaskRetentionDays > 3650) {
|
||||
return errors.New("AI_GATEWAY_TASK_RETENTION_DAYS must be between 1 and 3650")
|
||||
}
|
||||
if c.TaskAnalysisRetentionDays != 0 && (c.TaskAnalysisRetentionDays < 1 || (c.TaskRetentionDays != 0 && c.TaskAnalysisRetentionDays > c.TaskRetentionDays)) {
|
||||
return errors.New("AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS must be between 1 and AI_GATEWAY_TASK_RETENTION_DAYS")
|
||||
}
|
||||
if c.TaskCleanupIntervalSeconds != 0 && c.TaskCleanupIntervalSeconds < 60 {
|
||||
return errors.New("AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS must be at least 60")
|
||||
}
|
||||
if c.TaskCleanupBatchSize != 0 && (c.TaskCleanupBatchSize < 100 || c.TaskCleanupBatchSize > 5000) {
|
||||
return errors.New("AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE must be between 100 and 5000")
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(c.IdentitySecretStore)) {
|
||||
case "":
|
||||
case "file":
|
||||
|
||||
@@ -81,3 +81,22 @@ func TestValidateAsyncWorkerSettings(t *testing.T) {
|
||||
t.Fatalf("Validate() error = %v, want invalid non-integer hard limit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTaskHistorySettings(t *testing.T) {
|
||||
cfg := Load()
|
||||
cfg.TaskRetentionDays = 30
|
||||
cfg.TaskAnalysisRetentionDays = 31
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ANALYSIS_RETENTION") {
|
||||
t.Fatalf("Validate() error = %v, want invalid analysis retention", err)
|
||||
}
|
||||
cfg.TaskAnalysisRetentionDays = 7
|
||||
cfg.TaskCleanupIntervalSeconds = 59
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "CLEANUP_INTERVAL") {
|
||||
t.Fatalf("Validate() error = %v, want invalid cleanup interval", err)
|
||||
}
|
||||
cfg.TaskCleanupIntervalSeconds = 300
|
||||
cfg.TaskCleanupBatchSize = 5001
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "CLEANUP_BATCH") {
|
||||
t.Fatalf("Validate() error = %v, want invalid cleanup batch", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user