perf(storage): 极简化任务历史并增加保留治理

停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON,并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。

增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭,待兼容协议和异步恢复在线验证后单独启用。

验证:Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
2026-07-24 18:23:22 +08:00
parent 09375bfae7
commit 810dcfeee6
46 changed files with 2246 additions and 458 deletions
@@ -223,7 +223,6 @@ func (s *Server) createVolcesCompatibleTask(r *http.Request, user *auth.User, bo
return store.GatewayTask{}, err
}
task.CompatibilityProtocol = clients.ProtocolVolcesContents
task.CompatibilityPublicID = task.ID
if err := s.runner.EnqueueAsyncTask(r.Context(), task); err != nil {
return store.GatewayTask{}, &clients.ClientError{Code: "enqueue_failed", Message: err.Error(), StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
@@ -258,7 +257,6 @@ func (s *Server) waitForCompatibilitySubmission(r *http.Request, task store.Gate
Message: firstNonEmpty(current.ErrorMessage, current.Error, current.Message),
StatusCode: status,
Retryable: false,
Wire: compatibilitySubmissionWire(current),
}
}
select {
@@ -271,34 +269,6 @@ func (s *Server) waitForCompatibilitySubmission(r *http.Request, task store.Gate
}
}
func compatibilitySubmissionWire(task store.GatewayTask) *clients.WireResponse {
if task.CompatibilitySubmitHTTPStatus == 0 || strings.TrimSpace(task.CompatibilitySourceProtocol) == "" {
return nil
}
headers := map[string][]string{}
for name, value := range task.CompatibilitySubmitHeaders {
switch typed := value.(type) {
case []any:
for _, item := range typed {
if text := strings.TrimSpace(volcesCompatString(item)); text != "" {
headers[name] = append(headers[name], text)
}
}
case []string:
headers[name] = append([]string(nil), typed...)
case string:
headers[name] = []string{typed}
}
}
return &clients.WireResponse{
Protocol: task.CompatibilitySourceProtocol,
StatusCode: task.CompatibilitySubmitHTTPStatus,
Headers: headers,
Body: cloneVolcesCompatibleMap(task.CompatibilitySubmitBody),
Converted: task.CompatibilitySourceProtocol != task.CompatibilityProtocol,
}
}
func (s *Server) volcesCompatibleTaskForUser(w http.ResponseWriter, r *http.Request) (store.GatewayTask, bool) {
user, ok := auth.UserFromContext(r.Context())
if !ok || user == nil {
@@ -327,15 +297,30 @@ func isVolcesCompatibleTask(task store.GatewayTask) bool {
}
func volcesCompatibleTask(task store.GatewayTask) map[string]any {
if volcesTaskUsesNativeProtocol(task) {
if raw := cloneVolcesCompatibleMap(mapFromVolcesCompat(task.Result["raw"])); len(raw) > 0 {
return raw
response := map[string]any{}
for _, key := range []string{"model", "seed", "resolution", "ratio", "duration", "frames", "framespersecond"} {
if task.Result[key] != nil {
response[key] = task.Result[key]
}
if raw := cloneVolcesCompatibleMap(task.RemoteTaskPayload); len(raw) > 0 {
return raw
}
if content, ok := task.Result["content"].(map[string]any); ok && len(content) > 0 {
// Transitional read support for historical canonical results. New
// results only persist data[] and reconstruct this protocol field.
response["content"] = content
} else if data, ok := task.Result["data"].([]any); ok && len(data) > 0 {
if item, ok := data[0].(map[string]any); ok {
content := map[string]any{}
if videoURL := firstNonEmpty(volcesCompatString(item["url"]), volcesCompatString(item["video_url"])); videoURL != "" {
content["video_url"] = videoURL
}
if lastFrameURL := volcesCompatString(item["last_frame_url"]); lastFrameURL != "" {
content["last_frame_url"] = lastFrameURL
}
if len(content) > 0 {
response["content"] = content
}
}
}
response := map[string]any{}
response["id"] = volcesCompatiblePublicID(task)
response["model"] = firstNonEmpty(volcesCompatString(response["model"]), task.Model)
response["status"] = volcesCompatibleTaskStatus(task.Status)
@@ -346,8 +331,10 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
response[key] = task.Request[key]
}
}
if len(task.Usage) > 0 && response["usage"] == nil {
response["usage"] = task.Usage
if usage := volcesCompatibleUsage(task.Usage); len(usage) > 0 {
response["usage"] = usage
} else if legacyUsage, ok := task.Result["usage"].(map[string]any); ok && len(legacyUsage) > 0 {
response["usage"] = legacyUsage
}
if task.Status == "failed" || task.Status == "cancelled" {
response["error"] = map[string]any{"code": firstNonEmpty(task.ErrorCode, strings.ToUpper(task.Status)), "message": firstNonEmpty(task.ErrorMessage, task.Error, task.Message)}
@@ -355,19 +342,28 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
return response
}
func volcesCompatibleCreateResponse(task store.GatewayTask) map[string]any {
if volcesTaskUsesNativeProtocol(task) {
if raw := cloneVolcesCompatibleMap(task.RemoteTaskPayload); len(raw) > 0 {
return raw
func volcesCompatibleUsage(usage map[string]any) map[string]any {
out := map[string]any{}
for outputKey, inputKeys := range map[string][]string{
"prompt_tokens": {"inputTokens", "promptTokens"},
"completion_tokens": {"outputTokens", "completionTokens"},
"total_tokens": {"totalTokens"},
} {
for _, inputKey := range inputKeys {
if value := usage[inputKey]; value != nil {
out[outputKey] = value
break
}
}
}
return out
}
func volcesCompatibleCreateResponse(task store.GatewayTask) map[string]any {
return map[string]any{"id": volcesCompatiblePublicID(task)}
}
func volcesCompatiblePublicID(task store.GatewayTask) string {
if strings.TrimSpace(task.CompatibilityPublicID) != "" {
return strings.TrimSpace(task.CompatibilityPublicID)
}
if volcesTaskUsesNativeProtocol(task) && strings.TrimSpace(task.RemoteTaskID) != "" {
return strings.TrimSpace(task.RemoteTaskID)
}
@@ -386,11 +382,6 @@ func volcesTaskUsesNativeProtocol(task store.GatewayTask) bool {
return false
}
func mapFromVolcesCompat(value any) map[string]any {
result, _ := value.(map[string]any)
return result
}
func volcesCompatibleTaskStatus(status string) string {
switch strings.ToLower(strings.TrimSpace(status)) {
case "succeeded", "success", "completed":
@@ -406,21 +397,6 @@ func volcesCompatibleTaskStatus(status string) string {
}
}
func cloneVolcesCompatibleMap(source map[string]any) map[string]any {
if len(source) == 0 {
return nil
}
raw, err := json.Marshal(source)
if err != nil {
return map[string]any{}
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return map[string]any{}
}
return out
}
func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
status := http.StatusInternalServerError
var staged *gatewayTaskCreationError