perf(storage): 极简化任务历史并增加保留治理
停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON,并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。 增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭,待兼容协议和异步恢复在线验证后单独启用。 验证:Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
@@ -223,7 +223,6 @@ func (s *Server) createVolcesCompatibleTask(r *http.Request, user *auth.User, bo
|
||||
return store.GatewayTask{}, err
|
||||
}
|
||||
task.CompatibilityProtocol = clients.ProtocolVolcesContents
|
||||
task.CompatibilityPublicID = task.ID
|
||||
if err := s.runner.EnqueueAsyncTask(r.Context(), task); err != nil {
|
||||
return store.GatewayTask{}, &clients.ClientError{Code: "enqueue_failed", Message: err.Error(), StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
@@ -258,7 +257,6 @@ func (s *Server) waitForCompatibilitySubmission(r *http.Request, task store.Gate
|
||||
Message: firstNonEmpty(current.ErrorMessage, current.Error, current.Message),
|
||||
StatusCode: status,
|
||||
Retryable: false,
|
||||
Wire: compatibilitySubmissionWire(current),
|
||||
}
|
||||
}
|
||||
select {
|
||||
@@ -271,34 +269,6 @@ func (s *Server) waitForCompatibilitySubmission(r *http.Request, task store.Gate
|
||||
}
|
||||
}
|
||||
|
||||
func compatibilitySubmissionWire(task store.GatewayTask) *clients.WireResponse {
|
||||
if task.CompatibilitySubmitHTTPStatus == 0 || strings.TrimSpace(task.CompatibilitySourceProtocol) == "" {
|
||||
return nil
|
||||
}
|
||||
headers := map[string][]string{}
|
||||
for name, value := range task.CompatibilitySubmitHeaders {
|
||||
switch typed := value.(type) {
|
||||
case []any:
|
||||
for _, item := range typed {
|
||||
if text := strings.TrimSpace(volcesCompatString(item)); text != "" {
|
||||
headers[name] = append(headers[name], text)
|
||||
}
|
||||
}
|
||||
case []string:
|
||||
headers[name] = append([]string(nil), typed...)
|
||||
case string:
|
||||
headers[name] = []string{typed}
|
||||
}
|
||||
}
|
||||
return &clients.WireResponse{
|
||||
Protocol: task.CompatibilitySourceProtocol,
|
||||
StatusCode: task.CompatibilitySubmitHTTPStatus,
|
||||
Headers: headers,
|
||||
Body: cloneVolcesCompatibleMap(task.CompatibilitySubmitBody),
|
||||
Converted: task.CompatibilitySourceProtocol != task.CompatibilityProtocol,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) volcesCompatibleTaskForUser(w http.ResponseWriter, r *http.Request) (store.GatewayTask, bool) {
|
||||
user, ok := auth.UserFromContext(r.Context())
|
||||
if !ok || user == nil {
|
||||
@@ -327,15 +297,30 @@ func isVolcesCompatibleTask(task store.GatewayTask) bool {
|
||||
}
|
||||
|
||||
func volcesCompatibleTask(task store.GatewayTask) map[string]any {
|
||||
if volcesTaskUsesNativeProtocol(task) {
|
||||
if raw := cloneVolcesCompatibleMap(mapFromVolcesCompat(task.Result["raw"])); len(raw) > 0 {
|
||||
return raw
|
||||
response := map[string]any{}
|
||||
for _, key := range []string{"model", "seed", "resolution", "ratio", "duration", "frames", "framespersecond"} {
|
||||
if task.Result[key] != nil {
|
||||
response[key] = task.Result[key]
|
||||
}
|
||||
if raw := cloneVolcesCompatibleMap(task.RemoteTaskPayload); len(raw) > 0 {
|
||||
return raw
|
||||
}
|
||||
if content, ok := task.Result["content"].(map[string]any); ok && len(content) > 0 {
|
||||
// Transitional read support for historical canonical results. New
|
||||
// results only persist data[] and reconstruct this protocol field.
|
||||
response["content"] = content
|
||||
} else if data, ok := task.Result["data"].([]any); ok && len(data) > 0 {
|
||||
if item, ok := data[0].(map[string]any); ok {
|
||||
content := map[string]any{}
|
||||
if videoURL := firstNonEmpty(volcesCompatString(item["url"]), volcesCompatString(item["video_url"])); videoURL != "" {
|
||||
content["video_url"] = videoURL
|
||||
}
|
||||
if lastFrameURL := volcesCompatString(item["last_frame_url"]); lastFrameURL != "" {
|
||||
content["last_frame_url"] = lastFrameURL
|
||||
}
|
||||
if len(content) > 0 {
|
||||
response["content"] = content
|
||||
}
|
||||
}
|
||||
}
|
||||
response := map[string]any{}
|
||||
response["id"] = volcesCompatiblePublicID(task)
|
||||
response["model"] = firstNonEmpty(volcesCompatString(response["model"]), task.Model)
|
||||
response["status"] = volcesCompatibleTaskStatus(task.Status)
|
||||
@@ -346,8 +331,10 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
|
||||
response[key] = task.Request[key]
|
||||
}
|
||||
}
|
||||
if len(task.Usage) > 0 && response["usage"] == nil {
|
||||
response["usage"] = task.Usage
|
||||
if usage := volcesCompatibleUsage(task.Usage); len(usage) > 0 {
|
||||
response["usage"] = usage
|
||||
} else if legacyUsage, ok := task.Result["usage"].(map[string]any); ok && len(legacyUsage) > 0 {
|
||||
response["usage"] = legacyUsage
|
||||
}
|
||||
if task.Status == "failed" || task.Status == "cancelled" {
|
||||
response["error"] = map[string]any{"code": firstNonEmpty(task.ErrorCode, strings.ToUpper(task.Status)), "message": firstNonEmpty(task.ErrorMessage, task.Error, task.Message)}
|
||||
@@ -355,19 +342,28 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
|
||||
return response
|
||||
}
|
||||
|
||||
func volcesCompatibleCreateResponse(task store.GatewayTask) map[string]any {
|
||||
if volcesTaskUsesNativeProtocol(task) {
|
||||
if raw := cloneVolcesCompatibleMap(task.RemoteTaskPayload); len(raw) > 0 {
|
||||
return raw
|
||||
func volcesCompatibleUsage(usage map[string]any) map[string]any {
|
||||
out := map[string]any{}
|
||||
for outputKey, inputKeys := range map[string][]string{
|
||||
"prompt_tokens": {"inputTokens", "promptTokens"},
|
||||
"completion_tokens": {"outputTokens", "completionTokens"},
|
||||
"total_tokens": {"totalTokens"},
|
||||
} {
|
||||
for _, inputKey := range inputKeys {
|
||||
if value := usage[inputKey]; value != nil {
|
||||
out[outputKey] = value
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func volcesCompatibleCreateResponse(task store.GatewayTask) map[string]any {
|
||||
return map[string]any{"id": volcesCompatiblePublicID(task)}
|
||||
}
|
||||
|
||||
func volcesCompatiblePublicID(task store.GatewayTask) string {
|
||||
if strings.TrimSpace(task.CompatibilityPublicID) != "" {
|
||||
return strings.TrimSpace(task.CompatibilityPublicID)
|
||||
}
|
||||
if volcesTaskUsesNativeProtocol(task) && strings.TrimSpace(task.RemoteTaskID) != "" {
|
||||
return strings.TrimSpace(task.RemoteTaskID)
|
||||
}
|
||||
@@ -386,11 +382,6 @@ func volcesTaskUsesNativeProtocol(task store.GatewayTask) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func mapFromVolcesCompat(value any) map[string]any {
|
||||
result, _ := value.(map[string]any)
|
||||
return result
|
||||
}
|
||||
|
||||
func volcesCompatibleTaskStatus(status string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(status)) {
|
||||
case "succeeded", "success", "completed":
|
||||
@@ -406,21 +397,6 @@ func volcesCompatibleTaskStatus(status string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func cloneVolcesCompatibleMap(source map[string]any) map[string]any {
|
||||
if len(source) == 0 {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(source)
|
||||
if err != nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
|
||||
status := http.StatusInternalServerError
|
||||
var staged *gatewayTaskCreationError
|
||||
|
||||
Reference in New Issue
Block a user