perf(storage): 极简化任务历史并增加保留治理

停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON，并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭，待兼容协议和异步恢复在线验证后单独启用。验证：Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
2026-07-24 18:23:22 +08:00
parent 09375bfae7
commit 810dcfeee6
46 changed files with 2246 additions and 458 deletions
@@ -2,10 +2,10 @@ package runner

 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
-	"net/http"
 	"os"
 	"regexp"
 	"strconv"
@@ -88,6 +88,24 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
 	if cfg.AsyncWorkerRefreshIntervalSeconds == 0 {
 		cfg.AsyncWorkerRefreshIntervalSeconds = 5
 	}
+	if cfg.TaskProgressCallbackTimeoutMS == 0 {
+		cfg.TaskProgressCallbackTimeoutMS = 5000
+	}
+	if cfg.TaskProgressCallbackMaxAttempts == 0 {
+		cfg.TaskProgressCallbackMaxAttempts = 10
+	}
+	if cfg.TaskRetentionDays == 0 {
+		cfg.TaskRetentionDays = 30
+	}
+	if cfg.TaskAnalysisRetentionDays == 0 {
+		cfg.TaskAnalysisRetentionDays = 7
+	}
+	if cfg.TaskCleanupIntervalSeconds == 0 {
+		cfg.TaskCleanupIntervalSeconds = 300
+	}
+	if cfg.TaskCleanupBatchSize == 0 {
+		cfg.TaskCleanupBatchSize = 1000
+	}
 	httpClients := newHTTPClientCache()
 	scriptExecutor := &scriptengine.Executor{Logger: logger}
 	service := &Service{
@@ -130,6 +148,15 @@ func (s *Service) observeBillingEvent(event string) {
 	}
 }

+func (s *Service) observeTaskEventSkip(reason string) {
+	observer, ok := s.billingMetrics.(interface {
+		ObserveTaskEventSkip(string)
+	})
+	if ok {
+		observer.ObserveTaskEventSkip(reason)
+	}
+}
+
 func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
 	return s.execute(ctx, task, user, nil)
 }
@@ -569,6 +596,7 @@ candidatesLoop:
 			}
 			if err == nil {
 				attemptNo = nextAttemptNo
+				response.Result = canonicalTaskResult(response.Result)
 				var billings []any
 				finalAmount := fixedAmount(0)
 				finalAmountText := ""
@@ -1058,12 +1086,13 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 			if strings.TrimSpace(remoteTaskID) == "" {
 				return nil
 			}
-			if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, payload); err != nil {
+			checkpoint := minimalRemoteTaskCheckpoint(candidate.Provider, candidate.SpecType, payload)
+			if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, checkpoint); err != nil {
 				return err
 			}
 			task.RemoteTaskID = remoteTaskID
-			task.RemoteTaskPayload = payload
-			if err := s.persistCompatibilitySubmission(context.WithoutCancel(ctx), task, candidate, remoteTaskID, payload, submissionWire); err != nil {
+			task.RemoteTaskPayload = checkpoint
+			if err := s.persistCompatibilitySubmission(context.WithoutCancel(ctx), task, candidate, remoteTaskID, checkpoint, submissionWire); err != nil {
 				return err
 			}
 			return setSubmissionStatus("response_received")
@@ -1072,11 +1101,12 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 			if strings.TrimSpace(remoteTaskID) == "" {
 				return nil
 			}
-			if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, payload); err != nil {
+			checkpoint := minimalRemoteTaskCheckpoint(candidate.Provider, candidate.SpecType, payload)
+			if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, checkpoint); err != nil {
 				return err
 			}
 			task.RemoteTaskID = remoteTaskID
-			task.RemoteTaskPayload = payload
+			task.RemoteTaskPayload = checkpoint
 			return setSubmissionStatus("response_received")
 		},
 		OnUpstreamSubmissionStarted: func() error {
@@ -1258,6 +1288,59 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
 	return response, nil
 }

+func minimalRemoteTaskCheckpoint(provider string, specType string, payload map[string]any) map[string]any {
+	const maxBytes = 8192
+	provider = strings.ToLower(strings.TrimSpace(provider))
+	specType = strings.ToLower(strings.TrimSpace(specType))
+	allowedByProvider := map[string][]string{
+		"keling":     {"endpoint", "pollEndpoint", "mode", "taskType", "cleanupElementIds"},
+		"kling":      {"endpoint", "pollEndpoint", "mode", "taskType", "cleanupElementIds"},
+		"topaz":      {"phase", "targetResolution"},
+		"vectorizer": {"imageToken", "receipt"},
+	}
+	allowedKeys := allowedByProvider[provider]
+	if len(allowedKeys) == 0 {
+		allowedKeys = allowedByProvider[specType]
+	}
+	checkpoint := map[string]any{}
+	for _, key := range allowedKeys {
+		value, ok := payload[key]
+		if !ok || value == nil {
+			continue
+		}
+		if text, ok := value.(string); ok {
+			value = truncateText(text, 4096)
+		}
+		next := cloneMap(checkpoint)
+		next[key] = value
+		encoded, err := json.Marshal(next)
+		if err != nil || len(encoded) > maxBytes {
+			continue
+		}
+		checkpoint = next
+	}
+	return checkpoint
+}
+
+func canonicalTaskResult(result map[string]any) map[string]any {
+	canonical := cloneMap(result)
+	for _, key := range []string{
+		"raw",
+		"raw_data",
+		"rawData",
+		"provider_response",
+		"providerResponse",
+		"submit",
+		"file_retrieve",
+		"fileRetrieve",
+		"upstream_task_id",
+		"remote_task_id",
+	} {
+		delete(canonical, key)
+	}
+	return canonical
+}
+
 func (s *Service) persistCompatibilitySubmission(ctx context.Context, task store.GatewayTask, candidate store.RuntimeModelCandidate, remoteTaskID string, payload map[string]any, wire *clients.WireResponse) error {
 	targetProtocol := strings.TrimSpace(stringFromMap(task.Request, "_gateway_target_protocol"))
 	if targetProtocol == "" {
@@ -1267,32 +1350,9 @@ func (s *Service) persistCompatibilitySubmission(ctx context.Context, task store
 	if wire != nil && strings.TrimSpace(wire.Protocol) != "" {
 		sourceProtocol = strings.TrimSpace(wire.Protocol)
 	}
-	publicID := task.ID
-	if sourceProtocol == targetProtocol && strings.TrimSpace(remoteTaskID) != "" {
-		publicID = strings.TrimSpace(remoteTaskID)
-	}
-	submitBody := payload
-	if nested, ok := payload["submit"].(map[string]any); ok && len(nested) > 0 {
-		submitBody = nested
-	}
-	httpStatus := http.StatusOK
-	headers := map[string]any{}
-	if wire != nil {
-		httpStatus = wire.StatusCode
-		if len(wire.Body) > 0 {
-			submitBody = wire.Body
-		}
-		for name, values := range wire.Headers {
-			headers[name] = append([]string(nil), values...)
-		}
-	}
 	return s.store.SetTaskCompatibilitySubmission(ctx, task.ID, store.CompatibilitySubmission{
 		TargetProtocol: targetProtocol,
-		PublicID:       publicID,
 		SourceProtocol: sourceProtocol,
-		HTTPStatus:     httpStatus,
-		Headers:        headers,
-		Body:           submitBody,
 	})
 }

@@ -1327,9 +1387,10 @@ func compatibilitySourceProtocol(kind string, candidate store.RuntimeModelCandid
 }

 func (s *Service) recordTaskParameterPreprocessing(ctx context.Context, taskID string, attemptID string, attemptNo int, candidate store.RuntimeModelCandidate, log parameterPreprocessingLog) error {
-	if skipTaskParameterPreprocessingLog(log.ModelType) && !log.Changed {
+	if !log.Changed {
 		return nil
 	}
+	changes := minimalParameterPreprocessingChanges(log.Changes)
 	_, err := s.store.CreateTaskParamPreprocessingLog(ctx, store.CreateTaskParamPreprocessingLogInput{
 		TaskID:          taskID,
 		AttemptID:       attemptID,
@@ -1338,16 +1399,31 @@ func (s *Service) recordTaskParameterPreprocessing(ctx context.Context, taskID s
 		PlatformID:      candidate.PlatformID,
 		PlatformModelID: candidate.PlatformModelID,
 		ClientID:        candidate.ClientID,
-		Changed:         log.Changed,
+		Changed:         true,
 		ChangeCount:     len(log.Changes),
-		ActualInput:     log.Input,
-		ConvertedOutput: log.Output,
-		Changes:         log.Changes,
-		ModelSnapshot:   log.Model,
+		Changes:         changes,
 	})
 	return err
 }

+func minimalParameterPreprocessingChanges(changes []parameterPreprocessChange) []any {
+	const maxBytes = 1024
+	out := make([]any, 0, len(changes))
+	for _, change := range changes {
+		item := map[string]any{
+			"path":   truncateText(change.Path, 256),
+			"action": truncateText(change.Action, 64),
+		}
+		next := append(append([]any(nil), out...), item)
+		encoded, err := json.Marshal(next)
+		if err != nil || len(encoded) > maxBytes {
+			break
+		}
+		out = next
+	}
+	return out
+}
+
 func skipTaskParameterPreprocessingLog(modelType string) bool {
 	switch strings.TrimSpace(modelType) {
 	case "text_generate", "text_embedding", "text_rerank", "chat", "responses", "text":
@@ -1357,6 +1433,15 @@ func skipTaskParameterPreprocessingLog(modelType string) bool {
 	}
 }

+func truncateText(value string, maxRunes int) string {
+	value = strings.TrimSpace(value)
+	runes := []rune(value)
+	if len(runes) <= maxRunes {
+		return value
+	}
+	return string(runes[:maxRunes])
+}
+
 func (s *Service) clientFor(candidate store.RuntimeModelCandidate, simulated bool) clients.Client {
 	if simulated {
 		return s.clients["simulation"]
@@ -1628,17 +1713,6 @@ func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.Ga
 }

 func (s *Service) withAttemptHistory(ctx context.Context, taskID string, metrics map[string]any) map[string]any {
-	attempts, err := s.store.ListTaskAttempts(ctx, taskID)
-	if err != nil {
-		s.logger.Warn("list task attempts for metrics failed", "taskID", taskID, "error", err)
-		return metrics
-	}
-	if len(attempts) == 0 {
-		return metrics
-	}
-	metrics = mergeMetrics(metrics)
-	metrics["attemptCount"] = len(attempts)
-	metrics["attempts"] = summarizeAttempts(attempts)
 	return metrics
 }

@@ -1647,6 +1721,9 @@ func (s *Service) emit(ctx context.Context, taskID string, eventType string, sta
 	if err != nil {
 		return err
 	}
+	if event.ID == "" {
+		s.observeTaskEventSkip(event.SkippedReason)
+	}
 	if s.cfg.TaskProgressCallbackEnabled {
 		return s.store.QueueTaskCallback(ctx, event, s.cfg.TaskProgressCallbackURL)
 	}