perf(storage): 极简化任务历史并增加保留治理

停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON,并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。

增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭,待兼容协议和异步恢复在线验证后单独启用。

验证:Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
2026-07-24 18:23:22 +08:00
parent 09375bfae7
commit 810dcfeee6
46 changed files with 2246 additions and 458 deletions
+123 -46
View File
@@ -2,10 +2,10 @@ package runner
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"regexp"
"strconv"
@@ -88,6 +88,24 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
if cfg.AsyncWorkerRefreshIntervalSeconds == 0 {
cfg.AsyncWorkerRefreshIntervalSeconds = 5
}
if cfg.TaskProgressCallbackTimeoutMS == 0 {
cfg.TaskProgressCallbackTimeoutMS = 5000
}
if cfg.TaskProgressCallbackMaxAttempts == 0 {
cfg.TaskProgressCallbackMaxAttempts = 10
}
if cfg.TaskRetentionDays == 0 {
cfg.TaskRetentionDays = 30
}
if cfg.TaskAnalysisRetentionDays == 0 {
cfg.TaskAnalysisRetentionDays = 7
}
if cfg.TaskCleanupIntervalSeconds == 0 {
cfg.TaskCleanupIntervalSeconds = 300
}
if cfg.TaskCleanupBatchSize == 0 {
cfg.TaskCleanupBatchSize = 1000
}
httpClients := newHTTPClientCache()
scriptExecutor := &scriptengine.Executor{Logger: logger}
service := &Service{
@@ -130,6 +148,15 @@ func (s *Service) observeBillingEvent(event string) {
}
}
func (s *Service) observeTaskEventSkip(reason string) {
observer, ok := s.billingMetrics.(interface {
ObserveTaskEventSkip(string)
})
if ok {
observer.ObserveTaskEventSkip(reason)
}
}
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
return s.execute(ctx, task, user, nil)
}
@@ -569,6 +596,7 @@ candidatesLoop:
}
if err == nil {
attemptNo = nextAttemptNo
response.Result = canonicalTaskResult(response.Result)
var billings []any
finalAmount := fixedAmount(0)
finalAmountText := ""
@@ -1058,12 +1086,13 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
if strings.TrimSpace(remoteTaskID) == "" {
return nil
}
if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, payload); err != nil {
checkpoint := minimalRemoteTaskCheckpoint(candidate.Provider, candidate.SpecType, payload)
if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, checkpoint); err != nil {
return err
}
task.RemoteTaskID = remoteTaskID
task.RemoteTaskPayload = payload
if err := s.persistCompatibilitySubmission(context.WithoutCancel(ctx), task, candidate, remoteTaskID, payload, submissionWire); err != nil {
task.RemoteTaskPayload = checkpoint
if err := s.persistCompatibilitySubmission(context.WithoutCancel(ctx), task, candidate, remoteTaskID, checkpoint, submissionWire); err != nil {
return err
}
return setSubmissionStatus("response_received")
@@ -1072,11 +1101,12 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
if strings.TrimSpace(remoteTaskID) == "" {
return nil
}
if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, payload); err != nil {
checkpoint := minimalRemoteTaskCheckpoint(candidate.Provider, candidate.SpecType, payload)
if err := s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, checkpoint); err != nil {
return err
}
task.RemoteTaskID = remoteTaskID
task.RemoteTaskPayload = payload
task.RemoteTaskPayload = checkpoint
return setSubmissionStatus("response_received")
},
OnUpstreamSubmissionStarted: func() error {
@@ -1258,6 +1288,59 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
return response, nil
}
func minimalRemoteTaskCheckpoint(provider string, specType string, payload map[string]any) map[string]any {
const maxBytes = 8192
provider = strings.ToLower(strings.TrimSpace(provider))
specType = strings.ToLower(strings.TrimSpace(specType))
allowedByProvider := map[string][]string{
"keling": {"endpoint", "pollEndpoint", "mode", "taskType", "cleanupElementIds"},
"kling": {"endpoint", "pollEndpoint", "mode", "taskType", "cleanupElementIds"},
"topaz": {"phase", "targetResolution"},
"vectorizer": {"imageToken", "receipt"},
}
allowedKeys := allowedByProvider[provider]
if len(allowedKeys) == 0 {
allowedKeys = allowedByProvider[specType]
}
checkpoint := map[string]any{}
for _, key := range allowedKeys {
value, ok := payload[key]
if !ok || value == nil {
continue
}
if text, ok := value.(string); ok {
value = truncateText(text, 4096)
}
next := cloneMap(checkpoint)
next[key] = value
encoded, err := json.Marshal(next)
if err != nil || len(encoded) > maxBytes {
continue
}
checkpoint = next
}
return checkpoint
}
func canonicalTaskResult(result map[string]any) map[string]any {
canonical := cloneMap(result)
for _, key := range []string{
"raw",
"raw_data",
"rawData",
"provider_response",
"providerResponse",
"submit",
"file_retrieve",
"fileRetrieve",
"upstream_task_id",
"remote_task_id",
} {
delete(canonical, key)
}
return canonical
}
func (s *Service) persistCompatibilitySubmission(ctx context.Context, task store.GatewayTask, candidate store.RuntimeModelCandidate, remoteTaskID string, payload map[string]any, wire *clients.WireResponse) error {
targetProtocol := strings.TrimSpace(stringFromMap(task.Request, "_gateway_target_protocol"))
if targetProtocol == "" {
@@ -1267,32 +1350,9 @@ func (s *Service) persistCompatibilitySubmission(ctx context.Context, task store
if wire != nil && strings.TrimSpace(wire.Protocol) != "" {
sourceProtocol = strings.TrimSpace(wire.Protocol)
}
publicID := task.ID
if sourceProtocol == targetProtocol && strings.TrimSpace(remoteTaskID) != "" {
publicID = strings.TrimSpace(remoteTaskID)
}
submitBody := payload
if nested, ok := payload["submit"].(map[string]any); ok && len(nested) > 0 {
submitBody = nested
}
httpStatus := http.StatusOK
headers := map[string]any{}
if wire != nil {
httpStatus = wire.StatusCode
if len(wire.Body) > 0 {
submitBody = wire.Body
}
for name, values := range wire.Headers {
headers[name] = append([]string(nil), values...)
}
}
return s.store.SetTaskCompatibilitySubmission(ctx, task.ID, store.CompatibilitySubmission{
TargetProtocol: targetProtocol,
PublicID: publicID,
SourceProtocol: sourceProtocol,
HTTPStatus: httpStatus,
Headers: headers,
Body: submitBody,
})
}
@@ -1327,9 +1387,10 @@ func compatibilitySourceProtocol(kind string, candidate store.RuntimeModelCandid
}
func (s *Service) recordTaskParameterPreprocessing(ctx context.Context, taskID string, attemptID string, attemptNo int, candidate store.RuntimeModelCandidate, log parameterPreprocessingLog) error {
if skipTaskParameterPreprocessingLog(log.ModelType) && !log.Changed {
if !log.Changed {
return nil
}
changes := minimalParameterPreprocessingChanges(log.Changes)
_, err := s.store.CreateTaskParamPreprocessingLog(ctx, store.CreateTaskParamPreprocessingLogInput{
TaskID: taskID,
AttemptID: attemptID,
@@ -1338,16 +1399,31 @@ func (s *Service) recordTaskParameterPreprocessing(ctx context.Context, taskID s
PlatformID: candidate.PlatformID,
PlatformModelID: candidate.PlatformModelID,
ClientID: candidate.ClientID,
Changed: log.Changed,
Changed: true,
ChangeCount: len(log.Changes),
ActualInput: log.Input,
ConvertedOutput: log.Output,
Changes: log.Changes,
ModelSnapshot: log.Model,
Changes: changes,
})
return err
}
func minimalParameterPreprocessingChanges(changes []parameterPreprocessChange) []any {
const maxBytes = 1024
out := make([]any, 0, len(changes))
for _, change := range changes {
item := map[string]any{
"path": truncateText(change.Path, 256),
"action": truncateText(change.Action, 64),
}
next := append(append([]any(nil), out...), item)
encoded, err := json.Marshal(next)
if err != nil || len(encoded) > maxBytes {
break
}
out = next
}
return out
}
func skipTaskParameterPreprocessingLog(modelType string) bool {
switch strings.TrimSpace(modelType) {
case "text_generate", "text_embedding", "text_rerank", "chat", "responses", "text":
@@ -1357,6 +1433,15 @@ func skipTaskParameterPreprocessingLog(modelType string) bool {
}
}
func truncateText(value string, maxRunes int) string {
value = strings.TrimSpace(value)
runes := []rune(value)
if len(runes) <= maxRunes {
return value
}
return string(runes[:maxRunes])
}
func (s *Service) clientFor(candidate store.RuntimeModelCandidate, simulated bool) clients.Client {
if simulated {
return s.clients["simulation"]
@@ -1628,17 +1713,6 @@ func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.Ga
}
func (s *Service) withAttemptHistory(ctx context.Context, taskID string, metrics map[string]any) map[string]any {
attempts, err := s.store.ListTaskAttempts(ctx, taskID)
if err != nil {
s.logger.Warn("list task attempts for metrics failed", "taskID", taskID, "error", err)
return metrics
}
if len(attempts) == 0 {
return metrics
}
metrics = mergeMetrics(metrics)
metrics["attemptCount"] = len(attempts)
metrics["attempts"] = summarizeAttempts(attempts)
return metrics
}
@@ -1647,6 +1721,9 @@ func (s *Service) emit(ctx context.Context, taskID string, eventType string, sta
if err != nil {
return err
}
if event.ID == "" {
s.observeTaskEventSkip(event.SkippedReason)
}
if s.cfg.TaskProgressCallbackEnabled {
return s.store.QueueTaskCallback(ctx, event, s.cfg.TaskProgressCallbackURL)
}
@@ -0,0 +1,181 @@
package runner
import (
"bytes"
"context"
"encoding/json"
"fmt"
"hash/fnv"
"io"
"net/http"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
func (s *Service) StartTaskHistoryWorkers(ctx context.Context) {
if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" {
go s.runTaskCallbackWorker(ctx, "callback-"+uuid.NewString())
}
if s.cfg.TaskCleanupEnabled {
go s.runTaskHistoryCleanupWorker(ctx)
}
}
func (s *Service) runTaskCallbackWorker(ctx context.Context, workerID string) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
client := &http.Client{Timeout: time.Duration(s.cfg.TaskProgressCallbackTimeoutMS) * time.Millisecond}
for {
s.processTaskCallbackBatch(ctx, client, workerID)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (s *Service) processTaskCallbackBatch(ctx context.Context, client *http.Client, workerID string) {
items, err := s.store.ClaimTaskCallbacks(ctx, workerID, store.TaskCallbackBatchSize, store.TaskCallbackLockTTL)
if err != nil {
if ctx.Err() == nil {
s.logger.Error("claim task callbacks failed", "error_category", "task_callback_claim_failed")
}
return
}
for _, item := range items {
if ctx.Err() != nil {
return
}
statusCode, deliveryErr := deliverTaskCallback(ctx, client, item, s.cfg.ServerMainInternalToken)
if deliveryErr == nil && statusCode >= 200 && statusCode < 300 {
if err := s.store.MarkTaskCallbackDelivered(context.WithoutCancel(ctx), item); err != nil {
s.logger.Error("mark task callback delivered failed", "taskID", item.TaskID, "seq", item.Seq, "error_category", "task_callback_state_failed")
}
continue
}
retryable := deliveryErr != nil || statusCode == http.StatusRequestTimeout || statusCode == http.StatusTooManyRequests || statusCode >= 500
retry := retryable && item.Attempts < s.cfg.TaskProgressCallbackMaxAttempts
message := "callback_network_error"
if deliveryErr == nil {
message = fmt.Sprintf("callback_http_%d", statusCode)
}
nextAttemptAt := time.Now().Add(taskCallbackRetryDelay(item.ID, item.Attempts))
if err := s.store.MarkTaskCallbackFailed(context.WithoutCancel(ctx), item, retry, nextAttemptAt, message); err != nil {
s.logger.Error("mark task callback failed", "taskID", item.TaskID, "seq", item.Seq, "error_category", "task_callback_state_failed")
continue
}
if !retry {
s.logger.Warn("task callback moved to dead letter", "taskID", item.TaskID, "seq", item.Seq, "statusCode", statusCode, "error_category", "task_callback_failed")
}
}
}
func deliverTaskCallback(ctx context.Context, client *http.Client, item store.TaskCallbackDelivery, bearerToken string) (int, error) {
body, err := json.Marshal(map[string]any{
"taskId": item.TaskID,
"seq": item.Seq,
"eventType": item.EventType,
"status": item.TaskStatus,
"createdAt": item.CreatedAt.UTC().Format(time.RFC3339Nano),
})
if err != nil {
return 0, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, item.CallbackURL, bytes.NewReader(body))
if err != nil {
return 0, err
}
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", fmt.Sprintf("%s:%d", item.TaskID, item.Seq))
request.Header.Set("X-EasyAI-Event-Type", item.EventType)
if bearerToken != "" {
request.Header.Set("Authorization", "Bearer "+bearerToken)
}
response, err := client.Do(request)
if err != nil {
return 0, err
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
return response.StatusCode, nil
}
func taskCallbackRetryDelay(deliveryID string, attempt int) time.Duration {
delays := []time.Duration{
5 * time.Second,
30 * time.Second,
2 * time.Minute,
10 * time.Minute,
30 * time.Minute,
}
index := attempt - 1
if index < 0 {
index = 0
}
if index >= len(delays) {
index = len(delays) - 1
}
base := delays[index]
hash := fnv.New32a()
_, _ = hash.Write([]byte(deliveryID))
jitter := time.Duration(hash.Sum32()%21) * base / 100
return base + jitter
}
func (s *Service) runTaskHistoryCleanupWorker(ctx context.Context) {
interval := time.Duration(s.cfg.TaskCleanupIntervalSeconds) * time.Second
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
s.processTaskHistoryCleanup(ctx)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (s *Service) processTaskHistoryCleanup(ctx context.Context) {
for batch := 0; batch < 10; batch++ {
now := time.Now().UTC()
result, err := s.store.CleanupTaskHistory(
ctx,
now.AddDate(0, 0, -s.cfg.TaskAnalysisRetentionDays),
now.AddDate(0, 0, -s.cfg.TaskRetentionDays),
s.cfg.TaskCleanupBatchSize,
)
if err != nil {
if ctx.Err() == nil {
s.logger.Error("cleanup task history failed", "error_category", "task_history_cleanup_failed")
}
return
}
if result.Total() == 0 {
return
}
s.logger.Info("task history cleanup batch completed",
"compactedTasks", result.CompactedTasks,
"compactedCheckpoints", result.CompactedCheckpoints,
"compactedAttempts", result.CompactedAttempts,
"compactedEvents", result.CompactedEvents,
"compactedCallbacks", result.CompactedCallbacks,
"retiredLegacyCallbacks", result.RetiredCallbacks,
"compactedParamLogs", result.CompactedParamLogs,
"compactedClonedVoices", result.CompactedClonedVoices,
"deletedCallbacks", result.DeletedCallbacks,
"deletedParamLogs", result.DeletedParamLogs,
"deletedEvents", result.DeletedEvents,
"deletedAttempts", result.DeletedAttempts,
"deletedTasks", result.DeletedTasks,
)
select {
case <-ctx.Done():
return
case <-time.After(100 * time.Millisecond):
}
}
}
@@ -0,0 +1,118 @@
package runner
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestMinimalRemoteTaskCheckpointUsesAllowlist(t *testing.T) {
checkpoint := minimalRemoteTaskCheckpoint("keling", "", map[string]any{
"endpoint": "/videos/task",
"mode": "omni_video",
"cleanupElementIds": []any{"element-1"},
"phase": "uploaded",
"submit": map[string]any{"large": "provider response"},
"payload": map[string]any{"prompt": "duplicate request"},
"raw": map[string]any{"data": "duplicate result"},
"status": "processing",
})
if checkpoint["endpoint"] != "/videos/task" || checkpoint["mode"] != "omni_video" || checkpoint["cleanupElementIds"] == nil {
t.Fatalf("checkpoint lost required fields: %#v", checkpoint)
}
for _, forbidden := range []string{"phase", "submit", "payload", "raw", "status"} {
if _, ok := checkpoint[forbidden]; ok {
t.Fatalf("checkpoint contains forbidden field %q: %#v", forbidden, checkpoint)
}
}
if unknown := minimalRemoteTaskCheckpoint("unknown-provider", "", map[string]any{"endpoint": "/forbidden"}); len(unknown) != 0 {
t.Fatalf("unknown provider checkpoint=%#v, want empty", unknown)
}
}
func TestCanonicalTaskResultDropsProviderRawCopy(t *testing.T) {
result := canonicalTaskResult(map[string]any{
"id": "task",
"data": []any{map[string]any{"url": "https://example.com/output"}},
"raw": map[string]any{"provider": "duplicate"},
"raw_data": map[string]any{"provider": "duplicate"},
"provider_response": map[string]any{"provider": "duplicate"},
"upstream_task_id": "remote-task",
"compatibility_data": "kept",
})
if result["raw"] != nil || result["raw_data"] != nil || result["provider_response"] != nil ||
result["upstream_task_id"] != nil || result["id"] != "task" || result["data"] == nil ||
result["compatibility_data"] != "kept" {
t.Fatalf("canonical result=%#v", result)
}
}
func TestMinimalParameterPreprocessingChangesDropsValuesAndCapsSize(t *testing.T) {
changes := make([]parameterPreprocessChange, 0, 100)
for index := 0; index < 100; index++ {
changes = append(changes, parameterPreprocessChange{
Path: "input.image",
Action: "converted",
Before: map[string]any{"base64": "forbidden"},
After: map[string]any{"url": "forbidden"},
})
}
minimal := minimalParameterPreprocessingChanges(changes)
encoded, err := json.Marshal(minimal)
if err != nil {
t.Fatal(err)
}
if len(encoded) > 1024 {
t.Fatalf("minimal changes size=%d, want <=1024", len(encoded))
}
for _, raw := range minimal {
item := raw.(map[string]any)
if len(item) != 2 || item["path"] != "input.image" || item["action"] != "converted" {
t.Fatalf("unexpected minimal change: %#v", item)
}
}
}
func TestDeliverTaskCallbackSendsMinimalAuthenticatedBody(t *testing.T) {
createdAt := time.Date(2026, time.July, 24, 10, 0, 0, 0, time.UTC)
var received map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if request.Header.Get("Authorization") != "Bearer internal-token" {
t.Errorf("Authorization=%q", request.Header.Get("Authorization"))
}
if request.Header.Get("Idempotency-Key") != "task-id:3" {
t.Errorf("Idempotency-Key=%q", request.Header.Get("Idempotency-Key"))
}
if request.Header.Get("X-EasyAI-Event-Type") != "task.completed" {
t.Errorf("X-EasyAI-Event-Type=%q", request.Header.Get("X-EasyAI-Event-Type"))
}
if err := json.NewDecoder(request.Body).Decode(&received); err != nil {
t.Error(err)
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
status, err := deliverTaskCallback(context.Background(), server.Client(), store.TaskCallbackDelivery{
TaskID: "task-id",
Seq: 3,
CallbackURL: server.URL,
EventType: "task.completed",
TaskStatus: "succeeded",
CreatedAt: createdAt,
}, "internal-token")
if err != nil || status != http.StatusNoContent {
t.Fatalf("status=%d err=%v", status, err)
}
if len(received) != 5 {
t.Fatalf("callback fields=%#v, want exactly five fields", received)
}
if received["taskId"] != "task-id" || received["eventType"] != "task.completed" || received["status"] != "succeeded" {
t.Fatalf("callback body=%#v", received)
}
}
+1 -10
View File
@@ -167,16 +167,7 @@ func (s *Service) persistVoiceCloneResult(ctx context.Context, task store.Gatewa
DemoAudioURL: demoAudioURL,
Status: "active",
ExpiresAt: &expiresAt,
Metadata: map[string]any{
"request": map[string]any{
"textValidation": body["text_validation"],
"languageBoost": body["language_boost"],
"needNoiseReduction": body["need_noise_reduction"],
"needVolumeNormalization": body["need_volume_normalization"],
"aigcWatermark": body["aigc_watermark"],
},
"rawData": result["raw_data"],
},
Metadata: map[string]any{},
})
}