perf(worker): 缩短媒体完成链路
生成媒体在启用直传 OSS 时绕过后端存储通道,异步任务持久化成功后不再在 Worker 内二次水化 Base64。\n\n将任务事件与回调 outbox 合并到同一事务,降低跨地域同步复制提交次数并消除事件已落库但回调未登记的崩溃窗口。文件通道健康遥测改为异步复制提交,避免非关键状态占用执行槽。\n\n验证:Go 全量测试、go vet、gofmt、git diff --check、相对 447e7ed701 的迁移安全检查均通过。
This commit is contained in:
@@ -126,3 +126,51 @@ func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
|
||||
t.Fatalf("general upload channel=%+v", channel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
calls.Add(1)
|
||||
if request.Method != http.MethodPut {
|
||||
t.Errorf("method=%s, want PUT", request.Method)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := config.Config{
|
||||
MediaOSSDirectEnabled: true,
|
||||
MediaOSSEndpoint: server.URL,
|
||||
MediaOSSBucket: "media-bucket",
|
||||
MediaOSSAccessKeyID: "access-key",
|
||||
MediaOSSAccessKeySecret: "access-secret",
|
||||
MediaOSSPublicBaseURL: "https://cdn.example.com",
|
||||
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
|
||||
}
|
||||
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
|
||||
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(
|
||||
context.Background(),
|
||||
"task-direct-oss",
|
||||
&generatedInlineAsset{
|
||||
Bytes: []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a},
|
||||
ContentType: "image/png",
|
||||
Kind: "image",
|
||||
SourceKey: "b64_json",
|
||||
},
|
||||
0,
|
||||
[]store.FileStorageChannel{{ID: "unused-channel"}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("direct generated upload: %v", err)
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("direct OSS calls=%d, want 1", calls.Load())
|
||||
}
|
||||
if contentType != "image/png" || kind != "image" || strategy != "direct_aliyun_oss" {
|
||||
t.Fatalf("generated metadata=%s/%s/%s", contentType, kind, strategy)
|
||||
}
|
||||
channel, _ := upload["storageChannel"].(map[string]any)
|
||||
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
|
||||
t.Fatalf("storage channel=%+v", channel)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -952,6 +952,14 @@ candidatesLoop:
|
||||
}, isSimulation(task, candidate)); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if task.AsyncMode {
|
||||
// Async callers hydrate the canonical stored result through
|
||||
// the task detail endpoint. The River Worker does not consume
|
||||
// Result.Output, so downloading and rebuilding Base64 here
|
||||
// only retains the execution slot and media memory a second
|
||||
// time after the task is already durably complete.
|
||||
return Result{Task: finished, Output: response.Result}, nil
|
||||
}
|
||||
output, hydrateErr := s.HydrateTaskResult(ctx, task.ID, response.Result)
|
||||
if hydrateErr != nil {
|
||||
return Result{Task: finished}, hydrateErr
|
||||
@@ -2105,23 +2113,34 @@ func (s *Service) withAttemptHistory(ctx context.Context, taskID string, metrics
|
||||
}
|
||||
|
||||
func (s *Service) emit(ctx context.Context, taskID string, eventType string, status string, phase string, progress float64, message string, payload map[string]any, simulated bool) error {
|
||||
event, err := s.store.AddTaskEvent(ctx, taskID, eventType, status, phase, progress, message, payload, simulated)
|
||||
acceptanceURL, acceptance, callbackErr := s.store.AcceptanceCallbackURL(ctx, taskID)
|
||||
if callbackErr != nil {
|
||||
return callbackErr
|
||||
}
|
||||
callbackURL := ""
|
||||
if acceptance && acceptanceURL != "" {
|
||||
callbackURL = acceptanceURL
|
||||
} else if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" {
|
||||
callbackURL = s.cfg.TaskProgressCallbackURL
|
||||
}
|
||||
event, err := s.store.AddTaskEventWithCallback(
|
||||
ctx,
|
||||
taskID,
|
||||
eventType,
|
||||
status,
|
||||
phase,
|
||||
progress,
|
||||
message,
|
||||
payload,
|
||||
simulated,
|
||||
callbackURL,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if event.ID == "" {
|
||||
s.observeTaskEventSkip(event.SkippedReason)
|
||||
}
|
||||
acceptanceURL, acceptance, callbackErr := s.store.AcceptanceCallbackURL(ctx, taskID)
|
||||
if callbackErr != nil {
|
||||
return callbackErr
|
||||
}
|
||||
if acceptance && acceptanceURL != "" {
|
||||
return s.store.QueueTaskCallback(ctx, event, acceptanceURL)
|
||||
}
|
||||
if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" {
|
||||
return s.store.QueueTaskCallback(ctx, event, s.cfg.TaskProgressCallbackURL)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -167,9 +167,11 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
||||
var channels []store.FileStorageChannel
|
||||
channelsLoaded := false
|
||||
if needsUpload || rawNeedsUpload {
|
||||
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
|
||||
if err != nil {
|
||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
||||
if s.directOSS == nil {
|
||||
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
|
||||
if err != nil {
|
||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
||||
}
|
||||
}
|
||||
channelsLoaded = true
|
||||
}
|
||||
@@ -661,6 +663,10 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset
|
||||
Scene: store.FileStorageSceneImageResult,
|
||||
Source: "ai-gateway",
|
||||
}
|
||||
if s.directOSS != nil {
|
||||
upload, err := s.directOSS.upload(ctx, payload)
|
||||
return upload, contentType, kind, "direct_aliyun_oss", err
|
||||
}
|
||||
if len(channels) == 0 {
|
||||
upload, err := s.storeFileLocally(payload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
|
||||
return upload, contentType, kind, "local_static_inline_media", err
|
||||
@@ -683,6 +689,10 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as
|
||||
Scene: store.FileStorageSceneImageResult,
|
||||
Source: "ai-gateway",
|
||||
}
|
||||
if s.directOSS != nil {
|
||||
upload, err := s.directOSS.upload(ctx, uploadPayload)
|
||||
return upload, contentType, kind, "direct_aliyun_oss", err
|
||||
}
|
||||
if len(channels) == 0 {
|
||||
upload, err := s.storeFileLocally(uploadPayload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
|
||||
return upload, contentType, kind, "local_static_url_media", err
|
||||
|
||||
Reference in New Issue
Block a user