feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换,保留 server-main 兼容与环境 OSS 内存通道。

将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化,生产路径不再写入本机静态目录,历史本地资源仅保留只读兼容。

引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出,同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。

验证:go test ./... -count=1;go vet ./...;pnpm lint;pnpm test;pnpm build;pnpm openapi;tests/ci/migrations-test.sh。
This commit is contained in:
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
+78 -138
View File
@@ -12,7 +12,6 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
@@ -26,7 +25,6 @@ const (
localBinaryGenericBase64MinLength = 4096
localBinaryMaxDepth = 64
defaultLocalResultTTLHours = 24
defaultLocalResultMinFreeBytes = int64(10 * 1024 * 1024 * 1024)
defaultLocalResultMaxBytes = int64(256 * 1024 * 1024)
defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024)
)
@@ -41,41 +39,38 @@ type localBinaryDescriptor struct {
type localBinaryMaterializer struct {
service *Service
taskDir string
writeFiles bool
enforceLimits bool
totalBytes int64
seen map[string]struct{}
createdFiles []string
}
// materializeLocalBinaryResult replaces every inline binary value with a
// bounded placeholder after atomically writing and verifying the bytes locally.
// materializeLocalBinaryResult is retained as an internal compatibility seam,
// but new materialization always targets object storage. Historical local
// placeholders remain readable through HydrateTaskResult.
func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
if _, _, err := s.transformLocalBinaryResult(ctx, taskID, result, false, true); err != nil {
hadInline := TaskResultHasInlineBinary(result)
if !hadInline {
return result, false, nil
}
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
if err != nil {
return nil, false, err
}
return s.transformLocalBinaryResult(ctx, taskID, result, true, true)
return next, !TaskResultHasInlineBinary(next), nil
}
func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, result map[string]any, writeFiles bool, enforceLimits bool) (map[string]any, bool, error) {
root := s.localBinaryResultRoot()
taskDir := filepath.Join(root, safeLocalBinaryTaskDir(taskID))
func (s *Service) transformLocalBinaryResult(ctx context.Context, _ string, result map[string]any, _ bool, enforceLimits bool) (map[string]any, bool, error) {
materializer := &localBinaryMaterializer{
service: s,
taskDir: taskDir,
writeFiles: writeFiles,
enforceLimits: enforceLimits,
seen: map[string]struct{}{},
}
next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
if err != nil {
materializer.rollbackCreatedFiles()
return nil, false, err
}
mapped, ok := next.(map[string]any)
if !ok {
materializer.rollbackCreatedFiles()
return nil, false, &clients.ClientError{
Code: "result_binary_not_materialized",
Message: "generated result is not a JSON object",
@@ -83,28 +78,28 @@ func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string,
Retryable: false,
}
}
if changed && enforceLimits && !writeFiles {
if err := os.MkdirAll(root, 0o750); err != nil {
return nil, false, localBinaryStorageError(err)
}
if err := ensureLocalBinaryDiskHeadroom(root, materializer.totalBytes, s.localResultMinFreeBytes()); err != nil {
return nil, false, err
}
}
return mapped, changed, nil
}
// MaterializeTaskResultForStorage exposes the same verified materialization
// path to the explicit historical maintenance command.
func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
return s.materializeLocalBinaryResult(ctx, taskID, result)
hadInline := TaskResultHasInlineBinary(result)
if !hadInline {
return result, false, nil
}
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
if err != nil {
return nil, false, err
}
return next, hadInline && !TaskResultHasInlineBinary(next), nil
}
// CompactExpiredTaskResultForStorage computes the same deterministic
// placeholders without creating files that are already outside the recovery
// window.
// CompactExpiredTaskResultForStorage keeps the historical maintenance API but
// now uses the same object-storage path as live results. New GatewayBinary
// placeholders are never created; their parser remains read-only compatibility.
func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
return s.transformLocalBinaryResult(ctx, taskID, result, false, false)
return s.MaterializeTaskResultForStorage(ctx, taskID, result)
}
func TaskResultHasInlineBinary(result map[string]any) bool {
@@ -239,15 +234,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
Retryable: false,
}
}
if m.writeFiles {
created, err := m.service.writeLocalBinaryResult(m.taskDir, digestHex, payload)
if err != nil {
return nil, false, err
}
if created {
m.createdFiles = append(m.createdFiles, filepath.Join(m.taskDir, digestHex+".bin"))
}
}
m.seen[digestHex] = struct{}{}
m.totalBytes += size
}
@@ -261,94 +247,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
return localBinaryPlaceholder(descriptor), true, nil
}
func (m *localBinaryMaterializer) rollbackCreatedFiles() {
for _, path := range m.createdFiles {
_ = os.Remove(path)
}
_ = os.Remove(m.taskDir)
m.createdFiles = nil
}
func (s *Service) writeLocalBinaryResult(taskDir string, digestHex string, payload []byte) (bool, error) {
if err := os.MkdirAll(taskDir, 0o750); err != nil {
return false, localBinaryStorageError(err)
}
targetPath := filepath.Join(taskDir, digestHex+".bin")
if info, err := os.Stat(targetPath); err == nil {
if !info.IsDir() && info.Size() == int64(len(payload)) {
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err == nil {
now := time.Now()
if err := os.Chtimes(targetPath, now, now); err != nil {
return false, localBinaryStorageError(err)
}
return false, nil
}
}
return false, &clients.ClientError{
Code: "binary_result_corrupted",
Message: "existing local result file does not match its content hash",
StatusCode: 500,
Retryable: false,
}
} else if !errors.Is(err, os.ErrNotExist) {
return false, localBinaryStorageError(err)
}
if err := ensureLocalBinaryDiskHeadroom(taskDir, int64(len(payload)), s.localResultMinFreeBytes()); err != nil {
return false, err
}
tempFile, err := os.CreateTemp(taskDir, ".gateway-result-*")
if err != nil {
return false, localBinaryStorageError(err)
}
tempPath := tempFile.Name()
cleanup := func() {
_ = tempFile.Close()
_ = os.Remove(tempPath)
}
if err := tempFile.Chmod(0o640); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if _, err := tempFile.Write(payload); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if err := tempFile.Sync(); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if err := tempFile.Close(); err != nil {
_ = os.Remove(tempPath)
return false, localBinaryStorageError(err)
}
if err := os.Rename(tempPath, targetPath); err != nil {
_ = os.Remove(tempPath)
return false, localBinaryStorageError(err)
}
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err != nil {
_ = os.Remove(targetPath)
return false, err
}
return true, nil
}
func ensureLocalBinaryDiskHeadroom(path string, incomingBytes int64, minFreeBytes int64) error {
var stat syscall.Statfs_t
if err := syscall.Statfs(path, &stat); err != nil {
return localBinaryStorageError(err)
}
freeBytes := int64(stat.Bavail) * int64(stat.Bsize)
if freeBytes-incomingBytes < minFreeBytes {
return &clients.ClientError{
Code: "local_result_storage_unavailable",
Message: "local result storage does not have enough free space",
StatusCode: 503,
Retryable: false,
}
}
return nil
}
func localBinaryStorageError(err error) error {
return &clients.ClientError{
Code: "local_result_storage_unavailable",
@@ -406,6 +304,30 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
}
switch typed := value.(type) {
case map[string]any:
refreshedAccessURL := false
if asset, upload, ok := generatedResultUploadReference(typed); ok {
accessURL, err := s.requestAssetAccessURL(ctx, asset)
if err != nil {
return nil, false, err
}
next := make(map[string]any, len(typed))
for key, item := range typed {
next[key] = item
}
nextUpload := make(map[string]any, len(upload))
for key, item := range upload {
nextUpload[key] = item
}
nextUpload["url"] = accessURL
next["upload"] = nextUpload
for _, key := range []string{"url", "image_url", "video_url", "audio_url"} {
if _, exists := next[key]; exists {
next[key] = accessURL
}
}
typed = next
refreshedAccessURL = true
}
if ref, ok := generatedResultAssetReference(typed); ok {
payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
if err != nil {
@@ -418,7 +340,7 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
return encoded, true, nil
}
next := make(map[string]any, len(typed))
changed := false
changed := refreshedAccessURL
for key, childValue := range typed {
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
if err != nil {
@@ -465,6 +387,27 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
}
}
func generatedResultUploadReference(value map[string]any) (store.RequestAsset, map[string]any, bool) {
upload, ok := value["upload"].(map[string]any)
if !ok {
return store.RequestAsset{}, nil, false
}
objectKey := strings.TrimSpace(stringFromAny(upload["objectKey"]))
channel, _ := upload["storageChannel"].(map[string]any)
channelKey := strings.TrimSpace(stringFromAny(channel["channelKey"]))
if objectKey == "" || channelKey == "" {
return store.RequestAsset{}, nil, false
}
return store.RequestAsset{
URL: stringFromAny(upload["url"]),
StorageProvider: stringFromAny(channel["provider"]),
StorageChannelID: stringFromAny(channel["id"]),
StorageChannelKey: channelKey,
ObjectKey: objectKey,
AccessScope: stringFromAny(upload["accessScope"]),
}, upload, true
}
func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) {
ref, ok := value["assetRef"].(map[string]any)
if !ok {
@@ -475,10 +418,14 @@ func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bo
return store.RequestAsset{}, false
}
asset := store.RequestAsset{
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
StorageProvider: stringFromAny(ref["storageProvider"]),
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
StorageProvider: stringFromAny(ref["storageProvider"]),
StorageChannelID: stringFromAny(ref["storageChannelId"]),
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
ObjectKey: stringFromAny(ref["objectKey"]),
AccessScope: stringFromAny(ref["accessScope"]),
}
if size := floatFromAny(ref["size"]); size > 0 {
asset.ByteSize = int64(size)
@@ -767,13 +714,6 @@ func (s *Service) localResultTTLHours() int {
return s.cfg.LocalResultTTLHours
}
func (s *Service) localResultMinFreeBytes() int64 {
if s.cfg.LocalResultMinFreeBytes <= 0 {
return defaultLocalResultMinFreeBytes
}
return s.cfg.LocalResultMinFreeBytes
}
func (s *Service) localResultMaxBytes() int64 {
if s.cfg.LocalResultMaxBytes <= 0 {
return defaultLocalResultMaxBytes
+83 -31
View File
@@ -31,7 +31,21 @@ func newLocalBinaryTestService(t *testing.T) *Service {
}}
}
func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
func writeHistoricalLocalBinaryFixture(t *testing.T, service *Service, taskID string, payload []byte) string {
t.Helper()
digest := sha256.Sum256(payload)
taskDir := filepath.Join(service.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID))
if err := os.MkdirAll(taskDir, 0o750); err != nil {
t.Fatalf("create historical result fixture directory: %v", err)
}
path := filepath.Join(taskDir, hex.EncodeToString(digest[:])+".bin")
if err := os.WriteFile(path, payload, 0o640); err != nil {
t.Fatalf("write historical result fixture: %v", err)
}
return path
}
func TestHydrateHistoricalLocalBinaryResult(t *testing.T) {
service := newLocalBinaryTestService(t)
payload := []byte("one binary result shared across representations")
encoded := base64.StdEncoding.EncodeToString(payload)
@@ -49,7 +63,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
},
}
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input)
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-123", input, false, true)
if err != nil {
t.Fatalf("materialize local binary result: %v", err)
}
@@ -76,6 +90,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
if len(persistentJSON) > 32*1024 {
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
}
writeHistoricalLocalBinaryFixture(t, service, "task-123", payload)
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
entries, err := os.ReadDir(taskDir)
@@ -155,19 +170,53 @@ func TestHydrateGeneratedResultAssetReference(t *testing.T) {
assertClientErrorCode(t, err, "binary_result_corrupted")
}
func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) {
cfg := config.Config{
MediaOSSDirectEnabled: true,
MediaOSSEndpoint: "https://oss.example.com",
MediaOSSBucket: "media-bucket",
MediaOSSAccessKeyID: "access-id",
MediaOSSAccessKeySecret: "access-secret",
MediaOSSObjectPrefix: "media",
}
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
result := map[string]any{
"data": []any{map[string]any{
"url": "https://expired.example.com/result.png",
"image_url": "https://expired.example.com/result.png",
"upload": map[string]any{
"url": "https://expired.example.com/result.png",
"objectKey": "media/image_result/2026/08/04/hash.png",
"accessScope": "private",
"storageChannel": map[string]any{
"channelKey": "environment-direct-oss",
"provider": "aliyun_oss",
},
},
}},
}
hydrated, err := service.HydrateTaskResult(t.Context(), "task-private-url", result)
if err != nil {
t.Fatal(err)
}
item := hydrated["data"].([]any)[0].(map[string]any)
for _, key := range []string{"url", "image_url"} {
value := stringFromAny(item[key])
if !strings.Contains(value, "OSSAccessKeyId=access-id") || strings.Contains(value, "expired.example.com") {
t.Fatalf("%s was not refreshed: %q", key, value)
}
}
}
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultTTLHours = 1
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded})
persistent, _, err := service.transformLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}, false, true)
if err != nil {
t.Fatalf("materialize fixture: %v", err)
}
entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired"))
if err != nil || len(entries) != 1 {
t.Fatalf("read fixture result: entries=%v err=%v", entries, err)
}
path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name())
path := writeHistoricalLocalBinaryFixture(t, service, "task-expired", []byte("expiring result"))
old := time.Now().Add(-2 * time.Hour)
if err := os.Chtimes(path, old, old); err != nil {
t.Fatalf("age fixture: %v", err)
@@ -186,17 +235,17 @@ func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T)
assertClientErrorCode(t, err, "binary_result_corrupted")
}
func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
func TestHistoricalLocalBinaryFixtureEnforcesLimitsAndKeepsText(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultMaxBytes = 4
_, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{
_, _, err := service.transformLocalBinaryResult(context.Background(), "task-large", map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
})
}, false, true)
assertClientErrorCode(t, err, "binary_result_too_large")
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-text", map[string]any{
"message": "YWJjZA==",
})
}, false, true)
if err != nil || changed || persistent["message"] != "YWJjZA==" {
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
}
@@ -204,18 +253,30 @@ func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
service = newLocalBinaryTestService(t)
service.cfg.LocalResultMaxBytes = 1024
service.cfg.LocalResultMaxTaskBytes = 8
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
_, _, err = service.transformLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
})
}, false, true)
assertClientErrorCode(t, err, "binary_result_too_large")
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
}
}
func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
func TestCompactExpiredTaskResultUsesObjectStorageAndDoesNotWriteFiles(t *testing.T) {
var uploads int
storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
t.Fatalf("method=%s, want PUT", r.Method)
}
uploads++
w.WriteHeader(http.StatusOK)
}))
defer storageServer.Close()
service := newLocalBinaryTestService(t)
service.directOSS = &directOSSUploader{
endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
}
service.cfg.LocalResultMaxBytes = 1
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
@@ -225,17 +286,18 @@ func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
if err != nil {
t.Fatalf("compact expired result: %v", err)
}
if !changed || !localBinaryResultHasPlaceholders(persistent) {
t.Fatalf("expired result was not compacted: %+v", persistent)
if !changed || localBinaryResultHasPlaceholders(persistent) || TaskResultHasInlineBinary(persistent) {
t.Fatalf("expired result was not objectified: %+v", persistent)
}
if uploads != 1 {
t.Fatalf("object storage uploads=%d, want 1", uploads)
}
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("expired compaction must not create a task directory: %v", err)
}
_, err = service.HydrateTaskResult(context.Background(), "task-old", persistent)
assertClientErrorCode(t, err, "binary_result_expired")
}
func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
err := localBinaryStorageError(errors.New("write failed"))
assertClientErrorCode(t, err, "local_result_storage_unavailable")
if clients.IsRetryable(err) {
@@ -247,16 +309,6 @@ func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
t.Fatal("local result storage failure must not fail over to another provider")
}
service := newLocalBinaryTestService(t)
service.cfg.LocalResultMinFreeBytes = 1 << 62
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")),
})
assertClientErrorCode(t, err, "local_result_storage_unavailable")
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("disk preflight must not create partial task files: %v", statErr)
}
}
func bytesToAny(payload []byte) []any {
+23 -131
View File
@@ -1,26 +1,12 @@
package runner
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const directOSSUploadTimeout = 120 * time.Second
type directOSSUploader struct {
endpoint string
bucket string
@@ -28,8 +14,6 @@ type directOSSUploader struct {
accessKeySecret string
publicBaseURL string
objectPrefix string
client *http.Client
now func() time.Time
}
func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
@@ -43,133 +27,41 @@ func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret),
publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"),
objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"),
client: &http.Client{
Timeout: directOSSUploadTimeout,
Transport: &http.Transport{
MaxIdleConns: 256,
MaxIdleConnsPerHost: 256,
MaxConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
},
},
now: time.Now,
}
}
func directOSSScene(scene string) bool {
switch strings.TrimSpace(scene) {
case store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
case store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
return true
default:
return false
}
}
func (u *directOSSUploader) upload(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
func (u *directOSSUploader) fileStorageChannel() store.FileStorageChannel {
if u == nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: "direct OSS uploader is not configured", Retryable: false}
return store.FileStorageChannel{}
}
objectKey := u.objectKey(payload)
escapedKey := escapeOSSObjectKey(objectKey)
uploadURL := u.endpoint + "/" + escapedKey
contentType := strings.TrimSpace(payload.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
return store.FileStorageChannel{
ChannelKey: "environment-direct-oss",
Name: "Environment Direct OSS",
Provider: "aliyun_oss",
AccessKeyID: u.accessKeyID,
AccessKeySecret: u.accessKeySecret,
Priority: 10,
Status: "enabled",
Scenes: []string{store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult},
Config: map[string]any{
"endpoint": u.endpoint,
"bucket": u.bucket,
"publicBaseUrl": u.publicBaseURL,
"objectPrefix": u.objectPrefix,
},
RetryPolicy: map[string]any{
"enabled": true,
"maxRetries": 2,
"backoffSeconds": []any{0.25, 1.0},
},
}
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
if err := sleepWithContext(ctx, time.Duration(attempt*attempt)*200*time.Millisecond); err != nil {
return nil, err
}
}
status, err := u.put(ctx, uploadURL, escapedKey, contentType, payload.Bytes)
if err == nil && status >= 200 && status < 300 {
publicURL := u.publicBaseURL + "/" + escapedKey
return map[string]any{
"url": publicURL,
"fileName": objectKey,
"storageChannel": map[string]any{
"channelKey": "environment-direct-oss",
"name": "Environment Direct OSS",
"provider": "aliyun_oss_direct",
},
"assetStorage": map[string]any{
"scene": payload.Scene,
"source": firstNonEmptyString(payload.Source, "ai-gateway"),
"strategy": "direct_aliyun_oss",
},
}, nil
}
if err != nil {
lastErr = err
continue
}
lastErr = fmt.Errorf("HTTP %d", status)
if status != http.StatusTooManyRequests && status < 500 {
break
}
}
message := "direct OSS upload failed"
if lastErr != nil {
message += ": " + lastErr.Error()
}
return nil, &clients.ClientError{Code: "upload_failed", Message: message, Retryable: true}
}
func (u *directOSSUploader) put(ctx context.Context, uploadURL string, escapedKey string, contentType string, payload []byte) (int, error) {
date := u.now().UTC().Format(http.TimeFormat)
canonicalResource := "/" + u.bucket + "/" + escapedKey
stringToSign := "PUT\n\n" + contentType + "\n" + date + "\n" + canonicalResource
mac := hmac.New(sha1.New, []byte(u.accessKeySecret))
_, _ = mac.Write([]byte(stringToSign))
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(payload))
if err != nil {
return 0, err
}
req.Header.Set("Authorization", "OSS "+u.accessKeyID+":"+signature)
req.Header.Set("Content-Type", contentType)
req.Header.Set("Date", date)
resp, err := u.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
return resp.StatusCode, nil
}
func (u *directOSSUploader) objectKey(payload FileUploadPayload) string {
now := u.now().UTC()
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
baseName := strings.TrimSuffix(path.Base(payload.FileName), path.Ext(payload.FileName))
baseName = sanitizeGeneratedAssetNamePart(baseName)
if baseName == "" {
baseName = "gateway-media"
}
if len(baseName) > 48 {
baseName = baseName[:48]
}
return fmt.Sprintf(
"%s/%s/%04d/%02d/%02d/%s-%s%s",
u.objectPrefix,
strings.TrimSpace(payload.Scene),
now.Year(),
now.Month(),
now.Day(),
baseName,
randomHexSuffix(8),
extension,
)
}
func escapeOSSObjectKey(objectKey string) string {
parts := strings.Split(objectKey, "/")
for index, part := range parts {
parts[index] = url.PathEscape(part)
}
return strings.Join(parts, "/")
}
+15 -18
View File
@@ -12,13 +12,12 @@ import (
"strings"
"sync/atomic"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
func TestDirectOSSUploadSignsRequestAndReturnsPrivateSignedURL(t *testing.T) {
payload := []byte("production-isomorphic-image")
var calls atomic.Int64
var uploadedPath string
@@ -60,9 +59,6 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
}
uploader := newDirectOSSUploader(cfg)
uploader.now = func() time.Time {
return time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC)
}
service := &Service{cfg: cfg, directOSS: uploader}
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
Bytes: payload,
@@ -77,29 +73,30 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
if calls.Load() != 2 {
t.Fatalf("upload calls=%d, want one retry", calls.Load())
}
if !strings.HasPrefix(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/2026/07/30/input-") ||
!strings.HasSuffix(uploadedPath, ".png") {
if !strings.Contains(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/") ||
!strings.HasSuffix(uploadedPath, "/"+sha256Hex(payload)+".png") {
t.Fatalf("unexpected object path=%q", uploadedPath)
}
if got := stringFromAny(uploaded["url"]); got != "https://cdn.example.com"+uploadedPath {
t.Fatalf("public URL=%q", got)
if got := stringFromAny(uploaded["url"]); !strings.HasPrefix(got, server.URL+uploadedPath+"?") || !strings.Contains(got, "Signature=") {
t.Fatalf("private signed URL=%q", got)
}
if uploaded["accessScope"] != "private" {
t.Fatalf("request asset access scope=%v", uploaded["accessScope"])
}
channel, _ := uploaded["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
if stringFromAny(channel["provider"]) != "aliyun_oss" {
t.Fatalf("storage channel=%+v", channel)
}
}
func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
func TestDirectOSSPersistsGeneralUploadScene(t *testing.T) {
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
storageDir := t.TempDir()
cfg := config.Config{
LocalUploadedStorageDir: storageDir,
MediaOSSDirectEnabled: true,
MediaOSSEndpoint: server.URL,
MediaOSSBucket: "media-bucket",
@@ -118,11 +115,11 @@ func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
if err != nil {
t.Fatalf("general upload: %v", err)
}
if calls.Load() != 0 {
if calls.Load() != 1 {
t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load())
}
channel, _ := uploaded["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "local_static" {
if stringFromAny(channel["provider"]) != "aliyun_oss" {
t.Fatalf("general upload channel=%+v", channel)
}
}
@@ -158,7 +155,7 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
SourceKey: "b64_json",
},
0,
[]store.FileStorageChannel{{ID: "unused-channel"}},
[]store.FileStorageChannel{service.directOSS.fileStorageChannel()},
)
if err != nil {
t.Fatalf("direct generated upload: %v", err)
@@ -166,11 +163,11 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
if calls.Load() != 1 {
t.Fatalf("direct OSS calls=%d, want 1", calls.Load())
}
if contentType != "image/png" || kind != "image" || strategy != "direct_aliyun_oss" {
if contentType != "image/png" || kind != "image" || strategy != "upload_inline_media" {
t.Fatalf("generated metadata=%s/%s/%s", contentType, kind, strategy)
}
channel, _ := upload["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
if stringFromAny(channel["provider"]) != "aliyun_oss" {
t.Fatalf("storage channel=%+v", channel)
}
}
+530
View File
@@ -0,0 +1,530 @@
package runner
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
const (
objectStorageRequestTimeout = 120 * time.Second
objectStorageReadLimit = 256 << 20
objectStorageSignedURLTTL = 15 * time.Minute
)
var sharedObjectStorageHTTPClient = &http.Client{
Timeout: objectStorageRequestTimeout,
Transport: &http.Transport{
MaxIdleConns: 256,
MaxIdleConnsPerHost: 256,
MaxConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
},
}
type objectStorageAdapter struct {
channel store.FileStorageChannel
client *http.Client
now func() time.Time
}
type FileStorageChannelTestResult struct {
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
DurationMS int64 `json:"durationMs"`
}
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
// against one object-storage channel. The random probe avoids overwriting a
// content-addressed business object and no object key or credential is exposed.
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
startedAt := time.Now()
result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return result, err
}
payload := FileUploadPayload{
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
ContentType: "application/octet-stream",
FileName: "probe.bin",
Scene: store.FileStorageSceneUpload,
Source: "admin-connection-test",
}
upload, err := adapter.put(ctx, payload)
if err != nil {
return result, err
}
result.PutSucceeded = true
objectKey := stringFromAny(upload["objectKey"])
if objectKey == "" {
return result, storageClientError("storage_write_failed", "object storage probe did not return an object reference", 0, false)
}
if err := adapter.head(ctx, objectKey); err != nil {
_ = adapter.delete(context.WithoutCancel(ctx), objectKey)
return result, err
}
result.HeadSucceeded = true
if err := adapter.delete(ctx, objectKey); err != nil {
return result, err
}
result.DeleteSucceeded = true
result.DurationMS = time.Since(startedAt).Milliseconds()
return result, nil
}
func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAdapter, error) {
provider := strings.ToLower(strings.TrimSpace(channel.Provider))
if provider != "aliyun_oss" && provider != "s3" {
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported object storage provider", Retryable: false}
}
if objectStorageConfigString(channel.Config, "endpoint") == "" || objectStorageConfigString(channel.Config, "bucket") == "" {
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "object storage endpoint and bucket are required", Retryable: false}
}
if strings.TrimSpace(channel.AccessKeyID) == "" || strings.TrimSpace(channel.AccessKeySecret) == "" {
return nil, &clients.ClientError{Code: "storage_auth_failed", Message: "object storage credentials are not configured", Retryable: false}
}
return &objectStorageAdapter{
channel: channel,
client: sharedObjectStorageHTTPClient,
now: time.Now,
}, nil
}
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
objectKey := a.objectKey(payload)
requestURL, err := a.objectURL(objectKey)
if err != nil {
return nil, err
}
contentType := strings.TrimSpace(payload.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload.Bytes))
if err != nil {
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
}
req.Header.Set("Content-Type", contentType)
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
return nil, err
}
resp, err := a.client.Do(req)
if err != nil {
return nil, storageClientError("storage_write_failed", err.Error(), 0, true)
}
defer resp.Body.Close()
responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if readErr != nil {
return nil, storageClientError("storage_write_failed", readErr.Error(), resp.StatusCode, storageStatusRetryable(resp.StatusCode))
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, objectStorageHTTPError("storage_write_failed", resp.StatusCode, responseBody)
}
accessScope := objectStorageAccessScope(a.channel, payload.Scene)
publicURL := ""
if accessScope == "public" {
publicURL = a.publicURL(objectKey)
}
urlExpiresAt := ""
if publicURL == "" {
publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL)
if err != nil {
return nil, err
}
urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339)
}
digest := sha256.Sum256(payload.Bytes)
result := map[string]any{
"url": publicURL,
"fileName": path.Base(objectKey),
"objectKey": objectKey,
"contentType": contentType,
"size": len(payload.Bytes),
"sha256": hex.EncodeToString(digest[:]),
"accessScope": accessScope,
"storageChannel": map[string]any{
"id": a.channel.ID,
"channelKey": a.channel.ChannelKey,
"name": a.channel.Name,
"provider": a.channel.Provider,
},
}
if urlExpiresAt != "" {
result["urlExpiresAt"] = urlExpiresAt
}
return result, nil
}
func (a *objectStorageAdapter) get(ctx context.Context, objectKey string) ([]byte, error) {
requestURL, err := a.objectURL(objectKey)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
if err != nil {
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
}
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
return nil, err
}
resp, err := a.client.Do(req)
if err != nil {
return nil, storageClientError("storage_read_failed", err.Error(), 0, true)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
return nil, objectStorageHTTPError("storage_read_failed", resp.StatusCode, body)
}
payload, err := io.ReadAll(io.LimitReader(resp.Body, objectStorageReadLimit+1))
if err != nil {
return nil, storageClientError("storage_read_failed", err.Error(), resp.StatusCode, true)
}
if len(payload) > objectStorageReadLimit {
return nil, storageClientError("storage_read_failed", "stored object exceeds the read limit", resp.StatusCode, false)
}
return payload, nil
}
func (a *objectStorageAdapter) head(ctx context.Context, objectKey string) error {
return a.emptyObjectRequest(ctx, http.MethodHead, objectKey)
}
func (a *objectStorageAdapter) delete(ctx context.Context, objectKey string) error {
return a.emptyObjectRequest(ctx, http.MethodDelete, objectKey)
}
func (a *objectStorageAdapter) emptyObjectRequest(ctx context.Context, method string, objectKey string) error {
requestURL, err := a.objectURL(objectKey)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, method, requestURL, nil)
if err != nil {
return storageClientError("storage_config_invalid", err.Error(), 0, false)
}
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
return err
}
resp, err := a.client.Do(req)
if err != nil {
return storageClientError("storage_read_failed", err.Error(), 0, true)
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return objectStorageHTTPError("storage_read_failed", resp.StatusCode, nil)
}
return nil
}
func (a *objectStorageAdapter) presignGet(objectKey string, ttl time.Duration) (string, error) {
requestURL, err := a.objectURL(objectKey)
if err != nil {
return "", err
}
if ttl <= 0 {
ttl = objectStorageSignedURLTTL
}
if strings.EqualFold(a.channel.Provider, "s3") {
return a.presignS3Get(requestURL, ttl, a.now().UTC())
}
return a.presignOSSGet(requestURL, objectKey, ttl, a.now().UTC())
}
func (a *objectStorageAdapter) objectKey(payload FileUploadPayload) string {
now := a.now().UTC()
digest := sha256.Sum256(payload.Bytes)
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
prefix := strings.Trim(objectStorageConfigString(a.channel.Config, "objectPrefix"), "/")
parts := make([]string, 0, 7)
if prefix != "" {
parts = append(parts, prefix)
}
parts = append(parts,
firstNonEmptyString(strings.TrimSpace(payload.Scene), store.FileStorageSceneUpload),
fmt.Sprintf("%04d", now.Year()),
fmt.Sprintf("%02d", now.Month()),
fmt.Sprintf("%02d", now.Day()),
hex.EncodeToString(digest[:])+extension,
)
return strings.Join(parts, "/")
}
func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) {
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
}
bucket := objectStorageConfigString(a.channel.Config, "bucket")
if strings.EqualFold(a.channel.Provider, "s3") {
if objectStorageConfigBool(a.channel.Config, "forcePathStyle") {
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + bucket + "/" + strings.TrimLeft(objectKey, "/")
} else {
endpoint.Host = bucket + "." + endpoint.Host
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
}
} else {
hostname := endpoint.Hostname()
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
if port := endpoint.Port(); port != "" {
endpoint.Host = bucket + "." + hostname + ":" + port
} else {
endpoint.Host = bucket + "." + endpoint.Host
}
}
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
}
return endpoint.String(), nil
}
func (a *objectStorageAdapter) publicURL(objectKey string) string {
baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/")
if baseURL == "" {
baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/")
}
if baseURL == "" {
return ""
}
return baseURL + "/" + escapeObjectKey(objectKey)
}
func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now time.Time) error {
if strings.EqualFold(a.channel.Provider, "s3") {
a.signS3Request(req, payloadHash, now)
return nil
}
a.signOSSRequest(req, now)
return nil
}
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
date := now.UTC().Format(http.TimeFormat)
contentType := req.Header.Get("Content-Type")
canonicalHeaders := ""
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
req.Header.Set("x-oss-security-token", token)
canonicalHeaders = "x-oss-security-token:" + token + "\n"
}
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
req.Header.Set("Date", date)
}
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
parsed, err := url.Parse(requestURL)
if err != nil {
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
}
expires := strconv.FormatInt(now.Add(ttl).Unix(), 10)
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + "/" + strings.TrimLeft(objectKey, "/")
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), "GET\n\n\n"+expires+"\n"+canonicalResource)
query := parsed.Query()
query.Set("OSSAccessKeyId", a.channel.AccessKeyID)
query.Set("Expires", expires)
query.Set("Signature", signature)
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
query.Set("security-token", token)
}
parsed.RawQuery = query.Encode()
return parsed.String(), nil
}
func (a *objectStorageAdapter) signS3Request(req *http.Request, payloadHash string, now time.Time) {
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
amzDate := now.Format("20060102T150405Z")
date := now.Format("20060102")
req.Header.Set("x-amz-date", amzDate)
req.Header.Set("x-amz-content-sha256", payloadHash)
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
req.Header.Set("x-amz-security-token", token)
}
canonicalHeaders, signedHeaders := s3CanonicalHeaders(req)
canonicalRequest := strings.Join([]string{
req.Method,
s3CanonicalURI(req.URL),
s3CanonicalQuery(req.URL.Query()),
canonicalHeaders,
signedHeaders,
payloadHash,
}, "\n")
scope := date + "/" + region + "/s3/aws4_request"
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
signature := hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign))
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+a.channel.AccessKeyID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+signature)
}
func (a *objectStorageAdapter) presignS3Get(requestURL string, ttl time.Duration, now time.Time) (string, error) {
parsed, err := url.Parse(requestURL)
if err != nil {
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
}
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
amzDate := now.Format("20060102T150405Z")
date := now.Format("20060102")
scope := date + "/" + region + "/s3/aws4_request"
seconds := int64(ttl / time.Second)
if seconds < 1 {
seconds = 1
}
if seconds > 7*24*60*60 {
seconds = 7 * 24 * 60 * 60
}
query := parsed.Query()
query.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
query.Set("X-Amz-Credential", a.channel.AccessKeyID+"/"+scope)
query.Set("X-Amz-Date", amzDate)
query.Set("X-Amz-Expires", strconv.FormatInt(seconds, 10))
query.Set("X-Amz-SignedHeaders", "host")
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
query.Set("X-Amz-Security-Token", token)
}
canonicalRequest := "GET\n" + s3CanonicalURI(parsed) + "\n" + s3CanonicalQuery(query) + "\nhost:" + strings.ToLower(parsed.Host) + "\n\nhost\nUNSIGNED-PAYLOAD"
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
query.Set("X-Amz-Signature", hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign)))
parsed.RawQuery = query.Encode()
return parsed.String(), nil
}
func s3CanonicalHeaders(req *http.Request) (string, string) {
headers := map[string]string{"host": strings.ToLower(req.URL.Host)}
for key, values := range req.Header {
lower := strings.ToLower(strings.TrimSpace(key))
if lower == "content-type" || strings.HasPrefix(lower, "x-amz-") {
headers[lower] = strings.Join(values, ",")
}
}
keys := make([]string, 0, len(headers))
for key := range headers {
keys = append(keys, key)
}
sort.Strings(keys)
var canonical strings.Builder
for _, key := range keys {
canonical.WriteString(key)
canonical.WriteByte(':')
canonical.WriteString(strings.Join(strings.Fields(headers[key]), " "))
canonical.WriteByte('\n')
}
return canonical.String(), strings.Join(keys, ";")
}
func s3CanonicalURI(value *url.URL) string {
uri := value.EscapedPath()
if uri == "" {
return "/"
}
return uri
}
func s3CanonicalQuery(values url.Values) string {
return strings.ReplaceAll(values.Encode(), "+", "%20")
}
func s3SigningHMAC(secret string, date string, region string, stringToSign string) []byte {
dateKey := hmacSHA256([]byte("AWS4"+secret), date)
regionKey := hmacSHA256(dateKey, region)
serviceKey := hmacSHA256(regionKey, "s3")
signingKey := hmacSHA256(serviceKey, "aws4_request")
return hmacSHA256(signingKey, stringToSign)
}
func hmacSHA256(key []byte, value string) []byte {
mac := hmac.New(sha256.New, key)
_, _ = mac.Write([]byte(value))
return mac.Sum(nil)
}
func hmacSHA1Base64(key []byte, value string) string {
mac := hmac.New(sha1.New, key)
_, _ = mac.Write([]byte(value))
return base64.StdEncoding.EncodeToString(mac.Sum(nil))
}
func sha256Hex(value []byte) string {
digest := sha256.Sum256(value)
return hex.EncodeToString(digest[:])
}
func objectStorageConfigString(config map[string]any, key string) string {
if config == nil {
return ""
}
value, _ := config[key].(string)
return strings.TrimSpace(value)
}
func objectStorageConfigBool(config map[string]any, key string) bool {
if config == nil {
return false
}
value, _ := config[key].(bool)
return value
}
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
return scope
}
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
return "private"
}
if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" {
return "public"
}
return "private"
}
func escapeObjectKey(objectKey string) string {
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
for index, part := range parts {
parts[index] = url.PathEscape(part)
}
return strings.Join(parts, "/")
}
func storageStatusRetryable(status int) bool {
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
}
func objectStorageHTTPError(code string, status int, body []byte) error {
if status == http.StatusUnauthorized || status == http.StatusForbidden {
code = "storage_auth_failed"
}
message := http.StatusText(status)
if message == "" {
message = "object storage request failed"
}
// The provider response may contain object names, endpoints or credential
// diagnostics. Keep it out of the public error chain; channel health stores
// only the stable status description as well.
return storageClientError(code, message, status, storageStatusRetryable(status))
}
func storageClientError(code string, message string, status int, retryable bool) error {
return &clients.ClientError{Code: code, Message: message, StatusCode: status, Retryable: retryable}
}
@@ -0,0 +1,102 @@
package runner
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceemulator"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestLocalAcceptanceObjectStorageFailoverMatrix(t *testing.T) {
emulator := httptest.NewServer(acceptanceemulator.New(acceptanceemulator.Config{}).Handler())
defer emulator.Close()
service := &Service{}
payload := FileUploadPayload{
Bytes: []byte("acceptance-object"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset,
}
tests := []struct {
name string
channels []store.FileStorageChannel
winner string
wantErrCode string
}{
{
name: "oss_auth_to_s3",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-auth/bucket", "oss-auth", false),
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3", "s3-ok", false),
},
winner: "s3-ok",
},
{
name: "s3_auth_to_oss",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-auth", "s3-auth", false),
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-ok", false),
},
winner: "oss-ok",
},
{
name: "same_channel_transient_retry",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-transient", "s3-transient", true),
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-unused", false),
},
winner: "s3-transient",
},
{
name: "all_channels_failed",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-fail/bucket", "oss-fail", false),
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-fail", "s3-fail", false),
},
wantErrCode: "storage_write_failed",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := service.uploadFileWithFailover(t.Context(), payload, test.channels)
if test.wantErrCode != "" {
if clients.ErrorCode(err) != test.wantErrCode {
t.Fatalf("error=%v code=%s", err, clients.ErrorCode(err))
}
return
}
if err != nil {
t.Fatal(err)
}
channel, _ := result["storageChannel"].(map[string]any)
if got := stringFromAny(channel["channelKey"]); got != test.winner {
t.Fatalf("winner=%q, want %q", got, test.winner)
}
})
}
response, err := http.Get(emulator.URL + "/report")
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
var report acceptanceemulator.Report
if err := json.NewDecoder(response.Body).Decode(&report); err != nil {
t.Fatal(err)
}
if report.StoragePuts != 3 || report.StorageFailures != 5 {
t.Fatalf("unexpected storage acceptance report: %+v", report)
}
}
func acceptanceStorageChannel(provider string, endpoint string, key string, retry bool) store.FileStorageChannel {
channel := testObjectStorageChannel(provider, endpoint, key)
channel.Config["publicBaseUrl"] = ""
channel.RetryPolicy = map[string]any{"enabled": retry, "maxRetries": 0}
if retry {
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
}
return channel
}
@@ -0,0 +1,314 @@
package runner
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
type objectStorageMetricEvent struct {
event string
provider string
bytes int64
}
type objectStorageMetricsRecorder struct {
events []objectStorageMetricEvent
}
func (m *objectStorageMetricsRecorder) ObserveBillingEvent(string) {}
func (m *objectStorageMetricsRecorder) ObserveObjectStorage(event string, provider string, bytes int64, _ time.Duration) {
m.events = append(m.events, objectStorageMetricEvent{event: event, provider: provider, bytes: bytes})
}
func TestS3ObjectStorageUsesSigV4AndDeterministicObjectKey(t *testing.T) {
payload := []byte("same-media-payload")
var calls atomic.Int64
var requestPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
requestPath = r.URL.Path
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=access-id/") {
t.Errorf("missing SigV4 authorization: %q", r.Header.Get("Authorization"))
}
if r.Header.Get("x-amz-content-sha256") != sha256Hex(payload) {
t.Errorf("payload hash=%q", r.Header.Get("x-amz-content-sha256"))
}
body, _ := io.ReadAll(r.Body)
if string(body) != string(payload) {
t.Errorf("payload=%q", body)
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "s3-primary")
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC) }
result, err := adapter.put(t.Context(), FileUploadPayload{Bytes: payload, ContentType: "image/png", FileName: "ignored.jpg", Scene: store.FileStorageSceneImageResult})
if err != nil {
t.Fatal(err)
}
wantSuffix := "/bucket/media/image_result/2026/08/04/" + sha256Hex(payload) + ".png"
if requestPath != wantSuffix {
t.Fatalf("request path=%q, want %q", requestPath, wantSuffix)
}
if stringFromAny(result["objectKey"]) != strings.TrimPrefix(wantSuffix, "/bucket/") {
t.Fatalf("object key=%q", result["objectKey"])
}
if calls.Load() != 1 {
t.Fatalf("calls=%d", calls.Load())
}
}
func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
channel.Config["publicBaseUrl"] = "https://cdn.example"
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
result, err := adapter.put(t.Context(), FileUploadPayload{
Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset,
})
if err != nil {
t.Fatal(err)
}
if result["accessScope"] != "private" {
t.Fatalf("accessScope=%v, want private", result["accessScope"])
}
gotURL := stringFromAny(result["url"])
if strings.HasPrefix(gotURL, "https://cdn.example/") || !strings.Contains(gotURL, "X-Amz-Signature=") {
t.Fatalf("request asset URL must be signed and private: %s", gotURL)
}
}
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
channel.Config["forcePathStyle"] = false
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
value, err := adapter.objectURL("media/request_asset/object.png")
if err != nil {
t.Fatal(err)
}
if value != "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/request_asset/object.png" {
t.Fatalf("OSS object URL=%q", value)
}
}
func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) {
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
channel.Config["forcePathStyle"] = false
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/2026/08/04/OBJECT.png", nil)
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "image/png")
adapter.signOSSRequest(request, time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC))
if got, want := request.Header.Get("Authorization"), "OSS access-id:fohlh2SG7k7D+cmX48b7Keqa1Ok="; got != want {
t.Fatalf("authorization=%q, want %q", got, want)
}
if got := request.Header.Get("Date"); got != "Tue, 04 Aug 2026 01:02:03 GMT" {
t.Fatalf("date=%q", got)
}
}
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
var primaryCalls atomic.Int64
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
primaryCalls.Add(1)
http.Error(w, "temporary", http.StatusServiceUnavailable)
}))
defer primary.Close()
var secondaryCalls atomic.Int64
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
secondaryCalls.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer secondary.Close()
first := testObjectStorageChannel("s3", primary.URL, "s3-primary")
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
second := testObjectStorageChannel("aliyun_oss", secondary.URL, "oss-secondary")
second.RetryPolicy = map[string]any{"enabled": false}
metrics := &objectStorageMetricsRecorder{}
service := &Service{billingMetrics: metrics}
result, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second})
if err != nil {
t.Fatal(err)
}
channel, _ := result["storageChannel"].(map[string]any)
if stringFromAny(channel["channelKey"]) != "oss-secondary" {
t.Fatalf("unexpected winning channel: %+v", channel)
}
if primaryCalls.Load() != 2 || secondaryCalls.Load() != 1 {
t.Fatalf("calls primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
}
assertObjectStorageMetricEvent(t, metrics.events, "retry", "s3", 0)
assertObjectStorageMetricEvent(t, metrics.events, "failover", "s3", 0)
assertObjectStorageMetricEvent(t, metrics.events, "write_success", "aliyun_oss", int64(len("payload")))
}
func TestFileStorageChannelConnectionProbeWritesHeadsAndDeletes(t *testing.T) {
var methods []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
methods = append(methods, r.Method)
switch r.Method {
case http.MethodPut, http.MethodHead, http.MethodDelete:
w.WriteHeader(http.StatusOK)
default:
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
}
}))
defer server.Close()
result, err := (&Service{}).TestFileStorageChannel(t.Context(), testObjectStorageChannel("s3", server.URL, "probe"))
if err != nil {
t.Fatal(err)
}
if !result.PutSucceeded || !result.HeadSucceeded || !result.DeleteSucceeded || result.Provider != "s3" {
t.Fatalf("unexpected probe result: %+v", result)
}
if strings.Join(methods, ",") != "PUT,HEAD,DELETE" {
t.Fatalf("probe methods=%v", methods)
}
}
func TestObjectStorageReadUsesChannelRetryPolicy(t *testing.T) {
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 {
http.Error(w, "temporary", http.StatusServiceUnavailable)
return
}
_, _ = w.Write([]byte("stored-payload"))
}))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "s3-read")
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
payload, err := readObjectStorageWithRetries(t.Context(), adapter, "media/request_asset/object.png")
if err != nil || string(payload) != "stored-payload" || calls.Load() != 2 {
t.Fatalf("payload=%q calls=%d err=%v", payload, calls.Load(), err)
}
}
func TestObjectStorageAuthFailureSkipsSameChannelRetry(t *testing.T) {
var primaryCalls atomic.Int64
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
primaryCalls.Add(1)
http.Error(w, "forbidden details", http.StatusForbidden)
}))
defer primary.Close()
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer secondary.Close()
first := testObjectStorageChannel("aliyun_oss", primary.URL, "oss-primary")
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 3, "backoffSeconds": []any{0.001}}
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
service := &Service{}
if _, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second}); err != nil {
t.Fatal(err)
}
if primaryCalls.Load() != 1 {
t.Fatalf("auth failure retried %d times", primaryCalls.Load())
}
}
func TestRequestLevelUploadFailureDoesNotSwitchChannels(t *testing.T) {
var primaryCalls atomic.Int64
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
primaryCalls.Add(1)
http.Error(w, "unsupported media", http.StatusUnsupportedMediaType)
}))
defer primary.Close()
var secondaryCalls atomic.Int64
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
secondaryCalls.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer secondary.Close()
first := store.FileStorageChannel{
ChannelKey: "server-main-primary", Provider: "server_main_openapi", UploadURL: primary.URL,
APIKey: "test-key", RetryPolicy: map[string]any{"enabled": false},
}
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
_, err := (&Service{}).uploadFileWithFailover(t.Context(), FileUploadPayload{
Bytes: []byte("payload"), ContentType: "application/x-unsupported", Scene: store.FileStorageSceneUpload,
}, []store.FileStorageChannel{first, second})
if err == nil || clients.ErrorCode(err) != "upload_failed" {
t.Fatalf("unexpected request-level error: %v", err)
}
if primaryCalls.Load() != 1 || secondaryCalls.Load() != 0 {
t.Fatalf("request-level failure switched channels: primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
}
}
func TestAllObjectStorageChannelsFailWithStableError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "failure", http.StatusBadGateway) }))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "only")
channel.RetryPolicy = map[string]any{"enabled": false}
_, err := (&Service{}).uploadFileWithFailover(context.Background(), FileUploadPayload{Bytes: []byte("payload"), Scene: store.FileStorageSceneUpload}, []store.FileStorageChannel{channel})
if clients.ErrorCode(err) != "storage_write_failed" || strings.Contains(err.Error(), "failure") {
t.Fatalf("unexpected public storage error: %v", err)
}
}
func testObjectStorageChannel(provider string, endpoint string, channelKey string) store.FileStorageChannel {
return store.FileStorageChannel{
ChannelKey: channelKey,
Name: channelKey,
Provider: provider,
AccessKeyID: "access-id",
AccessKeySecret: "access-secret",
Priority: 100,
Config: map[string]any{
"endpoint": endpoint,
"bucket": "bucket",
"region": "cn-test-1",
"objectPrefix": "media",
"publicBaseUrl": "https://cdn.example.com",
"forcePathStyle": true,
},
}
}
func assertObjectStorageMetricEvent(t *testing.T, events []objectStorageMetricEvent, event string, provider string, bytes int64) {
t.Helper()
for _, item := range events {
if item.event == event && item.provider == provider && item.bytes == bytes {
return
}
}
t.Fatalf("missing metric event %s/%s/%d in %+v", event, provider, bytes, events)
}
+79 -6
View File
@@ -148,10 +148,14 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st
}
return base64.StdEncoding.EncodeToString(payload), nil
}
if strings.TrimSpace(asset.URL) == "" {
assetURL, err := s.requestAssetAccessURL(ctx, asset)
if err != nil {
return nil, err
}
if strings.TrimSpace(assetURL) == "" {
return nil, requestAssetExpiredError(asset)
}
return asset.URL, nil
return assetURL, nil
}
func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value string, path []string, candidate store.RuntimeModelCandidate) (any, error) {
@@ -224,10 +228,14 @@ func (s *Service) resolveRequestAsset(ctx context.Context, ref map[string]any) (
sha := stringFromAny(ref["sha256"])
contentType := stringFromAny(ref["contentType"])
asset := store.RequestAsset{
SHA256: sha,
ContentType: contentType,
URL: stringFromAny(ref["url"]),
StorageProvider: stringFromAny(ref["storageProvider"]),
SHA256: sha,
ContentType: contentType,
URL: stringFromAny(ref["url"]),
StorageProvider: stringFromAny(ref["storageProvider"]),
StorageChannelID: stringFromAny(ref["storageChannelId"]),
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
ObjectKey: stringFromAny(ref["objectKey"]),
AccessScope: stringFromAny(ref["accessScope"]),
}
if size := floatFromAny(ref["size"]); size > 0 {
asset.ByteSize = int64(size)
@@ -254,6 +262,17 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
if requestAssetIsExpired(asset, time.Now()) {
return nil, requestAssetExpiredError(asset)
}
if strings.TrimSpace(asset.ObjectKey) != "" {
channel, err := s.fileStorageChannelForAsset(ctx, asset)
if err != nil {
return nil, err
}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return nil, err
}
return readObjectStorageWithRetries(ctx, adapter, asset.ObjectKey)
}
if strings.TrimSpace(asset.LocalPath) != "" {
payload, err := os.ReadFile(asset.LocalPath)
if err != nil {
@@ -290,6 +309,60 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
return nil, requestAssetExpiredError(asset)
}
func readObjectStorageWithRetries(ctx context.Context, adapter *objectStorageAdapter, objectKey string) ([]byte, error) {
maxRetries, delays := uploadRetrySchedule(adapter.channel.RetryPolicy, adapter.channel.Provider)
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
payload, err := adapter.get(ctx, objectKey)
if err == nil {
return payload, nil
}
lastErr = err
if attempt >= maxRetries || !clients.IsRetryable(err) {
break
}
if err := sleepWithContext(ctx, retryDelayForAttempt(attempt, delays)); err != nil {
return nil, err
}
}
return nil, lastErr
}
func (s *Service) requestAssetAccessURL(ctx context.Context, asset store.RequestAsset) (string, error) {
if strings.TrimSpace(asset.ObjectKey) == "" {
return asset.URL, nil
}
channel, err := s.fileStorageChannelForAsset(ctx, asset)
if err != nil {
return "", err
}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return "", err
}
if strings.EqualFold(strings.TrimSpace(asset.AccessScope), "public") {
if value := adapter.publicURL(asset.ObjectKey); value != "" {
return value, nil
}
}
return adapter.presignGet(asset.ObjectKey, objectStorageSignedURLTTL)
}
func (s *Service) fileStorageChannelForAsset(ctx context.Context, asset store.RequestAsset) (store.FileStorageChannel, error) {
channelKey := strings.TrimSpace(asset.StorageChannelKey)
if channelKey == "environment-direct-oss" && s.directOSS != nil {
return s.directOSS.fileStorageChannel(), nil
}
if s.store == nil || channelKey == "" {
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
}
channel, err := s.store.GetFileStorageChannelByKey(ctx, channelKey)
if err != nil {
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
}
return channel, nil
}
func (s *Service) localPathFromRequestAssetURL(value string) string {
raw := strings.TrimSpace(value)
if raw == "" {
+9
View File
@@ -230,6 +230,15 @@ func (s *Service) observeTaskEventSkip(reason string) {
}
}
func (s *Service) observeObjectStorage(event string, provider string, bytes int, duration time.Duration) {
observer, ok := s.billingMetrics.(interface {
ObserveObjectStorage(string, string, int64, time.Duration)
})
if ok {
observer.ObserveObjectStorage(event, provider, int64(bytes), duration)
}
}
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
return s.execute(ctx, task, user, nil)
}
@@ -10,6 +10,7 @@ import (
"net/http"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
@@ -74,13 +75,19 @@ func (s *Service) processTaskCallbackBatch(ctx context.Context, client *http.Cli
}
func deliverTaskCallback(ctx context.Context, client *http.Client, item store.TaskCallbackDelivery, bearerToken string) (int, error) {
body, err := json.Marshal(map[string]any{
payload := map[string]any{
"taskId": item.TaskID,
"seq": item.Seq,
"eventType": item.EventType,
"status": item.TaskStatus,
"createdAt": item.CreatedAt.UTC().Format(time.RFC3339Nano),
})
}
if item.TaskStatus == "failed" || item.TaskStatus == "cancelled" {
standard := publicerror.WithIDs(publicerror.FromFields(item.TaskErrorCode, item.TaskErrorMessage, 0, false), item.TaskRequestID, item.TaskID)
publicerror.Observe(standard)
payload["error"] = standard
}
body, err := json.Marshal(payload)
if err != nil {
return 0, err
}
@@ -5,6 +5,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@@ -116,3 +117,33 @@ func TestDeliverTaskCallbackSendsMinimalAuthenticatedBody(t *testing.T) {
t.Fatalf("callback body=%#v", received)
}
}
func TestDeliverFailedTaskCallbackUsesStandardErrorWithoutSocketDetails(t *testing.T) {
var received map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if err := json.NewDecoder(request.Body).Decode(&received); err != nil {
t.Error(err)
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
status, err := deliverTaskCallback(t.Context(), server.Client(), store.TaskCallbackDelivery{
TaskID: "failed-task",
Seq: 4,
CallbackURL: server.URL,
EventType: "task.failed",
TaskStatus: "failed",
TaskErrorCode: "response_read_error",
TaskErrorMessage: raw,
CreatedAt: time.Now(),
}, "")
if err != nil || status != http.StatusNoContent {
t.Fatalf("status=%d err=%v", status, err)
}
errorValue, _ := received["error"].(map[string]any)
if errorValue["code"] != "upstream_connection_interrupted" || strings.Contains(stringFromAny(errorValue["message"]), "10.42.0.72") {
t.Fatalf("unsafe callback error: %#v", errorValue)
}
}
+139 -151
View File
@@ -17,25 +17,18 @@ import (
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const defaultServerMainOpenAPIUploadURL = "http://127.0.0.1:3001/v1/files/upload"
const maxGeneratedAssetFetchBytes = 256 << 20
const (
localStaticGeneratedPathPrefix = "/static/generated/"
localStaticUploadedPathPrefix = "/static/uploaded/"
)
type FileUploadPayload struct {
ContentType string
FileName string
@@ -136,14 +129,6 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
return nil, &clients.ClientError{Code: "acceptance_run_inactive", Message: err.Error(), Retryable: false}
}
}
if policy.LocalizeInlineMedia {
next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
if err != nil {
return nil, err
}
redactGeneratedResultRawData(next)
return next, nil
}
if len(data) == 0 && !rawNeedsUpload {
return s.finalizeGeneratedAssets(ctx, taskID, taskKind, result, policy, nil, false, 0)
}
@@ -179,11 +164,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
var channels []store.FileStorageChannel
channelsLoaded := false
if needsUpload || rawNeedsUpload {
if s.directOSS == nil {
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
if err != nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
}
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
if err != nil {
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "object storage configuration is unavailable", Retryable: true}
}
channelsLoaded = true
}
@@ -323,12 +306,12 @@ func (s *Service) finalizeGeneratedAssets(
if !TaskResultHasInlineBinary(next) {
return next, nil
}
persistent, _, err := s.materializeLocalBinaryResult(ctx, taskID, next)
if err != nil {
return nil, err
return nil, &clients.ClientError{
Code: "storage_write_failed",
Message: "generated binary result could not be written to object storage",
StatusCode: http.StatusServiceUnavailable,
Retryable: true,
}
redactGeneratedResultRawData(persistent)
return persistent, nil
}
func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
@@ -355,6 +338,9 @@ func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]
func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) {
switch typed := value.(type) {
case map[string]any:
if payload, contentType, ok := localBufferObjectBytes(typed); ok {
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, contentType, firstNonEmptyString(key, "buffer"), siblings, channels, index)
}
next := make(map[string]any, len(typed))
changed := false
for childKey, childValue := range typed {
@@ -372,6 +358,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
}
return value, false, nil
case []any:
if localBinaryKey(key) {
if payload, ok := bytesFromNumberArray(typed); ok {
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
}
}
next := make([]any, len(typed))
changed := false
for itemIndex, item := range typed {
@@ -388,6 +379,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
return next, true, nil
}
return value, false, nil
case []byte:
if len(typed) == 0 {
return value, false, nil
}
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
case string:
asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind)
if !ok {
@@ -404,19 +400,36 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
}
}
func (s *Service) uploadGeneratedBinaryValue(ctx context.Context, taskID string, taskKind string, payload []byte, contentType string, sourceKey string, siblings map[string]any, channels []store.FileStorageChannel, index *int) (any, bool, error) {
contentType = firstNonEmptyString(contentType, defaultContentTypeForRawMediaKey(sourceKey))
asset := &generatedInlineAsset{
Bytes: payload,
ContentType: contentType,
Kind: mediaKindForAsset(taskKind, siblings, sourceKey, contentType),
SourceKey: sourceKey,
}
upload, resolvedContentType, kind, strategy, err := s.uploadGeneratedAsset(ctx, taskID, asset, *index, channels)
if err != nil {
return nil, false, err
}
*index = *index + 1
return generatedRawMediaReference(asset, upload, resolvedContentType, kind, strategy), true, nil
}
func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) {
raw := strings.TrimSpace(value)
if raw == "" {
return nil, false
}
keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
if !generatedRawDataMediaPayloadKey(key) && !generatedContentTypeIsMedia(contentType) {
if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) {
return nil, false
}
if !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
if !keyLooksLikeMediaPayload && !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
return nil, false
}
payload, payloadContentType, ok, err := inlineMediaPayload(raw, generatedRawDataMediaPayloadKey(key))
payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload)
if err != nil || !ok || len(payload) == 0 {
return nil, false
}
@@ -454,6 +467,18 @@ func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]a
if provider := stringFromAny(channel["provider"]); provider != "" {
ref["storageProvider"] = provider
}
if id := stringFromAny(channel["id"]); id != "" {
ref["storageChannelId"] = id
}
if key := stringFromAny(channel["channelKey"]); key != "" {
ref["storageChannelKey"] = key
}
}
if objectKey := stringFromAny(upload["objectKey"]); objectKey != "" {
ref["objectKey"] = objectKey
}
if accessScope := stringFromAny(upload["accessScope"]); accessScope != "" {
ref["accessScope"] = accessScope
}
out := map[string]any{
"assetRef": ref,
@@ -650,6 +675,9 @@ func removeASCIIWhitespace(value string) string {
}
func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
if s.store == nil {
return defaultGeneratedAssetUploadPolicy(), nil
}
settings, err := s.store.GetFileStorageSettings(ctx)
if err != nil {
if store.IsUndefinedDatabaseObject(err) {
@@ -665,8 +693,6 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
switch policyName {
case store.FileStorageResultUploadPolicyUploadAll:
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
case store.FileStorageResultUploadPolicyUploadNone:
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
default:
return defaultGeneratedAssetUploadPolicy()
}
@@ -682,13 +708,8 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset
Scene: store.FileStorageSceneImageResult,
Source: "ai-gateway",
}
if s.directOSS != nil {
upload, err := s.directOSS.upload(ctx, payload)
return upload, contentType, kind, "direct_aliyun_oss", err
}
if len(channels) == 0 {
upload, err := s.storeFileLocally(payload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
return upload, contentType, kind, "local_static_inline_media", err
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
upload, err := s.uploadFileWithFailover(ctx, payload, channels)
return upload, contentType, kind, "upload_inline_media", err
@@ -708,95 +729,13 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as
Scene: store.FileStorageSceneImageResult,
Source: "ai-gateway",
}
if s.directOSS != nil {
upload, err := s.directOSS.upload(ctx, uploadPayload)
return upload, contentType, kind, "direct_aliyun_oss", err
}
if len(channels) == 0 {
upload, err := s.storeFileLocally(uploadPayload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
return upload, contentType, kind, "local_static_url_media", err
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels)
return upload, contentType, kind, "upload_url_media", err
}
func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string, fallbackStorageDir string, pathPrefix string) (map[string]any, error) {
storageDir = strings.TrimSpace(storageDir)
if storageDir == "" {
storageDir = fallbackStorageDir
}
if err := os.MkdirAll(storageDir, 0o755); err != nil {
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
}
fileName := filepath.Base(strings.TrimSpace(payload.FileName))
if fileName == "" || fileName == "." || fileName == ".." || fileName == string(filepath.Separator) {
kind := generatedAssetKindFromContentType("", payload.ContentType)
fileName = generatedAssetFileName("generated", 0, payload.ContentType, kind)
}
targetPath := filepath.Join(storageDir, fileName)
file, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
}
_, writeErr := file.Write(payload.Bytes)
closeErr := file.Close()
if writeErr != nil {
_ = os.Remove(targetPath)
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: writeErr.Error(), Retryable: true}
}
if closeErr != nil {
_ = os.Remove(targetPath)
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true}
}
expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339)
return map[string]any{
"url": s.localStaticFileURL(fileName, pathPrefix),
"fileName": fileName,
"contentType": payload.ContentType,
"size": len(payload.Bytes),
"expiresAt": expiresAt,
"storageChannel": map[string]any{
"id": "local-static",
"channelKey": "local-static",
"name": "AI Gateway local static storage",
"provider": "local_static",
},
}, nil
}
func (s *Service) localStaticAssetTTLHours() int {
if s.cfg.LocalTempAssetTTLHours <= 0 {
return 24
}
return s.cfg.LocalTempAssetTTLHours
}
func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string {
if strings.TrimSpace(pathPrefix) == "" {
pathPrefix = localStaticUploadedPathPrefix
}
path := pathPrefix + url.PathEscape(filepath.Base(fileName))
baseURL := strings.TrimRight(strings.TrimSpace(s.cfg.PublicBaseURL), "/")
if baseURL == "" {
return path
}
return baseURL + path
}
func localStaticUploadFileName(originalName string, contentType string) string {
baseName := filepath.Base(strings.TrimSpace(originalName))
originalExt := strings.ToLower(filepath.Ext(baseName))
namePart := strings.TrimSuffix(baseName, originalExt)
namePart = sanitizeGeneratedAssetNamePart(namePart)
if namePart == "" {
namePart = "gateway-upload"
}
if len(namePart) > 48 {
namePart = namePart[:48]
}
return fmt.Sprintf("%s-%s%s", namePart, randomHexSuffix(6), uploadFileExtension(contentType, originalExt))
}
func uploadFileExtension(contentType string, fallbackExt string) string {
normalized := normalizeGeneratedContentType(contentType)
if generatedContentTypeIsMedia(normalized) {
@@ -981,46 +920,41 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma
if strings.TrimSpace(payload.Scene) == "" {
payload.Scene = store.FileStorageSceneUpload
}
if s.directOSS != nil && directOSSScene(payload.Scene) {
return s.directOSS.upload(ctx, payload)
}
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
if err != nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
}
if len(channels) == 0 {
payload.FileName = localStaticUploadFileName(payload.FileName, payload.ContentType)
upload, err := s.storeFileLocally(payload, s.cfg.LocalUploadedStorageDir, config.DefaultLocalUploadedStorageDir, localStaticUploadedPathPrefix)
if err != nil {
return nil, err
}
upload["assetStorage"] = map[string]any{
"scene": payload.Scene,
"source": firstNonEmptyString(payload.Source, "ai-gateway-openapi"),
"strategy": "local_static_upload",
}
return upload, nil
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
return s.uploadFileWithFailover(ctx, payload, channels)
}
func (s *Service) activeFileStorageChannels(ctx context.Context, scene string) ([]store.FileStorageChannel, error) {
if s.store == nil {
return nil, nil
channels := make([]store.FileStorageChannel, 0)
if s.directOSS != nil && directOSSScene(scene) {
channels = append(channels, s.directOSS.fileStorageChannel())
}
channels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
if s.store == nil {
return channels, nil
}
storedChannels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
if err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
}
if len(channels) > 0 {
return channels, nil
}
return nil, nil
channels = append(channels, storedChannels...)
sort.SliceStable(channels, func(i, j int) bool {
if channels[i].Priority != channels[j].Priority {
return channels[i].Priority < channels[j].Priority
}
return channels[i].ChannelKey < channels[j].ChannelKey
})
return channels, nil
}
func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUploadPayload, channels []store.FileStorageChannel) (map[string]any, error) {
var lastErr error
for _, channel := range channels {
for index, channel := range channels {
upload, err := s.uploadWithChannelRetries(ctx, payload, channel)
if err == nil {
if s.store != nil {
@@ -1032,25 +966,59 @@ func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUpload
if s.store != nil {
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(ctx), channel.ID, err.Error())
}
if !storageFailureAllowsFailover(channel, err) {
return nil, err
}
if index+1 < len(channels) {
s.observeObjectStorage("failover", channel.Provider, 0, 0)
}
}
if lastErr != nil {
return nil, lastErr
s.observeObjectStorage("all_failed", "", 0, 0)
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "all configured object storage channels failed", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
func storageFailureAllowsFailover(channel store.FileStorageChannel, err error) bool {
code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
switch code {
case "upload_source_too_large", "upload_decode_failed", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio":
return false
}
var clientErr *clients.ClientError
if !errors.As(err, &clientErr) {
return true
}
switch clientErr.StatusCode {
case http.StatusRequestEntityTooLarge, http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity:
return false
case http.StatusBadRequest:
// Object-storage 400 responses commonly indicate endpoint/signature
// configuration and should move to the next channel. server-main 400 is
// the existing upload API's request validation result.
return !strings.EqualFold(strings.TrimSpace(channel.Provider), "server_main_openapi")
default:
return true
}
return nil, &clients.ClientError{Code: "upload_no_channel", Message: "no enabled file storage channel", Retryable: false}
}
func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy)
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy, channel.Provider)
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
startedAt := time.Now()
upload, err := s.uploadOnce(ctx, payload, channel)
if err == nil {
s.observeObjectStorage("write_success", channel.Provider, len(payload.Bytes), time.Since(startedAt))
return upload, nil
}
s.observeObjectStorage("write_failure", channel.Provider, 0, time.Since(startedAt))
lastErr = err
if attempt >= maxRetries || !clients.IsRetryable(err) {
break
}
s.observeObjectStorage("retry", channel.Provider, 0, 0)
delay := retryDelayForAttempt(attempt, delays)
if err := sleepWithContext(ctx, delay); err != nil {
return nil, err
@@ -1060,9 +1028,21 @@ func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUplo
}
func (s *Service) uploadOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
if strings.ToLower(strings.TrimSpace(channel.Provider)) != "server_main_openapi" {
return nil, &clients.ClientError{Code: "upload_unsupported_channel", Message: "unsupported file storage channel: " + channel.Provider, Retryable: false}
switch strings.ToLower(strings.TrimSpace(channel.Provider)) {
case "aliyun_oss", "s3":
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return nil, err
}
return adapter.put(ctx, payload)
case "server_main_openapi":
return s.uploadServerMainOnce(ctx, payload, channel)
default:
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported file storage channel", Retryable: false}
}
}
func (s *Service) uploadServerMainOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
uploadURL := strings.TrimSpace(channel.UploadURL)
if uploadURL == "" {
uploadURL = defaultServerMainOpenAPIUploadURL
@@ -1700,9 +1680,17 @@ func uniqueStringList(values []string) []string {
return out
}
func uploadRetrySchedule(policy map[string]any) (int, []time.Duration) {
func uploadRetrySchedule(policy map[string]any, providers ...string) (int, []time.Duration) {
provider := ""
if len(providers) > 0 {
provider = providers[0]
}
if policy == nil {
policy = defaultUploadRetryPolicy()
if strings.EqualFold(provider, "aliyun_oss") || strings.EqualFold(provider, "s3") {
policy = map[string]any{"enabled": true, "maxRetries": 2, "backoffSeconds": []any{0.25, 1.0}}
} else {
policy = defaultUploadRetryPolicy()
}
}
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
return 0, nil
@@ -1725,9 +1713,9 @@ func uploadRetryDelays(value any) []time.Duration {
}
delays := make([]time.Duration, 0, len(items))
for _, item := range items {
seconds := int(floatFromAny(item))
seconds := floatFromAny(item)
if seconds > 0 {
delays = append(delays, time.Duration(seconds)*time.Second)
delays = append(delays, time.Duration(seconds*float64(time.Second)))
}
}
return delays
+59 -139
View File
@@ -7,11 +7,11 @@ import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -227,38 +227,6 @@ func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
}
}
func TestGeneratedAssetDecisionPreservesInlineWhenPolicyUploadNone(t *testing.T) {
item := map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
}
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline != nil || decision.URL != nil {
t.Fatalf("upload_none should not transfer inline payloads: %+v", decision)
}
if len(decision.StripKeys) != 0 {
t.Fatalf("upload_none should preserve b64_json for the caller: %+v", decision.StripKeys)
}
}
func TestGeneratedAssetDecisionPreservesInlineAlongsideURLWhenPolicyUploadNone(t *testing.T) {
item := map[string]any{
"url": "https://cdn.example.com/generated.png",
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
}
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline != nil || decision.URL != nil || len(decision.StripKeys) != 0 {
t.Fatalf("upload_none should preserve the upstream URL and base64 fields unchanged: %+v", decision)
}
}
func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
tests := []struct {
name string
@@ -276,9 +244,9 @@ func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true, PreserveInlineMedia: false},
},
{
name: "upload none",
name: "legacy upload none becomes default",
policyName: store.FileStorageResultUploadPolicyUploadNone,
want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true},
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: false, PreserveInlineMedia: false},
},
}
@@ -330,8 +298,10 @@ func TestAcceptanceGeneratedMediaAllowsOnlyExactEmulatorOrigin(t *testing.T) {
}
func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
encoded := base64.StdEncoding.EncodeToString(payload)
result := map[string]any{
@@ -348,7 +318,7 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
"images.edits",
result,
defaultGeneratedAssetUploadPolicy(),
nil,
channels,
true,
0,
)
@@ -363,15 +333,8 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
if !ok {
t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested)
}
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-nested-binary-01-") {
t.Fatalf("unexpected local static URL: %s", urlValue)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("read generated storage: %v", err)
}
if len(entries) != 1 {
t.Fatalf("expected one localized result file, got %d", len(entries))
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") {
t.Fatalf("unexpected object storage URL: %s", urlValue)
}
}
@@ -405,9 +368,8 @@ func TestGeneratedAssetFileNameIsUniqueAndTyped(t *testing.T) {
}
}
func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
func TestUploadGeneratedAssetFailsWithoutObjectStorage(t *testing.T) {
service := &Service{}
payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
asset := &generatedInlineAsset{
Bytes: payload,
@@ -416,44 +378,14 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
SourceKey: "b64_json",
}
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
if err != nil {
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
if clients.ErrorCode(err) != "storage_write_failed" {
t.Fatalf("unexpected error: %v", err)
}
if contentType != "image/png" || kind != "image" || strategy != "local_static_inline_media" {
t.Fatalf("unexpected local upload metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
}
urlValue := stringFromAny(upload["url"])
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-123-01-") || !strings.HasSuffix(urlValue, ".png") {
t.Fatalf("unexpected local static URL: %s", urlValue)
}
expiresAt, err := time.Parse(time.RFC3339, stringFromAny(upload["expiresAt"]))
if err != nil {
t.Fatalf("local static upload should expose expiresAt: %+v", upload)
}
remaining := time.Until(expiresAt)
if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour+time.Minute {
t.Fatalf("local static upload should expire after one day, remaining=%s", remaining)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("failed to read local static dir: %v", err)
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
t.Fatalf("expected one PNG file in local static dir, got %+v", entries)
}
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
if err != nil {
t.Fatalf("failed to read local static file: %v", err)
}
if !bytes.Equal(stored, payload) {
t.Fatalf("stored payload does not match source payload")
}
}
func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
func TestUploadGeneratedAudioFailsWithoutObjectStorage(t *testing.T) {
service := &Service{}
asset := &generatedInlineAsset{
Bytes: []byte("inline audio payload"),
ContentType: "audio/mpeg",
@@ -461,32 +393,17 @@ func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
SourceKey: "content",
}
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
if err != nil {
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
if clients.ErrorCode(err) != "storage_write_failed" {
t.Fatalf("unexpected error: %v", err)
}
if contentType != "audio/mpeg" || kind != "audio" || strategy != "local_static_inline_media" {
t.Fatalf("unexpected local audio metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
}
urlValue := stringFromAny(upload["url"])
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-tts-01-") || !strings.HasSuffix(urlValue, ".mp3") {
t.Fatalf("unexpected local audio URL: %s", urlValue)
}
if stringFromAny(upload["expiresAt"]) == "" {
t.Fatalf("local audio static upload should expose expiresAt: %+v", upload)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("failed to read local static dir: %v", err)
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".mp3") {
t.Fatalf("expected one MP3 file in local static dir, got %+v", entries)
}
}
func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
raw := map[string]any{
"candidates": []any{
@@ -506,7 +423,7 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
}
index := 0
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), nil, &index)
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
if err != nil {
t.Fatalf("upload raw media: %v", err)
}
@@ -528,22 +445,47 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) {
t.Fatalf("unexpected asset ref: %+v", ref)
}
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-raw-01-") || !strings.HasSuffix(urlValue, ".png") {
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") || !strings.HasSuffix(urlValue, ".png") {
t.Fatalf("unexpected raw media URL: %s", urlValue)
}
if inlineData["data"] == base64.StdEncoding.EncodeToString(payload) {
t.Fatal("raw inlineData still contains base64 payload")
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("read generated storage: %v", err)
}
func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-buffer-result")}
raw := map[string]any{
"buffer": map[string]any{
"type": "Buffer", "mimeType": "image/png", "data": []any{float64(0x89), float64('P'), float64('N'), float64('G')},
},
"audio_bytes": []any{float64('I'), float64('D'), float64('3')},
"direct": []byte("direct bytes"),
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
t.Fatalf("expected one generated PNG, got %+v", entries)
index := 0
uploaded, changed, err := service.uploadGeneratedRawMediaValue(t.Context(), "task-buffer", "images.generations", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
if err != nil {
t.Fatal(err)
}
if !changed || index != 3 {
t.Fatalf("changed=%v uploads=%d", changed, index)
}
next := uploaded.(map[string]any)
for _, key := range []string{"buffer", "audio_bytes", "direct"} {
item, ok := next[key].(map[string]any)
if !ok || item["assetRef"] == nil || item["upload"] == nil {
t.Fatalf("%s was not objectified: %#v", key, next[key])
}
}
if TaskResultHasInlineBinary(next) {
t.Fatalf("objectified result still contains inline binary: %#v", next)
}
}
func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
func TestUploadFileFailsWithoutObjectStorageAndCreatesNoLocalFile(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{
LocalUploadedStorageDir: storageDir,
@@ -552,43 +494,21 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
}}
payload := []byte("%PDF-1.4")
upload, err := service.UploadFile(context.Background(), FileUploadPayload{
_, err := service.UploadFile(context.Background(), FileUploadPayload{
Bytes: payload,
ContentType: "application/pdf",
FileName: "用户文件.png",
Source: "playground",
})
if err != nil {
if clients.ErrorCode(err) != "storage_write_failed" {
t.Fatalf("unexpected error: %v", err)
}
urlValue := stringFromAny(upload["url"])
if !strings.HasPrefix(urlValue, "/static/uploaded/") || !strings.HasSuffix(urlValue, ".pdf") {
t.Fatalf("unexpected uploaded local static URL: %s", urlValue)
}
if stringFromAny(upload["expiresAt"]) == "" {
t.Fatalf("local uploaded static file should expose expiresAt: %+v", upload)
}
storageChannel, _ := upload["storageChannel"].(map[string]any)
if stringFromAny(storageChannel["provider"]) != "local_static" {
t.Fatalf("expected local static provider metadata, got %+v", upload["storageChannel"])
}
assetStorage, _ := upload["assetStorage"].(map[string]any)
if stringFromAny(assetStorage["strategy"]) != "local_static_upload" || stringFromAny(assetStorage["scene"]) != store.FileStorageSceneUpload {
t.Fatalf("unexpected upload asset storage metadata: %+v", assetStorage)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("failed to read uploaded static dir: %v", err)
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".pdf") {
t.Fatalf("expected one PDF file in uploaded static dir, got %+v", entries)
}
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
if err != nil {
t.Fatalf("failed to read uploaded static file: %v", err)
}
if !bytes.Equal(stored, payload) {
t.Fatalf("stored uploaded payload does not match source payload")
if len(entries) != 0 {
t.Fatalf("unexpected local files: %+v", entries)
}
}