feat(storage): 统一二进制对象存储与公开错误
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换,保留 server-main 兼容与环境 OSS 内存通道。 将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化,生产路径不再写入本机静态目录,历史本地资源仅保留只读兼容。 引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出,同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。 验证:go test ./... -count=1;go vet ./...;pnpm lint;pnpm test;pnpm build;pnpm openapi;tests/ci/migrations-test.sh。
This commit is contained in:
@@ -12,7 +12,6 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
@@ -26,7 +25,6 @@ const (
|
||||
localBinaryGenericBase64MinLength = 4096
|
||||
localBinaryMaxDepth = 64
|
||||
defaultLocalResultTTLHours = 24
|
||||
defaultLocalResultMinFreeBytes = int64(10 * 1024 * 1024 * 1024)
|
||||
defaultLocalResultMaxBytes = int64(256 * 1024 * 1024)
|
||||
defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024)
|
||||
)
|
||||
@@ -41,41 +39,38 @@ type localBinaryDescriptor struct {
|
||||
|
||||
type localBinaryMaterializer struct {
|
||||
service *Service
|
||||
taskDir string
|
||||
writeFiles bool
|
||||
enforceLimits bool
|
||||
totalBytes int64
|
||||
seen map[string]struct{}
|
||||
createdFiles []string
|
||||
}
|
||||
|
||||
// materializeLocalBinaryResult replaces every inline binary value with a
|
||||
// bounded placeholder after atomically writing and verifying the bytes locally.
|
||||
// materializeLocalBinaryResult is retained as an internal compatibility seam,
|
||||
// but new materialization always targets object storage. Historical local
|
||||
// placeholders remain readable through HydrateTaskResult.
|
||||
func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
||||
if _, _, err := s.transformLocalBinaryResult(ctx, taskID, result, false, true); err != nil {
|
||||
hadInline := TaskResultHasInlineBinary(result)
|
||||
if !hadInline {
|
||||
return result, false, nil
|
||||
}
|
||||
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return s.transformLocalBinaryResult(ctx, taskID, result, true, true)
|
||||
return next, !TaskResultHasInlineBinary(next), nil
|
||||
}
|
||||
|
||||
func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, result map[string]any, writeFiles bool, enforceLimits bool) (map[string]any, bool, error) {
|
||||
root := s.localBinaryResultRoot()
|
||||
taskDir := filepath.Join(root, safeLocalBinaryTaskDir(taskID))
|
||||
func (s *Service) transformLocalBinaryResult(ctx context.Context, _ string, result map[string]any, _ bool, enforceLimits bool) (map[string]any, bool, error) {
|
||||
materializer := &localBinaryMaterializer{
|
||||
service: s,
|
||||
taskDir: taskDir,
|
||||
writeFiles: writeFiles,
|
||||
enforceLimits: enforceLimits,
|
||||
seen: map[string]struct{}{},
|
||||
}
|
||||
next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
|
||||
if err != nil {
|
||||
materializer.rollbackCreatedFiles()
|
||||
return nil, false, err
|
||||
}
|
||||
mapped, ok := next.(map[string]any)
|
||||
if !ok {
|
||||
materializer.rollbackCreatedFiles()
|
||||
return nil, false, &clients.ClientError{
|
||||
Code: "result_binary_not_materialized",
|
||||
Message: "generated result is not a JSON object",
|
||||
@@ -83,28 +78,28 @@ func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
if changed && enforceLimits && !writeFiles {
|
||||
if err := os.MkdirAll(root, 0o750); err != nil {
|
||||
return nil, false, localBinaryStorageError(err)
|
||||
}
|
||||
if err := ensureLocalBinaryDiskHeadroom(root, materializer.totalBytes, s.localResultMinFreeBytes()); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
return mapped, changed, nil
|
||||
}
|
||||
|
||||
// MaterializeTaskResultForStorage exposes the same verified materialization
|
||||
// path to the explicit historical maintenance command.
|
||||
func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
||||
return s.materializeLocalBinaryResult(ctx, taskID, result)
|
||||
hadInline := TaskResultHasInlineBinary(result)
|
||||
if !hadInline {
|
||||
return result, false, nil
|
||||
}
|
||||
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return next, hadInline && !TaskResultHasInlineBinary(next), nil
|
||||
}
|
||||
|
||||
// CompactExpiredTaskResultForStorage computes the same deterministic
|
||||
// placeholders without creating files that are already outside the recovery
|
||||
// window.
|
||||
// CompactExpiredTaskResultForStorage keeps the historical maintenance API but
|
||||
// now uses the same object-storage path as live results. New GatewayBinary
|
||||
// placeholders are never created; their parser remains read-only compatibility.
|
||||
func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
||||
return s.transformLocalBinaryResult(ctx, taskID, result, false, false)
|
||||
return s.MaterializeTaskResultForStorage(ctx, taskID, result)
|
||||
}
|
||||
|
||||
func TaskResultHasInlineBinary(result map[string]any) bool {
|
||||
@@ -239,15 +234,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
if m.writeFiles {
|
||||
created, err := m.service.writeLocalBinaryResult(m.taskDir, digestHex, payload)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if created {
|
||||
m.createdFiles = append(m.createdFiles, filepath.Join(m.taskDir, digestHex+".bin"))
|
||||
}
|
||||
}
|
||||
m.seen[digestHex] = struct{}{}
|
||||
m.totalBytes += size
|
||||
}
|
||||
@@ -261,94 +247,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
|
||||
return localBinaryPlaceholder(descriptor), true, nil
|
||||
}
|
||||
|
||||
func (m *localBinaryMaterializer) rollbackCreatedFiles() {
|
||||
for _, path := range m.createdFiles {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
_ = os.Remove(m.taskDir)
|
||||
m.createdFiles = nil
|
||||
}
|
||||
|
||||
func (s *Service) writeLocalBinaryResult(taskDir string, digestHex string, payload []byte) (bool, error) {
|
||||
if err := os.MkdirAll(taskDir, 0o750); err != nil {
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
targetPath := filepath.Join(taskDir, digestHex+".bin")
|
||||
if info, err := os.Stat(targetPath); err == nil {
|
||||
if !info.IsDir() && info.Size() == int64(len(payload)) {
|
||||
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err == nil {
|
||||
now := time.Now()
|
||||
if err := os.Chtimes(targetPath, now, now); err != nil {
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return false, &clients.ClientError{
|
||||
Code: "binary_result_corrupted",
|
||||
Message: "existing local result file does not match its content hash",
|
||||
StatusCode: 500,
|
||||
Retryable: false,
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
if err := ensureLocalBinaryDiskHeadroom(taskDir, int64(len(payload)), s.localResultMinFreeBytes()); err != nil {
|
||||
return false, err
|
||||
}
|
||||
tempFile, err := os.CreateTemp(taskDir, ".gateway-result-*")
|
||||
if err != nil {
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
tempPath := tempFile.Name()
|
||||
cleanup := func() {
|
||||
_ = tempFile.Close()
|
||||
_ = os.Remove(tempPath)
|
||||
}
|
||||
if err := tempFile.Chmod(0o640); err != nil {
|
||||
cleanup()
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
if _, err := tempFile.Write(payload); err != nil {
|
||||
cleanup()
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
if err := tempFile.Sync(); err != nil {
|
||||
cleanup()
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
if err := tempFile.Close(); err != nil {
|
||||
_ = os.Remove(tempPath)
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
if err := os.Rename(tempPath, targetPath); err != nil {
|
||||
_ = os.Remove(tempPath)
|
||||
return false, localBinaryStorageError(err)
|
||||
}
|
||||
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err != nil {
|
||||
_ = os.Remove(targetPath)
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func ensureLocalBinaryDiskHeadroom(path string, incomingBytes int64, minFreeBytes int64) error {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &stat); err != nil {
|
||||
return localBinaryStorageError(err)
|
||||
}
|
||||
freeBytes := int64(stat.Bavail) * int64(stat.Bsize)
|
||||
if freeBytes-incomingBytes < minFreeBytes {
|
||||
return &clients.ClientError{
|
||||
Code: "local_result_storage_unavailable",
|
||||
Message: "local result storage does not have enough free space",
|
||||
StatusCode: 503,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func localBinaryStorageError(err error) error {
|
||||
return &clients.ClientError{
|
||||
Code: "local_result_storage_unavailable",
|
||||
@@ -406,6 +304,30 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
refreshedAccessURL := false
|
||||
if asset, upload, ok := generatedResultUploadReference(typed); ok {
|
||||
accessURL, err := s.requestAssetAccessURL(ctx, asset)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
next := make(map[string]any, len(typed))
|
||||
for key, item := range typed {
|
||||
next[key] = item
|
||||
}
|
||||
nextUpload := make(map[string]any, len(upload))
|
||||
for key, item := range upload {
|
||||
nextUpload[key] = item
|
||||
}
|
||||
nextUpload["url"] = accessURL
|
||||
next["upload"] = nextUpload
|
||||
for _, key := range []string{"url", "image_url", "video_url", "audio_url"} {
|
||||
if _, exists := next[key]; exists {
|
||||
next[key] = accessURL
|
||||
}
|
||||
}
|
||||
typed = next
|
||||
refreshedAccessURL = true
|
||||
}
|
||||
if ref, ok := generatedResultAssetReference(typed); ok {
|
||||
payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
|
||||
if err != nil {
|
||||
@@ -418,7 +340,7 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
|
||||
return encoded, true, nil
|
||||
}
|
||||
next := make(map[string]any, len(typed))
|
||||
changed := false
|
||||
changed := refreshedAccessURL
|
||||
for key, childValue := range typed {
|
||||
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
|
||||
if err != nil {
|
||||
@@ -465,6 +387,27 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
|
||||
}
|
||||
}
|
||||
|
||||
func generatedResultUploadReference(value map[string]any) (store.RequestAsset, map[string]any, bool) {
|
||||
upload, ok := value["upload"].(map[string]any)
|
||||
if !ok {
|
||||
return store.RequestAsset{}, nil, false
|
||||
}
|
||||
objectKey := strings.TrimSpace(stringFromAny(upload["objectKey"]))
|
||||
channel, _ := upload["storageChannel"].(map[string]any)
|
||||
channelKey := strings.TrimSpace(stringFromAny(channel["channelKey"]))
|
||||
if objectKey == "" || channelKey == "" {
|
||||
return store.RequestAsset{}, nil, false
|
||||
}
|
||||
return store.RequestAsset{
|
||||
URL: stringFromAny(upload["url"]),
|
||||
StorageProvider: stringFromAny(channel["provider"]),
|
||||
StorageChannelID: stringFromAny(channel["id"]),
|
||||
StorageChannelKey: channelKey,
|
||||
ObjectKey: objectKey,
|
||||
AccessScope: stringFromAny(upload["accessScope"]),
|
||||
}, upload, true
|
||||
}
|
||||
|
||||
func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) {
|
||||
ref, ok := value["assetRef"].(map[string]any)
|
||||
if !ok {
|
||||
@@ -475,10 +418,14 @@ func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bo
|
||||
return store.RequestAsset{}, false
|
||||
}
|
||||
asset := store.RequestAsset{
|
||||
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
|
||||
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
|
||||
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
|
||||
StorageProvider: stringFromAny(ref["storageProvider"]),
|
||||
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
|
||||
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
|
||||
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
|
||||
StorageProvider: stringFromAny(ref["storageProvider"]),
|
||||
StorageChannelID: stringFromAny(ref["storageChannelId"]),
|
||||
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
|
||||
ObjectKey: stringFromAny(ref["objectKey"]),
|
||||
AccessScope: stringFromAny(ref["accessScope"]),
|
||||
}
|
||||
if size := floatFromAny(ref["size"]); size > 0 {
|
||||
asset.ByteSize = int64(size)
|
||||
@@ -767,13 +714,6 @@ func (s *Service) localResultTTLHours() int {
|
||||
return s.cfg.LocalResultTTLHours
|
||||
}
|
||||
|
||||
func (s *Service) localResultMinFreeBytes() int64 {
|
||||
if s.cfg.LocalResultMinFreeBytes <= 0 {
|
||||
return defaultLocalResultMinFreeBytes
|
||||
}
|
||||
return s.cfg.LocalResultMinFreeBytes
|
||||
}
|
||||
|
||||
func (s *Service) localResultMaxBytes() int64 {
|
||||
if s.cfg.LocalResultMaxBytes <= 0 {
|
||||
return defaultLocalResultMaxBytes
|
||||
|
||||
@@ -31,7 +31,21 @@ func newLocalBinaryTestService(t *testing.T) *Service {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
|
||||
func writeHistoricalLocalBinaryFixture(t *testing.T, service *Service, taskID string, payload []byte) string {
|
||||
t.Helper()
|
||||
digest := sha256.Sum256(payload)
|
||||
taskDir := filepath.Join(service.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID))
|
||||
if err := os.MkdirAll(taskDir, 0o750); err != nil {
|
||||
t.Fatalf("create historical result fixture directory: %v", err)
|
||||
}
|
||||
path := filepath.Join(taskDir, hex.EncodeToString(digest[:])+".bin")
|
||||
if err := os.WriteFile(path, payload, 0o640); err != nil {
|
||||
t.Fatalf("write historical result fixture: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestHydrateHistoricalLocalBinaryResult(t *testing.T) {
|
||||
service := newLocalBinaryTestService(t)
|
||||
payload := []byte("one binary result shared across representations")
|
||||
encoded := base64.StdEncoding.EncodeToString(payload)
|
||||
@@ -49,7 +63,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input)
|
||||
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-123", input, false, true)
|
||||
if err != nil {
|
||||
t.Fatalf("materialize local binary result: %v", err)
|
||||
}
|
||||
@@ -76,6 +90,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
|
||||
if len(persistentJSON) > 32*1024 {
|
||||
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
|
||||
}
|
||||
writeHistoricalLocalBinaryFixture(t, service, "task-123", payload)
|
||||
|
||||
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
|
||||
entries, err := os.ReadDir(taskDir)
|
||||
@@ -155,19 +170,53 @@ func TestHydrateGeneratedResultAssetReference(t *testing.T) {
|
||||
assertClientErrorCode(t, err, "binary_result_corrupted")
|
||||
}
|
||||
|
||||
func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) {
|
||||
cfg := config.Config{
|
||||
MediaOSSDirectEnabled: true,
|
||||
MediaOSSEndpoint: "https://oss.example.com",
|
||||
MediaOSSBucket: "media-bucket",
|
||||
MediaOSSAccessKeyID: "access-id",
|
||||
MediaOSSAccessKeySecret: "access-secret",
|
||||
MediaOSSObjectPrefix: "media",
|
||||
}
|
||||
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
|
||||
result := map[string]any{
|
||||
"data": []any{map[string]any{
|
||||
"url": "https://expired.example.com/result.png",
|
||||
"image_url": "https://expired.example.com/result.png",
|
||||
"upload": map[string]any{
|
||||
"url": "https://expired.example.com/result.png",
|
||||
"objectKey": "media/image_result/2026/08/04/hash.png",
|
||||
"accessScope": "private",
|
||||
"storageChannel": map[string]any{
|
||||
"channelKey": "environment-direct-oss",
|
||||
"provider": "aliyun_oss",
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
hydrated, err := service.HydrateTaskResult(t.Context(), "task-private-url", result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := hydrated["data"].([]any)[0].(map[string]any)
|
||||
for _, key := range []string{"url", "image_url"} {
|
||||
value := stringFromAny(item[key])
|
||||
if !strings.Contains(value, "OSSAccessKeyId=access-id") || strings.Contains(value, "expired.example.com") {
|
||||
t.Fatalf("%s was not refreshed: %q", key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
|
||||
service := newLocalBinaryTestService(t)
|
||||
service.cfg.LocalResultTTLHours = 1
|
||||
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
|
||||
persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded})
|
||||
persistent, _, err := service.transformLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}, false, true)
|
||||
if err != nil {
|
||||
t.Fatalf("materialize fixture: %v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired"))
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("read fixture result: entries=%v err=%v", entries, err)
|
||||
}
|
||||
path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name())
|
||||
path := writeHistoricalLocalBinaryFixture(t, service, "task-expired", []byte("expiring result"))
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
if err := os.Chtimes(path, old, old); err != nil {
|
||||
t.Fatalf("age fixture: %v", err)
|
||||
@@ -186,17 +235,17 @@ func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T)
|
||||
assertClientErrorCode(t, err, "binary_result_corrupted")
|
||||
}
|
||||
|
||||
func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
|
||||
func TestHistoricalLocalBinaryFixtureEnforcesLimitsAndKeepsText(t *testing.T) {
|
||||
service := newLocalBinaryTestService(t)
|
||||
service.cfg.LocalResultMaxBytes = 4
|
||||
_, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{
|
||||
_, _, err := service.transformLocalBinaryResult(context.Background(), "task-large", map[string]any{
|
||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
|
||||
})
|
||||
}, false, true)
|
||||
assertClientErrorCode(t, err, "binary_result_too_large")
|
||||
|
||||
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{
|
||||
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-text", map[string]any{
|
||||
"message": "YWJjZA==",
|
||||
})
|
||||
}, false, true)
|
||||
if err != nil || changed || persistent["message"] != "YWJjZA==" {
|
||||
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
|
||||
}
|
||||
@@ -204,18 +253,30 @@ func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
|
||||
service = newLocalBinaryTestService(t)
|
||||
service.cfg.LocalResultMaxBytes = 1024
|
||||
service.cfg.LocalResultMaxTaskBytes = 8
|
||||
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
|
||||
_, _, err = service.transformLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
|
||||
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
|
||||
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
|
||||
})
|
||||
}, false, true)
|
||||
assertClientErrorCode(t, err, "binary_result_too_large")
|
||||
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
|
||||
func TestCompactExpiredTaskResultUsesObjectStorageAndDoesNotWriteFiles(t *testing.T) {
|
||||
var uploads int
|
||||
storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
t.Fatalf("method=%s, want PUT", r.Method)
|
||||
}
|
||||
uploads++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer storageServer.Close()
|
||||
service := newLocalBinaryTestService(t)
|
||||
service.directOSS = &directOSSUploader{
|
||||
endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
|
||||
}
|
||||
service.cfg.LocalResultMaxBytes = 1
|
||||
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
|
||||
|
||||
@@ -225,17 +286,18 @@ func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("compact expired result: %v", err)
|
||||
}
|
||||
if !changed || !localBinaryResultHasPlaceholders(persistent) {
|
||||
t.Fatalf("expired result was not compacted: %+v", persistent)
|
||||
if !changed || localBinaryResultHasPlaceholders(persistent) || TaskResultHasInlineBinary(persistent) {
|
||||
t.Fatalf("expired result was not objectified: %+v", persistent)
|
||||
}
|
||||
if uploads != 1 {
|
||||
t.Fatalf("object storage uploads=%d, want 1", uploads)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("expired compaction must not create a task directory: %v", err)
|
||||
}
|
||||
_, err = service.HydrateTaskResult(context.Background(), "task-old", persistent)
|
||||
assertClientErrorCode(t, err, "binary_result_expired")
|
||||
}
|
||||
|
||||
func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
||||
func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
||||
err := localBinaryStorageError(errors.New("write failed"))
|
||||
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
||||
if clients.IsRetryable(err) {
|
||||
@@ -247,16 +309,6 @@ func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
||||
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
|
||||
t.Fatal("local result storage failure must not fail over to another provider")
|
||||
}
|
||||
|
||||
service := newLocalBinaryTestService(t)
|
||||
service.cfg.LocalResultMinFreeBytes = 1 << 62
|
||||
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{
|
||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")),
|
||||
})
|
||||
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
||||
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("disk preflight must not create partial task files: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func bytesToAny(payload []byte) []any {
|
||||
|
||||
@@ -1,26 +1,12 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const directOSSUploadTimeout = 120 * time.Second
|
||||
|
||||
type directOSSUploader struct {
|
||||
endpoint string
|
||||
bucket string
|
||||
@@ -28,8 +14,6 @@ type directOSSUploader struct {
|
||||
accessKeySecret string
|
||||
publicBaseURL string
|
||||
objectPrefix string
|
||||
client *http.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
|
||||
@@ -43,133 +27,41 @@ func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
|
||||
accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret),
|
||||
publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"),
|
||||
objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"),
|
||||
client: &http.Client{
|
||||
Timeout: directOSSUploadTimeout,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 256,
|
||||
MaxIdleConnsPerHost: 256,
|
||||
MaxConnsPerHost: 256,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
},
|
||||
},
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func directOSSScene(scene string) bool {
|
||||
switch strings.TrimSpace(scene) {
|
||||
case store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
|
||||
case store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (u *directOSSUploader) upload(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
||||
func (u *directOSSUploader) fileStorageChannel() store.FileStorageChannel {
|
||||
if u == nil {
|
||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: "direct OSS uploader is not configured", Retryable: false}
|
||||
return store.FileStorageChannel{}
|
||||
}
|
||||
objectKey := u.objectKey(payload)
|
||||
escapedKey := escapeOSSObjectKey(objectKey)
|
||||
uploadURL := u.endpoint + "/" + escapedKey
|
||||
contentType := strings.TrimSpace(payload.ContentType)
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
return store.FileStorageChannel{
|
||||
ChannelKey: "environment-direct-oss",
|
||||
Name: "Environment Direct OSS",
|
||||
Provider: "aliyun_oss",
|
||||
AccessKeyID: u.accessKeyID,
|
||||
AccessKeySecret: u.accessKeySecret,
|
||||
Priority: 10,
|
||||
Status: "enabled",
|
||||
Scenes: []string{store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult},
|
||||
Config: map[string]any{
|
||||
"endpoint": u.endpoint,
|
||||
"bucket": u.bucket,
|
||||
"publicBaseUrl": u.publicBaseURL,
|
||||
"objectPrefix": u.objectPrefix,
|
||||
},
|
||||
RetryPolicy: map[string]any{
|
||||
"enabled": true,
|
||||
"maxRetries": 2,
|
||||
"backoffSeconds": []any{0.25, 1.0},
|
||||
},
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := sleepWithContext(ctx, time.Duration(attempt*attempt)*200*time.Millisecond); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
status, err := u.put(ctx, uploadURL, escapedKey, contentType, payload.Bytes)
|
||||
if err == nil && status >= 200 && status < 300 {
|
||||
publicURL := u.publicBaseURL + "/" + escapedKey
|
||||
return map[string]any{
|
||||
"url": publicURL,
|
||||
"fileName": objectKey,
|
||||
"storageChannel": map[string]any{
|
||||
"channelKey": "environment-direct-oss",
|
||||
"name": "Environment Direct OSS",
|
||||
"provider": "aliyun_oss_direct",
|
||||
},
|
||||
"assetStorage": map[string]any{
|
||||
"scene": payload.Scene,
|
||||
"source": firstNonEmptyString(payload.Source, "ai-gateway"),
|
||||
"strategy": "direct_aliyun_oss",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
lastErr = fmt.Errorf("HTTP %d", status)
|
||||
if status != http.StatusTooManyRequests && status < 500 {
|
||||
break
|
||||
}
|
||||
}
|
||||
message := "direct OSS upload failed"
|
||||
if lastErr != nil {
|
||||
message += ": " + lastErr.Error()
|
||||
}
|
||||
return nil, &clients.ClientError{Code: "upload_failed", Message: message, Retryable: true}
|
||||
}
|
||||
|
||||
func (u *directOSSUploader) put(ctx context.Context, uploadURL string, escapedKey string, contentType string, payload []byte) (int, error) {
|
||||
date := u.now().UTC().Format(http.TimeFormat)
|
||||
canonicalResource := "/" + u.bucket + "/" + escapedKey
|
||||
stringToSign := "PUT\n\n" + contentType + "\n" + date + "\n" + canonicalResource
|
||||
mac := hmac.New(sha1.New, []byte(u.accessKeySecret))
|
||||
_, _ = mac.Write([]byte(stringToSign))
|
||||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Set("Authorization", "OSS "+u.accessKeyID+":"+signature)
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Date", date)
|
||||
resp, err := u.client.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
|
||||
func (u *directOSSUploader) objectKey(payload FileUploadPayload) string {
|
||||
now := u.now().UTC()
|
||||
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
|
||||
baseName := strings.TrimSuffix(path.Base(payload.FileName), path.Ext(payload.FileName))
|
||||
baseName = sanitizeGeneratedAssetNamePart(baseName)
|
||||
if baseName == "" {
|
||||
baseName = "gateway-media"
|
||||
}
|
||||
if len(baseName) > 48 {
|
||||
baseName = baseName[:48]
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"%s/%s/%04d/%02d/%02d/%s-%s%s",
|
||||
u.objectPrefix,
|
||||
strings.TrimSpace(payload.Scene),
|
||||
now.Year(),
|
||||
now.Month(),
|
||||
now.Day(),
|
||||
baseName,
|
||||
randomHexSuffix(8),
|
||||
extension,
|
||||
)
|
||||
}
|
||||
|
||||
func escapeOSSObjectKey(objectKey string) string {
|
||||
parts := strings.Split(objectKey, "/")
|
||||
for index, part := range parts {
|
||||
parts[index] = url.PathEscape(part)
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
@@ -12,13 +12,12 @@ import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
|
||||
func TestDirectOSSUploadSignsRequestAndReturnsPrivateSignedURL(t *testing.T) {
|
||||
payload := []byte("production-isomorphic-image")
|
||||
var calls atomic.Int64
|
||||
var uploadedPath string
|
||||
@@ -60,9 +59,6 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
|
||||
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
|
||||
}
|
||||
uploader := newDirectOSSUploader(cfg)
|
||||
uploader.now = func() time.Time {
|
||||
return time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC)
|
||||
}
|
||||
service := &Service{cfg: cfg, directOSS: uploader}
|
||||
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
|
||||
Bytes: payload,
|
||||
@@ -77,29 +73,30 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
|
||||
if calls.Load() != 2 {
|
||||
t.Fatalf("upload calls=%d, want one retry", calls.Load())
|
||||
}
|
||||
if !strings.HasPrefix(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/2026/07/30/input-") ||
|
||||
!strings.HasSuffix(uploadedPath, ".png") {
|
||||
if !strings.Contains(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/") ||
|
||||
!strings.HasSuffix(uploadedPath, "/"+sha256Hex(payload)+".png") {
|
||||
t.Fatalf("unexpected object path=%q", uploadedPath)
|
||||
}
|
||||
if got := stringFromAny(uploaded["url"]); got != "https://cdn.example.com"+uploadedPath {
|
||||
t.Fatalf("public URL=%q", got)
|
||||
if got := stringFromAny(uploaded["url"]); !strings.HasPrefix(got, server.URL+uploadedPath+"?") || !strings.Contains(got, "Signature=") {
|
||||
t.Fatalf("private signed URL=%q", got)
|
||||
}
|
||||
if uploaded["accessScope"] != "private" {
|
||||
t.Fatalf("request asset access scope=%v", uploaded["accessScope"])
|
||||
}
|
||||
channel, _ := uploaded["storageChannel"].(map[string]any)
|
||||
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
|
||||
if stringFromAny(channel["provider"]) != "aliyun_oss" {
|
||||
t.Fatalf("storage channel=%+v", channel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
|
||||
func TestDirectOSSPersistsGeneralUploadScene(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
calls.Add(1)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
storageDir := t.TempDir()
|
||||
cfg := config.Config{
|
||||
LocalUploadedStorageDir: storageDir,
|
||||
MediaOSSDirectEnabled: true,
|
||||
MediaOSSEndpoint: server.URL,
|
||||
MediaOSSBucket: "media-bucket",
|
||||
@@ -118,11 +115,11 @@ func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("general upload: %v", err)
|
||||
}
|
||||
if calls.Load() != 0 {
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load())
|
||||
}
|
||||
channel, _ := uploaded["storageChannel"].(map[string]any)
|
||||
if stringFromAny(channel["provider"]) != "local_static" {
|
||||
if stringFromAny(channel["provider"]) != "aliyun_oss" {
|
||||
t.Fatalf("general upload channel=%+v", channel)
|
||||
}
|
||||
}
|
||||
@@ -158,7 +155,7 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
|
||||
SourceKey: "b64_json",
|
||||
},
|
||||
0,
|
||||
[]store.FileStorageChannel{{ID: "unused-channel"}},
|
||||
[]store.FileStorageChannel{service.directOSS.fileStorageChannel()},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("direct generated upload: %v", err)
|
||||
@@ -166,11 +163,11 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("direct OSS calls=%d, want 1", calls.Load())
|
||||
}
|
||||
if contentType != "image/png" || kind != "image" || strategy != "direct_aliyun_oss" {
|
||||
if contentType != "image/png" || kind != "image" || strategy != "upload_inline_media" {
|
||||
t.Fatalf("generated metadata=%s/%s/%s", contentType, kind, strategy)
|
||||
}
|
||||
channel, _ := upload["storageChannel"].(map[string]any)
|
||||
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
|
||||
if stringFromAny(channel["provider"]) != "aliyun_oss" {
|
||||
t.Fatalf("storage channel=%+v", channel)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,530 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
objectStorageRequestTimeout = 120 * time.Second
|
||||
objectStorageReadLimit = 256 << 20
|
||||
objectStorageSignedURLTTL = 15 * time.Minute
|
||||
)
|
||||
|
||||
var sharedObjectStorageHTTPClient = &http.Client{
|
||||
Timeout: objectStorageRequestTimeout,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 256,
|
||||
MaxIdleConnsPerHost: 256,
|
||||
MaxConnsPerHost: 256,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
type objectStorageAdapter struct {
|
||||
channel store.FileStorageChannel
|
||||
client *http.Client
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type FileStorageChannelTestResult struct {
|
||||
Provider string `json:"provider"`
|
||||
PutSucceeded bool `json:"putSucceeded"`
|
||||
HeadSucceeded bool `json:"headSucceeded"`
|
||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||
DurationMS int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
|
||||
// against one object-storage channel. The random probe avoids overwriting a
|
||||
// content-addressed business object and no object key or credential is exposed.
|
||||
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
|
||||
startedAt := time.Now()
|
||||
result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))}
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
payload := FileUploadPayload{
|
||||
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
|
||||
ContentType: "application/octet-stream",
|
||||
FileName: "probe.bin",
|
||||
Scene: store.FileStorageSceneUpload,
|
||||
Source: "admin-connection-test",
|
||||
}
|
||||
upload, err := adapter.put(ctx, payload)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.PutSucceeded = true
|
||||
objectKey := stringFromAny(upload["objectKey"])
|
||||
if objectKey == "" {
|
||||
return result, storageClientError("storage_write_failed", "object storage probe did not return an object reference", 0, false)
|
||||
}
|
||||
if err := adapter.head(ctx, objectKey); err != nil {
|
||||
_ = adapter.delete(context.WithoutCancel(ctx), objectKey)
|
||||
return result, err
|
||||
}
|
||||
result.HeadSucceeded = true
|
||||
if err := adapter.delete(ctx, objectKey); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.DeleteSucceeded = true
|
||||
result.DurationMS = time.Since(startedAt).Milliseconds()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAdapter, error) {
|
||||
provider := strings.ToLower(strings.TrimSpace(channel.Provider))
|
||||
if provider != "aliyun_oss" && provider != "s3" {
|
||||
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported object storage provider", Retryable: false}
|
||||
}
|
||||
if objectStorageConfigString(channel.Config, "endpoint") == "" || objectStorageConfigString(channel.Config, "bucket") == "" {
|
||||
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "object storage endpoint and bucket are required", Retryable: false}
|
||||
}
|
||||
if strings.TrimSpace(channel.AccessKeyID) == "" || strings.TrimSpace(channel.AccessKeySecret) == "" {
|
||||
return nil, &clients.ClientError{Code: "storage_auth_failed", Message: "object storage credentials are not configured", Retryable: false}
|
||||
}
|
||||
return &objectStorageAdapter{
|
||||
channel: channel,
|
||||
client: sharedObjectStorageHTTPClient,
|
||||
now: time.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
||||
objectKey := a.objectKey(payload)
|
||||
requestURL, err := a.objectURL(objectKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
contentType := strings.TrimSpace(payload.ContentType)
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload.Bytes))
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_write_failed", err.Error(), 0, true)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
if readErr != nil {
|
||||
return nil, storageClientError("storage_write_failed", readErr.Error(), resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, objectStorageHTTPError("storage_write_failed", resp.StatusCode, responseBody)
|
||||
}
|
||||
accessScope := objectStorageAccessScope(a.channel, payload.Scene)
|
||||
publicURL := ""
|
||||
if accessScope == "public" {
|
||||
publicURL = a.publicURL(objectKey)
|
||||
}
|
||||
urlExpiresAt := ""
|
||||
if publicURL == "" {
|
||||
publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339)
|
||||
}
|
||||
digest := sha256.Sum256(payload.Bytes)
|
||||
result := map[string]any{
|
||||
"url": publicURL,
|
||||
"fileName": path.Base(objectKey),
|
||||
"objectKey": objectKey,
|
||||
"contentType": contentType,
|
||||
"size": len(payload.Bytes),
|
||||
"sha256": hex.EncodeToString(digest[:]),
|
||||
"accessScope": accessScope,
|
||||
"storageChannel": map[string]any{
|
||||
"id": a.channel.ID,
|
||||
"channelKey": a.channel.ChannelKey,
|
||||
"name": a.channel.Name,
|
||||
"provider": a.channel.Provider,
|
||||
},
|
||||
}
|
||||
if urlExpiresAt != "" {
|
||||
result["urlExpiresAt"] = urlExpiresAt
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) get(ctx context.Context, objectKey string) ([]byte, error) {
|
||||
requestURL, err := a.objectURL(objectKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_read_failed", err.Error(), 0, true)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
return nil, objectStorageHTTPError("storage_read_failed", resp.StatusCode, body)
|
||||
}
|
||||
payload, err := io.ReadAll(io.LimitReader(resp.Body, objectStorageReadLimit+1))
|
||||
if err != nil {
|
||||
return nil, storageClientError("storage_read_failed", err.Error(), resp.StatusCode, true)
|
||||
}
|
||||
if len(payload) > objectStorageReadLimit {
|
||||
return nil, storageClientError("storage_read_failed", "stored object exceeds the read limit", resp.StatusCode, false)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) head(ctx context.Context, objectKey string) error {
|
||||
return a.emptyObjectRequest(ctx, http.MethodHead, objectKey)
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) delete(ctx context.Context, objectKey string) error {
|
||||
return a.emptyObjectRequest(ctx, http.MethodDelete, objectKey)
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) emptyObjectRequest(ctx context.Context, method string, objectKey string) error {
|
||||
requestURL, err := a.objectURL(objectKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, requestURL, nil)
|
||||
if err != nil {
|
||||
return storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return storageClientError("storage_read_failed", err.Error(), 0, true)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return objectStorageHTTPError("storage_read_failed", resp.StatusCode, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) presignGet(objectKey string, ttl time.Duration) (string, error) {
|
||||
requestURL, err := a.objectURL(objectKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = objectStorageSignedURLTTL
|
||||
}
|
||||
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||
return a.presignS3Get(requestURL, ttl, a.now().UTC())
|
||||
}
|
||||
return a.presignOSSGet(requestURL, objectKey, ttl, a.now().UTC())
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) objectKey(payload FileUploadPayload) string {
|
||||
now := a.now().UTC()
|
||||
digest := sha256.Sum256(payload.Bytes)
|
||||
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
|
||||
prefix := strings.Trim(objectStorageConfigString(a.channel.Config, "objectPrefix"), "/")
|
||||
parts := make([]string, 0, 7)
|
||||
if prefix != "" {
|
||||
parts = append(parts, prefix)
|
||||
}
|
||||
parts = append(parts,
|
||||
firstNonEmptyString(strings.TrimSpace(payload.Scene), store.FileStorageSceneUpload),
|
||||
fmt.Sprintf("%04d", now.Year()),
|
||||
fmt.Sprintf("%02d", now.Month()),
|
||||
fmt.Sprintf("%02d", now.Day()),
|
||||
hex.EncodeToString(digest[:])+extension,
|
||||
)
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) {
|
||||
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
|
||||
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
|
||||
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
|
||||
}
|
||||
bucket := objectStorageConfigString(a.channel.Config, "bucket")
|
||||
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||
if objectStorageConfigBool(a.channel.Config, "forcePathStyle") {
|
||||
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + bucket + "/" + strings.TrimLeft(objectKey, "/")
|
||||
} else {
|
||||
endpoint.Host = bucket + "." + endpoint.Host
|
||||
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
|
||||
}
|
||||
} else {
|
||||
hostname := endpoint.Hostname()
|
||||
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
|
||||
if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||
if port := endpoint.Port(); port != "" {
|
||||
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||
} else {
|
||||
endpoint.Host = bucket + "." + endpoint.Host
|
||||
}
|
||||
}
|
||||
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
|
||||
}
|
||||
return endpoint.String(), nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) publicURL(objectKey string) string {
|
||||
baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/")
|
||||
if baseURL == "" {
|
||||
baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/")
|
||||
}
|
||||
if baseURL == "" {
|
||||
return ""
|
||||
}
|
||||
return baseURL + "/" + escapeObjectKey(objectKey)
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now time.Time) error {
|
||||
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||
a.signS3Request(req, payloadHash, now)
|
||||
return nil
|
||||
}
|
||||
a.signOSSRequest(req, now)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
|
||||
date := now.UTC().Format(http.TimeFormat)
|
||||
contentType := req.Header.Get("Content-Type")
|
||||
canonicalHeaders := ""
|
||||
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||
req.Header.Set("x-oss-security-token", token)
|
||||
canonicalHeaders = "x-oss-security-token:" + token + "\n"
|
||||
}
|
||||
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
|
||||
stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
|
||||
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
|
||||
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
|
||||
req.Header.Set("Date", date)
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
|
||||
parsed, err := url.Parse(requestURL)
|
||||
if err != nil {
|
||||
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
expires := strconv.FormatInt(now.Add(ttl).Unix(), 10)
|
||||
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + "/" + strings.TrimLeft(objectKey, "/")
|
||||
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), "GET\n\n\n"+expires+"\n"+canonicalResource)
|
||||
query := parsed.Query()
|
||||
query.Set("OSSAccessKeyId", a.channel.AccessKeyID)
|
||||
query.Set("Expires", expires)
|
||||
query.Set("Signature", signature)
|
||||
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||
query.Set("security-token", token)
|
||||
}
|
||||
parsed.RawQuery = query.Encode()
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) signS3Request(req *http.Request, payloadHash string, now time.Time) {
|
||||
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
|
||||
amzDate := now.Format("20060102T150405Z")
|
||||
date := now.Format("20060102")
|
||||
req.Header.Set("x-amz-date", amzDate)
|
||||
req.Header.Set("x-amz-content-sha256", payloadHash)
|
||||
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||
req.Header.Set("x-amz-security-token", token)
|
||||
}
|
||||
canonicalHeaders, signedHeaders := s3CanonicalHeaders(req)
|
||||
canonicalRequest := strings.Join([]string{
|
||||
req.Method,
|
||||
s3CanonicalURI(req.URL),
|
||||
s3CanonicalQuery(req.URL.Query()),
|
||||
canonicalHeaders,
|
||||
signedHeaders,
|
||||
payloadHash,
|
||||
}, "\n")
|
||||
scope := date + "/" + region + "/s3/aws4_request"
|
||||
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
|
||||
signature := hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign))
|
||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+a.channel.AccessKeyID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+signature)
|
||||
}
|
||||
|
||||
func (a *objectStorageAdapter) presignS3Get(requestURL string, ttl time.Duration, now time.Time) (string, error) {
|
||||
parsed, err := url.Parse(requestURL)
|
||||
if err != nil {
|
||||
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||
}
|
||||
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
|
||||
amzDate := now.Format("20060102T150405Z")
|
||||
date := now.Format("20060102")
|
||||
scope := date + "/" + region + "/s3/aws4_request"
|
||||
seconds := int64(ttl / time.Second)
|
||||
if seconds < 1 {
|
||||
seconds = 1
|
||||
}
|
||||
if seconds > 7*24*60*60 {
|
||||
seconds = 7 * 24 * 60 * 60
|
||||
}
|
||||
query := parsed.Query()
|
||||
query.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
|
||||
query.Set("X-Amz-Credential", a.channel.AccessKeyID+"/"+scope)
|
||||
query.Set("X-Amz-Date", amzDate)
|
||||
query.Set("X-Amz-Expires", strconv.FormatInt(seconds, 10))
|
||||
query.Set("X-Amz-SignedHeaders", "host")
|
||||
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||
query.Set("X-Amz-Security-Token", token)
|
||||
}
|
||||
canonicalRequest := "GET\n" + s3CanonicalURI(parsed) + "\n" + s3CanonicalQuery(query) + "\nhost:" + strings.ToLower(parsed.Host) + "\n\nhost\nUNSIGNED-PAYLOAD"
|
||||
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
|
||||
query.Set("X-Amz-Signature", hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign)))
|
||||
parsed.RawQuery = query.Encode()
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
func s3CanonicalHeaders(req *http.Request) (string, string) {
|
||||
headers := map[string]string{"host": strings.ToLower(req.URL.Host)}
|
||||
for key, values := range req.Header {
|
||||
lower := strings.ToLower(strings.TrimSpace(key))
|
||||
if lower == "content-type" || strings.HasPrefix(lower, "x-amz-") {
|
||||
headers[lower] = strings.Join(values, ",")
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(headers))
|
||||
for key := range headers {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var canonical strings.Builder
|
||||
for _, key := range keys {
|
||||
canonical.WriteString(key)
|
||||
canonical.WriteByte(':')
|
||||
canonical.WriteString(strings.Join(strings.Fields(headers[key]), " "))
|
||||
canonical.WriteByte('\n')
|
||||
}
|
||||
return canonical.String(), strings.Join(keys, ";")
|
||||
}
|
||||
|
||||
func s3CanonicalURI(value *url.URL) string {
|
||||
uri := value.EscapedPath()
|
||||
if uri == "" {
|
||||
return "/"
|
||||
}
|
||||
return uri
|
||||
}
|
||||
|
||||
func s3CanonicalQuery(values url.Values) string {
|
||||
return strings.ReplaceAll(values.Encode(), "+", "%20")
|
||||
}
|
||||
|
||||
func s3SigningHMAC(secret string, date string, region string, stringToSign string) []byte {
|
||||
dateKey := hmacSHA256([]byte("AWS4"+secret), date)
|
||||
regionKey := hmacSHA256(dateKey, region)
|
||||
serviceKey := hmacSHA256(regionKey, "s3")
|
||||
signingKey := hmacSHA256(serviceKey, "aws4_request")
|
||||
return hmacSHA256(signingKey, stringToSign)
|
||||
}
|
||||
|
||||
func hmacSHA256(key []byte, value string) []byte {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
_, _ = mac.Write([]byte(value))
|
||||
return mac.Sum(nil)
|
||||
}
|
||||
|
||||
func hmacSHA1Base64(key []byte, value string) string {
|
||||
mac := hmac.New(sha1.New, key)
|
||||
_, _ = mac.Write([]byte(value))
|
||||
return base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
func sha256Hex(value []byte) string {
|
||||
digest := sha256.Sum256(value)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func objectStorageConfigString(config map[string]any, key string) string {
|
||||
if config == nil {
|
||||
return ""
|
||||
}
|
||||
value, _ := config[key].(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func objectStorageConfigBool(config map[string]any, key string) bool {
|
||||
if config == nil {
|
||||
return false
|
||||
}
|
||||
value, _ := config[key].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
|
||||
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
|
||||
return scope
|
||||
}
|
||||
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
|
||||
return "private"
|
||||
}
|
||||
if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" {
|
||||
return "public"
|
||||
}
|
||||
return "private"
|
||||
}
|
||||
|
||||
func escapeObjectKey(objectKey string) string {
|
||||
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
|
||||
for index, part := range parts {
|
||||
parts[index] = url.PathEscape(part)
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func storageStatusRetryable(status int) bool {
|
||||
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
|
||||
}
|
||||
|
||||
func objectStorageHTTPError(code string, status int, body []byte) error {
|
||||
if status == http.StatusUnauthorized || status == http.StatusForbidden {
|
||||
code = "storage_auth_failed"
|
||||
}
|
||||
message := http.StatusText(status)
|
||||
if message == "" {
|
||||
message = "object storage request failed"
|
||||
}
|
||||
// The provider response may contain object names, endpoints or credential
|
||||
// diagnostics. Keep it out of the public error chain; channel health stores
|
||||
// only the stable status description as well.
|
||||
return storageClientError(code, message, status, storageStatusRetryable(status))
|
||||
}
|
||||
|
||||
func storageClientError(code string, message string, status int, retryable bool) error {
|
||||
return &clients.ClientError{Code: code, Message: message, StatusCode: status, Retryable: retryable}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceemulator"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestLocalAcceptanceObjectStorageFailoverMatrix(t *testing.T) {
|
||||
emulator := httptest.NewServer(acceptanceemulator.New(acceptanceemulator.Config{}).Handler())
|
||||
defer emulator.Close()
|
||||
service := &Service{}
|
||||
payload := FileUploadPayload{
|
||||
Bytes: []byte("acceptance-object"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
channels []store.FileStorageChannel
|
||||
winner string
|
||||
wantErrCode string
|
||||
}{
|
||||
{
|
||||
name: "oss_auth_to_s3",
|
||||
channels: []store.FileStorageChannel{
|
||||
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-auth/bucket", "oss-auth", false),
|
||||
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3", "s3-ok", false),
|
||||
},
|
||||
winner: "s3-ok",
|
||||
},
|
||||
{
|
||||
name: "s3_auth_to_oss",
|
||||
channels: []store.FileStorageChannel{
|
||||
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-auth", "s3-auth", false),
|
||||
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-ok", false),
|
||||
},
|
||||
winner: "oss-ok",
|
||||
},
|
||||
{
|
||||
name: "same_channel_transient_retry",
|
||||
channels: []store.FileStorageChannel{
|
||||
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-transient", "s3-transient", true),
|
||||
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-unused", false),
|
||||
},
|
||||
winner: "s3-transient",
|
||||
},
|
||||
{
|
||||
name: "all_channels_failed",
|
||||
channels: []store.FileStorageChannel{
|
||||
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-fail/bucket", "oss-fail", false),
|
||||
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-fail", "s3-fail", false),
|
||||
},
|
||||
wantErrCode: "storage_write_failed",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result, err := service.uploadFileWithFailover(t.Context(), payload, test.channels)
|
||||
if test.wantErrCode != "" {
|
||||
if clients.ErrorCode(err) != test.wantErrCode {
|
||||
t.Fatalf("error=%v code=%s", err, clients.ErrorCode(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
channel, _ := result["storageChannel"].(map[string]any)
|
||||
if got := stringFromAny(channel["channelKey"]); got != test.winner {
|
||||
t.Fatalf("winner=%q, want %q", got, test.winner)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
response, err := http.Get(emulator.URL + "/report")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
var report acceptanceemulator.Report
|
||||
if err := json.NewDecoder(response.Body).Decode(&report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.StoragePuts != 3 || report.StorageFailures != 5 {
|
||||
t.Fatalf("unexpected storage acceptance report: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func acceptanceStorageChannel(provider string, endpoint string, key string, retry bool) store.FileStorageChannel {
|
||||
channel := testObjectStorageChannel(provider, endpoint, key)
|
||||
channel.Config["publicBaseUrl"] = ""
|
||||
channel.RetryPolicy = map[string]any{"enabled": retry, "maxRetries": 0}
|
||||
if retry {
|
||||
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
||||
}
|
||||
return channel
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
type objectStorageMetricEvent struct {
|
||||
event string
|
||||
provider string
|
||||
bytes int64
|
||||
}
|
||||
|
||||
type objectStorageMetricsRecorder struct {
|
||||
events []objectStorageMetricEvent
|
||||
}
|
||||
|
||||
func (m *objectStorageMetricsRecorder) ObserveBillingEvent(string) {}
|
||||
|
||||
func (m *objectStorageMetricsRecorder) ObserveObjectStorage(event string, provider string, bytes int64, _ time.Duration) {
|
||||
m.events = append(m.events, objectStorageMetricEvent{event: event, provider: provider, bytes: bytes})
|
||||
}
|
||||
|
||||
func TestS3ObjectStorageUsesSigV4AndDeterministicObjectKey(t *testing.T) {
|
||||
payload := []byte("same-media-payload")
|
||||
var calls atomic.Int64
|
||||
var requestPath string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
requestPath = r.URL.Path
|
||||
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=access-id/") {
|
||||
t.Errorf("missing SigV4 authorization: %q", r.Header.Get("Authorization"))
|
||||
}
|
||||
if r.Header.Get("x-amz-content-sha256") != sha256Hex(payload) {
|
||||
t.Errorf("payload hash=%q", r.Header.Get("x-amz-content-sha256"))
|
||||
}
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if string(body) != string(payload) {
|
||||
t.Errorf("payload=%q", body)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
channel := testObjectStorageChannel("s3", server.URL, "s3-primary")
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC) }
|
||||
result, err := adapter.put(t.Context(), FileUploadPayload{Bytes: payload, ContentType: "image/png", FileName: "ignored.jpg", Scene: store.FileStorageSceneImageResult})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantSuffix := "/bucket/media/image_result/2026/08/04/" + sha256Hex(payload) + ".png"
|
||||
if requestPath != wantSuffix {
|
||||
t.Fatalf("request path=%q, want %q", requestPath, wantSuffix)
|
||||
}
|
||||
if stringFromAny(result["objectKey"]) != strings.TrimPrefix(wantSuffix, "/bucket/") {
|
||||
t.Fatalf("object key=%q", result["objectKey"])
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("calls=%d", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
|
||||
channel.Config["publicBaseUrl"] = "https://cdn.example"
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
|
||||
|
||||
result, err := adapter.put(t.Context(), FileUploadPayload{
|
||||
Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result["accessScope"] != "private" {
|
||||
t.Fatalf("accessScope=%v, want private", result["accessScope"])
|
||||
}
|
||||
gotURL := stringFromAny(result["url"])
|
||||
if strings.HasPrefix(gotURL, "https://cdn.example/") || !strings.Contains(gotURL, "X-Amz-Signature=") {
|
||||
t.Fatalf("request asset URL must be signed and private: %s", gotURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
|
||||
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
||||
channel.Config["forcePathStyle"] = false
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := adapter.objectURL("media/request_asset/object.png")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/request_asset/object.png" {
|
||||
t.Fatalf("OSS object URL=%q", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) {
|
||||
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
||||
channel.Config["forcePathStyle"] = false
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/2026/08/04/OBJECT.png", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.Header.Set("Content-Type", "image/png")
|
||||
adapter.signOSSRequest(request, time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC))
|
||||
if got, want := request.Header.Get("Authorization"), "OSS access-id:fohlh2SG7k7D+cmX48b7Keqa1Ok="; got != want {
|
||||
t.Fatalf("authorization=%q, want %q", got, want)
|
||||
}
|
||||
if got := request.Header.Get("Date"); got != "Tue, 04 Aug 2026 01:02:03 GMT" {
|
||||
t.Fatalf("date=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
|
||||
var primaryCalls atomic.Int64
|
||||
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
primaryCalls.Add(1)
|
||||
http.Error(w, "temporary", http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer primary.Close()
|
||||
var secondaryCalls atomic.Int64
|
||||
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
secondaryCalls.Add(1)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer secondary.Close()
|
||||
|
||||
first := testObjectStorageChannel("s3", primary.URL, "s3-primary")
|
||||
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
||||
second := testObjectStorageChannel("aliyun_oss", secondary.URL, "oss-secondary")
|
||||
second.RetryPolicy = map[string]any{"enabled": false}
|
||||
metrics := &objectStorageMetricsRecorder{}
|
||||
service := &Service{billingMetrics: metrics}
|
||||
result, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
channel, _ := result["storageChannel"].(map[string]any)
|
||||
if stringFromAny(channel["channelKey"]) != "oss-secondary" {
|
||||
t.Fatalf("unexpected winning channel: %+v", channel)
|
||||
}
|
||||
if primaryCalls.Load() != 2 || secondaryCalls.Load() != 1 {
|
||||
t.Fatalf("calls primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
|
||||
}
|
||||
assertObjectStorageMetricEvent(t, metrics.events, "retry", "s3", 0)
|
||||
assertObjectStorageMetricEvent(t, metrics.events, "failover", "s3", 0)
|
||||
assertObjectStorageMetricEvent(t, metrics.events, "write_success", "aliyun_oss", int64(len("payload")))
|
||||
}
|
||||
|
||||
func TestFileStorageChannelConnectionProbeWritesHeadsAndDeletes(t *testing.T) {
|
||||
var methods []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
methods = append(methods, r.Method)
|
||||
switch r.Method {
|
||||
case http.MethodPut, http.MethodHead, http.MethodDelete:
|
||||
w.WriteHeader(http.StatusOK)
|
||||
default:
|
||||
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result, err := (&Service{}).TestFileStorageChannel(t.Context(), testObjectStorageChannel("s3", server.URL, "probe"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.PutSucceeded || !result.HeadSucceeded || !result.DeleteSucceeded || result.Provider != "s3" {
|
||||
t.Fatalf("unexpected probe result: %+v", result)
|
||||
}
|
||||
if strings.Join(methods, ",") != "PUT,HEAD,DELETE" {
|
||||
t.Fatalf("probe methods=%v", methods)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageReadUsesChannelRetryPolicy(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if calls.Add(1) == 1 {
|
||||
http.Error(w, "temporary", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("stored-payload"))
|
||||
}))
|
||||
defer server.Close()
|
||||
channel := testObjectStorageChannel("s3", server.URL, "s3-read")
|
||||
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload, err := readObjectStorageWithRetries(t.Context(), adapter, "media/request_asset/object.png")
|
||||
if err != nil || string(payload) != "stored-payload" || calls.Load() != 2 {
|
||||
t.Fatalf("payload=%q calls=%d err=%v", payload, calls.Load(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageAuthFailureSkipsSameChannelRetry(t *testing.T) {
|
||||
var primaryCalls atomic.Int64
|
||||
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
primaryCalls.Add(1)
|
||||
http.Error(w, "forbidden details", http.StatusForbidden)
|
||||
}))
|
||||
defer primary.Close()
|
||||
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||
defer secondary.Close()
|
||||
|
||||
first := testObjectStorageChannel("aliyun_oss", primary.URL, "oss-primary")
|
||||
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 3, "backoffSeconds": []any{0.001}}
|
||||
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
|
||||
service := &Service{}
|
||||
if _, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if primaryCalls.Load() != 1 {
|
||||
t.Fatalf("auth failure retried %d times", primaryCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestLevelUploadFailureDoesNotSwitchChannels(t *testing.T) {
|
||||
var primaryCalls atomic.Int64
|
||||
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
primaryCalls.Add(1)
|
||||
http.Error(w, "unsupported media", http.StatusUnsupportedMediaType)
|
||||
}))
|
||||
defer primary.Close()
|
||||
var secondaryCalls atomic.Int64
|
||||
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
secondaryCalls.Add(1)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer secondary.Close()
|
||||
|
||||
first := store.FileStorageChannel{
|
||||
ChannelKey: "server-main-primary", Provider: "server_main_openapi", UploadURL: primary.URL,
|
||||
APIKey: "test-key", RetryPolicy: map[string]any{"enabled": false},
|
||||
}
|
||||
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
|
||||
_, err := (&Service{}).uploadFileWithFailover(t.Context(), FileUploadPayload{
|
||||
Bytes: []byte("payload"), ContentType: "application/x-unsupported", Scene: store.FileStorageSceneUpload,
|
||||
}, []store.FileStorageChannel{first, second})
|
||||
if err == nil || clients.ErrorCode(err) != "upload_failed" {
|
||||
t.Fatalf("unexpected request-level error: %v", err)
|
||||
}
|
||||
if primaryCalls.Load() != 1 || secondaryCalls.Load() != 0 {
|
||||
t.Fatalf("request-level failure switched channels: primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllObjectStorageChannelsFailWithStableError(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "failure", http.StatusBadGateway) }))
|
||||
defer server.Close()
|
||||
channel := testObjectStorageChannel("s3", server.URL, "only")
|
||||
channel.RetryPolicy = map[string]any{"enabled": false}
|
||||
_, err := (&Service{}).uploadFileWithFailover(context.Background(), FileUploadPayload{Bytes: []byte("payload"), Scene: store.FileStorageSceneUpload}, []store.FileStorageChannel{channel})
|
||||
if clients.ErrorCode(err) != "storage_write_failed" || strings.Contains(err.Error(), "failure") {
|
||||
t.Fatalf("unexpected public storage error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testObjectStorageChannel(provider string, endpoint string, channelKey string) store.FileStorageChannel {
|
||||
return store.FileStorageChannel{
|
||||
ChannelKey: channelKey,
|
||||
Name: channelKey,
|
||||
Provider: provider,
|
||||
AccessKeyID: "access-id",
|
||||
AccessKeySecret: "access-secret",
|
||||
Priority: 100,
|
||||
Config: map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"bucket": "bucket",
|
||||
"region": "cn-test-1",
|
||||
"objectPrefix": "media",
|
||||
"publicBaseUrl": "https://cdn.example.com",
|
||||
"forcePathStyle": true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func assertObjectStorageMetricEvent(t *testing.T, events []objectStorageMetricEvent, event string, provider string, bytes int64) {
|
||||
t.Helper()
|
||||
for _, item := range events {
|
||||
if item.event == event && item.provider == provider && item.bytes == bytes {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing metric event %s/%s/%d in %+v", event, provider, bytes, events)
|
||||
}
|
||||
@@ -148,10 +148,14 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(payload), nil
|
||||
}
|
||||
if strings.TrimSpace(asset.URL) == "" {
|
||||
assetURL, err := s.requestAssetAccessURL(ctx, asset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(assetURL) == "" {
|
||||
return nil, requestAssetExpiredError(asset)
|
||||
}
|
||||
return asset.URL, nil
|
||||
return assetURL, nil
|
||||
}
|
||||
|
||||
func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value string, path []string, candidate store.RuntimeModelCandidate) (any, error) {
|
||||
@@ -224,10 +228,14 @@ func (s *Service) resolveRequestAsset(ctx context.Context, ref map[string]any) (
|
||||
sha := stringFromAny(ref["sha256"])
|
||||
contentType := stringFromAny(ref["contentType"])
|
||||
asset := store.RequestAsset{
|
||||
SHA256: sha,
|
||||
ContentType: contentType,
|
||||
URL: stringFromAny(ref["url"]),
|
||||
StorageProvider: stringFromAny(ref["storageProvider"]),
|
||||
SHA256: sha,
|
||||
ContentType: contentType,
|
||||
URL: stringFromAny(ref["url"]),
|
||||
StorageProvider: stringFromAny(ref["storageProvider"]),
|
||||
StorageChannelID: stringFromAny(ref["storageChannelId"]),
|
||||
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
|
||||
ObjectKey: stringFromAny(ref["objectKey"]),
|
||||
AccessScope: stringFromAny(ref["accessScope"]),
|
||||
}
|
||||
if size := floatFromAny(ref["size"]); size > 0 {
|
||||
asset.ByteSize = int64(size)
|
||||
@@ -254,6 +262,17 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
|
||||
if requestAssetIsExpired(asset, time.Now()) {
|
||||
return nil, requestAssetExpiredError(asset)
|
||||
}
|
||||
if strings.TrimSpace(asset.ObjectKey) != "" {
|
||||
channel, err := s.fileStorageChannelForAsset(ctx, asset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return readObjectStorageWithRetries(ctx, adapter, asset.ObjectKey)
|
||||
}
|
||||
if strings.TrimSpace(asset.LocalPath) != "" {
|
||||
payload, err := os.ReadFile(asset.LocalPath)
|
||||
if err != nil {
|
||||
@@ -290,6 +309,60 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
|
||||
return nil, requestAssetExpiredError(asset)
|
||||
}
|
||||
|
||||
func readObjectStorageWithRetries(ctx context.Context, adapter *objectStorageAdapter, objectKey string) ([]byte, error) {
|
||||
maxRetries, delays := uploadRetrySchedule(adapter.channel.RetryPolicy, adapter.channel.Provider)
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
payload, err := adapter.get(ctx, objectKey)
|
||||
if err == nil {
|
||||
return payload, nil
|
||||
}
|
||||
lastErr = err
|
||||
if attempt >= maxRetries || !clients.IsRetryable(err) {
|
||||
break
|
||||
}
|
||||
if err := sleepWithContext(ctx, retryDelayForAttempt(attempt, delays)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func (s *Service) requestAssetAccessURL(ctx context.Context, asset store.RequestAsset) (string, error) {
|
||||
if strings.TrimSpace(asset.ObjectKey) == "" {
|
||||
return asset.URL, nil
|
||||
}
|
||||
channel, err := s.fileStorageChannelForAsset(ctx, asset)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(asset.AccessScope), "public") {
|
||||
if value := adapter.publicURL(asset.ObjectKey); value != "" {
|
||||
return value, nil
|
||||
}
|
||||
}
|
||||
return adapter.presignGet(asset.ObjectKey, objectStorageSignedURLTTL)
|
||||
}
|
||||
|
||||
func (s *Service) fileStorageChannelForAsset(ctx context.Context, asset store.RequestAsset) (store.FileStorageChannel, error) {
|
||||
channelKey := strings.TrimSpace(asset.StorageChannelKey)
|
||||
if channelKey == "environment-direct-oss" && s.directOSS != nil {
|
||||
return s.directOSS.fileStorageChannel(), nil
|
||||
}
|
||||
if s.store == nil || channelKey == "" {
|
||||
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
|
||||
}
|
||||
channel, err := s.store.GetFileStorageChannelByKey(ctx, channelKey)
|
||||
if err != nil {
|
||||
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
|
||||
}
|
||||
return channel, nil
|
||||
}
|
||||
|
||||
func (s *Service) localPathFromRequestAssetURL(value string) string {
|
||||
raw := strings.TrimSpace(value)
|
||||
if raw == "" {
|
||||
|
||||
@@ -230,6 +230,15 @@ func (s *Service) observeTaskEventSkip(reason string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) observeObjectStorage(event string, provider string, bytes int, duration time.Duration) {
|
||||
observer, ok := s.billingMetrics.(interface {
|
||||
ObserveObjectStorage(string, string, int64, time.Duration)
|
||||
})
|
||||
if ok {
|
||||
observer.ObserveObjectStorage(event, provider, int64(bytes), duration)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
|
||||
return s.execute(ctx, task, user, nil)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -74,13 +75,19 @@ func (s *Service) processTaskCallbackBatch(ctx context.Context, client *http.Cli
|
||||
}
|
||||
|
||||
func deliverTaskCallback(ctx context.Context, client *http.Client, item store.TaskCallbackDelivery, bearerToken string) (int, error) {
|
||||
body, err := json.Marshal(map[string]any{
|
||||
payload := map[string]any{
|
||||
"taskId": item.TaskID,
|
||||
"seq": item.Seq,
|
||||
"eventType": item.EventType,
|
||||
"status": item.TaskStatus,
|
||||
"createdAt": item.CreatedAt.UTC().Format(time.RFC3339Nano),
|
||||
})
|
||||
}
|
||||
if item.TaskStatus == "failed" || item.TaskStatus == "cancelled" {
|
||||
standard := publicerror.WithIDs(publicerror.FromFields(item.TaskErrorCode, item.TaskErrorMessage, 0, false), item.TaskRequestID, item.TaskID)
|
||||
publicerror.Observe(standard)
|
||||
payload["error"] = standard
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -116,3 +117,33 @@ func TestDeliverTaskCallbackSendsMinimalAuthenticatedBody(t *testing.T) {
|
||||
t.Fatalf("callback body=%#v", received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliverFailedTaskCallbackUsesStandardErrorWithoutSocketDetails(t *testing.T) {
|
||||
var received map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
if err := json.NewDecoder(request.Body).Decode(&received); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
|
||||
status, err := deliverTaskCallback(t.Context(), server.Client(), store.TaskCallbackDelivery{
|
||||
TaskID: "failed-task",
|
||||
Seq: 4,
|
||||
CallbackURL: server.URL,
|
||||
EventType: "task.failed",
|
||||
TaskStatus: "failed",
|
||||
TaskErrorCode: "response_read_error",
|
||||
TaskErrorMessage: raw,
|
||||
CreatedAt: time.Now(),
|
||||
}, "")
|
||||
if err != nil || status != http.StatusNoContent {
|
||||
t.Fatalf("status=%d err=%v", status, err)
|
||||
}
|
||||
errorValue, _ := received["error"].(map[string]any)
|
||||
if errorValue["code"] != "upstream_connection_interrupted" || strings.Contains(stringFromAny(errorValue["message"]), "10.42.0.72") {
|
||||
t.Fatalf("unsafe callback error: %#v", errorValue)
|
||||
}
|
||||
}
|
||||
|
||||
+139
-151
@@ -17,25 +17,18 @@ import (
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const defaultServerMainOpenAPIUploadURL = "http://127.0.0.1:3001/v1/files/upload"
|
||||
const maxGeneratedAssetFetchBytes = 256 << 20
|
||||
|
||||
const (
|
||||
localStaticGeneratedPathPrefix = "/static/generated/"
|
||||
localStaticUploadedPathPrefix = "/static/uploaded/"
|
||||
)
|
||||
|
||||
type FileUploadPayload struct {
|
||||
ContentType string
|
||||
FileName string
|
||||
@@ -136,14 +129,6 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
||||
return nil, &clients.ClientError{Code: "acceptance_run_inactive", Message: err.Error(), Retryable: false}
|
||||
}
|
||||
}
|
||||
if policy.LocalizeInlineMedia {
|
||||
next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
redactGeneratedResultRawData(next)
|
||||
return next, nil
|
||||
}
|
||||
if len(data) == 0 && !rawNeedsUpload {
|
||||
return s.finalizeGeneratedAssets(ctx, taskID, taskKind, result, policy, nil, false, 0)
|
||||
}
|
||||
@@ -179,11 +164,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
||||
var channels []store.FileStorageChannel
|
||||
channelsLoaded := false
|
||||
if needsUpload || rawNeedsUpload {
|
||||
if s.directOSS == nil {
|
||||
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
|
||||
if err != nil {
|
||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
||||
}
|
||||
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
|
||||
if err != nil {
|
||||
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "object storage configuration is unavailable", Retryable: true}
|
||||
}
|
||||
channelsLoaded = true
|
||||
}
|
||||
@@ -323,12 +306,12 @@ func (s *Service) finalizeGeneratedAssets(
|
||||
if !TaskResultHasInlineBinary(next) {
|
||||
return next, nil
|
||||
}
|
||||
persistent, _, err := s.materializeLocalBinaryResult(ctx, taskID, next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, &clients.ClientError{
|
||||
Code: "storage_write_failed",
|
||||
Message: "generated binary result could not be written to object storage",
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Retryable: true,
|
||||
}
|
||||
redactGeneratedResultRawData(persistent)
|
||||
return persistent, nil
|
||||
}
|
||||
|
||||
func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
|
||||
@@ -355,6 +338,9 @@ func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]
|
||||
func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
if payload, contentType, ok := localBufferObjectBytes(typed); ok {
|
||||
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, contentType, firstNonEmptyString(key, "buffer"), siblings, channels, index)
|
||||
}
|
||||
next := make(map[string]any, len(typed))
|
||||
changed := false
|
||||
for childKey, childValue := range typed {
|
||||
@@ -372,6 +358,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
|
||||
}
|
||||
return value, false, nil
|
||||
case []any:
|
||||
if localBinaryKey(key) {
|
||||
if payload, ok := bytesFromNumberArray(typed); ok {
|
||||
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
|
||||
}
|
||||
}
|
||||
next := make([]any, len(typed))
|
||||
changed := false
|
||||
for itemIndex, item := range typed {
|
||||
@@ -388,6 +379,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
|
||||
return next, true, nil
|
||||
}
|
||||
return value, false, nil
|
||||
case []byte:
|
||||
if len(typed) == 0 {
|
||||
return value, false, nil
|
||||
}
|
||||
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
|
||||
case string:
|
||||
asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind)
|
||||
if !ok {
|
||||
@@ -404,19 +400,36 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) uploadGeneratedBinaryValue(ctx context.Context, taskID string, taskKind string, payload []byte, contentType string, sourceKey string, siblings map[string]any, channels []store.FileStorageChannel, index *int) (any, bool, error) {
|
||||
contentType = firstNonEmptyString(contentType, defaultContentTypeForRawMediaKey(sourceKey))
|
||||
asset := &generatedInlineAsset{
|
||||
Bytes: payload,
|
||||
ContentType: contentType,
|
||||
Kind: mediaKindForAsset(taskKind, siblings, sourceKey, contentType),
|
||||
SourceKey: sourceKey,
|
||||
}
|
||||
upload, resolvedContentType, kind, strategy, err := s.uploadGeneratedAsset(ctx, taskID, asset, *index, channels)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
*index = *index + 1
|
||||
return generatedRawMediaReference(asset, upload, resolvedContentType, kind, strategy), true, nil
|
||||
}
|
||||
|
||||
func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) {
|
||||
raw := strings.TrimSpace(value)
|
||||
if raw == "" {
|
||||
return nil, false
|
||||
}
|
||||
keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
|
||||
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
|
||||
if !generatedRawDataMediaPayloadKey(key) && !generatedContentTypeIsMedia(contentType) {
|
||||
if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) {
|
||||
return nil, false
|
||||
}
|
||||
if !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
|
||||
if !keyLooksLikeMediaPayload && !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
|
||||
return nil, false
|
||||
}
|
||||
payload, payloadContentType, ok, err := inlineMediaPayload(raw, generatedRawDataMediaPayloadKey(key))
|
||||
payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload)
|
||||
if err != nil || !ok || len(payload) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
@@ -454,6 +467,18 @@ func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]a
|
||||
if provider := stringFromAny(channel["provider"]); provider != "" {
|
||||
ref["storageProvider"] = provider
|
||||
}
|
||||
if id := stringFromAny(channel["id"]); id != "" {
|
||||
ref["storageChannelId"] = id
|
||||
}
|
||||
if key := stringFromAny(channel["channelKey"]); key != "" {
|
||||
ref["storageChannelKey"] = key
|
||||
}
|
||||
}
|
||||
if objectKey := stringFromAny(upload["objectKey"]); objectKey != "" {
|
||||
ref["objectKey"] = objectKey
|
||||
}
|
||||
if accessScope := stringFromAny(upload["accessScope"]); accessScope != "" {
|
||||
ref["accessScope"] = accessScope
|
||||
}
|
||||
out := map[string]any{
|
||||
"assetRef": ref,
|
||||
@@ -650,6 +675,9 @@ func removeASCIIWhitespace(value string) string {
|
||||
}
|
||||
|
||||
func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
|
||||
if s.store == nil {
|
||||
return defaultGeneratedAssetUploadPolicy(), nil
|
||||
}
|
||||
settings, err := s.store.GetFileStorageSettings(ctx)
|
||||
if err != nil {
|
||||
if store.IsUndefinedDatabaseObject(err) {
|
||||
@@ -665,8 +693,6 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
|
||||
switch policyName {
|
||||
case store.FileStorageResultUploadPolicyUploadAll:
|
||||
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
|
||||
case store.FileStorageResultUploadPolicyUploadNone:
|
||||
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
|
||||
default:
|
||||
return defaultGeneratedAssetUploadPolicy()
|
||||
}
|
||||
@@ -682,13 +708,8 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset
|
||||
Scene: store.FileStorageSceneImageResult,
|
||||
Source: "ai-gateway",
|
||||
}
|
||||
if s.directOSS != nil {
|
||||
upload, err := s.directOSS.upload(ctx, payload)
|
||||
return upload, contentType, kind, "direct_aliyun_oss", err
|
||||
}
|
||||
if len(channels) == 0 {
|
||||
upload, err := s.storeFileLocally(payload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
|
||||
return upload, contentType, kind, "local_static_inline_media", err
|
||||
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
upload, err := s.uploadFileWithFailover(ctx, payload, channels)
|
||||
return upload, contentType, kind, "upload_inline_media", err
|
||||
@@ -708,95 +729,13 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as
|
||||
Scene: store.FileStorageSceneImageResult,
|
||||
Source: "ai-gateway",
|
||||
}
|
||||
if s.directOSS != nil {
|
||||
upload, err := s.directOSS.upload(ctx, uploadPayload)
|
||||
return upload, contentType, kind, "direct_aliyun_oss", err
|
||||
}
|
||||
if len(channels) == 0 {
|
||||
upload, err := s.storeFileLocally(uploadPayload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
|
||||
return upload, contentType, kind, "local_static_url_media", err
|
||||
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels)
|
||||
return upload, contentType, kind, "upload_url_media", err
|
||||
}
|
||||
|
||||
func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string, fallbackStorageDir string, pathPrefix string) (map[string]any, error) {
|
||||
storageDir = strings.TrimSpace(storageDir)
|
||||
if storageDir == "" {
|
||||
storageDir = fallbackStorageDir
|
||||
}
|
||||
if err := os.MkdirAll(storageDir, 0o755); err != nil {
|
||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
|
||||
}
|
||||
fileName := filepath.Base(strings.TrimSpace(payload.FileName))
|
||||
if fileName == "" || fileName == "." || fileName == ".." || fileName == string(filepath.Separator) {
|
||||
kind := generatedAssetKindFromContentType("", payload.ContentType)
|
||||
fileName = generatedAssetFileName("generated", 0, payload.ContentType, kind)
|
||||
}
|
||||
targetPath := filepath.Join(storageDir, fileName)
|
||||
file, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
|
||||
}
|
||||
_, writeErr := file.Write(payload.Bytes)
|
||||
closeErr := file.Close()
|
||||
if writeErr != nil {
|
||||
_ = os.Remove(targetPath)
|
||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: writeErr.Error(), Retryable: true}
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(targetPath)
|
||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true}
|
||||
}
|
||||
expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339)
|
||||
return map[string]any{
|
||||
"url": s.localStaticFileURL(fileName, pathPrefix),
|
||||
"fileName": fileName,
|
||||
"contentType": payload.ContentType,
|
||||
"size": len(payload.Bytes),
|
||||
"expiresAt": expiresAt,
|
||||
"storageChannel": map[string]any{
|
||||
"id": "local-static",
|
||||
"channelKey": "local-static",
|
||||
"name": "AI Gateway local static storage",
|
||||
"provider": "local_static",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) localStaticAssetTTLHours() int {
|
||||
if s.cfg.LocalTempAssetTTLHours <= 0 {
|
||||
return 24
|
||||
}
|
||||
return s.cfg.LocalTempAssetTTLHours
|
||||
}
|
||||
|
||||
func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string {
|
||||
if strings.TrimSpace(pathPrefix) == "" {
|
||||
pathPrefix = localStaticUploadedPathPrefix
|
||||
}
|
||||
path := pathPrefix + url.PathEscape(filepath.Base(fileName))
|
||||
baseURL := strings.TrimRight(strings.TrimSpace(s.cfg.PublicBaseURL), "/")
|
||||
if baseURL == "" {
|
||||
return path
|
||||
}
|
||||
return baseURL + path
|
||||
}
|
||||
|
||||
func localStaticUploadFileName(originalName string, contentType string) string {
|
||||
baseName := filepath.Base(strings.TrimSpace(originalName))
|
||||
originalExt := strings.ToLower(filepath.Ext(baseName))
|
||||
namePart := strings.TrimSuffix(baseName, originalExt)
|
||||
namePart = sanitizeGeneratedAssetNamePart(namePart)
|
||||
if namePart == "" {
|
||||
namePart = "gateway-upload"
|
||||
}
|
||||
if len(namePart) > 48 {
|
||||
namePart = namePart[:48]
|
||||
}
|
||||
return fmt.Sprintf("%s-%s%s", namePart, randomHexSuffix(6), uploadFileExtension(contentType, originalExt))
|
||||
}
|
||||
|
||||
func uploadFileExtension(contentType string, fallbackExt string) string {
|
||||
normalized := normalizeGeneratedContentType(contentType)
|
||||
if generatedContentTypeIsMedia(normalized) {
|
||||
@@ -981,46 +920,41 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma
|
||||
if strings.TrimSpace(payload.Scene) == "" {
|
||||
payload.Scene = store.FileStorageSceneUpload
|
||||
}
|
||||
if s.directOSS != nil && directOSSScene(payload.Scene) {
|
||||
return s.directOSS.upload(ctx, payload)
|
||||
}
|
||||
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
|
||||
if err != nil {
|
||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
||||
}
|
||||
if len(channels) == 0 {
|
||||
payload.FileName = localStaticUploadFileName(payload.FileName, payload.ContentType)
|
||||
upload, err := s.storeFileLocally(payload, s.cfg.LocalUploadedStorageDir, config.DefaultLocalUploadedStorageDir, localStaticUploadedPathPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upload["assetStorage"] = map[string]any{
|
||||
"scene": payload.Scene,
|
||||
"source": firstNonEmptyString(payload.Source, "ai-gateway-openapi"),
|
||||
"strategy": "local_static_upload",
|
||||
}
|
||||
return upload, nil
|
||||
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
return s.uploadFileWithFailover(ctx, payload, channels)
|
||||
}
|
||||
|
||||
func (s *Service) activeFileStorageChannels(ctx context.Context, scene string) ([]store.FileStorageChannel, error) {
|
||||
if s.store == nil {
|
||||
return nil, nil
|
||||
channels := make([]store.FileStorageChannel, 0)
|
||||
if s.directOSS != nil && directOSSScene(scene) {
|
||||
channels = append(channels, s.directOSS.fileStorageChannel())
|
||||
}
|
||||
channels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
|
||||
if s.store == nil {
|
||||
return channels, nil
|
||||
}
|
||||
storedChannels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
|
||||
if err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||
return nil, err
|
||||
}
|
||||
if len(channels) > 0 {
|
||||
return channels, nil
|
||||
}
|
||||
return nil, nil
|
||||
channels = append(channels, storedChannels...)
|
||||
sort.SliceStable(channels, func(i, j int) bool {
|
||||
if channels[i].Priority != channels[j].Priority {
|
||||
return channels[i].Priority < channels[j].Priority
|
||||
}
|
||||
return channels[i].ChannelKey < channels[j].ChannelKey
|
||||
})
|
||||
return channels, nil
|
||||
}
|
||||
|
||||
func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUploadPayload, channels []store.FileStorageChannel) (map[string]any, error) {
|
||||
var lastErr error
|
||||
for _, channel := range channels {
|
||||
for index, channel := range channels {
|
||||
upload, err := s.uploadWithChannelRetries(ctx, payload, channel)
|
||||
if err == nil {
|
||||
if s.store != nil {
|
||||
@@ -1032,25 +966,59 @@ func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUpload
|
||||
if s.store != nil {
|
||||
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(ctx), channel.ID, err.Error())
|
||||
}
|
||||
if !storageFailureAllowsFailover(channel, err) {
|
||||
return nil, err
|
||||
}
|
||||
if index+1 < len(channels) {
|
||||
s.observeObjectStorage("failover", channel.Provider, 0, 0)
|
||||
}
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
s.observeObjectStorage("all_failed", "", 0, 0)
|
||||
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "all configured object storage channels failed", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||
}
|
||||
|
||||
func storageFailureAllowsFailover(channel store.FileStorageChannel, err error) bool {
|
||||
code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
|
||||
switch code {
|
||||
case "upload_source_too_large", "upload_decode_failed", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio":
|
||||
return false
|
||||
}
|
||||
var clientErr *clients.ClientError
|
||||
if !errors.As(err, &clientErr) {
|
||||
return true
|
||||
}
|
||||
switch clientErr.StatusCode {
|
||||
case http.StatusRequestEntityTooLarge, http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity:
|
||||
return false
|
||||
case http.StatusBadRequest:
|
||||
// Object-storage 400 responses commonly indicate endpoint/signature
|
||||
// configuration and should move to the next channel. server-main 400 is
|
||||
// the existing upload API's request validation result.
|
||||
return !strings.EqualFold(strings.TrimSpace(channel.Provider), "server_main_openapi")
|
||||
default:
|
||||
return true
|
||||
}
|
||||
return nil, &clients.ClientError{Code: "upload_no_channel", Message: "no enabled file storage channel", Retryable: false}
|
||||
}
|
||||
|
||||
func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
||||
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy)
|
||||
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy, channel.Provider)
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
startedAt := time.Now()
|
||||
upload, err := s.uploadOnce(ctx, payload, channel)
|
||||
if err == nil {
|
||||
s.observeObjectStorage("write_success", channel.Provider, len(payload.Bytes), time.Since(startedAt))
|
||||
return upload, nil
|
||||
}
|
||||
s.observeObjectStorage("write_failure", channel.Provider, 0, time.Since(startedAt))
|
||||
lastErr = err
|
||||
if attempt >= maxRetries || !clients.IsRetryable(err) {
|
||||
break
|
||||
}
|
||||
s.observeObjectStorage("retry", channel.Provider, 0, 0)
|
||||
delay := retryDelayForAttempt(attempt, delays)
|
||||
if err := sleepWithContext(ctx, delay); err != nil {
|
||||
return nil, err
|
||||
@@ -1060,9 +1028,21 @@ func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUplo
|
||||
}
|
||||
|
||||
func (s *Service) uploadOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
||||
if strings.ToLower(strings.TrimSpace(channel.Provider)) != "server_main_openapi" {
|
||||
return nil, &clients.ClientError{Code: "upload_unsupported_channel", Message: "unsupported file storage channel: " + channel.Provider, Retryable: false}
|
||||
switch strings.ToLower(strings.TrimSpace(channel.Provider)) {
|
||||
case "aliyun_oss", "s3":
|
||||
adapter, err := newObjectStorageAdapter(channel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return adapter.put(ctx, payload)
|
||||
case "server_main_openapi":
|
||||
return s.uploadServerMainOnce(ctx, payload, channel)
|
||||
default:
|
||||
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported file storage channel", Retryable: false}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) uploadServerMainOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
||||
uploadURL := strings.TrimSpace(channel.UploadURL)
|
||||
if uploadURL == "" {
|
||||
uploadURL = defaultServerMainOpenAPIUploadURL
|
||||
@@ -1700,9 +1680,17 @@ func uniqueStringList(values []string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func uploadRetrySchedule(policy map[string]any) (int, []time.Duration) {
|
||||
func uploadRetrySchedule(policy map[string]any, providers ...string) (int, []time.Duration) {
|
||||
provider := ""
|
||||
if len(providers) > 0 {
|
||||
provider = providers[0]
|
||||
}
|
||||
if policy == nil {
|
||||
policy = defaultUploadRetryPolicy()
|
||||
if strings.EqualFold(provider, "aliyun_oss") || strings.EqualFold(provider, "s3") {
|
||||
policy = map[string]any{"enabled": true, "maxRetries": 2, "backoffSeconds": []any{0.25, 1.0}}
|
||||
} else {
|
||||
policy = defaultUploadRetryPolicy()
|
||||
}
|
||||
}
|
||||
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
|
||||
return 0, nil
|
||||
@@ -1725,9 +1713,9 @@ func uploadRetryDelays(value any) []time.Duration {
|
||||
}
|
||||
delays := make([]time.Duration, 0, len(items))
|
||||
for _, item := range items {
|
||||
seconds := int(floatFromAny(item))
|
||||
seconds := floatFromAny(item)
|
||||
if seconds > 0 {
|
||||
delays = append(delays, time.Duration(seconds)*time.Second)
|
||||
delays = append(delays, time.Duration(seconds*float64(time.Second)))
|
||||
}
|
||||
}
|
||||
return delays
|
||||
|
||||
@@ -7,11 +7,11 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
@@ -227,38 +227,6 @@ func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedAssetDecisionPreservesInlineWhenPolicyUploadNone(t *testing.T) {
|
||||
item := map[string]any{
|
||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
|
||||
}
|
||||
|
||||
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if decision.Inline != nil || decision.URL != nil {
|
||||
t.Fatalf("upload_none should not transfer inline payloads: %+v", decision)
|
||||
}
|
||||
if len(decision.StripKeys) != 0 {
|
||||
t.Fatalf("upload_none should preserve b64_json for the caller: %+v", decision.StripKeys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedAssetDecisionPreservesInlineAlongsideURLWhenPolicyUploadNone(t *testing.T) {
|
||||
item := map[string]any{
|
||||
"url": "https://cdn.example.com/generated.png",
|
||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
|
||||
}
|
||||
|
||||
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if decision.Inline != nil || decision.URL != nil || len(decision.StripKeys) != 0 {
|
||||
t.Fatalf("upload_none should preserve the upstream URL and base64 fields unchanged: %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -276,9 +244,9 @@ func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
|
||||
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true, PreserveInlineMedia: false},
|
||||
},
|
||||
{
|
||||
name: "upload none",
|
||||
name: "legacy upload none becomes default",
|
||||
policyName: store.FileStorageResultUploadPolicyUploadNone,
|
||||
want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true},
|
||||
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: false, PreserveInlineMedia: false},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -330,8 +298,10 @@ func TestAcceptanceGeneratedMediaAllowsOnlyExactEmulatorOrigin(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||
defer server.Close()
|
||||
service := &Service{}
|
||||
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
|
||||
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
|
||||
encoded := base64.StdEncoding.EncodeToString(payload)
|
||||
result := map[string]any{
|
||||
@@ -348,7 +318,7 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
|
||||
"images.edits",
|
||||
result,
|
||||
defaultGeneratedAssetUploadPolicy(),
|
||||
nil,
|
||||
channels,
|
||||
true,
|
||||
0,
|
||||
)
|
||||
@@ -363,15 +333,8 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
|
||||
if !ok {
|
||||
t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested)
|
||||
}
|
||||
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-nested-binary-01-") {
|
||||
t.Fatalf("unexpected local static URL: %s", urlValue)
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read generated storage: %v", err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("expected one localized result file, got %d", len(entries))
|
||||
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") {
|
||||
t.Fatalf("unexpected object storage URL: %s", urlValue)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,9 +368,8 @@ func TestGeneratedAssetFileNameIsUniqueAndTyped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
||||
func TestUploadGeneratedAssetFailsWithoutObjectStorage(t *testing.T) {
|
||||
service := &Service{}
|
||||
payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
|
||||
asset := &generatedInlineAsset{
|
||||
Bytes: payload,
|
||||
@@ -416,44 +378,14 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
|
||||
SourceKey: "b64_json",
|
||||
}
|
||||
|
||||
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
|
||||
if err != nil {
|
||||
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
|
||||
if clients.ErrorCode(err) != "storage_write_failed" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if contentType != "image/png" || kind != "image" || strategy != "local_static_inline_media" {
|
||||
t.Fatalf("unexpected local upload metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
|
||||
}
|
||||
urlValue := stringFromAny(upload["url"])
|
||||
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-123-01-") || !strings.HasSuffix(urlValue, ".png") {
|
||||
t.Fatalf("unexpected local static URL: %s", urlValue)
|
||||
}
|
||||
expiresAt, err := time.Parse(time.RFC3339, stringFromAny(upload["expiresAt"]))
|
||||
if err != nil {
|
||||
t.Fatalf("local static upload should expose expiresAt: %+v", upload)
|
||||
}
|
||||
remaining := time.Until(expiresAt)
|
||||
if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour+time.Minute {
|
||||
t.Fatalf("local static upload should expire after one day, remaining=%s", remaining)
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read local static dir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
|
||||
t.Fatalf("expected one PNG file in local static dir, got %+v", entries)
|
||||
}
|
||||
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read local static file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(stored, payload) {
|
||||
t.Fatalf("stored payload does not match source payload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
||||
func TestUploadGeneratedAudioFailsWithoutObjectStorage(t *testing.T) {
|
||||
service := &Service{}
|
||||
asset := &generatedInlineAsset{
|
||||
Bytes: []byte("inline audio payload"),
|
||||
ContentType: "audio/mpeg",
|
||||
@@ -461,32 +393,17 @@ func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
|
||||
SourceKey: "content",
|
||||
}
|
||||
|
||||
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
|
||||
if err != nil {
|
||||
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
|
||||
if clients.ErrorCode(err) != "storage_write_failed" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if contentType != "audio/mpeg" || kind != "audio" || strategy != "local_static_inline_media" {
|
||||
t.Fatalf("unexpected local audio metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
|
||||
}
|
||||
urlValue := stringFromAny(upload["url"])
|
||||
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-tts-01-") || !strings.HasSuffix(urlValue, ".mp3") {
|
||||
t.Fatalf("unexpected local audio URL: %s", urlValue)
|
||||
}
|
||||
if stringFromAny(upload["expiresAt"]) == "" {
|
||||
t.Fatalf("local audio static upload should expose expiresAt: %+v", upload)
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read local static dir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".mp3") {
|
||||
t.Fatalf("expected one MP3 file in local static dir, got %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||
defer server.Close()
|
||||
service := &Service{}
|
||||
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
|
||||
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
|
||||
raw := map[string]any{
|
||||
"candidates": []any{
|
||||
@@ -506,7 +423,7 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
|
||||
}
|
||||
index := 0
|
||||
|
||||
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), nil, &index)
|
||||
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
|
||||
if err != nil {
|
||||
t.Fatalf("upload raw media: %v", err)
|
||||
}
|
||||
@@ -528,22 +445,47 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
|
||||
if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) {
|
||||
t.Fatalf("unexpected asset ref: %+v", ref)
|
||||
}
|
||||
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-raw-01-") || !strings.HasSuffix(urlValue, ".png") {
|
||||
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") || !strings.HasSuffix(urlValue, ".png") {
|
||||
t.Fatalf("unexpected raw media URL: %s", urlValue)
|
||||
}
|
||||
if inlineData["data"] == base64.StdEncoding.EncodeToString(payload) {
|
||||
t.Fatal("raw inlineData still contains base64 payload")
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read generated storage: %v", err)
|
||||
}
|
||||
|
||||
func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||
defer server.Close()
|
||||
service := &Service{}
|
||||
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-buffer-result")}
|
||||
raw := map[string]any{
|
||||
"buffer": map[string]any{
|
||||
"type": "Buffer", "mimeType": "image/png", "data": []any{float64(0x89), float64('P'), float64('N'), float64('G')},
|
||||
},
|
||||
"audio_bytes": []any{float64('I'), float64('D'), float64('3')},
|
||||
"direct": []byte("direct bytes"),
|
||||
}
|
||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
|
||||
t.Fatalf("expected one generated PNG, got %+v", entries)
|
||||
index := 0
|
||||
uploaded, changed, err := service.uploadGeneratedRawMediaValue(t.Context(), "task-buffer", "images.generations", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !changed || index != 3 {
|
||||
t.Fatalf("changed=%v uploads=%d", changed, index)
|
||||
}
|
||||
next := uploaded.(map[string]any)
|
||||
for _, key := range []string{"buffer", "audio_bytes", "direct"} {
|
||||
item, ok := next[key].(map[string]any)
|
||||
if !ok || item["assetRef"] == nil || item["upload"] == nil {
|
||||
t.Fatalf("%s was not objectified: %#v", key, next[key])
|
||||
}
|
||||
}
|
||||
if TaskResultHasInlineBinary(next) {
|
||||
t.Fatalf("objectified result still contains inline binary: %#v", next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
|
||||
func TestUploadFileFailsWithoutObjectStorageAndCreatesNoLocalFile(t *testing.T) {
|
||||
storageDir := t.TempDir()
|
||||
service := &Service{cfg: config.Config{
|
||||
LocalUploadedStorageDir: storageDir,
|
||||
@@ -552,43 +494,21 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
|
||||
}}
|
||||
payload := []byte("%PDF-1.4")
|
||||
|
||||
upload, err := service.UploadFile(context.Background(), FileUploadPayload{
|
||||
_, err := service.UploadFile(context.Background(), FileUploadPayload{
|
||||
Bytes: payload,
|
||||
ContentType: "application/pdf",
|
||||
FileName: "用户文件.png",
|
||||
Source: "playground",
|
||||
})
|
||||
if err != nil {
|
||||
if clients.ErrorCode(err) != "storage_write_failed" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
urlValue := stringFromAny(upload["url"])
|
||||
if !strings.HasPrefix(urlValue, "/static/uploaded/") || !strings.HasSuffix(urlValue, ".pdf") {
|
||||
t.Fatalf("unexpected uploaded local static URL: %s", urlValue)
|
||||
}
|
||||
if stringFromAny(upload["expiresAt"]) == "" {
|
||||
t.Fatalf("local uploaded static file should expose expiresAt: %+v", upload)
|
||||
}
|
||||
storageChannel, _ := upload["storageChannel"].(map[string]any)
|
||||
if stringFromAny(storageChannel["provider"]) != "local_static" {
|
||||
t.Fatalf("expected local static provider metadata, got %+v", upload["storageChannel"])
|
||||
}
|
||||
assetStorage, _ := upload["assetStorage"].(map[string]any)
|
||||
if stringFromAny(assetStorage["strategy"]) != "local_static_upload" || stringFromAny(assetStorage["scene"]) != store.FileStorageSceneUpload {
|
||||
t.Fatalf("unexpected upload asset storage metadata: %+v", assetStorage)
|
||||
}
|
||||
entries, err := os.ReadDir(storageDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read uploaded static dir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".pdf") {
|
||||
t.Fatalf("expected one PDF file in uploaded static dir, got %+v", entries)
|
||||
}
|
||||
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read uploaded static file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(stored, payload) {
|
||||
t.Fatalf("stored uploaded payload does not match source payload")
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("unexpected local files: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user