wangbo
0f0998cbcf
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。 将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。 引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。 验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
315 lines
12 KiB
Go
315 lines
12 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
type objectStorageMetricEvent struct {
|
|
event string
|
|
provider string
|
|
bytes int64
|
|
}
|
|
|
|
type objectStorageMetricsRecorder struct {
|
|
events []objectStorageMetricEvent
|
|
}
|
|
|
|
func (m *objectStorageMetricsRecorder) ObserveBillingEvent(string) {}
|
|
|
|
func (m *objectStorageMetricsRecorder) ObserveObjectStorage(event string, provider string, bytes int64, _ time.Duration) {
|
|
m.events = append(m.events, objectStorageMetricEvent{event: event, provider: provider, bytes: bytes})
|
|
}
|
|
|
|
func TestS3ObjectStorageUsesSigV4AndDeterministicObjectKey(t *testing.T) {
|
|
payload := []byte("same-media-payload")
|
|
var calls atomic.Int64
|
|
var requestPath string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
requestPath = r.URL.Path
|
|
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=access-id/") {
|
|
t.Errorf("missing SigV4 authorization: %q", r.Header.Get("Authorization"))
|
|
}
|
|
if r.Header.Get("x-amz-content-sha256") != sha256Hex(payload) {
|
|
t.Errorf("payload hash=%q", r.Header.Get("x-amz-content-sha256"))
|
|
}
|
|
body, _ := io.ReadAll(r.Body)
|
|
if string(body) != string(payload) {
|
|
t.Errorf("payload=%q", body)
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer server.Close()
|
|
|
|
channel := testObjectStorageChannel("s3", server.URL, "s3-primary")
|
|
adapter, err := newObjectStorageAdapter(channel)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC) }
|
|
result, err := adapter.put(t.Context(), FileUploadPayload{Bytes: payload, ContentType: "image/png", FileName: "ignored.jpg", Scene: store.FileStorageSceneImageResult})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantSuffix := "/bucket/media/image_result/2026/08/04/" + sha256Hex(payload) + ".png"
|
|
if requestPath != wantSuffix {
|
|
t.Fatalf("request path=%q, want %q", requestPath, wantSuffix)
|
|
}
|
|
if stringFromAny(result["objectKey"]) != strings.TrimPrefix(wantSuffix, "/bucket/") {
|
|
t.Fatalf("object key=%q", result["objectKey"])
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("calls=%d", calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer server.Close()
|
|
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
|
|
channel.Config["publicBaseUrl"] = "https://cdn.example"
|
|
adapter, err := newObjectStorageAdapter(channel)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
|
|
|
|
result, err := adapter.put(t.Context(), FileUploadPayload{
|
|
Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result["accessScope"] != "private" {
|
|
t.Fatalf("accessScope=%v, want private", result["accessScope"])
|
|
}
|
|
gotURL := stringFromAny(result["url"])
|
|
if strings.HasPrefix(gotURL, "https://cdn.example/") || !strings.Contains(gotURL, "X-Amz-Signature=") {
|
|
t.Fatalf("request asset URL must be signed and private: %s", gotURL)
|
|
}
|
|
}
|
|
|
|
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
|
|
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
|
channel.Config["forcePathStyle"] = false
|
|
adapter, err := newObjectStorageAdapter(channel)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value, err := adapter.objectURL("media/request_asset/object.png")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if value != "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/request_asset/object.png" {
|
|
t.Fatalf("OSS object URL=%q", value)
|
|
}
|
|
}
|
|
|
|
func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) {
|
|
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
|
channel.Config["forcePathStyle"] = false
|
|
adapter, err := newObjectStorageAdapter(channel)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/2026/08/04/OBJECT.png", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request.Header.Set("Content-Type", "image/png")
|
|
adapter.signOSSRequest(request, time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC))
|
|
if got, want := request.Header.Get("Authorization"), "OSS access-id:fohlh2SG7k7D+cmX48b7Keqa1Ok="; got != want {
|
|
t.Fatalf("authorization=%q, want %q", got, want)
|
|
}
|
|
if got := request.Header.Get("Date"); got != "Tue, 04 Aug 2026 01:02:03 GMT" {
|
|
t.Fatalf("date=%q", got)
|
|
}
|
|
}
|
|
|
|
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
|
|
var primaryCalls atomic.Int64
|
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
primaryCalls.Add(1)
|
|
http.Error(w, "temporary", http.StatusServiceUnavailable)
|
|
}))
|
|
defer primary.Close()
|
|
var secondaryCalls atomic.Int64
|
|
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
secondaryCalls.Add(1)
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer secondary.Close()
|
|
|
|
first := testObjectStorageChannel("s3", primary.URL, "s3-primary")
|
|
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
|
second := testObjectStorageChannel("aliyun_oss", secondary.URL, "oss-secondary")
|
|
second.RetryPolicy = map[string]any{"enabled": false}
|
|
metrics := &objectStorageMetricsRecorder{}
|
|
service := &Service{billingMetrics: metrics}
|
|
result, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
channel, _ := result["storageChannel"].(map[string]any)
|
|
if stringFromAny(channel["channelKey"]) != "oss-secondary" {
|
|
t.Fatalf("unexpected winning channel: %+v", channel)
|
|
}
|
|
if primaryCalls.Load() != 2 || secondaryCalls.Load() != 1 {
|
|
t.Fatalf("calls primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
|
|
}
|
|
assertObjectStorageMetricEvent(t, metrics.events, "retry", "s3", 0)
|
|
assertObjectStorageMetricEvent(t, metrics.events, "failover", "s3", 0)
|
|
assertObjectStorageMetricEvent(t, metrics.events, "write_success", "aliyun_oss", int64(len("payload")))
|
|
}
|
|
|
|
func TestFileStorageChannelConnectionProbeWritesHeadsAndDeletes(t *testing.T) {
|
|
var methods []string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
methods = append(methods, r.Method)
|
|
switch r.Method {
|
|
case http.MethodPut, http.MethodHead, http.MethodDelete:
|
|
w.WriteHeader(http.StatusOK)
|
|
default:
|
|
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
result, err := (&Service{}).TestFileStorageChannel(t.Context(), testObjectStorageChannel("s3", server.URL, "probe"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.PutSucceeded || !result.HeadSucceeded || !result.DeleteSucceeded || result.Provider != "s3" {
|
|
t.Fatalf("unexpected probe result: %+v", result)
|
|
}
|
|
if strings.Join(methods, ",") != "PUT,HEAD,DELETE" {
|
|
t.Fatalf("probe methods=%v", methods)
|
|
}
|
|
}
|
|
|
|
func TestObjectStorageReadUsesChannelRetryPolicy(t *testing.T) {
|
|
var calls atomic.Int64
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if calls.Add(1) == 1 {
|
|
http.Error(w, "temporary", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte("stored-payload"))
|
|
}))
|
|
defer server.Close()
|
|
channel := testObjectStorageChannel("s3", server.URL, "s3-read")
|
|
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
|
adapter, err := newObjectStorageAdapter(channel)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
payload, err := readObjectStorageWithRetries(t.Context(), adapter, "media/request_asset/object.png")
|
|
if err != nil || string(payload) != "stored-payload" || calls.Load() != 2 {
|
|
t.Fatalf("payload=%q calls=%d err=%v", payload, calls.Load(), err)
|
|
}
|
|
}
|
|
|
|
func TestObjectStorageAuthFailureSkipsSameChannelRetry(t *testing.T) {
|
|
var primaryCalls atomic.Int64
|
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
primaryCalls.Add(1)
|
|
http.Error(w, "forbidden details", http.StatusForbidden)
|
|
}))
|
|
defer primary.Close()
|
|
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
|
defer secondary.Close()
|
|
|
|
first := testObjectStorageChannel("aliyun_oss", primary.URL, "oss-primary")
|
|
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 3, "backoffSeconds": []any{0.001}}
|
|
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
|
|
service := &Service{}
|
|
if _, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if primaryCalls.Load() != 1 {
|
|
t.Fatalf("auth failure retried %d times", primaryCalls.Load())
|
|
}
|
|
}
|
|
|
|
func TestRequestLevelUploadFailureDoesNotSwitchChannels(t *testing.T) {
|
|
var primaryCalls atomic.Int64
|
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
primaryCalls.Add(1)
|
|
http.Error(w, "unsupported media", http.StatusUnsupportedMediaType)
|
|
}))
|
|
defer primary.Close()
|
|
var secondaryCalls atomic.Int64
|
|
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
secondaryCalls.Add(1)
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer secondary.Close()
|
|
|
|
first := store.FileStorageChannel{
|
|
ChannelKey: "server-main-primary", Provider: "server_main_openapi", UploadURL: primary.URL,
|
|
APIKey: "test-key", RetryPolicy: map[string]any{"enabled": false},
|
|
}
|
|
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
|
|
_, err := (&Service{}).uploadFileWithFailover(t.Context(), FileUploadPayload{
|
|
Bytes: []byte("payload"), ContentType: "application/x-unsupported", Scene: store.FileStorageSceneUpload,
|
|
}, []store.FileStorageChannel{first, second})
|
|
if err == nil || clients.ErrorCode(err) != "upload_failed" {
|
|
t.Fatalf("unexpected request-level error: %v", err)
|
|
}
|
|
if primaryCalls.Load() != 1 || secondaryCalls.Load() != 0 {
|
|
t.Fatalf("request-level failure switched channels: primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
|
|
}
|
|
}
|
|
|
|
func TestAllObjectStorageChannelsFailWithStableError(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "failure", http.StatusBadGateway) }))
|
|
defer server.Close()
|
|
channel := testObjectStorageChannel("s3", server.URL, "only")
|
|
channel.RetryPolicy = map[string]any{"enabled": false}
|
|
_, err := (&Service{}).uploadFileWithFailover(context.Background(), FileUploadPayload{Bytes: []byte("payload"), Scene: store.FileStorageSceneUpload}, []store.FileStorageChannel{channel})
|
|
if clients.ErrorCode(err) != "storage_write_failed" || strings.Contains(err.Error(), "failure") {
|
|
t.Fatalf("unexpected public storage error: %v", err)
|
|
}
|
|
}
|
|
|
|
func testObjectStorageChannel(provider string, endpoint string, channelKey string) store.FileStorageChannel {
|
|
return store.FileStorageChannel{
|
|
ChannelKey: channelKey,
|
|
Name: channelKey,
|
|
Provider: provider,
|
|
AccessKeyID: "access-id",
|
|
AccessKeySecret: "access-secret",
|
|
Priority: 100,
|
|
Config: map[string]any{
|
|
"endpoint": endpoint,
|
|
"bucket": "bucket",
|
|
"region": "cn-test-1",
|
|
"objectPrefix": "media",
|
|
"publicBaseUrl": "https://cdn.example.com",
|
|
"forcePathStyle": true,
|
|
},
|
|
}
|
|
}
|
|
|
|
func assertObjectStorageMetricEvent(t *testing.T, events []objectStorageMetricEvent, event string, provider string, bytes int64) {
|
|
t.Helper()
|
|
for _, item := range events {
|
|
if item.event == event && item.provider == provider && item.bytes == bytes {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("missing metric event %s/%s/%d in %+v", event, provider, bytes, events)
|
|
}
|