diff --git a/apps/api/cmd/acceptance-bootstrap/main.go b/apps/api/cmd/acceptance-bootstrap/main.go index 63ab1b3..ab9507f 100644 --- a/apps/api/cmd/acceptance-bootstrap/main.go +++ b/apps/api/cmd/acceptance-bootstrap/main.go @@ -161,6 +161,9 @@ func run(ctx context.Context, opts options) error { if err := ensureAcceptanceAccessRules(ctx, database, groupID); err != nil { return err } + if err := ensureAcceptanceObjectStorage(ctx, database, opts.emulatorBaseURL); err != nil { + return err + } geminiModel, videoModel, err := selectedModels(ctx, database) if err != nil { return err @@ -240,6 +243,73 @@ func run(ctx context.Context, opts options) error { return nil } +func ensureAcceptanceObjectStorage(ctx context.Context, database *store.Store, emulatorBaseURL string) error { + baseURL := strings.TrimRight(strings.TrimSpace(emulatorBaseURL), "/") + if baseURL == "" { + return errors.New("acceptance object storage requires the emulator URL") + } + channels := []struct { + key string + name string + provider string + endpoint string + priority int + pathStyle bool + }{ + { + key: "local-acceptance-oss", name: "Local Acceptance Aliyun OSS", provider: "aliyun_oss", + endpoint: baseURL + "/storage/oss/bucket", priority: 10, pathStyle: true, + }, + { + key: "local-acceptance-s3", name: "Local Acceptance S3", provider: "s3", + endpoint: baseURL + "/storage/s3", priority: 20, pathStyle: true, + }, + } + for _, channel := range channels { + _, err := database.Pool().Exec(ctx, ` +INSERT INTO file_storage_channels ( + channel_key, name, provider, credentials, config, retry_policy, priority, status +) +VALUES ( + $1, $2, $3, + '{"accessKeyId":"local-acceptance","accessKeySecret":"local-acceptance-secret"}'::jsonb, + jsonb_build_object( + 'endpoint', $4::text, + 'region', 'local-acceptance-1', + 'bucket', 'bucket', + 'objectPrefix', 'acceptance/media', + 'accessScope', 'private', + 'forcePathStyle', $5::boolean, + 'scenes', jsonb_build_array('upload', 'image_result', 'request_asset'), + 'acceptanceEmulatorOnly', true + ), + '{"enabled":true,"maxRetries":2,"backoffSeconds":[0.25,1],"strategy":"exponential"}'::jsonb, + $6, 'enabled' +) +ON CONFLICT (channel_key) DO UPDATE +SET name = EXCLUDED.name, + provider = EXCLUDED.provider, + credentials = EXCLUDED.credentials, + config = EXCLUDED.config, + retry_policy = EXCLUDED.retry_policy, + priority = EXCLUDED.priority, + status = 'enabled', + deleted_at = NULL, + last_error = NULL, + updated_at = now()`, channel.key, channel.name, channel.provider, channel.endpoint, channel.pathStyle, channel.priority) + if err != nil { + return fmt.Errorf("configure %s object storage channel: %w", channel.provider, err) + } + } + _, err := database.Pool().Exec(ctx, ` +INSERT INTO system_settings (setting_key, value) +VALUES ('file_storage', '{"resultUploadPolicy":"default"}'::jsonb) +ON CONFLICT (setting_key) DO UPDATE +SET value = jsonb_set(COALESCE(system_settings.value, '{}'::jsonb), '{resultUploadPolicy}', '"default"'::jsonb, true), + updated_at = now()`) + return err +} + func resetPreviousLocalRun(ctx context.Context, database *store.Store, opts options) error { mode, err := database.GetGatewayTrafficMode(ctx) if err != nil { diff --git a/apps/api/docs/swagger.json b/apps/api/docs/swagger.json index 6624ddb..e8a1159 100644 --- a/apps/api/docs/swagger.json +++ b/apps/api/docs/swagger.json @@ -3157,7 +3157,7 @@ "BearerAuth": [] } ], - "description": "创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。", + "description": "创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。", "consumes": [ "application/json" ], @@ -3354,6 +3354,70 @@ } } }, + "/api/admin/system/file-storage/channels/{channelID}/test": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "测试对象存储通道", + "parameters": [ + { + "type": "string", + "description": "文件存储通道 ID", + "name": "channelID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/httpapi.FileStorageChannelTestResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + } + } + } + }, "/api/admin/system/file-storage/settings": { "get": { "security": [ @@ -6712,7 +6776,7 @@ "BearerAuth": [] } ], - "description": "上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。", + "description": "上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。", "consumes": [ "multipart/form-data" ], @@ -11207,22 +11271,52 @@ "httpapi.ErrorPayload": { "type": "object", "properties": { + "action": { + "type": "string", + "example": "retry_after" + }, + "category": { + "type": "string", + "example": "rate_limit" + }, "code": { "type": "string", "example": "rate_limit" }, + "httpStatus": { + "type": "integer", + "example": 429 + }, "message": { "type": "string", "example": "invalid json body" }, "param": {}, + "requestId": { + "type": "string" + }, + "retryAfterSeconds": { + "type": "integer", + "example": 2 + }, + "retryable": { + "type": "boolean", + "example": true + }, "status": { "type": "integer", "example": 400 }, + "taskId": { + "type": "string" + }, "type": { "type": "string", "example": "invalid_request_error" + }, + "version": { + "type": "string", + "example": "v1" } } }, @@ -11237,6 +11331,26 @@ } } }, + "httpapi.FileStorageChannelTestResponse": { + "type": "object", + "properties": { + "deleteSucceeded": { + "type": "boolean" + }, + "durationMs": { + "type": "integer" + }, + "headSucceeded": { + "type": "boolean" + }, + "provider": { + "type": "string" + }, + "putSucceeded": { + "type": "boolean" + } + } + }, "httpapi.FileUploadData": { "type": "object", "properties": { @@ -11443,6 +11557,9 @@ "example": 0 }, "data": {}, + "error": { + "$ref": "#/definitions/publicerror.Error" + }, "message": { "type": "string", "example": "SUCCEED" @@ -11620,6 +11737,9 @@ "type": "string", "example": "invalid parameter" }, + "public_error": { + "$ref": "#/definitions/publicerror.Error" + }, "request_id": { "type": "string" } @@ -12809,20 +12929,7 @@ "type": "object", "properties": { "error": { - "$ref": "#/definitions/httpapi.VolcesErrorPayload" - } - } - }, - "httpapi.VolcesErrorPayload": { - "type": "object", - "properties": { - "code": { - "type": "string", - "example": "invalid_parameter" - }, - "message": { - "type": "string", - "example": "model is required" + "$ref": "#/definitions/publicerror.Error" } } }, @@ -13323,6 +13430,41 @@ "RevisionFailed" ] }, + "publicerror.Error": { + "type": "object", + "properties": { + "action": { + "type": "string" + }, + "category": { + "type": "string" + }, + "code": { + "type": "string" + }, + "httpStatus": { + "type": "integer" + }, + "message": { + "type": "string" + }, + "requestId": { + "type": "string" + }, + "retryAfterSeconds": { + "type": "integer" + }, + "retryable": { + "type": "boolean" + }, + "taskId": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, "runner.PortraitAssetCapability": { "type": "object", "properties": { @@ -13835,6 +13977,9 @@ "type": "object", "additionalProperties": {} }, + "publicError": { + "$ref": "#/definitions/publicerror.Error" + }, "remoteTaskId": { "type": "string" }, @@ -14771,6 +14916,15 @@ "store.FileStorageChannelInput": { "type": "object", "properties": { + "accessKey": { + "type": "string" + }, + "accessKeyId": { + "type": "string" + }, + "accessKeySecret": { + "type": "string" + }, "apiKey": { "type": "string" }, @@ -14800,6 +14954,12 @@ "type": "string" } }, + "secretKey": { + "type": "string" + }, + "sessionToken": { + "type": "string" + }, "status": { "type": "string" }, @@ -14971,6 +15131,9 @@ "type": "object", "additionalProperties": {} }, + "publicError": { + "$ref": "#/definitions/publicerror.Error" + }, "remoteTaskId": { "type": "string" }, @@ -16540,6 +16703,9 @@ "providerModelName": { "type": "string" }, + "publicError": { + "$ref": "#/definitions/publicerror.Error" + }, "queueKey": { "type": "string" }, diff --git a/apps/api/docs/swagger.yaml b/apps/api/docs/swagger.yaml index 83e7061..ced6113 100644 --- a/apps/api/docs/swagger.yaml +++ b/apps/api/docs/swagger.yaml @@ -481,19 +481,41 @@ definitions: type: object httpapi.ErrorPayload: properties: + action: + example: retry_after + type: string + category: + example: rate_limit + type: string code: example: rate_limit type: string + httpStatus: + example: 429 + type: integer message: example: invalid json body type: string param: {} + requestId: + type: string + retryAfterSeconds: + example: 2 + type: integer + retryable: + example: true + type: boolean status: example: 400 type: integer + taskId: + type: string type: example: invalid_request_error type: string + version: + example: v1 + type: string type: object httpapi.FileStorageChannelListResponse: properties: @@ -502,6 +524,19 @@ definitions: $ref: '#/definitions/store.FileStorageChannel' type: array type: object + httpapi.FileStorageChannelTestResponse: + properties: + deleteSucceeded: + type: boolean + durationMs: + type: integer + headSucceeded: + type: boolean + provider: + type: string + putSucceeded: + type: boolean + type: object httpapi.FileUploadData: properties: assetStorage: @@ -651,6 +686,8 @@ definitions: example: 0 type: integer data: {} + error: + $ref: '#/definitions/publicerror.Error' message: example: SUCCEED type: string @@ -775,6 +812,8 @@ definitions: message: example: invalid parameter type: string + public_error: + $ref: '#/definitions/publicerror.Error' request_id: type: string type: object @@ -1602,16 +1641,7 @@ definitions: httpapi.VolcesErrorEnvelope: properties: error: - $ref: '#/definitions/httpapi.VolcesErrorPayload' - type: object - httpapi.VolcesErrorPayload: - properties: - code: - example: invalid_parameter - type: string - message: - example: model is required - type: string + $ref: '#/definitions/publicerror.Error' type: object httpapi.WalletAdjustmentResponse: properties: @@ -1953,6 +1983,29 @@ definitions: - RevisionActive - RevisionSuperseded - RevisionFailed + publicerror.Error: + properties: + action: + type: string + category: + type: string + code: + type: string + httpStatus: + type: integer + message: + type: string + requestId: + type: string + retryAfterSeconds: + type: integer + retryable: + type: boolean + taskId: + type: string + version: + type: string + type: object runner.PortraitAssetCapability: properties: availablePlatformIds: @@ -2297,6 +2350,8 @@ definitions: pricingSnapshot: additionalProperties: {} type: object + publicError: + $ref: '#/definitions/publicerror.Error' remoteTaskId: type: string request: @@ -2929,6 +2984,12 @@ definitions: type: object store.FileStorageChannelInput: properties: + accessKey: + type: string + accessKeyId: + type: string + accessKeySecret: + type: string apiKey: type: string channelKey: @@ -2949,6 +3010,10 @@ definitions: items: type: string type: array + secretKey: + type: string + sessionToken: + type: string status: type: string uploadUrl: @@ -3064,6 +3129,8 @@ definitions: pricingSnapshot: additionalProperties: {} type: object + publicError: + $ref: '#/definitions/publicerror.Error' remoteTaskId: type: string request: @@ -4125,6 +4192,8 @@ definitions: type: string providerModelName: type: string + publicError: + $ref: '#/definitions/publicerror.Error' queueKey: type: string requestFingerprint: @@ -6358,7 +6427,7 @@ paths: post: consumes: - application/json - description: 创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。 + description: 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。 parameters: - description: 文件存储通道 in: body @@ -6485,6 +6554,47 @@ paths: summary: 更新文件存储通道 tags: - system + /api/admin/system/file-storage/channels/{channelID}/test: + post: + description: 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。 + parameters: + - description: 文件存储通道 ID + in: path + name: channelID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/httpapi.FileStorageChannelTestResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "404": + description: Not Found + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + security: + - BearerAuth: [] + summary: 测试对象存储通道 + tags: + - system /api/admin/system/file-storage/settings: get: description: 返回文件存储系统设置；数据库对象尚未创建时返回默认设置。 @@ -8652,7 +8762,7 @@ paths: post: consumes: - multipart/form-data - description: 上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。 + description: 上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。 parameters: - description: 要上传的文件 in: formData diff --git a/apps/api/internal/acceptanceemulator/server.go b/apps/api/internal/acceptanceemulator/server.go index 609095e..18e7151 100644 --- a/apps/api/internal/acceptanceemulator/server.go +++ b/apps/api/internal/acceptanceemulator/server.go @@ -51,6 +51,8 @@ type Server struct { callbacks map[string]map[int64]int videoByIdempotency map[string]string geminiIdempotency map[string]struct{} + storageObjects map[string]fixture + storageAttempts map[string]int pngSmall string pngLarge string pngPeak string @@ -75,6 +77,11 @@ type Report struct { DuplicateCallbacks int64 `json:"duplicateCallbacks"` MissingIdempotency int64 `json:"missingIdempotencyKeys"` DuplicateSubmissions int64 `json:"duplicateSubmissionAttempts"` + StoragePuts int64 `json:"storagePuts"` + StorageGets int64 `json:"storageGets"` + StorageHeads int64 `json:"storageHeads"` + StorageDeletes int64 `json:"storageDeletes"` + StorageFailures int64 `json:"storageFailures"` } type videoTask struct { @@ -117,6 +124,8 @@ func New(config Config) *Server { callbacks: map[string]map[int64]int{}, videoByIdempotency: map[string]string{}, geminiIdempotency: map[string]struct{}{}, + storageObjects: map[string]fixture{}, + storageAttempts: map[string]int{}, report: Report{ VideoReferenceCounts: map[string]int64{}, VideoRoleCounts: map[string]int64{}, @@ -139,9 +148,107 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /media/{asset}", s.getMedia) mux.HandleFunc("GET /fixtures/{asset}", s.getFixture) mux.HandleFunc("POST /callbacks", s.collectCallback) + mux.HandleFunc("PUT /storage/{profile}/{object...}", s.objectStorage) + mux.HandleFunc("GET /storage/{profile}/{object...}", s.objectStorage) + mux.HandleFunc("HEAD /storage/{profile}/{object...}", s.objectStorage) + mux.HandleFunc("DELETE /storage/{profile}/{object...}", s.objectStorage) return mux } +func (s *Server) objectStorage(w http.ResponseWriter, r *http.Request) { + profile := strings.ToLower(strings.TrimSpace(r.PathValue("profile"))) + object := strings.TrimLeft(strings.TrimSpace(r.PathValue("object")), "/") + if object == "" { + http.NotFound(w, r) + return + } + baseProfile := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(profile, "-transient"), "-auth"), "-fail") + if baseProfile != "oss" && baseProfile != "s3" { + http.NotFound(w, r) + return + } + key := profile + "/" + object + s.mu.Lock() + s.storageAttempts[key]++ + attempt := s.storageAttempts[key] + if strings.HasSuffix(profile, "-auth") { + s.report.StorageFailures++ + s.mu.Unlock() + http.Error(w, "forbidden", http.StatusForbidden) + return + } + if strings.HasSuffix(profile, "-fail") || (strings.HasSuffix(profile, "-transient") && attempt == 1) { + s.report.StorageFailures++ + s.mu.Unlock() + http.Error(w, "temporary storage failure", http.StatusServiceUnavailable) + return + } + switch r.Method { + case http.MethodPut: + s.mu.Unlock() + payload, err := io.ReadAll(io.LimitReader(r.Body, maxProtocolBodyBytes+1)) + if err != nil || len(payload) > maxProtocolBodyBytes { + http.Error(w, "invalid object", http.StatusBadRequest) + return + } + s.mu.Lock() + s.storageObjects[key] = fixture{ContentType: firstNonEmpty(r.Header.Get("Content-Type"), "application/octet-stream"), Payload: payload} + s.report.StoragePuts++ + s.mu.Unlock() + w.WriteHeader(http.StatusOK) + case http.MethodGet: + item, ok := s.storageObjects[key] + if ok { + s.report.StorageGets++ + } + s.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", item.ContentType) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(item.Payload) + case http.MethodHead: + item, ok := s.storageObjects[key] + if ok { + s.report.StorageHeads++ + } + s.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", item.ContentType) + w.Header().Set("Content-Length", strconv.Itoa(len(item.Payload))) + w.WriteHeader(http.StatusOK) + case http.MethodDelete: + _, ok := s.storageObjects[key] + delete(s.storageObjects, key) + if ok { + s.report.StorageDeletes++ + } + s.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusNoContent) + default: + s.mu.Unlock() + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + func (s *Server) health(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]any{"ok": true}) } diff --git a/apps/api/internal/acceptanceemulator/server_test.go b/apps/api/internal/acceptanceemulator/server_test.go index 0e21f7b..7696423 100644 --- a/apps/api/internal/acceptanceemulator/server_test.go +++ b/apps/api/internal/acceptanceemulator/server_test.go @@ -315,6 +315,86 @@ func TestVolcesProtocolAcceptsThreeSixAndNineReferenceImages(t *testing.T) { } } +func TestObjectStorageEmulationSupportsRetryLifecycleAndFaults(t *testing.T) { + server := httptest.NewServer(New(Config{}).Handler()) + defer server.Close() + objectURL := server.URL + "/storage/s3-transient/bucket/media/result.png" + + request, _ := http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes")) + request.Header.Set("Content-Type", "image/png") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("first transient PUT status=%d", response.StatusCode) + } + request, _ = http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes")) + request.Header.Set("Content-Type", "image/png") + response, err = http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("retried PUT status=%d", response.StatusCode) + } + + response, err = http.Get(objectURL) + if err != nil { + t.Fatal(err) + } + payload, _ := io.ReadAll(response.Body) + _ = response.Body.Close() + if response.StatusCode != http.StatusOK || string(payload) != "image-bytes" { + t.Fatalf("GET status=%d payload=%q", response.StatusCode, payload) + } + request, _ = http.NewRequest(http.MethodHead, objectURL, nil) + response, err = http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("HEAD status=%d", response.StatusCode) + } + request, _ = http.NewRequest(http.MethodDelete, objectURL, nil) + response, err = http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusNoContent { + t.Fatalf("DELETE status=%d", response.StatusCode) + } + + for profile, wantStatus := range map[string]int{"oss-auth": http.StatusForbidden, "s3-fail": http.StatusServiceUnavailable} { + request, _ = http.NewRequest(http.MethodPut, server.URL+"/storage/"+profile+"/probe.bin", strings.NewReader("probe")) + response, err = http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != wantStatus { + t.Fatalf("%s status=%d, want %d", profile, response.StatusCode, wantStatus) + } + } + + response, err = http.Get(server.URL + "/report") + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var report Report + if err := json.NewDecoder(response.Body).Decode(&report); err != nil { + t.Fatal(err) + } + if report.StoragePuts != 1 || report.StorageGets != 1 || report.StorageHeads != 1 || report.StorageDeletes != 1 || report.StorageFailures != 3 { + t.Fatalf("unexpected storage report: %+v", report) + } +} + func postIdempotent(url string, body []byte, key string) (*http.Response, error) { request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) if err != nil { diff --git a/apps/api/internal/httpapi/compat_protocol.go b/apps/api/internal/httpapi/compat_protocol.go index 0d61785..bd88faf 100644 --- a/apps/api/internal/httpapi/compat_protocol.go +++ b/apps/api/internal/httpapi/compat_protocol.go @@ -9,6 +9,7 @@ import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -97,11 +98,16 @@ func writeWireResponse(w http.ResponseWriter, wire *clients.WireResponse) { } func writeProtocolError(w http.ResponseWriter, protocol string, status int, message string, details map[string]any, code string) { + standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500) + standard = publicErrorWithRetryAfter(standard, details) + publicerror.Observe(standard) + status, message, code = standard.HTTPStatus, standard.Message, standard.Code + details = safePublicErrorDetails(details, standard, true) switch protocol { case clients.ProtocolGeminiGenerateContent: writeGeminiError(w, status, message, details, code) case clients.ProtocolVolcesContents: - writeVolcesError(w, status, message, code) + writeVolcesPublicError(w, standard) case clients.ProtocolKlingV1Omni, clients.ProtocolKlingV2Omni: writeKelingCompatError(w, "", newKelingCompatError(status, kelingCompatBusinessCode(code, message), message)) default: @@ -185,11 +191,11 @@ func googleRPCStatus(status int, code string) string { } func writeVolcesError(w http.ResponseWriter, status int, message string, code string) { - if strings.TrimSpace(code) == "" { - code = http.StatusText(status) - } - writeJSON(w, status, map[string]any{"error": map[string]any{ - "code": code, - "message": message, - }}) + standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500) + publicerror.Observe(standard) + writeVolcesPublicError(w, standard) +} + +func writeVolcesPublicError(w http.ResponseWriter, standard publicerror.Error) { + writeJSON(w, standard.HTTPStatus, map[string]any{"error": standard}) } diff --git a/apps/api/internal/httpapi/compat_protocol_test.go b/apps/api/internal/httpapi/compat_protocol_test.go index a49f6fc..0d17be4 100644 --- a/apps/api/internal/httpapi/compat_protocol_test.go +++ b/apps/api/internal/httpapi/compat_protocol_test.go @@ -47,7 +47,7 @@ func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) { } } -func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) { +func TestProtocolErrorsUseCompatibleShapesWithStandardPublicErrors(t *testing.T) { tests := []struct { name string protocol string @@ -78,10 +78,10 @@ func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) { name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusBadGateway, assertBody: func(t *testing.T, body map[string]any) { errorBody := requireObject(t, body["error"]) - if errorBody["code"] != "upstream_submission_unknown" { + if errorBody["code"] != "upstream_submission_unknown" || errorBody["httpStatus"] != float64(http.StatusBadGateway) || errorBody["retryable"] != true { t.Fatalf("unexpected Volces error: %+v", body) } - assertNoKeys(t, errorBody, "status", "retryable", "taskId", "gateway_status") + assertNoKeys(t, errorBody, "status", "taskId", "gateway_status") }, }, } @@ -135,6 +135,44 @@ func TestWireResponsePassthroughPreservesStatusUnknownFieldsAndAllowedHeaders(t } } +func TestCompatibilityErrorWritersNeverExposeTransportDetails(t *testing.T) { + raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer" + tests := []struct { + name string + write func(http.ResponseWriter) + }{ + {name: "openai", write: func(w http.ResponseWriter) { + writeProtocolError(w, clients.ProtocolOpenAIResponses, http.StatusOK, raw, map[string]any{ + "provider": "secret-provider", "endpoint": "https://private.example.invalid", "bucket": "private-bucket", + }, "response_read_error") + }}, + {name: "volces", write: func(w http.ResponseWriter) { + writeVolcesError(w, http.StatusOK, raw, "response_read_error") + }}, + {name: "kling", write: func(w http.ResponseWriter) { + writeKlingCompatError(w, http.StatusOK, raw, "response_read_error") + }}, + {name: "keling", write: func(w http.ResponseWriter) { + writeKelingCompatError(w, "request-1", newKelingCompatError(http.StatusOK, 5001, raw)) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + test.write(recorder) + if recorder.Code != http.StatusBadGateway { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + body := recorder.Body.String() + if strings.Contains(body, "10.42.0.72") || strings.Contains(body, "47.77.191.126") || + strings.Contains(body, "secret-provider") || strings.Contains(body, "private.example.invalid") || strings.Contains(body, "private-bucket") || + !strings.Contains(body, "upstream_connection_interrupted") { + t.Fatalf("transport details were not standardized: %s", body) + } + }) + } +} + func TestCompatibilityStatusMappings(t *testing.T) { for internal, want := range map[string]string{ "queued": "queued", "running": "running", "succeeded": "succeeded", "failed": "failed", "cancelled": "cancelled", diff --git a/apps/api/internal/httpapi/easyai_compat.go b/apps/api/internal/httpapi/easyai_compat.go index 2a25a0e..5beeb9d 100644 --- a/apps/api/internal/httpapi/easyai_compat.go +++ b/apps/api/internal/httpapi/easyai_compat.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/runner" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -58,14 +59,21 @@ func writeEasyAIAsyncError(w http.ResponseWriter, status int, message string, de if len(codes) > 0 { code = strings.TrimSpace(codes[0]) } + standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500) + standard = publicErrorWithRetryAfter(standard, details) + publicerror.Observe(standard) + status, message, code = standard.HTTPStatus, standard.Message, standard.Code errorPayload := map[string]any{ - "message": message, - "status": status, + "message": message, + "status": status, + "code": code, + "category": standard.Category, + "httpStatus": standard.HTTPStatus, + "retryable": standard.Retryable, + "action": standard.Action, + "version": standard.Version, } - if code != "" { - errorPayload["code"] = code - } - for key, value := range details { + for key, value := range safePublicErrorDetails(details, standard, false) { errorPayload[key] = value } response := map[string]any{ @@ -158,8 +166,14 @@ func easyAITaskResultResponse(task store.GatewayTask) map[string]any { if message != "" { response["message"] = message } - if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" { - response["code"] = code + if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" || status == "failed" { + standard := publicTaskError(task) + if code != "" && task.ErrorCode == "" { + standard = publicerror.WithIDs(publicerror.FromFields(code, message, 0, false), task.RequestID, task.ID) + } + response["code"] = standard.Code + response["message"] = standard.Message + response["error"] = standard } return response } diff --git a/apps/api/internal/httpapi/file_upload_handlers.go b/apps/api/internal/httpapi/file_upload_handlers.go index 0a02b14..fbbae9b 100644 --- a/apps/api/internal/httpapi/file_upload_handlers.go +++ b/apps/api/internal/httpapi/file_upload_handlers.go @@ -13,7 +13,7 @@ const maxGatewayUploadBytes = 256 << 20 // uploadFile godoc // @Summary 上传文件 -// @Description 上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。 +// @Description 上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。 // @Tags files // @Accept multipart/form-data // @Produce json @@ -55,11 +55,8 @@ func (s *Server) uploadFile(w http.ResponseWriter, r *http.Request) { }) if err != nil { s.logger.Error("upload file failed", "error", err) - status := http.StatusBadGateway - if clients.ErrorCode(err) == "upload_no_channel" { - status = http.StatusServiceUnavailable - } - writeError(w, status, err.Error()) + status := statusFromRunError(err) + writeError(w, status, err.Error(), clients.ErrorCode(err)) return } writeJSON(w, http.StatusOK, easyAIFileUploadResponse(upload)) diff --git a/apps/api/internal/httpapi/gemini_compat.go b/apps/api/internal/httpapi/gemini_compat.go index 85bb1ee..4b8a651 100644 --- a/apps/api/internal/httpapi/gemini_compat.go +++ b/apps/api/internal/httpapi/gemini_compat.go @@ -17,6 +17,7 @@ import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/runner" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -159,7 +160,7 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { if err != nil { s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err) status := http.StatusBadRequest - if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" { + if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" { status = http.StatusBadGateway } writeGeminiTaskError(status, err.Error(), nil, clients.ErrorCode(err)) @@ -318,14 +319,13 @@ func (s *Server) writeGeminiGenerateContentStream(runCtx context.Context, w http } applyRunErrorHeaders(w, runErr) if !nativePassthrough && !convertedFrames { - if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, clients.ProtocolGeminiGenerateContent) { - writeWireResponse(w, wire) - return - } writeGeminiTaskError(statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr)) } else if !nativePassthrough { - status := statusFromRunError(runErr) - writeGeminiSSEFrame(w, geminiErrorEnvelope(status, runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))) + standard := publicerror.FromFields(runErrorCode(runErr), runErrorMessage(runErr), statusFromRunError(runErr), clients.IsRetryable(runErr)) + standard = publicErrorWithRetryAfter(standard, runErrorDetails(runErr)) + publicerror.Observe(standard) + details := safePublicErrorDetails(runErrorDetails(runErr), standard, true) + writeGeminiSSEFrame(w, geminiErrorEnvelope(standard.HTTPStatus, standard.Message, details, standard.Code)) if flusher != nil { flusher.Flush() } @@ -683,10 +683,7 @@ func (s *Server) finalizeGeminiFilesUpload(w http.ResponseWriter, r *http.Reques }) if err != nil { s.logger.Error("Gemini files upload failed", "error", err) - status := http.StatusBadGateway - if clients.ErrorCode(err) == "upload_no_channel" { - status = http.StatusServiceUnavailable - } + status := statusFromRunError(err) writeProtocolError(w, clients.ProtocolGeminiGenerateContent, status, err.Error(), nil, clients.ErrorCode(err)) return } diff --git a/apps/api/internal/httpapi/handlers.go b/apps/api/internal/httpapi/handlers.go index 0bfaec3..765ec89 100644 --- a/apps/api/internal/httpapi/handlers.go +++ b/apps/api/internal/httpapi/handlers.go @@ -1154,7 +1154,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler { body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind) if err != nil { status := http.StatusBadRequest - if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" { + if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" { status = http.StatusBadGateway } writeTaskError(status, err.Error(), nil, clients.ErrorCode(err)) @@ -1195,7 +1195,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler { if err != nil { s.logger.Warn("prepare task request failed", "kind", kind, "error", err) status := http.StatusBadRequest - if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" { + if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" { status = http.StatusBadGateway } writeTaskError(status, err.Error(), nil, clients.ErrorCode(err)) @@ -1556,10 +1556,6 @@ func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.Response return } applyRunErrorHeaders(w, runErr) - if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, targetProtocol) { - writeWireResponse(w, wire) - return - } if targetProtocol != "" { writeProtocolError(w, targetProtocol, statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr)) return @@ -1673,6 +1669,8 @@ func modelNameFromValue(value any) string { func statusFromRunError(err error) int { switch { + case clients.ErrorCode(err) == "storage_write_failed" || clients.ErrorCode(err) == "storage_read_failed" || clients.ErrorCode(err) == "storage_config_invalid" || clients.ErrorCode(err) == "storage_auth_failed": + return http.StatusServiceUnavailable case clients.ErrorCode(err) == "binary_result_expired": return http.StatusGone case clients.ErrorCode(err) == "binary_result_corrupted" || clients.ErrorCode(err) == "result_binary_not_materialized": @@ -1949,7 +1947,7 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) { return } writeJSON(w, http.StatusOK, map[string]any{ - "items": result.Items, + "items": publicTaskList(result.Items), "total": result.Total, "page": result.Page, "pageSize": result.PageSize, @@ -2034,7 +2032,7 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) { writeStoredBinaryResultError(w, err) return } - writeJSON(w, http.StatusOK, task) + writeJSON(w, http.StatusOK, publicGatewayTask(task)) return } if store.IsNotFound(err) { diff --git a/apps/api/internal/httpapi/keling_compat_handlers.go b/apps/api/internal/httpapi/keling_compat_handlers.go index 994cc6e..a4d62b1 100644 --- a/apps/api/internal/httpapi/keling_compat_handlers.go +++ b/apps/api/internal/httpapi/keling_compat_handlers.go @@ -15,6 +15,7 @@ import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -66,17 +67,19 @@ type KelingOmniWatermarkInfo struct { } type KelingCompatibleEnvelope struct { - Code int `json:"code" example:"0"` - Message string `json:"message" example:"SUCCEED"` - RequestID string `json:"request_id"` - Data any `json:"data,omitempty"` + Code int `json:"code" example:"0"` + Message string `json:"message" example:"SUCCEED"` + RequestID string `json:"request_id"` + Data any `json:"data,omitempty"` + Error *publicerror.Error `json:"error,omitempty"` } type kelingCompatError struct { - HTTPStatus int - Code int - Message string - RequestID string + HTTPStatus int + Code int + Message string + RequestID string + PublicError *publicerror.Error } func (e *kelingCompatError) Error() string { @@ -200,10 +203,6 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) { } task, createErr = s.waitForCompatibilitySubmission(r, task) if createErr != nil { - if wire := clients.ErrorWireResponse(createErr); wireResponseMatches(wire, clients.ProtocolKlingV1Omni) { - writeWireResponse(w, wire) - return - } writeKelingCompatError(w, requestID, kelingCompatGatewayError(createErr)) return } @@ -690,11 +689,13 @@ func kelingCompatTaskData(task store.GatewayTask) map[string]any { "enabled": kelingCompatTaskWatermark(task.Request), }, } - if message := kelingCompatTaskMessage(task); message != "" { - data["task_status_msg"] = message - } if kelingCompatTaskStatus(task.Status) == "failed" { - data["task_status_code"] = kelingCompatBusinessCode(task.ErrorCode, kelingCompatTaskMessage(task)) + standard := publicTaskError(task) + data["task_status_msg"] = standard.Message + data["task_status_code"] = kelingCompatBusinessCode(standard.Code, standard.Message) + data["error"] = standard + } else if message := kelingCompatTaskMessage(task); message != "" { + data["task_status_msg"] = message } if videos := kelingCompatTaskVideos(task.Result); len(videos) > 0 { data["task_result"] = map[string]any{"videos": videos} @@ -770,6 +771,9 @@ func firstKelingCompatValue(values ...any) any { } func kelingCompatTaskMessage(task store.GatewayTask) string { + if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" || task.Error != "" { + return publicTaskError(task).Message + } return strings.TrimSpace(firstNonEmpty(task.ErrorMessage, task.Error, task.Message)) } @@ -825,8 +829,11 @@ func kelingCompatGatewayError(err error) *kelingCompatError { return newKelingCompatError(http.StatusInternalServerError, 5000, "unknown gateway error") } codeText := clients.ErrorCode(err) - businessCode := kelingCompatBusinessCode(codeText, err.Error()) - status := http.StatusInternalServerError + status := statusFromRunError(err) + standard := publicerror.FromFields(codeText, err.Error(), status, clients.IsRetryable(err)) + publicerror.Observe(standard) + businessCode := kelingCompatBusinessCode(standard.Code, standard.Message) + status = standard.HTTPStatus switch businessCode { case 1101: status = http.StatusPaymentRequired @@ -841,7 +848,9 @@ func kelingCompatGatewayError(err error) *kelingCompatError { case 5001: status = http.StatusBadGateway } - return newKelingCompatError(status, businessCode, err.Error()) + result := newKelingCompatError(status, businessCode, standard.Message) + result.PublicError = &standard + return result } func kelingCompatBusinessCode(errorCode string, message string) int { @@ -923,9 +932,18 @@ func writeKelingCompatError(w http.ResponseWriter, requestID string, err *keling if status == 0 { status = http.StatusInternalServerError } + standard := err.PublicError + if standard == nil { + value := publicerror.FromFields("", err.Message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500) + value = publicerror.WithIDs(value, requestID, "") + publicerror.Observe(value) + standard = &value + } + status = standard.HTTPStatus writeJSON(w, status, KelingCompatibleEnvelope{ Code: err.Code, - Message: err.Message, + Message: standard.Message, RequestID: requestID, + Error: standard, }) } diff --git a/apps/api/internal/httpapi/kling_compat.go b/apps/api/internal/httpapi/kling_compat.go index b2fc624..975425f 100644 --- a/apps/api/internal/httpapi/kling_compat.go +++ b/apps/api/internal/httpapi/kling_compat.go @@ -14,6 +14,7 @@ import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -191,10 +192,6 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v } task, err = s.waitForCompatibilitySubmission(r, task) if err != nil { - if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, targetProtocol) { - writeWireResponse(w, wire) - return - } writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err)) return } @@ -667,7 +664,9 @@ func klingV1TaskData(task store.GatewayTask) map[string]any { "created_at": task.CreatedAt.UnixMilli(), "updated_at": task.UpdatedAt.UnixMilli(), } if task.ErrorMessage != "" || task.Error != "" { - data["task_status_msg"] = firstNonEmpty(task.ErrorMessage, task.Error) + standard := publicTaskError(task) + data["task_status_msg"] = standard.Message + data["error"] = standard } if watermarkInfo, ok := task.Request["watermark_info"].(map[string]any); ok { data["watermark_info"] = watermarkInfo @@ -698,7 +697,9 @@ func klingV2TaskData(task store.GatewayTask) map[string]any { "external_id": task.ExternalTaskID, } if message := firstNonEmpty(task.ErrorMessage, task.Error); message != "" { - data["message"] = message + standard := publicTaskError(task) + data["message"] = standard.Message + data["error"] = standard } if outputs := klingV2Outputs(task); len(outputs) > 0 { data["outputs"] = outputs @@ -827,14 +828,14 @@ func decodeKlingJSON(r *http.Request, target any) error { } func writeKlingCompatError(w http.ResponseWriter, status int, message string, code string) { - if strings.TrimSpace(code) == "" { - code = "invalid_request" - } - writeJSON(w, status, map[string]any{ - "code": klingCompatErrorCode(status), - "message": message, - "request_id": "", - "error": code, + standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500) + publicerror.Observe(standard) + writeJSON(w, standard.HTTPStatus, map[string]any{ + "code": klingCompatErrorCode(standard.HTTPStatus), + "message": standard.Message, + "request_id": "", + "error": standard.Code, + "public_error": standard, }) } diff --git a/apps/api/internal/httpapi/openapi_models.go b/apps/api/internal/httpapi/openapi_models.go index c46afc2..8654507 100644 --- a/apps/api/internal/httpapi/openapi_models.go +++ b/apps/api/internal/httpapi/openapi_models.go @@ -2,6 +2,7 @@ package httpapi import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -32,11 +33,19 @@ type ErrorEnvelope struct { } type ErrorPayload struct { - Message string `json:"message" example:"invalid json body"` - Status int `json:"status" example:"400"` - Code string `json:"code,omitempty" example:"rate_limit"` - Type string `json:"type,omitempty" example:"invalid_request_error"` - Param any `json:"param,omitempty"` + Message string `json:"message" example:"invalid json body"` + Status int `json:"status" example:"400"` + Code string `json:"code,omitempty" example:"rate_limit"` + Category string `json:"category,omitempty" example:"rate_limit"` + HTTPStatus int `json:"httpStatus,omitempty" example:"429"` + Retryable bool `json:"retryable" example:"true"` + Action string `json:"action,omitempty" example:"retry_after"` + Version string `json:"version,omitempty" example:"v1"` + RetryAfterSeconds int `json:"retryAfterSeconds,omitempty" example:"2"` + RequestID string `json:"requestId,omitempty"` + TaskID string `json:"taskId,omitempty"` + Type string `json:"type,omitempty" example:"invalid_request_error"` + Param any `json:"param,omitempty"` } type OpenAIErrorEnvelope struct { @@ -62,12 +71,7 @@ type GeminiErrorStatus struct { } type VolcesErrorEnvelope struct { - Error VolcesErrorPayload `json:"error"` -} - -type VolcesErrorPayload struct { - Code string `json:"code" example:"invalid_parameter"` - Message string `json:"message" example:"model is required"` + Error publicerror.Error `json:"error"` } type VolcesContentsGenerationTaskResponse struct { @@ -82,10 +86,11 @@ type VolcesContentsGenerationTaskResponse struct { } type KlingErrorEnvelope struct { - Code int `json:"code" example:"1001"` - Message string `json:"message" example:"invalid parameter"` - RequestID string `json:"request_id"` - Error string `json:"error,omitempty" example:"invalid_parameter"` + Code int `json:"code" example:"1001"` + Message string `json:"message" example:"invalid parameter"` + RequestID string `json:"request_id"` + Error string `json:"error,omitempty" example:"invalid_parameter"` + PublicError *publicerror.Error `json:"public_error,omitempty"` } type AuthResponse struct { @@ -232,6 +237,14 @@ type FileStorageChannelListResponse struct { Items []store.FileStorageChannel `json:"items"` } +type FileStorageChannelTestResponse struct { + Provider string `json:"provider"` + PutSucceeded bool `json:"putSucceeded"` + HeadSucceeded bool `json:"headSucceeded"` + DeleteSucceeded bool `json:"deleteSucceeded"` + DurationMS int64 `json:"durationMs"` +} + type FileUploadResponse struct { ID string `json:"id,omitempty" example:"file_abc123"` URL string `json:"url,omitempty" example:"/static/uploaded/upload-abc123.png"` diff --git a/apps/api/internal/httpapi/public_errors.go b/apps/api/internal/httpapi/public_errors.go new file mode 100644 index 0000000..1fdcd6b --- /dev/null +++ b/apps/api/internal/httpapi/public_errors.go @@ -0,0 +1,120 @@ +package httpapi + +import ( + "net/http" + "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func publicGatewayTask(task store.GatewayTask) store.GatewayTask { + if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" { + value := publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error), storedTaskErrorStatus(task.ErrorCode), false) + if task.PublicError != nil && task.PublicError.Code != "" { + value = *task.PublicError + } + value = publicerror.WithIDs(value, task.RequestID, task.ID) + publicerror.Observe(value) + task.PublicError = &value + task.ErrorCode = value.Code + task.ErrorMessage = value.Message + task.Error = value.Message + } + for index := range task.Attempts { + attempt := &task.Attempts[index] + if attempt.ErrorCode == "" && attempt.ErrorMessage == "" { + continue + } + status := attempt.StatusCode + if status <= 0 { + status = http.StatusBadGateway + } + value := publicerror.FromFields(attempt.ErrorCode, attempt.ErrorMessage, status, attempt.Retryable) + if attempt.PublicError != nil && attempt.PublicError.Code != "" { + value = *attempt.PublicError + } + value = publicerror.WithIDs(value, attempt.RequestID, task.ID) + attempt.PublicError = &value + attempt.ErrorCode = value.Code + attempt.ErrorMessage = value.Message + } + return task +} + +func publicTaskError(task store.GatewayTask) publicerror.Error { + if task.PublicError != nil && task.PublicError.Code != "" { + return publicerror.WithIDs(*task.PublicError, task.RequestID, task.ID) + } + status := storedTaskErrorStatus(task.ErrorCode) + if status <= 0 { + status = http.StatusBadGateway + } + return publicerror.WithIDs(publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error, task.Message), status, false), task.RequestID, task.ID) +} + +func publicTaskList(items []store.GatewayTask) []store.GatewayTask { + out := make([]store.GatewayTask, len(items)) + for index, task := range items { + out[index] = publicGatewayTask(task) + } + return out +} + +func publicErrorMap(value publicerror.Error) map[string]any { + out := map[string]any{ + "code": value.Code, "message": value.Message, "category": value.Category, + "httpStatus": value.HTTPStatus, "retryable": value.Retryable, "action": value.Action, + "version": value.Version, + } + if value.RetryAfterSeconds > 0 { + out["retryAfterSeconds"] = value.RetryAfterSeconds + } + if requestID := strings.TrimSpace(value.RequestID); requestID != "" { + out["requestId"] = requestID + } + if taskID := strings.TrimSpace(value.TaskID); taskID != "" { + out["taskId"] = taskID + } + return out +} + +func publicErrorWithRetryAfter(value publicerror.Error, details map[string]any) publicerror.Error { + if value.RetryAfterSeconds > 0 || details == nil { + return value + } + switch typed := details["retryAfterSeconds"].(type) { + case int: + value.RetryAfterSeconds = typed + case int32: + value.RetryAfterSeconds = int(typed) + case int64: + value.RetryAfterSeconds = int(typed) + case float64: + value.RetryAfterSeconds = int(typed) + } + if value.RetryAfterSeconds < 0 { + value.RetryAfterSeconds = 0 + } + return value +} + +func safePublicErrorDetails(details map[string]any, value publicerror.Error, includePublicError bool) map[string]any { + out := map[string]any{} + for _, key := range []string{"param", "retryAfterSeconds", "recoveryAt", "rateLimit", "pricing"} { + if item, ok := details[key]; ok && item != nil { + out[key] = item + } + } + if value.Category == "request" { + for _, key := range []string{"reason", "diagnosticId"} { + if item, ok := details[key]; ok && item != nil { + out[key] = item + } + } + } + if includePublicError { + out["publicError"] = value + } + return out +} diff --git a/apps/api/internal/httpapi/request_preparation.go b/apps/api/internal/httpapi/request_preparation.go index d2e3852..b3fea37 100644 --- a/apps/api/internal/httpapi/request_preparation.go +++ b/apps/api/internal/httpapi/request_preparation.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "sort" - "strconv" "strings" "sync" "time" @@ -38,12 +37,6 @@ type decodedRequestAsset struct { ContentType string } -type requestAssetOptions struct { - RequirePublicURL bool - UploadScene string - Source string -} - type requestAssetLock struct { mu sync.Mutex refs int @@ -115,6 +108,19 @@ func (s *Server) prepareRequestAssetRefs(ctx context.Context, body map[string]an } func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path []string, siblings map[string]any) (any, error) { + if decoded, ok, err := requestAssetFromBinaryValue(requestAssetPathKey(path), path, value, siblings); err != nil { + return nil, err + } else if ok { + if err := s.acquireMediaRequestSlot(ctx); err != nil { + return nil, err + } + defer s.releaseMediaRequestSlot() + ref, err := s.ensureRequestAsset(ctx, decoded) + if err != nil { + return nil, err + } + return requestAssetWrapper(ref), nil + } switch typed := value.(type) { case map[string]any: if typed["assetRef"] != nil { @@ -156,6 +162,83 @@ func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path [ } } +func requestAssetFromBinaryValue(key string, path []string, value any, siblings map[string]any) (decodedRequestAsset, bool, error) { + var payload []byte + contentType := "" + switch typed := value.(type) { + case []byte: + payload = append([]byte(nil), typed...) + case map[string]any: + if !strings.EqualFold(strings.TrimSpace(stringFromRequestAny(typed["type"])), "buffer") { + return decodedRequestAsset{}, false, nil + } + contentType = firstNonEmptyRequestString(typed, "contentType", "mimeType", "mime_type") + switch data := typed["data"].(type) { + case []byte: + payload = append([]byte(nil), data...) + case []any: + var ok bool + payload, ok = requestBytesFromNumberArray(data) + if !ok { + return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data must contain byte values")) + } + default: + return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data is required")) + } + case []any: + if !strictRequestBinaryArrayField(key, path) { + return decodedRequestAsset{}, false, nil + } + var ok bool + payload, ok = requestBytesFromNumberArray(typed) + if !ok { + return decodedRequestAsset{}, false, nil + } + default: + return decodedRequestAsset{}, false, nil + } + if len(payload) == 0 { + return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("binary media payload is empty")) + } + return decodedRequestAsset{ + Bytes: payload, + ContentType: requestAssetContentType(contentType, payload, key, path, siblings), + }, true, nil +} + +func requestAssetPathKey(path []string) string { + for index := len(path) - 1; index >= 0; index-- { + value := strings.TrimSpace(path[index]) + if value != "" && !strings.HasPrefix(value, "[") { + return value + } + } + return "" +} + +func strictRequestBinaryArrayField(key string, path []string) bool { + value := strings.ToLower(strings.TrimSpace(key)) + if value == "bytes" || value == "buffer" || strings.Contains(value, "binary") || strings.Contains(value, "buffer") { + return true + } + return requestMediaKind(key, path, nil) != "" && (value == "data" || value == "content") +} + +func requestBytesFromNumberArray(values []any) ([]byte, bool) { + if len(values) == 0 { + return nil, false + } + payload := make([]byte, len(values)) + for index, value := range values { + number, ok := value.(float64) + if !ok || number < 0 || number > 255 || number != float64(byte(number)) { + return nil, false + } + payload[index] = byte(number) + } + return payload, true +} + func (s *Server) prepareRequestAssetField(ctx context.Context, key string, path []string, value any, siblings map[string]any) (map[string]any, bool, error) { text, ok := value.(string) if !ok { @@ -291,68 +374,43 @@ func requestAssetFromValue(key string, path []string, value any, siblings map[st } func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) { - return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{ - UploadScene: store.FileStorageSceneRequestAsset, - Source: "ai-gateway-request", - }) -} - -func (s *Server) ensurePublicRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) { - return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{ - RequirePublicURL: true, - UploadScene: store.FileStorageSceneUpload, - Source: "ai-gateway-form-data", - }) -} - -func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded decodedRequestAsset, options requestAssetOptions) (map[string]any, error) { sum := sha256.Sum256(decoded.Bytes) sha := hex.EncodeToString(sum[:]) contentType := strings.TrimSpace(decoded.ContentType) if contentType == "" { contentType = "application/octet-stream" } - release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + options.UploadScene + "\x00" + strconv.FormatBool(options.RequirePublicURL)) + release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + store.FileStorageSceneRequestAsset) defer release() now := time.Now() if existing, ok, err := s.store.FindRequestAsset(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) { return nil, err } else if ok && requestAssetStillUsable(existing, now) { ref := requestAssetRef(existing) - if !options.RequirePublicURL || requestAssetRefHasPublicURL(ref) { - if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) { - return nil, err - } - return ref, nil + if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) { + return nil, err } + return ref, nil } - uploadScene := strings.TrimSpace(options.UploadScene) - if uploadScene == "" { - uploadScene = store.FileStorageSceneRequestAsset - } - source := strings.TrimSpace(options.Source) - if source == "" { - source = "ai-gateway-request" - } upload, err := s.runner.UploadFile(ctx, runner.FileUploadPayload{ Bytes: decoded.Bytes, ContentType: contentType, FileName: requestAssetFileName(sha, contentType), - Scene: uploadScene, - Source: source, + Scene: store.FileStorageSceneRequestAsset, + Source: "ai-gateway-request", }) if err != nil { return nil, err } storageProvider := requestAssetStorageProvider(upload) + storageChannelID, storageChannelKey := requestAssetStorageChannel(upload) + objectKey := stringFromRequestAny(upload["objectKey"]) + accessScope := firstNonEmpty(stringFromRequestAny(upload["accessScope"]), "private") url := stringFromRequestAny(upload["url"]) if url == "" { return nil, &clients.ClientError{Code: "request_asset_upload_failed", Message: "file storage response did not include url", Retryable: false} } - if options.RequirePublicURL && !requestAssetURLIsPublic(storageProvider, url) { - return nil, &clients.ClientError{Code: "request_asset_public_url_required", Message: "multipart image assets require a public file storage URL; enable a non-local file storage channel for uploads", Retryable: false} - } var expiresAt *time.Time localPath := "" if storageProvider == "local_static" { @@ -361,23 +419,31 @@ func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded deco localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"])) } asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{ - SHA256: sha, - ContentType: contentType, - ByteSize: int64(len(decoded.Bytes)), - URL: url, - StorageProvider: storageProvider, - LocalPath: localPath, - ExpiresAt: expiresAt, + SHA256: sha, + ContentType: contentType, + ByteSize: int64(len(decoded.Bytes)), + URL: url, + StorageProvider: storageProvider, + StorageChannelID: storageChannelID, + StorageChannelKey: storageChannelKey, + ObjectKey: objectKey, + AccessScope: accessScope, + LocalPath: localPath, + ExpiresAt: expiresAt, }) if err != nil { if store.IsUndefinedDatabaseObject(err) { return map[string]any{ - "sha256": sha, - "url": url, - "contentType": contentType, - "size": len(decoded.Bytes), - "storageProvider": storageProvider, - "expiresAt": timePtrToRFC3339(expiresAt), + "sha256": sha, + "url": url, + "contentType": contentType, + "size": len(decoded.Bytes), + "storageProvider": storageProvider, + "storageChannelId": storageChannelID, + "storageChannelKey": storageChannelKey, + "objectKey": objectKey, + "accessScope": accessScope, + "expiresAt": timePtrToRFC3339(expiresAt), }, nil } return nil, err @@ -493,27 +559,19 @@ func requestAssetWrapper(ref map[string]any) map[string]any { func requestAssetRef(asset store.RequestAsset) map[string]any { return map[string]any{ - "sha256": asset.SHA256, - "url": asset.URL, - "contentType": asset.ContentType, - "size": asset.ByteSize, - "storageProvider": asset.StorageProvider, - "expiresAt": timePtrToRFC3339(asset.ExpiresAt), + "sha256": asset.SHA256, + "url": asset.URL, + "contentType": asset.ContentType, + "size": asset.ByteSize, + "storageProvider": asset.StorageProvider, + "storageChannelId": asset.StorageChannelID, + "storageChannelKey": asset.StorageChannelKey, + "objectKey": asset.ObjectKey, + "accessScope": asset.AccessScope, + "expiresAt": timePtrToRFC3339(asset.ExpiresAt), } } -func requestAssetRefHasPublicURL(ref map[string]any) bool { - return requestAssetURLIsPublic(stringFromRequestAny(ref["storageProvider"]), stringFromRequestAny(ref["url"])) -} - -func requestAssetURLIsPublic(storageProvider string, url string) bool { - if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") { - return false - } - lower := strings.ToLower(strings.TrimSpace(url)) - return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") -} - func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool { if asset.ExpiredAt != nil { return false @@ -543,6 +601,11 @@ func requestAssetStorageProvider(upload map[string]any) string { return "unknown" } +func requestAssetStorageChannel(upload map[string]any) (string, string) { + channel, _ := upload["storageChannel"].(map[string]any) + return stringFromRequestAny(channel["id"]), stringFromRequestAny(channel["channelKey"]) +} + func requestAssetLocalPath(storageDir string, fileName string) string { if strings.TrimSpace(storageDir) == "" { storageDir = config.DefaultLocalUploadedStorageDir diff --git a/apps/api/internal/httpapi/request_preparation_test.go b/apps/api/internal/httpapi/request_preparation_test.go index 9f56148..6479fb4 100644 --- a/apps/api/internal/httpapi/request_preparation_test.go +++ b/apps/api/internal/httpapi/request_preparation_test.go @@ -41,6 +41,33 @@ func TestRequestAssetFromValueDetectsDataURLAndRawBase64(t *testing.T) { } } +func TestRequestAssetFromBinaryValueDetectsBufferAndByteArray(t *testing.T) { + png := []any{float64(0x89), float64('P'), float64('N'), float64('G')} + decoded, ok, err := requestAssetFromBinaryValue("image", []string{"input", "image"}, map[string]any{ + "type": "Buffer", + "data": png, + "mimeType": "image/png", + }, nil) + if err != nil { + t.Fatal(err) + } + if !ok || decoded.ContentType != "image/png" || len(decoded.Bytes) != 4 || decoded.Bytes[0] != 0x89 { + t.Fatalf("unexpected Buffer asset: ok=%v decoded=%+v", ok, decoded) + } + + decoded, ok, err = requestAssetFromBinaryValue("bytes", []string{"input_audio", "bytes"}, png, map[string]any{"format": "mp3"}) + if err != nil { + t.Fatal(err) + } + if !ok || len(decoded.Bytes) != 4 { + t.Fatalf("unexpected byte array asset: ok=%v decoded=%+v", ok, decoded) + } + + if _, ok, err := requestAssetFromBinaryValue("values", []string{"embedding", "values"}, png, nil); err != nil || ok { + t.Fatalf("ordinary numeric arrays must remain JSON: ok=%v err=%v", ok, err) + } +} + func TestMediaRequestBodySlotLimitsPreAuthWorkAndCanReleaseEarly(t *testing.T) { server := &Server{mediaRequestBodySlots: make(chan struct{}, 2)} var critical atomic.Int64 diff --git a/apps/api/internal/httpapi/response.go b/apps/api/internal/httpapi/response.go index 504cf57..79c7126 100644 --- a/apps/api/internal/httpapi/response.go +++ b/apps/api/internal/httpapi/response.go @@ -5,6 +5,8 @@ import ( "fmt" "net/http" "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" ) func writeJSON(w http.ResponseWriter, status int, value any) { @@ -18,22 +20,31 @@ func writeError(w http.ResponseWriter, status int, message string, codes ...stri } func writeErrorWithDetails(w http.ResponseWriter, status int, message string, details map[string]any, codes ...string) { - errorPayload := map[string]any{ - "message": message, - "status": status, - } + code := "" if len(codes) > 0 { - if code := strings.TrimSpace(codes[0]); code != "" { - errorPayload["code"] = code - if code == "invalid_parameter" || code == "unsupported_response_parameter" { - errorPayload["type"] = "invalid_request_error" - if _, ok := details["param"]; !ok { - errorPayload["param"] = nil - } - } + code = strings.TrimSpace(codes[0]) + } + standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500) + standard = publicErrorWithRetryAfter(standard, details) + publicerror.Observe(standard) + status = standard.HTTPStatus + errorPayload := map[string]any{ + "message": standard.Message, + "status": status, + "code": standard.Code, + "category": standard.Category, + "httpStatus": standard.HTTPStatus, + "retryable": standard.Retryable, + "action": standard.Action, + "version": standard.Version, + } + if standard.Code == "invalid_parameter" || standard.Code == "unsupported_response_parameter" { + errorPayload["type"] = "invalid_request_error" + if _, ok := details["param"]; !ok { + errorPayload["param"] = nil } } - for key, value := range details { + for key, value := range safePublicErrorDetails(details, standard, false) { errorPayload[key] = value } writeJSON(w, status, map[string]any{"error": errorPayload}) diff --git a/apps/api/internal/httpapi/server.go b/apps/api/internal/httpapi/server.go index 43f156d..0cf3259 100644 --- a/apps/api/internal/httpapi/server.go +++ b/apps/api/internal/httpapi/server.go @@ -286,6 +286,7 @@ func NewServerWithStores( mux.Handle("GET /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listFileStorageChannels))) mux.Handle("POST /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createFileStorageChannel))) mux.Handle("PATCH /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageChannel))) + mux.Handle("POST /api/admin/system/file-storage/channels/{channelID}/test", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.testFileStorageChannel))) mux.Handle("DELETE /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteFileStorageChannel))) mux.Handle("GET /api/admin/platforms", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listPlatforms))) mux.Handle("POST /api/admin/platforms", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createPlatform))) diff --git a/apps/api/internal/httpapi/system_settings_handlers.go b/apps/api/internal/httpapi/system_settings_handlers.go index 2abdc7c..07eb9a6 100644 --- a/apps/api/internal/httpapi/system_settings_handlers.go +++ b/apps/api/internal/httpapi/system_settings_handlers.go @@ -1,10 +1,13 @@ package httpapi import ( + "context" "encoding/json" "net/http" + "net/url" "strings" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -74,6 +77,10 @@ func (s *Server) updateFileStorageSettings(w http.ResponseWriter, r *http.Reques writeError(w, http.StatusBadRequest, "invalid json body") return } + if legacyLocalStoragePolicy(input.ResultUploadPolicy) { + writeError(w, http.StatusBadRequest, "upload_none is no longer supported; configure an object storage channel") + return + } settings, err := s.store.UpdateFileStorageSettings(r.Context(), input) if err != nil { s.logger.Error("update file storage settings failed", "error", err) @@ -155,7 +162,7 @@ func (s *Server) updateClientCustomizationSettings(w http.ResponseWriter, r *htt // createFileStorageChannel godoc // @Summary 创建文件存储通道 -// @Description 创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。 +// @Description 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。 // @Tags system // @Accept json // @Produce json @@ -271,6 +278,46 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request w.WriteHeader(http.StatusNoContent) } +// testFileStorageChannel godoc +// @Summary 测试对象存储通道 +// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。 +// @Tags system +// @Produce json +// @Security BearerAuth +// @Param channelID path string true "文件存储通道 ID" +// @Success 200 {object} FileStorageChannelTestResponse +// @Failure 400 {object} ErrorEnvelope +// @Failure 401 {object} ErrorEnvelope +// @Failure 403 {object} ErrorEnvelope +// @Failure 404 {object} ErrorEnvelope +// @Failure 503 {object} ErrorEnvelope +// @Router /api/admin/system/file-storage/channels/{channelID}/test [post] +func (s *Server) testFileStorageChannel(w http.ResponseWriter, r *http.Request) { + channel, err := s.store.GetFileStorageChannel(r.Context(), r.PathValue("channelID")) + if err != nil { + if store.IsNotFound(err) { + writeError(w, http.StatusNotFound, "file storage channel not found") + return + } + s.logger.Error("get file storage channel for test failed", "error", err) + writeError(w, http.StatusInternalServerError, "get file storage channel failed") + return + } + if channel.Provider != "aliyun_oss" && channel.Provider != "s3" { + writeError(w, http.StatusBadRequest, "connection test is supported for aliyun_oss and s3 channels", "invalid_parameter") + return + } + result, err := s.runner.TestFileStorageChannel(r.Context(), channel) + if err != nil { + _ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(r.Context()), channel.ID, err.Error()) + s.logger.Warn("file storage channel test failed", "channel_id", channel.ID, "provider", channel.Provider, "error", err) + writeError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err)) + return + } + _ = s.store.MarkFileStorageChannelSuccess(context.WithoutCancel(r.Context()), channel.ID) + writeJSON(w, http.StatusOK, result) +} + func validateFileStorageChannelInput(input store.FileStorageChannelInput, existing *store.FileStorageChannel) string { provider := strings.ToLower(strings.TrimSpace(input.Provider)) if provider == "" { @@ -286,6 +333,9 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi if status != "enabled" && status != "disabled" { return "status must be enabled or disabled" } + if provider != "server_main_openapi" && provider != "aliyun_oss" && provider != "s3" { + return "provider must be server_main_openapi, aliyun_oss or s3" + } if provider == "server_main_openapi" { hasAPIKey := false if input.APIKey != nil { @@ -297,5 +347,83 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi return "server-main OpenAPI channel requires API key before enabling" } } + if provider == "aliyun_oss" || provider == "s3" { + if fileStorageConfigContainsCredential(input.Config) { + return "object storage credentials must use the write-only credential fields, not config" + } + endpoint := fileStorageConfigString(input.Config, "endpoint") + if endpoint == "" || fileStorageConfigString(input.Config, "region") == "" || fileStorageConfigString(input.Config, "bucket") == "" { + return "object storage channel requires config.endpoint, config.region and config.bucket" + } + if !validFileStorageBaseURL(endpoint) { + return "object storage config.endpoint must be an http or https URL without embedded credentials" + } + if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) { + return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials" + } + accessKeyID := input.AccessKeyID + if accessKeyID == nil { + accessKeyID = input.AccessKey + } + accessKeySecret := input.AccessKeySecret + if accessKeySecret == nil { + accessKeySecret = input.SecretKey + } + hasAccessKeyID := fileStorageCredentialPresent(accessKeyID, existing, func(item *store.FileStorageChannel) string { return item.AccessKeyID }) + hasAccessKeySecret := fileStorageCredentialPresent(accessKeySecret, existing, func(item *store.FileStorageChannel) string { return item.AccessKeySecret }) + if status == "enabled" && (!hasAccessKeyID || !hasAccessKeySecret) { + return "object storage channel requires accessKeyId and accessKeySecret before enabling" + } + } return "" } + +func validFileStorageBaseURL(value string) bool { + parsed, err := url.Parse(strings.TrimSpace(value)) + return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https") +} + +func fileStorageConfigContainsCredential(value any) bool { + switch typed := value.(type) { + case map[string]any: + for key, item := range typed { + normalized := strings.NewReplacer("_", "", "-", "", ".", "").Replace(strings.ToLower(strings.TrimSpace(key))) + switch normalized { + case "apikey", "accesskey", "accesskeyid", "accesskeysecret", "secretkey", "sessiontoken", "ststoken", "password", "credential", "credentials", "authorization": + return true + } + if fileStorageConfigContainsCredential(item) { + return true + } + } + case []any: + for _, item := range typed { + if fileStorageConfigContainsCredential(item) { + return true + } + } + } + return false +} + +func fileStorageConfigString(config map[string]any, key string) string { + value, _ := config[key].(string) + return strings.TrimSpace(value) +} + +func fileStorageCredentialPresent(input *string, existing *store.FileStorageChannel, current func(*store.FileStorageChannel) string) bool { + if input != nil { + return strings.TrimSpace(*input) != "" + } + return existing != nil && strings.TrimSpace(current(existing)) != "" +} + +func legacyLocalStoragePolicy(value string) bool { + normalized := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(value)), "-", "_") + switch normalized { + case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all": + return true + default: + return false + } +} diff --git a/apps/api/internal/httpapi/system_settings_handlers_test.go b/apps/api/internal/httpapi/system_settings_handlers_test.go new file mode 100644 index 0000000..d6d92b6 --- /dev/null +++ b/apps/api/internal/httpapi/system_settings_handlers_test.go @@ -0,0 +1,55 @@ +package httpapi + +import ( + "strings" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestValidateObjectStorageChannelAcceptsS3CredentialAliases(t *testing.T) { + accessKey := "access" + secretKey := "secret" + input := store.FileStorageChannelInput{ + ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled", + AccessKey: &accessKey, SecretKey: &secretKey, + Config: map[string]any{ + "endpoint": "https://s3.example.com", "region": "us-east-1", "bucket": "media", + }, + } + if message := validateFileStorageChannelInput(input, nil); message != "" { + t.Fatalf("valid S3 channel rejected: %s", message) + } +} + +func TestValidateObjectStorageChannelRejectsCredentialsInConfig(t *testing.T) { + accessKeyID := "access" + accessKeySecret := "secret" + input := store.FileStorageChannelInput{ + ChannelKey: "oss-primary", Name: "OSS primary", Provider: "aliyun_oss", Status: "enabled", + AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret, + Config: map[string]any{ + "endpoint": "https://oss-cn-hangzhou.aliyuncs.com", "region": "cn-hangzhou", "bucket": "media", + "nested": map[string]any{"session_token": "must-not-be-public"}, + }, + } + message := validateFileStorageChannelInput(input, nil) + if !strings.Contains(message, "write-only credential fields") { + t.Fatalf("credential-bearing config was accepted: %q", message) + } +} + +func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) { + accessKeyID := "access" + accessKeySecret := "secret" + input := store.FileStorageChannelInput{ + ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled", + AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret, + Config: map[string]any{ + "endpoint": "https://user:password@s3.example.com", "region": "us-east-1", "bucket": "media", + }, + } + if message := validateFileStorageChannelInput(input, nil); !strings.Contains(message, "without embedded credentials") { + t.Fatalf("credentialed endpoint was accepted: %q", message) + } +} diff --git a/apps/api/internal/httpapi/task_idempotency.go b/apps/api/internal/httpapi/task_idempotency.go index 467b7e1..32c84d6 100644 --- a/apps/api/internal/httpapi/task_idempotency.go +++ b/apps/api/internal/httpapi/task_idempotency.go @@ -69,6 +69,10 @@ func writeIdempotentTaskReplay(w http.ResponseWriter, task store.GatewayTask, co func storedTaskErrorStatus(code string) int { switch strings.TrimSpace(code) { + case "storage_write_failed", "storage_read_failed", "storage_config_invalid", "storage_auth_failed": + return http.StatusServiceUnavailable + case "binary_result_expired", "result_expired", "result_unavailable": + return http.StatusGone case "pricing_unavailable", "response_chain_unavailable", "billing_hold": return http.StatusServiceUnavailable case "insufficient_balance": diff --git a/apps/api/internal/httpapi/task_multipart.go b/apps/api/internal/httpapi/task_multipart.go index b1883e3..fec7ff5 100644 --- a/apps/api/internal/httpapi/task_multipart.go +++ b/apps/api/internal/httpapi/task_multipart.go @@ -255,7 +255,7 @@ func (s *Server) uploadImageEditMultipartAsset(ctx context.Context, field string if !strings.HasPrefix(strings.ToLower(contentType), "image/") { return nil, &clients.ClientError{Code: "invalid_multipart_image", Message: "image edit multipart files must be images", Retryable: false} } - ref, err := s.ensurePublicRequestAsset(ctx, decodedRequestAsset{ + ref, err := s.ensureRequestAsset(ctx, decodedRequestAsset{ Bytes: payload, ContentType: contentType, }) diff --git a/apps/api/internal/httpapi/volces_compat_handlers.go b/apps/api/internal/httpapi/volces_compat_handlers.go index d1f5ad4..6cea4de 100644 --- a/apps/api/internal/httpapi/volces_compat_handlers.go +++ b/apps/api/internal/httpapi/volces_compat_handlers.go @@ -352,7 +352,7 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any { response["usage"] = legacyUsage } if task.Status == "failed" || task.Status == "cancelled" { - response["error"] = map[string]any{"code": firstNonEmpty(task.ErrorCode, strings.ToUpper(task.Status)), "message": firstNonEmpty(task.ErrorMessage, task.Error, task.Message)} + response["error"] = publicErrorMap(publicTaskError(task)) } return response } @@ -431,10 +431,6 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) { } else if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) { status = http.StatusBadRequest } - if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, clients.ProtocolVolcesContents) { - writeWireResponse(w, wire) - return - } writeVolcesError(w, status, err.Error(), clients.ErrorCode(err)) } diff --git a/apps/api/internal/publicerror/metrics.go b/apps/api/internal/publicerror/metrics.go new file mode 100644 index 0000000..e8d1cda --- /dev/null +++ b/apps/api/internal/publicerror/metrics.go @@ -0,0 +1,73 @@ +package publicerror + +import "sync/atomic" + +type MetricCount struct { + Code string + Count uint64 +} + +var observed = struct { + upstreamConnectionInterrupted atomic.Uint64 + upstreamTimeout atomic.Uint64 + upstreamRateLimited atomic.Uint64 + upstreamUnavailable atomic.Uint64 + upstreamInvalidResponse atomic.Uint64 + upstreamAuthFailed atomic.Uint64 + upstreamRequestRejected atomic.Uint64 + storageWriteFailed atomic.Uint64 + storageReadFailed atomic.Uint64 + resultExpired atomic.Uint64 + resultUnavailable atomic.Uint64 + invalidParameter atomic.Uint64 + gatewayError atomic.Uint64 +}{} + +func Observe(value Error) { + switch value.Code { + case "upstream_connection_interrupted": + observed.upstreamConnectionInterrupted.Add(1) + case "upstream_timeout": + observed.upstreamTimeout.Add(1) + case "upstream_rate_limited": + observed.upstreamRateLimited.Add(1) + case "upstream_unavailable": + observed.upstreamUnavailable.Add(1) + case "upstream_invalid_response": + observed.upstreamInvalidResponse.Add(1) + case "upstream_auth_failed": + observed.upstreamAuthFailed.Add(1) + case "upstream_request_rejected": + observed.upstreamRequestRejected.Add(1) + case "storage_write_failed": + observed.storageWriteFailed.Add(1) + case "storage_read_failed": + observed.storageReadFailed.Add(1) + case "result_expired": + observed.resultExpired.Add(1) + case "result_unavailable": + observed.resultUnavailable.Add(1) + case "invalid_parameter": + observed.invalidParameter.Add(1) + default: + observed.gatewayError.Add(1) + } +} + +func MetricSnapshot() []MetricCount { + return []MetricCount{ + {Code: "upstream_connection_interrupted", Count: observed.upstreamConnectionInterrupted.Load()}, + {Code: "upstream_timeout", Count: observed.upstreamTimeout.Load()}, + {Code: "upstream_rate_limited", Count: observed.upstreamRateLimited.Load()}, + {Code: "upstream_unavailable", Count: observed.upstreamUnavailable.Load()}, + {Code: "upstream_invalid_response", Count: observed.upstreamInvalidResponse.Load()}, + {Code: "upstream_auth_failed", Count: observed.upstreamAuthFailed.Load()}, + {Code: "upstream_request_rejected", Count: observed.upstreamRequestRejected.Load()}, + {Code: "storage_write_failed", Count: observed.storageWriteFailed.Load()}, + {Code: "storage_read_failed", Count: observed.storageReadFailed.Load()}, + {Code: "result_expired", Count: observed.resultExpired.Load()}, + {Code: "result_unavailable", Count: observed.resultUnavailable.Load()}, + {Code: "invalid_parameter", Count: observed.invalidParameter.Load()}, + {Code: "gateway_error", Count: observed.gatewayError.Load()}, + } +} diff --git a/apps/api/internal/publicerror/public_error.go b/apps/api/internal/publicerror/public_error.go new file mode 100644 index 0000000..9e74398 --- /dev/null +++ b/apps/api/internal/publicerror/public_error.go @@ -0,0 +1,200 @@ +package publicerror + +import ( + "net/http" + "strings" +) + +const Version = "v1" + +type Error struct { + Code string `json:"code"` + Message string `json:"message"` + Category string `json:"category"` + HTTPStatus int `json:"httpStatus"` + Retryable bool `json:"retryable"` + Action string `json:"action"` + RetryAfterSeconds int `json:"retryAfterSeconds,omitempty"` + RequestID string `json:"requestId,omitempty"` + TaskID string `json:"taskId,omitempty"` + Version string `json:"version"` +} + +func FromFields(code string, message string, status int, retryable bool) Error { + originalCode := strings.TrimSpace(code) + code = strings.ToLower(originalCode) + message = strings.TrimSpace(message) + if status <= 0 { + status = statusFromCode(code) + } + if strings.HasPrefix(originalCode, "GATEWAY_") { + return newError(originalCode, safeRequestMessage(message), "gateway", status, retryable, retryAction(retryable)) + } + if mapped, ok := mappedError(code, message, status, retryable); ok { + return mapped + } + if sensitiveTransportMessage(message) { + return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry") + } + if code == "" { + code = defaultCodeForStatus(status) + } else { + code = originalCode + } + if message == "" { + message = defaultMessageForStatus(status) + } + category := "gateway" + action := "none" + if status == http.StatusTooManyRequests { + category, action, retryable = "rate_limit", "retry_after", true + } else if status >= 500 { + category = "upstream" + if retryable { + action = "retry" + } + } else if status >= 400 { + category, action = "request", "fix_request" + } + return newError(code, message, category, status, retryable, action) +} + +func WithIDs(value Error, requestID string, taskID string) Error { + value.RequestID = strings.TrimSpace(requestID) + value.TaskID = strings.TrimSpace(taskID) + return value +} + +func mappedError(code string, message string, status int, retryable bool) (Error, bool) { + switch code { + case "bad_request", "invalid_request", "invalid_parameter", "unsupported_kind", "unsupported_operation", "unsupported_response_parameter": + return newError(code, safeRequestMessage(message), "request", http.StatusBadRequest, false, "fix_request"), true + case "cancelled", "task_cancelled": + return newError(code, safeRequestMessage(message), "request", http.StatusConflict, false, "none"), true + case "validation_in_progress": + return newError(code, firstNonEmptyMessage(message, "New production tasks are paused while validation is running."), "gateway", http.StatusServiceUnavailable, true, "retry_after"), true + case "traffic_gate_unavailable": + return newError(code, "The gateway traffic admission service is temporarily unavailable.", "gateway", http.StatusServiceUnavailable, true, "retry"), true + case "response_read_error", "stream_read_error", "network", "connection_reset", "upstream_connection_interrupted", "unexpected_eof", "http2_stream_closed", "request_asset_fetch_failed": + return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry"), true + case "upstream_timeout", "timeout", "context_deadline_exceeded": + return newError("upstream_timeout", "The upstream service did not respond in time.", "upstream", http.StatusGatewayTimeout, true, "retry"), true + case "rate_limit", "upstream_rate_limited", "too_many_requests": + return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", http.StatusTooManyRequests, true, "retry_after"), true + case "upstream_unavailable", "upstream_overloaded", "service_unavailable", "bad_gateway", "server_error", "provider_failed": + if status > 0 && status < 500 { + break + } + return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true + case "upload_invalid_response", "invalid_response", "response_too_large", "invalid_upstream_response", "upstream_invalid_response": + return newError("upstream_invalid_response", "The upstream service returned an invalid or incomplete response.", "upstream", http.StatusBadGateway, true, "retry"), true + case "invalid_api_key", "authentication_error", "auth_failed", "missing_credentials", "upstream_auth_failed": + return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true + case "storage_write_failed", "storage_config_invalid", "storage_auth_failed", "upload_config_failed", "upload_no_channel", "upload_network", "upload_read_failed", "upload_failed", "local_result_storage_unavailable", "local_static_store_failed", "request_asset_upload_failed": + return newError("storage_write_failed", "The media asset could not be written to object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true + case "storage_read_failed", "upload_source_fetch_failed", "upload_source_read_failed": + return newError("storage_read_failed", "The media asset could not be read from object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true + case "binary_result_expired", "result_expired": + return newError("result_expired", "The generated result has expired and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true + case "result_unavailable", "request_asset_expired": + return newError("result_unavailable", "The generated result is no longer available and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true + case "binary_result_corrupted", "result_binary_not_materialized": + return newError("result_corrupted", "The stored result failed an integrity check.", "storage", http.StatusInternalServerError, false, "contact_support"), true + } + if status == http.StatusTooManyRequests { + return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", status, true, "retry_after"), true + } + if (status == http.StatusUnauthorized || status == http.StatusForbidden) && (strings.HasPrefix(code, "http_") || strings.Contains(code, "provider")) { + return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true + } + if strings.HasPrefix(code, "http_") || (code == "provider_failed" && status > 0 && status < 500) { + return newError("upstream_request_rejected", "The upstream service rejected the request.", "upstream", http.StatusBadRequest, false, "fix_request"), true + } + if status >= 500 && code != "upstream_submission_unknown" && (strings.Contains(code, "upstream") || strings.Contains(code, "provider")) { + return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true + } + if status >= 500 && code != "upstream_submission_unknown" { + return newError("gateway_error", defaultMessageForStatus(status), "gateway", status, retryable, retryAction(retryable)), true + } + return Error{}, false +} + +func safeRequestMessage(message string) string { + message = strings.TrimSpace(message) + if message == "" || sensitiveTransportMessage(message) { + return "The request parameters are invalid." + } + return message +} + +func firstNonEmptyMessage(message string, fallback string) string { + if message = strings.TrimSpace(message); message != "" && !sensitiveTransportMessage(message) { + return message + } + return fallback +} + +func retryAction(retryable bool) string { + if retryable { + return "retry" + } + return "contact_support" +} + +func newError(code string, message string, category string, status int, retryable bool, action string) Error { + return Error{Code: code, Message: message, Category: category, HTTPStatus: status, Retryable: retryable, Action: action, Version: Version} +} + +func sensitiveTransportMessage(message string) bool { + value := strings.ToLower(message) + for _, marker := range []string{"read tcp ", "write tcp ", "dial tcp ", "connection reset by peer", "unexpected eof", "http2:", "stream error", "context deadline exceeded"} { + if strings.Contains(value, marker) { + return true + } + } + return false +} + +func statusFromCode(code string) int { + switch code { + case "rate_limit", "upstream_rate_limited", "too_many_requests": + return http.StatusTooManyRequests + case "upstream_timeout", "timeout", "context_deadline_exceeded": + return http.StatusGatewayTimeout + case "binary_result_expired", "result_expired", "result_unavailable", "request_asset_expired": + return http.StatusGone + case "storage_write_failed", "storage_read_failed", "local_result_storage_unavailable": + return http.StatusServiceUnavailable + default: + return http.StatusBadGateway + } +} + +func defaultCodeForStatus(status int) string { + switch status { + case http.StatusBadRequest: + return "invalid_request" + case http.StatusUnauthorized: + return "unauthorized" + case http.StatusForbidden: + return "forbidden" + case http.StatusNotFound: + return "not_found" + case http.StatusConflict: + return "conflict" + case http.StatusTooManyRequests: + return "rate_limited" + default: + if status >= 500 { + return "gateway_error" + } + return "request_failed" + } +} + +func defaultMessageForStatus(status int) string { + if value := http.StatusText(status); value != "" { + return value + } + return "Request failed." +} diff --git a/apps/api/internal/publicerror/public_error_test.go b/apps/api/internal/publicerror/public_error_test.go new file mode 100644 index 0000000..374db31 --- /dev/null +++ b/apps/api/internal/publicerror/public_error_test.go @@ -0,0 +1,69 @@ +package publicerror + +import ( + "net/http" + "strings" + "testing" +) + +func TestTransportErrorsNeverExposeSocketDetails(t *testing.T) { + raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer" + got := FromFields("response_read_error", raw, http.StatusOK, true) + if got.Code != "upstream_connection_interrupted" || got.HTTPStatus != http.StatusBadGateway || !got.Retryable { + t.Fatalf("unexpected public error: %+v", got) + } + if strings.Contains(got.Message, "10.42.0.72") || strings.Contains(got.Message, "47.77.191.126") { + t.Fatalf("public error leaked socket details: %+v", got) + } +} + +func TestStorageAndExpiredErrorsHaveStableActions(t *testing.T) { + storage := FromFields("storage_write_failed", "secret endpoint", 0, true) + if storage.Category != "storage" || storage.Action != "retry" || storage.HTTPStatus != http.StatusServiceUnavailable { + t.Fatalf("unexpected storage error: %+v", storage) + } + expired := FromFields("binary_result_expired", "local binary result has expired", 0, false) + if expired.Code != "result_expired" || expired.Action != "resubmit" || expired.HTTPStatus != http.StatusGone { + t.Fatalf("unexpected expired error: %+v", expired) + } +} + +func TestUnknownServerErrorDoesNotExposeProviderBody(t *testing.T) { + raw := `provider rejected request: {"account":"secret-project","detail":"internal"}` + got := FromFields("vendor_opaque_error", raw, http.StatusBadGateway, true) + if got.Code != "gateway_error" || got.Message == raw || strings.Contains(got.Message, "secret-project") { + t.Fatalf("unknown provider error was exposed: %+v", got) + } +} + +func TestProviderHTTPErrorDoesNotExposeProviderBody(t *testing.T) { + raw := `{"error":{"message":"bucket private-a rejected secret-project"}}` + for _, test := range []struct { + code string + status int + wantCode string + wantStatus int + }{ + {code: "http_400", status: http.StatusBadRequest, wantCode: "upstream_request_rejected", wantStatus: http.StatusBadRequest}, + {code: "auth_failed", status: http.StatusUnauthorized, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway}, + {code: "provider_failed", status: http.StatusForbidden, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway}, + {code: "provider_failed", status: http.StatusTooManyRequests, wantCode: "upstream_rate_limited", wantStatus: http.StatusTooManyRequests}, + {code: "server_error", status: http.StatusBadGateway, wantCode: "upstream_unavailable", wantStatus: http.StatusServiceUnavailable}, + {code: "invalid_response", status: http.StatusOK, wantCode: "upstream_invalid_response", wantStatus: http.StatusBadGateway}, + } { + got := FromFields(test.code, raw, test.status, true) + if got.Code != test.wantCode || got.HTTPStatus != test.wantStatus { + t.Fatalf("%s: unexpected public error: %+v", test.code, got) + } + if strings.Contains(got.Message, "private-a") || strings.Contains(got.Message, "secret-project") { + t.Fatalf("%s: provider body leaked: %+v", test.code, got) + } + } +} + +func TestValidationGateKeepsStablePublicCode(t *testing.T) { + got := FromFields("validation_in_progress", "new production tasks are paused while validation is running", http.StatusServiceUnavailable, true) + if got.Code != "validation_in_progress" || got.HTTPStatus != http.StatusServiceUnavailable || !got.Retryable { + t.Fatalf("unexpected validation gate error: %+v", got) + } +} diff --git a/apps/api/internal/runner/binary_results.go b/apps/api/internal/runner/binary_results.go index 57f9d73..cd219e1 100644 --- a/apps/api/internal/runner/binary_results.go +++ b/apps/api/internal/runner/binary_results.go @@ -12,7 +12,6 @@ import ( "path/filepath" "strconv" "strings" - "syscall" "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" @@ -26,7 +25,6 @@ const ( localBinaryGenericBase64MinLength = 4096 localBinaryMaxDepth = 64 defaultLocalResultTTLHours = 24 - defaultLocalResultMinFreeBytes = int64(10 * 1024 * 1024 * 1024) defaultLocalResultMaxBytes = int64(256 * 1024 * 1024) defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024) ) @@ -41,41 +39,38 @@ type localBinaryDescriptor struct { type localBinaryMaterializer struct { service *Service - taskDir string - writeFiles bool enforceLimits bool totalBytes int64 seen map[string]struct{} - createdFiles []string } -// materializeLocalBinaryResult replaces every inline binary value with a -// bounded placeholder after atomically writing and verifying the bytes locally. +// materializeLocalBinaryResult is retained as an internal compatibility seam, +// but new materialization always targets object storage. Historical local +// placeholders remain readable through HydrateTaskResult. func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) { - if _, _, err := s.transformLocalBinaryResult(ctx, taskID, result, false, true); err != nil { + hadInline := TaskResultHasInlineBinary(result) + if !hadInline { + return result, false, nil + } + next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result) + if err != nil { return nil, false, err } - return s.transformLocalBinaryResult(ctx, taskID, result, true, true) + return next, !TaskResultHasInlineBinary(next), nil } -func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, result map[string]any, writeFiles bool, enforceLimits bool) (map[string]any, bool, error) { - root := s.localBinaryResultRoot() - taskDir := filepath.Join(root, safeLocalBinaryTaskDir(taskID)) +func (s *Service) transformLocalBinaryResult(ctx context.Context, _ string, result map[string]any, _ bool, enforceLimits bool) (map[string]any, bool, error) { materializer := &localBinaryMaterializer{ service: s, - taskDir: taskDir, - writeFiles: writeFiles, enforceLimits: enforceLimits, seen: map[string]struct{}{}, } next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0) if err != nil { - materializer.rollbackCreatedFiles() return nil, false, err } mapped, ok := next.(map[string]any) if !ok { - materializer.rollbackCreatedFiles() return nil, false, &clients.ClientError{ Code: "result_binary_not_materialized", Message: "generated result is not a JSON object", @@ -83,28 +78,28 @@ func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, Retryable: false, } } - if changed && enforceLimits && !writeFiles { - if err := os.MkdirAll(root, 0o750); err != nil { - return nil, false, localBinaryStorageError(err) - } - if err := ensureLocalBinaryDiskHeadroom(root, materializer.totalBytes, s.localResultMinFreeBytes()); err != nil { - return nil, false, err - } - } return mapped, changed, nil } // MaterializeTaskResultForStorage exposes the same verified materialization // path to the explicit historical maintenance command. func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) { - return s.materializeLocalBinaryResult(ctx, taskID, result) + hadInline := TaskResultHasInlineBinary(result) + if !hadInline { + return result, false, nil + } + next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result) + if err != nil { + return nil, false, err + } + return next, hadInline && !TaskResultHasInlineBinary(next), nil } -// CompactExpiredTaskResultForStorage computes the same deterministic -// placeholders without creating files that are already outside the recovery -// window. +// CompactExpiredTaskResultForStorage keeps the historical maintenance API but +// now uses the same object-storage path as live results. New GatewayBinary +// placeholders are never created; their parser remains read-only compatibility. func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) { - return s.transformLocalBinaryResult(ctx, taskID, result, false, false) + return s.MaterializeTaskResultForStorage(ctx, taskID, result) } func TaskResultHasInlineBinary(result map[string]any) bool { @@ -239,15 +234,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b Retryable: false, } } - if m.writeFiles { - created, err := m.service.writeLocalBinaryResult(m.taskDir, digestHex, payload) - if err != nil { - return nil, false, err - } - if created { - m.createdFiles = append(m.createdFiles, filepath.Join(m.taskDir, digestHex+".bin")) - } - } m.seen[digestHex] = struct{}{} m.totalBytes += size } @@ -261,94 +247,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b return localBinaryPlaceholder(descriptor), true, nil } -func (m *localBinaryMaterializer) rollbackCreatedFiles() { - for _, path := range m.createdFiles { - _ = os.Remove(path) - } - _ = os.Remove(m.taskDir) - m.createdFiles = nil -} - -func (s *Service) writeLocalBinaryResult(taskDir string, digestHex string, payload []byte) (bool, error) { - if err := os.MkdirAll(taskDir, 0o750); err != nil { - return false, localBinaryStorageError(err) - } - targetPath := filepath.Join(taskDir, digestHex+".bin") - if info, err := os.Stat(targetPath); err == nil { - if !info.IsDir() && info.Size() == int64(len(payload)) { - if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err == nil { - now := time.Now() - if err := os.Chtimes(targetPath, now, now); err != nil { - return false, localBinaryStorageError(err) - } - return false, nil - } - } - return false, &clients.ClientError{ - Code: "binary_result_corrupted", - Message: "existing local result file does not match its content hash", - StatusCode: 500, - Retryable: false, - } - } else if !errors.Is(err, os.ErrNotExist) { - return false, localBinaryStorageError(err) - } - if err := ensureLocalBinaryDiskHeadroom(taskDir, int64(len(payload)), s.localResultMinFreeBytes()); err != nil { - return false, err - } - tempFile, err := os.CreateTemp(taskDir, ".gateway-result-*") - if err != nil { - return false, localBinaryStorageError(err) - } - tempPath := tempFile.Name() - cleanup := func() { - _ = tempFile.Close() - _ = os.Remove(tempPath) - } - if err := tempFile.Chmod(0o640); err != nil { - cleanup() - return false, localBinaryStorageError(err) - } - if _, err := tempFile.Write(payload); err != nil { - cleanup() - return false, localBinaryStorageError(err) - } - if err := tempFile.Sync(); err != nil { - cleanup() - return false, localBinaryStorageError(err) - } - if err := tempFile.Close(); err != nil { - _ = os.Remove(tempPath) - return false, localBinaryStorageError(err) - } - if err := os.Rename(tempPath, targetPath); err != nil { - _ = os.Remove(tempPath) - return false, localBinaryStorageError(err) - } - if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err != nil { - _ = os.Remove(targetPath) - return false, err - } - return true, nil -} - -func ensureLocalBinaryDiskHeadroom(path string, incomingBytes int64, minFreeBytes int64) error { - var stat syscall.Statfs_t - if err := syscall.Statfs(path, &stat); err != nil { - return localBinaryStorageError(err) - } - freeBytes := int64(stat.Bavail) * int64(stat.Bsize) - if freeBytes-incomingBytes < minFreeBytes { - return &clients.ClientError{ - Code: "local_result_storage_unavailable", - Message: "local result storage does not have enough free space", - StatusCode: 503, - Retryable: false, - } - } - return nil -} - func localBinaryStorageError(err error) error { return &clients.ClientError{ Code: "local_result_storage_unavailable", @@ -406,6 +304,30 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va } switch typed := value.(type) { case map[string]any: + refreshedAccessURL := false + if asset, upload, ok := generatedResultUploadReference(typed); ok { + accessURL, err := s.requestAssetAccessURL(ctx, asset) + if err != nil { + return nil, false, err + } + next := make(map[string]any, len(typed)) + for key, item := range typed { + next[key] = item + } + nextUpload := make(map[string]any, len(upload)) + for key, item := range upload { + nextUpload[key] = item + } + nextUpload["url"] = accessURL + next["upload"] = nextUpload + for _, key := range []string{"url", "image_url", "video_url", "audio_url"} { + if _, exists := next[key]; exists { + next[key] = accessURL + } + } + typed = next + refreshedAccessURL = true + } if ref, ok := generatedResultAssetReference(typed); ok { payload, contentType, err := s.readGeneratedResultAsset(ctx, ref) if err != nil { @@ -418,7 +340,7 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va return encoded, true, nil } next := make(map[string]any, len(typed)) - changed := false + changed := refreshedAccessURL for key, childValue := range typed { child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1) if err != nil { @@ -465,6 +387,27 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va } } +func generatedResultUploadReference(value map[string]any) (store.RequestAsset, map[string]any, bool) { + upload, ok := value["upload"].(map[string]any) + if !ok { + return store.RequestAsset{}, nil, false + } + objectKey := strings.TrimSpace(stringFromAny(upload["objectKey"])) + channel, _ := upload["storageChannel"].(map[string]any) + channelKey := strings.TrimSpace(stringFromAny(channel["channelKey"])) + if objectKey == "" || channelKey == "" { + return store.RequestAsset{}, nil, false + } + return store.RequestAsset{ + URL: stringFromAny(upload["url"]), + StorageProvider: stringFromAny(channel["provider"]), + StorageChannelID: stringFromAny(channel["id"]), + StorageChannelKey: channelKey, + ObjectKey: objectKey, + AccessScope: stringFromAny(upload["accessScope"]), + }, upload, true +} + func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) { ref, ok := value["assetRef"].(map[string]any) if !ok { @@ -475,10 +418,14 @@ func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bo return store.RequestAsset{}, false } asset := store.RequestAsset{ - SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))), - ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])), - URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])), - StorageProvider: stringFromAny(ref["storageProvider"]), + SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))), + ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])), + URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])), + StorageProvider: stringFromAny(ref["storageProvider"]), + StorageChannelID: stringFromAny(ref["storageChannelId"]), + StorageChannelKey: stringFromAny(ref["storageChannelKey"]), + ObjectKey: stringFromAny(ref["objectKey"]), + AccessScope: stringFromAny(ref["accessScope"]), } if size := floatFromAny(ref["size"]); size > 0 { asset.ByteSize = int64(size) @@ -767,13 +714,6 @@ func (s *Service) localResultTTLHours() int { return s.cfg.LocalResultTTLHours } -func (s *Service) localResultMinFreeBytes() int64 { - if s.cfg.LocalResultMinFreeBytes <= 0 { - return defaultLocalResultMinFreeBytes - } - return s.cfg.LocalResultMinFreeBytes -} - func (s *Service) localResultMaxBytes() int64 { if s.cfg.LocalResultMaxBytes <= 0 { return defaultLocalResultMaxBytes diff --git a/apps/api/internal/runner/binary_results_test.go b/apps/api/internal/runner/binary_results_test.go index 45d69bf..0eea6e7 100644 --- a/apps/api/internal/runner/binary_results_test.go +++ b/apps/api/internal/runner/binary_results_test.go @@ -31,7 +31,21 @@ func newLocalBinaryTestService(t *testing.T) *Service { }} } -func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) { +func writeHistoricalLocalBinaryFixture(t *testing.T, service *Service, taskID string, payload []byte) string { + t.Helper() + digest := sha256.Sum256(payload) + taskDir := filepath.Join(service.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID)) + if err := os.MkdirAll(taskDir, 0o750); err != nil { + t.Fatalf("create historical result fixture directory: %v", err) + } + path := filepath.Join(taskDir, hex.EncodeToString(digest[:])+".bin") + if err := os.WriteFile(path, payload, 0o640); err != nil { + t.Fatalf("write historical result fixture: %v", err) + } + return path +} + +func TestHydrateHistoricalLocalBinaryResult(t *testing.T) { service := newLocalBinaryTestService(t) payload := []byte("one binary result shared across representations") encoded := base64.StdEncoding.EncodeToString(payload) @@ -49,7 +63,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) { }, } - persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input) + persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-123", input, false, true) if err != nil { t.Fatalf("materialize local binary result: %v", err) } @@ -76,6 +90,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) { if len(persistentJSON) > 32*1024 { t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON)) } + writeHistoricalLocalBinaryFixture(t, service, "task-123", payload) taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123") entries, err := os.ReadDir(taskDir) @@ -155,19 +170,53 @@ func TestHydrateGeneratedResultAssetReference(t *testing.T) { assertClientErrorCode(t, err, "binary_result_corrupted") } +func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) { + cfg := config.Config{ + MediaOSSDirectEnabled: true, + MediaOSSEndpoint: "https://oss.example.com", + MediaOSSBucket: "media-bucket", + MediaOSSAccessKeyID: "access-id", + MediaOSSAccessKeySecret: "access-secret", + MediaOSSObjectPrefix: "media", + } + service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)} + result := map[string]any{ + "data": []any{map[string]any{ + "url": "https://expired.example.com/result.png", + "image_url": "https://expired.example.com/result.png", + "upload": map[string]any{ + "url": "https://expired.example.com/result.png", + "objectKey": "media/image_result/2026/08/04/hash.png", + "accessScope": "private", + "storageChannel": map[string]any{ + "channelKey": "environment-direct-oss", + "provider": "aliyun_oss", + }, + }, + }}, + } + hydrated, err := service.HydrateTaskResult(t.Context(), "task-private-url", result) + if err != nil { + t.Fatal(err) + } + item := hydrated["data"].([]any)[0].(map[string]any) + for _, key := range []string{"url", "image_url"} { + value := stringFromAny(item[key]) + if !strings.Contains(value, "OSSAccessKeyId=access-id") || strings.Contains(value, "expired.example.com") { + t.Fatalf("%s was not refreshed: %q", key, value) + } + } +} + func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) { service := newLocalBinaryTestService(t) service.cfg.LocalResultTTLHours = 1 encoded := base64.StdEncoding.EncodeToString([]byte("expiring result")) - persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}) + persistent, _, err := service.transformLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}, false, true) if err != nil { t.Fatalf("materialize fixture: %v", err) } - entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired")) - if err != nil || len(entries) != 1 { - t.Fatalf("read fixture result: entries=%v err=%v", entries, err) - } - path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name()) + path := writeHistoricalLocalBinaryFixture(t, service, "task-expired", []byte("expiring result")) old := time.Now().Add(-2 * time.Hour) if err := os.Chtimes(path, old, old); err != nil { t.Fatalf("age fixture: %v", err) @@ -186,17 +235,17 @@ func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) assertClientErrorCode(t, err, "binary_result_corrupted") } -func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) { +func TestHistoricalLocalBinaryFixtureEnforcesLimitsAndKeepsText(t *testing.T) { service := newLocalBinaryTestService(t) service.cfg.LocalResultMaxBytes = 4 - _, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{ + _, _, err := service.transformLocalBinaryResult(context.Background(), "task-large", map[string]any{ "b64_json": base64.StdEncoding.EncodeToString([]byte("too large")), - }) + }, false, true) assertClientErrorCode(t, err, "binary_result_too_large") - persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{ + persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-text", map[string]any{ "message": "YWJjZA==", - }) + }, false, true) if err != nil || changed || persistent["message"] != "YWJjZA==" { t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err) } @@ -204,18 +253,30 @@ func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) { service = newLocalBinaryTestService(t) service.cfg.LocalResultMaxBytes = 1024 service.cfg.LocalResultMaxTaskBytes = 8 - _, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{ + _, _, err = service.transformLocalBinaryResult(context.Background(), "task-total-large", map[string]any{ "first_base64": base64.StdEncoding.EncodeToString([]byte("12345")), "second_base64": base64.StdEncoding.EncodeToString([]byte("67890")), - }) + }, false, true) assertClientErrorCode(t, err, "binary_result_too_large") if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) { t.Fatalf("preflight task limit must not create partial files: %v", statErr) } } -func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) { +func TestCompactExpiredTaskResultUsesObjectStorageAndDoesNotWriteFiles(t *testing.T) { + var uploads int + storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + t.Fatalf("method=%s, want PUT", r.Method) + } + uploads++ + w.WriteHeader(http.StatusOK) + })) + defer storageServer.Close() service := newLocalBinaryTestService(t) + service.directOSS = &directOSSUploader{ + endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media", + } service.cfg.LocalResultMaxBytes = 1 encoded := base64.StdEncoding.EncodeToString([]byte("already expired result")) @@ -225,17 +286,18 @@ func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) { if err != nil { t.Fatalf("compact expired result: %v", err) } - if !changed || !localBinaryResultHasPlaceholders(persistent) { - t.Fatalf("expired result was not compacted: %+v", persistent) + if !changed || localBinaryResultHasPlaceholders(persistent) || TaskResultHasInlineBinary(persistent) { + t.Fatalf("expired result was not objectified: %+v", persistent) + } + if uploads != 1 { + t.Fatalf("object storage uploads=%d, want 1", uploads) } if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) { t.Fatalf("expired compaction must not create a task directory: %v", err) } - _, err = service.HydrateTaskResult(context.Background(), "task-old", persistent) - assertClientErrorCode(t, err, "binary_result_expired") } -func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) { +func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) { err := localBinaryStorageError(errors.New("write failed")) assertClientErrorCode(t, err, "local_result_storage_unavailable") if clients.IsRetryable(err) { @@ -247,16 +309,6 @@ func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) { if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry { t.Fatal("local result storage failure must not fail over to another provider") } - - service := newLocalBinaryTestService(t) - service.cfg.LocalResultMinFreeBytes = 1 << 62 - _, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{ - "b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")), - }) - assertClientErrorCode(t, err, "local_result_storage_unavailable") - if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) { - t.Fatalf("disk preflight must not create partial task files: %v", statErr) - } } func bytesToAny(payload []byte) []any { diff --git a/apps/api/internal/runner/direct_oss.go b/apps/api/internal/runner/direct_oss.go index 8581fda..db10342 100644 --- a/apps/api/internal/runner/direct_oss.go +++ b/apps/api/internal/runner/direct_oss.go @@ -1,26 +1,12 @@ package runner import ( - "bytes" - "context" - "crypto/hmac" - "crypto/sha1" - "encoding/base64" - "fmt" - "io" - "net/http" - "net/url" - "path" "strings" - "time" - "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) -const directOSSUploadTimeout = 120 * time.Second - type directOSSUploader struct { endpoint string bucket string @@ -28,8 +14,6 @@ type directOSSUploader struct { accessKeySecret string publicBaseURL string objectPrefix string - client *http.Client - now func() time.Time } func newDirectOSSUploader(cfg config.Config) *directOSSUploader { @@ -43,133 +27,41 @@ func newDirectOSSUploader(cfg config.Config) *directOSSUploader { accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret), publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"), objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"), - client: &http.Client{ - Timeout: directOSSUploadTimeout, - Transport: &http.Transport{ - MaxIdleConns: 256, - MaxIdleConnsPerHost: 256, - MaxConnsPerHost: 256, - IdleConnTimeout: 90 * time.Second, - }, - }, - now: time.Now, } } func directOSSScene(scene string) bool { switch strings.TrimSpace(scene) { - case store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult: + case store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult: return true default: return false } } -func (u *directOSSUploader) upload(ctx context.Context, payload FileUploadPayload) (map[string]any, error) { +func (u *directOSSUploader) fileStorageChannel() store.FileStorageChannel { if u == nil { - return nil, &clients.ClientError{Code: "upload_config_failed", Message: "direct OSS uploader is not configured", Retryable: false} + return store.FileStorageChannel{} } - objectKey := u.objectKey(payload) - escapedKey := escapeOSSObjectKey(objectKey) - uploadURL := u.endpoint + "/" + escapedKey - contentType := strings.TrimSpace(payload.ContentType) - if contentType == "" { - contentType = "application/octet-stream" + return store.FileStorageChannel{ + ChannelKey: "environment-direct-oss", + Name: "Environment Direct OSS", + Provider: "aliyun_oss", + AccessKeyID: u.accessKeyID, + AccessKeySecret: u.accessKeySecret, + Priority: 10, + Status: "enabled", + Scenes: []string{store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult}, + Config: map[string]any{ + "endpoint": u.endpoint, + "bucket": u.bucket, + "publicBaseUrl": u.publicBaseURL, + "objectPrefix": u.objectPrefix, + }, + RetryPolicy: map[string]any{ + "enabled": true, + "maxRetries": 2, + "backoffSeconds": []any{0.25, 1.0}, + }, } - - var lastErr error - for attempt := 0; attempt < 3; attempt++ { - if attempt > 0 { - if err := sleepWithContext(ctx, time.Duration(attempt*attempt)*200*time.Millisecond); err != nil { - return nil, err - } - } - status, err := u.put(ctx, uploadURL, escapedKey, contentType, payload.Bytes) - if err == nil && status >= 200 && status < 300 { - publicURL := u.publicBaseURL + "/" + escapedKey - return map[string]any{ - "url": publicURL, - "fileName": objectKey, - "storageChannel": map[string]any{ - "channelKey": "environment-direct-oss", - "name": "Environment Direct OSS", - "provider": "aliyun_oss_direct", - }, - "assetStorage": map[string]any{ - "scene": payload.Scene, - "source": firstNonEmptyString(payload.Source, "ai-gateway"), - "strategy": "direct_aliyun_oss", - }, - }, nil - } - if err != nil { - lastErr = err - continue - } - lastErr = fmt.Errorf("HTTP %d", status) - if status != http.StatusTooManyRequests && status < 500 { - break - } - } - message := "direct OSS upload failed" - if lastErr != nil { - message += ": " + lastErr.Error() - } - return nil, &clients.ClientError{Code: "upload_failed", Message: message, Retryable: true} -} - -func (u *directOSSUploader) put(ctx context.Context, uploadURL string, escapedKey string, contentType string, payload []byte) (int, error) { - date := u.now().UTC().Format(http.TimeFormat) - canonicalResource := "/" + u.bucket + "/" + escapedKey - stringToSign := "PUT\n\n" + contentType + "\n" + date + "\n" + canonicalResource - mac := hmac.New(sha1.New, []byte(u.accessKeySecret)) - _, _ = mac.Write([]byte(stringToSign)) - signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) - - req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(payload)) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", "OSS "+u.accessKeyID+":"+signature) - req.Header.Set("Content-Type", contentType) - req.Header.Set("Date", date) - resp, err := u.client.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) - return resp.StatusCode, nil -} - -func (u *directOSSUploader) objectKey(payload FileUploadPayload) string { - now := u.now().UTC() - extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName)) - baseName := strings.TrimSuffix(path.Base(payload.FileName), path.Ext(payload.FileName)) - baseName = sanitizeGeneratedAssetNamePart(baseName) - if baseName == "" { - baseName = "gateway-media" - } - if len(baseName) > 48 { - baseName = baseName[:48] - } - return fmt.Sprintf( - "%s/%s/%04d/%02d/%02d/%s-%s%s", - u.objectPrefix, - strings.TrimSpace(payload.Scene), - now.Year(), - now.Month(), - now.Day(), - baseName, - randomHexSuffix(8), - extension, - ) -} - -func escapeOSSObjectKey(objectKey string) string { - parts := strings.Split(objectKey, "/") - for index, part := range parts { - parts[index] = url.PathEscape(part) - } - return strings.Join(parts, "/") } diff --git a/apps/api/internal/runner/direct_oss_test.go b/apps/api/internal/runner/direct_oss_test.go index 20330d8..b222cc1 100644 --- a/apps/api/internal/runner/direct_oss_test.go +++ b/apps/api/internal/runner/direct_oss_test.go @@ -12,13 +12,12 @@ import ( "strings" "sync/atomic" "testing" - "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) -func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) { +func TestDirectOSSUploadSignsRequestAndReturnsPrivateSignedURL(t *testing.T) { payload := []byte("production-isomorphic-image") var calls atomic.Int64 var uploadedPath string @@ -60,9 +59,6 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) { MediaOSSObjectPrefix: "easyai-ai-gateway/production/media", } uploader := newDirectOSSUploader(cfg) - uploader.now = func() time.Time { - return time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC) - } service := &Service{cfg: cfg, directOSS: uploader} uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{ Bytes: payload, @@ -77,29 +73,30 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) { if calls.Load() != 2 { t.Fatalf("upload calls=%d, want one retry", calls.Load()) } - if !strings.HasPrefix(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/2026/07/30/input-") || - !strings.HasSuffix(uploadedPath, ".png") { + if !strings.Contains(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/") || + !strings.HasSuffix(uploadedPath, "/"+sha256Hex(payload)+".png") { t.Fatalf("unexpected object path=%q", uploadedPath) } - if got := stringFromAny(uploaded["url"]); got != "https://cdn.example.com"+uploadedPath { - t.Fatalf("public URL=%q", got) + if got := stringFromAny(uploaded["url"]); !strings.HasPrefix(got, server.URL+uploadedPath+"?") || !strings.Contains(got, "Signature=") { + t.Fatalf("private signed URL=%q", got) + } + if uploaded["accessScope"] != "private" { + t.Fatalf("request asset access scope=%v", uploaded["accessScope"]) } channel, _ := uploaded["storageChannel"].(map[string]any) - if stringFromAny(channel["provider"]) != "aliyun_oss_direct" { + if stringFromAny(channel["provider"]) != "aliyun_oss" { t.Fatalf("storage channel=%+v", channel) } } -func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) { +func TestDirectOSSPersistsGeneralUploadScene(t *testing.T) { var calls atomic.Int64 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { calls.Add(1) w.WriteHeader(http.StatusOK) })) defer server.Close() - storageDir := t.TempDir() cfg := config.Config{ - LocalUploadedStorageDir: storageDir, MediaOSSDirectEnabled: true, MediaOSSEndpoint: server.URL, MediaOSSBucket: "media-bucket", @@ -118,11 +115,11 @@ func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) { if err != nil { t.Fatalf("general upload: %v", err) } - if calls.Load() != 0 { + if calls.Load() != 1 { t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load()) } channel, _ := uploaded["storageChannel"].(map[string]any) - if stringFromAny(channel["provider"]) != "local_static" { + if stringFromAny(channel["provider"]) != "aliyun_oss" { t.Fatalf("general upload channel=%+v", channel) } } @@ -158,7 +155,7 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) { SourceKey: "b64_json", }, 0, - []store.FileStorageChannel{{ID: "unused-channel"}}, + []store.FileStorageChannel{service.directOSS.fileStorageChannel()}, ) if err != nil { t.Fatalf("direct generated upload: %v", err) @@ -166,11 +163,11 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) { if calls.Load() != 1 { t.Fatalf("direct OSS calls=%d, want 1", calls.Load()) } - if contentType != "image/png" || kind != "image" || strategy != "direct_aliyun_oss" { + if contentType != "image/png" || kind != "image" || strategy != "upload_inline_media" { t.Fatalf("generated metadata=%s/%s/%s", contentType, kind, strategy) } channel, _ := upload["storageChannel"].(map[string]any) - if stringFromAny(channel["provider"]) != "aliyun_oss_direct" { + if stringFromAny(channel["provider"]) != "aliyun_oss" { t.Fatalf("storage channel=%+v", channel) } } diff --git a/apps/api/internal/runner/object_storage.go b/apps/api/internal/runner/object_storage.go new file mode 100644 index 0000000..e5c5f06 --- /dev/null +++ b/apps/api/internal/runner/object_storage.go @@ -0,0 +1,530 @@ +package runner + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net" + "net/http" + "net/url" + "path" + "sort" + "strconv" + "strings" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" + "github.com/google/uuid" +) + +const ( + objectStorageRequestTimeout = 120 * time.Second + objectStorageReadLimit = 256 << 20 + objectStorageSignedURLTTL = 15 * time.Minute +) + +var sharedObjectStorageHTTPClient = &http.Client{ + Timeout: objectStorageRequestTimeout, + Transport: &http.Transport{ + MaxIdleConns: 256, + MaxIdleConnsPerHost: 256, + MaxConnsPerHost: 256, + IdleConnTimeout: 90 * time.Second, + }, +} + +type objectStorageAdapter struct { + channel store.FileStorageChannel + client *http.Client + now func() time.Time +} + +type FileStorageChannelTestResult struct { + Provider string `json:"provider"` + PutSucceeded bool `json:"putSucceeded"` + HeadSucceeded bool `json:"headSucceeded"` + DeleteSucceeded bool `json:"deleteSucceeded"` + DurationMS int64 `json:"durationMs"` +} + +// TestFileStorageChannel performs an isolated write, metadata read and cleanup +// against one object-storage channel. The random probe avoids overwriting a +// content-addressed business object and no object key or credential is exposed. +func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) { + startedAt := time.Now() + result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))} + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + return result, err + } + payload := FileUploadPayload{ + Bytes: []byte("easyai-storage-probe:" + uuid.NewString()), + ContentType: "application/octet-stream", + FileName: "probe.bin", + Scene: store.FileStorageSceneUpload, + Source: "admin-connection-test", + } + upload, err := adapter.put(ctx, payload) + if err != nil { + return result, err + } + result.PutSucceeded = true + objectKey := stringFromAny(upload["objectKey"]) + if objectKey == "" { + return result, storageClientError("storage_write_failed", "object storage probe did not return an object reference", 0, false) + } + if err := adapter.head(ctx, objectKey); err != nil { + _ = adapter.delete(context.WithoutCancel(ctx), objectKey) + return result, err + } + result.HeadSucceeded = true + if err := adapter.delete(ctx, objectKey); err != nil { + return result, err + } + result.DeleteSucceeded = true + result.DurationMS = time.Since(startedAt).Milliseconds() + return result, nil +} + +func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAdapter, error) { + provider := strings.ToLower(strings.TrimSpace(channel.Provider)) + if provider != "aliyun_oss" && provider != "s3" { + return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported object storage provider", Retryable: false} + } + if objectStorageConfigString(channel.Config, "endpoint") == "" || objectStorageConfigString(channel.Config, "bucket") == "" { + return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "object storage endpoint and bucket are required", Retryable: false} + } + if strings.TrimSpace(channel.AccessKeyID) == "" || strings.TrimSpace(channel.AccessKeySecret) == "" { + return nil, &clients.ClientError{Code: "storage_auth_failed", Message: "object storage credentials are not configured", Retryable: false} + } + return &objectStorageAdapter{ + channel: channel, + client: sharedObjectStorageHTTPClient, + now: time.Now, + }, nil +} + +func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) { + objectKey := a.objectKey(payload) + requestURL, err := a.objectURL(objectKey) + if err != nil { + return nil, err + } + contentType := strings.TrimSpace(payload.ContentType) + if contentType == "" { + contentType = "application/octet-stream" + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload.Bytes)) + if err != nil { + return nil, storageClientError("storage_config_invalid", err.Error(), 0, false) + } + req.Header.Set("Content-Type", contentType) + if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil { + return nil, err + } + resp, err := a.client.Do(req) + if err != nil { + return nil, storageClientError("storage_write_failed", err.Error(), 0, true) + } + defer resp.Body.Close() + responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + if readErr != nil { + return nil, storageClientError("storage_write_failed", readErr.Error(), resp.StatusCode, storageStatusRetryable(resp.StatusCode)) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, objectStorageHTTPError("storage_write_failed", resp.StatusCode, responseBody) + } + accessScope := objectStorageAccessScope(a.channel, payload.Scene) + publicURL := "" + if accessScope == "public" { + publicURL = a.publicURL(objectKey) + } + urlExpiresAt := "" + if publicURL == "" { + publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL) + if err != nil { + return nil, err + } + urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339) + } + digest := sha256.Sum256(payload.Bytes) + result := map[string]any{ + "url": publicURL, + "fileName": path.Base(objectKey), + "objectKey": objectKey, + "contentType": contentType, + "size": len(payload.Bytes), + "sha256": hex.EncodeToString(digest[:]), + "accessScope": accessScope, + "storageChannel": map[string]any{ + "id": a.channel.ID, + "channelKey": a.channel.ChannelKey, + "name": a.channel.Name, + "provider": a.channel.Provider, + }, + } + if urlExpiresAt != "" { + result["urlExpiresAt"] = urlExpiresAt + } + return result, nil +} + +func (a *objectStorageAdapter) get(ctx context.Context, objectKey string) ([]byte, error) { + requestURL, err := a.objectURL(objectKey) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, storageClientError("storage_config_invalid", err.Error(), 0, false) + } + if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil { + return nil, err + } + resp, err := a.client.Do(req) + if err != nil { + return nil, storageClientError("storage_read_failed", err.Error(), 0, true) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + return nil, objectStorageHTTPError("storage_read_failed", resp.StatusCode, body) + } + payload, err := io.ReadAll(io.LimitReader(resp.Body, objectStorageReadLimit+1)) + if err != nil { + return nil, storageClientError("storage_read_failed", err.Error(), resp.StatusCode, true) + } + if len(payload) > objectStorageReadLimit { + return nil, storageClientError("storage_read_failed", "stored object exceeds the read limit", resp.StatusCode, false) + } + return payload, nil +} + +func (a *objectStorageAdapter) head(ctx context.Context, objectKey string) error { + return a.emptyObjectRequest(ctx, http.MethodHead, objectKey) +} + +func (a *objectStorageAdapter) delete(ctx context.Context, objectKey string) error { + return a.emptyObjectRequest(ctx, http.MethodDelete, objectKey) +} + +func (a *objectStorageAdapter) emptyObjectRequest(ctx context.Context, method string, objectKey string) error { + requestURL, err := a.objectURL(objectKey) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, method, requestURL, nil) + if err != nil { + return storageClientError("storage_config_invalid", err.Error(), 0, false) + } + if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil { + return err + } + resp, err := a.client.Do(req) + if err != nil { + return storageClientError("storage_read_failed", err.Error(), 0, true) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return objectStorageHTTPError("storage_read_failed", resp.StatusCode, nil) + } + return nil +} + +func (a *objectStorageAdapter) presignGet(objectKey string, ttl time.Duration) (string, error) { + requestURL, err := a.objectURL(objectKey) + if err != nil { + return "", err + } + if ttl <= 0 { + ttl = objectStorageSignedURLTTL + } + if strings.EqualFold(a.channel.Provider, "s3") { + return a.presignS3Get(requestURL, ttl, a.now().UTC()) + } + return a.presignOSSGet(requestURL, objectKey, ttl, a.now().UTC()) +} + +func (a *objectStorageAdapter) objectKey(payload FileUploadPayload) string { + now := a.now().UTC() + digest := sha256.Sum256(payload.Bytes) + extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName)) + prefix := strings.Trim(objectStorageConfigString(a.channel.Config, "objectPrefix"), "/") + parts := make([]string, 0, 7) + if prefix != "" { + parts = append(parts, prefix) + } + parts = append(parts, + firstNonEmptyString(strings.TrimSpace(payload.Scene), store.FileStorageSceneUpload), + fmt.Sprintf("%04d", now.Year()), + fmt.Sprintf("%02d", now.Month()), + fmt.Sprintf("%02d", now.Day()), + hex.EncodeToString(digest[:])+extension, + ) + return strings.Join(parts, "/") +} + +func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) { + endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/")) + if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil { + return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false) + } + bucket := objectStorageConfigString(a.channel.Config, "bucket") + if strings.EqualFold(a.channel.Provider, "s3") { + if objectStorageConfigBool(a.channel.Config, "forcePathStyle") { + endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + bucket + "/" + strings.TrimLeft(objectKey, "/") + } else { + endpoint.Host = bucket + "." + endpoint.Host + endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/") + } + } else { + hostname := endpoint.Hostname() + forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle") + if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") { + if port := endpoint.Port(); port != "" { + endpoint.Host = bucket + "." + hostname + ":" + port + } else { + endpoint.Host = bucket + "." + endpoint.Host + } + } + endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/") + } + return endpoint.String(), nil +} + +func (a *objectStorageAdapter) publicURL(objectKey string) string { + baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/") + if baseURL == "" { + baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/") + } + if baseURL == "" { + return "" + } + return baseURL + "/" + escapeObjectKey(objectKey) +} + +func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now time.Time) error { + if strings.EqualFold(a.channel.Provider, "s3") { + a.signS3Request(req, payloadHash, now) + return nil + } + a.signOSSRequest(req, now) + return nil +} + +func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) { + date := now.UTC().Format(http.TimeFormat) + contentType := req.Header.Get("Content-Type") + canonicalHeaders := "" + if token := strings.TrimSpace(a.channel.SessionToken); token != "" { + req.Header.Set("x-oss-security-token", token) + canonicalHeaders = "x-oss-security-token:" + token + "\n" + } + canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath() + stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource + signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign) + req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature) + req.Header.Set("Date", date) +} + +func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) { + parsed, err := url.Parse(requestURL) + if err != nil { + return "", storageClientError("storage_config_invalid", err.Error(), 0, false) + } + expires := strconv.FormatInt(now.Add(ttl).Unix(), 10) + canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + "/" + strings.TrimLeft(objectKey, "/") + signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), "GET\n\n\n"+expires+"\n"+canonicalResource) + query := parsed.Query() + query.Set("OSSAccessKeyId", a.channel.AccessKeyID) + query.Set("Expires", expires) + query.Set("Signature", signature) + if token := strings.TrimSpace(a.channel.SessionToken); token != "" { + query.Set("security-token", token) + } + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func (a *objectStorageAdapter) signS3Request(req *http.Request, payloadHash string, now time.Time) { + region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1") + amzDate := now.Format("20060102T150405Z") + date := now.Format("20060102") + req.Header.Set("x-amz-date", amzDate) + req.Header.Set("x-amz-content-sha256", payloadHash) + if token := strings.TrimSpace(a.channel.SessionToken); token != "" { + req.Header.Set("x-amz-security-token", token) + } + canonicalHeaders, signedHeaders := s3CanonicalHeaders(req) + canonicalRequest := strings.Join([]string{ + req.Method, + s3CanonicalURI(req.URL), + s3CanonicalQuery(req.URL.Query()), + canonicalHeaders, + signedHeaders, + payloadHash, + }, "\n") + scope := date + "/" + region + "/s3/aws4_request" + stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest)) + signature := hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign)) + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+a.channel.AccessKeyID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+signature) +} + +func (a *objectStorageAdapter) presignS3Get(requestURL string, ttl time.Duration, now time.Time) (string, error) { + parsed, err := url.Parse(requestURL) + if err != nil { + return "", storageClientError("storage_config_invalid", err.Error(), 0, false) + } + region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1") + amzDate := now.Format("20060102T150405Z") + date := now.Format("20060102") + scope := date + "/" + region + "/s3/aws4_request" + seconds := int64(ttl / time.Second) + if seconds < 1 { + seconds = 1 + } + if seconds > 7*24*60*60 { + seconds = 7 * 24 * 60 * 60 + } + query := parsed.Query() + query.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256") + query.Set("X-Amz-Credential", a.channel.AccessKeyID+"/"+scope) + query.Set("X-Amz-Date", amzDate) + query.Set("X-Amz-Expires", strconv.FormatInt(seconds, 10)) + query.Set("X-Amz-SignedHeaders", "host") + if token := strings.TrimSpace(a.channel.SessionToken); token != "" { + query.Set("X-Amz-Security-Token", token) + } + canonicalRequest := "GET\n" + s3CanonicalURI(parsed) + "\n" + s3CanonicalQuery(query) + "\nhost:" + strings.ToLower(parsed.Host) + "\n\nhost\nUNSIGNED-PAYLOAD" + stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest)) + query.Set("X-Amz-Signature", hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign))) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func s3CanonicalHeaders(req *http.Request) (string, string) { + headers := map[string]string{"host": strings.ToLower(req.URL.Host)} + for key, values := range req.Header { + lower := strings.ToLower(strings.TrimSpace(key)) + if lower == "content-type" || strings.HasPrefix(lower, "x-amz-") { + headers[lower] = strings.Join(values, ",") + } + } + keys := make([]string, 0, len(headers)) + for key := range headers { + keys = append(keys, key) + } + sort.Strings(keys) + var canonical strings.Builder + for _, key := range keys { + canonical.WriteString(key) + canonical.WriteByte(':') + canonical.WriteString(strings.Join(strings.Fields(headers[key]), " ")) + canonical.WriteByte('\n') + } + return canonical.String(), strings.Join(keys, ";") +} + +func s3CanonicalURI(value *url.URL) string { + uri := value.EscapedPath() + if uri == "" { + return "/" + } + return uri +} + +func s3CanonicalQuery(values url.Values) string { + return strings.ReplaceAll(values.Encode(), "+", "%20") +} + +func s3SigningHMAC(secret string, date string, region string, stringToSign string) []byte { + dateKey := hmacSHA256([]byte("AWS4"+secret), date) + regionKey := hmacSHA256(dateKey, region) + serviceKey := hmacSHA256(regionKey, "s3") + signingKey := hmacSHA256(serviceKey, "aws4_request") + return hmacSHA256(signingKey, stringToSign) +} + +func hmacSHA256(key []byte, value string) []byte { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write([]byte(value)) + return mac.Sum(nil) +} + +func hmacSHA1Base64(key []byte, value string) string { + mac := hmac.New(sha1.New, key) + _, _ = mac.Write([]byte(value)) + return base64.StdEncoding.EncodeToString(mac.Sum(nil)) +} + +func sha256Hex(value []byte) string { + digest := sha256.Sum256(value) + return hex.EncodeToString(digest[:]) +} + +func objectStorageConfigString(config map[string]any, key string) string { + if config == nil { + return "" + } + value, _ := config[key].(string) + return strings.TrimSpace(value) +} + +func objectStorageConfigBool(config map[string]any, key string) bool { + if config == nil { + return false + } + value, _ := config[key].(bool) + return value +} + +func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string { + if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" { + return scope + } + if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset { + return "private" + } + if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" { + return "public" + } + return "private" +} + +func escapeObjectKey(objectKey string) string { + parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/") + for index, part := range parts { + parts[index] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +func storageStatusRetryable(status int) bool { + return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500 +} + +func objectStorageHTTPError(code string, status int, body []byte) error { + if status == http.StatusUnauthorized || status == http.StatusForbidden { + code = "storage_auth_failed" + } + message := http.StatusText(status) + if message == "" { + message = "object storage request failed" + } + // The provider response may contain object names, endpoints or credential + // diagnostics. Keep it out of the public error chain; channel health stores + // only the stable status description as well. + return storageClientError(code, message, status, storageStatusRetryable(status)) +} + +func storageClientError(code string, message string, status int, retryable bool) error { + return &clients.ClientError{Code: code, Message: message, StatusCode: status, Retryable: retryable} +} diff --git a/apps/api/internal/runner/object_storage_acceptance_test.go b/apps/api/internal/runner/object_storage_acceptance_test.go new file mode 100644 index 0000000..b8b2a9a --- /dev/null +++ b/apps/api/internal/runner/object_storage_acceptance_test.go @@ -0,0 +1,102 @@ +package runner + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceemulator" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestLocalAcceptanceObjectStorageFailoverMatrix(t *testing.T) { + emulator := httptest.NewServer(acceptanceemulator.New(acceptanceemulator.Config{}).Handler()) + defer emulator.Close() + service := &Service{} + payload := FileUploadPayload{ + Bytes: []byte("acceptance-object"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset, + } + + tests := []struct { + name string + channels []store.FileStorageChannel + winner string + wantErrCode string + }{ + { + name: "oss_auth_to_s3", + channels: []store.FileStorageChannel{ + acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-auth/bucket", "oss-auth", false), + acceptanceStorageChannel("s3", emulator.URL+"/storage/s3", "s3-ok", false), + }, + winner: "s3-ok", + }, + { + name: "s3_auth_to_oss", + channels: []store.FileStorageChannel{ + acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-auth", "s3-auth", false), + acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-ok", false), + }, + winner: "oss-ok", + }, + { + name: "same_channel_transient_retry", + channels: []store.FileStorageChannel{ + acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-transient", "s3-transient", true), + acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-unused", false), + }, + winner: "s3-transient", + }, + { + name: "all_channels_failed", + channels: []store.FileStorageChannel{ + acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-fail/bucket", "oss-fail", false), + acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-fail", "s3-fail", false), + }, + wantErrCode: "storage_write_failed", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := service.uploadFileWithFailover(t.Context(), payload, test.channels) + if test.wantErrCode != "" { + if clients.ErrorCode(err) != test.wantErrCode { + t.Fatalf("error=%v code=%s", err, clients.ErrorCode(err)) + } + return + } + if err != nil { + t.Fatal(err) + } + channel, _ := result["storageChannel"].(map[string]any) + if got := stringFromAny(channel["channelKey"]); got != test.winner { + t.Fatalf("winner=%q, want %q", got, test.winner) + } + }) + } + + response, err := http.Get(emulator.URL + "/report") + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var report acceptanceemulator.Report + if err := json.NewDecoder(response.Body).Decode(&report); err != nil { + t.Fatal(err) + } + if report.StoragePuts != 3 || report.StorageFailures != 5 { + t.Fatalf("unexpected storage acceptance report: %+v", report) + } +} + +func acceptanceStorageChannel(provider string, endpoint string, key string, retry bool) store.FileStorageChannel { + channel := testObjectStorageChannel(provider, endpoint, key) + channel.Config["publicBaseUrl"] = "" + channel.RetryPolicy = map[string]any{"enabled": retry, "maxRetries": 0} + if retry { + channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}} + } + return channel +} diff --git a/apps/api/internal/runner/object_storage_test.go b/apps/api/internal/runner/object_storage_test.go new file mode 100644 index 0000000..29e7904 --- /dev/null +++ b/apps/api/internal/runner/object_storage_test.go @@ -0,0 +1,314 @@ +package runner + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +type objectStorageMetricEvent struct { + event string + provider string + bytes int64 +} + +type objectStorageMetricsRecorder struct { + events []objectStorageMetricEvent +} + +func (m *objectStorageMetricsRecorder) ObserveBillingEvent(string) {} + +func (m *objectStorageMetricsRecorder) ObserveObjectStorage(event string, provider string, bytes int64, _ time.Duration) { + m.events = append(m.events, objectStorageMetricEvent{event: event, provider: provider, bytes: bytes}) +} + +func TestS3ObjectStorageUsesSigV4AndDeterministicObjectKey(t *testing.T) { + payload := []byte("same-media-payload") + var calls atomic.Int64 + var requestPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + requestPath = r.URL.Path + if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=access-id/") { + t.Errorf("missing SigV4 authorization: %q", r.Header.Get("Authorization")) + } + if r.Header.Get("x-amz-content-sha256") != sha256Hex(payload) { + t.Errorf("payload hash=%q", r.Header.Get("x-amz-content-sha256")) + } + body, _ := io.ReadAll(r.Body) + if string(body) != string(payload) { + t.Errorf("payload=%q", body) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + channel := testObjectStorageChannel("s3", server.URL, "s3-primary") + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + t.Fatal(err) + } + adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC) } + result, err := adapter.put(t.Context(), FileUploadPayload{Bytes: payload, ContentType: "image/png", FileName: "ignored.jpg", Scene: store.FileStorageSceneImageResult}) + if err != nil { + t.Fatal(err) + } + wantSuffix := "/bucket/media/image_result/2026/08/04/" + sha256Hex(payload) + ".png" + if requestPath != wantSuffix { + t.Fatalf("request path=%q, want %q", requestPath, wantSuffix) + } + if stringFromAny(result["objectKey"]) != strings.TrimPrefix(wantSuffix, "/bucket/") { + t.Fatalf("object key=%q", result["objectKey"]) + } + if calls.Load() != 1 { + t.Fatalf("calls=%d", calls.Load()) + } +} + +func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + channel := testObjectStorageChannel("s3", server.URL, "s3-private") + channel.Config["publicBaseUrl"] = "https://cdn.example" + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + t.Fatal(err) + } + adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) } + + result, err := adapter.put(t.Context(), FileUploadPayload{ + Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset, + }) + if err != nil { + t.Fatal(err) + } + if result["accessScope"] != "private" { + t.Fatalf("accessScope=%v, want private", result["accessScope"]) + } + gotURL := stringFromAny(result["url"]) + if strings.HasPrefix(gotURL, "https://cdn.example/") || !strings.Contains(gotURL, "X-Amz-Signature=") { + t.Fatalf("request asset URL must be signed and private: %s", gotURL) + } +} + +func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) { + channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary") + channel.Config["forcePathStyle"] = false + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + t.Fatal(err) + } + value, err := adapter.objectURL("media/request_asset/object.png") + if err != nil { + t.Fatal(err) + } + if value != "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/request_asset/object.png" { + t.Fatalf("OSS object URL=%q", value) + } +} + +func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) { + channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary") + channel.Config["forcePathStyle"] = false + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + t.Fatal(err) + } + request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/2026/08/04/OBJECT.png", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Content-Type", "image/png") + adapter.signOSSRequest(request, time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC)) + if got, want := request.Header.Get("Authorization"), "OSS access-id:fohlh2SG7k7D+cmX48b7Keqa1Ok="; got != want { + t.Fatalf("authorization=%q, want %q", got, want) + } + if got := request.Header.Get("Date"); got != "Tue, 04 Aug 2026 01:02:03 GMT" { + t.Fatalf("date=%q", got) + } +} + +func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) { + var primaryCalls atomic.Int64 + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + primaryCalls.Add(1) + http.Error(w, "temporary", http.StatusServiceUnavailable) + })) + defer primary.Close() + var secondaryCalls atomic.Int64 + secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + secondaryCalls.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer secondary.Close() + + first := testObjectStorageChannel("s3", primary.URL, "s3-primary") + first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}} + second := testObjectStorageChannel("aliyun_oss", secondary.URL, "oss-secondary") + second.RetryPolicy = map[string]any{"enabled": false} + metrics := &objectStorageMetricsRecorder{} + service := &Service{billingMetrics: metrics} + result, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second}) + if err != nil { + t.Fatal(err) + } + channel, _ := result["storageChannel"].(map[string]any) + if stringFromAny(channel["channelKey"]) != "oss-secondary" { + t.Fatalf("unexpected winning channel: %+v", channel) + } + if primaryCalls.Load() != 2 || secondaryCalls.Load() != 1 { + t.Fatalf("calls primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load()) + } + assertObjectStorageMetricEvent(t, metrics.events, "retry", "s3", 0) + assertObjectStorageMetricEvent(t, metrics.events, "failover", "s3", 0) + assertObjectStorageMetricEvent(t, metrics.events, "write_success", "aliyun_oss", int64(len("payload"))) +} + +func TestFileStorageChannelConnectionProbeWritesHeadsAndDeletes(t *testing.T) { + var methods []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + switch r.Method { + case http.MethodPut, http.MethodHead, http.MethodDelete: + w.WriteHeader(http.StatusOK) + default: + http.Error(w, "unexpected method", http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + result, err := (&Service{}).TestFileStorageChannel(t.Context(), testObjectStorageChannel("s3", server.URL, "probe")) + if err != nil { + t.Fatal(err) + } + if !result.PutSucceeded || !result.HeadSucceeded || !result.DeleteSucceeded || result.Provider != "s3" { + t.Fatalf("unexpected probe result: %+v", result) + } + if strings.Join(methods, ",") != "PUT,HEAD,DELETE" { + t.Fatalf("probe methods=%v", methods) + } +} + +func TestObjectStorageReadUsesChannelRetryPolicy(t *testing.T) { + var calls atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + http.Error(w, "temporary", http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte("stored-payload")) + })) + defer server.Close() + channel := testObjectStorageChannel("s3", server.URL, "s3-read") + channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}} + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + t.Fatal(err) + } + payload, err := readObjectStorageWithRetries(t.Context(), adapter, "media/request_asset/object.png") + if err != nil || string(payload) != "stored-payload" || calls.Load() != 2 { + t.Fatalf("payload=%q calls=%d err=%v", payload, calls.Load(), err) + } +} + +func TestObjectStorageAuthFailureSkipsSameChannelRetry(t *testing.T) { + var primaryCalls atomic.Int64 + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + primaryCalls.Add(1) + http.Error(w, "forbidden details", http.StatusForbidden) + })) + defer primary.Close() + secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + defer secondary.Close() + + first := testObjectStorageChannel("aliyun_oss", primary.URL, "oss-primary") + first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 3, "backoffSeconds": []any{0.001}} + second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary") + service := &Service{} + if _, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second}); err != nil { + t.Fatal(err) + } + if primaryCalls.Load() != 1 { + t.Fatalf("auth failure retried %d times", primaryCalls.Load()) + } +} + +func TestRequestLevelUploadFailureDoesNotSwitchChannels(t *testing.T) { + var primaryCalls atomic.Int64 + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + primaryCalls.Add(1) + http.Error(w, "unsupported media", http.StatusUnsupportedMediaType) + })) + defer primary.Close() + var secondaryCalls atomic.Int64 + secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + secondaryCalls.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer secondary.Close() + + first := store.FileStorageChannel{ + ChannelKey: "server-main-primary", Provider: "server_main_openapi", UploadURL: primary.URL, + APIKey: "test-key", RetryPolicy: map[string]any{"enabled": false}, + } + second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary") + _, err := (&Service{}).uploadFileWithFailover(t.Context(), FileUploadPayload{ + Bytes: []byte("payload"), ContentType: "application/x-unsupported", Scene: store.FileStorageSceneUpload, + }, []store.FileStorageChannel{first, second}) + if err == nil || clients.ErrorCode(err) != "upload_failed" { + t.Fatalf("unexpected request-level error: %v", err) + } + if primaryCalls.Load() != 1 || secondaryCalls.Load() != 0 { + t.Fatalf("request-level failure switched channels: primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load()) + } +} + +func TestAllObjectStorageChannelsFailWithStableError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "failure", http.StatusBadGateway) })) + defer server.Close() + channel := testObjectStorageChannel("s3", server.URL, "only") + channel.RetryPolicy = map[string]any{"enabled": false} + _, err := (&Service{}).uploadFileWithFailover(context.Background(), FileUploadPayload{Bytes: []byte("payload"), Scene: store.FileStorageSceneUpload}, []store.FileStorageChannel{channel}) + if clients.ErrorCode(err) != "storage_write_failed" || strings.Contains(err.Error(), "failure") { + t.Fatalf("unexpected public storage error: %v", err) + } +} + +func testObjectStorageChannel(provider string, endpoint string, channelKey string) store.FileStorageChannel { + return store.FileStorageChannel{ + ChannelKey: channelKey, + Name: channelKey, + Provider: provider, + AccessKeyID: "access-id", + AccessKeySecret: "access-secret", + Priority: 100, + Config: map[string]any{ + "endpoint": endpoint, + "bucket": "bucket", + "region": "cn-test-1", + "objectPrefix": "media", + "publicBaseUrl": "https://cdn.example.com", + "forcePathStyle": true, + }, + } +} + +func assertObjectStorageMetricEvent(t *testing.T, events []objectStorageMetricEvent, event string, provider string, bytes int64) { + t.Helper() + for _, item := range events { + if item.event == event && item.provider == provider && item.bytes == bytes { + return + } + } + t.Fatalf("missing metric event %s/%s/%d in %+v", event, provider, bytes, events) +} diff --git a/apps/api/internal/runner/request_assets.go b/apps/api/internal/runner/request_assets.go index 85670e8..b65fcfc 100644 --- a/apps/api/internal/runner/request_assets.go +++ b/apps/api/internal/runner/request_assets.go @@ -148,10 +148,14 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st } return base64.StdEncoding.EncodeToString(payload), nil } - if strings.TrimSpace(asset.URL) == "" { + assetURL, err := s.requestAssetAccessURL(ctx, asset) + if err != nil { + return nil, err + } + if strings.TrimSpace(assetURL) == "" { return nil, requestAssetExpiredError(asset) } - return asset.URL, nil + return assetURL, nil } func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value string, path []string, candidate store.RuntimeModelCandidate) (any, error) { @@ -224,10 +228,14 @@ func (s *Service) resolveRequestAsset(ctx context.Context, ref map[string]any) ( sha := stringFromAny(ref["sha256"]) contentType := stringFromAny(ref["contentType"]) asset := store.RequestAsset{ - SHA256: sha, - ContentType: contentType, - URL: stringFromAny(ref["url"]), - StorageProvider: stringFromAny(ref["storageProvider"]), + SHA256: sha, + ContentType: contentType, + URL: stringFromAny(ref["url"]), + StorageProvider: stringFromAny(ref["storageProvider"]), + StorageChannelID: stringFromAny(ref["storageChannelId"]), + StorageChannelKey: stringFromAny(ref["storageChannelKey"]), + ObjectKey: stringFromAny(ref["objectKey"]), + AccessScope: stringFromAny(ref["accessScope"]), } if size := floatFromAny(ref["size"]); size > 0 { asset.ByteSize = int64(size) @@ -254,6 +262,17 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request if requestAssetIsExpired(asset, time.Now()) { return nil, requestAssetExpiredError(asset) } + if strings.TrimSpace(asset.ObjectKey) != "" { + channel, err := s.fileStorageChannelForAsset(ctx, asset) + if err != nil { + return nil, err + } + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + return nil, err + } + return readObjectStorageWithRetries(ctx, adapter, asset.ObjectKey) + } if strings.TrimSpace(asset.LocalPath) != "" { payload, err := os.ReadFile(asset.LocalPath) if err != nil { @@ -290,6 +309,60 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request return nil, requestAssetExpiredError(asset) } +func readObjectStorageWithRetries(ctx context.Context, adapter *objectStorageAdapter, objectKey string) ([]byte, error) { + maxRetries, delays := uploadRetrySchedule(adapter.channel.RetryPolicy, adapter.channel.Provider) + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + payload, err := adapter.get(ctx, objectKey) + if err == nil { + return payload, nil + } + lastErr = err + if attempt >= maxRetries || !clients.IsRetryable(err) { + break + } + if err := sleepWithContext(ctx, retryDelayForAttempt(attempt, delays)); err != nil { + return nil, err + } + } + return nil, lastErr +} + +func (s *Service) requestAssetAccessURL(ctx context.Context, asset store.RequestAsset) (string, error) { + if strings.TrimSpace(asset.ObjectKey) == "" { + return asset.URL, nil + } + channel, err := s.fileStorageChannelForAsset(ctx, asset) + if err != nil { + return "", err + } + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + return "", err + } + if strings.EqualFold(strings.TrimSpace(asset.AccessScope), "public") { + if value := adapter.publicURL(asset.ObjectKey); value != "" { + return value, nil + } + } + return adapter.presignGet(asset.ObjectKey, objectStorageSignedURLTTL) +} + +func (s *Service) fileStorageChannelForAsset(ctx context.Context, asset store.RequestAsset) (store.FileStorageChannel, error) { + channelKey := strings.TrimSpace(asset.StorageChannelKey) + if channelKey == "environment-direct-oss" && s.directOSS != nil { + return s.directOSS.fileStorageChannel(), nil + } + if s.store == nil || channelKey == "" { + return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true} + } + channel, err := s.store.GetFileStorageChannelByKey(ctx, channelKey) + if err != nil { + return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true} + } + return channel, nil +} + func (s *Service) localPathFromRequestAssetURL(value string) string { raw := strings.TrimSpace(value) if raw == "" { diff --git a/apps/api/internal/runner/service.go b/apps/api/internal/runner/service.go index 5fcb50a..d27b324 100644 --- a/apps/api/internal/runner/service.go +++ b/apps/api/internal/runner/service.go @@ -230,6 +230,15 @@ func (s *Service) observeTaskEventSkip(reason string) { } } +func (s *Service) observeObjectStorage(event string, provider string, bytes int, duration time.Duration) { + observer, ok := s.billingMetrics.(interface { + ObserveObjectStorage(string, string, int64, time.Duration) + }) + if ok { + observer.ObserveObjectStorage(event, provider, int64(bytes), duration) + } +} + func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) { return s.execute(ctx, task, user, nil) } diff --git a/apps/api/internal/runner/task_history_workers.go b/apps/api/internal/runner/task_history_workers.go index 78f62dd..d49461d 100644 --- a/apps/api/internal/runner/task_history_workers.go +++ b/apps/api/internal/runner/task_history_workers.go @@ -10,6 +10,7 @@ import ( "net/http" "time" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" "github.com/google/uuid" ) @@ -74,13 +75,19 @@ func (s *Service) processTaskCallbackBatch(ctx context.Context, client *http.Cli } func deliverTaskCallback(ctx context.Context, client *http.Client, item store.TaskCallbackDelivery, bearerToken string) (int, error) { - body, err := json.Marshal(map[string]any{ + payload := map[string]any{ "taskId": item.TaskID, "seq": item.Seq, "eventType": item.EventType, "status": item.TaskStatus, "createdAt": item.CreatedAt.UTC().Format(time.RFC3339Nano), - }) + } + if item.TaskStatus == "failed" || item.TaskStatus == "cancelled" { + standard := publicerror.WithIDs(publicerror.FromFields(item.TaskErrorCode, item.TaskErrorMessage, 0, false), item.TaskRequestID, item.TaskID) + publicerror.Observe(standard) + payload["error"] = standard + } + body, err := json.Marshal(payload) if err != nil { return 0, err } diff --git a/apps/api/internal/runner/task_history_workers_test.go b/apps/api/internal/runner/task_history_workers_test.go index 795a645..1ce2cb7 100644 --- a/apps/api/internal/runner/task_history_workers_test.go +++ b/apps/api/internal/runner/task_history_workers_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -116,3 +117,33 @@ func TestDeliverTaskCallbackSendsMinimalAuthenticatedBody(t *testing.T) { t.Fatalf("callback body=%#v", received) } } + +func TestDeliverFailedTaskCallbackUsesStandardErrorWithoutSocketDetails(t *testing.T) { + var received map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if err := json.NewDecoder(request.Body).Decode(&received); err != nil { + t.Error(err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer" + status, err := deliverTaskCallback(t.Context(), server.Client(), store.TaskCallbackDelivery{ + TaskID: "failed-task", + Seq: 4, + CallbackURL: server.URL, + EventType: "task.failed", + TaskStatus: "failed", + TaskErrorCode: "response_read_error", + TaskErrorMessage: raw, + CreatedAt: time.Now(), + }, "") + if err != nil || status != http.StatusNoContent { + t.Fatalf("status=%d err=%v", status, err) + } + errorValue, _ := received["error"].(map[string]any) + if errorValue["code"] != "upstream_connection_interrupted" || strings.Contains(stringFromAny(errorValue["message"]), "10.42.0.72") { + t.Fatalf("unsafe callback error: %#v", errorValue) + } +} diff --git a/apps/api/internal/runner/upload.go b/apps/api/internal/runner/upload.go index 7c505a3..fdfed64 100644 --- a/apps/api/internal/runner/upload.go +++ b/apps/api/internal/runner/upload.go @@ -17,25 +17,18 @@ import ( "net/http" "net/textproto" "net/url" - "os" - "path/filepath" + "sort" "strings" "sync" "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" - "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) const defaultServerMainOpenAPIUploadURL = "http://127.0.0.1:3001/v1/files/upload" const maxGeneratedAssetFetchBytes = 256 << 20 -const ( - localStaticGeneratedPathPrefix = "/static/generated/" - localStaticUploadedPathPrefix = "/static/uploaded/" -) - type FileUploadPayload struct { ContentType string FileName string @@ -136,14 +129,6 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task return nil, &clients.ClientError{Code: "acceptance_run_inactive", Message: err.Error(), Retryable: false} } } - if policy.LocalizeInlineMedia { - next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result) - if err != nil { - return nil, err - } - redactGeneratedResultRawData(next) - return next, nil - } if len(data) == 0 && !rawNeedsUpload { return s.finalizeGeneratedAssets(ctx, taskID, taskKind, result, policy, nil, false, 0) } @@ -179,11 +164,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task var channels []store.FileStorageChannel channelsLoaded := false if needsUpload || rawNeedsUpload { - if s.directOSS == nil { - channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult) - if err != nil { - return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true} - } + channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult) + if err != nil { + return nil, &clients.ClientError{Code: "storage_write_failed", Message: "object storage configuration is unavailable", Retryable: true} } channelsLoaded = true } @@ -323,12 +306,12 @@ func (s *Service) finalizeGeneratedAssets( if !TaskResultHasInlineBinary(next) { return next, nil } - persistent, _, err := s.materializeLocalBinaryResult(ctx, taskID, next) - if err != nil { - return nil, err + return nil, &clients.ClientError{ + Code: "storage_write_failed", + Message: "generated binary result could not be written to object storage", + StatusCode: http.StatusServiceUnavailable, + Retryable: true, } - redactGeneratedResultRawData(persistent) - return persistent, nil } func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool { @@ -355,6 +338,9 @@ func generatedRawValueHasInlineMedia(value any, key string, siblings map[string] func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) { switch typed := value.(type) { case map[string]any: + if payload, contentType, ok := localBufferObjectBytes(typed); ok { + return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, contentType, firstNonEmptyString(key, "buffer"), siblings, channels, index) + } next := make(map[string]any, len(typed)) changed := false for childKey, childValue := range typed { @@ -372,6 +358,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin } return value, false, nil case []any: + if localBinaryKey(key) { + if payload, ok := bytesFromNumberArray(typed); ok { + return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index) + } + } next := make([]any, len(typed)) changed := false for itemIndex, item := range typed { @@ -388,6 +379,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin return next, true, nil } return value, false, nil + case []byte: + if len(typed) == 0 { + return value, false, nil + } + return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index) case string: asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind) if !ok { @@ -404,19 +400,36 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin } } +func (s *Service) uploadGeneratedBinaryValue(ctx context.Context, taskID string, taskKind string, payload []byte, contentType string, sourceKey string, siblings map[string]any, channels []store.FileStorageChannel, index *int) (any, bool, error) { + contentType = firstNonEmptyString(contentType, defaultContentTypeForRawMediaKey(sourceKey)) + asset := &generatedInlineAsset{ + Bytes: payload, + ContentType: contentType, + Kind: mediaKindForAsset(taskKind, siblings, sourceKey, contentType), + SourceKey: sourceKey, + } + upload, resolvedContentType, kind, strategy, err := s.uploadGeneratedAsset(ctx, taskID, asset, *index, channels) + if err != nil { + return nil, false, err + } + *index = *index + 1 + return generatedRawMediaReference(asset, upload, resolvedContentType, kind, strategy), true, nil +} + func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) { raw := strings.TrimSpace(value) if raw == "" { return nil, false } + keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key) contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key)) - if !generatedRawDataMediaPayloadKey(key) && !generatedContentTypeIsMedia(contentType) { + if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) { return nil, false } - if !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 { + if !keyLooksLikeMediaPayload && !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 { return nil, false } - payload, payloadContentType, ok, err := inlineMediaPayload(raw, generatedRawDataMediaPayloadKey(key)) + payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload) if err != nil || !ok || len(payload) == 0 { return nil, false } @@ -454,6 +467,18 @@ func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]a if provider := stringFromAny(channel["provider"]); provider != "" { ref["storageProvider"] = provider } + if id := stringFromAny(channel["id"]); id != "" { + ref["storageChannelId"] = id + } + if key := stringFromAny(channel["channelKey"]); key != "" { + ref["storageChannelKey"] = key + } + } + if objectKey := stringFromAny(upload["objectKey"]); objectKey != "" { + ref["objectKey"] = objectKey + } + if accessScope := stringFromAny(upload["accessScope"]); accessScope != "" { + ref["accessScope"] = accessScope } out := map[string]any{ "assetRef": ref, @@ -650,6 +675,9 @@ func removeASCIIWhitespace(value string) string { } func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) { + if s.store == nil { + return defaultGeneratedAssetUploadPolicy(), nil + } settings, err := s.store.GetFileStorageSettings(ctx) if err != nil { if store.IsUndefinedDatabaseObject(err) { @@ -665,8 +693,6 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP switch policyName { case store.FileStorageResultUploadPolicyUploadAll: return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true} - case store.FileStorageResultUploadPolicyUploadNone: - return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true} default: return defaultGeneratedAssetUploadPolicy() } @@ -682,13 +708,8 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset Scene: store.FileStorageSceneImageResult, Source: "ai-gateway", } - if s.directOSS != nil { - upload, err := s.directOSS.upload(ctx, payload) - return upload, contentType, kind, "direct_aliyun_oss", err - } if len(channels) == 0 { - upload, err := s.storeFileLocally(payload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix) - return upload, contentType, kind, "local_static_inline_media", err + return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true} } upload, err := s.uploadFileWithFailover(ctx, payload, channels) return upload, contentType, kind, "upload_inline_media", err @@ -708,95 +729,13 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as Scene: store.FileStorageSceneImageResult, Source: "ai-gateway", } - if s.directOSS != nil { - upload, err := s.directOSS.upload(ctx, uploadPayload) - return upload, contentType, kind, "direct_aliyun_oss", err - } if len(channels) == 0 { - upload, err := s.storeFileLocally(uploadPayload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix) - return upload, contentType, kind, "local_static_url_media", err + return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true} } upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels) return upload, contentType, kind, "upload_url_media", err } -func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string, fallbackStorageDir string, pathPrefix string) (map[string]any, error) { - storageDir = strings.TrimSpace(storageDir) - if storageDir == "" { - storageDir = fallbackStorageDir - } - if err := os.MkdirAll(storageDir, 0o755); err != nil { - return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true} - } - fileName := filepath.Base(strings.TrimSpace(payload.FileName)) - if fileName == "" || fileName == "." || fileName == ".." || fileName == string(filepath.Separator) { - kind := generatedAssetKindFromContentType("", payload.ContentType) - fileName = generatedAssetFileName("generated", 0, payload.ContentType, kind) - } - targetPath := filepath.Join(storageDir, fileName) - file, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if err != nil { - return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true} - } - _, writeErr := file.Write(payload.Bytes) - closeErr := file.Close() - if writeErr != nil { - _ = os.Remove(targetPath) - return nil, &clients.ClientError{Code: "local_static_store_failed", Message: writeErr.Error(), Retryable: true} - } - if closeErr != nil { - _ = os.Remove(targetPath) - return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true} - } - expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339) - return map[string]any{ - "url": s.localStaticFileURL(fileName, pathPrefix), - "fileName": fileName, - "contentType": payload.ContentType, - "size": len(payload.Bytes), - "expiresAt": expiresAt, - "storageChannel": map[string]any{ - "id": "local-static", - "channelKey": "local-static", - "name": "AI Gateway local static storage", - "provider": "local_static", - }, - }, nil -} - -func (s *Service) localStaticAssetTTLHours() int { - if s.cfg.LocalTempAssetTTLHours <= 0 { - return 24 - } - return s.cfg.LocalTempAssetTTLHours -} - -func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string { - if strings.TrimSpace(pathPrefix) == "" { - pathPrefix = localStaticUploadedPathPrefix - } - path := pathPrefix + url.PathEscape(filepath.Base(fileName)) - baseURL := strings.TrimRight(strings.TrimSpace(s.cfg.PublicBaseURL), "/") - if baseURL == "" { - return path - } - return baseURL + path -} - -func localStaticUploadFileName(originalName string, contentType string) string { - baseName := filepath.Base(strings.TrimSpace(originalName)) - originalExt := strings.ToLower(filepath.Ext(baseName)) - namePart := strings.TrimSuffix(baseName, originalExt) - namePart = sanitizeGeneratedAssetNamePart(namePart) - if namePart == "" { - namePart = "gateway-upload" - } - if len(namePart) > 48 { - namePart = namePart[:48] - } - return fmt.Sprintf("%s-%s%s", namePart, randomHexSuffix(6), uploadFileExtension(contentType, originalExt)) -} - func uploadFileExtension(contentType string, fallbackExt string) string { normalized := normalizeGeneratedContentType(contentType) if generatedContentTypeIsMedia(normalized) { @@ -981,46 +920,41 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma if strings.TrimSpace(payload.Scene) == "" { payload.Scene = store.FileStorageSceneUpload } - if s.directOSS != nil && directOSSScene(payload.Scene) { - return s.directOSS.upload(ctx, payload) - } channels, err := s.activeFileStorageChannels(ctx, payload.Scene) if err != nil { return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true} } if len(channels) == 0 { - payload.FileName = localStaticUploadFileName(payload.FileName, payload.ContentType) - upload, err := s.storeFileLocally(payload, s.cfg.LocalUploadedStorageDir, config.DefaultLocalUploadedStorageDir, localStaticUploadedPathPrefix) - if err != nil { - return nil, err - } - upload["assetStorage"] = map[string]any{ - "scene": payload.Scene, - "source": firstNonEmptyString(payload.Source, "ai-gateway-openapi"), - "strategy": "local_static_upload", - } - return upload, nil + return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true} } return s.uploadFileWithFailover(ctx, payload, channels) } func (s *Service) activeFileStorageChannels(ctx context.Context, scene string) ([]store.FileStorageChannel, error) { - if s.store == nil { - return nil, nil + channels := make([]store.FileStorageChannel, 0) + if s.directOSS != nil && directOSSScene(scene) { + channels = append(channels, s.directOSS.fileStorageChannel()) } - channels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene) + if s.store == nil { + return channels, nil + } + storedChannels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene) if err != nil && !store.IsUndefinedDatabaseObject(err) { return nil, err } - if len(channels) > 0 { - return channels, nil - } - return nil, nil + channels = append(channels, storedChannels...) + sort.SliceStable(channels, func(i, j int) bool { + if channels[i].Priority != channels[j].Priority { + return channels[i].Priority < channels[j].Priority + } + return channels[i].ChannelKey < channels[j].ChannelKey + }) + return channels, nil } func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUploadPayload, channels []store.FileStorageChannel) (map[string]any, error) { var lastErr error - for _, channel := range channels { + for index, channel := range channels { upload, err := s.uploadWithChannelRetries(ctx, payload, channel) if err == nil { if s.store != nil { @@ -1032,25 +966,59 @@ func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUpload if s.store != nil { _ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(ctx), channel.ID, err.Error()) } + if !storageFailureAllowsFailover(channel, err) { + return nil, err + } + if index+1 < len(channels) { + s.observeObjectStorage("failover", channel.Provider, 0, 0) + } } if lastErr != nil { - return nil, lastErr + s.observeObjectStorage("all_failed", "", 0, 0) + return nil, &clients.ClientError{Code: "storage_write_failed", Message: "all configured object storage channels failed", StatusCode: http.StatusServiceUnavailable, Retryable: true} + } + return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true} +} + +func storageFailureAllowsFailover(channel store.FileStorageChannel, err error) bool { + code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err))) + switch code { + case "upload_source_too_large", "upload_decode_failed", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio": + return false + } + var clientErr *clients.ClientError + if !errors.As(err, &clientErr) { + return true + } + switch clientErr.StatusCode { + case http.StatusRequestEntityTooLarge, http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity: + return false + case http.StatusBadRequest: + // Object-storage 400 responses commonly indicate endpoint/signature + // configuration and should move to the next channel. server-main 400 is + // the existing upload API's request validation result. + return !strings.EqualFold(strings.TrimSpace(channel.Provider), "server_main_openapi") + default: + return true } - return nil, &clients.ClientError{Code: "upload_no_channel", Message: "no enabled file storage channel", Retryable: false} } func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) { - maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy) + maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy, channel.Provider) var lastErr error for attempt := 0; attempt <= maxRetries; attempt++ { + startedAt := time.Now() upload, err := s.uploadOnce(ctx, payload, channel) if err == nil { + s.observeObjectStorage("write_success", channel.Provider, len(payload.Bytes), time.Since(startedAt)) return upload, nil } + s.observeObjectStorage("write_failure", channel.Provider, 0, time.Since(startedAt)) lastErr = err if attempt >= maxRetries || !clients.IsRetryable(err) { break } + s.observeObjectStorage("retry", channel.Provider, 0, 0) delay := retryDelayForAttempt(attempt, delays) if err := sleepWithContext(ctx, delay); err != nil { return nil, err @@ -1060,9 +1028,21 @@ func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUplo } func (s *Service) uploadOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) { - if strings.ToLower(strings.TrimSpace(channel.Provider)) != "server_main_openapi" { - return nil, &clients.ClientError{Code: "upload_unsupported_channel", Message: "unsupported file storage channel: " + channel.Provider, Retryable: false} + switch strings.ToLower(strings.TrimSpace(channel.Provider)) { + case "aliyun_oss", "s3": + adapter, err := newObjectStorageAdapter(channel) + if err != nil { + return nil, err + } + return adapter.put(ctx, payload) + case "server_main_openapi": + return s.uploadServerMainOnce(ctx, payload, channel) + default: + return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported file storage channel", Retryable: false} } +} + +func (s *Service) uploadServerMainOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) { uploadURL := strings.TrimSpace(channel.UploadURL) if uploadURL == "" { uploadURL = defaultServerMainOpenAPIUploadURL @@ -1700,9 +1680,17 @@ func uniqueStringList(values []string) []string { return out } -func uploadRetrySchedule(policy map[string]any) (int, []time.Duration) { +func uploadRetrySchedule(policy map[string]any, providers ...string) (int, []time.Duration) { + provider := "" + if len(providers) > 0 { + provider = providers[0] + } if policy == nil { - policy = defaultUploadRetryPolicy() + if strings.EqualFold(provider, "aliyun_oss") || strings.EqualFold(provider, "s3") { + policy = map[string]any{"enabled": true, "maxRetries": 2, "backoffSeconds": []any{0.25, 1.0}} + } else { + policy = defaultUploadRetryPolicy() + } } if enabled, ok := policy["enabled"].(bool); ok && !enabled { return 0, nil @@ -1725,9 +1713,9 @@ func uploadRetryDelays(value any) []time.Duration { } delays := make([]time.Duration, 0, len(items)) for _, item := range items { - seconds := int(floatFromAny(item)) + seconds := floatFromAny(item) if seconds > 0 { - delays = append(delays, time.Duration(seconds)*time.Second) + delays = append(delays, time.Duration(seconds*float64(time.Second))) } } return delays diff --git a/apps/api/internal/runner/upload_test.go b/apps/api/internal/runner/upload_test.go index 4abc756..342b65e 100644 --- a/apps/api/internal/runner/upload_test.go +++ b/apps/api/internal/runner/upload_test.go @@ -7,11 +7,11 @@ import ( "net/http" "net/http/httptest" "os" - "path/filepath" "strings" "testing" "time" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -227,38 +227,6 @@ func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) { } } -func TestGeneratedAssetDecisionPreservesInlineWhenPolicyUploadNone(t *testing.T) { - item := map[string]any{ - "b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")), - } - - decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if decision.Inline != nil || decision.URL != nil { - t.Fatalf("upload_none should not transfer inline payloads: %+v", decision) - } - if len(decision.StripKeys) != 0 { - t.Fatalf("upload_none should preserve b64_json for the caller: %+v", decision.StripKeys) - } -} - -func TestGeneratedAssetDecisionPreservesInlineAlongsideURLWhenPolicyUploadNone(t *testing.T) { - item := map[string]any{ - "url": "https://cdn.example.com/generated.png", - "b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")), - } - - decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if decision.Inline != nil || decision.URL != nil || len(decision.StripKeys) != 0 { - t.Fatalf("upload_none should preserve the upstream URL and base64 fields unchanged: %+v", decision) - } -} - func TestGeneratedAssetUploadPolicyFromName(t *testing.T) { tests := []struct { name string @@ -276,9 +244,9 @@ func TestGeneratedAssetUploadPolicyFromName(t *testing.T) { want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true, PreserveInlineMedia: false}, }, { - name: "upload none", + name: "legacy upload none becomes default", policyName: store.FileStorageResultUploadPolicyUploadNone, - want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}, + want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: false, PreserveInlineMedia: false}, }, } @@ -330,8 +298,10 @@ func TestAcceptanceGeneratedMediaAllowsOnlyExactEmulatorOrigin(t *testing.T) { } func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *testing.T) { - storageDir := t.TempDir() - service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + defer server.Close() + service := &Service{} + channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")} payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...) encoded := base64.StdEncoding.EncodeToString(payload) result := map[string]any{ @@ -348,7 +318,7 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t "images.edits", result, defaultGeneratedAssetUploadPolicy(), - nil, + channels, true, 0, ) @@ -363,15 +333,8 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t if !ok { t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested) } - if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-nested-binary-01-") { - t.Fatalf("unexpected local static URL: %s", urlValue) - } - entries, err := os.ReadDir(storageDir) - if err != nil { - t.Fatalf("read generated storage: %v", err) - } - if len(entries) != 1 { - t.Fatalf("expected one localized result file, got %d", len(entries)) + if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") { + t.Fatalf("unexpected object storage URL: %s", urlValue) } } @@ -405,9 +368,8 @@ func TestGeneratedAssetFileNameIsUniqueAndTyped(t *testing.T) { } } -func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) { - storageDir := t.TempDir() - service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}} +func TestUploadGeneratedAssetFailsWithoutObjectStorage(t *testing.T) { + service := &Service{} payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0} asset := &generatedInlineAsset{ Bytes: payload, @@ -416,44 +378,14 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) { SourceKey: "b64_json", } - upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil) - if err != nil { + _, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil) + if clients.ErrorCode(err) != "storage_write_failed" { t.Fatalf("unexpected error: %v", err) } - if contentType != "image/png" || kind != "image" || strategy != "local_static_inline_media" { - t.Fatalf("unexpected local upload metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy) - } - urlValue := stringFromAny(upload["url"]) - if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-123-01-") || !strings.HasSuffix(urlValue, ".png") { - t.Fatalf("unexpected local static URL: %s", urlValue) - } - expiresAt, err := time.Parse(time.RFC3339, stringFromAny(upload["expiresAt"])) - if err != nil { - t.Fatalf("local static upload should expose expiresAt: %+v", upload) - } - remaining := time.Until(expiresAt) - if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour+time.Minute { - t.Fatalf("local static upload should expire after one day, remaining=%s", remaining) - } - entries, err := os.ReadDir(storageDir) - if err != nil { - t.Fatalf("failed to read local static dir: %v", err) - } - if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") { - t.Fatalf("expected one PNG file in local static dir, got %+v", entries) - } - stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name())) - if err != nil { - t.Fatalf("failed to read local static file: %v", err) - } - if !bytes.Equal(stored, payload) { - t.Fatalf("stored payload does not match source payload") - } } -func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) { - storageDir := t.TempDir() - service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}} +func TestUploadGeneratedAudioFailsWithoutObjectStorage(t *testing.T) { + service := &Service{} asset := &generatedInlineAsset{ Bytes: []byte("inline audio payload"), ContentType: "audio/mpeg", @@ -461,32 +393,17 @@ func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) { SourceKey: "content", } - upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil) - if err != nil { + _, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil) + if clients.ErrorCode(err) != "storage_write_failed" { t.Fatalf("unexpected error: %v", err) } - if contentType != "audio/mpeg" || kind != "audio" || strategy != "local_static_inline_media" { - t.Fatalf("unexpected local audio metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy) - } - urlValue := stringFromAny(upload["url"]) - if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-tts-01-") || !strings.HasSuffix(urlValue, ".mp3") { - t.Fatalf("unexpected local audio URL: %s", urlValue) - } - if stringFromAny(upload["expiresAt"]) == "" { - t.Fatalf("local audio static upload should expose expiresAt: %+v", upload) - } - entries, err := os.ReadDir(storageDir) - if err != nil { - t.Fatalf("failed to read local static dir: %v", err) - } - if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".mp3") { - t.Fatalf("expected one MP3 file in local static dir, got %+v", entries) - } } func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) { - storageDir := t.TempDir() - service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + defer server.Close() + service := &Service{} + channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")} payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...) raw := map[string]any{ "candidates": []any{ @@ -506,7 +423,7 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes } index := 0 - uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), nil, &index) + uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index) if err != nil { t.Fatalf("upload raw media: %v", err) } @@ -528,22 +445,47 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) { t.Fatalf("unexpected asset ref: %+v", ref) } - if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-raw-01-") || !strings.HasSuffix(urlValue, ".png") { + if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") || !strings.HasSuffix(urlValue, ".png") { t.Fatalf("unexpected raw media URL: %s", urlValue) } if inlineData["data"] == base64.StdEncoding.EncodeToString(payload) { t.Fatal("raw inlineData still contains base64 payload") } - entries, err := os.ReadDir(storageDir) - if err != nil { - t.Fatalf("read generated storage: %v", err) +} + +func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + defer server.Close() + service := &Service{} + channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-buffer-result")} + raw := map[string]any{ + "buffer": map[string]any{ + "type": "Buffer", "mimeType": "image/png", "data": []any{float64(0x89), float64('P'), float64('N'), float64('G')}, + }, + "audio_bytes": []any{float64('I'), float64('D'), float64('3')}, + "direct": []byte("direct bytes"), } - if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") { - t.Fatalf("expected one generated PNG, got %+v", entries) + index := 0 + uploaded, changed, err := service.uploadGeneratedRawMediaValue(t.Context(), "task-buffer", "images.generations", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index) + if err != nil { + t.Fatal(err) + } + if !changed || index != 3 { + t.Fatalf("changed=%v uploads=%d", changed, index) + } + next := uploaded.(map[string]any) + for _, key := range []string{"buffer", "audio_bytes", "direct"} { + item, ok := next[key].(map[string]any) + if !ok || item["assetRef"] == nil || item["upload"] == nil { + t.Fatalf("%s was not objectified: %#v", key, next[key]) + } + } + if TaskResultHasInlineBinary(next) { + t.Fatalf("objectified result still contains inline binary: %#v", next) } } -func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) { +func TestUploadFileFailsWithoutObjectStorageAndCreatesNoLocalFile(t *testing.T) { storageDir := t.TempDir() service := &Service{cfg: config.Config{ LocalUploadedStorageDir: storageDir, @@ -552,43 +494,21 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) { }} payload := []byte("%PDF-1.4") - upload, err := service.UploadFile(context.Background(), FileUploadPayload{ + _, err := service.UploadFile(context.Background(), FileUploadPayload{ Bytes: payload, ContentType: "application/pdf", FileName: "用户文件.png", Source: "playground", }) - if err != nil { + if clients.ErrorCode(err) != "storage_write_failed" { t.Fatalf("unexpected error: %v", err) } - urlValue := stringFromAny(upload["url"]) - if !strings.HasPrefix(urlValue, "/static/uploaded/") || !strings.HasSuffix(urlValue, ".pdf") { - t.Fatalf("unexpected uploaded local static URL: %s", urlValue) - } - if stringFromAny(upload["expiresAt"]) == "" { - t.Fatalf("local uploaded static file should expose expiresAt: %+v", upload) - } - storageChannel, _ := upload["storageChannel"].(map[string]any) - if stringFromAny(storageChannel["provider"]) != "local_static" { - t.Fatalf("expected local static provider metadata, got %+v", upload["storageChannel"]) - } - assetStorage, _ := upload["assetStorage"].(map[string]any) - if stringFromAny(assetStorage["strategy"]) != "local_static_upload" || stringFromAny(assetStorage["scene"]) != store.FileStorageSceneUpload { - t.Fatalf("unexpected upload asset storage metadata: %+v", assetStorage) - } entries, err := os.ReadDir(storageDir) if err != nil { t.Fatalf("failed to read uploaded static dir: %v", err) } - if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".pdf") { - t.Fatalf("expected one PDF file in uploaded static dir, got %+v", entries) - } - stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name())) - if err != nil { - t.Fatalf("failed to read uploaded static file: %v", err) - } - if !bytes.Equal(stored, payload) { - t.Fatalf("stored uploaded payload does not match source payload") + if len(entries) != 0 { + t.Fatalf("unexpected local files: %+v", entries) } } diff --git a/apps/api/internal/securityevents/metrics.go b/apps/api/internal/securityevents/metrics.go index 8fb7e68..c2cd446 100644 --- a/apps/api/internal/securityevents/metrics.go +++ b/apps/api/internal/securityevents/metrics.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -86,6 +87,19 @@ type Metrics struct { candidateDisabledSkipped atomic.Uint64 candidateAllFullQueued atomic.Uint64 candidateRoutingOther atomic.Uint64 + storageAliyunWrites atomic.Uint64 + storageAliyunFailures atomic.Uint64 + storageAliyunWriteNanos atomic.Uint64 + storageS3Writes atomic.Uint64 + storageS3Failures atomic.Uint64 + storageS3WriteNanos atomic.Uint64 + storageServerMainWrites atomic.Uint64 + storageServerMainFailures atomic.Uint64 + storageServerMainWriteNanos atomic.Uint64 + storageChannelRetries atomic.Uint64 + storageChannelFailovers atomic.Uint64 + storageAllChannelsFailed atomic.Uint64 + storageObjectifiedBytes atomic.Uint64 taskAdmissionWaitBuckets [11]atomic.Uint64 taskAdmissionWaitMicros atomic.Uint64 } @@ -265,6 +279,40 @@ func (m *Metrics) ObserveCandidateRouting(reason string) { } } +func (m *Metrics) ObserveObjectStorage(event string, provider string, bytes int64, duration time.Duration) { + provider = strings.ToLower(strings.TrimSpace(provider)) + if duration < 0 { + duration = 0 + } + switch event { + case "write_success", "write_failure": + var writes, failures, nanos *atomic.Uint64 + switch provider { + case "aliyun_oss": + writes, failures, nanos = &m.storageAliyunWrites, &m.storageAliyunFailures, &m.storageAliyunWriteNanos + case "s3": + writes, failures, nanos = &m.storageS3Writes, &m.storageS3Failures, &m.storageS3WriteNanos + case "server_main_openapi": + writes, failures, nanos = &m.storageServerMainWrites, &m.storageServerMainFailures, &m.storageServerMainWriteNanos + default: + return + } + writes.Add(1) + nanos.Add(uint64(duration)) + if event == "write_failure" { + failures.Add(1) + } else if bytes > 0 { + m.storageObjectifiedBytes.Add(uint64(bytes)) + } + case "retry": + m.storageChannelRetries.Add(1) + case "failover": + m.storageChannelFailovers.Add(1) + case "all_failed": + m.storageAllChannelsFailed.Add(1) + } +} + func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) { if wait < 0 { wait = 0 @@ -471,6 +519,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str {"all_full_queued", m.candidateAllFullQueued.Load()}, {"other", m.candidateRoutingOther.Load()}, }) + storageProtocolCounters(w, "easyai_gateway_storage_write_attempts_total", "Object storage write attempts by bounded protocol.", + m.storageAliyunWrites.Load(), m.storageS3Writes.Load(), m.storageServerMainWrites.Load()) + storageProtocolCounters(w, "easyai_gateway_storage_write_failures_total", "Object storage write failures by bounded protocol.", + m.storageAliyunFailures.Load(), m.storageS3Failures.Load(), m.storageServerMainFailures.Load()) + storageProtocolDurationCounters(w, + m.storageAliyunWriteNanos.Load(), m.storageS3WriteNanos.Load(), m.storageServerMainWriteNanos.Load()) + plainCounter(w, "easyai_gateway_storage_channel_retries_total", "Retries performed within the current storage channel.", m.storageChannelRetries.Load()) + plainCounter(w, "easyai_gateway_storage_channel_failovers_total", "Switches to the next configured storage channel.", m.storageChannelFailovers.Load()) + plainCounter(w, "easyai_gateway_storage_all_channels_failed_total", "Storage writes where every eligible channel failed.", m.storageAllChannelsFailed.Load()) + plainCounter(w, "easyai_gateway_storage_objectified_bytes_total", "Binary bytes successfully written through storage channels.", m.storageObjectifiedBytes.Load()) + publicErrorCounters(w, publicerror.MetricSnapshot()) platformModelRateLimitUtilizationGauges(w, modelRateLimits) plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections)) plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections)) @@ -567,6 +626,29 @@ func plainFloatGauge(w http.ResponseWriter, name, help string, value float64) { fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value) } +func storageProtocolCounters(w http.ResponseWriter, name, help string, aliyunOSS, s3, serverMain uint64) { + fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n", name, help, name) + fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %d\n", name, aliyunOSS) + fmt.Fprintf(w, "%s{protocol=\"s3\"} %d\n", name, s3) + fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %d\n", name, serverMain) +} + +func storageProtocolDurationCounters(w http.ResponseWriter, aliyunOSS, s3, serverMain uint64) { + const name = "easyai_gateway_storage_write_duration_seconds_total" + fmt.Fprintf(w, "# HELP %s Total time spent in object storage write attempts by bounded protocol.\n# TYPE %s counter\n", name, name) + fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %.6f\n", name, float64(aliyunOSS)/float64(time.Second)) + fmt.Fprintf(w, "%s{protocol=\"s3\"} %.6f\n", name, float64(s3)/float64(time.Second)) + fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %.6f\n", name, float64(serverMain)/float64(time.Second)) +} + +func publicErrorCounters(w http.ResponseWriter, values []publicerror.MetricCount) { + const name = "easyai_gateway_public_errors_total" + fmt.Fprintf(w, "# HELP %s Public error responses by bounded stable code.\n# TYPE %s counter\n", name, name) + for _, value := range values { + fmt.Fprintf(w, "%s{code=\"%s\"} %d\n", name, value.Code, value.Count) + } +} + func platformModelRateLimitUtilizationGauges(w http.ResponseWriter, statuses []store.ModelRateLimitStatus) { const name = "easyai_gateway_platform_model_rate_limit_utilization" fmt.Fprintf(w, "# HELP %s Cluster-wide platform model quota utilization.\n# TYPE %s gauge\n", name, name) diff --git a/apps/api/internal/securityevents/metrics_test.go b/apps/api/internal/securityevents/metrics_test.go index 8fc0221..cfbc6c4 100644 --- a/apps/api/internal/securityevents/metrics_test.go +++ b/apps/api/internal/securityevents/metrics_test.go @@ -52,6 +52,11 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) { metrics.ObserveCandidateRouting("cooldown_skipped") metrics.ObserveCandidateRouting("disabled_skipped") metrics.ObserveCandidateRouting("all_full_queued") + metrics.ObserveObjectStorage("write_success", "aliyun_oss", 128, 25*time.Millisecond) + metrics.ObserveObjectStorage("write_failure", "s3", 0, 10*time.Millisecond) + metrics.ObserveObjectStorage("retry", "s3", 0, 0) + metrics.ObserveObjectStorage("failover", "s3", 0, 0) + metrics.ObserveObjectStorage("all_failed", "", 0, 0) metrics.ObserveAsyncWorkerResize("success") metrics.ObserveConcurrencyLeaseRenewal("success") metrics.ObserveConcurrencyLeaseRenewal("lost") @@ -98,6 +103,13 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) { `easyai_gateway_candidate_routing_total{outcome="cooldown_skipped"} 1`, `easyai_gateway_candidate_routing_total{outcome="disabled_skipped"} 1`, `easyai_gateway_candidate_routing_total{outcome="all_full_queued"} 1`, + `easyai_gateway_storage_write_attempts_total{protocol="aliyun_oss"} 1`, + `easyai_gateway_storage_write_failures_total{protocol="s3"} 1`, + `easyai_gateway_storage_write_duration_seconds_total{protocol="aliyun_oss"} 0.025000`, + `easyai_gateway_storage_channel_retries_total 1`, + `easyai_gateway_storage_channel_failovers_total 1`, + `easyai_gateway_storage_all_channels_failed_total 1`, + `easyai_gateway_storage_objectified_bytes_total 128`, `easyai_gateway_platform_model_rate_limit_utilization{platform_model_id="platform-model-1",metric="concurrent"} 0.750000`, `easyai_gateway_async_worker_resizes_total{outcome="success"} 1`, `easyai_gateway_concurrency_lease_renewals_total{outcome="success"} 1`, diff --git a/apps/api/internal/store/file_storage_channels.go b/apps/api/internal/store/file_storage_channels.go index 041bac7..993a0af 100644 --- a/apps/api/internal/store/file_storage_channels.go +++ b/apps/api/internal/store/file_storage_channels.go @@ -39,6 +39,9 @@ type FileStorageChannel struct { Provider string `json:"provider"` UploadURL string `json:"uploadUrl,omitempty"` APIKey string `json:"-"` + AccessKeyID string `json:"-"` + AccessKeySecret string `json:"-"` + SessionToken string `json:"-"` CredentialsPreview map[string]any `json:"credentialsPreview,omitempty"` Scenes []string `json:"scenes,omitempty"` Config map[string]any `json:"config,omitempty"` @@ -53,16 +56,21 @@ type FileStorageChannel struct { } type FileStorageChannelInput struct { - ChannelKey string `json:"channelKey"` - Name string `json:"name"` - Provider string `json:"provider"` - UploadURL string `json:"uploadUrl"` - APIKey *string `json:"apiKey"` - Scenes []string `json:"scenes"` - Config map[string]any `json:"config"` - RetryPolicy map[string]any `json:"retryPolicy"` - Priority int `json:"priority"` - Status string `json:"status"` + ChannelKey string `json:"channelKey"` + Name string `json:"name"` + Provider string `json:"provider"` + UploadURL string `json:"uploadUrl"` + APIKey *string `json:"apiKey"` + AccessKey *string `json:"accessKey"` + AccessKeyID *string `json:"accessKeyId"` + AccessKeySecret *string `json:"accessKeySecret"` + SecretKey *string `json:"secretKey"` + SessionToken *string `json:"sessionToken"` + Scenes []string `json:"scenes"` + Config map[string]any `json:"config"` + RetryPolicy map[string]any `json:"retryPolicy"` + Priority int `json:"priority"` + Status string `json:"status"` } type FileStorageSettings struct { @@ -156,11 +164,18 @@ WHERE id = $1::uuid AND deleted_at IS NULL`, id)) } +func (s *Store) GetFileStorageChannelByKey(ctx context.Context, channelKey string) (FileStorageChannel, error) { + return scanFileStorageChannel(s.pool.QueryRow(ctx, ` +SELECT `+fileStorageChannelColumns+` +FROM file_storage_channels +WHERE channel_key = $1`, strings.TrimSpace(channelKey))) +} + func (s *Store) CreateFileStorageChannel(ctx context.Context, input FileStorageChannelInput) (FileStorageChannel, error) { input = normalizeFileStorageChannelInput(input) credentials, _ := json.Marshal(credentialsFromFileStorageInput(input)) config, _ := json.Marshal(configFromFileStorageInput(input)) - retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy)) + retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider)) return scanFileStorageChannel(s.pool.QueryRow(ctx, ` INSERT INTO file_storage_channels ( @@ -182,10 +197,10 @@ RETURNING `+fileStorageChannelColumns, func (s *Store) UpdateFileStorageChannel(ctx context.Context, id string, input FileStorageChannelInput) (FileStorageChannel, error) { input = normalizeFileStorageChannelInput(input) - replaceCredentials := input.APIKey != nil + replaceCredentials := fileStorageInputReplacesCredentials(input) credentials, _ := json.Marshal(credentialsFromFileStorageInput(input)) config, _ := json.Marshal(configFromFileStorageInput(input)) - retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy)) + retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider)) return scanFileStorageChannel(s.pool.QueryRow(ctx, ` UPDATE file_storage_channels @@ -193,7 +208,7 @@ SET channel_key = $2, name = $3, provider = $4, upload_url = NULLIF($5, ''), - credentials = CASE WHEN $6::boolean THEN $7 ELSE credentials END, + credentials = CASE WHEN $6::boolean THEN credentials || $7::jsonb ELSE credentials END, config = $8, retry_policy = $9, priority = $10, @@ -307,6 +322,9 @@ func scanFileStorageChannel(scanner fileStorageChannelScanner) (FileStorageChann } credentialObject := decodeObject(credentials) item.APIKey = stringFromObject(credentialObject, "apiKey") + item.AccessKeyID = stringFromObject(credentialObject, "accessKeyId") + item.AccessKeySecret = stringFromObject(credentialObject, "accessKeySecret") + item.SessionToken = stringFromObject(credentialObject, "sessionToken") item.CredentialsPreview = maskCredentialsPreview(credentials) configObject := decodeObject(config) item.Scenes = fileStorageScenesFromConfig(configObject) @@ -324,6 +342,17 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage apiKey := strings.TrimSpace(*input.APIKey) input.APIKey = &apiKey } + if input.AccessKeyID == nil && input.AccessKey != nil { + input.AccessKeyID = input.AccessKey + } + if input.AccessKeySecret == nil && input.SecretKey != nil { + input.AccessKeySecret = input.SecretKey + } + for _, value := range []*string{input.AccessKeyID, input.AccessKeySecret, input.SessionToken} { + if value != nil { + *value = strings.TrimSpace(*value) + } + } input.Scenes = normalizeFileStorageScenes(input.Scenes) input.Status = strings.ToLower(strings.TrimSpace(input.Status)) if input.Provider == "" { @@ -342,11 +371,24 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage } func credentialsFromFileStorageInput(input FileStorageChannelInput) map[string]any { - apiKey := fileStorageInputAPIKey(input) - if apiKey == "" { - return map[string]any{} + credentials := map[string]any{} + if input.APIKey != nil { + credentials["apiKey"] = fileStorageInputAPIKey(input) } - return map[string]any{"apiKey": apiKey} + if input.AccessKeyID != nil { + credentials["accessKeyId"] = strings.TrimSpace(*input.AccessKeyID) + } + if input.AccessKeySecret != nil { + credentials["accessKeySecret"] = strings.TrimSpace(*input.AccessKeySecret) + } + if input.SessionToken != nil { + credentials["sessionToken"] = strings.TrimSpace(*input.SessionToken) + } + return credentials +} + +func fileStorageInputReplacesCredentials(input FileStorageChannelInput) bool { + return input.APIKey != nil || input.AccessKeyID != nil || input.AccessKeySecret != nil || input.AccessKey != nil || input.SecretKey != nil || input.SessionToken != nil } func fileStorageInputAPIKey(input FileStorageChannelInput) string { @@ -549,7 +591,10 @@ func NormalizeFileStorageResultUploadPolicy(policy string) string { case "upload_all", "all", "always", "all_upload": return FileStorageResultUploadPolicyUploadAll case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all": - return FileStorageResultUploadPolicyUploadNone + // Local result persistence is no longer a supported write path. Preserve + // compatibility with historical settings by normalizing them to the safe + // object-storage policy. + return FileStorageResultUploadPolicyDefault default: return FileStorageResultUploadPolicyDefault } @@ -601,10 +646,22 @@ func defaultFileStorageScenes() []string { return []string{FileStorageSceneUpload, FileStorageSceneImageResult, FileStorageSceneRequestAsset} } -func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any) map[string]any { +func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any, providers ...string) map[string]any { if len(policy) > 0 { return policy } + provider := "" + if len(providers) > 0 { + provider = strings.ToLower(strings.TrimSpace(providers[0])) + } + if provider == "aliyun_oss" || provider == "s3" { + return map[string]any{ + "enabled": true, + "maxRetries": 2, + "backoffSeconds": []any{0.25, 1.0}, + "strategy": "exponential", + } + } return map[string]any{ "enabled": true, "maxRetries": 3, diff --git a/apps/api/internal/store/file_storage_channels_test.go b/apps/api/internal/store/file_storage_channels_test.go index 61dbafb..1b9e21c 100644 --- a/apps/api/internal/store/file_storage_channels_test.go +++ b/apps/api/internal/store/file_storage_channels_test.go @@ -1,7 +1,9 @@ package store import ( + "encoding/json" "reflect" + "strings" "testing" ) @@ -15,3 +17,32 @@ func TestDefaultFileStorageScenesIncludeCrossNodeRequestAssets(t *testing.T) { t.Fatalf("default file storage scenes = %#v, want %#v", got, want) } } + +func TestFileStorageChannelJSONNeverIncludesCredentials(t *testing.T) { + channel := FileStorageChannel{ + Provider: "s3", APIKey: "api-key-secret", AccessKeyID: "access-key-id", + AccessKeySecret: "access-key-secret", SessionToken: "session-token", + CredentialsPreview: map[string]any{"accessKeyId": "acce…-id", "accessKeySecret": "acce…cret"}, + } + payload, err := json.Marshal(channel) + if err != nil { + t.Fatal(err) + } + encoded := string(payload) + for _, secret := range []string{"api-key-secret", "access-key-id", "access-key-secret", "session-token"} { + if strings.Contains(encoded, secret) { + t.Fatalf("channel JSON leaked credential %q: %s", secret, encoded) + } + } + if !strings.Contains(encoded, "credentialsPreview") { + t.Fatalf("channel JSON omitted masked preview: %s", encoded) + } +} + +func TestLegacyUploadNoneNormalizesToObjectStorageDefault(t *testing.T) { + for _, value := range []string{"upload_none", "none", "never", "skip_all"} { + if got := NormalizeFileStorageResultUploadPolicy(value); got != FileStorageResultUploadPolicyDefault { + t.Fatalf("policy %q normalized to %q", value, got) + } + } +} diff --git a/apps/api/internal/store/postgres.go b/apps/api/internal/store/postgres.go index 03c62db..706cbbd 100644 --- a/apps/api/internal/store/postgres.go +++ b/apps/api/internal/store/postgres.go @@ -14,6 +14,7 @@ import ( "unicode" "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" @@ -568,73 +569,74 @@ type CreateTaskResult struct { } type GatewayTask struct { - ID string `json:"id"` - ExternalTaskID string `json:"externalTaskId,omitempty"` - Kind string `json:"kind"` - RunMode string `json:"runMode"` - AcceptanceRunID string `json:"acceptanceRunId,omitempty"` - UserID string `json:"userId"` - GatewayUserID string `json:"gatewayUserId,omitempty"` - UserSource string `json:"userSource,omitempty"` - GatewayTenantID string `json:"gatewayTenantId,omitempty"` - TenantID string `json:"tenantId,omitempty"` - TenantKey string `json:"tenantKey,omitempty"` - APIKeyID string `json:"apiKeyId,omitempty"` - APIKeyName string `json:"apiKeyName,omitempty"` - APIKeyPrefix string `json:"apiKeyPrefix,omitempty"` - UserGroupID string `json:"userGroupId,omitempty"` - UserGroupKey string `json:"userGroupKey,omitempty"` - Model string `json:"model"` - ModelType string `json:"modelType,omitempty"` - RequestedModel string `json:"requestedModel,omitempty"` - ResolvedModel string `json:"resolvedModel,omitempty"` - RequestID string `json:"requestId,omitempty"` - ConversationID string `json:"conversationId,omitempty"` - NewMessageCount int `json:"newMessageCount,omitempty"` - Request map[string]any `json:"request,omitempty"` - AsyncMode bool `json:"asyncMode"` - RiverJobID int64 `json:"riverJobId,omitempty"` - Status string `json:"status"` - QueueKey string `json:"-"` - Priority int `json:"-"` - Cancellable *bool `json:"cancellable,omitempty"` - Submitted *bool `json:"submitted,omitempty"` - Message string `json:"message,omitempty"` - AttemptCount int `json:"attemptCount"` - RemoteTaskID string `json:"remoteTaskId,omitempty"` - RemoteTaskPayload map[string]any `json:"-"` - Result map[string]any `json:"result,omitempty"` - Billings []any `json:"billings,omitempty"` - Usage map[string]any `json:"usage"` - Metrics map[string]any `json:"metrics"` - BillingSummary map[string]any `json:"billingSummary"` - FinalChargeAmount float64 `json:"finalChargeAmount"` - BillingVersion string `json:"billingVersion"` - BillingStatus string `json:"billingStatus"` - BillingCurrency string `json:"billingCurrency"` - PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"` - RequestFingerprint string `json:"requestFingerprint,omitempty"` - ReservationAmount float64 `json:"reservationAmount"` - ExecutionToken string `json:"-"` - ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"` - BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"` - BillingSettledAt string `json:"billingSettledAt,omitempty"` - ResponseStartedAt string `json:"responseStartedAt,omitempty"` - ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` - ResponseDurationMS int64 `json:"responseDurationMs"` - FinishedAt string `json:"finishedAt,omitempty"` - Error string `json:"error,omitempty"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"` - CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"` - CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"` - CompatibilitySubmitHTTPStatus int `json:"compatibilitySubmitHttpStatus,omitempty"` - CompatibilitySubmitHeaders map[string]any `json:"compatibilitySubmitHeaders,omitempty"` - CompatibilitySubmitBody map[string]any `json:"compatibilitySubmitBody,omitempty"` - Attempts []TaskAttempt `json:"attempts,omitempty"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + ExternalTaskID string `json:"externalTaskId,omitempty"` + Kind string `json:"kind"` + RunMode string `json:"runMode"` + AcceptanceRunID string `json:"acceptanceRunId,omitempty"` + UserID string `json:"userId"` + GatewayUserID string `json:"gatewayUserId,omitempty"` + UserSource string `json:"userSource,omitempty"` + GatewayTenantID string `json:"gatewayTenantId,omitempty"` + TenantID string `json:"tenantId,omitempty"` + TenantKey string `json:"tenantKey,omitempty"` + APIKeyID string `json:"apiKeyId,omitempty"` + APIKeyName string `json:"apiKeyName,omitempty"` + APIKeyPrefix string `json:"apiKeyPrefix,omitempty"` + UserGroupID string `json:"userGroupId,omitempty"` + UserGroupKey string `json:"userGroupKey,omitempty"` + Model string `json:"model"` + ModelType string `json:"modelType,omitempty"` + RequestedModel string `json:"requestedModel,omitempty"` + ResolvedModel string `json:"resolvedModel,omitempty"` + RequestID string `json:"requestId,omitempty"` + ConversationID string `json:"conversationId,omitempty"` + NewMessageCount int `json:"newMessageCount,omitempty"` + Request map[string]any `json:"request,omitempty"` + AsyncMode bool `json:"asyncMode"` + RiverJobID int64 `json:"riverJobId,omitempty"` + Status string `json:"status"` + QueueKey string `json:"-"` + Priority int `json:"-"` + Cancellable *bool `json:"cancellable,omitempty"` + Submitted *bool `json:"submitted,omitempty"` + Message string `json:"message,omitempty"` + AttemptCount int `json:"attemptCount"` + RemoteTaskID string `json:"remoteTaskId,omitempty"` + RemoteTaskPayload map[string]any `json:"-"` + Result map[string]any `json:"result,omitempty"` + Billings []any `json:"billings,omitempty"` + Usage map[string]any `json:"usage"` + Metrics map[string]any `json:"metrics"` + BillingSummary map[string]any `json:"billingSummary"` + FinalChargeAmount float64 `json:"finalChargeAmount"` + BillingVersion string `json:"billingVersion"` + BillingStatus string `json:"billingStatus"` + BillingCurrency string `json:"billingCurrency"` + PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"` + RequestFingerprint string `json:"requestFingerprint,omitempty"` + ReservationAmount float64 `json:"reservationAmount"` + ExecutionToken string `json:"-"` + ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"` + BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"` + BillingSettledAt string `json:"billingSettledAt,omitempty"` + ResponseStartedAt string `json:"responseStartedAt,omitempty"` + ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` + ResponseDurationMS int64 `json:"responseDurationMs"` + FinishedAt string `json:"finishedAt,omitempty"` + Error string `json:"error,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + PublicError *publicerror.Error `json:"publicError,omitempty"` + CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"` + CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"` + CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"` + CompatibilitySubmitHTTPStatus int `json:"compatibilitySubmitHttpStatus,omitempty"` + CompatibilitySubmitHeaders map[string]any `json:"compatibilitySubmitHeaders,omitempty"` + CompatibilitySubmitBody map[string]any `json:"compatibilitySubmitBody,omitempty"` + Attempts []TaskAttempt `json:"attempts,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } const gatewayTaskColumns = ` @@ -656,7 +658,7 @@ COALESCE(reservation_amount, 0)::float8, COALESCE(execution_token::text, ''), COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''), COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''), COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''), -COALESCE(error_code, ''), COALESCE(error_message, ''), +COALESCE(error_code, ''), COALESCE(error_message, ''), COALESCE(public_error, '{}'::jsonb), COALESCE(compatibility_protocol, ''), COALESCE(compatibility_public_id, ''), COALESCE(compatibility_source_protocol, ''), COALESCE(compatibility_submit_http_status, 0), COALESCE(compatibility_submit_headers, '{}'::jsonb), COALESCE(compatibility_submit_body, '{}'::jsonb), @@ -679,39 +681,40 @@ type TaskEvent struct { } type TaskAttempt struct { - ID string `json:"id"` - TaskID string `json:"taskId"` - AttemptNo int `json:"attemptNo"` - PlatformID string `json:"platformId,omitempty"` - PlatformName string `json:"platformName,omitempty"` - Provider string `json:"provider,omitempty"` - PlatformModelID string `json:"platformModelId,omitempty"` - ModelName string `json:"modelName,omitempty"` - ProviderModelName string `json:"providerModelName,omitempty"` - ModelAlias string `json:"modelAlias,omitempty"` - ModelType string `json:"modelType,omitempty"` - ClientID string `json:"clientId,omitempty"` - QueueKey string `json:"queueKey"` - Status string `json:"status"` - Retryable bool `json:"retryable"` - Simulated bool `json:"simulated"` - RequestID string `json:"requestId,omitempty"` - StatusCode int `json:"statusCode,omitempty"` - Usage map[string]any `json:"usage,omitempty"` - Metrics map[string]any `json:"metrics,omitempty"` - RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"` - ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"` - ResponseStartedAt string `json:"responseStartedAt,omitempty"` - ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` - ResponseDurationMS int64 `json:"responseDurationMs"` - PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"` - RequestFingerprint string `json:"requestFingerprint,omitempty"` - UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"` - UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - StartedAt time.Time `json:"startedAt"` - FinishedAt string `json:"finishedAt,omitempty"` + ID string `json:"id"` + TaskID string `json:"taskId"` + AttemptNo int `json:"attemptNo"` + PlatformID string `json:"platformId,omitempty"` + PlatformName string `json:"platformName,omitempty"` + Provider string `json:"provider,omitempty"` + PlatformModelID string `json:"platformModelId,omitempty"` + ModelName string `json:"modelName,omitempty"` + ProviderModelName string `json:"providerModelName,omitempty"` + ModelAlias string `json:"modelAlias,omitempty"` + ModelType string `json:"modelType,omitempty"` + ClientID string `json:"clientId,omitempty"` + QueueKey string `json:"queueKey"` + Status string `json:"status"` + Retryable bool `json:"retryable"` + Simulated bool `json:"simulated"` + RequestID string `json:"requestId,omitempty"` + StatusCode int `json:"statusCode,omitempty"` + Usage map[string]any `json:"usage,omitempty"` + Metrics map[string]any `json:"metrics,omitempty"` + RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"` + ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"` + ResponseStartedAt string `json:"responseStartedAt,omitempty"` + ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` + ResponseDurationMS int64 `json:"responseDurationMs"` + PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"` + RequestFingerprint string `json:"requestFingerprint,omitempty"` + UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"` + UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + PublicError *publicerror.Error `json:"publicError,omitempty"` + StartedAt time.Time `json:"startedAt"` + FinishedAt string `json:"finishedAt,omitempty"` } type TaskParamPreprocessingLog struct { @@ -2240,6 +2243,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) { var remoteTaskPayloadBytes []byte var compatibilitySubmitHeadersBytes []byte var compatibilitySubmitBodyBytes []byte + var publicErrorBytes []byte if err := scanner.Scan( &task.ID, &task.ExternalTaskID, @@ -2295,6 +2299,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) { &task.Error, &task.ErrorCode, &task.ErrorMessage, + &publicErrorBytes, &task.CompatibilityProtocol, &task.CompatibilityPublicID, &task.CompatibilitySourceProtocol, @@ -2317,9 +2322,25 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) { task.PricingSnapshot = decodeObject(pricingSnapshotBytes) task.CompatibilitySubmitHeaders = decodeObject(compatibilitySubmitHeadersBytes) task.CompatibilitySubmitBody = decodeObject(compatibilitySubmitBodyBytes) + if len(publicErrorBytes) > 0 { + var snapshot publicerror.Error + if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" { + task.PublicError = &snapshot + } + } return task, nil } +func encodePublicErrorSnapshot(code string, message string, status int, retryable bool, requestID string, taskID string) []byte { + if strings.TrimSpace(code) == "" && strings.TrimSpace(message) == "" { + return []byte("null") + } + snapshot := publicerror.FromFields(code, message, status, retryable) + snapshot = publicerror.WithIDs(snapshot, requestID, taskID) + payload, _ := json.Marshal(snapshot) + return payload +} + func (s *Store) ListTaskEvents(ctx context.Context, taskID string) ([]TaskEvent, error) { rows, err := s.pool.Query(ctx, ` SELECT id::text, task_id::text, seq, event_type, COALESCE(status, ''), COALESCE(phase, ''), diff --git a/apps/api/internal/store/request_assets.go b/apps/api/internal/store/request_assets.go index 56a03e0..92439ca 100644 --- a/apps/api/internal/store/request_assets.go +++ b/apps/api/internal/store/request_assets.go @@ -9,33 +9,43 @@ import ( ) type RequestAsset struct { - ID string `json:"id"` - SHA256 string `json:"sha256"` - ContentType string `json:"contentType"` - ByteSize int64 `json:"byteSize"` - URL string `json:"url"` - StorageProvider string `json:"storageProvider"` - LocalPath string `json:"localPath,omitempty"` - ExpiresAt *time.Time `json:"expiresAt,omitempty"` - ExpiredAt *time.Time `json:"expiredAt,omitempty"` - RefCount int `json:"refCount"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + SHA256 string `json:"sha256"` + ContentType string `json:"contentType"` + ByteSize int64 `json:"byteSize"` + URL string `json:"url"` + StorageProvider string `json:"storageProvider"` + StorageChannelID string `json:"storageChannelId,omitempty"` + StorageChannelKey string `json:"storageChannelKey,omitempty"` + ObjectKey string `json:"objectKey,omitempty"` + AccessScope string `json:"accessScope,omitempty"` + LocalPath string `json:"localPath,omitempty"` + ExpiresAt *time.Time `json:"expiresAt,omitempty"` + ExpiredAt *time.Time `json:"expiredAt,omitempty"` + RefCount int `json:"refCount"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } type RequestAssetInput struct { - SHA256 string - ContentType string - ByteSize int64 - URL string - StorageProvider string - LocalPath string - ExpiresAt *time.Time + SHA256 string + ContentType string + ByteSize int64 + URL string + StorageProvider string + StorageChannelID string + StorageChannelKey string + ObjectKey string + AccessScope string + LocalPath string + ExpiresAt *time.Time } func (s *Store) FindRequestAsset(ctx context.Context, sha256 string, contentType string) (RequestAsset, bool, error) { asset, err := scanRequestAsset(s.pool.QueryRow(ctx, ` SELECT id::text, sha256, content_type, byte_size, url, storage_provider, + COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''), + COALESCE(object_key, ''), COALESCE(access_scope, 'private'), COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at FROM gateway_request_assets WHERE sha256 = $1 AND content_type = $2`, sha256, contentType)) @@ -51,25 +61,37 @@ WHERE sha256 = $1 AND content_type = $2`, sha256, contentType)) func (s *Store) UpsertRequestAsset(ctx context.Context, input RequestAssetInput) (RequestAsset, error) { return scanRequestAsset(s.pool.QueryRow(ctx, ` INSERT INTO gateway_request_assets ( - sha256, content_type, byte_size, url, storage_provider, local_path, expires_at, expired_at, ref_count + sha256, content_type, byte_size, url, storage_provider, storage_channel_id, + storage_channel_key, object_key, access_scope, local_path, expires_at, expired_at, ref_count ) -VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), $7, NULL, 1) +VALUES ($1, $2, $3, $4, $5, NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''), + COALESCE(NULLIF($9, ''), 'private'), NULLIF($10, ''), $11, NULL, 1) ON CONFLICT (sha256, content_type) DO UPDATE SET byte_size = EXCLUDED.byte_size, url = EXCLUDED.url, storage_provider = EXCLUDED.storage_provider, + storage_channel_id = EXCLUDED.storage_channel_id, + storage_channel_key = EXCLUDED.storage_channel_key, + object_key = EXCLUDED.object_key, + access_scope = EXCLUDED.access_scope, local_path = EXCLUDED.local_path, expires_at = EXCLUDED.expires_at, expired_at = NULL, ref_count = gateway_request_assets.ref_count + 1, updated_at = now() RETURNING id::text, sha256, content_type, byte_size, url, storage_provider, + COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''), + COALESCE(object_key, ''), COALESCE(access_scope, 'private'), COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at`, input.SHA256, input.ContentType, input.ByteSize, input.URL, input.StorageProvider, + input.StorageChannelID, + input.StorageChannelKey, + input.ObjectKey, + input.AccessScope, input.LocalPath, input.ExpiresAt, )) @@ -110,6 +132,10 @@ func scanRequestAsset(scanner interface{ Scan(dest ...any) error }) (RequestAsse &asset.ByteSize, &asset.URL, &asset.StorageProvider, + &asset.StorageChannelID, + &asset.StorageChannelKey, + &asset.ObjectKey, + &asset.AccessScope, &localPath, &expiresAt, &expiredAt, diff --git a/apps/api/internal/store/task_history.go b/apps/api/internal/store/task_history.go index d144d05..f8a6f33 100644 --- a/apps/api/internal/store/task_history.go +++ b/apps/api/internal/store/task_history.go @@ -13,17 +13,20 @@ const ( ) type TaskCallbackDelivery struct { - ID string - TaskID string - EventID string - Seq int64 - CallbackURL string - Status string - Attempts int - LockToken string - EventType string - TaskStatus string - CreatedAt time.Time + ID string + TaskID string + EventID string + Seq int64 + CallbackURL string + Status string + Attempts int + LockToken string + EventType string + TaskStatus string + TaskRequestID string + TaskErrorCode string + TaskErrorMessage string + CreatedAt time.Time } type TaskHistoryCleanupResult struct { @@ -67,9 +70,13 @@ func (s *Store) ClaimTaskCallbacks(ctx context.Context, workerID string, limit i } rows, err := s.pool.Query(ctx, ` WITH picked AS ( - SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at + SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at, + COALESCE(task.request_id, '') AS task_request_id, + COALESCE(task.error_code, '') AS task_error_code, + COALESCE(task.error_message, task.error, '') AS task_error_message FROM gateway_task_callback_outbox outbox JOIN gateway_task_events event ON event.id = outbox.event_id + JOIN gateway_tasks task ON task.id = outbox.task_id WHERE outbox.created_at >= COALESCE(( SELECT applied_at FROM schema_migrations @@ -101,7 +108,7 @@ WHERE outbox.id = picked.id RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, ''), outbox.seq, outbox.callback_url, outbox.status, outbox.attempts, COALESCE(outbox.lock_token::text, ''), picked.event_type, picked.task_status, - picked.created_at`, + picked.created_at, picked.task_request_id, picked.task_error_code, picked.task_error_message`, workerID, limit, int(staleAfter/time.Second)) if err != nil { return nil, err @@ -122,6 +129,9 @@ RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, &item.EventType, &item.TaskStatus, &item.CreatedAt, + &item.TaskRequestID, + &item.TaskErrorCode, + &item.TaskErrorMessage, ); err != nil { return nil, err } diff --git a/apps/api/internal/store/tasks_runtime.go b/apps/api/internal/store/tasks_runtime.go index 18c79f7..b5fd34f 100644 --- a/apps/api/internal/store/tasks_runtime.go +++ b/apps/api/internal/store/tasks_runtime.go @@ -11,6 +11,7 @@ import ( "unicode/utf8" "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror" "github.com/jackc/pgx/v5" ) @@ -544,12 +545,14 @@ WHERE id = $1::uuid } func (s *Store) FailQueuedTask(ctx context.Context, taskID string, code string, message string) (GatewayTask, error) { + publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID) tag, err := s.pool.Exec(ctx, ` UPDATE gateway_tasks SET status = 'failed', error = NULL, error_code = NULLIF($2, ''), error_message = NULLIF($3, ''), + public_error = $4::jsonb, billing_status = CASE WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required' ELSE 'released' @@ -563,7 +566,7 @@ SET status = 'failed', finished_at = now(), updated_at = now() WHERE id = $1::uuid - AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048)) + AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048), string(publicErrorJSON)) if err != nil { return GatewayTask{}, err } @@ -686,6 +689,7 @@ SET status = 'queued', error = NULL, error_code = NULL, error_message = NULL, + public_error = NULL, updated_at = now() WHERE id = $1::uuid AND status IN ('queued', 'running') @@ -722,6 +726,7 @@ SET status = 'queued', error = NULL, error_code = NULL, error_message = NULL, + public_error = NULL, updated_at = now() WHERE task.id = $1::uuid AND task.status IN ('queued', 'running') @@ -1371,6 +1376,7 @@ SELECT a.id::text, a.task_id::text, a.attempt_no, a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb), COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''), COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''), + COALESCE(a.public_error, '{}'::jsonb), COALESCE(a.pricing_snapshot, '{}'::jsonb), COALESCE(a.request_fingerprint, ''), a.upstream_submission_status, COALESCE(a.upstream_submission_updated_at::text, ''), a.started_at, COALESCE(a.finished_at::text, '') @@ -1403,6 +1409,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) { var requestBytes []byte var responseBytes []byte var pricingSnapshotBytes []byte + var publicErrorBytes []byte if err := scanner.Scan( &item.ID, &item.TaskID, @@ -1430,6 +1437,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) { &item.ResponseDurationMS, &item.ErrorCode, &item.ErrorMessage, + &publicErrorBytes, &pricingSnapshotBytes, &item.RequestFingerprint, &item.UpstreamSubmissionStatus, @@ -1444,6 +1452,12 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) { item.RequestSnapshot = decodeObject(requestBytes) item.ResponseSnapshot = decodeObject(responseBytes) item.PricingSnapshot = decodeObject(pricingSnapshotBytes) + if len(publicErrorBytes) > 0 { + var snapshot publicerror.Error + if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" { + item.PublicError = &snapshot + } + } enrichTaskAttemptFromMetrics(&item) return item, nil } @@ -1534,6 +1548,7 @@ func (s *Store) FinishTaskAttempt(ctx context.Context, input FinishTaskAttemptIn } usageJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptUsage(input.Usage))) metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptMetrics(input.Metrics))) + publicErrorJSON := encodePublicErrorSnapshot(input.ErrorCode, input.ErrorMessage, statusCode, input.Retryable, input.RequestID, "") _, err := s.pool.Exec(ctx, ` UPDATE gateway_task_attempts SET status = $2::text, @@ -1550,6 +1565,7 @@ SET status = $2::text, response_duration_ms = $8, error_code = NULLIF($9::text, ''), error_message = NULLIF(left($10::text, 2048), ''), + public_error = CASE WHEN $2::text = 'failed' THEN $14::jsonb ELSE NULL END, upstream_submission_status = COALESCE(NULLIF($13::text, ''), upstream_submission_status), upstream_submission_updated_at = CASE WHEN NULLIF($13::text, '') IS NULL THEN upstream_submission_updated_at @@ -1570,6 +1586,7 @@ WHERE id = $1::uuid`, usageJSON, metricsJSON, input.UpstreamSubmissionStatus, + string(publicErrorJSON), ) return err } @@ -1651,6 +1668,7 @@ SET status = 'succeeded', response_duration_ms = $5, error_code = NULL, error_message = NULL, + public_error = NULL, finished_at = now() WHERE id = $1::uuid`, input.AttemptID, input.RequestID, @@ -1687,6 +1705,7 @@ SET status = 'succeeded', error = NULL, error_code = NULL, error_message = NULL, + public_error = NULL, locked_by = NULL, locked_at = NULL, heartbeat_at = NULL, @@ -2049,6 +2068,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics))) resultJSON, _ := json.Marshal(minimalTaskResult(nil)) message := truncateUTF8Bytes(input.Message, 2048) + publicErrorJSON := encodePublicErrorSnapshot(input.Code, message, 0, true, input.RequestID, input.TaskID) finalizedAdmission := false err := s.beginTransaction(ctx, func(tx pgx.Tx) error { tag, err := tx.Exec(ctx, ` @@ -2057,6 +2077,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn error = NULL, error_code = NULLIF($3::text, ''), error_message = NULLIF($2::text, ''), + public_error = $11::jsonb, request_id = NULLIF($4::text, ''), metrics = $5::jsonb, response_started_at = $6::timestamptz, @@ -2090,6 +2111,7 @@ WHERE id = $1::uuid input.ResponseDurationMS, string(resultJSON), input.ExecutionToken, + string(publicErrorJSON), ) if err != nil { return err diff --git a/apps/api/migrations/0103_object_storage_references.sql b/apps/api/migrations/0103_object_storage_references.sql new file mode 100644 index 0000000..ad6f5b0 --- /dev/null +++ b/apps/api/migrations/0103_object_storage_references.sql @@ -0,0 +1,21 @@ +ALTER TABLE gateway_request_assets + ADD COLUMN IF NOT EXISTS storage_channel_id uuid REFERENCES file_storage_channels(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS storage_channel_key text, + ADD COLUMN IF NOT EXISTS object_key text, + ADD COLUMN IF NOT EXISTS access_scope text DEFAULT 'private'; + +ALTER TABLE gateway_tasks + ADD COLUMN IF NOT EXISTS public_error jsonb; + +ALTER TABLE gateway_task_attempts + ADD COLUMN IF NOT EXISTS public_error jsonb; + +CREATE INDEX IF NOT EXISTS idx_gateway_request_assets_object + ON gateway_request_assets(storage_channel_key, object_key) + WHERE object_key IS NOT NULL; + +UPDATE system_settings +SET value = jsonb_set(COALESCE(value, '{}'::jsonb), '{resultUploadPolicy}', '"default"'::jsonb, true), + updated_at = now() +WHERE setting_key = 'file_storage' + AND COALESCE(value->>'resultUploadPolicy', '') IN ('upload_none', 'none', 'never', 'disabled', 'no_upload', 'skip', 'skip_all'); diff --git a/apps/api/project.json b/apps/api/project.json index be25e56..b953c9a 100644 --- a/apps/api/project.json +++ b/apps/api/project.json @@ -24,7 +24,7 @@ "outputs": ["{projectRoot}/docs/swagger.json", "{projectRoot}/docs/swagger.yaml"], "options": { "cwd": "apps/api", - "command": "go run github.com/swaggo/swag/cmd/swag@v1.16.4 init --parseInternal -d ./cmd/gateway,./internal/httpapi,./internal/store,./internal/auth,./internal/identity,./internal/runner -g main.go -o docs --outputTypes json,yaml" + "command": "go run github.com/swaggo/swag/cmd/swag@v1.16.4 init --parseInternal -d ./cmd/gateway,./internal/httpapi,./internal/store,./internal/auth,./internal/identity,./internal/runner,./internal/publicerror -g main.go -o docs --outputTypes json,yaml" } }, "test": { diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 595de68..d94552f 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -111,6 +111,7 @@ import { updateClientCustomizationSettings, updateFileStorageChannel, updateFileStorageSettings, + testFileStorageChannel, updateGatewayUser, updatePlatform, updatePlatformDynamicPriority, @@ -1182,6 +1183,22 @@ export function App() { } } + async function verifyFileStorageChannel(channelId: string) { + setCoreState('loading'); + setCoreMessage(''); + try { + const result = await testFileStorageChannel(token, channelId); + const refreshed = await listFileStorageChannels(token); + setFileStorageChannels(refreshed.items); + setCoreState('ready'); + setCoreMessage(`对象存储连接测试通过，Put / Head / Delete 均成功，耗时 ${result.durationMs}ms。`); + } catch (err) { + setCoreState('error'); + setCoreMessage(err instanceof Error ? err.message : '对象存储连接测试失败'); + throw err; + } + } + async function saveClientCustomizationSettings(input: ClientCustomizationSettingsUpdateRequest) { setCoreState('loading'); setCoreMessage(''); @@ -1614,6 +1631,7 @@ export function App() { onSaveAccessRule={saveAccessRule} onSaveFileStorageChannel={saveFileStorageChannel} onSaveFileStorageSettings={saveFileStorageSettings} + onTestFileStorageChannel={verifyFileStorageChannel} onSaveClientCustomizationSettings={saveClientCustomizationSettings} onConnectSecurityEvents={connectSecurityEvents} onDisconnectSecurityEvents={disconnectSecurityEvents} diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts index 98d2639..9f38252 100644 --- a/apps/web/src/api.ts +++ b/apps/web/src/api.ts @@ -12,6 +12,7 @@ import type { ClientCustomizationSettingsUpdateRequest, CreatedGatewayApiKey, FileStorageChannel, + FileStorageChannelTestResult, FileStorageSettings, FileStorageSettingsUpdateRequest, FileStorageChannelUpsertRequest, @@ -1276,6 +1277,16 @@ export async function deleteFileStorageChannel(token: string, channelId: string) }); } +export async function testFileStorageChannel( + token: string, + channelId: string, +): Promise { + return request(`/api/admin/system/file-storage/channels/${channelId}/test`, { + method: 'POST', + token, + }); +} + async function request( path: string, options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record; signal?: AbortSignal; timeoutMs?: number } = {}, diff --git a/apps/web/src/pages/AdminPage.tsx b/apps/web/src/pages/AdminPage.tsx index 64cf042..5d52a9c 100644 --- a/apps/web/src/pages/AdminPage.tsx +++ b/apps/web/src/pages/AdminPage.tsx @@ -92,6 +92,7 @@ export function AdminPage(props: { onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise; onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise; onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise; + onTestFileStorageChannel: (channelId: string) => Promise; onConnectSecurityEvents: (transmitterIssuer: string) => Promise; onDisconnectSecurityEvents: () => Promise; onRefreshSecurityEvents: () => Promise; @@ -214,6 +215,7 @@ export function AdminPage(props: { onDeleteFileStorageChannel={props.onDeleteFileStorageChannel} onSaveFileStorageChannel={props.onSaveFileStorageChannel} onSaveFileStorageSettings={props.onSaveFileStorageSettings} + onTestFileStorageChannel={props.onTestFileStorageChannel} onSaveClientCustomizationSettings={props.onSaveClientCustomizationSettings} /> )} diff --git a/apps/web/src/pages/admin/SystemSettingsPanel.tsx b/apps/web/src/pages/admin/SystemSettingsPanel.tsx index f0c3c11..1b792f8 100644 --- a/apps/web/src/pages/admin/SystemSettingsPanel.tsx +++ b/apps/web/src/pages/admin/SystemSettingsPanel.tsx @@ -15,6 +15,10 @@ type ClientCustomizationForm = { }; type FileStorageChannelForm = { + accessKeyId: string; + accessKeyIdPreview: string; + accessKeySecret: string; + accessKeySecretPreview: string; apiKey: string; apiKeyPreview: string; channelKey: string; @@ -23,13 +27,22 @@ type FileStorageChannelForm = { priority: string; provider: string; retryPolicyJson: string; + sessionToken: string; + sessionTokenPreview: string; scenes: string[]; status: string; uploadUrl: string; }; const defaultUploadUrl = 'http://127.0.0.1:3001/v1/files/upload'; -const defaultRetryPolicy = { +const defaultObjectStorageRetryPolicy = { + enabled: true, + maxRetries: 2, + backoffSeconds: [0.25, 1], + strategy: 'exponential', +}; + +const defaultServerMainRetryPolicy = { enabled: true, maxRetries: 3, backoffSeconds: [60, 120, 180], @@ -39,7 +52,7 @@ const defaultRetryPolicy = { const providerOptions = [ { value: 'server_main_openapi', label: 'server-main OpenAPI' }, { value: 'aliyun_oss', label: '阿里云 OSS' }, - { value: 'tencent_cos', label: '腾讯云 COS' }, + { value: 's3', label: 'S3 / S3 兼容存储' }, ]; const defaultScenes = ['upload', 'image_result', 'request_asset']; @@ -52,7 +65,6 @@ const sceneOptions = [ const resultUploadPolicyOptions = [ { value: 'default', label: '默认：仅非链接资源转存', description: 'URL 结果直接保存；base64 / buffer 等结果转存后保存 URL' }, { value: 'upload_all', label: '全部转存', description: 'URL、base64、buffer 等生成媒体结果都会转存到当前文件渠道' }, - { value: 'upload_none', label: '不做外部转存', description: 'base64 / buffer 临时写入本地静态文件；数据库仅保存占位符，默认 24 小时内按需恢复' }, ]; export function SystemSettingsPanel(props: { @@ -66,6 +78,7 @@ export function SystemSettingsPanel(props: { onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise; onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise; onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise; + onTestFileStorageChannel: (channelId: string) => Promise; }) { const [activeTab, setActiveTab] = useState('fileStorage'); const [dialogOpen, setDialogOpen] = useState(false); @@ -75,6 +88,7 @@ export function SystemSettingsPanel(props: { const [clientCustomizationForm, setClientCustomizationForm] = useState(() => clientCustomizationSettingsToForm(props.clientCustomizationSettings)); const [settingsPolicy, setSettingsPolicy] = useState(() => normalizeResultUploadPolicy(props.settings?.resultUploadPolicy)); const [localError, setLocalError] = useState(''); + const [testingChannelId, setTestingChannelId] = useState(''); useEffect(() => { setSettingsPolicy(normalizeResultUploadPolicy(props.settings?.resultUploadPolicy)); @@ -130,6 +144,18 @@ export function SystemSettingsPanel(props: { } } + async function testChannel(channel: FileStorageChannel) { + setLocalError(''); + setTestingChannelId(channel.id); + try { + await props.onTestFileStorageChannel(channel.id); + } catch (err) { + setLocalError(err instanceof Error ? err.message : '对象存储连接测试失败'); + } finally { + setTestingChannelId(''); + } + } + async function saveSettings() { setLocalError(''); try { @@ -154,7 +180,7 @@ export function SystemSettingsPanel(props: {
系统设置 -

集中维护网关级配置；文件存储渠道按优先级轮转，单渠道使用 60/120/180 秒退避重试。

+

集中维护网关级配置；文件存储渠道按优先级轮转，对象存储默认使用 250ms / 1s 退避。

{props.channels.length} 个文件渠道 @@ -177,7 +203,7 @@ export function SystemSettingsPanel(props: {
全局生成媒体转存策略 - 对图片、音频和视频生成结果统一生效；启用转存但没有可用渠道时，回退到保留 24 小时的网关本地静态资源。 + 对图片、音频和视频生成结果统一生效；没有可用对象存储时任务会安全失败，不再写入网关本机目录。
- } + {form.provider !== 'server_main_openapi' && <> + + + + }