BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(storage): 统一二进制对象存储与公开错误

Browse Source 
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。

将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。

引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。

验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
This commit is contained in:
wangbo
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/cmd/acceptance-bootstrap/main.go
+70
View File
@@ -161,6 +161,9 @@ func run(ctx context.Context, opts options) error {
if err := ensureAcceptanceAccessRules(ctx, database, groupID); err != nil {
return err
}
if err := ensureAcceptanceObjectStorage(ctx, database, opts.emulatorBaseURL); err != nil {
return err
}
geminiModel, videoModel, err := selectedModels(ctx, database)
if err != nil {
return err
@@ -240,6 +243,73 @@ func run(ctx context.Context, opts options) error {
return nil
}
func ensureAcceptanceObjectStorage(ctx context.Context, database *store.Store, emulatorBaseURL string) error {
baseURL := strings.TrimRight(strings.TrimSpace(emulatorBaseURL), "/")
if baseURL == "" {
return errors.New("acceptance object storage requires the emulator URL")
}
channels := []struct {
key string
name string
provider string
endpoint string
priority int
pathStyle bool
}{
{
key: "local-acceptance-oss", name: "Local Acceptance Aliyun OSS", provider: "aliyun_oss",
endpoint: baseURL + "/storage/oss/bucket", priority: 10, pathStyle: true,
},
{
key: "local-acceptance-s3", name: "Local Acceptance S3", provider: "s3",
endpoint: baseURL + "/storage/s3", priority: 20, pathStyle: true,
},
}
for _, channel := range channels {
_, err := database.Pool().Exec(ctx, `
INSERT INTO file_storage_channels (
channel_key, name, provider, credentials, config, retry_policy, priority, status
)
VALUES (
$1, $2, $3,
'{"accessKeyId":"local-acceptance","accessKeySecret":"local-acceptance-secret"}'::jsonb,
jsonb_build_object(
'endpoint', $4::text,
'region', 'local-acceptance-1',
'bucket', 'bucket',
'objectPrefix', 'acceptance/media',
'accessScope', 'private',
'forcePathStyle', $5::boolean,
'scenes', jsonb_build_array('upload', 'image_result', 'request_asset'),
'acceptanceEmulatorOnly', true
),
'{"enabled":true,"maxRetries":2,"backoffSeconds":[0.25,1],"strategy":"exponential"}'::jsonb,
$6, 'enabled'
)
ON CONFLICT (channel_key) DO UPDATE
SET name = EXCLUDED.name,
provider = EXCLUDED.provider,
credentials = EXCLUDED.credentials,
config = EXCLUDED.config,
retry_policy = EXCLUDED.retry_policy,
priority = EXCLUDED.priority,
status = 'enabled',
deleted_at = NULL,
last_error = NULL,
updated_at = now()`, channel.key, channel.name, channel.provider, channel.endpoint, channel.pathStyle, channel.priority)
if err != nil {
return fmt.Errorf("configure %s object storage channel: %w", channel.provider, err)
}
}
_, err := database.Pool().Exec(ctx, `
INSERT INTO system_settings (setting_key, value)
VALUES ('file_storage', '{"resultUploadPolicy":"default"}'::jsonb)
ON CONFLICT (setting_key) DO UPDATE
SET value = jsonb_set(COALESCE(system_settings.value, '{}'::jsonb), '{resultUploadPolicy}', '"default"'::jsonb, true),
updated_at = now()`)
return err
}
func resetPreviousLocalRun(ctx context.Context, database *store.Store, opts options) error {
mode, err := database.GetGatewayTrafficMode(ctx)
if err != nil {
apps/api/docs/swagger.json
+182 -16
View File
@@ -3157,7 +3157,7 @@
"BearerAuth": []
}
],
"description": "创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。",
"description": "创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。",
"consumes": [
"application/json"
],
@@ -3354,6 +3354,70 @@
}
}
},
"/api/admin/system/file-storage/channels/{channelID}/test": {
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。",
"produces": [
"application/json"
],
"tags": [
"system"
],
"summary": "测试对象存储通道",
"parameters": [
{
"type": "string",
"description": "文件存储通道 ID",
"name": "channelID",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/httpapi.FileStorageChannelTestResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"503": {
"description": "Service Unavailable",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
}
}
}
},
"/api/admin/system/file-storage/settings": {
"get": {
"security": [
@@ -6712,7 +6776,7 @@
"BearerAuth": []
}
],
"description": "上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。",
"description": "上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。",
"consumes": [
"multipart/form-data"
],
@@ -11207,22 +11271,52 @@
"httpapi.ErrorPayload": {
"type": "object",
"properties": {
"action": {
"type": "string",
"example": "retry_after"
},
"category": {
"type": "string",
"example": "rate_limit"
},
"code": {
"type": "string",
"example": "rate_limit"
},
"httpStatus": {
"type": "integer",
"example": 429
},
"message": {
"type": "string",
"example": "invalid json body"
},
"param": {},
"requestId": {
"type": "string"
},
"retryAfterSeconds": {
"type": "integer",
"example": 2
},
"retryable": {
"type": "boolean",
"example": true
},
"status": {
"type": "integer",
"example": 400
},
"taskId": {
"type": "string"
},
"type": {
"type": "string",
"example": "invalid_request_error"
},
"version": {
"type": "string",
"example": "v1"
}
}
},
@@ -11237,6 +11331,26 @@
}
}
},
"httpapi.FileStorageChannelTestResponse": {
"type": "object",
"properties": {
"deleteSucceeded": {
"type": "boolean"
},
"durationMs": {
"type": "integer"
},
"headSucceeded": {
"type": "boolean"
},
"provider": {
"type": "string"
},
"putSucceeded": {
"type": "boolean"
}
}
},
"httpapi.FileUploadData": {
"type": "object",
"properties": {
@@ -11443,6 +11557,9 @@
"example": 0
},
"data": {},
"error": {
"$ref": "#/definitions/publicerror.Error"
},
"message": {
"type": "string",
"example": "SUCCEED"
@@ -11620,6 +11737,9 @@
"type": "string",
"example": "invalid parameter"
},
"public_error": {
"$ref": "#/definitions/publicerror.Error"
},
"request_id": {
"type": "string"
}
@@ -12809,20 +12929,7 @@
"type": "object",
"properties": {
"error": {
"$ref": "#/definitions/httpapi.VolcesErrorPayload"
}
}
},
"httpapi.VolcesErrorPayload": {
"type": "object",
"properties": {
"code": {
"type": "string",
"example": "invalid_parameter"
},
"message": {
"type": "string",
"example": "model is required"
"$ref": "#/definitions/publicerror.Error"
}
}
},
@@ -13323,6 +13430,41 @@
"RevisionFailed"
]
},
"publicerror.Error": {
"type": "object",
"properties": {
"action": {
"type": "string"
},
"category": {
"type": "string"
},
"code": {
"type": "string"
},
"httpStatus": {
"type": "integer"
},
"message": {
"type": "string"
},
"requestId": {
"type": "string"
},
"retryAfterSeconds": {
"type": "integer"
},
"retryable": {
"type": "boolean"
},
"taskId": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"runner.PortraitAssetCapability": {
"type": "object",
"properties": {
@@ -13835,6 +13977,9 @@
"type": "object",
"additionalProperties": {}
},
"publicError": {
"$ref": "#/definitions/publicerror.Error"
},
"remoteTaskId": {
"type": "string"
},
@@ -14771,6 +14916,15 @@
"store.FileStorageChannelInput": {
"type": "object",
"properties": {
"accessKey": {
"type": "string"
},
"accessKeyId": {
"type": "string"
},
"accessKeySecret": {
"type": "string"
},
"apiKey": {
"type": "string"
},
@@ -14800,6 +14954,12 @@
"type": "string"
}
},
"secretKey": {
"type": "string"
},
"sessionToken": {
"type": "string"
},
"status": {
"type": "string"
},
@@ -14971,6 +15131,9 @@
"type": "object",
"additionalProperties": {}
},
"publicError": {
"$ref": "#/definitions/publicerror.Error"
},
"remoteTaskId": {
"type": "string"
},
@@ -16540,6 +16703,9 @@
"providerModelName": {
"type": "string"
},
"publicError": {
"$ref": "#/definitions/publicerror.Error"
},
"queueKey": {
"type": "string"
},
apps/api/docs/swagger.yaml
+122 -12
View File
@@ -481,19 +481,41 @@ definitions:
type: object
httpapi.ErrorPayload:
properties:
action:
example: retry_after
type: string
category:
example: rate_limit
type: string
code:
example: rate_limit
type: string
httpStatus:
example: 429
type: integer
message:
example: invalid json body
type: string
param: {}
requestId:
type: string
retryAfterSeconds:
example: 2
type: integer
retryable:
example: true
type: boolean
status:
example: 400
type: integer
taskId:
type: string
type:
example: invalid_request_error
type: string
version:
example: v1
type: string
type: object
httpapi.FileStorageChannelListResponse:
properties:
@@ -502,6 +524,19 @@ definitions:
$ref: '#/definitions/store.FileStorageChannel'
type: array
type: object
httpapi.FileStorageChannelTestResponse:
properties:
deleteSucceeded:
type: boolean
durationMs:
type: integer
headSucceeded:
type: boolean
provider:
type: string
putSucceeded:
type: boolean
type: object
httpapi.FileUploadData:
properties:
assetStorage:
@@ -651,6 +686,8 @@ definitions:
example: 0
type: integer
data: {}
error:
$ref: '#/definitions/publicerror.Error'
message:
example: SUCCEED
type: string
@@ -775,6 +812,8 @@ definitions:
message:
example: invalid parameter
type: string
public_error:
$ref: '#/definitions/publicerror.Error'
request_id:
type: string
type: object
@@ -1602,16 +1641,7 @@ definitions:
httpapi.VolcesErrorEnvelope:
properties:
error:
$ref: '#/definitions/httpapi.VolcesErrorPayload'
type: object
httpapi.VolcesErrorPayload:
properties:
code:
example: invalid_parameter
type: string
message:
example: model is required
type: string
$ref: '#/definitions/publicerror.Error'
type: object
httpapi.WalletAdjustmentResponse:
properties:
@@ -1953,6 +1983,29 @@ definitions:
- RevisionActive
- RevisionSuperseded
- RevisionFailed
publicerror.Error:
properties:
action:
type: string
category:
type: string
code:
type: string
httpStatus:
type: integer
message:
type: string
requestId:
type: string
retryAfterSeconds:
type: integer
retryable:
type: boolean
taskId:
type: string
version:
type: string
type: object
runner.PortraitAssetCapability:
properties:
availablePlatformIds:
@@ -2297,6 +2350,8 @@ definitions:
pricingSnapshot:
additionalProperties: {}
type: object
publicError:
$ref: '#/definitions/publicerror.Error'
remoteTaskId:
type: string
request:
@@ -2929,6 +2984,12 @@ definitions:
type: object
store.FileStorageChannelInput:
properties:
accessKey:
type: string
accessKeyId:
type: string
accessKeySecret:
type: string
apiKey:
type: string
channelKey:
@@ -2949,6 +3010,10 @@ definitions:
items:
type: string
type: array
secretKey:
type: string
sessionToken:
type: string
status:
type: string
uploadUrl:
@@ -3064,6 +3129,8 @@ definitions:
pricingSnapshot:
additionalProperties: {}
type: object
publicError:
$ref: '#/definitions/publicerror.Error'
remoteTaskId:
type: string
request:
@@ -4125,6 +4192,8 @@ definitions:
type: string
providerModelName:
type: string
publicError:
$ref: '#/definitions/publicerror.Error'
queueKey:
type: string
requestFingerprint:
@@ -6358,7 +6427,7 @@ paths:
post:
consumes:
- application/json
description: 创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。
description: 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。
parameters:
- description: 文件存储通道
in: body
@@ -6485,6 +6554,47 @@ paths:
summary: 更新文件存储通道
tags:
- system
/api/admin/system/file-storage/channels/{channelID}/test:
post:
description: 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。
parameters:
- description: 文件存储通道 ID
in: path
name: channelID
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/httpapi.FileStorageChannelTestResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"403":
description: Forbidden
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"404":
description: Not Found
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
security:
- BearerAuth: []
summary: 测试对象存储通道
tags:
- system
/api/admin/system/file-storage/settings:
get:
description: 返回文件存储系统设置；数据库对象尚未创建时返回默认设置。
@@ -8652,7 +8762,7 @@ paths:
post:
consumes:
- multipart/form-data
description: 上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。
description: 上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。
parameters:
- description: 要上传的文件
in: formData
apps/api/internal/acceptanceemulator/server.go
+107
View File
@@ -51,6 +51,8 @@ type Server struct {
callbacks map[string]map[int64]int
videoByIdempotency map[string]string
geminiIdempotency map[string]struct{}
storageObjects map[string]fixture
storageAttempts map[string]int
pngSmall string
pngLarge string
pngPeak string
@@ -75,6 +77,11 @@ type Report struct {
DuplicateCallbacks int64 `json:"duplicateCallbacks"`
MissingIdempotency int64 `json:"missingIdempotencyKeys"`
DuplicateSubmissions int64 `json:"duplicateSubmissionAttempts"`
StoragePuts int64 `json:"storagePuts"`
StorageGets int64 `json:"storageGets"`
StorageHeads int64 `json:"storageHeads"`
StorageDeletes int64 `json:"storageDeletes"`
StorageFailures int64 `json:"storageFailures"`
}
type videoTask struct {
@@ -117,6 +124,8 @@ func New(config Config) *Server {
callbacks: map[string]map[int64]int{},
videoByIdempotency: map[string]string{},
geminiIdempotency: map[string]struct{}{},
storageObjects: map[string]fixture{},
storageAttempts: map[string]int{},
report: Report{
VideoReferenceCounts: map[string]int64{},
VideoRoleCounts: map[string]int64{},
@@ -139,9 +148,107 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("GET /media/{asset}", s.getMedia)
mux.HandleFunc("GET /fixtures/{asset}", s.getFixture)
mux.HandleFunc("POST /callbacks", s.collectCallback)
mux.HandleFunc("PUT /storage/{profile}/{object...}", s.objectStorage)
mux.HandleFunc("GET /storage/{profile}/{object...}", s.objectStorage)
mux.HandleFunc("HEAD /storage/{profile}/{object...}", s.objectStorage)
mux.HandleFunc("DELETE /storage/{profile}/{object...}", s.objectStorage)
return mux
}
func (s *Server) objectStorage(w http.ResponseWriter, r *http.Request) {
profile := strings.ToLower(strings.TrimSpace(r.PathValue("profile")))
object := strings.TrimLeft(strings.TrimSpace(r.PathValue("object")), "/")
if object == "" {
http.NotFound(w, r)
return
}
baseProfile := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(profile, "-transient"), "-auth"), "-fail")
if baseProfile != "oss" && baseProfile != "s3" {
http.NotFound(w, r)
return
}
key := profile + "/" + object
s.mu.Lock()
s.storageAttempts[key]++
attempt := s.storageAttempts[key]
if strings.HasSuffix(profile, "-auth") {
s.report.StorageFailures++
s.mu.Unlock()
http.Error(w, "forbidden", http.StatusForbidden)
return
}
if strings.HasSuffix(profile, "-fail") || (strings.HasSuffix(profile, "-transient") && attempt == 1) {
s.report.StorageFailures++
s.mu.Unlock()
http.Error(w, "temporary storage failure", http.StatusServiceUnavailable)
return
}
switch r.Method {
case http.MethodPut:
s.mu.Unlock()
payload, err := io.ReadAll(io.LimitReader(r.Body, maxProtocolBodyBytes+1))
if err != nil || len(payload) > maxProtocolBodyBytes {
http.Error(w, "invalid object", http.StatusBadRequest)
return
}
s.mu.Lock()
s.storageObjects[key] = fixture{ContentType: firstNonEmpty(r.Header.Get("Content-Type"), "application/octet-stream"), Payload: payload}
s.report.StoragePuts++
s.mu.Unlock()
w.WriteHeader(http.StatusOK)
case http.MethodGet:
item, ok := s.storageObjects[key]
if ok {
s.report.StorageGets++
}
s.mu.Unlock()
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", item.ContentType)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(item.Payload)
case http.MethodHead:
item, ok := s.storageObjects[key]
if ok {
s.report.StorageHeads++
}
s.mu.Unlock()
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", item.ContentType)
w.Header().Set("Content-Length", strconv.Itoa(len(item.Payload)))
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
_, ok := s.storageObjects[key]
delete(s.storageObjects, key)
if ok {
s.report.StorageDeletes++
}
s.mu.Unlock()
if !ok {
http.NotFound(w, r)
return
}
w.WriteHeader(http.StatusNoContent)
default:
s.mu.Unlock()
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if value = strings.TrimSpace(value); value != "" {
return value
}
}
return ""
}
func (s *Server) health(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
}
apps/api/internal/acceptanceemulator/server_test.go
+80
View File
@@ -315,6 +315,86 @@ func TestVolcesProtocolAcceptsThreeSixAndNineReferenceImages(t *testing.T) {
}
}
func TestObjectStorageEmulationSupportsRetryLifecycleAndFaults(t *testing.T) {
server := httptest.NewServer(New(Config{}).Handler())
defer server.Close()
objectURL := server.URL + "/storage/s3-transient/bucket/media/result.png"
request, _ := http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes"))
request.Header.Set("Content-Type", "image/png")
response, err := http.DefaultClient.Do(request)
if err != nil {
t.Fatal(err)
}
_ = response.Body.Close()
if response.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("first transient PUT status=%d", response.StatusCode)
}
request, _ = http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes"))
request.Header.Set("Content-Type", "image/png")
response, err = http.DefaultClient.Do(request)
if err != nil {
t.Fatal(err)
}
_ = response.Body.Close()
if response.StatusCode != http.StatusOK {
t.Fatalf("retried PUT status=%d", response.StatusCode)
}
response, err = http.Get(objectURL)
if err != nil {
t.Fatal(err)
}
payload, _ := io.ReadAll(response.Body)
_ = response.Body.Close()
if response.StatusCode != http.StatusOK || string(payload) != "image-bytes" {
t.Fatalf("GET status=%d payload=%q", response.StatusCode, payload)
}
request, _ = http.NewRequest(http.MethodHead, objectURL, nil)
response, err = http.DefaultClient.Do(request)
if err != nil {
t.Fatal(err)
}
_ = response.Body.Close()
if response.StatusCode != http.StatusOK {
t.Fatalf("HEAD status=%d", response.StatusCode)
}
request, _ = http.NewRequest(http.MethodDelete, objectURL, nil)
response, err = http.DefaultClient.Do(request)
if err != nil {
t.Fatal(err)
}
_ = response.Body.Close()
if response.StatusCode != http.StatusNoContent {
t.Fatalf("DELETE status=%d", response.StatusCode)
}
for profile, wantStatus := range map[string]int{"oss-auth": http.StatusForbidden, "s3-fail": http.StatusServiceUnavailable} {
request, _ = http.NewRequest(http.MethodPut, server.URL+"/storage/"+profile+"/probe.bin", strings.NewReader("probe"))
response, err = http.DefaultClient.Do(request)
if err != nil {
t.Fatal(err)
}
_ = response.Body.Close()
if response.StatusCode != wantStatus {
t.Fatalf("%s status=%d, want %d", profile, response.StatusCode, wantStatus)
}
}
response, err = http.Get(server.URL + "/report")
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
var report Report
if err := json.NewDecoder(response.Body).Decode(&report); err != nil {
t.Fatal(err)
}
if report.StoragePuts != 1 || report.StorageGets != 1 || report.StorageHeads != 1 || report.StorageDeletes != 1 || report.StorageFailures != 3 {
t.Fatalf("unexpected storage report: %+v", report)
}
}
func postIdempotent(url string, body []byte, key string) (*http.Response, error) {
request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
if err != nil {
apps/api/internal/httpapi/compat_protocol.go
+14 -8
View File
@@ -9,6 +9,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -97,11 +98,16 @@ func writeWireResponse(w http.ResponseWriter, wire *clients.WireResponse) {
}
func writeProtocolError(w http.ResponseWriter, protocol string, status int, message string, details map[string]any, code string) {
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
standard = publicErrorWithRetryAfter(standard, details)
publicerror.Observe(standard)
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
details = safePublicErrorDetails(details, standard, true)
switch protocol {
case clients.ProtocolGeminiGenerateContent:
writeGeminiError(w, status, message, details, code)
case clients.ProtocolVolcesContents:
writeVolcesError(w, status, message, code)
writeVolcesPublicError(w, standard)
case clients.ProtocolKlingV1Omni, clients.ProtocolKlingV2Omni:
writeKelingCompatError(w, "", newKelingCompatError(status, kelingCompatBusinessCode(code, message), message))
default:
@@ -185,11 +191,11 @@ func googleRPCStatus(status int, code string) string {
}
func writeVolcesError(w http.ResponseWriter, status int, message string, code string) {
if strings.TrimSpace(code) == "" {
code = http.StatusText(status)
}
writeJSON(w, status, map[string]any{"error": map[string]any{
"code": code,
"message": message,
}})
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
publicerror.Observe(standard)
writeVolcesPublicError(w, standard)
}
func writeVolcesPublicError(w http.ResponseWriter, standard publicerror.Error) {
writeJSON(w, standard.HTTPStatus, map[string]any{"error": standard})
}
apps/api/internal/httpapi/compat_protocol_test.go
+41 -3
View File
@@ -47,7 +47,7 @@ func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) {
}
}
func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
func TestProtocolErrorsUseCompatibleShapesWithStandardPublicErrors(t *testing.T) {
tests := []struct {
name string
protocol string
@@ -78,10 +78,10 @@ func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusBadGateway,
assertBody: func(t *testing.T, body map[string]any) {
errorBody := requireObject(t, body["error"])
if errorBody["code"] != "upstream_submission_unknown" {
if errorBody["code"] != "upstream_submission_unknown" || errorBody["httpStatus"] != float64(http.StatusBadGateway) || errorBody["retryable"] != true {
t.Fatalf("unexpected Volces error: %+v", body)
}
assertNoKeys(t, errorBody, "status", "retryable", "taskId", "gateway_status")
assertNoKeys(t, errorBody, "status", "taskId", "gateway_status")
},
},
}
@@ -135,6 +135,44 @@ func TestWireResponsePassthroughPreservesStatusUnknownFieldsAndAllowedHeaders(t
}
}
func TestCompatibilityErrorWritersNeverExposeTransportDetails(t *testing.T) {
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
tests := []struct {
name string
write func(http.ResponseWriter)
}{
{name: "openai", write: func(w http.ResponseWriter) {
writeProtocolError(w, clients.ProtocolOpenAIResponses, http.StatusOK, raw, map[string]any{
"provider": "secret-provider", "endpoint": "https://private.example.invalid", "bucket": "private-bucket",
}, "response_read_error")
}},
{name: "volces", write: func(w http.ResponseWriter) {
writeVolcesError(w, http.StatusOK, raw, "response_read_error")
}},
{name: "kling", write: func(w http.ResponseWriter) {
writeKlingCompatError(w, http.StatusOK, raw, "response_read_error")
}},
{name: "keling", write: func(w http.ResponseWriter) {
writeKelingCompatError(w, "request-1", newKelingCompatError(http.StatusOK, 5001, raw))
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
recorder := httptest.NewRecorder()
test.write(recorder)
if recorder.Code != http.StatusBadGateway {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
body := recorder.Body.String()
if strings.Contains(body, "10.42.0.72") || strings.Contains(body, "47.77.191.126") ||
strings.Contains(body, "secret-provider") || strings.Contains(body, "private.example.invalid") || strings.Contains(body, "private-bucket") ||
!strings.Contains(body, "upstream_connection_interrupted") {
t.Fatalf("transport details were not standardized: %s", body)
}
})
}
}
func TestCompatibilityStatusMappings(t *testing.T) {
for internal, want := range map[string]string{
"queued": "queued", "running": "running", "succeeded": "succeeded", "failed": "failed", "cancelled": "cancelled",
apps/api/internal/httpapi/easyai_compat.go
+22 -8
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -58,14 +59,21 @@ func writeEasyAIAsyncError(w http.ResponseWriter, status int, message string, de
if len(codes) > 0 {
code = strings.TrimSpace(codes[0])
}
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
standard = publicErrorWithRetryAfter(standard, details)
publicerror.Observe(standard)
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
errorPayload := map[string]any{
"message": message,
"status": status,
"message": message,
"status": status,
"code": code,
"category": standard.Category,
"httpStatus": standard.HTTPStatus,
"retryable": standard.Retryable,
"action": standard.Action,
"version": standard.Version,
}
if code != "" {
errorPayload["code"] = code
}
for key, value := range details {
for key, value := range safePublicErrorDetails(details, standard, false) {
errorPayload[key] = value
}
response := map[string]any{
@@ -158,8 +166,14 @@ func easyAITaskResultResponse(task store.GatewayTask) map[string]any {
if message != "" {
response["message"] = message
}
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" {
response["code"] = code
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" || status == "failed" {
standard := publicTaskError(task)
if code != "" && task.ErrorCode == "" {
standard = publicerror.WithIDs(publicerror.FromFields(code, message, 0, false), task.RequestID, task.ID)
}
response["code"] = standard.Code
response["message"] = standard.Message
response["error"] = standard
}
return response
}
apps/api/internal/httpapi/file_upload_handlers.go
+3 -6
View File
@@ -13,7 +13,7 @@ const maxGatewayUploadBytes = 256 << 20
// uploadFile godoc
// @Summary 上传文件
// @Description 上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。
// @Description 上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。
// @Tags files
// @Accept multipart/form-data
// @Produce json
@@ -55,11 +55,8 @@ func (s *Server) uploadFile(w http.ResponseWriter, r *http.Request) {
})
if err != nil {
s.logger.Error("upload file failed", "error", err)
status := http.StatusBadGateway
if clients.ErrorCode(err) == "upload_no_channel" {
status = http.StatusServiceUnavailable
}
writeError(w, status, err.Error())
status := statusFromRunError(err)
writeError(w, status, err.Error(), clients.ErrorCode(err))
return
}
writeJSON(w, http.StatusOK, easyAIFileUploadResponse(upload))
apps/api/internal/httpapi/gemini_compat.go
+8 -11
View File
@@ -17,6 +17,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -159,7 +160,7 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
if err != nil {
s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err)
status := http.StatusBadRequest
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
status = http.StatusBadGateway
}
writeGeminiTaskError(status, err.Error(), nil, clients.ErrorCode(err))
@@ -318,14 +319,13 @@ func (s *Server) writeGeminiGenerateContentStream(runCtx context.Context, w http
}
applyRunErrorHeaders(w, runErr)
if !nativePassthrough && !convertedFrames {
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, clients.ProtocolGeminiGenerateContent) {
writeWireResponse(w, wire)
return
}
writeGeminiTaskError(statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
} else if !nativePassthrough {
status := statusFromRunError(runErr)
writeGeminiSSEFrame(w, geminiErrorEnvelope(status, runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr)))
standard := publicerror.FromFields(runErrorCode(runErr), runErrorMessage(runErr), statusFromRunError(runErr), clients.IsRetryable(runErr))
standard = publicErrorWithRetryAfter(standard, runErrorDetails(runErr))
publicerror.Observe(standard)
details := safePublicErrorDetails(runErrorDetails(runErr), standard, true)
writeGeminiSSEFrame(w, geminiErrorEnvelope(standard.HTTPStatus, standard.Message, details, standard.Code))
if flusher != nil {
flusher.Flush()
}
@@ -683,10 +683,7 @@ func (s *Server) finalizeGeminiFilesUpload(w http.ResponseWriter, r *http.Reques
})
if err != nil {
s.logger.Error("Gemini files upload failed", "error", err)
status := http.StatusBadGateway
if clients.ErrorCode(err) == "upload_no_channel" {
status = http.StatusServiceUnavailable
}
status := statusFromRunError(err)
writeProtocolError(w, clients.ProtocolGeminiGenerateContent, status, err.Error(), nil, clients.ErrorCode(err))
return
}
apps/api/internal/httpapi/handlers.go
+6 -8
View File
@@ -1154,7 +1154,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind)
if err != nil {
status := http.StatusBadRequest
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
status = http.StatusBadGateway
}
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
@@ -1195,7 +1195,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
if err != nil {
s.logger.Warn("prepare task request failed", "kind", kind, "error", err)
status := http.StatusBadRequest
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
status = http.StatusBadGateway
}
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
@@ -1556,10 +1556,6 @@ func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.Response
return
}
applyRunErrorHeaders(w, runErr)
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, targetProtocol) {
writeWireResponse(w, wire)
return
}
if targetProtocol != "" {
writeProtocolError(w, targetProtocol, statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
return
@@ -1673,6 +1669,8 @@ func modelNameFromValue(value any) string {
func statusFromRunError(err error) int {
switch {
case clients.ErrorCode(err) == "storage_write_failed" || clients.ErrorCode(err) == "storage_read_failed" || clients.ErrorCode(err) == "storage_config_invalid" || clients.ErrorCode(err) == "storage_auth_failed":
return http.StatusServiceUnavailable
case clients.ErrorCode(err) == "binary_result_expired":
return http.StatusGone
case clients.ErrorCode(err) == "binary_result_corrupted" || clients.ErrorCode(err) == "result_binary_not_materialized":
@@ -1949,7 +1947,7 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) {
return
}
writeJSON(w, http.StatusOK, map[string]any{
"items": result.Items,
"items": publicTaskList(result.Items),
"total": result.Total,
"page": result.Page,
"pageSize": result.PageSize,
@@ -2034,7 +2032,7 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
writeStoredBinaryResultError(w, err)
return
}
writeJSON(w, http.StatusOK, task)
writeJSON(w, http.StatusOK, publicGatewayTask(task))
return
}
if store.IsNotFound(err) {
apps/api/internal/httpapi/keling_compat_handlers.go
+38 -20
View File
@@ -15,6 +15,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -66,17 +67,19 @@ type KelingOmniWatermarkInfo struct {
}
type KelingCompatibleEnvelope struct {
Code int `json:"code" example:"0"`
Message string `json:"message" example:"SUCCEED"`
RequestID string `json:"request_id"`
Data any `json:"data,omitempty"`
Code int `json:"code" example:"0"`
Message string `json:"message" example:"SUCCEED"`
RequestID string `json:"request_id"`
Data any `json:"data,omitempty"`
Error *publicerror.Error `json:"error,omitempty"`
}
type kelingCompatError struct {
HTTPStatus int
Code int
Message string
RequestID string
HTTPStatus int
Code int
Message string
RequestID string
PublicError *publicerror.Error
}
func (e *kelingCompatError) Error() string {
@@ -200,10 +203,6 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
}
task, createErr = s.waitForCompatibilitySubmission(r, task)
if createErr != nil {
if wire := clients.ErrorWireResponse(createErr); wireResponseMatches(wire, clients.ProtocolKlingV1Omni) {
writeWireResponse(w, wire)
return
}
writeKelingCompatError(w, requestID, kelingCompatGatewayError(createErr))
return
}
@@ -690,11 +689,13 @@ func kelingCompatTaskData(task store.GatewayTask) map[string]any {
"enabled": kelingCompatTaskWatermark(task.Request),
},
}
if message := kelingCompatTaskMessage(task); message != "" {
data["task_status_msg"] = message
}
if kelingCompatTaskStatus(task.Status) == "failed" {
data["task_status_code"] = kelingCompatBusinessCode(task.ErrorCode, kelingCompatTaskMessage(task))
standard := publicTaskError(task)
data["task_status_msg"] = standard.Message
data["task_status_code"] = kelingCompatBusinessCode(standard.Code, standard.Message)
data["error"] = standard
} else if message := kelingCompatTaskMessage(task); message != "" {
data["task_status_msg"] = message
}
if videos := kelingCompatTaskVideos(task.Result); len(videos) > 0 {
data["task_result"] = map[string]any{"videos": videos}
@@ -770,6 +771,9 @@ func firstKelingCompatValue(values ...any) any {
}
func kelingCompatTaskMessage(task store.GatewayTask) string {
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" || task.Error != "" {
return publicTaskError(task).Message
}
return strings.TrimSpace(firstNonEmpty(task.ErrorMessage, task.Error, task.Message))
}
@@ -825,8 +829,11 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
return newKelingCompatError(http.StatusInternalServerError, 5000, "unknown gateway error")
}
codeText := clients.ErrorCode(err)
businessCode := kelingCompatBusinessCode(codeText, err.Error())
status := http.StatusInternalServerError
status := statusFromRunError(err)
standard := publicerror.FromFields(codeText, err.Error(), status, clients.IsRetryable(err))
publicerror.Observe(standard)
businessCode := kelingCompatBusinessCode(standard.Code, standard.Message)
status = standard.HTTPStatus
switch businessCode {
case 1101:
status = http.StatusPaymentRequired
@@ -841,7 +848,9 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
case 5001:
status = http.StatusBadGateway
}
return newKelingCompatError(status, businessCode, err.Error())
result := newKelingCompatError(status, businessCode, standard.Message)
result.PublicError = &standard
return result
}
func kelingCompatBusinessCode(errorCode string, message string) int {
@@ -923,9 +932,18 @@ func writeKelingCompatError(w http.ResponseWriter, requestID string, err *keling
if status == 0 {
status = http.StatusInternalServerError
}
standard := err.PublicError
if standard == nil {
value := publicerror.FromFields("", err.Message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
value = publicerror.WithIDs(value, requestID, "")
publicerror.Observe(value)
standard = &value
}
status = standard.HTTPStatus
writeJSON(w, status, KelingCompatibleEnvelope{
Code: err.Code,
Message: err.Message,
Message: standard.Message,
RequestID: requestID,
Error: standard,
})
}
apps/api/internal/httpapi/kling_compat.go
+15 -14
View File
@@ -14,6 +14,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -191,10 +192,6 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
}
task, err = s.waitForCompatibilitySubmission(r, task)
if err != nil {
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, targetProtocol) {
writeWireResponse(w, wire)
return
}
writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
return
}
@@ -667,7 +664,9 @@ func klingV1TaskData(task store.GatewayTask) map[string]any {
"created_at": task.CreatedAt.UnixMilli(), "updated_at": task.UpdatedAt.UnixMilli(),
}
if task.ErrorMessage != "" || task.Error != "" {
data["task_status_msg"] = firstNonEmpty(task.ErrorMessage, task.Error)
standard := publicTaskError(task)
data["task_status_msg"] = standard.Message
data["error"] = standard
}
if watermarkInfo, ok := task.Request["watermark_info"].(map[string]any); ok {
data["watermark_info"] = watermarkInfo
@@ -698,7 +697,9 @@ func klingV2TaskData(task store.GatewayTask) map[string]any {
"external_id": task.ExternalTaskID,
}
if message := firstNonEmpty(task.ErrorMessage, task.Error); message != "" {
data["message"] = message
standard := publicTaskError(task)
data["message"] = standard.Message
data["error"] = standard
}
if outputs := klingV2Outputs(task); len(outputs) > 0 {
data["outputs"] = outputs
@@ -827,14 +828,14 @@ func decodeKlingJSON(r *http.Request, target any) error {
}
func writeKlingCompatError(w http.ResponseWriter, status int, message string, code string) {
if strings.TrimSpace(code) == "" {
code = "invalid_request"
}
writeJSON(w, status, map[string]any{
"code": klingCompatErrorCode(status),
"message": message,
"request_id": "",
"error": code,
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
publicerror.Observe(standard)
writeJSON(w, standard.HTTPStatus, map[string]any{
"code": klingCompatErrorCode(standard.HTTPStatus),
"message": standard.Message,
"request_id": "",
"error": standard.Code,
"public_error": standard,
})
}
apps/api/internal/httpapi/openapi_models.go
+28 -15
View File
@@ -2,6 +2,7 @@ package httpapi
import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -32,11 +33,19 @@ type ErrorEnvelope struct {
}
type ErrorPayload struct {
Message string `json:"message" example:"invalid json body"`
Status int `json:"status" example:"400"`
Code string `json:"code,omitempty" example:"rate_limit"`
Type string `json:"type,omitempty" example:"invalid_request_error"`
Param any `json:"param,omitempty"`
Message string `json:"message" example:"invalid json body"`
Status int `json:"status" example:"400"`
Code string `json:"code,omitempty" example:"rate_limit"`
Category string `json:"category,omitempty" example:"rate_limit"`
HTTPStatus int `json:"httpStatus,omitempty" example:"429"`
Retryable bool `json:"retryable" example:"true"`
Action string `json:"action,omitempty" example:"retry_after"`
Version string `json:"version,omitempty" example:"v1"`
RetryAfterSeconds int `json:"retryAfterSeconds,omitempty" example:"2"`
RequestID string `json:"requestId,omitempty"`
TaskID string `json:"taskId,omitempty"`
Type string `json:"type,omitempty" example:"invalid_request_error"`
Param any `json:"param,omitempty"`
}
type OpenAIErrorEnvelope struct {
@@ -62,12 +71,7 @@ type GeminiErrorStatus struct {
}
type VolcesErrorEnvelope struct {
Error VolcesErrorPayload `json:"error"`
}
type VolcesErrorPayload struct {
Code string `json:"code" example:"invalid_parameter"`
Message string `json:"message" example:"model is required"`
Error publicerror.Error `json:"error"`
}
type VolcesContentsGenerationTaskResponse struct {
@@ -82,10 +86,11 @@ type VolcesContentsGenerationTaskResponse struct {
}
type KlingErrorEnvelope struct {
Code int `json:"code" example:"1001"`
Message string `json:"message" example:"invalid parameter"`
RequestID string `json:"request_id"`
Error string `json:"error,omitempty" example:"invalid_parameter"`
Code int `json:"code" example:"1001"`
Message string `json:"message" example:"invalid parameter"`
RequestID string `json:"request_id"`
Error string `json:"error,omitempty" example:"invalid_parameter"`
PublicError *publicerror.Error `json:"public_error,omitempty"`
}
type AuthResponse struct {
@@ -232,6 +237,14 @@ type FileStorageChannelListResponse struct {
Items []store.FileStorageChannel `json:"items"`
}
type FileStorageChannelTestResponse struct {
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
DurationMS int64 `json:"durationMs"`
}
type FileUploadResponse struct {
ID string `json:"id,omitempty" example:"file_abc123"`
URL string `json:"url,omitempty" example:"/static/uploaded/upload-abc123.png"`
apps/api/internal/httpapi/public_errors.go
+120
View File
@@ -0,0 +1,120 @@
package httpapi
import (
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func publicGatewayTask(task store.GatewayTask) store.GatewayTask {
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" {
value := publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error), storedTaskErrorStatus(task.ErrorCode), false)
if task.PublicError != nil && task.PublicError.Code != "" {
value = *task.PublicError
}
value = publicerror.WithIDs(value, task.RequestID, task.ID)
publicerror.Observe(value)
task.PublicError = &value
task.ErrorCode = value.Code
task.ErrorMessage = value.Message
task.Error = value.Message
}
for index := range task.Attempts {
attempt := &task.Attempts[index]
if attempt.ErrorCode == "" && attempt.ErrorMessage == "" {
continue
}
status := attempt.StatusCode
if status <= 0 {
status = http.StatusBadGateway
}
value := publicerror.FromFields(attempt.ErrorCode, attempt.ErrorMessage, status, attempt.Retryable)
if attempt.PublicError != nil && attempt.PublicError.Code != "" {
value = *attempt.PublicError
}
value = publicerror.WithIDs(value, attempt.RequestID, task.ID)
attempt.PublicError = &value
attempt.ErrorCode = value.Code
attempt.ErrorMessage = value.Message
}
return task
}
func publicTaskError(task store.GatewayTask) publicerror.Error {
if task.PublicError != nil && task.PublicError.Code != "" {
return publicerror.WithIDs(*task.PublicError, task.RequestID, task.ID)
}
status := storedTaskErrorStatus(task.ErrorCode)
if status <= 0 {
status = http.StatusBadGateway
}
return publicerror.WithIDs(publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error, task.Message), status, false), task.RequestID, task.ID)
}
func publicTaskList(items []store.GatewayTask) []store.GatewayTask {
out := make([]store.GatewayTask, len(items))
for index, task := range items {
out[index] = publicGatewayTask(task)
}
return out
}
func publicErrorMap(value publicerror.Error) map[string]any {
out := map[string]any{
"code": value.Code, "message": value.Message, "category": value.Category,
"httpStatus": value.HTTPStatus, "retryable": value.Retryable, "action": value.Action,
"version": value.Version,
}
if value.RetryAfterSeconds > 0 {
out["retryAfterSeconds"] = value.RetryAfterSeconds
}
if requestID := strings.TrimSpace(value.RequestID); requestID != "" {
out["requestId"] = requestID
}
if taskID := strings.TrimSpace(value.TaskID); taskID != "" {
out["taskId"] = taskID
}
return out
}
func publicErrorWithRetryAfter(value publicerror.Error, details map[string]any) publicerror.Error {
if value.RetryAfterSeconds > 0 || details == nil {
return value
}
switch typed := details["retryAfterSeconds"].(type) {
case int:
value.RetryAfterSeconds = typed
case int32:
value.RetryAfterSeconds = int(typed)
case int64:
value.RetryAfterSeconds = int(typed)
case float64:
value.RetryAfterSeconds = int(typed)
}
if value.RetryAfterSeconds < 0 {
value.RetryAfterSeconds = 0
}
return value
}
func safePublicErrorDetails(details map[string]any, value publicerror.Error, includePublicError bool) map[string]any {
out := map[string]any{}
for _, key := range []string{"param", "retryAfterSeconds", "recoveryAt", "rateLimit", "pricing"} {
if item, ok := details[key]; ok && item != nil {
out[key] = item
}
}
if value.Category == "request" {
for _, key := range []string{"reason", "diagnosticId"} {
if item, ok := details[key]; ok && item != nil {
out[key] = item
}
}
}
if includePublicError {
out["publicError"] = value
}
return out
}
apps/api/internal/httpapi/request_preparation.go
+135 -72
View File
@@ -12,7 +12,6 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -38,12 +37,6 @@ type decodedRequestAsset struct {
ContentType string
}
type requestAssetOptions struct {
RequirePublicURL bool
UploadScene string
Source string
}
type requestAssetLock struct {
mu sync.Mutex
refs int
@@ -115,6 +108,19 @@ func (s *Server) prepareRequestAssetRefs(ctx context.Context, body map[string]an
}
func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path []string, siblings map[string]any) (any, error) {
if decoded, ok, err := requestAssetFromBinaryValue(requestAssetPathKey(path), path, value, siblings); err != nil {
return nil, err
} else if ok {
if err := s.acquireMediaRequestSlot(ctx); err != nil {
return nil, err
}
defer s.releaseMediaRequestSlot()
ref, err := s.ensureRequestAsset(ctx, decoded)
if err != nil {
return nil, err
}
return requestAssetWrapper(ref), nil
}
switch typed := value.(type) {
case map[string]any:
if typed["assetRef"] != nil {
@@ -156,6 +162,83 @@ func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path [
}
}
func requestAssetFromBinaryValue(key string, path []string, value any, siblings map[string]any) (decodedRequestAsset, bool, error) {
var payload []byte
contentType := ""
switch typed := value.(type) {
case []byte:
payload = append([]byte(nil), typed...)
case map[string]any:
if !strings.EqualFold(strings.TrimSpace(stringFromRequestAny(typed["type"])), "buffer") {
return decodedRequestAsset{}, false, nil
}
contentType = firstNonEmptyRequestString(typed, "contentType", "mimeType", "mime_type")
switch data := typed["data"].(type) {
case []byte:
payload = append([]byte(nil), data...)
case []any:
var ok bool
payload, ok = requestBytesFromNumberArray(data)
if !ok {
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data must contain byte values"))
}
default:
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data is required"))
}
case []any:
if !strictRequestBinaryArrayField(key, path) {
return decodedRequestAsset{}, false, nil
}
var ok bool
payload, ok = requestBytesFromNumberArray(typed)
if !ok {
return decodedRequestAsset{}, false, nil
}
default:
return decodedRequestAsset{}, false, nil
}
if len(payload) == 0 {
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("binary media payload is empty"))
}
return decodedRequestAsset{
Bytes: payload,
ContentType: requestAssetContentType(contentType, payload, key, path, siblings),
}, true, nil
}
func requestAssetPathKey(path []string) string {
for index := len(path) - 1; index >= 0; index-- {
value := strings.TrimSpace(path[index])
if value != "" && !strings.HasPrefix(value, "[") {
return value
}
}
return ""
}
func strictRequestBinaryArrayField(key string, path []string) bool {
value := strings.ToLower(strings.TrimSpace(key))
if value == "bytes" || value == "buffer" || strings.Contains(value, "binary") || strings.Contains(value, "buffer") {
return true
}
return requestMediaKind(key, path, nil) != "" && (value == "data" || value == "content")
}
func requestBytesFromNumberArray(values []any) ([]byte, bool) {
if len(values) == 0 {
return nil, false
}
payload := make([]byte, len(values))
for index, value := range values {
number, ok := value.(float64)
if !ok || number < 0 || number > 255 || number != float64(byte(number)) {
return nil, false
}
payload[index] = byte(number)
}
return payload, true
}
func (s *Server) prepareRequestAssetField(ctx context.Context, key string, path []string, value any, siblings map[string]any) (map[string]any, bool, error) {
text, ok := value.(string)
if !ok {
@@ -291,68 +374,43 @@ func requestAssetFromValue(key string, path []string, value any, siblings map[st
}
func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
UploadScene: store.FileStorageSceneRequestAsset,
Source: "ai-gateway-request",
})
}
func (s *Server) ensurePublicRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
RequirePublicURL: true,
UploadScene: store.FileStorageSceneUpload,
Source: "ai-gateway-form-data",
})
}
func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded decodedRequestAsset, options requestAssetOptions) (map[string]any, error) {
sum := sha256.Sum256(decoded.Bytes)
sha := hex.EncodeToString(sum[:])
contentType := strings.TrimSpace(decoded.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
}
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + options.UploadScene + "\x00" + strconv.FormatBool(options.RequirePublicURL))
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + store.FileStorageSceneRequestAsset)
defer release()
now := time.Now()
if existing, ok, err := s.store.FindRequestAsset(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
} else if ok && requestAssetStillUsable(existing, now) {
ref := requestAssetRef(existing)
if !options.RequirePublicURL || requestAssetRefHasPublicURL(ref) {
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
}
return ref, nil
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
}
return ref, nil
}
uploadScene := strings.TrimSpace(options.UploadScene)
if uploadScene == "" {
uploadScene = store.FileStorageSceneRequestAsset
}
source := strings.TrimSpace(options.Source)
if source == "" {
source = "ai-gateway-request"
}
upload, err := s.runner.UploadFile(ctx, runner.FileUploadPayload{
Bytes: decoded.Bytes,
ContentType: contentType,
FileName: requestAssetFileName(sha, contentType),
Scene: uploadScene,
Source: source,
Scene: store.FileStorageSceneRequestAsset,
Source: "ai-gateway-request",
})
if err != nil {
return nil, err
}
storageProvider := requestAssetStorageProvider(upload)
storageChannelID, storageChannelKey := requestAssetStorageChannel(upload)
objectKey := stringFromRequestAny(upload["objectKey"])
accessScope := firstNonEmpty(stringFromRequestAny(upload["accessScope"]), "private")
url := stringFromRequestAny(upload["url"])
if url == "" {
return nil, &clients.ClientError{Code: "request_asset_upload_failed", Message: "file storage response did not include url", Retryable: false}
}
if options.RequirePublicURL && !requestAssetURLIsPublic(storageProvider, url) {
return nil, &clients.ClientError{Code: "request_asset_public_url_required", Message: "multipart image assets require a public file storage URL; enable a non-local file storage channel for uploads", Retryable: false}
}
var expiresAt *time.Time
localPath := ""
if storageProvider == "local_static" {
@@ -361,23 +419,31 @@ func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded deco
localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
}
asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
SHA256: sha,
ContentType: contentType,
ByteSize: int64(len(decoded.Bytes)),
URL: url,
StorageProvider: storageProvider,
LocalPath: localPath,
ExpiresAt: expiresAt,
SHA256: sha,
ContentType: contentType,
ByteSize: int64(len(decoded.Bytes)),
URL: url,
StorageProvider: storageProvider,
StorageChannelID: storageChannelID,
StorageChannelKey: storageChannelKey,
ObjectKey: objectKey,
AccessScope: accessScope,
LocalPath: localPath,
ExpiresAt: expiresAt,
})
if err != nil {
if store.IsUndefinedDatabaseObject(err) {
return map[string]any{
"sha256": sha,
"url": url,
"contentType": contentType,
"size": len(decoded.Bytes),
"storageProvider": storageProvider,
"expiresAt": timePtrToRFC3339(expiresAt),
"sha256": sha,
"url": url,
"contentType": contentType,
"size": len(decoded.Bytes),
"storageProvider": storageProvider,
"storageChannelId": storageChannelID,
"storageChannelKey": storageChannelKey,
"objectKey": objectKey,
"accessScope": accessScope,
"expiresAt": timePtrToRFC3339(expiresAt),
}, nil
}
return nil, err
@@ -493,27 +559,19 @@ func requestAssetWrapper(ref map[string]any) map[string]any {
func requestAssetRef(asset store.RequestAsset) map[string]any {
return map[string]any{
"sha256": asset.SHA256,
"url": asset.URL,
"contentType": asset.ContentType,
"size": asset.ByteSize,
"storageProvider": asset.StorageProvider,
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
"sha256": asset.SHA256,
"url": asset.URL,
"contentType": asset.ContentType,
"size": asset.ByteSize,
"storageProvider": asset.StorageProvider,
"storageChannelId": asset.StorageChannelID,
"storageChannelKey": asset.StorageChannelKey,
"objectKey": asset.ObjectKey,
"accessScope": asset.AccessScope,
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
}
}
func requestAssetRefHasPublicURL(ref map[string]any) bool {
return requestAssetURLIsPublic(stringFromRequestAny(ref["storageProvider"]), stringFromRequestAny(ref["url"]))
}
func requestAssetURLIsPublic(storageProvider string, url string) bool {
if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") {
return false
}
lower := strings.ToLower(strings.TrimSpace(url))
return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
}
func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
if asset.ExpiredAt != nil {
return false
@@ -543,6 +601,11 @@ func requestAssetStorageProvider(upload map[string]any) string {
return "unknown"
}
func requestAssetStorageChannel(upload map[string]any) (string, string) {
channel, _ := upload["storageChannel"].(map[string]any)
return stringFromRequestAny(channel["id"]), stringFromRequestAny(channel["channelKey"])
}
func requestAssetLocalPath(storageDir string, fileName string) string {
if strings.TrimSpace(storageDir) == "" {
storageDir = config.DefaultLocalUploadedStorageDir
apps/api/internal/httpapi/request_preparation_test.go
+27
View File
@@ -41,6 +41,33 @@ func TestRequestAssetFromValueDetectsDataURLAndRawBase64(t *testing.T) {
}
}
func TestRequestAssetFromBinaryValueDetectsBufferAndByteArray(t *testing.T) {
png := []any{float64(0x89), float64('P'), float64('N'), float64('G')}
decoded, ok, err := requestAssetFromBinaryValue("image", []string{"input", "image"}, map[string]any{
"type": "Buffer",
"data": png,
"mimeType": "image/png",
}, nil)
if err != nil {
t.Fatal(err)
}
if !ok || decoded.ContentType != "image/png" || len(decoded.Bytes) != 4 || decoded.Bytes[0] != 0x89 {
t.Fatalf("unexpected Buffer asset: ok=%v decoded=%+v", ok, decoded)
}
decoded, ok, err = requestAssetFromBinaryValue("bytes", []string{"input_audio", "bytes"}, png, map[string]any{"format": "mp3"})
if err != nil {
t.Fatal(err)
}
if !ok || len(decoded.Bytes) != 4 {
t.Fatalf("unexpected byte array asset: ok=%v decoded=%+v", ok, decoded)
}
if _, ok, err := requestAssetFromBinaryValue("values", []string{"embedding", "values"}, png, nil); err != nil || ok {
t.Fatalf("ordinary numeric arrays must remain JSON: ok=%v err=%v", ok, err)
}
}
func TestMediaRequestBodySlotLimitsPreAuthWorkAndCanReleaseEarly(t *testing.T) {
server := &Server{mediaRequestBodySlots: make(chan struct{}, 2)}
var critical atomic.Int64
apps/api/internal/httpapi/response.go
+24 -13
View File
@@ -5,6 +5,8 @@ import (
"fmt"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
)
func writeJSON(w http.ResponseWriter, status int, value any) {
@@ -18,22 +20,31 @@ func writeError(w http.ResponseWriter, status int, message string, codes ...stri
}
func writeErrorWithDetails(w http.ResponseWriter, status int, message string, details map[string]any, codes ...string) {
errorPayload := map[string]any{
"message": message,
"status": status,
}
code := ""
if len(codes) > 0 {
if code := strings.TrimSpace(codes[0]); code != "" {
errorPayload["code"] = code
if code == "invalid_parameter" || code == "unsupported_response_parameter" {
errorPayload["type"] = "invalid_request_error"
if _, ok := details["param"]; !ok {
errorPayload["param"] = nil
}
}
code = strings.TrimSpace(codes[0])
}
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
standard = publicErrorWithRetryAfter(standard, details)
publicerror.Observe(standard)
status = standard.HTTPStatus
errorPayload := map[string]any{
"message": standard.Message,
"status": status,
"code": standard.Code,
"category": standard.Category,
"httpStatus": standard.HTTPStatus,
"retryable": standard.Retryable,
"action": standard.Action,
"version": standard.Version,
}
if standard.Code == "invalid_parameter" || standard.Code == "unsupported_response_parameter" {
errorPayload["type"] = "invalid_request_error"
if _, ok := details["param"]; !ok {
errorPayload["param"] = nil
}
}
for key, value := range details {
for key, value := range safePublicErrorDetails(details, standard, false) {
errorPayload[key] = value
}
writeJSON(w, status, map[string]any{"error": errorPayload})
apps/api/internal/httpapi/server.go
+1
View File
@@ -286,6 +286,7 @@ func NewServerWithStores(
mux.Handle("GET /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listFileStorageChannels)))
mux.Handle("POST /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createFileStorageChannel)))
mux.Handle("PATCH /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageChannel)))
mux.Handle("POST /api/admin/system/file-storage/channels/{channelID}/test", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.testFileStorageChannel)))
mux.Handle("DELETE /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteFileStorageChannel)))
mux.Handle("GET /api/admin/platforms", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listPlatforms)))
mux.Handle("POST /api/admin/platforms", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createPlatform)))
apps/api/internal/httpapi/system_settings_handlers.go
+129 -1
View File
@@ -1,10 +1,13 @@
package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/url"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -74,6 +77,10 @@ func (s *Server) updateFileStorageSettings(w http.ResponseWriter, r *http.Reques
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
if legacyLocalStoragePolicy(input.ResultUploadPolicy) {
writeError(w, http.StatusBadRequest, "upload_none is no longer supported; configure an object storage channel")
return
}
settings, err := s.store.UpdateFileStorageSettings(r.Context(), input)
if err != nil {
s.logger.Error("update file storage settings failed", "error", err)
@@ -155,7 +162,7 @@ func (s *Server) updateClientCustomizationSettings(w http.ResponseWriter, r *htt
// createFileStorageChannel godoc
// @Summary 创建文件存储通道
// @Description 创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。
// @Description 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。
// @Tags system
// @Accept json
// @Produce json
@@ -271,6 +278,46 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request
w.WriteHeader(http.StatusNoContent)
}
// testFileStorageChannel godoc
// @Summary 测试对象存储通道
// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。
// @Tags system
// @Produce json
// @Security BearerAuth
// @Param channelID path string true "文件存储通道 ID"
// @Success 200 {object} FileStorageChannelTestResponse
// @Failure 400 {object} ErrorEnvelope
// @Failure 401 {object} ErrorEnvelope
// @Failure 403 {object} ErrorEnvelope
// @Failure 404 {object} ErrorEnvelope
// @Failure 503 {object} ErrorEnvelope
// @Router /api/admin/system/file-storage/channels/{channelID}/test [post]
func (s *Server) testFileStorageChannel(w http.ResponseWriter, r *http.Request) {
channel, err := s.store.GetFileStorageChannel(r.Context(), r.PathValue("channelID"))
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusNotFound, "file storage channel not found")
return
}
s.logger.Error("get file storage channel for test failed", "error", err)
writeError(w, http.StatusInternalServerError, "get file storage channel failed")
return
}
if channel.Provider != "aliyun_oss" && channel.Provider != "s3" {
writeError(w, http.StatusBadRequest, "connection test is supported for aliyun_oss and s3 channels", "invalid_parameter")
return
}
result, err := s.runner.TestFileStorageChannel(r.Context(), channel)
if err != nil {
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(r.Context()), channel.ID, err.Error())
s.logger.Warn("file storage channel test failed", "channel_id", channel.ID, "provider", channel.Provider, "error", err)
writeError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
return
}
_ = s.store.MarkFileStorageChannelSuccess(context.WithoutCancel(r.Context()), channel.ID)
writeJSON(w, http.StatusOK, result)
}
func validateFileStorageChannelInput(input store.FileStorageChannelInput, existing *store.FileStorageChannel) string {
provider := strings.ToLower(strings.TrimSpace(input.Provider))
if provider == "" {
@@ -286,6 +333,9 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
if status != "enabled" && status != "disabled" {
return "status must be enabled or disabled"
}
if provider != "server_main_openapi" && provider != "aliyun_oss" && provider != "s3" {
return "provider must be server_main_openapi, aliyun_oss or s3"
}
if provider == "server_main_openapi" {
hasAPIKey := false
if input.APIKey != nil {
@@ -297,5 +347,83 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
return "server-main OpenAPI channel requires API key before enabling"
}
}
if provider == "aliyun_oss" || provider == "s3" {
if fileStorageConfigContainsCredential(input.Config) {
return "object storage credentials must use the write-only credential fields, not config"
}
endpoint := fileStorageConfigString(input.Config, "endpoint")
if endpoint == "" || fileStorageConfigString(input.Config, "region") == "" || fileStorageConfigString(input.Config, "bucket") == "" {
return "object storage channel requires config.endpoint, config.region and config.bucket"
}
if !validFileStorageBaseURL(endpoint) {
return "object storage config.endpoint must be an http or https URL without embedded credentials"
}
if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
}
accessKeyID := input.AccessKeyID
if accessKeyID == nil {
accessKeyID = input.AccessKey
}
accessKeySecret := input.AccessKeySecret
if accessKeySecret == nil {
accessKeySecret = input.SecretKey
}
hasAccessKeyID := fileStorageCredentialPresent(accessKeyID, existing, func(item *store.FileStorageChannel) string { return item.AccessKeyID })
hasAccessKeySecret := fileStorageCredentialPresent(accessKeySecret, existing, func(item *store.FileStorageChannel) string { return item.AccessKeySecret })
if status == "enabled" && (!hasAccessKeyID || !hasAccessKeySecret) {
return "object storage channel requires accessKeyId and accessKeySecret before enabling"
}
}
return ""
}
func validFileStorageBaseURL(value string) bool {
parsed, err := url.Parse(strings.TrimSpace(value))
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
}
func fileStorageConfigContainsCredential(value any) bool {
switch typed := value.(type) {
case map[string]any:
for key, item := range typed {
normalized := strings.NewReplacer("_", "", "-", "", ".", "").Replace(strings.ToLower(strings.TrimSpace(key)))
switch normalized {
case "apikey", "accesskey", "accesskeyid", "accesskeysecret", "secretkey", "sessiontoken", "ststoken", "password", "credential", "credentials", "authorization":
return true
}
if fileStorageConfigContainsCredential(item) {
return true
}
}
case []any:
for _, item := range typed {
if fileStorageConfigContainsCredential(item) {
return true
}
}
}
return false
}
func fileStorageConfigString(config map[string]any, key string) string {
value, _ := config[key].(string)
return strings.TrimSpace(value)
}
func fileStorageCredentialPresent(input *string, existing *store.FileStorageChannel, current func(*store.FileStorageChannel) string) bool {
if input != nil {
return strings.TrimSpace(*input) != ""
}
return existing != nil && strings.TrimSpace(current(existing)) != ""
}
func legacyLocalStoragePolicy(value string) bool {
normalized := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(value)), "-", "_")
switch normalized {
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
return true
default:
return false
}
}
apps/api/internal/httpapi/system_settings_handlers_test.go
+55
View File
@@ -0,0 +1,55 @@
package httpapi
import (
"strings"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestValidateObjectStorageChannelAcceptsS3CredentialAliases(t *testing.T) {
accessKey := "access"
secretKey := "secret"
input := store.FileStorageChannelInput{
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
AccessKey: &accessKey, SecretKey: &secretKey,
Config: map[string]any{
"endpoint": "https://s3.example.com", "region": "us-east-1", "bucket": "media",
},
}
if message := validateFileStorageChannelInput(input, nil); message != "" {
t.Fatalf("valid S3 channel rejected: %s", message)
}
}
func TestValidateObjectStorageChannelRejectsCredentialsInConfig(t *testing.T) {
accessKeyID := "access"
accessKeySecret := "secret"
input := store.FileStorageChannelInput{
ChannelKey: "oss-primary", Name: "OSS primary", Provider: "aliyun_oss", Status: "enabled",
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
Config: map[string]any{
"endpoint": "https://oss-cn-hangzhou.aliyuncs.com", "region": "cn-hangzhou", "bucket": "media",
"nested": map[string]any{"session_token": "must-not-be-public"},
},
}
message := validateFileStorageChannelInput(input, nil)
if !strings.Contains(message, "write-only credential fields") {
t.Fatalf("credential-bearing config was accepted: %q", message)
}
}
func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) {
accessKeyID := "access"
accessKeySecret := "secret"
input := store.FileStorageChannelInput{
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
Config: map[string]any{
"endpoint": "https://user:password@s3.example.com", "region": "us-east-1", "bucket": "media",
},
}
if message := validateFileStorageChannelInput(input, nil); !strings.Contains(message, "without embedded credentials") {
t.Fatalf("credentialed endpoint was accepted: %q", message)
}
}
apps/api/internal/httpapi/task_idempotency.go
+4
View File
@@ -69,6 +69,10 @@ func writeIdempotentTaskReplay(w http.ResponseWriter, task store.GatewayTask, co
func storedTaskErrorStatus(code string) int {
switch strings.TrimSpace(code) {
case "storage_write_failed", "storage_read_failed", "storage_config_invalid", "storage_auth_failed":
return http.StatusServiceUnavailable
case "binary_result_expired", "result_expired", "result_unavailable":
return http.StatusGone
case "pricing_unavailable", "response_chain_unavailable", "billing_hold":
return http.StatusServiceUnavailable
case "insufficient_balance":
apps/api/internal/httpapi/task_multipart.go
+1 -1
View File
@@ -255,7 +255,7 @@ func (s *Server) uploadImageEditMultipartAsset(ctx context.Context, field string
if !strings.HasPrefix(strings.ToLower(contentType), "image/") {
return nil, &clients.ClientError{Code: "invalid_multipart_image", Message: "image edit multipart files must be images", Retryable: false}
}
ref, err := s.ensurePublicRequestAsset(ctx, decodedRequestAsset{
ref, err := s.ensureRequestAsset(ctx, decodedRequestAsset{
Bytes: payload,
ContentType: contentType,
})
apps/api/internal/httpapi/volces_compat_handlers.go
+1 -5
View File
@@ -352,7 +352,7 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
response["usage"] = legacyUsage
}
if task.Status == "failed" || task.Status == "cancelled" {
response["error"] = map[string]any{"code": firstNonEmpty(task.ErrorCode, strings.ToUpper(task.Status)), "message": firstNonEmpty(task.ErrorMessage, task.Error, task.Message)}
response["error"] = publicErrorMap(publicTaskError(task))
}
return response
}
@@ -431,10 +431,6 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
} else if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
status = http.StatusBadRequest
}
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, clients.ProtocolVolcesContents) {
writeWireResponse(w, wire)
return
}
writeVolcesError(w, status, err.Error(), clients.ErrorCode(err))
}
apps/api/internal/publicerror/metrics.go
+73
View File
@@ -0,0 +1,73 @@
package publicerror
import "sync/atomic"
type MetricCount struct {
Code string
Count uint64
}
var observed = struct {
upstreamConnectionInterrupted atomic.Uint64
upstreamTimeout atomic.Uint64
upstreamRateLimited atomic.Uint64
upstreamUnavailable atomic.Uint64
upstreamInvalidResponse atomic.Uint64
upstreamAuthFailed atomic.Uint64
upstreamRequestRejected atomic.Uint64
storageWriteFailed atomic.Uint64
storageReadFailed atomic.Uint64
resultExpired atomic.Uint64
resultUnavailable atomic.Uint64
invalidParameter atomic.Uint64
gatewayError atomic.Uint64
}{}
func Observe(value Error) {
switch value.Code {
case "upstream_connection_interrupted":
observed.upstreamConnectionInterrupted.Add(1)
case "upstream_timeout":
observed.upstreamTimeout.Add(1)
case "upstream_rate_limited":
observed.upstreamRateLimited.Add(1)
case "upstream_unavailable":
observed.upstreamUnavailable.Add(1)
case "upstream_invalid_response":
observed.upstreamInvalidResponse.Add(1)
case "upstream_auth_failed":
observed.upstreamAuthFailed.Add(1)
case "upstream_request_rejected":
observed.upstreamRequestRejected.Add(1)
case "storage_write_failed":
observed.storageWriteFailed.Add(1)
case "storage_read_failed":
observed.storageReadFailed.Add(1)
case "result_expired":
observed.resultExpired.Add(1)
case "result_unavailable":
observed.resultUnavailable.Add(1)
case "invalid_parameter":
observed.invalidParameter.Add(1)
default:
observed.gatewayError.Add(1)
}
}
func MetricSnapshot() []MetricCount {
return []MetricCount{
{Code: "upstream_connection_interrupted", Count: observed.upstreamConnectionInterrupted.Load()},
{Code: "upstream_timeout", Count: observed.upstreamTimeout.Load()},
{Code: "upstream_rate_limited", Count: observed.upstreamRateLimited.Load()},
{Code: "upstream_unavailable", Count: observed.upstreamUnavailable.Load()},
{Code: "upstream_invalid_response", Count: observed.upstreamInvalidResponse.Load()},
{Code: "upstream_auth_failed", Count: observed.upstreamAuthFailed.Load()},
{Code: "upstream_request_rejected", Count: observed.upstreamRequestRejected.Load()},
{Code: "storage_write_failed", Count: observed.storageWriteFailed.Load()},
{Code: "storage_read_failed", Count: observed.storageReadFailed.Load()},
{Code: "result_expired", Count: observed.resultExpired.Load()},
{Code: "result_unavailable", Count: observed.resultUnavailable.Load()},
{Code: "invalid_parameter", Count: observed.invalidParameter.Load()},
{Code: "gateway_error", Count: observed.gatewayError.Load()},
}
}
apps/api/internal/publicerror/public_error.go
+200
View File
@@ -0,0 +1,200 @@
package publicerror
import (
"net/http"
"strings"
)
const Version = "v1"
type Error struct {
Code string `json:"code"`
Message string `json:"message"`
Category string `json:"category"`
HTTPStatus int `json:"httpStatus"`
Retryable bool `json:"retryable"`
Action string `json:"action"`
RetryAfterSeconds int `json:"retryAfterSeconds,omitempty"`
RequestID string `json:"requestId,omitempty"`
TaskID string `json:"taskId,omitempty"`
Version string `json:"version"`
}
func FromFields(code string, message string, status int, retryable bool) Error {
originalCode := strings.TrimSpace(code)
code = strings.ToLower(originalCode)
message = strings.TrimSpace(message)
if status <= 0 {
status = statusFromCode(code)
}
if strings.HasPrefix(originalCode, "GATEWAY_") {
return newError(originalCode, safeRequestMessage(message), "gateway", status, retryable, retryAction(retryable))
}
if mapped, ok := mappedError(code, message, status, retryable); ok {
return mapped
}
if sensitiveTransportMessage(message) {
return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry")
}
if code == "" {
code = defaultCodeForStatus(status)
} else {
code = originalCode
}
if message == "" {
message = defaultMessageForStatus(status)
}
category := "gateway"
action := "none"
if status == http.StatusTooManyRequests {
category, action, retryable = "rate_limit", "retry_after", true
} else if status >= 500 {
category = "upstream"
if retryable {
action = "retry"
}
} else if status >= 400 {
category, action = "request", "fix_request"
}
return newError(code, message, category, status, retryable, action)
}
func WithIDs(value Error, requestID string, taskID string) Error {
value.RequestID = strings.TrimSpace(requestID)
value.TaskID = strings.TrimSpace(taskID)
return value
}
func mappedError(code string, message string, status int, retryable bool) (Error, bool) {
switch code {
case "bad_request", "invalid_request", "invalid_parameter", "unsupported_kind", "unsupported_operation", "unsupported_response_parameter":
return newError(code, safeRequestMessage(message), "request", http.StatusBadRequest, false, "fix_request"), true
case "cancelled", "task_cancelled":
return newError(code, safeRequestMessage(message), "request", http.StatusConflict, false, "none"), true
case "validation_in_progress":
return newError(code, firstNonEmptyMessage(message, "New production tasks are paused while validation is running."), "gateway", http.StatusServiceUnavailable, true, "retry_after"), true
case "traffic_gate_unavailable":
return newError(code, "The gateway traffic admission service is temporarily unavailable.", "gateway", http.StatusServiceUnavailable, true, "retry"), true
case "response_read_error", "stream_read_error", "network", "connection_reset", "upstream_connection_interrupted", "unexpected_eof", "http2_stream_closed", "request_asset_fetch_failed":
return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry"), true
case "upstream_timeout", "timeout", "context_deadline_exceeded":
return newError("upstream_timeout", "The upstream service did not respond in time.", "upstream", http.StatusGatewayTimeout, true, "retry"), true
case "rate_limit", "upstream_rate_limited", "too_many_requests":
return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", http.StatusTooManyRequests, true, "retry_after"), true
case "upstream_unavailable", "upstream_overloaded", "service_unavailable", "bad_gateway", "server_error", "provider_failed":
if status > 0 && status < 500 {
break
}
return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true
case "upload_invalid_response", "invalid_response", "response_too_large", "invalid_upstream_response", "upstream_invalid_response":
return newError("upstream_invalid_response", "The upstream service returned an invalid or incomplete response.", "upstream", http.StatusBadGateway, true, "retry"), true
case "invalid_api_key", "authentication_error", "auth_failed", "missing_credentials", "upstream_auth_failed":
return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true
case "storage_write_failed", "storage_config_invalid", "storage_auth_failed", "upload_config_failed", "upload_no_channel", "upload_network", "upload_read_failed", "upload_failed", "local_result_storage_unavailable", "local_static_store_failed", "request_asset_upload_failed":
return newError("storage_write_failed", "The media asset could not be written to object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true
case "storage_read_failed", "upload_source_fetch_failed", "upload_source_read_failed":
return newError("storage_read_failed", "The media asset could not be read from object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true
case "binary_result_expired", "result_expired":
return newError("result_expired", "The generated result has expired and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true
case "result_unavailable", "request_asset_expired":
return newError("result_unavailable", "The generated result is no longer available and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true
case "binary_result_corrupted", "result_binary_not_materialized":
return newError("result_corrupted", "The stored result failed an integrity check.", "storage", http.StatusInternalServerError, false, "contact_support"), true
}
if status == http.StatusTooManyRequests {
return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", status, true, "retry_after"), true
}
if (status == http.StatusUnauthorized || status == http.StatusForbidden) && (strings.HasPrefix(code, "http_") || strings.Contains(code, "provider")) {
return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true
}
if strings.HasPrefix(code, "http_") || (code == "provider_failed" && status > 0 && status < 500) {
return newError("upstream_request_rejected", "The upstream service rejected the request.", "upstream", http.StatusBadRequest, false, "fix_request"), true
}
if status >= 500 && code != "upstream_submission_unknown" && (strings.Contains(code, "upstream") || strings.Contains(code, "provider")) {
return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true
}
if status >= 500 && code != "upstream_submission_unknown" {
return newError("gateway_error", defaultMessageForStatus(status), "gateway", status, retryable, retryAction(retryable)), true
}
return Error{}, false
}
func safeRequestMessage(message string) string {
message = strings.TrimSpace(message)
if message == "" || sensitiveTransportMessage(message) {
return "The request parameters are invalid."
}
return message
}
func firstNonEmptyMessage(message string, fallback string) string {
if message = strings.TrimSpace(message); message != "" && !sensitiveTransportMessage(message) {
return message
}
return fallback
}
func retryAction(retryable bool) string {
if retryable {
return "retry"
}
return "contact_support"
}
func newError(code string, message string, category string, status int, retryable bool, action string) Error {
return Error{Code: code, Message: message, Category: category, HTTPStatus: status, Retryable: retryable, Action: action, Version: Version}
}
func sensitiveTransportMessage(message string) bool {
value := strings.ToLower(message)
for _, marker := range []string{"read tcp ", "write tcp ", "dial tcp ", "connection reset by peer", "unexpected eof", "http2:", "stream error", "context deadline exceeded"} {
if strings.Contains(value, marker) {
return true
}
}
return false
}
func statusFromCode(code string) int {
switch code {
case "rate_limit", "upstream_rate_limited", "too_many_requests":
return http.StatusTooManyRequests
case "upstream_timeout", "timeout", "context_deadline_exceeded":
return http.StatusGatewayTimeout
case "binary_result_expired", "result_expired", "result_unavailable", "request_asset_expired":
return http.StatusGone
case "storage_write_failed", "storage_read_failed", "local_result_storage_unavailable":
return http.StatusServiceUnavailable
default:
return http.StatusBadGateway
}
}
func defaultCodeForStatus(status int) string {
switch status {
case http.StatusBadRequest:
return "invalid_request"
case http.StatusUnauthorized:
return "unauthorized"
case http.StatusForbidden:
return "forbidden"
case http.StatusNotFound:
return "not_found"
case http.StatusConflict:
return "conflict"
case http.StatusTooManyRequests:
return "rate_limited"
default:
if status >= 500 {
return "gateway_error"
}
return "request_failed"
}
}
func defaultMessageForStatus(status int) string {
if value := http.StatusText(status); value != "" {
return value
}
return "Request failed."
}
apps/api/internal/publicerror/public_error_test.go
+69
View File
@@ -0,0 +1,69 @@
package publicerror
import (
"net/http"
"strings"
"testing"
)
func TestTransportErrorsNeverExposeSocketDetails(t *testing.T) {
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
got := FromFields("response_read_error", raw, http.StatusOK, true)
if got.Code != "upstream_connection_interrupted" || got.HTTPStatus != http.StatusBadGateway || !got.Retryable {
t.Fatalf("unexpected public error: %+v", got)
}
if strings.Contains(got.Message, "10.42.0.72") || strings.Contains(got.Message, "47.77.191.126") {
t.Fatalf("public error leaked socket details: %+v", got)
}
}
func TestStorageAndExpiredErrorsHaveStableActions(t *testing.T) {
storage := FromFields("storage_write_failed", "secret endpoint", 0, true)
if storage.Category != "storage" || storage.Action != "retry" || storage.HTTPStatus != http.StatusServiceUnavailable {
t.Fatalf("unexpected storage error: %+v", storage)
}
expired := FromFields("binary_result_expired", "local binary result has expired", 0, false)
if expired.Code != "result_expired" || expired.Action != "resubmit" || expired.HTTPStatus != http.StatusGone {
t.Fatalf("unexpected expired error: %+v", expired)
}
}
func TestUnknownServerErrorDoesNotExposeProviderBody(t *testing.T) {
raw := `provider rejected request: {"account":"secret-project","detail":"internal"}`
got := FromFields("vendor_opaque_error", raw, http.StatusBadGateway, true)
if got.Code != "gateway_error" || got.Message == raw || strings.Contains(got.Message, "secret-project") {
t.Fatalf("unknown provider error was exposed: %+v", got)
}
}
func TestProviderHTTPErrorDoesNotExposeProviderBody(t *testing.T) {
raw := `{"error":{"message":"bucket private-a rejected secret-project"}}`
for _, test := range []struct {
code string
status int
wantCode string
wantStatus int
}{
{code: "http_400", status: http.StatusBadRequest, wantCode: "upstream_request_rejected", wantStatus: http.StatusBadRequest},
{code: "auth_failed", status: http.StatusUnauthorized, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway},
{code: "provider_failed", status: http.StatusForbidden, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway},
{code: "provider_failed", status: http.StatusTooManyRequests, wantCode: "upstream_rate_limited", wantStatus: http.StatusTooManyRequests},
{code: "server_error", status: http.StatusBadGateway, wantCode: "upstream_unavailable", wantStatus: http.StatusServiceUnavailable},
{code: "invalid_response", status: http.StatusOK, wantCode: "upstream_invalid_response", wantStatus: http.StatusBadGateway},
} {
got := FromFields(test.code, raw, test.status, true)
if got.Code != test.wantCode || got.HTTPStatus != test.wantStatus {
t.Fatalf("%s: unexpected public error: %+v", test.code, got)
}
if strings.Contains(got.Message, "private-a") || strings.Contains(got.Message, "secret-project") {
t.Fatalf("%s: provider body leaked: %+v", test.code, got)
}
}
}
func TestValidationGateKeepsStablePublicCode(t *testing.T) {
got := FromFields("validation_in_progress", "new production tasks are paused while validation is running", http.StatusServiceUnavailable, true)
if got.Code != "validation_in_progress" || got.HTTPStatus != http.StatusServiceUnavailable || !got.Retryable {
t.Fatalf("unexpected validation gate error: %+v", got)
}
}
apps/api/internal/runner/binary_results.go
+78 -138
View File
@@ -12,7 +12,6 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
@@ -26,7 +25,6 @@ const (
localBinaryGenericBase64MinLength = 4096
localBinaryMaxDepth = 64
defaultLocalResultTTLHours = 24
defaultLocalResultMinFreeBytes = int64(10 * 1024 * 1024 * 1024)
defaultLocalResultMaxBytes = int64(256 * 1024 * 1024)
defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024)
)
@@ -41,41 +39,38 @@ type localBinaryDescriptor struct {
type localBinaryMaterializer struct {
service *Service
taskDir string
writeFiles bool
enforceLimits bool
totalBytes int64
seen map[string]struct{}
createdFiles []string
}
// materializeLocalBinaryResult replaces every inline binary value with a
// bounded placeholder after atomically writing and verifying the bytes locally.
// materializeLocalBinaryResult is retained as an internal compatibility seam,
// but new materialization always targets object storage. Historical local
// placeholders remain readable through HydrateTaskResult.
func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
if _, _, err := s.transformLocalBinaryResult(ctx, taskID, result, false, true); err != nil {
hadInline := TaskResultHasInlineBinary(result)
if !hadInline {
return result, false, nil
}
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
if err != nil {
return nil, false, err
}
return s.transformLocalBinaryResult(ctx, taskID, result, true, true)
return next, !TaskResultHasInlineBinary(next), nil
}
func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, result map[string]any, writeFiles bool, enforceLimits bool) (map[string]any, bool, error) {
root := s.localBinaryResultRoot()
taskDir := filepath.Join(root, safeLocalBinaryTaskDir(taskID))
func (s *Service) transformLocalBinaryResult(ctx context.Context, _ string, result map[string]any, _ bool, enforceLimits bool) (map[string]any, bool, error) {
materializer := &localBinaryMaterializer{
service: s,
taskDir: taskDir,
writeFiles: writeFiles,
enforceLimits: enforceLimits,
seen: map[string]struct{}{},
}
next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
if err != nil {
materializer.rollbackCreatedFiles()
return nil, false, err
}
mapped, ok := next.(map[string]any)
if !ok {
materializer.rollbackCreatedFiles()
return nil, false, &clients.ClientError{
Code: "result_binary_not_materialized",
Message: "generated result is not a JSON object",
@@ -83,28 +78,28 @@ func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string,
Retryable: false,
}
}
if changed && enforceLimits && !writeFiles {
if err := os.MkdirAll(root, 0o750); err != nil {
return nil, false, localBinaryStorageError(err)
}
if err := ensureLocalBinaryDiskHeadroom(root, materializer.totalBytes, s.localResultMinFreeBytes()); err != nil {
return nil, false, err
}
}
return mapped, changed, nil
}
// MaterializeTaskResultForStorage exposes the same verified materialization
// path to the explicit historical maintenance command.
func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
return s.materializeLocalBinaryResult(ctx, taskID, result)
hadInline := TaskResultHasInlineBinary(result)
if !hadInline {
return result, false, nil
}
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
if err != nil {
return nil, false, err
}
return next, hadInline && !TaskResultHasInlineBinary(next), nil
}
// CompactExpiredTaskResultForStorage computes the same deterministic
// placeholders without creating files that are already outside the recovery
// window.
// CompactExpiredTaskResultForStorage keeps the historical maintenance API but
// now uses the same object-storage path as live results. New GatewayBinary
// placeholders are never created; their parser remains read-only compatibility.
func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
return s.transformLocalBinaryResult(ctx, taskID, result, false, false)
return s.MaterializeTaskResultForStorage(ctx, taskID, result)
}
func TaskResultHasInlineBinary(result map[string]any) bool {
@@ -239,15 +234,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
Retryable: false,
}
}
if m.writeFiles {
created, err := m.service.writeLocalBinaryResult(m.taskDir, digestHex, payload)
if err != nil {
return nil, false, err
}
if created {
m.createdFiles = append(m.createdFiles, filepath.Join(m.taskDir, digestHex+".bin"))
}
}
m.seen[digestHex] = struct{}{}
m.totalBytes += size
}
@@ -261,94 +247,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
return localBinaryPlaceholder(descriptor), true, nil
}
func (m *localBinaryMaterializer) rollbackCreatedFiles() {
for _, path := range m.createdFiles {
_ = os.Remove(path)
}
_ = os.Remove(m.taskDir)
m.createdFiles = nil
}
func (s *Service) writeLocalBinaryResult(taskDir string, digestHex string, payload []byte) (bool, error) {
if err := os.MkdirAll(taskDir, 0o750); err != nil {
return false, localBinaryStorageError(err)
}
targetPath := filepath.Join(taskDir, digestHex+".bin")
if info, err := os.Stat(targetPath); err == nil {
if !info.IsDir() && info.Size() == int64(len(payload)) {
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err == nil {
now := time.Now()
if err := os.Chtimes(targetPath, now, now); err != nil {
return false, localBinaryStorageError(err)
}
return false, nil
}
}
return false, &clients.ClientError{
Code: "binary_result_corrupted",
Message: "existing local result file does not match its content hash",
StatusCode: 500,
Retryable: false,
}
} else if !errors.Is(err, os.ErrNotExist) {
return false, localBinaryStorageError(err)
}
if err := ensureLocalBinaryDiskHeadroom(taskDir, int64(len(payload)), s.localResultMinFreeBytes()); err != nil {
return false, err
}
tempFile, err := os.CreateTemp(taskDir, ".gateway-result-*")
if err != nil {
return false, localBinaryStorageError(err)
}
tempPath := tempFile.Name()
cleanup := func() {
_ = tempFile.Close()
_ = os.Remove(tempPath)
}
if err := tempFile.Chmod(0o640); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if _, err := tempFile.Write(payload); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if err := tempFile.Sync(); err != nil {
cleanup()
return false, localBinaryStorageError(err)
}
if err := tempFile.Close(); err != nil {
_ = os.Remove(tempPath)
return false, localBinaryStorageError(err)
}
if err := os.Rename(tempPath, targetPath); err != nil {
_ = os.Remove(tempPath)
return false, localBinaryStorageError(err)
}
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err != nil {
_ = os.Remove(targetPath)
return false, err
}
return true, nil
}
func ensureLocalBinaryDiskHeadroom(path string, incomingBytes int64, minFreeBytes int64) error {
var stat syscall.Statfs_t
if err := syscall.Statfs(path, &stat); err != nil {
return localBinaryStorageError(err)
}
freeBytes := int64(stat.Bavail) * int64(stat.Bsize)
if freeBytes-incomingBytes < minFreeBytes {
return &clients.ClientError{
Code: "local_result_storage_unavailable",
Message: "local result storage does not have enough free space",
StatusCode: 503,
Retryable: false,
}
}
return nil
}
func localBinaryStorageError(err error) error {
return &clients.ClientError{
Code: "local_result_storage_unavailable",
@@ -406,6 +304,30 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
}
switch typed := value.(type) {
case map[string]any:
refreshedAccessURL := false
if asset, upload, ok := generatedResultUploadReference(typed); ok {
accessURL, err := s.requestAssetAccessURL(ctx, asset)
if err != nil {
return nil, false, err
}
next := make(map[string]any, len(typed))
for key, item := range typed {
next[key] = item
}
nextUpload := make(map[string]any, len(upload))
for key, item := range upload {
nextUpload[key] = item
}
nextUpload["url"] = accessURL
next["upload"] = nextUpload
for _, key := range []string{"url", "image_url", "video_url", "audio_url"} {
if _, exists := next[key]; exists {
next[key] = accessURL
}
}
typed = next
refreshedAccessURL = true
}
if ref, ok := generatedResultAssetReference(typed); ok {
payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
if err != nil {
@@ -418,7 +340,7 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
return encoded, true, nil
}
next := make(map[string]any, len(typed))
changed := false
changed := refreshedAccessURL
for key, childValue := range typed {
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
if err != nil {
@@ -465,6 +387,27 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
}
}
func generatedResultUploadReference(value map[string]any) (store.RequestAsset, map[string]any, bool) {
upload, ok := value["upload"].(map[string]any)
if !ok {
return store.RequestAsset{}, nil, false
}
objectKey := strings.TrimSpace(stringFromAny(upload["objectKey"]))
channel, _ := upload["storageChannel"].(map[string]any)
channelKey := strings.TrimSpace(stringFromAny(channel["channelKey"]))
if objectKey == "" || channelKey == "" {
return store.RequestAsset{}, nil, false
}
return store.RequestAsset{
URL: stringFromAny(upload["url"]),
StorageProvider: stringFromAny(channel["provider"]),
StorageChannelID: stringFromAny(channel["id"]),
StorageChannelKey: channelKey,
ObjectKey: objectKey,
AccessScope: stringFromAny(upload["accessScope"]),
}, upload, true
}
func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) {
ref, ok := value["assetRef"].(map[string]any)
if !ok {
@@ -475,10 +418,14 @@ func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bo
return store.RequestAsset{}, false
}
asset := store.RequestAsset{
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
StorageProvider: stringFromAny(ref["storageProvider"]),
SHA256: strings.ToLower(strings.TrimSpace(stringFromAny(ref["sha256"]))),
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
StorageProvider: stringFromAny(ref["storageProvider"]),
StorageChannelID: stringFromAny(ref["storageChannelId"]),
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
ObjectKey: stringFromAny(ref["objectKey"]),
AccessScope: stringFromAny(ref["accessScope"]),
}
if size := floatFromAny(ref["size"]); size > 0 {
asset.ByteSize = int64(size)
@@ -767,13 +714,6 @@ func (s *Service) localResultTTLHours() int {
return s.cfg.LocalResultTTLHours
}
func (s *Service) localResultMinFreeBytes() int64 {
if s.cfg.LocalResultMinFreeBytes <= 0 {
return defaultLocalResultMinFreeBytes
}
return s.cfg.LocalResultMinFreeBytes
}
func (s *Service) localResultMaxBytes() int64 {
if s.cfg.LocalResultMaxBytes <= 0 {
return defaultLocalResultMaxBytes
apps/api/internal/runner/binary_results_test.go
+83 -31
View File
@@ -31,7 +31,21 @@ func newLocalBinaryTestService(t *testing.T) *Service {
}}
}
func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
func writeHistoricalLocalBinaryFixture(t *testing.T, service *Service, taskID string, payload []byte) string {
t.Helper()
digest := sha256.Sum256(payload)
taskDir := filepath.Join(service.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID))
if err := os.MkdirAll(taskDir, 0o750); err != nil {
t.Fatalf("create historical result fixture directory: %v", err)
}
path := filepath.Join(taskDir, hex.EncodeToString(digest[:])+".bin")
if err := os.WriteFile(path, payload, 0o640); err != nil {
t.Fatalf("write historical result fixture: %v", err)
}
return path
}
func TestHydrateHistoricalLocalBinaryResult(t *testing.T) {
service := newLocalBinaryTestService(t)
payload := []byte("one binary result shared across representations")
encoded := base64.StdEncoding.EncodeToString(payload)
@@ -49,7 +63,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
},
}
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input)
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-123", input, false, true)
if err != nil {
t.Fatalf("materialize local binary result: %v", err)
}
@@ -76,6 +90,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
if len(persistentJSON) > 32*1024 {
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
}
writeHistoricalLocalBinaryFixture(t, service, "task-123", payload)
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
entries, err := os.ReadDir(taskDir)
@@ -155,19 +170,53 @@ func TestHydrateGeneratedResultAssetReference(t *testing.T) {
assertClientErrorCode(t, err, "binary_result_corrupted")
}
func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) {
cfg := config.Config{
MediaOSSDirectEnabled: true,
MediaOSSEndpoint: "https://oss.example.com",
MediaOSSBucket: "media-bucket",
MediaOSSAccessKeyID: "access-id",
MediaOSSAccessKeySecret: "access-secret",
MediaOSSObjectPrefix: "media",
}
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
result := map[string]any{
"data": []any{map[string]any{
"url": "https://expired.example.com/result.png",
"image_url": "https://expired.example.com/result.png",
"upload": map[string]any{
"url": "https://expired.example.com/result.png",
"objectKey": "media/image_result/2026/08/04/hash.png",
"accessScope": "private",
"storageChannel": map[string]any{
"channelKey": "environment-direct-oss",
"provider": "aliyun_oss",
},
},
}},
}
hydrated, err := service.HydrateTaskResult(t.Context(), "task-private-url", result)
if err != nil {
t.Fatal(err)
}
item := hydrated["data"].([]any)[0].(map[string]any)
for _, key := range []string{"url", "image_url"} {
value := stringFromAny(item[key])
if !strings.Contains(value, "OSSAccessKeyId=access-id") || strings.Contains(value, "expired.example.com") {
t.Fatalf("%s was not refreshed: %q", key, value)
}
}
}
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultTTLHours = 1
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded})
persistent, _, err := service.transformLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}, false, true)
if err != nil {
t.Fatalf("materialize fixture: %v", err)
}
entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired"))
if err != nil || len(entries) != 1 {
t.Fatalf("read fixture result: entries=%v err=%v", entries, err)
}
path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name())
path := writeHistoricalLocalBinaryFixture(t, service, "task-expired", []byte("expiring result"))
old := time.Now().Add(-2 * time.Hour)
if err := os.Chtimes(path, old, old); err != nil {
t.Fatalf("age fixture: %v", err)
@@ -186,17 +235,17 @@ func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T)
assertClientErrorCode(t, err, "binary_result_corrupted")
}
func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
func TestHistoricalLocalBinaryFixtureEnforcesLimitsAndKeepsText(t *testing.T) {
service := newLocalBinaryTestService(t)
service.cfg.LocalResultMaxBytes = 4
_, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{
_, _, err := service.transformLocalBinaryResult(context.Background(), "task-large", map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
})
}, false, true)
assertClientErrorCode(t, err, "binary_result_too_large")
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-text", map[string]any{
"message": "YWJjZA==",
})
}, false, true)
if err != nil || changed || persistent["message"] != "YWJjZA==" {
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
}
@@ -204,18 +253,30 @@ func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
service = newLocalBinaryTestService(t)
service.cfg.LocalResultMaxBytes = 1024
service.cfg.LocalResultMaxTaskBytes = 8
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
_, _, err = service.transformLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
})
}, false, true)
assertClientErrorCode(t, err, "binary_result_too_large")
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
}
}
func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
func TestCompactExpiredTaskResultUsesObjectStorageAndDoesNotWriteFiles(t *testing.T) {
var uploads int
storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
t.Fatalf("method=%s, want PUT", r.Method)
}
uploads++
w.WriteHeader(http.StatusOK)
}))
defer storageServer.Close()
service := newLocalBinaryTestService(t)
service.directOSS = &directOSSUploader{
endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
}
service.cfg.LocalResultMaxBytes = 1
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
@@ -225,17 +286,18 @@ func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
if err != nil {
t.Fatalf("compact expired result: %v", err)
}
if !changed || !localBinaryResultHasPlaceholders(persistent) {
t.Fatalf("expired result was not compacted: %+v", persistent)
if !changed || localBinaryResultHasPlaceholders(persistent) || TaskResultHasInlineBinary(persistent) {
t.Fatalf("expired result was not objectified: %+v", persistent)
}
if uploads != 1 {
t.Fatalf("object storage uploads=%d, want 1", uploads)
}
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("expired compaction must not create a task directory: %v", err)
}
_, err = service.HydrateTaskResult(context.Background(), "task-old", persistent)
assertClientErrorCode(t, err, "binary_result_expired")
}
func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
err := localBinaryStorageError(errors.New("write failed"))
assertClientErrorCode(t, err, "local_result_storage_unavailable")
if clients.IsRetryable(err) {
@@ -247,16 +309,6 @@ func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
t.Fatal("local result storage failure must not fail over to another provider")
}
service := newLocalBinaryTestService(t)
service.cfg.LocalResultMinFreeBytes = 1 << 62
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")),
})
assertClientErrorCode(t, err, "local_result_storage_unavailable")
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("disk preflight must not create partial task files: %v", statErr)
}
}
func bytesToAny(payload []byte) []any {
apps/api/internal/runner/direct_oss.go
+23 -131
View File
@@ -1,26 +1,12 @@
package runner
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const directOSSUploadTimeout = 120 * time.Second
type directOSSUploader struct {
endpoint string
bucket string
@@ -28,8 +14,6 @@ type directOSSUploader struct {
accessKeySecret string
publicBaseURL string
objectPrefix string
client *http.Client
now func() time.Time
}
func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
@@ -43,133 +27,41 @@ func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret),
publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"),
objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"),
client: &http.Client{
Timeout: directOSSUploadTimeout,
Transport: &http.Transport{
MaxIdleConns: 256,
MaxIdleConnsPerHost: 256,
MaxConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
},
},
now: time.Now,
}
}
func directOSSScene(scene string) bool {
switch strings.TrimSpace(scene) {
case store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
case store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
return true
default:
return false
}
}
func (u *directOSSUploader) upload(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
func (u *directOSSUploader) fileStorageChannel() store.FileStorageChannel {
if u == nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: "direct OSS uploader is not configured", Retryable: false}
return store.FileStorageChannel{}
}
objectKey := u.objectKey(payload)
escapedKey := escapeOSSObjectKey(objectKey)
uploadURL := u.endpoint + "/" + escapedKey
contentType := strings.TrimSpace(payload.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
return store.FileStorageChannel{
ChannelKey: "environment-direct-oss",
Name: "Environment Direct OSS",
Provider: "aliyun_oss",
AccessKeyID: u.accessKeyID,
AccessKeySecret: u.accessKeySecret,
Priority: 10,
Status: "enabled",
Scenes: []string{store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult},
Config: map[string]any{
"endpoint": u.endpoint,
"bucket": u.bucket,
"publicBaseUrl": u.publicBaseURL,
"objectPrefix": u.objectPrefix,
},
RetryPolicy: map[string]any{
"enabled": true,
"maxRetries": 2,
"backoffSeconds": []any{0.25, 1.0},
},
}
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
if err := sleepWithContext(ctx, time.Duration(attempt*attempt)*200*time.Millisecond); err != nil {
return nil, err
}
}
status, err := u.put(ctx, uploadURL, escapedKey, contentType, payload.Bytes)
if err == nil && status >= 200 && status < 300 {
publicURL := u.publicBaseURL + "/" + escapedKey
return map[string]any{
"url": publicURL,
"fileName": objectKey,
"storageChannel": map[string]any{
"channelKey": "environment-direct-oss",
"name": "Environment Direct OSS",
"provider": "aliyun_oss_direct",
},
"assetStorage": map[string]any{
"scene": payload.Scene,
"source": firstNonEmptyString(payload.Source, "ai-gateway"),
"strategy": "direct_aliyun_oss",
},
}, nil
}
if err != nil {
lastErr = err
continue
}
lastErr = fmt.Errorf("HTTP %d", status)
if status != http.StatusTooManyRequests && status < 500 {
break
}
}
message := "direct OSS upload failed"
if lastErr != nil {
message += ": " + lastErr.Error()
}
return nil, &clients.ClientError{Code: "upload_failed", Message: message, Retryable: true}
}
func (u *directOSSUploader) put(ctx context.Context, uploadURL string, escapedKey string, contentType string, payload []byte) (int, error) {
date := u.now().UTC().Format(http.TimeFormat)
canonicalResource := "/" + u.bucket + "/" + escapedKey
stringToSign := "PUT\n\n" + contentType + "\n" + date + "\n" + canonicalResource
mac := hmac.New(sha1.New, []byte(u.accessKeySecret))
_, _ = mac.Write([]byte(stringToSign))
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(payload))
if err != nil {
return 0, err
}
req.Header.Set("Authorization", "OSS "+u.accessKeyID+":"+signature)
req.Header.Set("Content-Type", contentType)
req.Header.Set("Date", date)
resp, err := u.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
return resp.StatusCode, nil
}
func (u *directOSSUploader) objectKey(payload FileUploadPayload) string {
now := u.now().UTC()
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
baseName := strings.TrimSuffix(path.Base(payload.FileName), path.Ext(payload.FileName))
baseName = sanitizeGeneratedAssetNamePart(baseName)
if baseName == "" {
baseName = "gateway-media"
}
if len(baseName) > 48 {
baseName = baseName[:48]
}
return fmt.Sprintf(
"%s/%s/%04d/%02d/%02d/%s-%s%s",
u.objectPrefix,
strings.TrimSpace(payload.Scene),
now.Year(),
now.Month(),
now.Day(),
baseName,
randomHexSuffix(8),
extension,
)
}
func escapeOSSObjectKey(objectKey string) string {
parts := strings.Split(objectKey, "/")
for index, part := range parts {
parts[index] = url.PathEscape(part)
}
return strings.Join(parts, "/")
}
apps/api/internal/runner/direct_oss_test.go
+15 -18
View File
@@ -12,13 +12,12 @@ import (
"strings"
"sync/atomic"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
func TestDirectOSSUploadSignsRequestAndReturnsPrivateSignedURL(t *testing.T) {
payload := []byte("production-isomorphic-image")
var calls atomic.Int64
var uploadedPath string
@@ -60,9 +59,6 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
}
uploader := newDirectOSSUploader(cfg)
uploader.now = func() time.Time {
return time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC)
}
service := &Service{cfg: cfg, directOSS: uploader}
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
Bytes: payload,
@@ -77,29 +73,30 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
if calls.Load() != 2 {
t.Fatalf("upload calls=%d, want one retry", calls.Load())
}
if !strings.HasPrefix(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/2026/07/30/input-") ||
!strings.HasSuffix(uploadedPath, ".png") {
if !strings.Contains(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/") ||
!strings.HasSuffix(uploadedPath, "/"+sha256Hex(payload)+".png") {
t.Fatalf("unexpected object path=%q", uploadedPath)
}
if got := stringFromAny(uploaded["url"]); got != "https://cdn.example.com"+uploadedPath {
t.Fatalf("public URL=%q", got)
if got := stringFromAny(uploaded["url"]); !strings.HasPrefix(got, server.URL+uploadedPath+"?") || !strings.Contains(got, "Signature=") {
t.Fatalf("private signed URL=%q", got)
}
if uploaded["accessScope"] != "private" {
t.Fatalf("request asset access scope=%v", uploaded["accessScope"])
}
channel, _ := uploaded["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
if stringFromAny(channel["provider"]) != "aliyun_oss" {
t.Fatalf("storage channel=%+v", channel)
}
}
func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
func TestDirectOSSPersistsGeneralUploadScene(t *testing.T) {
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
storageDir := t.TempDir()
cfg := config.Config{
LocalUploadedStorageDir: storageDir,
MediaOSSDirectEnabled: true,
MediaOSSEndpoint: server.URL,
MediaOSSBucket: "media-bucket",
@@ -118,11 +115,11 @@ func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
if err != nil {
t.Fatalf("general upload: %v", err)
}
if calls.Load() != 0 {
if calls.Load() != 1 {
t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load())
}
channel, _ := uploaded["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "local_static" {
if stringFromAny(channel["provider"]) != "aliyun_oss" {
t.Fatalf("general upload channel=%+v", channel)
}
}
@@ -158,7 +155,7 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
SourceKey: "b64_json",
},
0,
[]store.FileStorageChannel{{ID: "unused-channel"}},
[]store.FileStorageChannel{service.directOSS.fileStorageChannel()},
)
if err != nil {
t.Fatalf("direct generated upload: %v", err)
@@ -166,11 +163,11 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
if calls.Load() != 1 {
t.Fatalf("direct OSS calls=%d, want 1", calls.Load())
}
if contentType != "image/png" || kind != "image" || strategy != "direct_aliyun_oss" {
if contentType != "image/png" || kind != "image" || strategy != "upload_inline_media" {
t.Fatalf("generated metadata=%s/%s/%s", contentType, kind, strategy)
}
channel, _ := upload["storageChannel"].(map[string]any)
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
if stringFromAny(channel["provider"]) != "aliyun_oss" {
t.Fatalf("storage channel=%+v", channel)
}
}
apps/api/internal/runner/object_storage.go
+530
View File
@@ -0,0 +1,530 @@
package runner
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
const (
objectStorageRequestTimeout = 120 * time.Second
objectStorageReadLimit = 256 << 20
objectStorageSignedURLTTL = 15 * time.Minute
)
var sharedObjectStorageHTTPClient = &http.Client{
Timeout: objectStorageRequestTimeout,
Transport: &http.Transport{
MaxIdleConns: 256,
MaxIdleConnsPerHost: 256,
MaxConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
},
}
type objectStorageAdapter struct {
channel store.FileStorageChannel
client *http.Client
now func() time.Time
}
type FileStorageChannelTestResult struct {
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
DurationMS int64 `json:"durationMs"`
}
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
// against one object-storage channel. The random probe avoids overwriting a
// content-addressed business object and no object key or credential is exposed.
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
startedAt := time.Now()
result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return result, err
}
payload := FileUploadPayload{
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
ContentType: "application/octet-stream",
FileName: "probe.bin",
Scene: store.FileStorageSceneUpload,
Source: "admin-connection-test",
}
upload, err := adapter.put(ctx, payload)
if err != nil {
return result, err
}
result.PutSucceeded = true
objectKey := stringFromAny(upload["objectKey"])
if objectKey == "" {
return result, storageClientError("storage_write_failed", "object storage probe did not return an object reference", 0, false)
}
if err := adapter.head(ctx, objectKey); err != nil {
_ = adapter.delete(context.WithoutCancel(ctx), objectKey)
return result, err
}
result.HeadSucceeded = true
if err := adapter.delete(ctx, objectKey); err != nil {
return result, err
}
result.DeleteSucceeded = true
result.DurationMS = time.Since(startedAt).Milliseconds()
return result, nil
}
func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAdapter, error) {
provider := strings.ToLower(strings.TrimSpace(channel.Provider))
if provider != "aliyun_oss" && provider != "s3" {
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported object storage provider", Retryable: false}
}
if objectStorageConfigString(channel.Config, "endpoint") == "" || objectStorageConfigString(channel.Config, "bucket") == "" {
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "object storage endpoint and bucket are required", Retryable: false}
}
if strings.TrimSpace(channel.AccessKeyID) == "" || strings.TrimSpace(channel.AccessKeySecret) == "" {
return nil, &clients.ClientError{Code: "storage_auth_failed", Message: "object storage credentials are not configured", Retryable: false}
}
return &objectStorageAdapter{
channel: channel,
client: sharedObjectStorageHTTPClient,
now: time.Now,
}, nil
}
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
objectKey := a.objectKey(payload)
requestURL, err := a.objectURL(objectKey)
if err != nil {
return nil, err
}
contentType := strings.TrimSpace(payload.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload.Bytes))
if err != nil {
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
}
req.Header.Set("Content-Type", contentType)
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
return nil, err
}
resp, err := a.client.Do(req)
if err != nil {
return nil, storageClientError("storage_write_failed", err.Error(), 0, true)
}
defer resp.Body.Close()
responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if readErr != nil {
return nil, storageClientError("storage_write_failed", readErr.Error(), resp.StatusCode, storageStatusRetryable(resp.StatusCode))
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, objectStorageHTTPError("storage_write_failed", resp.StatusCode, responseBody)
}
accessScope := objectStorageAccessScope(a.channel, payload.Scene)
publicURL := ""
if accessScope == "public" {
publicURL = a.publicURL(objectKey)
}
urlExpiresAt := ""
if publicURL == "" {
publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL)
if err != nil {
return nil, err
}
urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339)
}
digest := sha256.Sum256(payload.Bytes)
result := map[string]any{
"url": publicURL,
"fileName": path.Base(objectKey),
"objectKey": objectKey,
"contentType": contentType,
"size": len(payload.Bytes),
"sha256": hex.EncodeToString(digest[:]),
"accessScope": accessScope,
"storageChannel": map[string]any{
"id": a.channel.ID,
"channelKey": a.channel.ChannelKey,
"name": a.channel.Name,
"provider": a.channel.Provider,
},
}
if urlExpiresAt != "" {
result["urlExpiresAt"] = urlExpiresAt
}
return result, nil
}
func (a *objectStorageAdapter) get(ctx context.Context, objectKey string) ([]byte, error) {
requestURL, err := a.objectURL(objectKey)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
if err != nil {
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
}
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
return nil, err
}
resp, err := a.client.Do(req)
if err != nil {
return nil, storageClientError("storage_read_failed", err.Error(), 0, true)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
return nil, objectStorageHTTPError("storage_read_failed", resp.StatusCode, body)
}
payload, err := io.ReadAll(io.LimitReader(resp.Body, objectStorageReadLimit+1))
if err != nil {
return nil, storageClientError("storage_read_failed", err.Error(), resp.StatusCode, true)
}
if len(payload) > objectStorageReadLimit {
return nil, storageClientError("storage_read_failed", "stored object exceeds the read limit", resp.StatusCode, false)
}
return payload, nil
}
func (a *objectStorageAdapter) head(ctx context.Context, objectKey string) error {
return a.emptyObjectRequest(ctx, http.MethodHead, objectKey)
}
func (a *objectStorageAdapter) delete(ctx context.Context, objectKey string) error {
return a.emptyObjectRequest(ctx, http.MethodDelete, objectKey)
}
func (a *objectStorageAdapter) emptyObjectRequest(ctx context.Context, method string, objectKey string) error {
requestURL, err := a.objectURL(objectKey)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, method, requestURL, nil)
if err != nil {
return storageClientError("storage_config_invalid", err.Error(), 0, false)
}
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
return err
}
resp, err := a.client.Do(req)
if err != nil {
return storageClientError("storage_read_failed", err.Error(), 0, true)
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return objectStorageHTTPError("storage_read_failed", resp.StatusCode, nil)
}
return nil
}
func (a *objectStorageAdapter) presignGet(objectKey string, ttl time.Duration) (string, error) {
requestURL, err := a.objectURL(objectKey)
if err != nil {
return "", err
}
if ttl <= 0 {
ttl = objectStorageSignedURLTTL
}
if strings.EqualFold(a.channel.Provider, "s3") {
return a.presignS3Get(requestURL, ttl, a.now().UTC())
}
return a.presignOSSGet(requestURL, objectKey, ttl, a.now().UTC())
}
func (a *objectStorageAdapter) objectKey(payload FileUploadPayload) string {
now := a.now().UTC()
digest := sha256.Sum256(payload.Bytes)
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
prefix := strings.Trim(objectStorageConfigString(a.channel.Config, "objectPrefix"), "/")
parts := make([]string, 0, 7)
if prefix != "" {
parts = append(parts, prefix)
}
parts = append(parts,
firstNonEmptyString(strings.TrimSpace(payload.Scene), store.FileStorageSceneUpload),
fmt.Sprintf("%04d", now.Year()),
fmt.Sprintf("%02d", now.Month()),
fmt.Sprintf("%02d", now.Day()),
hex.EncodeToString(digest[:])+extension,
)
return strings.Join(parts, "/")
}
func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) {
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
}
bucket := objectStorageConfigString(a.channel.Config, "bucket")
if strings.EqualFold(a.channel.Provider, "s3") {
if objectStorageConfigBool(a.channel.Config, "forcePathStyle") {
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + bucket + "/" + strings.TrimLeft(objectKey, "/")
} else {
endpoint.Host = bucket + "." + endpoint.Host
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
}
} else {
hostname := endpoint.Hostname()
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
if port := endpoint.Port(); port != "" {
endpoint.Host = bucket + "." + hostname + ":" + port
} else {
endpoint.Host = bucket + "." + endpoint.Host
}
}
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
}
return endpoint.String(), nil
}
func (a *objectStorageAdapter) publicURL(objectKey string) string {
baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/")
if baseURL == "" {
baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/")
}
if baseURL == "" {
return ""
}
return baseURL + "/" + escapeObjectKey(objectKey)
}
func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now time.Time) error {
if strings.EqualFold(a.channel.Provider, "s3") {
a.signS3Request(req, payloadHash, now)
return nil
}
a.signOSSRequest(req, now)
return nil
}
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
date := now.UTC().Format(http.TimeFormat)
contentType := req.Header.Get("Content-Type")
canonicalHeaders := ""
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
req.Header.Set("x-oss-security-token", token)
canonicalHeaders = "x-oss-security-token:" + token + "\n"
}
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
req.Header.Set("Date", date)
}
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
parsed, err := url.Parse(requestURL)
if err != nil {
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
}
expires := strconv.FormatInt(now.Add(ttl).Unix(), 10)
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + "/" + strings.TrimLeft(objectKey, "/")
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), "GET\n\n\n"+expires+"\n"+canonicalResource)
query := parsed.Query()
query.Set("OSSAccessKeyId", a.channel.AccessKeyID)
query.Set("Expires", expires)
query.Set("Signature", signature)
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
query.Set("security-token", token)
}
parsed.RawQuery = query.Encode()
return parsed.String(), nil
}
func (a *objectStorageAdapter) signS3Request(req *http.Request, payloadHash string, now time.Time) {
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
amzDate := now.Format("20060102T150405Z")
date := now.Format("20060102")
req.Header.Set("x-amz-date", amzDate)
req.Header.Set("x-amz-content-sha256", payloadHash)
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
req.Header.Set("x-amz-security-token", token)
}
canonicalHeaders, signedHeaders := s3CanonicalHeaders(req)
canonicalRequest := strings.Join([]string{
req.Method,
s3CanonicalURI(req.URL),
s3CanonicalQuery(req.URL.Query()),
canonicalHeaders,
signedHeaders,
payloadHash,
}, "\n")
scope := date + "/" + region + "/s3/aws4_request"
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
signature := hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign))
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+a.channel.AccessKeyID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+signature)
}
func (a *objectStorageAdapter) presignS3Get(requestURL string, ttl time.Duration, now time.Time) (string, error) {
parsed, err := url.Parse(requestURL)
if err != nil {
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
}
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
amzDate := now.Format("20060102T150405Z")
date := now.Format("20060102")
scope := date + "/" + region + "/s3/aws4_request"
seconds := int64(ttl / time.Second)
if seconds < 1 {
seconds = 1
}
if seconds > 7*24*60*60 {
seconds = 7 * 24 * 60 * 60
}
query := parsed.Query()
query.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
query.Set("X-Amz-Credential", a.channel.AccessKeyID+"/"+scope)
query.Set("X-Amz-Date", amzDate)
query.Set("X-Amz-Expires", strconv.FormatInt(seconds, 10))
query.Set("X-Amz-SignedHeaders", "host")
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
query.Set("X-Amz-Security-Token", token)
}
canonicalRequest := "GET\n" + s3CanonicalURI(parsed) + "\n" + s3CanonicalQuery(query) + "\nhost:" + strings.ToLower(parsed.Host) + "\n\nhost\nUNSIGNED-PAYLOAD"
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
query.Set("X-Amz-Signature", hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign)))
parsed.RawQuery = query.Encode()
return parsed.String(), nil
}
func s3CanonicalHeaders(req *http.Request) (string, string) {
headers := map[string]string{"host": strings.ToLower(req.URL.Host)}
for key, values := range req.Header {
lower := strings.ToLower(strings.TrimSpace(key))
if lower == "content-type" || strings.HasPrefix(lower, "x-amz-") {
headers[lower] = strings.Join(values, ",")
}
}
keys := make([]string, 0, len(headers))
for key := range headers {
keys = append(keys, key)
}
sort.Strings(keys)
var canonical strings.Builder
for _, key := range keys {
canonical.WriteString(key)
canonical.WriteByte(':')
canonical.WriteString(strings.Join(strings.Fields(headers[key]), " "))
canonical.WriteByte('\n')
}
return canonical.String(), strings.Join(keys, ";")
}
func s3CanonicalURI(value *url.URL) string {
uri := value.EscapedPath()
if uri == "" {
return "/"
}
return uri
}
func s3CanonicalQuery(values url.Values) string {
return strings.ReplaceAll(values.Encode(), "+", "%20")
}
func s3SigningHMAC(secret string, date string, region string, stringToSign string) []byte {
dateKey := hmacSHA256([]byte("AWS4"+secret), date)
regionKey := hmacSHA256(dateKey, region)
serviceKey := hmacSHA256(regionKey, "s3")
signingKey := hmacSHA256(serviceKey, "aws4_request")
return hmacSHA256(signingKey, stringToSign)
}
func hmacSHA256(key []byte, value string) []byte {
mac := hmac.New(sha256.New, key)
_, _ = mac.Write([]byte(value))
return mac.Sum(nil)
}
func hmacSHA1Base64(key []byte, value string) string {
mac := hmac.New(sha1.New, key)
_, _ = mac.Write([]byte(value))
return base64.StdEncoding.EncodeToString(mac.Sum(nil))
}
func sha256Hex(value []byte) string {
digest := sha256.Sum256(value)
return hex.EncodeToString(digest[:])
}
func objectStorageConfigString(config map[string]any, key string) string {
if config == nil {
return ""
}
value, _ := config[key].(string)
return strings.TrimSpace(value)
}
func objectStorageConfigBool(config map[string]any, key string) bool {
if config == nil {
return false
}
value, _ := config[key].(bool)
return value
}
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
return scope
}
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
return "private"
}
if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" {
return "public"
}
return "private"
}
func escapeObjectKey(objectKey string) string {
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
for index, part := range parts {
parts[index] = url.PathEscape(part)
}
return strings.Join(parts, "/")
}
func storageStatusRetryable(status int) bool {
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
}
func objectStorageHTTPError(code string, status int, body []byte) error {
if status == http.StatusUnauthorized || status == http.StatusForbidden {
code = "storage_auth_failed"
}
message := http.StatusText(status)
if message == "" {
message = "object storage request failed"
}
// The provider response may contain object names, endpoints or credential
// diagnostics. Keep it out of the public error chain; channel health stores
// only the stable status description as well.
return storageClientError(code, message, status, storageStatusRetryable(status))
}
func storageClientError(code string, message string, status int, retryable bool) error {
return &clients.ClientError{Code: code, Message: message, StatusCode: status, Retryable: retryable}
}
apps/api/internal/runner/object_storage_acceptance_test.go
+102
View File
@@ -0,0 +1,102 @@
package runner
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceemulator"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestLocalAcceptanceObjectStorageFailoverMatrix(t *testing.T) {
emulator := httptest.NewServer(acceptanceemulator.New(acceptanceemulator.Config{}).Handler())
defer emulator.Close()
service := &Service{}
payload := FileUploadPayload{
Bytes: []byte("acceptance-object"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset,
}
tests := []struct {
name string
channels []store.FileStorageChannel
winner string
wantErrCode string
}{
{
name: "oss_auth_to_s3",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-auth/bucket", "oss-auth", false),
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3", "s3-ok", false),
},
winner: "s3-ok",
},
{
name: "s3_auth_to_oss",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-auth", "s3-auth", false),
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-ok", false),
},
winner: "oss-ok",
},
{
name: "same_channel_transient_retry",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-transient", "s3-transient", true),
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-unused", false),
},
winner: "s3-transient",
},
{
name: "all_channels_failed",
channels: []store.FileStorageChannel{
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-fail/bucket", "oss-fail", false),
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-fail", "s3-fail", false),
},
wantErrCode: "storage_write_failed",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := service.uploadFileWithFailover(t.Context(), payload, test.channels)
if test.wantErrCode != "" {
if clients.ErrorCode(err) != test.wantErrCode {
t.Fatalf("error=%v code=%s", err, clients.ErrorCode(err))
}
return
}
if err != nil {
t.Fatal(err)
}
channel, _ := result["storageChannel"].(map[string]any)
if got := stringFromAny(channel["channelKey"]); got != test.winner {
t.Fatalf("winner=%q, want %q", got, test.winner)
}
})
}
response, err := http.Get(emulator.URL + "/report")
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
var report acceptanceemulator.Report
if err := json.NewDecoder(response.Body).Decode(&report); err != nil {
t.Fatal(err)
}
if report.StoragePuts != 3 || report.StorageFailures != 5 {
t.Fatalf("unexpected storage acceptance report: %+v", report)
}
}
func acceptanceStorageChannel(provider string, endpoint string, key string, retry bool) store.FileStorageChannel {
channel := testObjectStorageChannel(provider, endpoint, key)
channel.Config["publicBaseUrl"] = ""
channel.RetryPolicy = map[string]any{"enabled": retry, "maxRetries": 0}
if retry {
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
}
return channel
}
apps/api/internal/runner/object_storage_test.go
+314
View File
@@ -0,0 +1,314 @@
package runner
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
type objectStorageMetricEvent struct {
event string
provider string
bytes int64
}
type objectStorageMetricsRecorder struct {
events []objectStorageMetricEvent
}
func (m *objectStorageMetricsRecorder) ObserveBillingEvent(string) {}
func (m *objectStorageMetricsRecorder) ObserveObjectStorage(event string, provider string, bytes int64, _ time.Duration) {
m.events = append(m.events, objectStorageMetricEvent{event: event, provider: provider, bytes: bytes})
}
func TestS3ObjectStorageUsesSigV4AndDeterministicObjectKey(t *testing.T) {
payload := []byte("same-media-payload")
var calls atomic.Int64
var requestPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
requestPath = r.URL.Path
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=access-id/") {
t.Errorf("missing SigV4 authorization: %q", r.Header.Get("Authorization"))
}
if r.Header.Get("x-amz-content-sha256") != sha256Hex(payload) {
t.Errorf("payload hash=%q", r.Header.Get("x-amz-content-sha256"))
}
body, _ := io.ReadAll(r.Body)
if string(body) != string(payload) {
t.Errorf("payload=%q", body)
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "s3-primary")
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC) }
result, err := adapter.put(t.Context(), FileUploadPayload{Bytes: payload, ContentType: "image/png", FileName: "ignored.jpg", Scene: store.FileStorageSceneImageResult})
if err != nil {
t.Fatal(err)
}
wantSuffix := "/bucket/media/image_result/2026/08/04/" + sha256Hex(payload) + ".png"
if requestPath != wantSuffix {
t.Fatalf("request path=%q, want %q", requestPath, wantSuffix)
}
if stringFromAny(result["objectKey"]) != strings.TrimPrefix(wantSuffix, "/bucket/") {
t.Fatalf("object key=%q", result["objectKey"])
}
if calls.Load() != 1 {
t.Fatalf("calls=%d", calls.Load())
}
}
func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
channel.Config["publicBaseUrl"] = "https://cdn.example"
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
result, err := adapter.put(t.Context(), FileUploadPayload{
Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset,
})
if err != nil {
t.Fatal(err)
}
if result["accessScope"] != "private" {
t.Fatalf("accessScope=%v, want private", result["accessScope"])
}
gotURL := stringFromAny(result["url"])
if strings.HasPrefix(gotURL, "https://cdn.example/") || !strings.Contains(gotURL, "X-Amz-Signature=") {
t.Fatalf("request asset URL must be signed and private: %s", gotURL)
}
}
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
channel.Config["forcePathStyle"] = false
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
value, err := adapter.objectURL("media/request_asset/object.png")
if err != nil {
t.Fatal(err)
}
if value != "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/request_asset/object.png" {
t.Fatalf("OSS object URL=%q", value)
}
}
func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) {
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
channel.Config["forcePathStyle"] = false
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/2026/08/04/OBJECT.png", nil)
if err != nil {
t.Fatal(err)
}
request.Header.Set("Content-Type", "image/png")
adapter.signOSSRequest(request, time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC))
if got, want := request.Header.Get("Authorization"), "OSS access-id:fohlh2SG7k7D+cmX48b7Keqa1Ok="; got != want {
t.Fatalf("authorization=%q, want %q", got, want)
}
if got := request.Header.Get("Date"); got != "Tue, 04 Aug 2026 01:02:03 GMT" {
t.Fatalf("date=%q", got)
}
}
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
var primaryCalls atomic.Int64
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
primaryCalls.Add(1)
http.Error(w, "temporary", http.StatusServiceUnavailable)
}))
defer primary.Close()
var secondaryCalls atomic.Int64
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
secondaryCalls.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer secondary.Close()
first := testObjectStorageChannel("s3", primary.URL, "s3-primary")
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
second := testObjectStorageChannel("aliyun_oss", secondary.URL, "oss-secondary")
second.RetryPolicy = map[string]any{"enabled": false}
metrics := &objectStorageMetricsRecorder{}
service := &Service{billingMetrics: metrics}
result, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second})
if err != nil {
t.Fatal(err)
}
channel, _ := result["storageChannel"].(map[string]any)
if stringFromAny(channel["channelKey"]) != "oss-secondary" {
t.Fatalf("unexpected winning channel: %+v", channel)
}
if primaryCalls.Load() != 2 || secondaryCalls.Load() != 1 {
t.Fatalf("calls primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
}
assertObjectStorageMetricEvent(t, metrics.events, "retry", "s3", 0)
assertObjectStorageMetricEvent(t, metrics.events, "failover", "s3", 0)
assertObjectStorageMetricEvent(t, metrics.events, "write_success", "aliyun_oss", int64(len("payload")))
}
func TestFileStorageChannelConnectionProbeWritesHeadsAndDeletes(t *testing.T) {
var methods []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
methods = append(methods, r.Method)
switch r.Method {
case http.MethodPut, http.MethodHead, http.MethodDelete:
w.WriteHeader(http.StatusOK)
default:
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
}
}))
defer server.Close()
result, err := (&Service{}).TestFileStorageChannel(t.Context(), testObjectStorageChannel("s3", server.URL, "probe"))
if err != nil {
t.Fatal(err)
}
if !result.PutSucceeded || !result.HeadSucceeded || !result.DeleteSucceeded || result.Provider != "s3" {
t.Fatalf("unexpected probe result: %+v", result)
}
if strings.Join(methods, ",") != "PUT,HEAD,DELETE" {
t.Fatalf("probe methods=%v", methods)
}
}
func TestObjectStorageReadUsesChannelRetryPolicy(t *testing.T) {
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 {
http.Error(w, "temporary", http.StatusServiceUnavailable)
return
}
_, _ = w.Write([]byte("stored-payload"))
}))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "s3-read")
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
t.Fatal(err)
}
payload, err := readObjectStorageWithRetries(t.Context(), adapter, "media/request_asset/object.png")
if err != nil || string(payload) != "stored-payload" || calls.Load() != 2 {
t.Fatalf("payload=%q calls=%d err=%v", payload, calls.Load(), err)
}
}
func TestObjectStorageAuthFailureSkipsSameChannelRetry(t *testing.T) {
var primaryCalls atomic.Int64
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
primaryCalls.Add(1)
http.Error(w, "forbidden details", http.StatusForbidden)
}))
defer primary.Close()
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer secondary.Close()
first := testObjectStorageChannel("aliyun_oss", primary.URL, "oss-primary")
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 3, "backoffSeconds": []any{0.001}}
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
service := &Service{}
if _, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second}); err != nil {
t.Fatal(err)
}
if primaryCalls.Load() != 1 {
t.Fatalf("auth failure retried %d times", primaryCalls.Load())
}
}
func TestRequestLevelUploadFailureDoesNotSwitchChannels(t *testing.T) {
var primaryCalls atomic.Int64
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
primaryCalls.Add(1)
http.Error(w, "unsupported media", http.StatusUnsupportedMediaType)
}))
defer primary.Close()
var secondaryCalls atomic.Int64
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
secondaryCalls.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer secondary.Close()
first := store.FileStorageChannel{
ChannelKey: "server-main-primary", Provider: "server_main_openapi", UploadURL: primary.URL,
APIKey: "test-key", RetryPolicy: map[string]any{"enabled": false},
}
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
_, err := (&Service{}).uploadFileWithFailover(t.Context(), FileUploadPayload{
Bytes: []byte("payload"), ContentType: "application/x-unsupported", Scene: store.FileStorageSceneUpload,
}, []store.FileStorageChannel{first, second})
if err == nil || clients.ErrorCode(err) != "upload_failed" {
t.Fatalf("unexpected request-level error: %v", err)
}
if primaryCalls.Load() != 1 || secondaryCalls.Load() != 0 {
t.Fatalf("request-level failure switched channels: primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
}
}
func TestAllObjectStorageChannelsFailWithStableError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "failure", http.StatusBadGateway) }))
defer server.Close()
channel := testObjectStorageChannel("s3", server.URL, "only")
channel.RetryPolicy = map[string]any{"enabled": false}
_, err := (&Service{}).uploadFileWithFailover(context.Background(), FileUploadPayload{Bytes: []byte("payload"), Scene: store.FileStorageSceneUpload}, []store.FileStorageChannel{channel})
if clients.ErrorCode(err) != "storage_write_failed" || strings.Contains(err.Error(), "failure") {
t.Fatalf("unexpected public storage error: %v", err)
}
}
func testObjectStorageChannel(provider string, endpoint string, channelKey string) store.FileStorageChannel {
return store.FileStorageChannel{
ChannelKey: channelKey,
Name: channelKey,
Provider: provider,
AccessKeyID: "access-id",
AccessKeySecret: "access-secret",
Priority: 100,
Config: map[string]any{
"endpoint": endpoint,
"bucket": "bucket",
"region": "cn-test-1",
"objectPrefix": "media",
"publicBaseUrl": "https://cdn.example.com",
"forcePathStyle": true,
},
}
}
func assertObjectStorageMetricEvent(t *testing.T, events []objectStorageMetricEvent, event string, provider string, bytes int64) {
t.Helper()
for _, item := range events {
if item.event == event && item.provider == provider && item.bytes == bytes {
return
}
}
t.Fatalf("missing metric event %s/%s/%d in %+v", event, provider, bytes, events)
}
apps/api/internal/runner/request_assets.go
+79 -6
View File
@@ -148,10 +148,14 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st
}
return base64.StdEncoding.EncodeToString(payload), nil
}
if strings.TrimSpace(asset.URL) == "" {
assetURL, err := s.requestAssetAccessURL(ctx, asset)
if err != nil {
return nil, err
}
if strings.TrimSpace(assetURL) == "" {
return nil, requestAssetExpiredError(asset)
}
return asset.URL, nil
return assetURL, nil
}
func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value string, path []string, candidate store.RuntimeModelCandidate) (any, error) {
@@ -224,10 +228,14 @@ func (s *Service) resolveRequestAsset(ctx context.Context, ref map[string]any) (
sha := stringFromAny(ref["sha256"])
contentType := stringFromAny(ref["contentType"])
asset := store.RequestAsset{
SHA256: sha,
ContentType: contentType,
URL: stringFromAny(ref["url"]),
StorageProvider: stringFromAny(ref["storageProvider"]),
SHA256: sha,
ContentType: contentType,
URL: stringFromAny(ref["url"]),
StorageProvider: stringFromAny(ref["storageProvider"]),
StorageChannelID: stringFromAny(ref["storageChannelId"]),
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
ObjectKey: stringFromAny(ref["objectKey"]),
AccessScope: stringFromAny(ref["accessScope"]),
}
if size := floatFromAny(ref["size"]); size > 0 {
asset.ByteSize = int64(size)
@@ -254,6 +262,17 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
if requestAssetIsExpired(asset, time.Now()) {
return nil, requestAssetExpiredError(asset)
}
if strings.TrimSpace(asset.ObjectKey) != "" {
channel, err := s.fileStorageChannelForAsset(ctx, asset)
if err != nil {
return nil, err
}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return nil, err
}
return readObjectStorageWithRetries(ctx, adapter, asset.ObjectKey)
}
if strings.TrimSpace(asset.LocalPath) != "" {
payload, err := os.ReadFile(asset.LocalPath)
if err != nil {
@@ -290,6 +309,60 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
return nil, requestAssetExpiredError(asset)
}
func readObjectStorageWithRetries(ctx context.Context, adapter *objectStorageAdapter, objectKey string) ([]byte, error) {
maxRetries, delays := uploadRetrySchedule(adapter.channel.RetryPolicy, adapter.channel.Provider)
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
payload, err := adapter.get(ctx, objectKey)
if err == nil {
return payload, nil
}
lastErr = err
if attempt >= maxRetries || !clients.IsRetryable(err) {
break
}
if err := sleepWithContext(ctx, retryDelayForAttempt(attempt, delays)); err != nil {
return nil, err
}
}
return nil, lastErr
}
func (s *Service) requestAssetAccessURL(ctx context.Context, asset store.RequestAsset) (string, error) {
if strings.TrimSpace(asset.ObjectKey) == "" {
return asset.URL, nil
}
channel, err := s.fileStorageChannelForAsset(ctx, asset)
if err != nil {
return "", err
}
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return "", err
}
if strings.EqualFold(strings.TrimSpace(asset.AccessScope), "public") {
if value := adapter.publicURL(asset.ObjectKey); value != "" {
return value, nil
}
}
return adapter.presignGet(asset.ObjectKey, objectStorageSignedURLTTL)
}
func (s *Service) fileStorageChannelForAsset(ctx context.Context, asset store.RequestAsset) (store.FileStorageChannel, error) {
channelKey := strings.TrimSpace(asset.StorageChannelKey)
if channelKey == "environment-direct-oss" && s.directOSS != nil {
return s.directOSS.fileStorageChannel(), nil
}
if s.store == nil || channelKey == "" {
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
}
channel, err := s.store.GetFileStorageChannelByKey(ctx, channelKey)
if err != nil {
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
}
return channel, nil
}
func (s *Service) localPathFromRequestAssetURL(value string) string {
raw := strings.TrimSpace(value)
if raw == "" {
apps/api/internal/runner/service.go
+9
View File
@@ -230,6 +230,15 @@ func (s *Service) observeTaskEventSkip(reason string) {
}
}
func (s *Service) observeObjectStorage(event string, provider string, bytes int, duration time.Duration) {
observer, ok := s.billingMetrics.(interface {
ObserveObjectStorage(string, string, int64, time.Duration)
})
if ok {
observer.ObserveObjectStorage(event, provider, int64(bytes), duration)
}
}
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
return s.execute(ctx, task, user, nil)
}
apps/api/internal/runner/task_history_workers.go
+9 -2
View File
@@ -10,6 +10,7 @@ import (
"net/http"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
@@ -74,13 +75,19 @@ func (s *Service) processTaskCallbackBatch(ctx context.Context, client *http.Cli
}
func deliverTaskCallback(ctx context.Context, client *http.Client, item store.TaskCallbackDelivery, bearerToken string) (int, error) {
body, err := json.Marshal(map[string]any{
payload := map[string]any{
"taskId": item.TaskID,
"seq": item.Seq,
"eventType": item.EventType,
"status": item.TaskStatus,
"createdAt": item.CreatedAt.UTC().Format(time.RFC3339Nano),
})
}
if item.TaskStatus == "failed" || item.TaskStatus == "cancelled" {
standard := publicerror.WithIDs(publicerror.FromFields(item.TaskErrorCode, item.TaskErrorMessage, 0, false), item.TaskRequestID, item.TaskID)
publicerror.Observe(standard)
payload["error"] = standard
}
body, err := json.Marshal(payload)
if err != nil {
return 0, err
}
apps/api/internal/runner/task_history_workers_test.go
+31
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@@ -116,3 +117,33 @@ func TestDeliverTaskCallbackSendsMinimalAuthenticatedBody(t *testing.T) {
t.Fatalf("callback body=%#v", received)
}
}
func TestDeliverFailedTaskCallbackUsesStandardErrorWithoutSocketDetails(t *testing.T) {
var received map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if err := json.NewDecoder(request.Body).Decode(&received); err != nil {
t.Error(err)
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
status, err := deliverTaskCallback(t.Context(), server.Client(), store.TaskCallbackDelivery{
TaskID: "failed-task",
Seq: 4,
CallbackURL: server.URL,
EventType: "task.failed",
TaskStatus: "failed",
TaskErrorCode: "response_read_error",
TaskErrorMessage: raw,
CreatedAt: time.Now(),
}, "")
if err != nil || status != http.StatusNoContent {
t.Fatalf("status=%d err=%v", status, err)
}
errorValue, _ := received["error"].(map[string]any)
if errorValue["code"] != "upstream_connection_interrupted" || strings.Contains(stringFromAny(errorValue["message"]), "10.42.0.72") {
t.Fatalf("unsafe callback error: %#v", errorValue)
}
}
apps/api/internal/runner/upload.go
+139 -151
View File
@@ -17,25 +17,18 @@ import (
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const defaultServerMainOpenAPIUploadURL = "http://127.0.0.1:3001/v1/files/upload"
const maxGeneratedAssetFetchBytes = 256 << 20
const (
localStaticGeneratedPathPrefix = "/static/generated/"
localStaticUploadedPathPrefix = "/static/uploaded/"
)
type FileUploadPayload struct {
ContentType string
FileName string
@@ -136,14 +129,6 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
return nil, &clients.ClientError{Code: "acceptance_run_inactive", Message: err.Error(), Retryable: false}
}
}
if policy.LocalizeInlineMedia {
next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
if err != nil {
return nil, err
}
redactGeneratedResultRawData(next)
return next, nil
}
if len(data) == 0 && !rawNeedsUpload {
return s.finalizeGeneratedAssets(ctx, taskID, taskKind, result, policy, nil, false, 0)
}
@@ -179,11 +164,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
var channels []store.FileStorageChannel
channelsLoaded := false
if needsUpload || rawNeedsUpload {
if s.directOSS == nil {
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
if err != nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
}
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
if err != nil {
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "object storage configuration is unavailable", Retryable: true}
}
channelsLoaded = true
}
@@ -323,12 +306,12 @@ func (s *Service) finalizeGeneratedAssets(
if !TaskResultHasInlineBinary(next) {
return next, nil
}
persistent, _, err := s.materializeLocalBinaryResult(ctx, taskID, next)
if err != nil {
return nil, err
return nil, &clients.ClientError{
Code: "storage_write_failed",
Message: "generated binary result could not be written to object storage",
StatusCode: http.StatusServiceUnavailable,
Retryable: true,
}
redactGeneratedResultRawData(persistent)
return persistent, nil
}
func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
@@ -355,6 +338,9 @@ func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]
func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) {
switch typed := value.(type) {
case map[string]any:
if payload, contentType, ok := localBufferObjectBytes(typed); ok {
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, contentType, firstNonEmptyString(key, "buffer"), siblings, channels, index)
}
next := make(map[string]any, len(typed))
changed := false
for childKey, childValue := range typed {
@@ -372,6 +358,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
}
return value, false, nil
case []any:
if localBinaryKey(key) {
if payload, ok := bytesFromNumberArray(typed); ok {
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
}
}
next := make([]any, len(typed))
changed := false
for itemIndex, item := range typed {
@@ -388,6 +379,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
return next, true, nil
}
return value, false, nil
case []byte:
if len(typed) == 0 {
return value, false, nil
}
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
case string:
asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind)
if !ok {
@@ -404,19 +400,36 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
}
}
func (s *Service) uploadGeneratedBinaryValue(ctx context.Context, taskID string, taskKind string, payload []byte, contentType string, sourceKey string, siblings map[string]any, channels []store.FileStorageChannel, index *int) (any, bool, error) {
contentType = firstNonEmptyString(contentType, defaultContentTypeForRawMediaKey(sourceKey))
asset := &generatedInlineAsset{
Bytes: payload,
ContentType: contentType,
Kind: mediaKindForAsset(taskKind, siblings, sourceKey, contentType),
SourceKey: sourceKey,
}
upload, resolvedContentType, kind, strategy, err := s.uploadGeneratedAsset(ctx, taskID, asset, *index, channels)
if err != nil {
return nil, false, err
}
*index = *index + 1
return generatedRawMediaReference(asset, upload, resolvedContentType, kind, strategy), true, nil
}
func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) {
raw := strings.TrimSpace(value)
if raw == "" {
return nil, false
}
keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
if !generatedRawDataMediaPayloadKey(key) && !generatedContentTypeIsMedia(contentType) {
if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) {
return nil, false
}
if !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
if !keyLooksLikeMediaPayload && !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
return nil, false
}
payload, payloadContentType, ok, err := inlineMediaPayload(raw, generatedRawDataMediaPayloadKey(key))
payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload)
if err != nil || !ok || len(payload) == 0 {
return nil, false
}
@@ -454,6 +467,18 @@ func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]a
if provider := stringFromAny(channel["provider"]); provider != "" {
ref["storageProvider"] = provider
}
if id := stringFromAny(channel["id"]); id != "" {
ref["storageChannelId"] = id
}
if key := stringFromAny(channel["channelKey"]); key != "" {
ref["storageChannelKey"] = key
}
}
if objectKey := stringFromAny(upload["objectKey"]); objectKey != "" {
ref["objectKey"] = objectKey
}
if accessScope := stringFromAny(upload["accessScope"]); accessScope != "" {
ref["accessScope"] = accessScope
}
out := map[string]any{
"assetRef": ref,
@@ -650,6 +675,9 @@ func removeASCIIWhitespace(value string) string {
}
func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
if s.store == nil {
return defaultGeneratedAssetUploadPolicy(), nil
}
settings, err := s.store.GetFileStorageSettings(ctx)
if err != nil {
if store.IsUndefinedDatabaseObject(err) {
@@ -665,8 +693,6 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
switch policyName {
case store.FileStorageResultUploadPolicyUploadAll:
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
case store.FileStorageResultUploadPolicyUploadNone:
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
default:
return defaultGeneratedAssetUploadPolicy()
}
@@ -682,13 +708,8 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset
Scene: store.FileStorageSceneImageResult,
Source: "ai-gateway",
}
if s.directOSS != nil {
upload, err := s.directOSS.upload(ctx, payload)
return upload, contentType, kind, "direct_aliyun_oss", err
}
if len(channels) == 0 {
upload, err := s.storeFileLocally(payload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
return upload, contentType, kind, "local_static_inline_media", err
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
upload, err := s.uploadFileWithFailover(ctx, payload, channels)
return upload, contentType, kind, "upload_inline_media", err
@@ -708,95 +729,13 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as
Scene: store.FileStorageSceneImageResult,
Source: "ai-gateway",
}
if s.directOSS != nil {
upload, err := s.directOSS.upload(ctx, uploadPayload)
return upload, contentType, kind, "direct_aliyun_oss", err
}
if len(channels) == 0 {
upload, err := s.storeFileLocally(uploadPayload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
return upload, contentType, kind, "local_static_url_media", err
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels)
return upload, contentType, kind, "upload_url_media", err
}
func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string, fallbackStorageDir string, pathPrefix string) (map[string]any, error) {
storageDir = strings.TrimSpace(storageDir)
if storageDir == "" {
storageDir = fallbackStorageDir
}
if err := os.MkdirAll(storageDir, 0o755); err != nil {
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
}
fileName := filepath.Base(strings.TrimSpace(payload.FileName))
if fileName == "" || fileName == "." || fileName == ".." || fileName == string(filepath.Separator) {
kind := generatedAssetKindFromContentType("", payload.ContentType)
fileName = generatedAssetFileName("generated", 0, payload.ContentType, kind)
}
targetPath := filepath.Join(storageDir, fileName)
file, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
}
_, writeErr := file.Write(payload.Bytes)
closeErr := file.Close()
if writeErr != nil {
_ = os.Remove(targetPath)
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: writeErr.Error(), Retryable: true}
}
if closeErr != nil {
_ = os.Remove(targetPath)
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true}
}
expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339)
return map[string]any{
"url": s.localStaticFileURL(fileName, pathPrefix),
"fileName": fileName,
"contentType": payload.ContentType,
"size": len(payload.Bytes),
"expiresAt": expiresAt,
"storageChannel": map[string]any{
"id": "local-static",
"channelKey": "local-static",
"name": "AI Gateway local static storage",
"provider": "local_static",
},
}, nil
}
func (s *Service) localStaticAssetTTLHours() int {
if s.cfg.LocalTempAssetTTLHours <= 0 {
return 24
}
return s.cfg.LocalTempAssetTTLHours
}
func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string {
if strings.TrimSpace(pathPrefix) == "" {
pathPrefix = localStaticUploadedPathPrefix
}
path := pathPrefix + url.PathEscape(filepath.Base(fileName))
baseURL := strings.TrimRight(strings.TrimSpace(s.cfg.PublicBaseURL), "/")
if baseURL == "" {
return path
}
return baseURL + path
}
func localStaticUploadFileName(originalName string, contentType string) string {
baseName := filepath.Base(strings.TrimSpace(originalName))
originalExt := strings.ToLower(filepath.Ext(baseName))
namePart := strings.TrimSuffix(baseName, originalExt)
namePart = sanitizeGeneratedAssetNamePart(namePart)
if namePart == "" {
namePart = "gateway-upload"
}
if len(namePart) > 48 {
namePart = namePart[:48]
}
return fmt.Sprintf("%s-%s%s", namePart, randomHexSuffix(6), uploadFileExtension(contentType, originalExt))
}
func uploadFileExtension(contentType string, fallbackExt string) string {
normalized := normalizeGeneratedContentType(contentType)
if generatedContentTypeIsMedia(normalized) {
@@ -981,46 +920,41 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma
if strings.TrimSpace(payload.Scene) == "" {
payload.Scene = store.FileStorageSceneUpload
}
if s.directOSS != nil && directOSSScene(payload.Scene) {
return s.directOSS.upload(ctx, payload)
}
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
if err != nil {
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
}
if len(channels) == 0 {
payload.FileName = localStaticUploadFileName(payload.FileName, payload.ContentType)
upload, err := s.storeFileLocally(payload, s.cfg.LocalUploadedStorageDir, config.DefaultLocalUploadedStorageDir, localStaticUploadedPathPrefix)
if err != nil {
return nil, err
}
upload["assetStorage"] = map[string]any{
"scene": payload.Scene,
"source": firstNonEmptyString(payload.Source, "ai-gateway-openapi"),
"strategy": "local_static_upload",
}
return upload, nil
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
return s.uploadFileWithFailover(ctx, payload, channels)
}
func (s *Service) activeFileStorageChannels(ctx context.Context, scene string) ([]store.FileStorageChannel, error) {
if s.store == nil {
return nil, nil
channels := make([]store.FileStorageChannel, 0)
if s.directOSS != nil && directOSSScene(scene) {
channels = append(channels, s.directOSS.fileStorageChannel())
}
channels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
if s.store == nil {
return channels, nil
}
storedChannels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
if err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
}
if len(channels) > 0 {
return channels, nil
}
return nil, nil
channels = append(channels, storedChannels...)
sort.SliceStable(channels, func(i, j int) bool {
if channels[i].Priority != channels[j].Priority {
return channels[i].Priority < channels[j].Priority
}
return channels[i].ChannelKey < channels[j].ChannelKey
})
return channels, nil
}
func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUploadPayload, channels []store.FileStorageChannel) (map[string]any, error) {
var lastErr error
for _, channel := range channels {
for index, channel := range channels {
upload, err := s.uploadWithChannelRetries(ctx, payload, channel)
if err == nil {
if s.store != nil {
@@ -1032,25 +966,59 @@ func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUpload
if s.store != nil {
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(ctx), channel.ID, err.Error())
}
if !storageFailureAllowsFailover(channel, err) {
return nil, err
}
if index+1 < len(channels) {
s.observeObjectStorage("failover", channel.Provider, 0, 0)
}
}
if lastErr != nil {
return nil, lastErr
s.observeObjectStorage("all_failed", "", 0, 0)
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "all configured object storage channels failed", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
}
func storageFailureAllowsFailover(channel store.FileStorageChannel, err error) bool {
code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
switch code {
case "upload_source_too_large", "upload_decode_failed", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio":
return false
}
var clientErr *clients.ClientError
if !errors.As(err, &clientErr) {
return true
}
switch clientErr.StatusCode {
case http.StatusRequestEntityTooLarge, http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity:
return false
case http.StatusBadRequest:
// Object-storage 400 responses commonly indicate endpoint/signature
// configuration and should move to the next channel. server-main 400 is
// the existing upload API's request validation result.
return !strings.EqualFold(strings.TrimSpace(channel.Provider), "server_main_openapi")
default:
return true
}
return nil, &clients.ClientError{Code: "upload_no_channel", Message: "no enabled file storage channel", Retryable: false}
}
func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy)
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy, channel.Provider)
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
startedAt := time.Now()
upload, err := s.uploadOnce(ctx, payload, channel)
if err == nil {
s.observeObjectStorage("write_success", channel.Provider, len(payload.Bytes), time.Since(startedAt))
return upload, nil
}
s.observeObjectStorage("write_failure", channel.Provider, 0, time.Since(startedAt))
lastErr = err
if attempt >= maxRetries || !clients.IsRetryable(err) {
break
}
s.observeObjectStorage("retry", channel.Provider, 0, 0)
delay := retryDelayForAttempt(attempt, delays)
if err := sleepWithContext(ctx, delay); err != nil {
return nil, err
@@ -1060,9 +1028,21 @@ func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUplo
}
func (s *Service) uploadOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
if strings.ToLower(strings.TrimSpace(channel.Provider)) != "server_main_openapi" {
return nil, &clients.ClientError{Code: "upload_unsupported_channel", Message: "unsupported file storage channel: " + channel.Provider, Retryable: false}
switch strings.ToLower(strings.TrimSpace(channel.Provider)) {
case "aliyun_oss", "s3":
adapter, err := newObjectStorageAdapter(channel)
if err != nil {
return nil, err
}
return adapter.put(ctx, payload)
case "server_main_openapi":
return s.uploadServerMainOnce(ctx, payload, channel)
default:
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported file storage channel", Retryable: false}
}
}
func (s *Service) uploadServerMainOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
uploadURL := strings.TrimSpace(channel.UploadURL)
if uploadURL == "" {
uploadURL = defaultServerMainOpenAPIUploadURL
@@ -1700,9 +1680,17 @@ func uniqueStringList(values []string) []string {
return out
}
func uploadRetrySchedule(policy map[string]any) (int, []time.Duration) {
func uploadRetrySchedule(policy map[string]any, providers ...string) (int, []time.Duration) {
provider := ""
if len(providers) > 0 {
provider = providers[0]
}
if policy == nil {
policy = defaultUploadRetryPolicy()
if strings.EqualFold(provider, "aliyun_oss") || strings.EqualFold(provider, "s3") {
policy = map[string]any{"enabled": true, "maxRetries": 2, "backoffSeconds": []any{0.25, 1.0}}
} else {
policy = defaultUploadRetryPolicy()
}
}
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
return 0, nil
@@ -1725,9 +1713,9 @@ func uploadRetryDelays(value any) []time.Duration {
}
delays := make([]time.Duration, 0, len(items))
for _, item := range items {
seconds := int(floatFromAny(item))
seconds := floatFromAny(item)
if seconds > 0 {
delays = append(delays, time.Duration(seconds)*time.Second)
delays = append(delays, time.Duration(seconds*float64(time.Second)))
}
}
return delays
apps/api/internal/runner/upload_test.go
+59 -139
View File
@@ -7,11 +7,11 @@ import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -227,38 +227,6 @@ func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
}
}
func TestGeneratedAssetDecisionPreservesInlineWhenPolicyUploadNone(t *testing.T) {
item := map[string]any{
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
}
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline != nil || decision.URL != nil {
t.Fatalf("upload_none should not transfer inline payloads: %+v", decision)
}
if len(decision.StripKeys) != 0 {
t.Fatalf("upload_none should preserve b64_json for the caller: %+v", decision.StripKeys)
}
}
func TestGeneratedAssetDecisionPreservesInlineAlongsideURLWhenPolicyUploadNone(t *testing.T) {
item := map[string]any{
"url": "https://cdn.example.com/generated.png",
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
}
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decision.Inline != nil || decision.URL != nil || len(decision.StripKeys) != 0 {
t.Fatalf("upload_none should preserve the upstream URL and base64 fields unchanged: %+v", decision)
}
}
func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
tests := []struct {
name string
@@ -276,9 +244,9 @@ func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true, PreserveInlineMedia: false},
},
{
name: "upload none",
name: "legacy upload none becomes default",
policyName: store.FileStorageResultUploadPolicyUploadNone,
want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true},
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: false, PreserveInlineMedia: false},
},
}
@@ -330,8 +298,10 @@ func TestAcceptanceGeneratedMediaAllowsOnlyExactEmulatorOrigin(t *testing.T) {
}
func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
encoded := base64.StdEncoding.EncodeToString(payload)
result := map[string]any{
@@ -348,7 +318,7 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
"images.edits",
result,
defaultGeneratedAssetUploadPolicy(),
nil,
channels,
true,
0,
)
@@ -363,15 +333,8 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
if !ok {
t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested)
}
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-nested-binary-01-") {
t.Fatalf("unexpected local static URL: %s", urlValue)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("read generated storage: %v", err)
}
if len(entries) != 1 {
t.Fatalf("expected one localized result file, got %d", len(entries))
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") {
t.Fatalf("unexpected object storage URL: %s", urlValue)
}
}
@@ -405,9 +368,8 @@ func TestGeneratedAssetFileNameIsUniqueAndTyped(t *testing.T) {
}
}
func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
func TestUploadGeneratedAssetFailsWithoutObjectStorage(t *testing.T) {
service := &Service{}
payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
asset := &generatedInlineAsset{
Bytes: payload,
@@ -416,44 +378,14 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
SourceKey: "b64_json",
}
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
if err != nil {
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
if clients.ErrorCode(err) != "storage_write_failed" {
t.Fatalf("unexpected error: %v", err)
}
if contentType != "image/png" || kind != "image" || strategy != "local_static_inline_media" {
t.Fatalf("unexpected local upload metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
}
urlValue := stringFromAny(upload["url"])
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-123-01-") || !strings.HasSuffix(urlValue, ".png") {
t.Fatalf("unexpected local static URL: %s", urlValue)
}
expiresAt, err := time.Parse(time.RFC3339, stringFromAny(upload["expiresAt"]))
if err != nil {
t.Fatalf("local static upload should expose expiresAt: %+v", upload)
}
remaining := time.Until(expiresAt)
if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour+time.Minute {
t.Fatalf("local static upload should expire after one day, remaining=%s", remaining)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("failed to read local static dir: %v", err)
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
t.Fatalf("expected one PNG file in local static dir, got %+v", entries)
}
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
if err != nil {
t.Fatalf("failed to read local static file: %v", err)
}
if !bytes.Equal(stored, payload) {
t.Fatalf("stored payload does not match source payload")
}
}
func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
func TestUploadGeneratedAudioFailsWithoutObjectStorage(t *testing.T) {
service := &Service{}
asset := &generatedInlineAsset{
Bytes: []byte("inline audio payload"),
ContentType: "audio/mpeg",
@@ -461,32 +393,17 @@ func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
SourceKey: "content",
}
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
if err != nil {
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
if clients.ErrorCode(err) != "storage_write_failed" {
t.Fatalf("unexpected error: %v", err)
}
if contentType != "audio/mpeg" || kind != "audio" || strategy != "local_static_inline_media" {
t.Fatalf("unexpected local audio metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
}
urlValue := stringFromAny(upload["url"])
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-tts-01-") || !strings.HasSuffix(urlValue, ".mp3") {
t.Fatalf("unexpected local audio URL: %s", urlValue)
}
if stringFromAny(upload["expiresAt"]) == "" {
t.Fatalf("local audio static upload should expose expiresAt: %+v", upload)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("failed to read local static dir: %v", err)
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".mp3") {
t.Fatalf("expected one MP3 file in local static dir, got %+v", entries)
}
}
func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
raw := map[string]any{
"candidates": []any{
@@ -506,7 +423,7 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
}
index := 0
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), nil, &index)
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
if err != nil {
t.Fatalf("upload raw media: %v", err)
}
@@ -528,22 +445,47 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) {
t.Fatalf("unexpected asset ref: %+v", ref)
}
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-raw-01-") || !strings.HasSuffix(urlValue, ".png") {
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") || !strings.HasSuffix(urlValue, ".png") {
t.Fatalf("unexpected raw media URL: %s", urlValue)
}
if inlineData["data"] == base64.StdEncoding.EncodeToString(payload) {
t.Fatal("raw inlineData still contains base64 payload")
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("read generated storage: %v", err)
}
func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-buffer-result")}
raw := map[string]any{
"buffer": map[string]any{
"type": "Buffer", "mimeType": "image/png", "data": []any{float64(0x89), float64('P'), float64('N'), float64('G')},
},
"audio_bytes": []any{float64('I'), float64('D'), float64('3')},
"direct": []byte("direct bytes"),
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
t.Fatalf("expected one generated PNG, got %+v", entries)
index := 0
uploaded, changed, err := service.uploadGeneratedRawMediaValue(t.Context(), "task-buffer", "images.generations", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
if err != nil {
t.Fatal(err)
}
if !changed || index != 3 {
t.Fatalf("changed=%v uploads=%d", changed, index)
}
next := uploaded.(map[string]any)
for _, key := range []string{"buffer", "audio_bytes", "direct"} {
item, ok := next[key].(map[string]any)
if !ok || item["assetRef"] == nil || item["upload"] == nil {
t.Fatalf("%s was not objectified: %#v", key, next[key])
}
}
if TaskResultHasInlineBinary(next) {
t.Fatalf("objectified result still contains inline binary: %#v", next)
}
}
func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
func TestUploadFileFailsWithoutObjectStorageAndCreatesNoLocalFile(t *testing.T) {
storageDir := t.TempDir()
service := &Service{cfg: config.Config{
LocalUploadedStorageDir: storageDir,
@@ -552,43 +494,21 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
}}
payload := []byte("%PDF-1.4")
upload, err := service.UploadFile(context.Background(), FileUploadPayload{
_, err := service.UploadFile(context.Background(), FileUploadPayload{
Bytes: payload,
ContentType: "application/pdf",
FileName: "用户文件.png",
Source: "playground",
})
if err != nil {
if clients.ErrorCode(err) != "storage_write_failed" {
t.Fatalf("unexpected error: %v", err)
}
urlValue := stringFromAny(upload["url"])
if !strings.HasPrefix(urlValue, "/static/uploaded/") || !strings.HasSuffix(urlValue, ".pdf") {
t.Fatalf("unexpected uploaded local static URL: %s", urlValue)
}
if stringFromAny(upload["expiresAt"]) == "" {
t.Fatalf("local uploaded static file should expose expiresAt: %+v", upload)
}
storageChannel, _ := upload["storageChannel"].(map[string]any)
if stringFromAny(storageChannel["provider"]) != "local_static" {
t.Fatalf("expected local static provider metadata, got %+v", upload["storageChannel"])
}
assetStorage, _ := upload["assetStorage"].(map[string]any)
if stringFromAny(assetStorage["strategy"]) != "local_static_upload" || stringFromAny(assetStorage["scene"]) != store.FileStorageSceneUpload {
t.Fatalf("unexpected upload asset storage metadata: %+v", assetStorage)
}
entries, err := os.ReadDir(storageDir)
if err != nil {
t.Fatalf("failed to read uploaded static dir: %v", err)
}
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".pdf") {
t.Fatalf("expected one PDF file in uploaded static dir, got %+v", entries)
}
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
if err != nil {
t.Fatalf("failed to read uploaded static file: %v", err)
}
if !bytes.Equal(stored, payload) {
t.Fatalf("stored uploaded payload does not match source payload")
if len(entries) != 0 {
t.Fatalf("unexpected local files: %+v", entries)
}
}
apps/api/internal/securityevents/metrics.go
+82
View File
@@ -9,6 +9,7 @@ import (
"sync/atomic"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -86,6 +87,19 @@ type Metrics struct {
candidateDisabledSkipped atomic.Uint64
candidateAllFullQueued atomic.Uint64
candidateRoutingOther atomic.Uint64
storageAliyunWrites atomic.Uint64
storageAliyunFailures atomic.Uint64
storageAliyunWriteNanos atomic.Uint64
storageS3Writes atomic.Uint64
storageS3Failures atomic.Uint64
storageS3WriteNanos atomic.Uint64
storageServerMainWrites atomic.Uint64
storageServerMainFailures atomic.Uint64
storageServerMainWriteNanos atomic.Uint64
storageChannelRetries atomic.Uint64
storageChannelFailovers atomic.Uint64
storageAllChannelsFailed atomic.Uint64
storageObjectifiedBytes atomic.Uint64
taskAdmissionWaitBuckets [11]atomic.Uint64
taskAdmissionWaitMicros atomic.Uint64
}
@@ -265,6 +279,40 @@ func (m *Metrics) ObserveCandidateRouting(reason string) {
}
}
func (m *Metrics) ObserveObjectStorage(event string, provider string, bytes int64, duration time.Duration) {
provider = strings.ToLower(strings.TrimSpace(provider))
if duration < 0 {
duration = 0
}
switch event {
case "write_success", "write_failure":
var writes, failures, nanos *atomic.Uint64
switch provider {
case "aliyun_oss":
writes, failures, nanos = &m.storageAliyunWrites, &m.storageAliyunFailures, &m.storageAliyunWriteNanos
case "s3":
writes, failures, nanos = &m.storageS3Writes, &m.storageS3Failures, &m.storageS3WriteNanos
case "server_main_openapi":
writes, failures, nanos = &m.storageServerMainWrites, &m.storageServerMainFailures, &m.storageServerMainWriteNanos
default:
return
}
writes.Add(1)
nanos.Add(uint64(duration))
if event == "write_failure" {
failures.Add(1)
} else if bytes > 0 {
m.storageObjectifiedBytes.Add(uint64(bytes))
}
case "retry":
m.storageChannelRetries.Add(1)
case "failover":
m.storageChannelFailovers.Add(1)
case "all_failed":
m.storageAllChannelsFailed.Add(1)
}
}
func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) {
if wait < 0 {
wait = 0
@@ -471,6 +519,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
{"all_full_queued", m.candidateAllFullQueued.Load()},
{"other", m.candidateRoutingOther.Load()},
})
storageProtocolCounters(w, "easyai_gateway_storage_write_attempts_total", "Object storage write attempts by bounded protocol.",
m.storageAliyunWrites.Load(), m.storageS3Writes.Load(), m.storageServerMainWrites.Load())
storageProtocolCounters(w, "easyai_gateway_storage_write_failures_total", "Object storage write failures by bounded protocol.",
m.storageAliyunFailures.Load(), m.storageS3Failures.Load(), m.storageServerMainFailures.Load())
storageProtocolDurationCounters(w,
m.storageAliyunWriteNanos.Load(), m.storageS3WriteNanos.Load(), m.storageServerMainWriteNanos.Load())
plainCounter(w, "easyai_gateway_storage_channel_retries_total", "Retries performed within the current storage channel.", m.storageChannelRetries.Load())
plainCounter(w, "easyai_gateway_storage_channel_failovers_total", "Switches to the next configured storage channel.", m.storageChannelFailovers.Load())
plainCounter(w, "easyai_gateway_storage_all_channels_failed_total", "Storage writes where every eligible channel failed.", m.storageAllChannelsFailed.Load())
plainCounter(w, "easyai_gateway_storage_objectified_bytes_total", "Binary bytes successfully written through storage channels.", m.storageObjectifiedBytes.Load())
publicErrorCounters(w, publicerror.MetricSnapshot())
platformModelRateLimitUtilizationGauges(w, modelRateLimits)
plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections))
plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections))
@@ -567,6 +626,29 @@ func plainFloatGauge(w http.ResponseWriter, name, help string, value float64) {
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value)
}
func storageProtocolCounters(w http.ResponseWriter, name, help string, aliyunOSS, s3, serverMain uint64) {
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n", name, help, name)
fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %d\n", name, aliyunOSS)
fmt.Fprintf(w, "%s{protocol=\"s3\"} %d\n", name, s3)
fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %d\n", name, serverMain)
}
func storageProtocolDurationCounters(w http.ResponseWriter, aliyunOSS, s3, serverMain uint64) {
const name = "easyai_gateway_storage_write_duration_seconds_total"
fmt.Fprintf(w, "# HELP %s Total time spent in object storage write attempts by bounded protocol.\n# TYPE %s counter\n", name, name)
fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %.6f\n", name, float64(aliyunOSS)/float64(time.Second))
fmt.Fprintf(w, "%s{protocol=\"s3\"} %.6f\n", name, float64(s3)/float64(time.Second))
fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %.6f\n", name, float64(serverMain)/float64(time.Second))
}
func publicErrorCounters(w http.ResponseWriter, values []publicerror.MetricCount) {
const name = "easyai_gateway_public_errors_total"
fmt.Fprintf(w, "# HELP %s Public error responses by bounded stable code.\n# TYPE %s counter\n", name, name)
for _, value := range values {
fmt.Fprintf(w, "%s{code=\"%s\"} %d\n", name, value.Code, value.Count)
}
}
func platformModelRateLimitUtilizationGauges(w http.ResponseWriter, statuses []store.ModelRateLimitStatus) {
const name = "easyai_gateway_platform_model_rate_limit_utilization"
fmt.Fprintf(w, "# HELP %s Cluster-wide platform model quota utilization.\n# TYPE %s gauge\n", name, name)
apps/api/internal/securityevents/metrics_test.go
+12
View File
@@ -52,6 +52,11 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
metrics.ObserveCandidateRouting("cooldown_skipped")
metrics.ObserveCandidateRouting("disabled_skipped")
metrics.ObserveCandidateRouting("all_full_queued")
metrics.ObserveObjectStorage("write_success", "aliyun_oss", 128, 25*time.Millisecond)
metrics.ObserveObjectStorage("write_failure", "s3", 0, 10*time.Millisecond)
metrics.ObserveObjectStorage("retry", "s3", 0, 0)
metrics.ObserveObjectStorage("failover", "s3", 0, 0)
metrics.ObserveObjectStorage("all_failed", "", 0, 0)
metrics.ObserveAsyncWorkerResize("success")
metrics.ObserveConcurrencyLeaseRenewal("success")
metrics.ObserveConcurrencyLeaseRenewal("lost")
@@ -98,6 +103,13 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
`easyai_gateway_candidate_routing_total{outcome="cooldown_skipped"} 1`,
`easyai_gateway_candidate_routing_total{outcome="disabled_skipped"} 1`,
`easyai_gateway_candidate_routing_total{outcome="all_full_queued"} 1`,
`easyai_gateway_storage_write_attempts_total{protocol="aliyun_oss"} 1`,
`easyai_gateway_storage_write_failures_total{protocol="s3"} 1`,
`easyai_gateway_storage_write_duration_seconds_total{protocol="aliyun_oss"} 0.025000`,
`easyai_gateway_storage_channel_retries_total 1`,
`easyai_gateway_storage_channel_failovers_total 1`,
`easyai_gateway_storage_all_channels_failed_total 1`,
`easyai_gateway_storage_objectified_bytes_total 128`,
`easyai_gateway_platform_model_rate_limit_utilization{platform_model_id="platform-model-1",metric="concurrent"} 0.750000`,
`easyai_gateway_async_worker_resizes_total{outcome="success"} 1`,
`easyai_gateway_concurrency_lease_renewals_total{outcome="success"} 1`,
apps/api/internal/store/file_storage_channels.go
+77 -20
View File
@@ -39,6 +39,9 @@ type FileStorageChannel struct {
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl,omitempty"`
APIKey string `json:"-"`
AccessKeyID string `json:"-"`
AccessKeySecret string `json:"-"`
SessionToken string `json:"-"`
CredentialsPreview map[string]any `json:"credentialsPreview,omitempty"`
Scenes []string `json:"scenes,omitempty"`
Config map[string]any `json:"config,omitempty"`
@@ -53,16 +56,21 @@ type FileStorageChannel struct {
}
type FileStorageChannelInput struct {
ChannelKey string `json:"channelKey"`
Name string `json:"name"`
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl"`
APIKey *string `json:"apiKey"`
Scenes []string `json:"scenes"`
Config map[string]any `json:"config"`
RetryPolicy map[string]any `json:"retryPolicy"`
Priority int `json:"priority"`
Status string `json:"status"`
ChannelKey string `json:"channelKey"`
Name string `json:"name"`
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl"`
APIKey *string `json:"apiKey"`
AccessKey *string `json:"accessKey"`
AccessKeyID *string `json:"accessKeyId"`
AccessKeySecret *string `json:"accessKeySecret"`
SecretKey *string `json:"secretKey"`
SessionToken *string `json:"sessionToken"`
Scenes []string `json:"scenes"`
Config map[string]any `json:"config"`
RetryPolicy map[string]any `json:"retryPolicy"`
Priority int `json:"priority"`
Status string `json:"status"`
}
type FileStorageSettings struct {
@@ -156,11 +164,18 @@ WHERE id = $1::uuid
AND deleted_at IS NULL`, id))
}
func (s *Store) GetFileStorageChannelByKey(ctx context.Context, channelKey string) (FileStorageChannel, error) {
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
SELECT `+fileStorageChannelColumns+`
FROM file_storage_channels
WHERE channel_key = $1`, strings.TrimSpace(channelKey)))
}
func (s *Store) CreateFileStorageChannel(ctx context.Context, input FileStorageChannelInput) (FileStorageChannel, error) {
input = normalizeFileStorageChannelInput(input)
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
config, _ := json.Marshal(configFromFileStorageInput(input))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider))
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
INSERT INTO file_storage_channels (
@@ -182,10 +197,10 @@ RETURNING `+fileStorageChannelColumns,
func (s *Store) UpdateFileStorageChannel(ctx context.Context, id string, input FileStorageChannelInput) (FileStorageChannel, error) {
input = normalizeFileStorageChannelInput(input)
replaceCredentials := input.APIKey != nil
replaceCredentials := fileStorageInputReplacesCredentials(input)
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
config, _ := json.Marshal(configFromFileStorageInput(input))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider))
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
UPDATE file_storage_channels
@@ -193,7 +208,7 @@ SET channel_key = $2,
name = $3,
provider = $4,
upload_url = NULLIF($5, ''),
credentials = CASE WHEN $6::boolean THEN $7 ELSE credentials END,
credentials = CASE WHEN $6::boolean THEN credentials || $7::jsonb ELSE credentials END,
config = $8,
retry_policy = $9,
priority = $10,
@@ -307,6 +322,9 @@ func scanFileStorageChannel(scanner fileStorageChannelScanner) (FileStorageChann
}
credentialObject := decodeObject(credentials)
item.APIKey = stringFromObject(credentialObject, "apiKey")
item.AccessKeyID = stringFromObject(credentialObject, "accessKeyId")
item.AccessKeySecret = stringFromObject(credentialObject, "accessKeySecret")
item.SessionToken = stringFromObject(credentialObject, "sessionToken")
item.CredentialsPreview = maskCredentialsPreview(credentials)
configObject := decodeObject(config)
item.Scenes = fileStorageScenesFromConfig(configObject)
@@ -324,6 +342,17 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage
apiKey := strings.TrimSpace(*input.APIKey)
input.APIKey = &apiKey
}
if input.AccessKeyID == nil && input.AccessKey != nil {
input.AccessKeyID = input.AccessKey
}
if input.AccessKeySecret == nil && input.SecretKey != nil {
input.AccessKeySecret = input.SecretKey
}
for _, value := range []*string{input.AccessKeyID, input.AccessKeySecret, input.SessionToken} {
if value != nil {
*value = strings.TrimSpace(*value)
}
}
input.Scenes = normalizeFileStorageScenes(input.Scenes)
input.Status = strings.ToLower(strings.TrimSpace(input.Status))
if input.Provider == "" {
@@ -342,11 +371,24 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage
}
func credentialsFromFileStorageInput(input FileStorageChannelInput) map[string]any {
apiKey := fileStorageInputAPIKey(input)
if apiKey == "" {
return map[string]any{}
credentials := map[string]any{}
if input.APIKey != nil {
credentials["apiKey"] = fileStorageInputAPIKey(input)
}
return map[string]any{"apiKey": apiKey}
if input.AccessKeyID != nil {
credentials["accessKeyId"] = strings.TrimSpace(*input.AccessKeyID)
}
if input.AccessKeySecret != nil {
credentials["accessKeySecret"] = strings.TrimSpace(*input.AccessKeySecret)
}
if input.SessionToken != nil {
credentials["sessionToken"] = strings.TrimSpace(*input.SessionToken)
}
return credentials
}
func fileStorageInputReplacesCredentials(input FileStorageChannelInput) bool {
return input.APIKey != nil || input.AccessKeyID != nil || input.AccessKeySecret != nil || input.AccessKey != nil || input.SecretKey != nil || input.SessionToken != nil
}
func fileStorageInputAPIKey(input FileStorageChannelInput) string {
@@ -549,7 +591,10 @@ func NormalizeFileStorageResultUploadPolicy(policy string) string {
case "upload_all", "all", "always", "all_upload":
return FileStorageResultUploadPolicyUploadAll
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
return FileStorageResultUploadPolicyUploadNone
// Local result persistence is no longer a supported write path. Preserve
// compatibility with historical settings by normalizing them to the safe
// object-storage policy.
return FileStorageResultUploadPolicyDefault
default:
return FileStorageResultUploadPolicyDefault
}
@@ -601,10 +646,22 @@ func defaultFileStorageScenes() []string {
return []string{FileStorageSceneUpload, FileStorageSceneImageResult, FileStorageSceneRequestAsset}
}
func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any) map[string]any {
func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any, providers ...string) map[string]any {
if len(policy) > 0 {
return policy
}
provider := ""
if len(providers) > 0 {
provider = strings.ToLower(strings.TrimSpace(providers[0]))
}
if provider == "aliyun_oss" || provider == "s3" {
return map[string]any{
"enabled": true,
"maxRetries": 2,
"backoffSeconds": []any{0.25, 1.0},
"strategy": "exponential",
}
}
return map[string]any{
"enabled": true,
"maxRetries": 3,
apps/api/internal/store/file_storage_channels_test.go
+31
View File
@@ -1,7 +1,9 @@
package store
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
@@ -15,3 +17,32 @@ func TestDefaultFileStorageScenesIncludeCrossNodeRequestAssets(t *testing.T) {
t.Fatalf("default file storage scenes = %#v, want %#v", got, want)
}
}
func TestFileStorageChannelJSONNeverIncludesCredentials(t *testing.T) {
channel := FileStorageChannel{
Provider: "s3", APIKey: "api-key-secret", AccessKeyID: "access-key-id",
AccessKeySecret: "access-key-secret", SessionToken: "session-token",
CredentialsPreview: map[string]any{"accessKeyId": "acce…-id", "accessKeySecret": "acce…cret"},
}
payload, err := json.Marshal(channel)
if err != nil {
t.Fatal(err)
}
encoded := string(payload)
for _, secret := range []string{"api-key-secret", "access-key-id", "access-key-secret", "session-token"} {
if strings.Contains(encoded, secret) {
t.Fatalf("channel JSON leaked credential %q: %s", secret, encoded)
}
}
if !strings.Contains(encoded, "credentialsPreview") {
t.Fatalf("channel JSON omitted masked preview: %s", encoded)
}
}
func TestLegacyUploadNoneNormalizesToObjectStorageDefault(t *testing.T) {
for _, value := range []string{"upload_none", "none", "never", "skip_all"} {
if got := NormalizeFileStorageResultUploadPolicy(value); got != FileStorageResultUploadPolicyDefault {
t.Fatalf("policy %q normalized to %q", value, got)
}
}
}
apps/api/internal/store/postgres.go
+122 -101
View File
@@ -14,6 +14,7 @@ import (
"unicode"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
@@ -568,73 +569,74 @@ type CreateTaskResult struct {
}
type GatewayTask struct {
ID string `json:"id"`
ExternalTaskID string `json:"externalTaskId,omitempty"`
Kind string `json:"kind"`
RunMode string `json:"runMode"`
AcceptanceRunID string `json:"acceptanceRunId,omitempty"`
UserID string `json:"userId"`
GatewayUserID string `json:"gatewayUserId,omitempty"`
UserSource string `json:"userSource,omitempty"`
GatewayTenantID string `json:"gatewayTenantId,omitempty"`
TenantID string `json:"tenantId,omitempty"`
TenantKey string `json:"tenantKey,omitempty"`
APIKeyID string `json:"apiKeyId,omitempty"`
APIKeyName string `json:"apiKeyName,omitempty"`
APIKeyPrefix string `json:"apiKeyPrefix,omitempty"`
UserGroupID string `json:"userGroupId,omitempty"`
UserGroupKey string `json:"userGroupKey,omitempty"`
Model string `json:"model"`
ModelType string `json:"modelType,omitempty"`
RequestedModel string `json:"requestedModel,omitempty"`
ResolvedModel string `json:"resolvedModel,omitempty"`
RequestID string `json:"requestId,omitempty"`
ConversationID string `json:"conversationId,omitempty"`
NewMessageCount int `json:"newMessageCount,omitempty"`
Request map[string]any `json:"request,omitempty"`
AsyncMode bool `json:"asyncMode"`
RiverJobID int64 `json:"riverJobId,omitempty"`
Status string `json:"status"`
QueueKey string `json:"-"`
Priority int `json:"-"`
Cancellable *bool `json:"cancellable,omitempty"`
Submitted *bool `json:"submitted,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attemptCount"`
RemoteTaskID string `json:"remoteTaskId,omitempty"`
RemoteTaskPayload map[string]any `json:"-"`
Result map[string]any `json:"result,omitempty"`
Billings []any `json:"billings,omitempty"`
Usage map[string]any `json:"usage"`
Metrics map[string]any `json:"metrics"`
BillingSummary map[string]any `json:"billingSummary"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
BillingVersion string `json:"billingVersion"`
BillingStatus string `json:"billingStatus"`
BillingCurrency string `json:"billingCurrency"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
ReservationAmount float64 `json:"reservationAmount"`
ExecutionToken string `json:"-"`
ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"`
BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"`
BillingSettledAt string `json:"billingSettledAt,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
FinishedAt string `json:"finishedAt,omitempty"`
Error string `json:"error,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"`
CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"`
CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"`
CompatibilitySubmitHTTPStatus int `json:"compatibilitySubmitHttpStatus,omitempty"`
CompatibilitySubmitHeaders map[string]any `json:"compatibilitySubmitHeaders,omitempty"`
CompatibilitySubmitBody map[string]any `json:"compatibilitySubmitBody,omitempty"`
Attempts []TaskAttempt `json:"attempts,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
ExternalTaskID string `json:"externalTaskId,omitempty"`
Kind string `json:"kind"`
RunMode string `json:"runMode"`
AcceptanceRunID string `json:"acceptanceRunId,omitempty"`
UserID string `json:"userId"`
GatewayUserID string `json:"gatewayUserId,omitempty"`
UserSource string `json:"userSource,omitempty"`
GatewayTenantID string `json:"gatewayTenantId,omitempty"`
TenantID string `json:"tenantId,omitempty"`
TenantKey string `json:"tenantKey,omitempty"`
APIKeyID string `json:"apiKeyId,omitempty"`
APIKeyName string `json:"apiKeyName,omitempty"`
APIKeyPrefix string `json:"apiKeyPrefix,omitempty"`
UserGroupID string `json:"userGroupId,omitempty"`
UserGroupKey string `json:"userGroupKey,omitempty"`
Model string `json:"model"`
ModelType string `json:"modelType,omitempty"`
RequestedModel string `json:"requestedModel,omitempty"`
ResolvedModel string `json:"resolvedModel,omitempty"`
RequestID string `json:"requestId,omitempty"`
ConversationID string `json:"conversationId,omitempty"`
NewMessageCount int `json:"newMessageCount,omitempty"`
Request map[string]any `json:"request,omitempty"`
AsyncMode bool `json:"asyncMode"`
RiverJobID int64 `json:"riverJobId,omitempty"`
Status string `json:"status"`
QueueKey string `json:"-"`
Priority int `json:"-"`
Cancellable *bool `json:"cancellable,omitempty"`
Submitted *bool `json:"submitted,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attemptCount"`
RemoteTaskID string `json:"remoteTaskId,omitempty"`
RemoteTaskPayload map[string]any `json:"-"`
Result map[string]any `json:"result,omitempty"`
Billings []any `json:"billings,omitempty"`
Usage map[string]any `json:"usage"`
Metrics map[string]any `json:"metrics"`
BillingSummary map[string]any `json:"billingSummary"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
BillingVersion string `json:"billingVersion"`
BillingStatus string `json:"billingStatus"`
BillingCurrency string `json:"billingCurrency"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
ReservationAmount float64 `json:"reservationAmount"`
ExecutionToken string `json:"-"`
ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"`
BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"`
BillingSettledAt string `json:"billingSettledAt,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
FinishedAt string `json:"finishedAt,omitempty"`
Error string `json:"error,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
PublicError *publicerror.Error `json:"publicError,omitempty"`
CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"`
CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"`
CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"`
CompatibilitySubmitHTTPStatus int `json:"compatibilitySubmitHttpStatus,omitempty"`
CompatibilitySubmitHeaders map[string]any `json:"compatibilitySubmitHeaders,omitempty"`
CompatibilitySubmitBody map[string]any `json:"compatibilitySubmitBody,omitempty"`
Attempts []TaskAttempt `json:"attempts,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
const gatewayTaskColumns = `
@@ -656,7 +658,7 @@ COALESCE(reservation_amount, 0)::float8, COALESCE(execution_token::text, ''),
COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''),
COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''),
COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''),
COALESCE(error_code, ''), COALESCE(error_message, ''),
COALESCE(error_code, ''), COALESCE(error_message, ''), COALESCE(public_error, '{}'::jsonb),
COALESCE(compatibility_protocol, ''), COALESCE(compatibility_public_id, ''),
COALESCE(compatibility_source_protocol, ''), COALESCE(compatibility_submit_http_status, 0),
COALESCE(compatibility_submit_headers, '{}'::jsonb), COALESCE(compatibility_submit_body, '{}'::jsonb),
@@ -679,39 +681,40 @@ type TaskEvent struct {
}
type TaskAttempt struct {
ID string `json:"id"`
TaskID string `json:"taskId"`
AttemptNo int `json:"attemptNo"`
PlatformID string `json:"platformId,omitempty"`
PlatformName string `json:"platformName,omitempty"`
Provider string `json:"provider,omitempty"`
PlatformModelID string `json:"platformModelId,omitempty"`
ModelName string `json:"modelName,omitempty"`
ProviderModelName string `json:"providerModelName,omitempty"`
ModelAlias string `json:"modelAlias,omitempty"`
ModelType string `json:"modelType,omitempty"`
ClientID string `json:"clientId,omitempty"`
QueueKey string `json:"queueKey"`
Status string `json:"status"`
Retryable bool `json:"retryable"`
Simulated bool `json:"simulated"`
RequestID string `json:"requestId,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Usage map[string]any `json:"usage,omitempty"`
Metrics map[string]any `json:"metrics,omitempty"`
RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"`
ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"`
UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
StartedAt time.Time `json:"startedAt"`
FinishedAt string `json:"finishedAt,omitempty"`
ID string `json:"id"`
TaskID string `json:"taskId"`
AttemptNo int `json:"attemptNo"`
PlatformID string `json:"platformId,omitempty"`
PlatformName string `json:"platformName,omitempty"`
Provider string `json:"provider,omitempty"`
PlatformModelID string `json:"platformModelId,omitempty"`
ModelName string `json:"modelName,omitempty"`
ProviderModelName string `json:"providerModelName,omitempty"`
ModelAlias string `json:"modelAlias,omitempty"`
ModelType string `json:"modelType,omitempty"`
ClientID string `json:"clientId,omitempty"`
QueueKey string `json:"queueKey"`
Status string `json:"status"`
Retryable bool `json:"retryable"`
Simulated bool `json:"simulated"`
RequestID string `json:"requestId,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Usage map[string]any `json:"usage,omitempty"`
Metrics map[string]any `json:"metrics,omitempty"`
RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"`
ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"`
UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
PublicError *publicerror.Error `json:"publicError,omitempty"`
StartedAt time.Time `json:"startedAt"`
FinishedAt string `json:"finishedAt,omitempty"`
}
type TaskParamPreprocessingLog struct {
@@ -2240,6 +2243,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
var remoteTaskPayloadBytes []byte
var compatibilitySubmitHeadersBytes []byte
var compatibilitySubmitBodyBytes []byte
var publicErrorBytes []byte
if err := scanner.Scan(
&task.ID,
&task.ExternalTaskID,
@@ -2295,6 +2299,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
&task.Error,
&task.ErrorCode,
&task.ErrorMessage,
&publicErrorBytes,
&task.CompatibilityProtocol,
&task.CompatibilityPublicID,
&task.CompatibilitySourceProtocol,
@@ -2317,9 +2322,25 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
task.PricingSnapshot = decodeObject(pricingSnapshotBytes)
task.CompatibilitySubmitHeaders = decodeObject(compatibilitySubmitHeadersBytes)
task.CompatibilitySubmitBody = decodeObject(compatibilitySubmitBodyBytes)
if len(publicErrorBytes) > 0 {
var snapshot publicerror.Error
if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" {
task.PublicError = &snapshot
}
}
return task, nil
}
func encodePublicErrorSnapshot(code string, message string, status int, retryable bool, requestID string, taskID string) []byte {
if strings.TrimSpace(code) == "" && strings.TrimSpace(message) == "" {
return []byte("null")
}
snapshot := publicerror.FromFields(code, message, status, retryable)
snapshot = publicerror.WithIDs(snapshot, requestID, taskID)
payload, _ := json.Marshal(snapshot)
return payload
}
func (s *Store) ListTaskEvents(ctx context.Context, taskID string) ([]TaskEvent, error) {
rows, err := s.pool.Query(ctx, `
SELECT id::text, task_id::text, seq, event_type, COALESCE(status, ''), COALESCE(phase, ''),
apps/api/internal/store/request_assets.go
+47 -21
View File
@@ -9,33 +9,43 @@ import (
)
type RequestAsset struct {
ID string `json:"id"`
SHA256 string `json:"sha256"`
ContentType string `json:"contentType"`
ByteSize int64 `json:"byteSize"`
URL string `json:"url"`
StorageProvider string `json:"storageProvider"`
LocalPath string `json:"localPath,omitempty"`
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
RefCount int `json:"refCount"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
SHA256 string `json:"sha256"`
ContentType string `json:"contentType"`
ByteSize int64 `json:"byteSize"`
URL string `json:"url"`
StorageProvider string `json:"storageProvider"`
StorageChannelID string `json:"storageChannelId,omitempty"`
StorageChannelKey string `json:"storageChannelKey,omitempty"`
ObjectKey string `json:"objectKey,omitempty"`
AccessScope string `json:"accessScope,omitempty"`
LocalPath string `json:"localPath,omitempty"`
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
RefCount int `json:"refCount"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type RequestAssetInput struct {
SHA256 string
ContentType string
ByteSize int64
URL string
StorageProvider string
LocalPath string
ExpiresAt *time.Time
SHA256 string
ContentType string
ByteSize int64
URL string
StorageProvider string
StorageChannelID string
StorageChannelKey string
ObjectKey string
AccessScope string
LocalPath string
ExpiresAt *time.Time
}
func (s *Store) FindRequestAsset(ctx context.Context, sha256 string, contentType string) (RequestAsset, bool, error) {
asset, err := scanRequestAsset(s.pool.QueryRow(ctx, `
SELECT id::text, sha256, content_type, byte_size, url, storage_provider,
COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''),
COALESCE(object_key, ''), COALESCE(access_scope, 'private'),
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at
FROM gateway_request_assets
WHERE sha256 = $1 AND content_type = $2`, sha256, contentType))
@@ -51,25 +61,37 @@ WHERE sha256 = $1 AND content_type = $2`, sha256, contentType))
func (s *Store) UpsertRequestAsset(ctx context.Context, input RequestAssetInput) (RequestAsset, error) {
return scanRequestAsset(s.pool.QueryRow(ctx, `
INSERT INTO gateway_request_assets (
sha256, content_type, byte_size, url, storage_provider, local_path, expires_at, expired_at, ref_count
sha256, content_type, byte_size, url, storage_provider, storage_channel_id,
storage_channel_key, object_key, access_scope, local_path, expires_at, expired_at, ref_count
)
VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), $7, NULL, 1)
VALUES ($1, $2, $3, $4, $5, NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''),
COALESCE(NULLIF($9, ''), 'private'), NULLIF($10, ''), $11, NULL, 1)
ON CONFLICT (sha256, content_type) DO UPDATE
SET byte_size = EXCLUDED.byte_size,
url = EXCLUDED.url,
storage_provider = EXCLUDED.storage_provider,
storage_channel_id = EXCLUDED.storage_channel_id,
storage_channel_key = EXCLUDED.storage_channel_key,
object_key = EXCLUDED.object_key,
access_scope = EXCLUDED.access_scope,
local_path = EXCLUDED.local_path,
expires_at = EXCLUDED.expires_at,
expired_at = NULL,
ref_count = gateway_request_assets.ref_count + 1,
updated_at = now()
RETURNING id::text, sha256, content_type, byte_size, url, storage_provider,
COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''),
COALESCE(object_key, ''), COALESCE(access_scope, 'private'),
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at`,
input.SHA256,
input.ContentType,
input.ByteSize,
input.URL,
input.StorageProvider,
input.StorageChannelID,
input.StorageChannelKey,
input.ObjectKey,
input.AccessScope,
input.LocalPath,
input.ExpiresAt,
))
@@ -110,6 +132,10 @@ func scanRequestAsset(scanner interface{ Scan(dest ...any) error }) (RequestAsse
&asset.ByteSize,
&asset.URL,
&asset.StorageProvider,
&asset.StorageChannelID,
&asset.StorageChannelKey,
&asset.ObjectKey,
&asset.AccessScope,
&localPath,
&expiresAt,
&expiredAt,
apps/api/internal/store/task_history.go
+23 -13
View File
@@ -13,17 +13,20 @@ const (
)
type TaskCallbackDelivery struct {
ID string
TaskID string
EventID string
Seq int64
CallbackURL string
Status string
Attempts int
LockToken string
EventType string
TaskStatus string
CreatedAt time.Time
ID string
TaskID string
EventID string
Seq int64
CallbackURL string
Status string
Attempts int
LockToken string
EventType string
TaskStatus string
TaskRequestID string
TaskErrorCode string
TaskErrorMessage string
CreatedAt time.Time
}
type TaskHistoryCleanupResult struct {
@@ -67,9 +70,13 @@ func (s *Store) ClaimTaskCallbacks(ctx context.Context, workerID string, limit i
}
rows, err := s.pool.Query(ctx, `
WITH picked AS (
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at,
COALESCE(task.request_id, '') AS task_request_id,
COALESCE(task.error_code, '') AS task_error_code,
COALESCE(task.error_message, task.error, '') AS task_error_message
FROM gateway_task_callback_outbox outbox
JOIN gateway_task_events event ON event.id = outbox.event_id
JOIN gateway_tasks task ON task.id = outbox.task_id
WHERE outbox.created_at >= COALESCE((
SELECT applied_at
FROM schema_migrations
@@ -101,7 +108,7 @@ WHERE outbox.id = picked.id
RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, ''),
outbox.seq, outbox.callback_url, outbox.status, outbox.attempts,
COALESCE(outbox.lock_token::text, ''), picked.event_type, picked.task_status,
picked.created_at`,
picked.created_at, picked.task_request_id, picked.task_error_code, picked.task_error_message`,
workerID, limit, int(staleAfter/time.Second))
if err != nil {
return nil, err
@@ -122,6 +129,9 @@ RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text,
&item.EventType,
&item.TaskStatus,
&item.CreatedAt,
&item.TaskRequestID,
&item.TaskErrorCode,
&item.TaskErrorMessage,
); err != nil {
return nil, err
}
apps/api/internal/store/tasks_runtime.go
+23 -1
View File
@@ -11,6 +11,7 @@ import (
"unicode/utf8"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/jackc/pgx/v5"
)
@@ -544,12 +545,14 @@ WHERE id = $1::uuid
}
func (s *Store) FailQueuedTask(ctx context.Context, taskID string, code string, message string) (GatewayTask, error) {
publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
tag, err := s.pool.Exec(ctx, `
UPDATE gateway_tasks
SET status = 'failed',
error = NULL,
error_code = NULLIF($2, ''),
error_message = NULLIF($3, ''),
public_error = $4::jsonb,
billing_status = CASE
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
ELSE 'released'
@@ -563,7 +566,7 @@ SET status = 'failed',
finished_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048))
AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048), string(publicErrorJSON))
if err != nil {
return GatewayTask{}, err
}
@@ -686,6 +689,7 @@ SET status = 'queued',
error = NULL,
error_code = NULL,
error_message = NULL,
public_error = NULL,
updated_at = now()
WHERE id = $1::uuid
AND status IN ('queued', 'running')
@@ -722,6 +726,7 @@ SET status = 'queued',
error = NULL,
error_code = NULL,
error_message = NULL,
public_error = NULL,
updated_at = now()
WHERE task.id = $1::uuid
AND task.status IN ('queued', 'running')
@@ -1371,6 +1376,7 @@ SELECT a.id::text, a.task_id::text, a.attempt_no,
a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb),
COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''),
COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''),
COALESCE(a.public_error, '{}'::jsonb),
COALESCE(a.pricing_snapshot, '{}'::jsonb), COALESCE(a.request_fingerprint, ''),
a.upstream_submission_status, COALESCE(a.upstream_submission_updated_at::text, ''),
a.started_at, COALESCE(a.finished_at::text, '')
@@ -1403,6 +1409,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
var requestBytes []byte
var responseBytes []byte
var pricingSnapshotBytes []byte
var publicErrorBytes []byte
if err := scanner.Scan(
&item.ID,
&item.TaskID,
@@ -1430,6 +1437,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
&item.ResponseDurationMS,
&item.ErrorCode,
&item.ErrorMessage,
&publicErrorBytes,
&pricingSnapshotBytes,
&item.RequestFingerprint,
&item.UpstreamSubmissionStatus,
@@ -1444,6 +1452,12 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
item.RequestSnapshot = decodeObject(requestBytes)
item.ResponseSnapshot = decodeObject(responseBytes)
item.PricingSnapshot = decodeObject(pricingSnapshotBytes)
if len(publicErrorBytes) > 0 {
var snapshot publicerror.Error
if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" {
item.PublicError = &snapshot
}
}
enrichTaskAttemptFromMetrics(&item)
return item, nil
}
@@ -1534,6 +1548,7 @@ func (s *Store) FinishTaskAttempt(ctx context.Context, input FinishTaskAttemptIn
}
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptUsage(input.Usage)))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptMetrics(input.Metrics)))
publicErrorJSON := encodePublicErrorSnapshot(input.ErrorCode, input.ErrorMessage, statusCode, input.Retryable, input.RequestID, "")
_, err := s.pool.Exec(ctx, `
UPDATE gateway_task_attempts
SET status = $2::text,
@@ -1550,6 +1565,7 @@ SET status = $2::text,
response_duration_ms = $8,
error_code = NULLIF($9::text, ''),
error_message = NULLIF(left($10::text, 2048), ''),
public_error = CASE WHEN $2::text = 'failed' THEN $14::jsonb ELSE NULL END,
upstream_submission_status = COALESCE(NULLIF($13::text, ''), upstream_submission_status),
upstream_submission_updated_at = CASE
WHEN NULLIF($13::text, '') IS NULL THEN upstream_submission_updated_at
@@ -1570,6 +1586,7 @@ WHERE id = $1::uuid`,
usageJSON,
metricsJSON,
input.UpstreamSubmissionStatus,
string(publicErrorJSON),
)
return err
}
@@ -1651,6 +1668,7 @@ SET status = 'succeeded',
response_duration_ms = $5,
error_code = NULL,
error_message = NULL,
public_error = NULL,
finished_at = now()
WHERE id = $1::uuid`,
input.AttemptID, input.RequestID,
@@ -1687,6 +1705,7 @@ SET status = 'succeeded',
error = NULL,
error_code = NULL,
error_message = NULL,
public_error = NULL,
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
@@ -2049,6 +2068,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
message := truncateUTF8Bytes(input.Message, 2048)
publicErrorJSON := encodePublicErrorSnapshot(input.Code, message, 0, true, input.RequestID, input.TaskID)
finalizedAdmission := false
err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
tag, err := tx.Exec(ctx, `
@@ -2057,6 +2077,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
error = NULL,
error_code = NULLIF($3::text, ''),
error_message = NULLIF($2::text, ''),
public_error = $11::jsonb,
request_id = NULLIF($4::text, ''),
metrics = $5::jsonb,
response_started_at = $6::timestamptz,
@@ -2090,6 +2111,7 @@ WHERE id = $1::uuid
input.ResponseDurationMS,
string(resultJSON),
input.ExecutionToken,
string(publicErrorJSON),
)
if err != nil {
return err
apps/api/migrations/0103_object_storage_references.sql
+21
View File
@@ -0,0 +1,21 @@
ALTER TABLE gateway_request_assets
ADD COLUMN IF NOT EXISTS storage_channel_id uuid REFERENCES file_storage_channels(id) ON DELETE SET NULL,
ADD COLUMN IF NOT EXISTS storage_channel_key text,
ADD COLUMN IF NOT EXISTS object_key text,
ADD COLUMN IF NOT EXISTS access_scope text DEFAULT 'private';
ALTER TABLE gateway_tasks
ADD COLUMN IF NOT EXISTS public_error jsonb;
ALTER TABLE gateway_task_attempts
ADD COLUMN IF NOT EXISTS public_error jsonb;
CREATE INDEX IF NOT EXISTS idx_gateway_request_assets_object
ON gateway_request_assets(storage_channel_key, object_key)
WHERE object_key IS NOT NULL;
UPDATE system_settings
SET value = jsonb_set(COALESCE(value, '{}'::jsonb), '{resultUploadPolicy}', '"default"'::jsonb, true),
updated_at = now()
WHERE setting_key = 'file_storage'
AND COALESCE(value->>'resultUploadPolicy', '') IN ('upload_none', 'none', 'never', 'disabled', 'no_upload', 'skip', 'skip_all');
apps/api/project.json
+1 -1
View File
@@ -24,7 +24,7 @@
"outputs": ["{projectRoot}/docs/swagger.json", "{projectRoot}/docs/swagger.yaml"],
"options": {
"cwd": "apps/api",
"command": "go run github.com/swaggo/swag/cmd/swag@v1.16.4 init --parseInternal -d ./cmd/gateway,./internal/httpapi,./internal/store,./internal/auth,./internal/identity,./internal/runner -g main.go -o docs --outputTypes json,yaml"
"command": "go run github.com/swaggo/swag/cmd/swag@v1.16.4 init --parseInternal -d ./cmd/gateway,./internal/httpapi,./internal/store,./internal/auth,./internal/identity,./internal/runner,./internal/publicerror -g main.go -o docs --outputTypes json,yaml"
}
},
"test": {
apps/web/src/App.tsx
+18
View File
@@ -111,6 +111,7 @@ import {
updateClientCustomizationSettings,
updateFileStorageChannel,
updateFileStorageSettings,
testFileStorageChannel,
updateGatewayUser,
updatePlatform,
updatePlatformDynamicPriority,
@@ -1182,6 +1183,22 @@ export function App() {
}
}
async function verifyFileStorageChannel(channelId: string) {
setCoreState('loading');
setCoreMessage('');
try {
const result = await testFileStorageChannel(token, channelId);
const refreshed = await listFileStorageChannels(token);
setFileStorageChannels(refreshed.items);
setCoreState('ready');
setCoreMessage(`对象存储连接测试通过，Put / Head / Delete 均成功，耗时 ${result.durationMs}ms。`);
} catch (err) {
setCoreState('error');
setCoreMessage(err instanceof Error ? err.message : '对象存储连接测试失败');
throw err;
}
}
async function saveClientCustomizationSettings(input: ClientCustomizationSettingsUpdateRequest) {
setCoreState('loading');
setCoreMessage('');
@@ -1614,6 +1631,7 @@ export function App() {
onSaveAccessRule={saveAccessRule}
onSaveFileStorageChannel={saveFileStorageChannel}
onSaveFileStorageSettings={saveFileStorageSettings}
onTestFileStorageChannel={verifyFileStorageChannel}
onSaveClientCustomizationSettings={saveClientCustomizationSettings}
onConnectSecurityEvents={connectSecurityEvents}
onDisconnectSecurityEvents={disconnectSecurityEvents}
apps/web/src/api.ts
+11
View File
@@ -12,6 +12,7 @@ import type {
ClientCustomizationSettingsUpdateRequest,
CreatedGatewayApiKey,
FileStorageChannel,
FileStorageChannelTestResult,
FileStorageSettings,
FileStorageSettingsUpdateRequest,
FileStorageChannelUpsertRequest,
@@ -1276,6 +1277,16 @@ export async function deleteFileStorageChannel(token: string, channelId: string)
});
}
export async function testFileStorageChannel(
token: string,
channelId: string,
): Promise<FileStorageChannelTestResult> {
return request<FileStorageChannelTestResult>(`/api/admin/system/file-storage/channels/${channelId}/test`, {
method: 'POST',
token,
});
}
async function request<T>(
path: string,
options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record<string, string>; signal?: AbortSignal; timeoutMs?: number } = {},
apps/web/src/pages/AdminPage.tsx
+2
View File
@@ -92,6 +92,7 @@ export function AdminPage(props: {
onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise<void>;
onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise<void>;
onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise<void>;
onTestFileStorageChannel: (channelId: string) => Promise<void>;
onConnectSecurityEvents: (transmitterIssuer: string) => Promise<void>;
onDisconnectSecurityEvents: () => Promise<void>;
onRefreshSecurityEvents: () => Promise<void>;
@@ -214,6 +215,7 @@ export function AdminPage(props: {
onDeleteFileStorageChannel={props.onDeleteFileStorageChannel}
onSaveFileStorageChannel={props.onSaveFileStorageChannel}
onSaveFileStorageSettings={props.onSaveFileStorageSettings}
onTestFileStorageChannel={props.onTestFileStorageChannel}
onSaveClientCustomizationSettings={props.onSaveClientCustomizationSettings}
/>
)}
apps/web/src/pages/admin/SystemSettingsPanel.tsx
+101 -18
View File
@@ -15,6 +15,10 @@ type ClientCustomizationForm = {
};
type FileStorageChannelForm = {
accessKeyId: string;
accessKeyIdPreview: string;
accessKeySecret: string;
accessKeySecretPreview: string;
apiKey: string;
apiKeyPreview: string;
channelKey: string;
@@ -23,13 +27,22 @@ type FileStorageChannelForm = {
priority: string;
provider: string;
retryPolicyJson: string;
sessionToken: string;
sessionTokenPreview: string;
scenes: string[];
status: string;
uploadUrl: string;
};
const defaultUploadUrl = 'http://127.0.0.1:3001/v1/files/upload';
const defaultRetryPolicy = {
const defaultObjectStorageRetryPolicy = {
enabled: true,
maxRetries: 2,
backoffSeconds: [0.25, 1],
strategy: 'exponential',
};
const defaultServerMainRetryPolicy = {
enabled: true,
maxRetries: 3,
backoffSeconds: [60, 120, 180],
@@ -39,7 +52,7 @@ const defaultRetryPolicy = {
const providerOptions = [
{ value: 'server_main_openapi', label: 'server-main OpenAPI' },
{ value: 'aliyun_oss', label: '阿里云 OSS' },
{ value: 'tencent_cos', label: '腾讯云 COS' },
{ value: 's3', label: 'S3 / S3 兼容存储' },
];
const defaultScenes = ['upload', 'image_result', 'request_asset'];
@@ -52,7 +65,6 @@ const sceneOptions = [
const resultUploadPolicyOptions = [
{ value: 'default', label: '默认：仅非链接资源转存', description: 'URL 结果直接保存；base64 / buffer 等结果转存后保存 URL' },
{ value: 'upload_all', label: '全部转存', description: 'URL、base64、buffer 等生成媒体结果都会转存到当前文件渠道' },
{ value: 'upload_none', label: '不做外部转存', description: 'base64 / buffer 临时写入本地静态文件；数据库仅保存占位符，默认 24 小时内按需恢复' },
];
export function SystemSettingsPanel(props: {
@@ -66,6 +78,7 @@ export function SystemSettingsPanel(props: {
onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise<void>;
onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise<void>;
onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise<void>;
onTestFileStorageChannel: (channelId: string) => Promise<void>;
}) {
const [activeTab, setActiveTab] = useState<SystemSettingsTab>('fileStorage');
const [dialogOpen, setDialogOpen] = useState(false);
@@ -75,6 +88,7 @@ export function SystemSettingsPanel(props: {
const [clientCustomizationForm, setClientCustomizationForm] = useState<ClientCustomizationForm>(() => clientCustomizationSettingsToForm(props.clientCustomizationSettings));
const [settingsPolicy, setSettingsPolicy] = useState(() => normalizeResultUploadPolicy(props.settings?.resultUploadPolicy));
const [localError, setLocalError] = useState('');
const [testingChannelId, setTestingChannelId] = useState('');
useEffect(() => {
setSettingsPolicy(normalizeResultUploadPolicy(props.settings?.resultUploadPolicy));
@@ -130,6 +144,18 @@ export function SystemSettingsPanel(props: {
}
}
async function testChannel(channel: FileStorageChannel) {
setLocalError('');
setTestingChannelId(channel.id);
try {
await props.onTestFileStorageChannel(channel.id);
} catch (err) {
setLocalError(err instanceof Error ? err.message : '对象存储连接测试失败');
} finally {
setTestingChannelId('');
}
}
async function saveSettings() {
setLocalError('');
try {
@@ -154,7 +180,7 @@ export function SystemSettingsPanel(props: {
<CardHeader>
<div>
<CardTitle>系统设置</CardTitle>
<p className="mutedText">集中维护网关级配置；文件存储渠道按优先级轮转，单渠道使用 60/120/180 秒退避重试。</p>
<p className="mutedText">集中维护网关级配置；文件存储渠道按优先级轮转，对象存储默认使用 250ms / 1s 退避。</p>
</div>
<Badge variant="secondary">{props.channels.length} 个文件渠道</Badge>
</CardHeader>
@@ -177,7 +203,7 @@ export function SystemSettingsPanel(props: {
<div className="fileStorageSettingsCard">
<div>
<strong>全局生成媒体转存策略</strong>
<span>对图片、音频和视频生成结果统一生效；启用转存但没有可用渠道时，回退到保留 24 小时的网关本地静态资源。</span>
<span>对图片、音频和视频生成结果统一生效；没有可用对象存储时任务会安全失败，不再写入网关本机目录。</span>
</div>
<Label>
策略
@@ -195,7 +221,7 @@ export function SystemSettingsPanel(props: {
<div className="fileStorageToolbar">
<div>
<strong>文件存储渠道</strong>
<span>server-main OpenAPI 渠道只需要上传路由和 API Key。</span>
<span>支持 server-main OpenAPI、阿里云 OSS 与 S3 协议；按优先级重试下一个渠道。</span>
</div>
<Button type="button" onClick={openCreateDialog}>
<Plus size={15} />
@@ -224,6 +250,12 @@ export function SystemSettingsPanel(props: {
{channel.lastError && <span>最近错误: {channel.lastError}</span>}
</div>
<footer>
{channel.provider !== 'server_main_openapi' && (
<Button type="button" variant="outline" size="sm" disabled={testingChannelId === channel.id} onClick={() => testChannel(channel)}>
<ShieldCheck size={14} />
{testingChannelId === channel.id ? '测试中…' : '连接测试'}
</Button>
)}
<Button type="button" variant="outline" size="sm" onClick={() => editChannel(channel)}>
<Pencil size={14} />
修改
@@ -312,7 +344,16 @@ export function SystemSettingsPanel(props: {
</Label>
<Label>
渠道类型
<Select value={form.provider} onChange={(event) => setForm({ ...form, provider: event.target.value })}>
<Select value={form.provider} onChange={(event) => {
const provider = event.target.value;
setForm({
...form,
provider,
retryPolicyJson: editingChannel
? form.retryPolicyJson
: stringifyJson(defaultRetryPolicyForProvider(provider)),
});
}}>
{providerOptions.map((item) => <option value={item.value} key={item.value}>{item.label}</option>)}
</Select>
</Label>
@@ -337,15 +378,29 @@ export function SystemSettingsPanel(props: {
))}
</div>
</Label>
<Label className="spanTwo">
{form.provider === 'server_main_openapi' && <Label className="spanTwo">
上传路由
<Input value={form.uploadUrl} onChange={(event) => setForm({ ...form, uploadUrl: event.target.value })} placeholder={defaultUploadUrl} />
</Label>
<Label className="platformCredentialField">
</Label>}
{form.provider === 'server_main_openapi' && <Label className="platformCredentialField">
API Key
<Input value={form.apiKey} onChange={(event) => setForm({ ...form, apiKey: event.target.value })} placeholder={credentialInputPlaceholder(form.apiKeyPreview)} />
<small>保持脱敏值不变表示不修改；填写新值表示覆盖；清空后保存表示清除。</small>
</Label>
</Label>}
{form.provider !== 'server_main_openapi' && <>
<Label>
Access Key ID
<Input value={form.accessKeyId} onChange={(event) => setForm({ ...form, accessKeyId: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeyIdPreview)} />
</Label>
<Label>
Access Key Secret
<Input type="password" value={form.accessKeySecret} onChange={(event) => setForm({ ...form, accessKeySecret: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeySecretPreview)} />
</Label>
<Label className="spanTwo">
Session Token（可选）
<Input type="password" value={form.sessionToken} onChange={(event) => setForm({ ...form, sessionToken: event.target.value })} placeholder={credentialInputPlaceholder(form.sessionTokenPreview)} />
</Label>
</>}
<Label>
优先级
<Input type="number" min={1} value={form.priority} onChange={(event) => setForm({ ...form, priority: event.target.value })} />
@@ -403,6 +458,10 @@ function clientCustomizationFormToPayload(form: ClientCustomizationForm): Client
function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
return {
accessKeyId: '',
accessKeyIdPreview: '',
accessKeySecret: '',
accessKeySecretPreview: '',
apiKey: '',
apiKeyPreview: '',
channelKey,
@@ -410,7 +469,9 @@ function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
name: 'server-main OpenAPI',
priority: '100',
provider: 'server_main_openapi',
retryPolicyJson: stringifyJson(defaultRetryPolicy),
retryPolicyJson: stringifyJson(defaultServerMainRetryPolicy),
sessionToken: '',
sessionTokenPreview: '',
scenes: defaultScenes,
status: 'disabled',
uploadUrl: defaultUploadUrl,
@@ -420,6 +481,10 @@ function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
const preview = apiKeyPreview(channel);
return {
accessKeyId: credentialPreview(channel, 'accessKeyId'),
accessKeyIdPreview: credentialPreview(channel, 'accessKeyId'),
accessKeySecret: credentialPreview(channel, 'accessKeySecret'),
accessKeySecretPreview: credentialPreview(channel, 'accessKeySecret'),
apiKey: preview,
apiKeyPreview: preview,
channelKey: channel.channelKey,
@@ -427,7 +492,9 @@ function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
name: channel.name,
priority: String(channel.priority || 100),
provider: channel.provider || 'server_main_openapi',
retryPolicyJson: stringifyJson(channel.retryPolicy ?? defaultRetryPolicy),
retryPolicyJson: stringifyJson(channel.retryPolicy ?? defaultRetryPolicyForProvider(channel.provider)),
sessionToken: credentialPreview(channel, 'sessionToken'),
sessionTokenPreview: credentialPreview(channel, 'sessionToken'),
scenes: normalizeScenes(channel.scenes),
status: channel.status || 'disabled',
uploadUrl: channel.uploadUrl || defaultUploadUrl,
@@ -436,6 +503,8 @@ function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRequest {
return {
accessKeyId: credentialPayloadValue(form.accessKeyId, form.accessKeyIdPreview),
accessKeySecret: credentialPayloadValue(form.accessKeySecret, form.accessKeySecretPreview),
apiKey: apiKeyPayloadValue(form),
channelKey: form.channelKey.trim(),
config: parseJsonObject(form.configJson, '扩展配置 JSON'),
@@ -443,6 +512,7 @@ function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRe
priority: Number(form.priority) || 100,
provider: form.provider,
retryPolicy: parseJsonObject(form.retryPolicyJson, '重试策略 JSON'),
sessionToken: credentialPayloadValue(form.sessionToken, form.sessionTokenPreview),
scenes: normalizeScenes(form.scenes),
status: form.status,
uploadUrl: form.uploadUrl.trim(),
@@ -495,8 +565,8 @@ function nextScenes(current: string[], scene: string, checked: boolean) {
}
function retryPolicySummary(policy?: Record<string, unknown>) {
const maxRetries = numberFromUnknown(policy?.maxRetries) || 3;
const backoff = Array.isArray(policy?.backoffSeconds) ? policy?.backoffSeconds.join('/') : '60/120/180';
const maxRetries = numberFromUnknown(policy?.maxRetries) || 2;
const backoff = Array.isArray(policy?.backoffSeconds) ? policy?.backoffSeconds.join('/') : '0.25/1';
return `${maxRetries} 次 · ${backoff}s`;
}
@@ -506,9 +576,22 @@ function apiKeyPreview(channel: FileStorageChannel) {
}
function apiKeyPayloadValue(form: FileStorageChannelForm) {
const value = form.apiKey.trim();
if (form.apiKeyPreview && value === form.apiKeyPreview) return undefined;
return value || (form.apiKeyPreview ? '' : undefined);
return credentialPayloadValue(form.apiKey, form.apiKeyPreview);
}
function credentialPreview(channel: FileStorageChannel, key: string) {
const value = channel.credentialsPreview?.[key];
return typeof value === 'string' ? value : '';
}
function credentialPayloadValue(value: string, preview: string) {
const normalized = value.trim();
if (preview && normalized === preview) return undefined;
return normalized || (preview ? '' : undefined);
}
function defaultRetryPolicyForProvider(provider: string) {
return provider === 'server_main_openapi' ? defaultServerMainRetryPolicy : defaultObjectStorageRetryPolicy;
}
function credentialInputPlaceholder(preview: string) {
packages/contracts/src/index.ts
+32 -4
View File
@@ -1039,7 +1039,7 @@ export interface FileStorageChannel {
id: string;
channelKey: string;
name: string;
provider: 'server_main_openapi' | 'aliyun_oss' | 'tencent_cos' | string;
provider: 'server_main_openapi' | 'aliyun_oss' | 's3' | string;
uploadUrl?: string;
credentialsPreview?: Record<string, unknown>;
scenes?: string[];
@@ -1057,9 +1057,14 @@ export interface FileStorageChannel {
export interface FileStorageChannelUpsertRequest {
channelKey: string;
name: string;
provider?: 'server_main_openapi' | 'aliyun_oss' | 'tencent_cos' | string;
provider?: 'server_main_openapi' | 'aliyun_oss' | 's3' | string;
uploadUrl?: string;
apiKey?: string;
accessKey?: string;
accessKeyId?: string;
accessKeySecret?: string;
secretKey?: string;
sessionToken?: string;
scenes?: string[];
config?: Record<string, unknown>;
retryPolicy?: Record<string, unknown>;
@@ -1067,12 +1072,20 @@ export interface FileStorageChannelUpsertRequest {
status?: 'enabled' | 'disabled' | string;
}
export interface FileStorageChannelTestResult {
provider: 'aliyun_oss' | 's3' | string;
putSucceeded: boolean;
headSucceeded: boolean;
deleteSucceeded: boolean;
durationMs: number;
}
export interface FileStorageSettings {
resultUploadPolicy: 'default' | 'upload_all' | 'upload_none' | string;
resultUploadPolicy: 'default' | 'upload_all' | string;
}
export interface FileStorageSettingsUpdateRequest {
resultUploadPolicy: 'default' | 'upload_all' | 'upload_none' | string;
resultUploadPolicy: 'default' | 'upload_all' | string;
}
export interface ClientCustomizationSettings {
@@ -1238,6 +1251,19 @@ export interface IdentityRevisionPolicyUpdate {
sessionRefreshSeconds?: number;
}
export interface PublicErrorV1 {
code: string;
message: string;
category: string;
httpStatus: number;
retryable: boolean;
action: string;
retryAfterSeconds?: number;
requestId?: string;
taskId?: string;
version: 'v1' | string;
}
export interface GatewayTask {
id: string;
kind: string;
@@ -1284,6 +1310,7 @@ export interface GatewayTask {
error?: string;
errorCode?: string;
errorMessage?: string;
publicError?: PublicErrorV1;
attempts?: GatewayTaskAttempt[];
createdAt: string;
updatedAt: string;
@@ -1367,6 +1394,7 @@ export interface GatewayTaskAttempt {
upstreamSubmissionUpdatedAt?: string;
errorCode?: string;
errorMessage?: string;
publicError?: PublicErrorV1;
startedAt: string;
finishedAt?: string;
}