feat(storage): 统一二进制对象存储与公开错误
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。 将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。 引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。 验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
This commit is contained in:
@@ -161,6 +161,9 @@ func run(ctx context.Context, opts options) error {
|
|||||||
if err := ensureAcceptanceAccessRules(ctx, database, groupID); err != nil {
|
if err := ensureAcceptanceAccessRules(ctx, database, groupID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := ensureAcceptanceObjectStorage(ctx, database, opts.emulatorBaseURL); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
geminiModel, videoModel, err := selectedModels(ctx, database)
|
geminiModel, videoModel, err := selectedModels(ctx, database)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -240,6 +243,73 @@ func run(ctx context.Context, opts options) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ensureAcceptanceObjectStorage(ctx context.Context, database *store.Store, emulatorBaseURL string) error {
|
||||||
|
baseURL := strings.TrimRight(strings.TrimSpace(emulatorBaseURL), "/")
|
||||||
|
if baseURL == "" {
|
||||||
|
return errors.New("acceptance object storage requires the emulator URL")
|
||||||
|
}
|
||||||
|
channels := []struct {
|
||||||
|
key string
|
||||||
|
name string
|
||||||
|
provider string
|
||||||
|
endpoint string
|
||||||
|
priority int
|
||||||
|
pathStyle bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
key: "local-acceptance-oss", name: "Local Acceptance Aliyun OSS", provider: "aliyun_oss",
|
||||||
|
endpoint: baseURL + "/storage/oss/bucket", priority: 10, pathStyle: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "local-acceptance-s3", name: "Local Acceptance S3", provider: "s3",
|
||||||
|
endpoint: baseURL + "/storage/s3", priority: 20, pathStyle: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, channel := range channels {
|
||||||
|
_, err := database.Pool().Exec(ctx, `
|
||||||
|
INSERT INTO file_storage_channels (
|
||||||
|
channel_key, name, provider, credentials, config, retry_policy, priority, status
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
$1, $2, $3,
|
||||||
|
'{"accessKeyId":"local-acceptance","accessKeySecret":"local-acceptance-secret"}'::jsonb,
|
||||||
|
jsonb_build_object(
|
||||||
|
'endpoint', $4::text,
|
||||||
|
'region', 'local-acceptance-1',
|
||||||
|
'bucket', 'bucket',
|
||||||
|
'objectPrefix', 'acceptance/media',
|
||||||
|
'accessScope', 'private',
|
||||||
|
'forcePathStyle', $5::boolean,
|
||||||
|
'scenes', jsonb_build_array('upload', 'image_result', 'request_asset'),
|
||||||
|
'acceptanceEmulatorOnly', true
|
||||||
|
),
|
||||||
|
'{"enabled":true,"maxRetries":2,"backoffSeconds":[0.25,1],"strategy":"exponential"}'::jsonb,
|
||||||
|
$6, 'enabled'
|
||||||
|
)
|
||||||
|
ON CONFLICT (channel_key) DO UPDATE
|
||||||
|
SET name = EXCLUDED.name,
|
||||||
|
provider = EXCLUDED.provider,
|
||||||
|
credentials = EXCLUDED.credentials,
|
||||||
|
config = EXCLUDED.config,
|
||||||
|
retry_policy = EXCLUDED.retry_policy,
|
||||||
|
priority = EXCLUDED.priority,
|
||||||
|
status = 'enabled',
|
||||||
|
deleted_at = NULL,
|
||||||
|
last_error = NULL,
|
||||||
|
updated_at = now()`, channel.key, channel.name, channel.provider, channel.endpoint, channel.pathStyle, channel.priority)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("configure %s object storage channel: %w", channel.provider, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err := database.Pool().Exec(ctx, `
|
||||||
|
INSERT INTO system_settings (setting_key, value)
|
||||||
|
VALUES ('file_storage', '{"resultUploadPolicy":"default"}'::jsonb)
|
||||||
|
ON CONFLICT (setting_key) DO UPDATE
|
||||||
|
SET value = jsonb_set(COALESCE(system_settings.value, '{}'::jsonb), '{resultUploadPolicy}', '"default"'::jsonb, true),
|
||||||
|
updated_at = now()`)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func resetPreviousLocalRun(ctx context.Context, database *store.Store, opts options) error {
|
func resetPreviousLocalRun(ctx context.Context, database *store.Store, opts options) error {
|
||||||
mode, err := database.GetGatewayTrafficMode(ctx)
|
mode, err := database.GetGatewayTrafficMode(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+182
-16
@@ -3157,7 +3157,7 @@
|
|||||||
"BearerAuth": []
|
"BearerAuth": []
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"description": "创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。",
|
"description": "创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。",
|
||||||
"consumes": [
|
"consumes": [
|
||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
@@ -3354,6 +3354,70 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/api/admin/system/file-storage/channels/{channelID}/test": {
|
||||||
|
"post": {
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"BearerAuth": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。",
|
||||||
|
"produces": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"system"
|
||||||
|
],
|
||||||
|
"summary": "测试对象存储通道",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"description": "文件存储通道 ID",
|
||||||
|
"name": "channelID",
|
||||||
|
"in": "path",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "OK",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/httpapi.FileStorageChannelTestResponse"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"description": "Bad Request",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"401": {
|
||||||
|
"description": "Unauthorized",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"403": {
|
||||||
|
"description": "Forbidden",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"404": {
|
||||||
|
"description": "Not Found",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"503": {
|
||||||
|
"description": "Service Unavailable",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"/api/admin/system/file-storage/settings": {
|
"/api/admin/system/file-storage/settings": {
|
||||||
"get": {
|
"get": {
|
||||||
"security": [
|
"security": [
|
||||||
@@ -6712,7 +6776,7 @@
|
|||||||
"BearerAuth": []
|
"BearerAuth": []
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"description": "上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。",
|
"description": "上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。",
|
||||||
"consumes": [
|
"consumes": [
|
||||||
"multipart/form-data"
|
"multipart/form-data"
|
||||||
],
|
],
|
||||||
@@ -11207,22 +11271,52 @@
|
|||||||
"httpapi.ErrorPayload": {
|
"httpapi.ErrorPayload": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"action": {
|
||||||
|
"type": "string",
|
||||||
|
"example": "retry_after"
|
||||||
|
},
|
||||||
|
"category": {
|
||||||
|
"type": "string",
|
||||||
|
"example": "rate_limit"
|
||||||
|
},
|
||||||
"code": {
|
"code": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"example": "rate_limit"
|
"example": "rate_limit"
|
||||||
},
|
},
|
||||||
|
"httpStatus": {
|
||||||
|
"type": "integer",
|
||||||
|
"example": 429
|
||||||
|
},
|
||||||
"message": {
|
"message": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"example": "invalid json body"
|
"example": "invalid json body"
|
||||||
},
|
},
|
||||||
"param": {},
|
"param": {},
|
||||||
|
"requestId": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"retryAfterSeconds": {
|
||||||
|
"type": "integer",
|
||||||
|
"example": 2
|
||||||
|
},
|
||||||
|
"retryable": {
|
||||||
|
"type": "boolean",
|
||||||
|
"example": true
|
||||||
|
},
|
||||||
"status": {
|
"status": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"example": 400
|
"example": 400
|
||||||
},
|
},
|
||||||
|
"taskId": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"type": {
|
"type": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"example": "invalid_request_error"
|
"example": "invalid_request_error"
|
||||||
|
},
|
||||||
|
"version": {
|
||||||
|
"type": "string",
|
||||||
|
"example": "v1"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -11237,6 +11331,26 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"httpapi.FileStorageChannelTestResponse": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"deleteSucceeded": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"durationMs": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"headSucceeded": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"provider": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"putSucceeded": {
|
||||||
|
"type": "boolean"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"httpapi.FileUploadData": {
|
"httpapi.FileUploadData": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
@@ -11443,6 +11557,9 @@
|
|||||||
"example": 0
|
"example": 0
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
|
"error": {
|
||||||
|
"$ref": "#/definitions/publicerror.Error"
|
||||||
|
},
|
||||||
"message": {
|
"message": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"example": "SUCCEED"
|
"example": "SUCCEED"
|
||||||
@@ -11620,6 +11737,9 @@
|
|||||||
"type": "string",
|
"type": "string",
|
||||||
"example": "invalid parameter"
|
"example": "invalid parameter"
|
||||||
},
|
},
|
||||||
|
"public_error": {
|
||||||
|
"$ref": "#/definitions/publicerror.Error"
|
||||||
|
},
|
||||||
"request_id": {
|
"request_id": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
@@ -12809,20 +12929,7 @@
|
|||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"error": {
|
"error": {
|
||||||
"$ref": "#/definitions/httpapi.VolcesErrorPayload"
|
"$ref": "#/definitions/publicerror.Error"
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"httpapi.VolcesErrorPayload": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"code": {
|
|
||||||
"type": "string",
|
|
||||||
"example": "invalid_parameter"
|
|
||||||
},
|
|
||||||
"message": {
|
|
||||||
"type": "string",
|
|
||||||
"example": "model is required"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -13323,6 +13430,41 @@
|
|||||||
"RevisionFailed"
|
"RevisionFailed"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"publicerror.Error": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"action": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"category": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"code": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"httpStatus": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"requestId": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"retryAfterSeconds": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"retryable": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"taskId": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"version": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"runner.PortraitAssetCapability": {
|
"runner.PortraitAssetCapability": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
@@ -13835,6 +13977,9 @@
|
|||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": {}
|
"additionalProperties": {}
|
||||||
},
|
},
|
||||||
|
"publicError": {
|
||||||
|
"$ref": "#/definitions/publicerror.Error"
|
||||||
|
},
|
||||||
"remoteTaskId": {
|
"remoteTaskId": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
@@ -14771,6 +14916,15 @@
|
|||||||
"store.FileStorageChannelInput": {
|
"store.FileStorageChannelInput": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"accessKey": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"accessKeyId": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"accessKeySecret": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"apiKey": {
|
"apiKey": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
@@ -14800,6 +14954,12 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"secretKey": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"sessionToken": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"status": {
|
"status": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
@@ -14971,6 +15131,9 @@
|
|||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": {}
|
"additionalProperties": {}
|
||||||
},
|
},
|
||||||
|
"publicError": {
|
||||||
|
"$ref": "#/definitions/publicerror.Error"
|
||||||
|
},
|
||||||
"remoteTaskId": {
|
"remoteTaskId": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
@@ -16540,6 +16703,9 @@
|
|||||||
"providerModelName": {
|
"providerModelName": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"publicError": {
|
||||||
|
"$ref": "#/definitions/publicerror.Error"
|
||||||
|
},
|
||||||
"queueKey": {
|
"queueKey": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
|||||||
+122
-12
@@ -481,19 +481,41 @@ definitions:
|
|||||||
type: object
|
type: object
|
||||||
httpapi.ErrorPayload:
|
httpapi.ErrorPayload:
|
||||||
properties:
|
properties:
|
||||||
|
action:
|
||||||
|
example: retry_after
|
||||||
|
type: string
|
||||||
|
category:
|
||||||
|
example: rate_limit
|
||||||
|
type: string
|
||||||
code:
|
code:
|
||||||
example: rate_limit
|
example: rate_limit
|
||||||
type: string
|
type: string
|
||||||
|
httpStatus:
|
||||||
|
example: 429
|
||||||
|
type: integer
|
||||||
message:
|
message:
|
||||||
example: invalid json body
|
example: invalid json body
|
||||||
type: string
|
type: string
|
||||||
param: {}
|
param: {}
|
||||||
|
requestId:
|
||||||
|
type: string
|
||||||
|
retryAfterSeconds:
|
||||||
|
example: 2
|
||||||
|
type: integer
|
||||||
|
retryable:
|
||||||
|
example: true
|
||||||
|
type: boolean
|
||||||
status:
|
status:
|
||||||
example: 400
|
example: 400
|
||||||
type: integer
|
type: integer
|
||||||
|
taskId:
|
||||||
|
type: string
|
||||||
type:
|
type:
|
||||||
example: invalid_request_error
|
example: invalid_request_error
|
||||||
type: string
|
type: string
|
||||||
|
version:
|
||||||
|
example: v1
|
||||||
|
type: string
|
||||||
type: object
|
type: object
|
||||||
httpapi.FileStorageChannelListResponse:
|
httpapi.FileStorageChannelListResponse:
|
||||||
properties:
|
properties:
|
||||||
@@ -502,6 +524,19 @@ definitions:
|
|||||||
$ref: '#/definitions/store.FileStorageChannel'
|
$ref: '#/definitions/store.FileStorageChannel'
|
||||||
type: array
|
type: array
|
||||||
type: object
|
type: object
|
||||||
|
httpapi.FileStorageChannelTestResponse:
|
||||||
|
properties:
|
||||||
|
deleteSucceeded:
|
||||||
|
type: boolean
|
||||||
|
durationMs:
|
||||||
|
type: integer
|
||||||
|
headSucceeded:
|
||||||
|
type: boolean
|
||||||
|
provider:
|
||||||
|
type: string
|
||||||
|
putSucceeded:
|
||||||
|
type: boolean
|
||||||
|
type: object
|
||||||
httpapi.FileUploadData:
|
httpapi.FileUploadData:
|
||||||
properties:
|
properties:
|
||||||
assetStorage:
|
assetStorage:
|
||||||
@@ -651,6 +686,8 @@ definitions:
|
|||||||
example: 0
|
example: 0
|
||||||
type: integer
|
type: integer
|
||||||
data: {}
|
data: {}
|
||||||
|
error:
|
||||||
|
$ref: '#/definitions/publicerror.Error'
|
||||||
message:
|
message:
|
||||||
example: SUCCEED
|
example: SUCCEED
|
||||||
type: string
|
type: string
|
||||||
@@ -775,6 +812,8 @@ definitions:
|
|||||||
message:
|
message:
|
||||||
example: invalid parameter
|
example: invalid parameter
|
||||||
type: string
|
type: string
|
||||||
|
public_error:
|
||||||
|
$ref: '#/definitions/publicerror.Error'
|
||||||
request_id:
|
request_id:
|
||||||
type: string
|
type: string
|
||||||
type: object
|
type: object
|
||||||
@@ -1602,16 +1641,7 @@ definitions:
|
|||||||
httpapi.VolcesErrorEnvelope:
|
httpapi.VolcesErrorEnvelope:
|
||||||
properties:
|
properties:
|
||||||
error:
|
error:
|
||||||
$ref: '#/definitions/httpapi.VolcesErrorPayload'
|
$ref: '#/definitions/publicerror.Error'
|
||||||
type: object
|
|
||||||
httpapi.VolcesErrorPayload:
|
|
||||||
properties:
|
|
||||||
code:
|
|
||||||
example: invalid_parameter
|
|
||||||
type: string
|
|
||||||
message:
|
|
||||||
example: model is required
|
|
||||||
type: string
|
|
||||||
type: object
|
type: object
|
||||||
httpapi.WalletAdjustmentResponse:
|
httpapi.WalletAdjustmentResponse:
|
||||||
properties:
|
properties:
|
||||||
@@ -1953,6 +1983,29 @@ definitions:
|
|||||||
- RevisionActive
|
- RevisionActive
|
||||||
- RevisionSuperseded
|
- RevisionSuperseded
|
||||||
- RevisionFailed
|
- RevisionFailed
|
||||||
|
publicerror.Error:
|
||||||
|
properties:
|
||||||
|
action:
|
||||||
|
type: string
|
||||||
|
category:
|
||||||
|
type: string
|
||||||
|
code:
|
||||||
|
type: string
|
||||||
|
httpStatus:
|
||||||
|
type: integer
|
||||||
|
message:
|
||||||
|
type: string
|
||||||
|
requestId:
|
||||||
|
type: string
|
||||||
|
retryAfterSeconds:
|
||||||
|
type: integer
|
||||||
|
retryable:
|
||||||
|
type: boolean
|
||||||
|
taskId:
|
||||||
|
type: string
|
||||||
|
version:
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
runner.PortraitAssetCapability:
|
runner.PortraitAssetCapability:
|
||||||
properties:
|
properties:
|
||||||
availablePlatformIds:
|
availablePlatformIds:
|
||||||
@@ -2297,6 +2350,8 @@ definitions:
|
|||||||
pricingSnapshot:
|
pricingSnapshot:
|
||||||
additionalProperties: {}
|
additionalProperties: {}
|
||||||
type: object
|
type: object
|
||||||
|
publicError:
|
||||||
|
$ref: '#/definitions/publicerror.Error'
|
||||||
remoteTaskId:
|
remoteTaskId:
|
||||||
type: string
|
type: string
|
||||||
request:
|
request:
|
||||||
@@ -2929,6 +2984,12 @@ definitions:
|
|||||||
type: object
|
type: object
|
||||||
store.FileStorageChannelInput:
|
store.FileStorageChannelInput:
|
||||||
properties:
|
properties:
|
||||||
|
accessKey:
|
||||||
|
type: string
|
||||||
|
accessKeyId:
|
||||||
|
type: string
|
||||||
|
accessKeySecret:
|
||||||
|
type: string
|
||||||
apiKey:
|
apiKey:
|
||||||
type: string
|
type: string
|
||||||
channelKey:
|
channelKey:
|
||||||
@@ -2949,6 +3010,10 @@ definitions:
|
|||||||
items:
|
items:
|
||||||
type: string
|
type: string
|
||||||
type: array
|
type: array
|
||||||
|
secretKey:
|
||||||
|
type: string
|
||||||
|
sessionToken:
|
||||||
|
type: string
|
||||||
status:
|
status:
|
||||||
type: string
|
type: string
|
||||||
uploadUrl:
|
uploadUrl:
|
||||||
@@ -3064,6 +3129,8 @@ definitions:
|
|||||||
pricingSnapshot:
|
pricingSnapshot:
|
||||||
additionalProperties: {}
|
additionalProperties: {}
|
||||||
type: object
|
type: object
|
||||||
|
publicError:
|
||||||
|
$ref: '#/definitions/publicerror.Error'
|
||||||
remoteTaskId:
|
remoteTaskId:
|
||||||
type: string
|
type: string
|
||||||
request:
|
request:
|
||||||
@@ -4125,6 +4192,8 @@ definitions:
|
|||||||
type: string
|
type: string
|
||||||
providerModelName:
|
providerModelName:
|
||||||
type: string
|
type: string
|
||||||
|
publicError:
|
||||||
|
$ref: '#/definitions/publicerror.Error'
|
||||||
queueKey:
|
queueKey:
|
||||||
type: string
|
type: string
|
||||||
requestFingerprint:
|
requestFingerprint:
|
||||||
@@ -6358,7 +6427,7 @@ paths:
|
|||||||
post:
|
post:
|
||||||
consumes:
|
consumes:
|
||||||
- application/json
|
- application/json
|
||||||
description: 创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。
|
description: 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。
|
||||||
parameters:
|
parameters:
|
||||||
- description: 文件存储通道
|
- description: 文件存储通道
|
||||||
in: body
|
in: body
|
||||||
@@ -6485,6 +6554,47 @@ paths:
|
|||||||
summary: 更新文件存储通道
|
summary: 更新文件存储通道
|
||||||
tags:
|
tags:
|
||||||
- system
|
- system
|
||||||
|
/api/admin/system/file-storage/channels/{channelID}/test:
|
||||||
|
post:
|
||||||
|
description: 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。
|
||||||
|
parameters:
|
||||||
|
- description: 文件存储通道 ID
|
||||||
|
in: path
|
||||||
|
name: channelID
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
produces:
|
||||||
|
- application/json
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: OK
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/httpapi.FileStorageChannelTestResponse'
|
||||||
|
"400":
|
||||||
|
description: Bad Request
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||||
|
"401":
|
||||||
|
description: Unauthorized
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||||
|
"403":
|
||||||
|
description: Forbidden
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||||
|
"404":
|
||||||
|
description: Not Found
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||||
|
"503":
|
||||||
|
description: Service Unavailable
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||||
|
security:
|
||||||
|
- BearerAuth: []
|
||||||
|
summary: 测试对象存储通道
|
||||||
|
tags:
|
||||||
|
- system
|
||||||
/api/admin/system/file-storage/settings:
|
/api/admin/system/file-storage/settings:
|
||||||
get:
|
get:
|
||||||
description: 返回文件存储系统设置；数据库对象尚未创建时返回默认设置。
|
description: 返回文件存储系统设置；数据库对象尚未创建时返回默认设置。
|
||||||
@@ -8652,7 +8762,7 @@ paths:
|
|||||||
post:
|
post:
|
||||||
consumes:
|
consumes:
|
||||||
- multipart/form-data
|
- multipart/form-data
|
||||||
description: 上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。
|
description: 上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。
|
||||||
parameters:
|
parameters:
|
||||||
- description: 要上传的文件
|
- description: 要上传的文件
|
||||||
in: formData
|
in: formData
|
||||||
|
|||||||
@@ -51,6 +51,8 @@ type Server struct {
|
|||||||
callbacks map[string]map[int64]int
|
callbacks map[string]map[int64]int
|
||||||
videoByIdempotency map[string]string
|
videoByIdempotency map[string]string
|
||||||
geminiIdempotency map[string]struct{}
|
geminiIdempotency map[string]struct{}
|
||||||
|
storageObjects map[string]fixture
|
||||||
|
storageAttempts map[string]int
|
||||||
pngSmall string
|
pngSmall string
|
||||||
pngLarge string
|
pngLarge string
|
||||||
pngPeak string
|
pngPeak string
|
||||||
@@ -75,6 +77,11 @@ type Report struct {
|
|||||||
DuplicateCallbacks int64 `json:"duplicateCallbacks"`
|
DuplicateCallbacks int64 `json:"duplicateCallbacks"`
|
||||||
MissingIdempotency int64 `json:"missingIdempotencyKeys"`
|
MissingIdempotency int64 `json:"missingIdempotencyKeys"`
|
||||||
DuplicateSubmissions int64 `json:"duplicateSubmissionAttempts"`
|
DuplicateSubmissions int64 `json:"duplicateSubmissionAttempts"`
|
||||||
|
StoragePuts int64 `json:"storagePuts"`
|
||||||
|
StorageGets int64 `json:"storageGets"`
|
||||||
|
StorageHeads int64 `json:"storageHeads"`
|
||||||
|
StorageDeletes int64 `json:"storageDeletes"`
|
||||||
|
StorageFailures int64 `json:"storageFailures"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type videoTask struct {
|
type videoTask struct {
|
||||||
@@ -117,6 +124,8 @@ func New(config Config) *Server {
|
|||||||
callbacks: map[string]map[int64]int{},
|
callbacks: map[string]map[int64]int{},
|
||||||
videoByIdempotency: map[string]string{},
|
videoByIdempotency: map[string]string{},
|
||||||
geminiIdempotency: map[string]struct{}{},
|
geminiIdempotency: map[string]struct{}{},
|
||||||
|
storageObjects: map[string]fixture{},
|
||||||
|
storageAttempts: map[string]int{},
|
||||||
report: Report{
|
report: Report{
|
||||||
VideoReferenceCounts: map[string]int64{},
|
VideoReferenceCounts: map[string]int64{},
|
||||||
VideoRoleCounts: map[string]int64{},
|
VideoRoleCounts: map[string]int64{},
|
||||||
@@ -139,9 +148,107 @@ func (s *Server) Handler() http.Handler {
|
|||||||
mux.HandleFunc("GET /media/{asset}", s.getMedia)
|
mux.HandleFunc("GET /media/{asset}", s.getMedia)
|
||||||
mux.HandleFunc("GET /fixtures/{asset}", s.getFixture)
|
mux.HandleFunc("GET /fixtures/{asset}", s.getFixture)
|
||||||
mux.HandleFunc("POST /callbacks", s.collectCallback)
|
mux.HandleFunc("POST /callbacks", s.collectCallback)
|
||||||
|
mux.HandleFunc("PUT /storage/{profile}/{object...}", s.objectStorage)
|
||||||
|
mux.HandleFunc("GET /storage/{profile}/{object...}", s.objectStorage)
|
||||||
|
mux.HandleFunc("HEAD /storage/{profile}/{object...}", s.objectStorage)
|
||||||
|
mux.HandleFunc("DELETE /storage/{profile}/{object...}", s.objectStorage)
|
||||||
return mux
|
return mux
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Server) objectStorage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
profile := strings.ToLower(strings.TrimSpace(r.PathValue("profile")))
|
||||||
|
object := strings.TrimLeft(strings.TrimSpace(r.PathValue("object")), "/")
|
||||||
|
if object == "" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
baseProfile := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(profile, "-transient"), "-auth"), "-fail")
|
||||||
|
if baseProfile != "oss" && baseProfile != "s3" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key := profile + "/" + object
|
||||||
|
s.mu.Lock()
|
||||||
|
s.storageAttempts[key]++
|
||||||
|
attempt := s.storageAttempts[key]
|
||||||
|
if strings.HasSuffix(profile, "-auth") {
|
||||||
|
s.report.StorageFailures++
|
||||||
|
s.mu.Unlock()
|
||||||
|
http.Error(w, "forbidden", http.StatusForbidden)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(profile, "-fail") || (strings.HasSuffix(profile, "-transient") && attempt == 1) {
|
||||||
|
s.report.StorageFailures++
|
||||||
|
s.mu.Unlock()
|
||||||
|
http.Error(w, "temporary storage failure", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodPut:
|
||||||
|
s.mu.Unlock()
|
||||||
|
payload, err := io.ReadAll(io.LimitReader(r.Body, maxProtocolBodyBytes+1))
|
||||||
|
if err != nil || len(payload) > maxProtocolBodyBytes {
|
||||||
|
http.Error(w, "invalid object", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
s.storageObjects[key] = fixture{ContentType: firstNonEmpty(r.Header.Get("Content-Type"), "application/octet-stream"), Payload: payload}
|
||||||
|
s.report.StoragePuts++
|
||||||
|
s.mu.Unlock()
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case http.MethodGet:
|
||||||
|
item, ok := s.storageObjects[key]
|
||||||
|
if ok {
|
||||||
|
s.report.StorageGets++
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", item.ContentType)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = w.Write(item.Payload)
|
||||||
|
case http.MethodHead:
|
||||||
|
item, ok := s.storageObjects[key]
|
||||||
|
if ok {
|
||||||
|
s.report.StorageHeads++
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", item.ContentType)
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(item.Payload)))
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case http.MethodDelete:
|
||||||
|
_, ok := s.storageObjects[key]
|
||||||
|
delete(s.storageObjects, key)
|
||||||
|
if ok {
|
||||||
|
s.report.StorageDeletes++
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
default:
|
||||||
|
s.mu.Unlock()
|
||||||
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstNonEmpty(values ...string) string {
|
||||||
|
for _, value := range values {
|
||||||
|
if value = strings.TrimSpace(value); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Server) health(w http.ResponseWriter, _ *http.Request) {
|
func (s *Server) health(w http.ResponseWriter, _ *http.Request) {
|
||||||
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -315,6 +315,86 @@ func TestVolcesProtocolAcceptsThreeSixAndNineReferenceImages(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestObjectStorageEmulationSupportsRetryLifecycleAndFaults(t *testing.T) {
|
||||||
|
server := httptest.NewServer(New(Config{}).Handler())
|
||||||
|
defer server.Close()
|
||||||
|
objectURL := server.URL + "/storage/s3-transient/bucket/media/result.png"
|
||||||
|
|
||||||
|
request, _ := http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes"))
|
||||||
|
request.Header.Set("Content-Type", "image/png")
|
||||||
|
response, err := http.DefaultClient.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("first transient PUT status=%d", response.StatusCode)
|
||||||
|
}
|
||||||
|
request, _ = http.NewRequest(http.MethodPut, objectURL, strings.NewReader("image-bytes"))
|
||||||
|
request.Header.Set("Content-Type", "image/png")
|
||||||
|
response, err = http.DefaultClient.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("retried PUT status=%d", response.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err = http.Get(objectURL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
payload, _ := io.ReadAll(response.Body)
|
||||||
|
_ = response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK || string(payload) != "image-bytes" {
|
||||||
|
t.Fatalf("GET status=%d payload=%q", response.StatusCode, payload)
|
||||||
|
}
|
||||||
|
request, _ = http.NewRequest(http.MethodHead, objectURL, nil)
|
||||||
|
response, err = http.DefaultClient.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("HEAD status=%d", response.StatusCode)
|
||||||
|
}
|
||||||
|
request, _ = http.NewRequest(http.MethodDelete, objectURL, nil)
|
||||||
|
response, err = http.DefaultClient.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusNoContent {
|
||||||
|
t.Fatalf("DELETE status=%d", response.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
for profile, wantStatus := range map[string]int{"oss-auth": http.StatusForbidden, "s3-fail": http.StatusServiceUnavailable} {
|
||||||
|
request, _ = http.NewRequest(http.MethodPut, server.URL+"/storage/"+profile+"/probe.bin", strings.NewReader("probe"))
|
||||||
|
response, err = http.DefaultClient.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = response.Body.Close()
|
||||||
|
if response.StatusCode != wantStatus {
|
||||||
|
t.Fatalf("%s status=%d, want %d", profile, response.StatusCode, wantStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err = http.Get(server.URL + "/report")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
var report Report
|
||||||
|
if err := json.NewDecoder(response.Body).Decode(&report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if report.StoragePuts != 1 || report.StorageGets != 1 || report.StorageHeads != 1 || report.StorageDeletes != 1 || report.StorageFailures != 3 {
|
||||||
|
t.Fatalf("unexpected storage report: %+v", report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func postIdempotent(url string, body []byte, key string) (*http.Response, error) {
|
func postIdempotent(url string, body []byte, key string) (*http.Response, error) {
|
||||||
request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
|
request, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -97,11 +98,16 @@ func writeWireResponse(w http.ResponseWriter, wire *clients.WireResponse) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeProtocolError(w http.ResponseWriter, protocol string, status int, message string, details map[string]any, code string) {
|
func writeProtocolError(w http.ResponseWriter, protocol string, status int, message string, details map[string]any, code string) {
|
||||||
|
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||||
|
standard = publicErrorWithRetryAfter(standard, details)
|
||||||
|
publicerror.Observe(standard)
|
||||||
|
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
|
||||||
|
details = safePublicErrorDetails(details, standard, true)
|
||||||
switch protocol {
|
switch protocol {
|
||||||
case clients.ProtocolGeminiGenerateContent:
|
case clients.ProtocolGeminiGenerateContent:
|
||||||
writeGeminiError(w, status, message, details, code)
|
writeGeminiError(w, status, message, details, code)
|
||||||
case clients.ProtocolVolcesContents:
|
case clients.ProtocolVolcesContents:
|
||||||
writeVolcesError(w, status, message, code)
|
writeVolcesPublicError(w, standard)
|
||||||
case clients.ProtocolKlingV1Omni, clients.ProtocolKlingV2Omni:
|
case clients.ProtocolKlingV1Omni, clients.ProtocolKlingV2Omni:
|
||||||
writeKelingCompatError(w, "", newKelingCompatError(status, kelingCompatBusinessCode(code, message), message))
|
writeKelingCompatError(w, "", newKelingCompatError(status, kelingCompatBusinessCode(code, message), message))
|
||||||
default:
|
default:
|
||||||
@@ -185,11 +191,11 @@ func googleRPCStatus(status int, code string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeVolcesError(w http.ResponseWriter, status int, message string, code string) {
|
func writeVolcesError(w http.ResponseWriter, status int, message string, code string) {
|
||||||
if strings.TrimSpace(code) == "" {
|
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||||
code = http.StatusText(status)
|
publicerror.Observe(standard)
|
||||||
}
|
writeVolcesPublicError(w, standard)
|
||||||
writeJSON(w, status, map[string]any{"error": map[string]any{
|
}
|
||||||
"code": code,
|
|
||||||
"message": message,
|
func writeVolcesPublicError(w http.ResponseWriter, standard publicerror.Error) {
|
||||||
}})
|
writeJSON(w, standard.HTTPStatus, map[string]any{"error": standard})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
|
func TestProtocolErrorsUseCompatibleShapesWithStandardPublicErrors(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
protocol string
|
protocol string
|
||||||
@@ -78,10 +78,10 @@ func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
|
|||||||
name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusBadGateway,
|
name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusBadGateway,
|
||||||
assertBody: func(t *testing.T, body map[string]any) {
|
assertBody: func(t *testing.T, body map[string]any) {
|
||||||
errorBody := requireObject(t, body["error"])
|
errorBody := requireObject(t, body["error"])
|
||||||
if errorBody["code"] != "upstream_submission_unknown" {
|
if errorBody["code"] != "upstream_submission_unknown" || errorBody["httpStatus"] != float64(http.StatusBadGateway) || errorBody["retryable"] != true {
|
||||||
t.Fatalf("unexpected Volces error: %+v", body)
|
t.Fatalf("unexpected Volces error: %+v", body)
|
||||||
}
|
}
|
||||||
assertNoKeys(t, errorBody, "status", "retryable", "taskId", "gateway_status")
|
assertNoKeys(t, errorBody, "status", "taskId", "gateway_status")
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -135,6 +135,44 @@ func TestWireResponsePassthroughPreservesStatusUnknownFieldsAndAllowedHeaders(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCompatibilityErrorWritersNeverExposeTransportDetails(t *testing.T) {
|
||||||
|
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
write func(http.ResponseWriter)
|
||||||
|
}{
|
||||||
|
{name: "openai", write: func(w http.ResponseWriter) {
|
||||||
|
writeProtocolError(w, clients.ProtocolOpenAIResponses, http.StatusOK, raw, map[string]any{
|
||||||
|
"provider": "secret-provider", "endpoint": "https://private.example.invalid", "bucket": "private-bucket",
|
||||||
|
}, "response_read_error")
|
||||||
|
}},
|
||||||
|
{name: "volces", write: func(w http.ResponseWriter) {
|
||||||
|
writeVolcesError(w, http.StatusOK, raw, "response_read_error")
|
||||||
|
}},
|
||||||
|
{name: "kling", write: func(w http.ResponseWriter) {
|
||||||
|
writeKlingCompatError(w, http.StatusOK, raw, "response_read_error")
|
||||||
|
}},
|
||||||
|
{name: "keling", write: func(w http.ResponseWriter) {
|
||||||
|
writeKelingCompatError(w, "request-1", newKelingCompatError(http.StatusOK, 5001, raw))
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
test.write(recorder)
|
||||||
|
if recorder.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||||
|
}
|
||||||
|
body := recorder.Body.String()
|
||||||
|
if strings.Contains(body, "10.42.0.72") || strings.Contains(body, "47.77.191.126") ||
|
||||||
|
strings.Contains(body, "secret-provider") || strings.Contains(body, "private.example.invalid") || strings.Contains(body, "private-bucket") ||
|
||||||
|
!strings.Contains(body, "upstream_connection_interrupted") {
|
||||||
|
t.Fatalf("transport details were not standardized: %s", body)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCompatibilityStatusMappings(t *testing.T) {
|
func TestCompatibilityStatusMappings(t *testing.T) {
|
||||||
for internal, want := range map[string]string{
|
for internal, want := range map[string]string{
|
||||||
"queued": "queued", "running": "running", "succeeded": "succeeded", "failed": "failed", "cancelled": "cancelled",
|
"queued": "queued", "running": "running", "succeeded": "succeeded", "failed": "failed", "cancelled": "cancelled",
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
@@ -58,14 +59,21 @@ func writeEasyAIAsyncError(w http.ResponseWriter, status int, message string, de
|
|||||||
if len(codes) > 0 {
|
if len(codes) > 0 {
|
||||||
code = strings.TrimSpace(codes[0])
|
code = strings.TrimSpace(codes[0])
|
||||||
}
|
}
|
||||||
|
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||||
|
standard = publicErrorWithRetryAfter(standard, details)
|
||||||
|
publicerror.Observe(standard)
|
||||||
|
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
|
||||||
errorPayload := map[string]any{
|
errorPayload := map[string]any{
|
||||||
"message": message,
|
"message": message,
|
||||||
"status": status,
|
"status": status,
|
||||||
|
"code": code,
|
||||||
|
"category": standard.Category,
|
||||||
|
"httpStatus": standard.HTTPStatus,
|
||||||
|
"retryable": standard.Retryable,
|
||||||
|
"action": standard.Action,
|
||||||
|
"version": standard.Version,
|
||||||
}
|
}
|
||||||
if code != "" {
|
for key, value := range safePublicErrorDetails(details, standard, false) {
|
||||||
errorPayload["code"] = code
|
|
||||||
}
|
|
||||||
for key, value := range details {
|
|
||||||
errorPayload[key] = value
|
errorPayload[key] = value
|
||||||
}
|
}
|
||||||
response := map[string]any{
|
response := map[string]any{
|
||||||
@@ -158,8 +166,14 @@ func easyAITaskResultResponse(task store.GatewayTask) map[string]any {
|
|||||||
if message != "" {
|
if message != "" {
|
||||||
response["message"] = message
|
response["message"] = message
|
||||||
}
|
}
|
||||||
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" {
|
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" || status == "failed" {
|
||||||
response["code"] = code
|
standard := publicTaskError(task)
|
||||||
|
if code != "" && task.ErrorCode == "" {
|
||||||
|
standard = publicerror.WithIDs(publicerror.FromFields(code, message, 0, false), task.RequestID, task.ID)
|
||||||
|
}
|
||||||
|
response["code"] = standard.Code
|
||||||
|
response["message"] = standard.Message
|
||||||
|
response["error"] = standard
|
||||||
}
|
}
|
||||||
return response
|
return response
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ const maxGatewayUploadBytes = 256 << 20
|
|||||||
|
|
||||||
// uploadFile godoc
|
// uploadFile godoc
|
||||||
// @Summary 上传文件
|
// @Summary 上传文件
|
||||||
// @Description 上传文件到配置的文件存储通道；没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。
|
// @Description 上传文件到配置的对象存储通道；所有通道失败时返回标准化存储错误，不写入本机静态目录。单文件最大 256MiB。
|
||||||
// @Tags files
|
// @Tags files
|
||||||
// @Accept multipart/form-data
|
// @Accept multipart/form-data
|
||||||
// @Produce json
|
// @Produce json
|
||||||
@@ -55,11 +55,8 @@ func (s *Server) uploadFile(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("upload file failed", "error", err)
|
s.logger.Error("upload file failed", "error", err)
|
||||||
status := http.StatusBadGateway
|
status := statusFromRunError(err)
|
||||||
if clients.ErrorCode(err) == "upload_no_channel" {
|
writeError(w, status, err.Error(), clients.ErrorCode(err))
|
||||||
status = http.StatusServiceUnavailable
|
|
||||||
}
|
|
||||||
writeError(w, status, err.Error())
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, easyAIFileUploadResponse(upload))
|
writeJSON(w, http.StatusOK, easyAIFileUploadResponse(upload))
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
@@ -159,7 +160,7 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err)
|
s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err)
|
||||||
status := http.StatusBadRequest
|
status := http.StatusBadRequest
|
||||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
|
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
|
||||||
status = http.StatusBadGateway
|
status = http.StatusBadGateway
|
||||||
}
|
}
|
||||||
writeGeminiTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
writeGeminiTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
||||||
@@ -318,14 +319,13 @@ func (s *Server) writeGeminiGenerateContentStream(runCtx context.Context, w http
|
|||||||
}
|
}
|
||||||
applyRunErrorHeaders(w, runErr)
|
applyRunErrorHeaders(w, runErr)
|
||||||
if !nativePassthrough && !convertedFrames {
|
if !nativePassthrough && !convertedFrames {
|
||||||
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, clients.ProtocolGeminiGenerateContent) {
|
|
||||||
writeWireResponse(w, wire)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeGeminiTaskError(statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
|
writeGeminiTaskError(statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
|
||||||
} else if !nativePassthrough {
|
} else if !nativePassthrough {
|
||||||
status := statusFromRunError(runErr)
|
standard := publicerror.FromFields(runErrorCode(runErr), runErrorMessage(runErr), statusFromRunError(runErr), clients.IsRetryable(runErr))
|
||||||
writeGeminiSSEFrame(w, geminiErrorEnvelope(status, runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr)))
|
standard = publicErrorWithRetryAfter(standard, runErrorDetails(runErr))
|
||||||
|
publicerror.Observe(standard)
|
||||||
|
details := safePublicErrorDetails(runErrorDetails(runErr), standard, true)
|
||||||
|
writeGeminiSSEFrame(w, geminiErrorEnvelope(standard.HTTPStatus, standard.Message, details, standard.Code))
|
||||||
if flusher != nil {
|
if flusher != nil {
|
||||||
flusher.Flush()
|
flusher.Flush()
|
||||||
}
|
}
|
||||||
@@ -683,10 +683,7 @@ func (s *Server) finalizeGeminiFilesUpload(w http.ResponseWriter, r *http.Reques
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Gemini files upload failed", "error", err)
|
s.logger.Error("Gemini files upload failed", "error", err)
|
||||||
status := http.StatusBadGateway
|
status := statusFromRunError(err)
|
||||||
if clients.ErrorCode(err) == "upload_no_channel" {
|
|
||||||
status = http.StatusServiceUnavailable
|
|
||||||
}
|
|
||||||
writeProtocolError(w, clients.ProtocolGeminiGenerateContent, status, err.Error(), nil, clients.ErrorCode(err))
|
writeProtocolError(w, clients.ProtocolGeminiGenerateContent, status, err.Error(), nil, clients.ErrorCode(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1154,7 +1154,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
|
|||||||
body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind)
|
body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
status := http.StatusBadRequest
|
status := http.StatusBadRequest
|
||||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
|
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
|
||||||
status = http.StatusBadGateway
|
status = http.StatusBadGateway
|
||||||
}
|
}
|
||||||
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
||||||
@@ -1195,7 +1195,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Warn("prepare task request failed", "kind", kind, "error", err)
|
s.logger.Warn("prepare task request failed", "kind", kind, "error", err)
|
||||||
status := http.StatusBadRequest
|
status := http.StatusBadRequest
|
||||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
|
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
|
||||||
status = http.StatusBadGateway
|
status = http.StatusBadGateway
|
||||||
}
|
}
|
||||||
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
||||||
@@ -1556,10 +1556,6 @@ func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.Response
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
applyRunErrorHeaders(w, runErr)
|
applyRunErrorHeaders(w, runErr)
|
||||||
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, targetProtocol) {
|
|
||||||
writeWireResponse(w, wire)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if targetProtocol != "" {
|
if targetProtocol != "" {
|
||||||
writeProtocolError(w, targetProtocol, statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
|
writeProtocolError(w, targetProtocol, statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
|
||||||
return
|
return
|
||||||
@@ -1673,6 +1669,8 @@ func modelNameFromValue(value any) string {
|
|||||||
|
|
||||||
func statusFromRunError(err error) int {
|
func statusFromRunError(err error) int {
|
||||||
switch {
|
switch {
|
||||||
|
case clients.ErrorCode(err) == "storage_write_failed" || clients.ErrorCode(err) == "storage_read_failed" || clients.ErrorCode(err) == "storage_config_invalid" || clients.ErrorCode(err) == "storage_auth_failed":
|
||||||
|
return http.StatusServiceUnavailable
|
||||||
case clients.ErrorCode(err) == "binary_result_expired":
|
case clients.ErrorCode(err) == "binary_result_expired":
|
||||||
return http.StatusGone
|
return http.StatusGone
|
||||||
case clients.ErrorCode(err) == "binary_result_corrupted" || clients.ErrorCode(err) == "result_binary_not_materialized":
|
case clients.ErrorCode(err) == "binary_result_corrupted" || clients.ErrorCode(err) == "result_binary_not_materialized":
|
||||||
@@ -1949,7 +1947,7 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, map[string]any{
|
writeJSON(w, http.StatusOK, map[string]any{
|
||||||
"items": result.Items,
|
"items": publicTaskList(result.Items),
|
||||||
"total": result.Total,
|
"total": result.Total,
|
||||||
"page": result.Page,
|
"page": result.Page,
|
||||||
"pageSize": result.PageSize,
|
"pageSize": result.PageSize,
|
||||||
@@ -2034,7 +2032,7 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeStoredBinaryResultError(w, err)
|
writeStoredBinaryResultError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, task)
|
writeJSON(w, http.StatusOK, publicGatewayTask(task))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if store.IsNotFound(err) {
|
if store.IsNotFound(err) {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -70,6 +71,7 @@ type KelingCompatibleEnvelope struct {
|
|||||||
Message string `json:"message" example:"SUCCEED"`
|
Message string `json:"message" example:"SUCCEED"`
|
||||||
RequestID string `json:"request_id"`
|
RequestID string `json:"request_id"`
|
||||||
Data any `json:"data,omitempty"`
|
Data any `json:"data,omitempty"`
|
||||||
|
Error *publicerror.Error `json:"error,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type kelingCompatError struct {
|
type kelingCompatError struct {
|
||||||
@@ -77,6 +79,7 @@ type kelingCompatError struct {
|
|||||||
Code int
|
Code int
|
||||||
Message string
|
Message string
|
||||||
RequestID string
|
RequestID string
|
||||||
|
PublicError *publicerror.Error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *kelingCompatError) Error() string {
|
func (e *kelingCompatError) Error() string {
|
||||||
@@ -200,10 +203,6 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
task, createErr = s.waitForCompatibilitySubmission(r, task)
|
task, createErr = s.waitForCompatibilitySubmission(r, task)
|
||||||
if createErr != nil {
|
if createErr != nil {
|
||||||
if wire := clients.ErrorWireResponse(createErr); wireResponseMatches(wire, clients.ProtocolKlingV1Omni) {
|
|
||||||
writeWireResponse(w, wire)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeKelingCompatError(w, requestID, kelingCompatGatewayError(createErr))
|
writeKelingCompatError(w, requestID, kelingCompatGatewayError(createErr))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -690,11 +689,13 @@ func kelingCompatTaskData(task store.GatewayTask) map[string]any {
|
|||||||
"enabled": kelingCompatTaskWatermark(task.Request),
|
"enabled": kelingCompatTaskWatermark(task.Request),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if message := kelingCompatTaskMessage(task); message != "" {
|
|
||||||
data["task_status_msg"] = message
|
|
||||||
}
|
|
||||||
if kelingCompatTaskStatus(task.Status) == "failed" {
|
if kelingCompatTaskStatus(task.Status) == "failed" {
|
||||||
data["task_status_code"] = kelingCompatBusinessCode(task.ErrorCode, kelingCompatTaskMessage(task))
|
standard := publicTaskError(task)
|
||||||
|
data["task_status_msg"] = standard.Message
|
||||||
|
data["task_status_code"] = kelingCompatBusinessCode(standard.Code, standard.Message)
|
||||||
|
data["error"] = standard
|
||||||
|
} else if message := kelingCompatTaskMessage(task); message != "" {
|
||||||
|
data["task_status_msg"] = message
|
||||||
}
|
}
|
||||||
if videos := kelingCompatTaskVideos(task.Result); len(videos) > 0 {
|
if videos := kelingCompatTaskVideos(task.Result); len(videos) > 0 {
|
||||||
data["task_result"] = map[string]any{"videos": videos}
|
data["task_result"] = map[string]any{"videos": videos}
|
||||||
@@ -770,6 +771,9 @@ func firstKelingCompatValue(values ...any) any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func kelingCompatTaskMessage(task store.GatewayTask) string {
|
func kelingCompatTaskMessage(task store.GatewayTask) string {
|
||||||
|
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" || task.Error != "" {
|
||||||
|
return publicTaskError(task).Message
|
||||||
|
}
|
||||||
return strings.TrimSpace(firstNonEmpty(task.ErrorMessage, task.Error, task.Message))
|
return strings.TrimSpace(firstNonEmpty(task.ErrorMessage, task.Error, task.Message))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -825,8 +829,11 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
|
|||||||
return newKelingCompatError(http.StatusInternalServerError, 5000, "unknown gateway error")
|
return newKelingCompatError(http.StatusInternalServerError, 5000, "unknown gateway error")
|
||||||
}
|
}
|
||||||
codeText := clients.ErrorCode(err)
|
codeText := clients.ErrorCode(err)
|
||||||
businessCode := kelingCompatBusinessCode(codeText, err.Error())
|
status := statusFromRunError(err)
|
||||||
status := http.StatusInternalServerError
|
standard := publicerror.FromFields(codeText, err.Error(), status, clients.IsRetryable(err))
|
||||||
|
publicerror.Observe(standard)
|
||||||
|
businessCode := kelingCompatBusinessCode(standard.Code, standard.Message)
|
||||||
|
status = standard.HTTPStatus
|
||||||
switch businessCode {
|
switch businessCode {
|
||||||
case 1101:
|
case 1101:
|
||||||
status = http.StatusPaymentRequired
|
status = http.StatusPaymentRequired
|
||||||
@@ -841,7 +848,9 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
|
|||||||
case 5001:
|
case 5001:
|
||||||
status = http.StatusBadGateway
|
status = http.StatusBadGateway
|
||||||
}
|
}
|
||||||
return newKelingCompatError(status, businessCode, err.Error())
|
result := newKelingCompatError(status, businessCode, standard.Message)
|
||||||
|
result.PublicError = &standard
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func kelingCompatBusinessCode(errorCode string, message string) int {
|
func kelingCompatBusinessCode(errorCode string, message string) int {
|
||||||
@@ -923,9 +932,18 @@ func writeKelingCompatError(w http.ResponseWriter, requestID string, err *keling
|
|||||||
if status == 0 {
|
if status == 0 {
|
||||||
status = http.StatusInternalServerError
|
status = http.StatusInternalServerError
|
||||||
}
|
}
|
||||||
|
standard := err.PublicError
|
||||||
|
if standard == nil {
|
||||||
|
value := publicerror.FromFields("", err.Message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||||
|
value = publicerror.WithIDs(value, requestID, "")
|
||||||
|
publicerror.Observe(value)
|
||||||
|
standard = &value
|
||||||
|
}
|
||||||
|
status = standard.HTTPStatus
|
||||||
writeJSON(w, status, KelingCompatibleEnvelope{
|
writeJSON(w, status, KelingCompatibleEnvelope{
|
||||||
Code: err.Code,
|
Code: err.Code,
|
||||||
Message: err.Message,
|
Message: standard.Message,
|
||||||
RequestID: requestID,
|
RequestID: requestID,
|
||||||
|
Error: standard,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -191,10 +192,6 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
|
|||||||
}
|
}
|
||||||
task, err = s.waitForCompatibilitySubmission(r, task)
|
task, err = s.waitForCompatibilitySubmission(r, task)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, targetProtocol) {
|
|
||||||
writeWireResponse(w, wire)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
|
writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -667,7 +664,9 @@ func klingV1TaskData(task store.GatewayTask) map[string]any {
|
|||||||
"created_at": task.CreatedAt.UnixMilli(), "updated_at": task.UpdatedAt.UnixMilli(),
|
"created_at": task.CreatedAt.UnixMilli(), "updated_at": task.UpdatedAt.UnixMilli(),
|
||||||
}
|
}
|
||||||
if task.ErrorMessage != "" || task.Error != "" {
|
if task.ErrorMessage != "" || task.Error != "" {
|
||||||
data["task_status_msg"] = firstNonEmpty(task.ErrorMessage, task.Error)
|
standard := publicTaskError(task)
|
||||||
|
data["task_status_msg"] = standard.Message
|
||||||
|
data["error"] = standard
|
||||||
}
|
}
|
||||||
if watermarkInfo, ok := task.Request["watermark_info"].(map[string]any); ok {
|
if watermarkInfo, ok := task.Request["watermark_info"].(map[string]any); ok {
|
||||||
data["watermark_info"] = watermarkInfo
|
data["watermark_info"] = watermarkInfo
|
||||||
@@ -698,7 +697,9 @@ func klingV2TaskData(task store.GatewayTask) map[string]any {
|
|||||||
"external_id": task.ExternalTaskID,
|
"external_id": task.ExternalTaskID,
|
||||||
}
|
}
|
||||||
if message := firstNonEmpty(task.ErrorMessage, task.Error); message != "" {
|
if message := firstNonEmpty(task.ErrorMessage, task.Error); message != "" {
|
||||||
data["message"] = message
|
standard := publicTaskError(task)
|
||||||
|
data["message"] = standard.Message
|
||||||
|
data["error"] = standard
|
||||||
}
|
}
|
||||||
if outputs := klingV2Outputs(task); len(outputs) > 0 {
|
if outputs := klingV2Outputs(task); len(outputs) > 0 {
|
||||||
data["outputs"] = outputs
|
data["outputs"] = outputs
|
||||||
@@ -827,14 +828,14 @@ func decodeKlingJSON(r *http.Request, target any) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeKlingCompatError(w http.ResponseWriter, status int, message string, code string) {
|
func writeKlingCompatError(w http.ResponseWriter, status int, message string, code string) {
|
||||||
if strings.TrimSpace(code) == "" {
|
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||||
code = "invalid_request"
|
publicerror.Observe(standard)
|
||||||
}
|
writeJSON(w, standard.HTTPStatus, map[string]any{
|
||||||
writeJSON(w, status, map[string]any{
|
"code": klingCompatErrorCode(standard.HTTPStatus),
|
||||||
"code": klingCompatErrorCode(status),
|
"message": standard.Message,
|
||||||
"message": message,
|
|
||||||
"request_id": "",
|
"request_id": "",
|
||||||
"error": code,
|
"error": standard.Code,
|
||||||
|
"public_error": standard,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package httpapi
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -35,6 +36,14 @@ type ErrorPayload struct {
|
|||||||
Message string `json:"message" example:"invalid json body"`
|
Message string `json:"message" example:"invalid json body"`
|
||||||
Status int `json:"status" example:"400"`
|
Status int `json:"status" example:"400"`
|
||||||
Code string `json:"code,omitempty" example:"rate_limit"`
|
Code string `json:"code,omitempty" example:"rate_limit"`
|
||||||
|
Category string `json:"category,omitempty" example:"rate_limit"`
|
||||||
|
HTTPStatus int `json:"httpStatus,omitempty" example:"429"`
|
||||||
|
Retryable bool `json:"retryable" example:"true"`
|
||||||
|
Action string `json:"action,omitempty" example:"retry_after"`
|
||||||
|
Version string `json:"version,omitempty" example:"v1"`
|
||||||
|
RetryAfterSeconds int `json:"retryAfterSeconds,omitempty" example:"2"`
|
||||||
|
RequestID string `json:"requestId,omitempty"`
|
||||||
|
TaskID string `json:"taskId,omitempty"`
|
||||||
Type string `json:"type,omitempty" example:"invalid_request_error"`
|
Type string `json:"type,omitempty" example:"invalid_request_error"`
|
||||||
Param any `json:"param,omitempty"`
|
Param any `json:"param,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -62,12 +71,7 @@ type GeminiErrorStatus struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type VolcesErrorEnvelope struct {
|
type VolcesErrorEnvelope struct {
|
||||||
Error VolcesErrorPayload `json:"error"`
|
Error publicerror.Error `json:"error"`
|
||||||
}
|
|
||||||
|
|
||||||
type VolcesErrorPayload struct {
|
|
||||||
Code string `json:"code" example:"invalid_parameter"`
|
|
||||||
Message string `json:"message" example:"model is required"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type VolcesContentsGenerationTaskResponse struct {
|
type VolcesContentsGenerationTaskResponse struct {
|
||||||
@@ -86,6 +90,7 @@ type KlingErrorEnvelope struct {
|
|||||||
Message string `json:"message" example:"invalid parameter"`
|
Message string `json:"message" example:"invalid parameter"`
|
||||||
RequestID string `json:"request_id"`
|
RequestID string `json:"request_id"`
|
||||||
Error string `json:"error,omitempty" example:"invalid_parameter"`
|
Error string `json:"error,omitempty" example:"invalid_parameter"`
|
||||||
|
PublicError *publicerror.Error `json:"public_error,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type AuthResponse struct {
|
type AuthResponse struct {
|
||||||
@@ -232,6 +237,14 @@ type FileStorageChannelListResponse struct {
|
|||||||
Items []store.FileStorageChannel `json:"items"`
|
Items []store.FileStorageChannel `json:"items"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type FileStorageChannelTestResponse struct {
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
PutSucceeded bool `json:"putSucceeded"`
|
||||||
|
HeadSucceeded bool `json:"headSucceeded"`
|
||||||
|
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||||
|
DurationMS int64 `json:"durationMs"`
|
||||||
|
}
|
||||||
|
|
||||||
type FileUploadResponse struct {
|
type FileUploadResponse struct {
|
||||||
ID string `json:"id,omitempty" example:"file_abc123"`
|
ID string `json:"id,omitempty" example:"file_abc123"`
|
||||||
URL string `json:"url,omitempty" example:"/static/uploaded/upload-abc123.png"`
|
URL string `json:"url,omitempty" example:"/static/uploaded/upload-abc123.png"`
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func publicGatewayTask(task store.GatewayTask) store.GatewayTask {
|
||||||
|
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" {
|
||||||
|
value := publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error), storedTaskErrorStatus(task.ErrorCode), false)
|
||||||
|
if task.PublicError != nil && task.PublicError.Code != "" {
|
||||||
|
value = *task.PublicError
|
||||||
|
}
|
||||||
|
value = publicerror.WithIDs(value, task.RequestID, task.ID)
|
||||||
|
publicerror.Observe(value)
|
||||||
|
task.PublicError = &value
|
||||||
|
task.ErrorCode = value.Code
|
||||||
|
task.ErrorMessage = value.Message
|
||||||
|
task.Error = value.Message
|
||||||
|
}
|
||||||
|
for index := range task.Attempts {
|
||||||
|
attempt := &task.Attempts[index]
|
||||||
|
if attempt.ErrorCode == "" && attempt.ErrorMessage == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
status := attempt.StatusCode
|
||||||
|
if status <= 0 {
|
||||||
|
status = http.StatusBadGateway
|
||||||
|
}
|
||||||
|
value := publicerror.FromFields(attempt.ErrorCode, attempt.ErrorMessage, status, attempt.Retryable)
|
||||||
|
if attempt.PublicError != nil && attempt.PublicError.Code != "" {
|
||||||
|
value = *attempt.PublicError
|
||||||
|
}
|
||||||
|
value = publicerror.WithIDs(value, attempt.RequestID, task.ID)
|
||||||
|
attempt.PublicError = &value
|
||||||
|
attempt.ErrorCode = value.Code
|
||||||
|
attempt.ErrorMessage = value.Message
|
||||||
|
}
|
||||||
|
return task
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicTaskError(task store.GatewayTask) publicerror.Error {
|
||||||
|
if task.PublicError != nil && task.PublicError.Code != "" {
|
||||||
|
return publicerror.WithIDs(*task.PublicError, task.RequestID, task.ID)
|
||||||
|
}
|
||||||
|
status := storedTaskErrorStatus(task.ErrorCode)
|
||||||
|
if status <= 0 {
|
||||||
|
status = http.StatusBadGateway
|
||||||
|
}
|
||||||
|
return publicerror.WithIDs(publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error, task.Message), status, false), task.RequestID, task.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicTaskList(items []store.GatewayTask) []store.GatewayTask {
|
||||||
|
out := make([]store.GatewayTask, len(items))
|
||||||
|
for index, task := range items {
|
||||||
|
out[index] = publicGatewayTask(task)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicErrorMap(value publicerror.Error) map[string]any {
|
||||||
|
out := map[string]any{
|
||||||
|
"code": value.Code, "message": value.Message, "category": value.Category,
|
||||||
|
"httpStatus": value.HTTPStatus, "retryable": value.Retryable, "action": value.Action,
|
||||||
|
"version": value.Version,
|
||||||
|
}
|
||||||
|
if value.RetryAfterSeconds > 0 {
|
||||||
|
out["retryAfterSeconds"] = value.RetryAfterSeconds
|
||||||
|
}
|
||||||
|
if requestID := strings.TrimSpace(value.RequestID); requestID != "" {
|
||||||
|
out["requestId"] = requestID
|
||||||
|
}
|
||||||
|
if taskID := strings.TrimSpace(value.TaskID); taskID != "" {
|
||||||
|
out["taskId"] = taskID
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicErrorWithRetryAfter(value publicerror.Error, details map[string]any) publicerror.Error {
|
||||||
|
if value.RetryAfterSeconds > 0 || details == nil {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
switch typed := details["retryAfterSeconds"].(type) {
|
||||||
|
case int:
|
||||||
|
value.RetryAfterSeconds = typed
|
||||||
|
case int32:
|
||||||
|
value.RetryAfterSeconds = int(typed)
|
||||||
|
case int64:
|
||||||
|
value.RetryAfterSeconds = int(typed)
|
||||||
|
case float64:
|
||||||
|
value.RetryAfterSeconds = int(typed)
|
||||||
|
}
|
||||||
|
if value.RetryAfterSeconds < 0 {
|
||||||
|
value.RetryAfterSeconds = 0
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func safePublicErrorDetails(details map[string]any, value publicerror.Error, includePublicError bool) map[string]any {
|
||||||
|
out := map[string]any{}
|
||||||
|
for _, key := range []string{"param", "retryAfterSeconds", "recoveryAt", "rateLimit", "pricing"} {
|
||||||
|
if item, ok := details[key]; ok && item != nil {
|
||||||
|
out[key] = item
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if value.Category == "request" {
|
||||||
|
for _, key := range []string{"reason", "diagnosticId"} {
|
||||||
|
if item, ok := details[key]; ok && item != nil {
|
||||||
|
out[key] = item
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if includePublicError {
|
||||||
|
out["publicError"] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -38,12 +37,6 @@ type decodedRequestAsset struct {
|
|||||||
ContentType string
|
ContentType string
|
||||||
}
|
}
|
||||||
|
|
||||||
type requestAssetOptions struct {
|
|
||||||
RequirePublicURL bool
|
|
||||||
UploadScene string
|
|
||||||
Source string
|
|
||||||
}
|
|
||||||
|
|
||||||
type requestAssetLock struct {
|
type requestAssetLock struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
refs int
|
refs int
|
||||||
@@ -115,6 +108,19 @@ func (s *Server) prepareRequestAssetRefs(ctx context.Context, body map[string]an
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path []string, siblings map[string]any) (any, error) {
|
func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path []string, siblings map[string]any) (any, error) {
|
||||||
|
if decoded, ok, err := requestAssetFromBinaryValue(requestAssetPathKey(path), path, value, siblings); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if ok {
|
||||||
|
if err := s.acquireMediaRequestSlot(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer s.releaseMediaRequestSlot()
|
||||||
|
ref, err := s.ensureRequestAsset(ctx, decoded)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return requestAssetWrapper(ref), nil
|
||||||
|
}
|
||||||
switch typed := value.(type) {
|
switch typed := value.(type) {
|
||||||
case map[string]any:
|
case map[string]any:
|
||||||
if typed["assetRef"] != nil {
|
if typed["assetRef"] != nil {
|
||||||
@@ -156,6 +162,83 @@ func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path [
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func requestAssetFromBinaryValue(key string, path []string, value any, siblings map[string]any) (decodedRequestAsset, bool, error) {
|
||||||
|
var payload []byte
|
||||||
|
contentType := ""
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case []byte:
|
||||||
|
payload = append([]byte(nil), typed...)
|
||||||
|
case map[string]any:
|
||||||
|
if !strings.EqualFold(strings.TrimSpace(stringFromRequestAny(typed["type"])), "buffer") {
|
||||||
|
return decodedRequestAsset{}, false, nil
|
||||||
|
}
|
||||||
|
contentType = firstNonEmptyRequestString(typed, "contentType", "mimeType", "mime_type")
|
||||||
|
switch data := typed["data"].(type) {
|
||||||
|
case []byte:
|
||||||
|
payload = append([]byte(nil), data...)
|
||||||
|
case []any:
|
||||||
|
var ok bool
|
||||||
|
payload, ok = requestBytesFromNumberArray(data)
|
||||||
|
if !ok {
|
||||||
|
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data must contain byte values"))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data is required"))
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
if !strictRequestBinaryArrayField(key, path) {
|
||||||
|
return decodedRequestAsset{}, false, nil
|
||||||
|
}
|
||||||
|
var ok bool
|
||||||
|
payload, ok = requestBytesFromNumberArray(typed)
|
||||||
|
if !ok {
|
||||||
|
return decodedRequestAsset{}, false, nil
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return decodedRequestAsset{}, false, nil
|
||||||
|
}
|
||||||
|
if len(payload) == 0 {
|
||||||
|
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("binary media payload is empty"))
|
||||||
|
}
|
||||||
|
return decodedRequestAsset{
|
||||||
|
Bytes: payload,
|
||||||
|
ContentType: requestAssetContentType(contentType, payload, key, path, siblings),
|
||||||
|
}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestAssetPathKey(path []string) string {
|
||||||
|
for index := len(path) - 1; index >= 0; index-- {
|
||||||
|
value := strings.TrimSpace(path[index])
|
||||||
|
if value != "" && !strings.HasPrefix(value, "[") {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func strictRequestBinaryArrayField(key string, path []string) bool {
|
||||||
|
value := strings.ToLower(strings.TrimSpace(key))
|
||||||
|
if value == "bytes" || value == "buffer" || strings.Contains(value, "binary") || strings.Contains(value, "buffer") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return requestMediaKind(key, path, nil) != "" && (value == "data" || value == "content")
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestBytesFromNumberArray(values []any) ([]byte, bool) {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
payload := make([]byte, len(values))
|
||||||
|
for index, value := range values {
|
||||||
|
number, ok := value.(float64)
|
||||||
|
if !ok || number < 0 || number > 255 || number != float64(byte(number)) {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
payload[index] = byte(number)
|
||||||
|
}
|
||||||
|
return payload, true
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Server) prepareRequestAssetField(ctx context.Context, key string, path []string, value any, siblings map[string]any) (map[string]any, bool, error) {
|
func (s *Server) prepareRequestAssetField(ctx context.Context, key string, path []string, value any, siblings map[string]any) (map[string]any, bool, error) {
|
||||||
text, ok := value.(string)
|
text, ok := value.(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -291,68 +374,43 @@ func requestAssetFromValue(key string, path []string, value any, siblings map[st
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
|
func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
|
||||||
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
|
|
||||||
UploadScene: store.FileStorageSceneRequestAsset,
|
|
||||||
Source: "ai-gateway-request",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) ensurePublicRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
|
|
||||||
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
|
|
||||||
RequirePublicURL: true,
|
|
||||||
UploadScene: store.FileStorageSceneUpload,
|
|
||||||
Source: "ai-gateway-form-data",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded decodedRequestAsset, options requestAssetOptions) (map[string]any, error) {
|
|
||||||
sum := sha256.Sum256(decoded.Bytes)
|
sum := sha256.Sum256(decoded.Bytes)
|
||||||
sha := hex.EncodeToString(sum[:])
|
sha := hex.EncodeToString(sum[:])
|
||||||
contentType := strings.TrimSpace(decoded.ContentType)
|
contentType := strings.TrimSpace(decoded.ContentType)
|
||||||
if contentType == "" {
|
if contentType == "" {
|
||||||
contentType = "application/octet-stream"
|
contentType = "application/octet-stream"
|
||||||
}
|
}
|
||||||
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + options.UploadScene + "\x00" + strconv.FormatBool(options.RequirePublicURL))
|
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + store.FileStorageSceneRequestAsset)
|
||||||
defer release()
|
defer release()
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
if existing, ok, err := s.store.FindRequestAsset(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
if existing, ok, err := s.store.FindRequestAsset(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||||
return nil, err
|
return nil, err
|
||||||
} else if ok && requestAssetStillUsable(existing, now) {
|
} else if ok && requestAssetStillUsable(existing, now) {
|
||||||
ref := requestAssetRef(existing)
|
ref := requestAssetRef(existing)
|
||||||
if !options.RequirePublicURL || requestAssetRefHasPublicURL(ref) {
|
|
||||||
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return ref, nil
|
return ref, nil
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
uploadScene := strings.TrimSpace(options.UploadScene)
|
|
||||||
if uploadScene == "" {
|
|
||||||
uploadScene = store.FileStorageSceneRequestAsset
|
|
||||||
}
|
|
||||||
source := strings.TrimSpace(options.Source)
|
|
||||||
if source == "" {
|
|
||||||
source = "ai-gateway-request"
|
|
||||||
}
|
|
||||||
upload, err := s.runner.UploadFile(ctx, runner.FileUploadPayload{
|
upload, err := s.runner.UploadFile(ctx, runner.FileUploadPayload{
|
||||||
Bytes: decoded.Bytes,
|
Bytes: decoded.Bytes,
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
FileName: requestAssetFileName(sha, contentType),
|
FileName: requestAssetFileName(sha, contentType),
|
||||||
Scene: uploadScene,
|
Scene: store.FileStorageSceneRequestAsset,
|
||||||
Source: source,
|
Source: "ai-gateway-request",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
storageProvider := requestAssetStorageProvider(upload)
|
storageProvider := requestAssetStorageProvider(upload)
|
||||||
|
storageChannelID, storageChannelKey := requestAssetStorageChannel(upload)
|
||||||
|
objectKey := stringFromRequestAny(upload["objectKey"])
|
||||||
|
accessScope := firstNonEmpty(stringFromRequestAny(upload["accessScope"]), "private")
|
||||||
url := stringFromRequestAny(upload["url"])
|
url := stringFromRequestAny(upload["url"])
|
||||||
if url == "" {
|
if url == "" {
|
||||||
return nil, &clients.ClientError{Code: "request_asset_upload_failed", Message: "file storage response did not include url", Retryable: false}
|
return nil, &clients.ClientError{Code: "request_asset_upload_failed", Message: "file storage response did not include url", Retryable: false}
|
||||||
}
|
}
|
||||||
if options.RequirePublicURL && !requestAssetURLIsPublic(storageProvider, url) {
|
|
||||||
return nil, &clients.ClientError{Code: "request_asset_public_url_required", Message: "multipart image assets require a public file storage URL; enable a non-local file storage channel for uploads", Retryable: false}
|
|
||||||
}
|
|
||||||
var expiresAt *time.Time
|
var expiresAt *time.Time
|
||||||
localPath := ""
|
localPath := ""
|
||||||
if storageProvider == "local_static" {
|
if storageProvider == "local_static" {
|
||||||
@@ -366,6 +424,10 @@ func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded deco
|
|||||||
ByteSize: int64(len(decoded.Bytes)),
|
ByteSize: int64(len(decoded.Bytes)),
|
||||||
URL: url,
|
URL: url,
|
||||||
StorageProvider: storageProvider,
|
StorageProvider: storageProvider,
|
||||||
|
StorageChannelID: storageChannelID,
|
||||||
|
StorageChannelKey: storageChannelKey,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
AccessScope: accessScope,
|
||||||
LocalPath: localPath,
|
LocalPath: localPath,
|
||||||
ExpiresAt: expiresAt,
|
ExpiresAt: expiresAt,
|
||||||
})
|
})
|
||||||
@@ -377,6 +439,10 @@ func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded deco
|
|||||||
"contentType": contentType,
|
"contentType": contentType,
|
||||||
"size": len(decoded.Bytes),
|
"size": len(decoded.Bytes),
|
||||||
"storageProvider": storageProvider,
|
"storageProvider": storageProvider,
|
||||||
|
"storageChannelId": storageChannelID,
|
||||||
|
"storageChannelKey": storageChannelKey,
|
||||||
|
"objectKey": objectKey,
|
||||||
|
"accessScope": accessScope,
|
||||||
"expiresAt": timePtrToRFC3339(expiresAt),
|
"expiresAt": timePtrToRFC3339(expiresAt),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -498,22 +564,14 @@ func requestAssetRef(asset store.RequestAsset) map[string]any {
|
|||||||
"contentType": asset.ContentType,
|
"contentType": asset.ContentType,
|
||||||
"size": asset.ByteSize,
|
"size": asset.ByteSize,
|
||||||
"storageProvider": asset.StorageProvider,
|
"storageProvider": asset.StorageProvider,
|
||||||
|
"storageChannelId": asset.StorageChannelID,
|
||||||
|
"storageChannelKey": asset.StorageChannelKey,
|
||||||
|
"objectKey": asset.ObjectKey,
|
||||||
|
"accessScope": asset.AccessScope,
|
||||||
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
|
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func requestAssetRefHasPublicURL(ref map[string]any) bool {
|
|
||||||
return requestAssetURLIsPublic(stringFromRequestAny(ref["storageProvider"]), stringFromRequestAny(ref["url"]))
|
|
||||||
}
|
|
||||||
|
|
||||||
func requestAssetURLIsPublic(storageProvider string, url string) bool {
|
|
||||||
if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
lower := strings.ToLower(strings.TrimSpace(url))
|
|
||||||
return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
|
|
||||||
}
|
|
||||||
|
|
||||||
func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
|
func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
|
||||||
if asset.ExpiredAt != nil {
|
if asset.ExpiredAt != nil {
|
||||||
return false
|
return false
|
||||||
@@ -543,6 +601,11 @@ func requestAssetStorageProvider(upload map[string]any) string {
|
|||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func requestAssetStorageChannel(upload map[string]any) (string, string) {
|
||||||
|
channel, _ := upload["storageChannel"].(map[string]any)
|
||||||
|
return stringFromRequestAny(channel["id"]), stringFromRequestAny(channel["channelKey"])
|
||||||
|
}
|
||||||
|
|
||||||
func requestAssetLocalPath(storageDir string, fileName string) string {
|
func requestAssetLocalPath(storageDir string, fileName string) string {
|
||||||
if strings.TrimSpace(storageDir) == "" {
|
if strings.TrimSpace(storageDir) == "" {
|
||||||
storageDir = config.DefaultLocalUploadedStorageDir
|
storageDir = config.DefaultLocalUploadedStorageDir
|
||||||
|
|||||||
@@ -41,6 +41,33 @@ func TestRequestAssetFromValueDetectsDataURLAndRawBase64(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRequestAssetFromBinaryValueDetectsBufferAndByteArray(t *testing.T) {
|
||||||
|
png := []any{float64(0x89), float64('P'), float64('N'), float64('G')}
|
||||||
|
decoded, ok, err := requestAssetFromBinaryValue("image", []string{"input", "image"}, map[string]any{
|
||||||
|
"type": "Buffer",
|
||||||
|
"data": png,
|
||||||
|
"mimeType": "image/png",
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !ok || decoded.ContentType != "image/png" || len(decoded.Bytes) != 4 || decoded.Bytes[0] != 0x89 {
|
||||||
|
t.Fatalf("unexpected Buffer asset: ok=%v decoded=%+v", ok, decoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
decoded, ok, err = requestAssetFromBinaryValue("bytes", []string{"input_audio", "bytes"}, png, map[string]any{"format": "mp3"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !ok || len(decoded.Bytes) != 4 {
|
||||||
|
t.Fatalf("unexpected byte array asset: ok=%v decoded=%+v", ok, decoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok, err := requestAssetFromBinaryValue("values", []string{"embedding", "values"}, png, nil); err != nil || ok {
|
||||||
|
t.Fatalf("ordinary numeric arrays must remain JSON: ok=%v err=%v", ok, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMediaRequestBodySlotLimitsPreAuthWorkAndCanReleaseEarly(t *testing.T) {
|
func TestMediaRequestBodySlotLimitsPreAuthWorkAndCanReleaseEarly(t *testing.T) {
|
||||||
server := &Server{mediaRequestBodySlots: make(chan struct{}, 2)}
|
server := &Server{mediaRequestBodySlots: make(chan struct{}, 2)}
|
||||||
var critical atomic.Int64
|
var critical atomic.Int64
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
)
|
)
|
||||||
|
|
||||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||||
@@ -18,22 +20,31 @@ func writeError(w http.ResponseWriter, status int, message string, codes ...stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeErrorWithDetails(w http.ResponseWriter, status int, message string, details map[string]any, codes ...string) {
|
func writeErrorWithDetails(w http.ResponseWriter, status int, message string, details map[string]any, codes ...string) {
|
||||||
errorPayload := map[string]any{
|
code := ""
|
||||||
"message": message,
|
|
||||||
"status": status,
|
|
||||||
}
|
|
||||||
if len(codes) > 0 {
|
if len(codes) > 0 {
|
||||||
if code := strings.TrimSpace(codes[0]); code != "" {
|
code = strings.TrimSpace(codes[0])
|
||||||
errorPayload["code"] = code
|
}
|
||||||
if code == "invalid_parameter" || code == "unsupported_response_parameter" {
|
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||||
|
standard = publicErrorWithRetryAfter(standard, details)
|
||||||
|
publicerror.Observe(standard)
|
||||||
|
status = standard.HTTPStatus
|
||||||
|
errorPayload := map[string]any{
|
||||||
|
"message": standard.Message,
|
||||||
|
"status": status,
|
||||||
|
"code": standard.Code,
|
||||||
|
"category": standard.Category,
|
||||||
|
"httpStatus": standard.HTTPStatus,
|
||||||
|
"retryable": standard.Retryable,
|
||||||
|
"action": standard.Action,
|
||||||
|
"version": standard.Version,
|
||||||
|
}
|
||||||
|
if standard.Code == "invalid_parameter" || standard.Code == "unsupported_response_parameter" {
|
||||||
errorPayload["type"] = "invalid_request_error"
|
errorPayload["type"] = "invalid_request_error"
|
||||||
if _, ok := details["param"]; !ok {
|
if _, ok := details["param"]; !ok {
|
||||||
errorPayload["param"] = nil
|
errorPayload["param"] = nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
for key, value := range safePublicErrorDetails(details, standard, false) {
|
||||||
}
|
|
||||||
for key, value := range details {
|
|
||||||
errorPayload[key] = value
|
errorPayload[key] = value
|
||||||
}
|
}
|
||||||
writeJSON(w, status, map[string]any{"error": errorPayload})
|
writeJSON(w, status, map[string]any{"error": errorPayload})
|
||||||
|
|||||||
@@ -286,6 +286,7 @@ func NewServerWithStores(
|
|||||||
mux.Handle("GET /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listFileStorageChannels)))
|
mux.Handle("GET /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listFileStorageChannels)))
|
||||||
mux.Handle("POST /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createFileStorageChannel)))
|
mux.Handle("POST /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createFileStorageChannel)))
|
||||||
mux.Handle("PATCH /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageChannel)))
|
mux.Handle("PATCH /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageChannel)))
|
||||||
|
mux.Handle("POST /api/admin/system/file-storage/channels/{channelID}/test", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.testFileStorageChannel)))
|
||||||
mux.Handle("DELETE /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteFileStorageChannel)))
|
mux.Handle("DELETE /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteFileStorageChannel)))
|
||||||
mux.Handle("GET /api/admin/platforms", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listPlatforms)))
|
mux.Handle("GET /api/admin/platforms", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listPlatforms)))
|
||||||
mux.Handle("POST /api/admin/platforms", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createPlatform)))
|
mux.Handle("POST /api/admin/platforms", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createPlatform)))
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
package httpapi
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -74,6 +77,10 @@ func (s *Server) updateFileStorageSettings(w http.ResponseWriter, r *http.Reques
|
|||||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if legacyLocalStoragePolicy(input.ResultUploadPolicy) {
|
||||||
|
writeError(w, http.StatusBadRequest, "upload_none is no longer supported; configure an object storage channel")
|
||||||
|
return
|
||||||
|
}
|
||||||
settings, err := s.store.UpdateFileStorageSettings(r.Context(), input)
|
settings, err := s.store.UpdateFileStorageSettings(r.Context(), input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("update file storage settings failed", "error", err)
|
s.logger.Error("update file storage settings failed", "error", err)
|
||||||
@@ -155,7 +162,7 @@ func (s *Server) updateClientCustomizationSettings(w http.ResponseWriter, r *htt
|
|||||||
|
|
||||||
// createFileStorageChannel godoc
|
// createFileStorageChannel godoc
|
||||||
// @Summary 创建文件存储通道
|
// @Summary 创建文件存储通道
|
||||||
// @Description 创建文件存储通道，当前主要用于配置 server-main OpenAPI 上传通道。
|
// @Description 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。
|
||||||
// @Tags system
|
// @Tags system
|
||||||
// @Accept json
|
// @Accept json
|
||||||
// @Produce json
|
// @Produce json
|
||||||
@@ -271,6 +278,46 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request
|
|||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// testFileStorageChannel godoc
|
||||||
|
// @Summary 测试对象存储通道
|
||||||
|
// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针，不返回凭据或对象键。
|
||||||
|
// @Tags system
|
||||||
|
// @Produce json
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Param channelID path string true "文件存储通道 ID"
|
||||||
|
// @Success 200 {object} FileStorageChannelTestResponse
|
||||||
|
// @Failure 400 {object} ErrorEnvelope
|
||||||
|
// @Failure 401 {object} ErrorEnvelope
|
||||||
|
// @Failure 403 {object} ErrorEnvelope
|
||||||
|
// @Failure 404 {object} ErrorEnvelope
|
||||||
|
// @Failure 503 {object} ErrorEnvelope
|
||||||
|
// @Router /api/admin/system/file-storage/channels/{channelID}/test [post]
|
||||||
|
func (s *Server) testFileStorageChannel(w http.ResponseWriter, r *http.Request) {
|
||||||
|
channel, err := s.store.GetFileStorageChannel(r.Context(), r.PathValue("channelID"))
|
||||||
|
if err != nil {
|
||||||
|
if store.IsNotFound(err) {
|
||||||
|
writeError(w, http.StatusNotFound, "file storage channel not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.logger.Error("get file storage channel for test failed", "error", err)
|
||||||
|
writeError(w, http.StatusInternalServerError, "get file storage channel failed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if channel.Provider != "aliyun_oss" && channel.Provider != "s3" {
|
||||||
|
writeError(w, http.StatusBadRequest, "connection test is supported for aliyun_oss and s3 channels", "invalid_parameter")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
result, err := s.runner.TestFileStorageChannel(r.Context(), channel)
|
||||||
|
if err != nil {
|
||||||
|
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(r.Context()), channel.ID, err.Error())
|
||||||
|
s.logger.Warn("file storage channel test failed", "channel_id", channel.ID, "provider", channel.Provider, "error", err)
|
||||||
|
writeError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = s.store.MarkFileStorageChannelSuccess(context.WithoutCancel(r.Context()), channel.ID)
|
||||||
|
writeJSON(w, http.StatusOK, result)
|
||||||
|
}
|
||||||
|
|
||||||
func validateFileStorageChannelInput(input store.FileStorageChannelInput, existing *store.FileStorageChannel) string {
|
func validateFileStorageChannelInput(input store.FileStorageChannelInput, existing *store.FileStorageChannel) string {
|
||||||
provider := strings.ToLower(strings.TrimSpace(input.Provider))
|
provider := strings.ToLower(strings.TrimSpace(input.Provider))
|
||||||
if provider == "" {
|
if provider == "" {
|
||||||
@@ -286,6 +333,9 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
|||||||
if status != "enabled" && status != "disabled" {
|
if status != "enabled" && status != "disabled" {
|
||||||
return "status must be enabled or disabled"
|
return "status must be enabled or disabled"
|
||||||
}
|
}
|
||||||
|
if provider != "server_main_openapi" && provider != "aliyun_oss" && provider != "s3" {
|
||||||
|
return "provider must be server_main_openapi, aliyun_oss or s3"
|
||||||
|
}
|
||||||
if provider == "server_main_openapi" {
|
if provider == "server_main_openapi" {
|
||||||
hasAPIKey := false
|
hasAPIKey := false
|
||||||
if input.APIKey != nil {
|
if input.APIKey != nil {
|
||||||
@@ -297,5 +347,83 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
|||||||
return "server-main OpenAPI channel requires API key before enabling"
|
return "server-main OpenAPI channel requires API key before enabling"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if provider == "aliyun_oss" || provider == "s3" {
|
||||||
|
if fileStorageConfigContainsCredential(input.Config) {
|
||||||
|
return "object storage credentials must use the write-only credential fields, not config"
|
||||||
|
}
|
||||||
|
endpoint := fileStorageConfigString(input.Config, "endpoint")
|
||||||
|
if endpoint == "" || fileStorageConfigString(input.Config, "region") == "" || fileStorageConfigString(input.Config, "bucket") == "" {
|
||||||
|
return "object storage channel requires config.endpoint, config.region and config.bucket"
|
||||||
|
}
|
||||||
|
if !validFileStorageBaseURL(endpoint) {
|
||||||
|
return "object storage config.endpoint must be an http or https URL without embedded credentials"
|
||||||
|
}
|
||||||
|
if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
|
||||||
|
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
|
||||||
|
}
|
||||||
|
accessKeyID := input.AccessKeyID
|
||||||
|
if accessKeyID == nil {
|
||||||
|
accessKeyID = input.AccessKey
|
||||||
|
}
|
||||||
|
accessKeySecret := input.AccessKeySecret
|
||||||
|
if accessKeySecret == nil {
|
||||||
|
accessKeySecret = input.SecretKey
|
||||||
|
}
|
||||||
|
hasAccessKeyID := fileStorageCredentialPresent(accessKeyID, existing, func(item *store.FileStorageChannel) string { return item.AccessKeyID })
|
||||||
|
hasAccessKeySecret := fileStorageCredentialPresent(accessKeySecret, existing, func(item *store.FileStorageChannel) string { return item.AccessKeySecret })
|
||||||
|
if status == "enabled" && (!hasAccessKeyID || !hasAccessKeySecret) {
|
||||||
|
return "object storage channel requires accessKeyId and accessKeySecret before enabling"
|
||||||
|
}
|
||||||
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validFileStorageBaseURL(value string) bool {
|
||||||
|
parsed, err := url.Parse(strings.TrimSpace(value))
|
||||||
|
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileStorageConfigContainsCredential(value any) bool {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
for key, item := range typed {
|
||||||
|
normalized := strings.NewReplacer("_", "", "-", "", ".", "").Replace(strings.ToLower(strings.TrimSpace(key)))
|
||||||
|
switch normalized {
|
||||||
|
case "apikey", "accesskey", "accesskeyid", "accesskeysecret", "secretkey", "sessiontoken", "ststoken", "password", "credential", "credentials", "authorization":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if fileStorageConfigContainsCredential(item) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
for _, item := range typed {
|
||||||
|
if fileStorageConfigContainsCredential(item) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileStorageConfigString(config map[string]any, key string) string {
|
||||||
|
value, _ := config[key].(string)
|
||||||
|
return strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileStorageCredentialPresent(input *string, existing *store.FileStorageChannel, current func(*store.FileStorageChannel) string) bool {
|
||||||
|
if input != nil {
|
||||||
|
return strings.TrimSpace(*input) != ""
|
||||||
|
}
|
||||||
|
return existing != nil && strings.TrimSpace(current(existing)) != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func legacyLocalStoragePolicy(value string) bool {
|
||||||
|
normalized := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(value)), "-", "_")
|
||||||
|
switch normalized {
|
||||||
|
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateObjectStorageChannelAcceptsS3CredentialAliases(t *testing.T) {
|
||||||
|
accessKey := "access"
|
||||||
|
secretKey := "secret"
|
||||||
|
input := store.FileStorageChannelInput{
|
||||||
|
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
|
||||||
|
AccessKey: &accessKey, SecretKey: &secretKey,
|
||||||
|
Config: map[string]any{
|
||||||
|
"endpoint": "https://s3.example.com", "region": "us-east-1", "bucket": "media",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if message := validateFileStorageChannelInput(input, nil); message != "" {
|
||||||
|
t.Fatalf("valid S3 channel rejected: %s", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateObjectStorageChannelRejectsCredentialsInConfig(t *testing.T) {
|
||||||
|
accessKeyID := "access"
|
||||||
|
accessKeySecret := "secret"
|
||||||
|
input := store.FileStorageChannelInput{
|
||||||
|
ChannelKey: "oss-primary", Name: "OSS primary", Provider: "aliyun_oss", Status: "enabled",
|
||||||
|
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
|
||||||
|
Config: map[string]any{
|
||||||
|
"endpoint": "https://oss-cn-hangzhou.aliyuncs.com", "region": "cn-hangzhou", "bucket": "media",
|
||||||
|
"nested": map[string]any{"session_token": "must-not-be-public"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
message := validateFileStorageChannelInput(input, nil)
|
||||||
|
if !strings.Contains(message, "write-only credential fields") {
|
||||||
|
t.Fatalf("credential-bearing config was accepted: %q", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) {
|
||||||
|
accessKeyID := "access"
|
||||||
|
accessKeySecret := "secret"
|
||||||
|
input := store.FileStorageChannelInput{
|
||||||
|
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
|
||||||
|
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
|
||||||
|
Config: map[string]any{
|
||||||
|
"endpoint": "https://user:password@s3.example.com", "region": "us-east-1", "bucket": "media",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if message := validateFileStorageChannelInput(input, nil); !strings.Contains(message, "without embedded credentials") {
|
||||||
|
t.Fatalf("credentialed endpoint was accepted: %q", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -69,6 +69,10 @@ func writeIdempotentTaskReplay(w http.ResponseWriter, task store.GatewayTask, co
|
|||||||
|
|
||||||
func storedTaskErrorStatus(code string) int {
|
func storedTaskErrorStatus(code string) int {
|
||||||
switch strings.TrimSpace(code) {
|
switch strings.TrimSpace(code) {
|
||||||
|
case "storage_write_failed", "storage_read_failed", "storage_config_invalid", "storage_auth_failed":
|
||||||
|
return http.StatusServiceUnavailable
|
||||||
|
case "binary_result_expired", "result_expired", "result_unavailable":
|
||||||
|
return http.StatusGone
|
||||||
case "pricing_unavailable", "response_chain_unavailable", "billing_hold":
|
case "pricing_unavailable", "response_chain_unavailable", "billing_hold":
|
||||||
return http.StatusServiceUnavailable
|
return http.StatusServiceUnavailable
|
||||||
case "insufficient_balance":
|
case "insufficient_balance":
|
||||||
|
|||||||
@@ -255,7 +255,7 @@ func (s *Server) uploadImageEditMultipartAsset(ctx context.Context, field string
|
|||||||
if !strings.HasPrefix(strings.ToLower(contentType), "image/") {
|
if !strings.HasPrefix(strings.ToLower(contentType), "image/") {
|
||||||
return nil, &clients.ClientError{Code: "invalid_multipart_image", Message: "image edit multipart files must be images", Retryable: false}
|
return nil, &clients.ClientError{Code: "invalid_multipart_image", Message: "image edit multipart files must be images", Retryable: false}
|
||||||
}
|
}
|
||||||
ref, err := s.ensurePublicRequestAsset(ctx, decodedRequestAsset{
|
ref, err := s.ensureRequestAsset(ctx, decodedRequestAsset{
|
||||||
Bytes: payload,
|
Bytes: payload,
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -352,7 +352,7 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
|
|||||||
response["usage"] = legacyUsage
|
response["usage"] = legacyUsage
|
||||||
}
|
}
|
||||||
if task.Status == "failed" || task.Status == "cancelled" {
|
if task.Status == "failed" || task.Status == "cancelled" {
|
||||||
response["error"] = map[string]any{"code": firstNonEmpty(task.ErrorCode, strings.ToUpper(task.Status)), "message": firstNonEmpty(task.ErrorMessage, task.Error, task.Message)}
|
response["error"] = publicErrorMap(publicTaskError(task))
|
||||||
}
|
}
|
||||||
return response
|
return response
|
||||||
}
|
}
|
||||||
@@ -431,10 +431,6 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
|
|||||||
} else if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
|
} else if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
|
||||||
status = http.StatusBadRequest
|
status = http.StatusBadRequest
|
||||||
}
|
}
|
||||||
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, clients.ProtocolVolcesContents) {
|
|
||||||
writeWireResponse(w, wire)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeVolcesError(w, status, err.Error(), clients.ErrorCode(err))
|
writeVolcesError(w, status, err.Error(), clients.ErrorCode(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package publicerror
|
||||||
|
|
||||||
|
import "sync/atomic"
|
||||||
|
|
||||||
|
type MetricCount struct {
|
||||||
|
Code string
|
||||||
|
Count uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
var observed = struct {
|
||||||
|
upstreamConnectionInterrupted atomic.Uint64
|
||||||
|
upstreamTimeout atomic.Uint64
|
||||||
|
upstreamRateLimited atomic.Uint64
|
||||||
|
upstreamUnavailable atomic.Uint64
|
||||||
|
upstreamInvalidResponse atomic.Uint64
|
||||||
|
upstreamAuthFailed atomic.Uint64
|
||||||
|
upstreamRequestRejected atomic.Uint64
|
||||||
|
storageWriteFailed atomic.Uint64
|
||||||
|
storageReadFailed atomic.Uint64
|
||||||
|
resultExpired atomic.Uint64
|
||||||
|
resultUnavailable atomic.Uint64
|
||||||
|
invalidParameter atomic.Uint64
|
||||||
|
gatewayError atomic.Uint64
|
||||||
|
}{}
|
||||||
|
|
||||||
|
func Observe(value Error) {
|
||||||
|
switch value.Code {
|
||||||
|
case "upstream_connection_interrupted":
|
||||||
|
observed.upstreamConnectionInterrupted.Add(1)
|
||||||
|
case "upstream_timeout":
|
||||||
|
observed.upstreamTimeout.Add(1)
|
||||||
|
case "upstream_rate_limited":
|
||||||
|
observed.upstreamRateLimited.Add(1)
|
||||||
|
case "upstream_unavailable":
|
||||||
|
observed.upstreamUnavailable.Add(1)
|
||||||
|
case "upstream_invalid_response":
|
||||||
|
observed.upstreamInvalidResponse.Add(1)
|
||||||
|
case "upstream_auth_failed":
|
||||||
|
observed.upstreamAuthFailed.Add(1)
|
||||||
|
case "upstream_request_rejected":
|
||||||
|
observed.upstreamRequestRejected.Add(1)
|
||||||
|
case "storage_write_failed":
|
||||||
|
observed.storageWriteFailed.Add(1)
|
||||||
|
case "storage_read_failed":
|
||||||
|
observed.storageReadFailed.Add(1)
|
||||||
|
case "result_expired":
|
||||||
|
observed.resultExpired.Add(1)
|
||||||
|
case "result_unavailable":
|
||||||
|
observed.resultUnavailable.Add(1)
|
||||||
|
case "invalid_parameter":
|
||||||
|
observed.invalidParameter.Add(1)
|
||||||
|
default:
|
||||||
|
observed.gatewayError.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func MetricSnapshot() []MetricCount {
|
||||||
|
return []MetricCount{
|
||||||
|
{Code: "upstream_connection_interrupted", Count: observed.upstreamConnectionInterrupted.Load()},
|
||||||
|
{Code: "upstream_timeout", Count: observed.upstreamTimeout.Load()},
|
||||||
|
{Code: "upstream_rate_limited", Count: observed.upstreamRateLimited.Load()},
|
||||||
|
{Code: "upstream_unavailable", Count: observed.upstreamUnavailable.Load()},
|
||||||
|
{Code: "upstream_invalid_response", Count: observed.upstreamInvalidResponse.Load()},
|
||||||
|
{Code: "upstream_auth_failed", Count: observed.upstreamAuthFailed.Load()},
|
||||||
|
{Code: "upstream_request_rejected", Count: observed.upstreamRequestRejected.Load()},
|
||||||
|
{Code: "storage_write_failed", Count: observed.storageWriteFailed.Load()},
|
||||||
|
{Code: "storage_read_failed", Count: observed.storageReadFailed.Load()},
|
||||||
|
{Code: "result_expired", Count: observed.resultExpired.Load()},
|
||||||
|
{Code: "result_unavailable", Count: observed.resultUnavailable.Load()},
|
||||||
|
{Code: "invalid_parameter", Count: observed.invalidParameter.Load()},
|
||||||
|
{Code: "gateway_error", Count: observed.gatewayError.Load()},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,200 @@
|
|||||||
|
package publicerror
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Version = "v1"
|
||||||
|
|
||||||
|
type Error struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Category string `json:"category"`
|
||||||
|
HTTPStatus int `json:"httpStatus"`
|
||||||
|
Retryable bool `json:"retryable"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
RetryAfterSeconds int `json:"retryAfterSeconds,omitempty"`
|
||||||
|
RequestID string `json:"requestId,omitempty"`
|
||||||
|
TaskID string `json:"taskId,omitempty"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func FromFields(code string, message string, status int, retryable bool) Error {
|
||||||
|
originalCode := strings.TrimSpace(code)
|
||||||
|
code = strings.ToLower(originalCode)
|
||||||
|
message = strings.TrimSpace(message)
|
||||||
|
if status <= 0 {
|
||||||
|
status = statusFromCode(code)
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(originalCode, "GATEWAY_") {
|
||||||
|
return newError(originalCode, safeRequestMessage(message), "gateway", status, retryable, retryAction(retryable))
|
||||||
|
}
|
||||||
|
if mapped, ok := mappedError(code, message, status, retryable); ok {
|
||||||
|
return mapped
|
||||||
|
}
|
||||||
|
if sensitiveTransportMessage(message) {
|
||||||
|
return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry")
|
||||||
|
}
|
||||||
|
if code == "" {
|
||||||
|
code = defaultCodeForStatus(status)
|
||||||
|
} else {
|
||||||
|
code = originalCode
|
||||||
|
}
|
||||||
|
if message == "" {
|
||||||
|
message = defaultMessageForStatus(status)
|
||||||
|
}
|
||||||
|
category := "gateway"
|
||||||
|
action := "none"
|
||||||
|
if status == http.StatusTooManyRequests {
|
||||||
|
category, action, retryable = "rate_limit", "retry_after", true
|
||||||
|
} else if status >= 500 {
|
||||||
|
category = "upstream"
|
||||||
|
if retryable {
|
||||||
|
action = "retry"
|
||||||
|
}
|
||||||
|
} else if status >= 400 {
|
||||||
|
category, action = "request", "fix_request"
|
||||||
|
}
|
||||||
|
return newError(code, message, category, status, retryable, action)
|
||||||
|
}
|
||||||
|
|
||||||
|
func WithIDs(value Error, requestID string, taskID string) Error {
|
||||||
|
value.RequestID = strings.TrimSpace(requestID)
|
||||||
|
value.TaskID = strings.TrimSpace(taskID)
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func mappedError(code string, message string, status int, retryable bool) (Error, bool) {
|
||||||
|
switch code {
|
||||||
|
case "bad_request", "invalid_request", "invalid_parameter", "unsupported_kind", "unsupported_operation", "unsupported_response_parameter":
|
||||||
|
return newError(code, safeRequestMessage(message), "request", http.StatusBadRequest, false, "fix_request"), true
|
||||||
|
case "cancelled", "task_cancelled":
|
||||||
|
return newError(code, safeRequestMessage(message), "request", http.StatusConflict, false, "none"), true
|
||||||
|
case "validation_in_progress":
|
||||||
|
return newError(code, firstNonEmptyMessage(message, "New production tasks are paused while validation is running."), "gateway", http.StatusServiceUnavailable, true, "retry_after"), true
|
||||||
|
case "traffic_gate_unavailable":
|
||||||
|
return newError(code, "The gateway traffic admission service is temporarily unavailable.", "gateway", http.StatusServiceUnavailable, true, "retry"), true
|
||||||
|
case "response_read_error", "stream_read_error", "network", "connection_reset", "upstream_connection_interrupted", "unexpected_eof", "http2_stream_closed", "request_asset_fetch_failed":
|
||||||
|
return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry"), true
|
||||||
|
case "upstream_timeout", "timeout", "context_deadline_exceeded":
|
||||||
|
return newError("upstream_timeout", "The upstream service did not respond in time.", "upstream", http.StatusGatewayTimeout, true, "retry"), true
|
||||||
|
case "rate_limit", "upstream_rate_limited", "too_many_requests":
|
||||||
|
return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", http.StatusTooManyRequests, true, "retry_after"), true
|
||||||
|
case "upstream_unavailable", "upstream_overloaded", "service_unavailable", "bad_gateway", "server_error", "provider_failed":
|
||||||
|
if status > 0 && status < 500 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true
|
||||||
|
case "upload_invalid_response", "invalid_response", "response_too_large", "invalid_upstream_response", "upstream_invalid_response":
|
||||||
|
return newError("upstream_invalid_response", "The upstream service returned an invalid or incomplete response.", "upstream", http.StatusBadGateway, true, "retry"), true
|
||||||
|
case "invalid_api_key", "authentication_error", "auth_failed", "missing_credentials", "upstream_auth_failed":
|
||||||
|
return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true
|
||||||
|
case "storage_write_failed", "storage_config_invalid", "storage_auth_failed", "upload_config_failed", "upload_no_channel", "upload_network", "upload_read_failed", "upload_failed", "local_result_storage_unavailable", "local_static_store_failed", "request_asset_upload_failed":
|
||||||
|
return newError("storage_write_failed", "The media asset could not be written to object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true
|
||||||
|
case "storage_read_failed", "upload_source_fetch_failed", "upload_source_read_failed":
|
||||||
|
return newError("storage_read_failed", "The media asset could not be read from object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true
|
||||||
|
case "binary_result_expired", "result_expired":
|
||||||
|
return newError("result_expired", "The generated result has expired and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true
|
||||||
|
case "result_unavailable", "request_asset_expired":
|
||||||
|
return newError("result_unavailable", "The generated result is no longer available and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true
|
||||||
|
case "binary_result_corrupted", "result_binary_not_materialized":
|
||||||
|
return newError("result_corrupted", "The stored result failed an integrity check.", "storage", http.StatusInternalServerError, false, "contact_support"), true
|
||||||
|
}
|
||||||
|
if status == http.StatusTooManyRequests {
|
||||||
|
return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", status, true, "retry_after"), true
|
||||||
|
}
|
||||||
|
if (status == http.StatusUnauthorized || status == http.StatusForbidden) && (strings.HasPrefix(code, "http_") || strings.Contains(code, "provider")) {
|
||||||
|
return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(code, "http_") || (code == "provider_failed" && status > 0 && status < 500) {
|
||||||
|
return newError("upstream_request_rejected", "The upstream service rejected the request.", "upstream", http.StatusBadRequest, false, "fix_request"), true
|
||||||
|
}
|
||||||
|
if status >= 500 && code != "upstream_submission_unknown" && (strings.Contains(code, "upstream") || strings.Contains(code, "provider")) {
|
||||||
|
return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true
|
||||||
|
}
|
||||||
|
if status >= 500 && code != "upstream_submission_unknown" {
|
||||||
|
return newError("gateway_error", defaultMessageForStatus(status), "gateway", status, retryable, retryAction(retryable)), true
|
||||||
|
}
|
||||||
|
return Error{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeRequestMessage(message string) string {
|
||||||
|
message = strings.TrimSpace(message)
|
||||||
|
if message == "" || sensitiveTransportMessage(message) {
|
||||||
|
return "The request parameters are invalid."
|
||||||
|
}
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstNonEmptyMessage(message string, fallback string) string {
|
||||||
|
if message = strings.TrimSpace(message); message != "" && !sensitiveTransportMessage(message) {
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
func retryAction(retryable bool) string {
|
||||||
|
if retryable {
|
||||||
|
return "retry"
|
||||||
|
}
|
||||||
|
return "contact_support"
|
||||||
|
}
|
||||||
|
|
||||||
|
func newError(code string, message string, category string, status int, retryable bool, action string) Error {
|
||||||
|
return Error{Code: code, Message: message, Category: category, HTTPStatus: status, Retryable: retryable, Action: action, Version: Version}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sensitiveTransportMessage(message string) bool {
|
||||||
|
value := strings.ToLower(message)
|
||||||
|
for _, marker := range []string{"read tcp ", "write tcp ", "dial tcp ", "connection reset by peer", "unexpected eof", "http2:", "stream error", "context deadline exceeded"} {
|
||||||
|
if strings.Contains(value, marker) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func statusFromCode(code string) int {
|
||||||
|
switch code {
|
||||||
|
case "rate_limit", "upstream_rate_limited", "too_many_requests":
|
||||||
|
return http.StatusTooManyRequests
|
||||||
|
case "upstream_timeout", "timeout", "context_deadline_exceeded":
|
||||||
|
return http.StatusGatewayTimeout
|
||||||
|
case "binary_result_expired", "result_expired", "result_unavailable", "request_asset_expired":
|
||||||
|
return http.StatusGone
|
||||||
|
case "storage_write_failed", "storage_read_failed", "local_result_storage_unavailable":
|
||||||
|
return http.StatusServiceUnavailable
|
||||||
|
default:
|
||||||
|
return http.StatusBadGateway
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultCodeForStatus(status int) string {
|
||||||
|
switch status {
|
||||||
|
case http.StatusBadRequest:
|
||||||
|
return "invalid_request"
|
||||||
|
case http.StatusUnauthorized:
|
||||||
|
return "unauthorized"
|
||||||
|
case http.StatusForbidden:
|
||||||
|
return "forbidden"
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return "not_found"
|
||||||
|
case http.StatusConflict:
|
||||||
|
return "conflict"
|
||||||
|
case http.StatusTooManyRequests:
|
||||||
|
return "rate_limited"
|
||||||
|
default:
|
||||||
|
if status >= 500 {
|
||||||
|
return "gateway_error"
|
||||||
|
}
|
||||||
|
return "request_failed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultMessageForStatus(status int) string {
|
||||||
|
if value := http.StatusText(status); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return "Request failed."
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package publicerror
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTransportErrorsNeverExposeSocketDetails(t *testing.T) {
|
||||||
|
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
|
||||||
|
got := FromFields("response_read_error", raw, http.StatusOK, true)
|
||||||
|
if got.Code != "upstream_connection_interrupted" || got.HTTPStatus != http.StatusBadGateway || !got.Retryable {
|
||||||
|
t.Fatalf("unexpected public error: %+v", got)
|
||||||
|
}
|
||||||
|
if strings.Contains(got.Message, "10.42.0.72") || strings.Contains(got.Message, "47.77.191.126") {
|
||||||
|
t.Fatalf("public error leaked socket details: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageAndExpiredErrorsHaveStableActions(t *testing.T) {
|
||||||
|
storage := FromFields("storage_write_failed", "secret endpoint", 0, true)
|
||||||
|
if storage.Category != "storage" || storage.Action != "retry" || storage.HTTPStatus != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("unexpected storage error: %+v", storage)
|
||||||
|
}
|
||||||
|
expired := FromFields("binary_result_expired", "local binary result has expired", 0, false)
|
||||||
|
if expired.Code != "result_expired" || expired.Action != "resubmit" || expired.HTTPStatus != http.StatusGone {
|
||||||
|
t.Fatalf("unexpected expired error: %+v", expired)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnknownServerErrorDoesNotExposeProviderBody(t *testing.T) {
|
||||||
|
raw := `provider rejected request: {"account":"secret-project","detail":"internal"}`
|
||||||
|
got := FromFields("vendor_opaque_error", raw, http.StatusBadGateway, true)
|
||||||
|
if got.Code != "gateway_error" || got.Message == raw || strings.Contains(got.Message, "secret-project") {
|
||||||
|
t.Fatalf("unknown provider error was exposed: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProviderHTTPErrorDoesNotExposeProviderBody(t *testing.T) {
|
||||||
|
raw := `{"error":{"message":"bucket private-a rejected secret-project"}}`
|
||||||
|
for _, test := range []struct {
|
||||||
|
code string
|
||||||
|
status int
|
||||||
|
wantCode string
|
||||||
|
wantStatus int
|
||||||
|
}{
|
||||||
|
{code: "http_400", status: http.StatusBadRequest, wantCode: "upstream_request_rejected", wantStatus: http.StatusBadRequest},
|
||||||
|
{code: "auth_failed", status: http.StatusUnauthorized, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway},
|
||||||
|
{code: "provider_failed", status: http.StatusForbidden, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway},
|
||||||
|
{code: "provider_failed", status: http.StatusTooManyRequests, wantCode: "upstream_rate_limited", wantStatus: http.StatusTooManyRequests},
|
||||||
|
{code: "server_error", status: http.StatusBadGateway, wantCode: "upstream_unavailable", wantStatus: http.StatusServiceUnavailable},
|
||||||
|
{code: "invalid_response", status: http.StatusOK, wantCode: "upstream_invalid_response", wantStatus: http.StatusBadGateway},
|
||||||
|
} {
|
||||||
|
got := FromFields(test.code, raw, test.status, true)
|
||||||
|
if got.Code != test.wantCode || got.HTTPStatus != test.wantStatus {
|
||||||
|
t.Fatalf("%s: unexpected public error: %+v", test.code, got)
|
||||||
|
}
|
||||||
|
if strings.Contains(got.Message, "private-a") || strings.Contains(got.Message, "secret-project") {
|
||||||
|
t.Fatalf("%s: provider body leaked: %+v", test.code, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationGateKeepsStablePublicCode(t *testing.T) {
|
||||||
|
got := FromFields("validation_in_progress", "new production tasks are paused while validation is running", http.StatusServiceUnavailable, true)
|
||||||
|
if got.Code != "validation_in_progress" || got.HTTPStatus != http.StatusServiceUnavailable || !got.Retryable {
|
||||||
|
t.Fatalf("unexpected validation gate error: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
@@ -26,7 +25,6 @@ const (
|
|||||||
localBinaryGenericBase64MinLength = 4096
|
localBinaryGenericBase64MinLength = 4096
|
||||||
localBinaryMaxDepth = 64
|
localBinaryMaxDepth = 64
|
||||||
defaultLocalResultTTLHours = 24
|
defaultLocalResultTTLHours = 24
|
||||||
defaultLocalResultMinFreeBytes = int64(10 * 1024 * 1024 * 1024)
|
|
||||||
defaultLocalResultMaxBytes = int64(256 * 1024 * 1024)
|
defaultLocalResultMaxBytes = int64(256 * 1024 * 1024)
|
||||||
defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024)
|
defaultLocalResultMaxTaskBytes = int64(512 * 1024 * 1024)
|
||||||
)
|
)
|
||||||
@@ -41,41 +39,38 @@ type localBinaryDescriptor struct {
|
|||||||
|
|
||||||
type localBinaryMaterializer struct {
|
type localBinaryMaterializer struct {
|
||||||
service *Service
|
service *Service
|
||||||
taskDir string
|
|
||||||
writeFiles bool
|
|
||||||
enforceLimits bool
|
enforceLimits bool
|
||||||
totalBytes int64
|
totalBytes int64
|
||||||
seen map[string]struct{}
|
seen map[string]struct{}
|
||||||
createdFiles []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// materializeLocalBinaryResult replaces every inline binary value with a
|
// materializeLocalBinaryResult is retained as an internal compatibility seam,
|
||||||
// bounded placeholder after atomically writing and verifying the bytes locally.
|
// but new materialization always targets object storage. Historical local
|
||||||
|
// placeholders remain readable through HydrateTaskResult.
|
||||||
func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
func (s *Service) materializeLocalBinaryResult(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
||||||
if _, _, err := s.transformLocalBinaryResult(ctx, taskID, result, false, true); err != nil {
|
hadInline := TaskResultHasInlineBinary(result)
|
||||||
|
if !hadInline {
|
||||||
|
return result, false, nil
|
||||||
|
}
|
||||||
|
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
|
||||||
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
return s.transformLocalBinaryResult(ctx, taskID, result, true, true)
|
return next, !TaskResultHasInlineBinary(next), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string, result map[string]any, writeFiles bool, enforceLimits bool) (map[string]any, bool, error) {
|
func (s *Service) transformLocalBinaryResult(ctx context.Context, _ string, result map[string]any, _ bool, enforceLimits bool) (map[string]any, bool, error) {
|
||||||
root := s.localBinaryResultRoot()
|
|
||||||
taskDir := filepath.Join(root, safeLocalBinaryTaskDir(taskID))
|
|
||||||
materializer := &localBinaryMaterializer{
|
materializer := &localBinaryMaterializer{
|
||||||
service: s,
|
service: s,
|
||||||
taskDir: taskDir,
|
|
||||||
writeFiles: writeFiles,
|
|
||||||
enforceLimits: enforceLimits,
|
enforceLimits: enforceLimits,
|
||||||
seen: map[string]struct{}{},
|
seen: map[string]struct{}{},
|
||||||
}
|
}
|
||||||
next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
|
next, changed, err := materializer.materializeValue(ctx, result, "", nil, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
materializer.rollbackCreatedFiles()
|
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
mapped, ok := next.(map[string]any)
|
mapped, ok := next.(map[string]any)
|
||||||
if !ok {
|
if !ok {
|
||||||
materializer.rollbackCreatedFiles()
|
|
||||||
return nil, false, &clients.ClientError{
|
return nil, false, &clients.ClientError{
|
||||||
Code: "result_binary_not_materialized",
|
Code: "result_binary_not_materialized",
|
||||||
Message: "generated result is not a JSON object",
|
Message: "generated result is not a JSON object",
|
||||||
@@ -83,28 +78,28 @@ func (s *Service) transformLocalBinaryResult(ctx context.Context, taskID string,
|
|||||||
Retryable: false,
|
Retryable: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if changed && enforceLimits && !writeFiles {
|
|
||||||
if err := os.MkdirAll(root, 0o750); err != nil {
|
|
||||||
return nil, false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
if err := ensureLocalBinaryDiskHeadroom(root, materializer.totalBytes, s.localResultMinFreeBytes()); err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return mapped, changed, nil
|
return mapped, changed, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MaterializeTaskResultForStorage exposes the same verified materialization
|
// MaterializeTaskResultForStorage exposes the same verified materialization
|
||||||
// path to the explicit historical maintenance command.
|
// path to the explicit historical maintenance command.
|
||||||
func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
func (s *Service) MaterializeTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
||||||
return s.materializeLocalBinaryResult(ctx, taskID, result)
|
hadInline := TaskResultHasInlineBinary(result)
|
||||||
|
if !hadInline {
|
||||||
|
return result, false, nil
|
||||||
|
}
|
||||||
|
next, err := s.uploadGeneratedAssets(ctx, taskID, "", "", result)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return next, hadInline && !TaskResultHasInlineBinary(next), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompactExpiredTaskResultForStorage computes the same deterministic
|
// CompactExpiredTaskResultForStorage keeps the historical maintenance API but
|
||||||
// placeholders without creating files that are already outside the recovery
|
// now uses the same object-storage path as live results. New GatewayBinary
|
||||||
// window.
|
// placeholders are never created; their parser remains read-only compatibility.
|
||||||
func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
func (s *Service) CompactExpiredTaskResultForStorage(ctx context.Context, taskID string, result map[string]any) (map[string]any, bool, error) {
|
||||||
return s.transformLocalBinaryResult(ctx, taskID, result, false, false)
|
return s.MaterializeTaskResultForStorage(ctx, taskID, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TaskResultHasInlineBinary(result map[string]any) bool {
|
func TaskResultHasInlineBinary(result map[string]any) bool {
|
||||||
@@ -239,15 +234,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
|
|||||||
Retryable: false,
|
Retryable: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if m.writeFiles {
|
|
||||||
created, err := m.service.writeLocalBinaryResult(m.taskDir, digestHex, payload)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
if created {
|
|
||||||
m.createdFiles = append(m.createdFiles, filepath.Join(m.taskDir, digestHex+".bin"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
m.seen[digestHex] = struct{}{}
|
m.seen[digestHex] = struct{}{}
|
||||||
m.totalBytes += size
|
m.totalBytes += size
|
||||||
}
|
}
|
||||||
@@ -261,94 +247,6 @@ func (m *localBinaryMaterializer) persistBinary(ctx context.Context, payload []b
|
|||||||
return localBinaryPlaceholder(descriptor), true, nil
|
return localBinaryPlaceholder(descriptor), true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *localBinaryMaterializer) rollbackCreatedFiles() {
|
|
||||||
for _, path := range m.createdFiles {
|
|
||||||
_ = os.Remove(path)
|
|
||||||
}
|
|
||||||
_ = os.Remove(m.taskDir)
|
|
||||||
m.createdFiles = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Service) writeLocalBinaryResult(taskDir string, digestHex string, payload []byte) (bool, error) {
|
|
||||||
if err := os.MkdirAll(taskDir, 0o750); err != nil {
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
targetPath := filepath.Join(taskDir, digestHex+".bin")
|
|
||||||
if info, err := os.Stat(targetPath); err == nil {
|
|
||||||
if !info.IsDir() && info.Size() == int64(len(payload)) {
|
|
||||||
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err == nil {
|
|
||||||
now := time.Now()
|
|
||||||
if err := os.Chtimes(targetPath, now, now); err != nil {
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false, &clients.ClientError{
|
|
||||||
Code: "binary_result_corrupted",
|
|
||||||
Message: "existing local result file does not match its content hash",
|
|
||||||
StatusCode: 500,
|
|
||||||
Retryable: false,
|
|
||||||
}
|
|
||||||
} else if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
if err := ensureLocalBinaryDiskHeadroom(taskDir, int64(len(payload)), s.localResultMinFreeBytes()); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
tempFile, err := os.CreateTemp(taskDir, ".gateway-result-*")
|
|
||||||
if err != nil {
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
tempPath := tempFile.Name()
|
|
||||||
cleanup := func() {
|
|
||||||
_ = tempFile.Close()
|
|
||||||
_ = os.Remove(tempPath)
|
|
||||||
}
|
|
||||||
if err := tempFile.Chmod(0o640); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
if _, err := tempFile.Write(payload); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
if err := tempFile.Sync(); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
if err := tempFile.Close(); err != nil {
|
|
||||||
_ = os.Remove(tempPath)
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
if err := os.Rename(tempPath, targetPath); err != nil {
|
|
||||||
_ = os.Remove(tempPath)
|
|
||||||
return false, localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
if err := verifyLocalBinaryFile(targetPath, digestHex, int64(len(payload))); err != nil {
|
|
||||||
_ = os.Remove(targetPath)
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ensureLocalBinaryDiskHeadroom(path string, incomingBytes int64, minFreeBytes int64) error {
|
|
||||||
var stat syscall.Statfs_t
|
|
||||||
if err := syscall.Statfs(path, &stat); err != nil {
|
|
||||||
return localBinaryStorageError(err)
|
|
||||||
}
|
|
||||||
freeBytes := int64(stat.Bavail) * int64(stat.Bsize)
|
|
||||||
if freeBytes-incomingBytes < minFreeBytes {
|
|
||||||
return &clients.ClientError{
|
|
||||||
Code: "local_result_storage_unavailable",
|
|
||||||
Message: "local result storage does not have enough free space",
|
|
||||||
StatusCode: 503,
|
|
||||||
Retryable: false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func localBinaryStorageError(err error) error {
|
func localBinaryStorageError(err error) error {
|
||||||
return &clients.ClientError{
|
return &clients.ClientError{
|
||||||
Code: "local_result_storage_unavailable",
|
Code: "local_result_storage_unavailable",
|
||||||
@@ -406,6 +304,30 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
|
|||||||
}
|
}
|
||||||
switch typed := value.(type) {
|
switch typed := value.(type) {
|
||||||
case map[string]any:
|
case map[string]any:
|
||||||
|
refreshedAccessURL := false
|
||||||
|
if asset, upload, ok := generatedResultUploadReference(typed); ok {
|
||||||
|
accessURL, err := s.requestAssetAccessURL(ctx, asset)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
next := make(map[string]any, len(typed))
|
||||||
|
for key, item := range typed {
|
||||||
|
next[key] = item
|
||||||
|
}
|
||||||
|
nextUpload := make(map[string]any, len(upload))
|
||||||
|
for key, item := range upload {
|
||||||
|
nextUpload[key] = item
|
||||||
|
}
|
||||||
|
nextUpload["url"] = accessURL
|
||||||
|
next["upload"] = nextUpload
|
||||||
|
for _, key := range []string{"url", "image_url", "video_url", "audio_url"} {
|
||||||
|
if _, exists := next[key]; exists {
|
||||||
|
next[key] = accessURL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
typed = next
|
||||||
|
refreshedAccessURL = true
|
||||||
|
}
|
||||||
if ref, ok := generatedResultAssetReference(typed); ok {
|
if ref, ok := generatedResultAssetReference(typed); ok {
|
||||||
payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
|
payload, contentType, err := s.readGeneratedResultAsset(ctx, ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -418,7 +340,7 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
|
|||||||
return encoded, true, nil
|
return encoded, true, nil
|
||||||
}
|
}
|
||||||
next := make(map[string]any, len(typed))
|
next := make(map[string]any, len(typed))
|
||||||
changed := false
|
changed := refreshedAccessURL
|
||||||
for key, childValue := range typed {
|
for key, childValue := range typed {
|
||||||
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
|
child, childChanged, err := s.hydrateLocalBinaryValue(ctx, taskID, childValue, depth+1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -465,6 +387,27 @@ func (s *Service) hydrateLocalBinaryValue(ctx context.Context, taskID string, va
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func generatedResultUploadReference(value map[string]any) (store.RequestAsset, map[string]any, bool) {
|
||||||
|
upload, ok := value["upload"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return store.RequestAsset{}, nil, false
|
||||||
|
}
|
||||||
|
objectKey := strings.TrimSpace(stringFromAny(upload["objectKey"]))
|
||||||
|
channel, _ := upload["storageChannel"].(map[string]any)
|
||||||
|
channelKey := strings.TrimSpace(stringFromAny(channel["channelKey"]))
|
||||||
|
if objectKey == "" || channelKey == "" {
|
||||||
|
return store.RequestAsset{}, nil, false
|
||||||
|
}
|
||||||
|
return store.RequestAsset{
|
||||||
|
URL: stringFromAny(upload["url"]),
|
||||||
|
StorageProvider: stringFromAny(channel["provider"]),
|
||||||
|
StorageChannelID: stringFromAny(channel["id"]),
|
||||||
|
StorageChannelKey: channelKey,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
AccessScope: stringFromAny(upload["accessScope"]),
|
||||||
|
}, upload, true
|
||||||
|
}
|
||||||
|
|
||||||
func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) {
|
func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bool) {
|
||||||
ref, ok := value["assetRef"].(map[string]any)
|
ref, ok := value["assetRef"].(map[string]any)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -479,6 +422,10 @@ func generatedResultAssetReference(value map[string]any) (store.RequestAsset, bo
|
|||||||
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
|
ContentType: firstNonEmptyString(stringFromAny(ref["contentType"]), stringFromAny(storage["contentType"])),
|
||||||
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
|
URL: firstNonEmptyString(stringFromAny(ref["url"]), stringFromAny(value["url"])),
|
||||||
StorageProvider: stringFromAny(ref["storageProvider"]),
|
StorageProvider: stringFromAny(ref["storageProvider"]),
|
||||||
|
StorageChannelID: stringFromAny(ref["storageChannelId"]),
|
||||||
|
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
|
||||||
|
ObjectKey: stringFromAny(ref["objectKey"]),
|
||||||
|
AccessScope: stringFromAny(ref["accessScope"]),
|
||||||
}
|
}
|
||||||
if size := floatFromAny(ref["size"]); size > 0 {
|
if size := floatFromAny(ref["size"]); size > 0 {
|
||||||
asset.ByteSize = int64(size)
|
asset.ByteSize = int64(size)
|
||||||
@@ -767,13 +714,6 @@ func (s *Service) localResultTTLHours() int {
|
|||||||
return s.cfg.LocalResultTTLHours
|
return s.cfg.LocalResultTTLHours
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) localResultMinFreeBytes() int64 {
|
|
||||||
if s.cfg.LocalResultMinFreeBytes <= 0 {
|
|
||||||
return defaultLocalResultMinFreeBytes
|
|
||||||
}
|
|
||||||
return s.cfg.LocalResultMinFreeBytes
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Service) localResultMaxBytes() int64 {
|
func (s *Service) localResultMaxBytes() int64 {
|
||||||
if s.cfg.LocalResultMaxBytes <= 0 {
|
if s.cfg.LocalResultMaxBytes <= 0 {
|
||||||
return defaultLocalResultMaxBytes
|
return defaultLocalResultMaxBytes
|
||||||
|
|||||||
@@ -31,7 +31,21 @@ func newLocalBinaryTestService(t *testing.T) *Service {
|
|||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
|
func writeHistoricalLocalBinaryFixture(t *testing.T, service *Service, taskID string, payload []byte) string {
|
||||||
|
t.Helper()
|
||||||
|
digest := sha256.Sum256(payload)
|
||||||
|
taskDir := filepath.Join(service.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID))
|
||||||
|
if err := os.MkdirAll(taskDir, 0o750); err != nil {
|
||||||
|
t.Fatalf("create historical result fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(taskDir, hex.EncodeToString(digest[:])+".bin")
|
||||||
|
if err := os.WriteFile(path, payload, 0o640); err != nil {
|
||||||
|
t.Fatalf("write historical result fixture: %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydrateHistoricalLocalBinaryResult(t *testing.T) {
|
||||||
service := newLocalBinaryTestService(t)
|
service := newLocalBinaryTestService(t)
|
||||||
payload := []byte("one binary result shared across representations")
|
payload := []byte("one binary result shared across representations")
|
||||||
encoded := base64.StdEncoding.EncodeToString(payload)
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
||||||
@@ -49,7 +63,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input)
|
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-123", input, false, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("materialize local binary result: %v", err)
|
t.Fatalf("materialize local binary result: %v", err)
|
||||||
}
|
}
|
||||||
@@ -76,6 +90,7 @@ func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
|
|||||||
if len(persistentJSON) > 32*1024 {
|
if len(persistentJSON) > 32*1024 {
|
||||||
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
|
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
|
||||||
}
|
}
|
||||||
|
writeHistoricalLocalBinaryFixture(t, service, "task-123", payload)
|
||||||
|
|
||||||
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
|
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
|
||||||
entries, err := os.ReadDir(taskDir)
|
entries, err := os.ReadDir(taskDir)
|
||||||
@@ -155,19 +170,53 @@ func TestHydrateGeneratedResultAssetReference(t *testing.T) {
|
|||||||
assertClientErrorCode(t, err, "binary_result_corrupted")
|
assertClientErrorCode(t, err, "binary_result_corrupted")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) {
|
||||||
|
cfg := config.Config{
|
||||||
|
MediaOSSDirectEnabled: true,
|
||||||
|
MediaOSSEndpoint: "https://oss.example.com",
|
||||||
|
MediaOSSBucket: "media-bucket",
|
||||||
|
MediaOSSAccessKeyID: "access-id",
|
||||||
|
MediaOSSAccessKeySecret: "access-secret",
|
||||||
|
MediaOSSObjectPrefix: "media",
|
||||||
|
}
|
||||||
|
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
|
||||||
|
result := map[string]any{
|
||||||
|
"data": []any{map[string]any{
|
||||||
|
"url": "https://expired.example.com/result.png",
|
||||||
|
"image_url": "https://expired.example.com/result.png",
|
||||||
|
"upload": map[string]any{
|
||||||
|
"url": "https://expired.example.com/result.png",
|
||||||
|
"objectKey": "media/image_result/2026/08/04/hash.png",
|
||||||
|
"accessScope": "private",
|
||||||
|
"storageChannel": map[string]any{
|
||||||
|
"channelKey": "environment-direct-oss",
|
||||||
|
"provider": "aliyun_oss",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
hydrated, err := service.HydrateTaskResult(t.Context(), "task-private-url", result)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
item := hydrated["data"].([]any)[0].(map[string]any)
|
||||||
|
for _, key := range []string{"url", "image_url"} {
|
||||||
|
value := stringFromAny(item[key])
|
||||||
|
if !strings.Contains(value, "OSSAccessKeyId=access-id") || strings.Contains(value, "expired.example.com") {
|
||||||
|
t.Fatalf("%s was not refreshed: %q", key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
|
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
|
||||||
service := newLocalBinaryTestService(t)
|
service := newLocalBinaryTestService(t)
|
||||||
service.cfg.LocalResultTTLHours = 1
|
service.cfg.LocalResultTTLHours = 1
|
||||||
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
|
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
|
||||||
persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded})
|
persistent, _, err := service.transformLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}, false, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("materialize fixture: %v", err)
|
t.Fatalf("materialize fixture: %v", err)
|
||||||
}
|
}
|
||||||
entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired"))
|
path := writeHistoricalLocalBinaryFixture(t, service, "task-expired", []byte("expiring result"))
|
||||||
if err != nil || len(entries) != 1 {
|
|
||||||
t.Fatalf("read fixture result: entries=%v err=%v", entries, err)
|
|
||||||
}
|
|
||||||
path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name())
|
|
||||||
old := time.Now().Add(-2 * time.Hour)
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
if err := os.Chtimes(path, old, old); err != nil {
|
if err := os.Chtimes(path, old, old); err != nil {
|
||||||
t.Fatalf("age fixture: %v", err)
|
t.Fatalf("age fixture: %v", err)
|
||||||
@@ -186,17 +235,17 @@ func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T)
|
|||||||
assertClientErrorCode(t, err, "binary_result_corrupted")
|
assertClientErrorCode(t, err, "binary_result_corrupted")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
|
func TestHistoricalLocalBinaryFixtureEnforcesLimitsAndKeepsText(t *testing.T) {
|
||||||
service := newLocalBinaryTestService(t)
|
service := newLocalBinaryTestService(t)
|
||||||
service.cfg.LocalResultMaxBytes = 4
|
service.cfg.LocalResultMaxBytes = 4
|
||||||
_, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{
|
_, _, err := service.transformLocalBinaryResult(context.Background(), "task-large", map[string]any{
|
||||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
|
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
|
||||||
})
|
}, false, true)
|
||||||
assertClientErrorCode(t, err, "binary_result_too_large")
|
assertClientErrorCode(t, err, "binary_result_too_large")
|
||||||
|
|
||||||
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{
|
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-text", map[string]any{
|
||||||
"message": "YWJjZA==",
|
"message": "YWJjZA==",
|
||||||
})
|
}, false, true)
|
||||||
if err != nil || changed || persistent["message"] != "YWJjZA==" {
|
if err != nil || changed || persistent["message"] != "YWJjZA==" {
|
||||||
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
|
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
|
||||||
}
|
}
|
||||||
@@ -204,18 +253,30 @@ func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
|
|||||||
service = newLocalBinaryTestService(t)
|
service = newLocalBinaryTestService(t)
|
||||||
service.cfg.LocalResultMaxBytes = 1024
|
service.cfg.LocalResultMaxBytes = 1024
|
||||||
service.cfg.LocalResultMaxTaskBytes = 8
|
service.cfg.LocalResultMaxTaskBytes = 8
|
||||||
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
|
_, _, err = service.transformLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
|
||||||
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
|
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
|
||||||
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
|
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
|
||||||
})
|
}, false, true)
|
||||||
assertClientErrorCode(t, err, "binary_result_too_large")
|
assertClientErrorCode(t, err, "binary_result_too_large")
|
||||||
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
|
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
|
||||||
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
|
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
|
func TestCompactExpiredTaskResultUsesObjectStorageAndDoesNotWriteFiles(t *testing.T) {
|
||||||
|
var uploads int
|
||||||
|
storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPut {
|
||||||
|
t.Fatalf("method=%s, want PUT", r.Method)
|
||||||
|
}
|
||||||
|
uploads++
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer storageServer.Close()
|
||||||
service := newLocalBinaryTestService(t)
|
service := newLocalBinaryTestService(t)
|
||||||
|
service.directOSS = &directOSSUploader{
|
||||||
|
endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
|
||||||
|
}
|
||||||
service.cfg.LocalResultMaxBytes = 1
|
service.cfg.LocalResultMaxBytes = 1
|
||||||
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
|
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
|
||||||
|
|
||||||
@@ -225,17 +286,18 @@ func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("compact expired result: %v", err)
|
t.Fatalf("compact expired result: %v", err)
|
||||||
}
|
}
|
||||||
if !changed || !localBinaryResultHasPlaceholders(persistent) {
|
if !changed || localBinaryResultHasPlaceholders(persistent) || TaskResultHasInlineBinary(persistent) {
|
||||||
t.Fatalf("expired result was not compacted: %+v", persistent)
|
t.Fatalf("expired result was not objectified: %+v", persistent)
|
||||||
|
}
|
||||||
|
if uploads != 1 {
|
||||||
|
t.Fatalf("object storage uploads=%d, want 1", uploads)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
|
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
|
||||||
t.Fatalf("expired compaction must not create a task directory: %v", err)
|
t.Fatalf("expired compaction must not create a task directory: %v", err)
|
||||||
}
|
}
|
||||||
_, err = service.HydrateTaskResult(context.Background(), "task-old", persistent)
|
|
||||||
assertClientErrorCode(t, err, "binary_result_expired")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
||||||
err := localBinaryStorageError(errors.New("write failed"))
|
err := localBinaryStorageError(errors.New("write failed"))
|
||||||
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
||||||
if clients.IsRetryable(err) {
|
if clients.IsRetryable(err) {
|
||||||
@@ -247,16 +309,6 @@ func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
|||||||
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
|
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
|
||||||
t.Fatal("local result storage failure must not fail over to another provider")
|
t.Fatal("local result storage failure must not fail over to another provider")
|
||||||
}
|
}
|
||||||
|
|
||||||
service := newLocalBinaryTestService(t)
|
|
||||||
service.cfg.LocalResultMinFreeBytes = 1 << 62
|
|
||||||
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{
|
|
||||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")),
|
|
||||||
})
|
|
||||||
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
|
||||||
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) {
|
|
||||||
t.Fatalf("disk preflight must not create partial task files: %v", statErr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func bytesToAny(payload []byte) []any {
|
func bytesToAny(payload []byte) []any {
|
||||||
|
|||||||
@@ -1,26 +1,12 @@
|
|||||||
package runner
|
package runner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/hmac"
|
|
||||||
"crypto/sha1"
|
|
||||||
"encoding/base64"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"path"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
const directOSSUploadTimeout = 120 * time.Second
|
|
||||||
|
|
||||||
type directOSSUploader struct {
|
type directOSSUploader struct {
|
||||||
endpoint string
|
endpoint string
|
||||||
bucket string
|
bucket string
|
||||||
@@ -28,8 +14,6 @@ type directOSSUploader struct {
|
|||||||
accessKeySecret string
|
accessKeySecret string
|
||||||
publicBaseURL string
|
publicBaseURL string
|
||||||
objectPrefix string
|
objectPrefix string
|
||||||
client *http.Client
|
|
||||||
now func() time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
|
func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
|
||||||
@@ -43,133 +27,41 @@ func newDirectOSSUploader(cfg config.Config) *directOSSUploader {
|
|||||||
accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret),
|
accessKeySecret: strings.TrimSpace(cfg.MediaOSSAccessKeySecret),
|
||||||
publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"),
|
publicBaseURL: strings.TrimRight(cfg.MediaOSSPublicBaseURL, "/"),
|
||||||
objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"),
|
objectPrefix: strings.Trim(cfg.MediaOSSObjectPrefix, "/"),
|
||||||
client: &http.Client{
|
|
||||||
Timeout: directOSSUploadTimeout,
|
|
||||||
Transport: &http.Transport{
|
|
||||||
MaxIdleConns: 256,
|
|
||||||
MaxIdleConnsPerHost: 256,
|
|
||||||
MaxConnsPerHost: 256,
|
|
||||||
IdleConnTimeout: 90 * time.Second,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
now: time.Now,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func directOSSScene(scene string) bool {
|
func directOSSScene(scene string) bool {
|
||||||
switch strings.TrimSpace(scene) {
|
switch strings.TrimSpace(scene) {
|
||||||
case store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
|
case store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult:
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *directOSSUploader) upload(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
func (u *directOSSUploader) fileStorageChannel() store.FileStorageChannel {
|
||||||
if u == nil {
|
if u == nil {
|
||||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: "direct OSS uploader is not configured", Retryable: false}
|
return store.FileStorageChannel{}
|
||||||
}
|
}
|
||||||
objectKey := u.objectKey(payload)
|
return store.FileStorageChannel{
|
||||||
escapedKey := escapeOSSObjectKey(objectKey)
|
ChannelKey: "environment-direct-oss",
|
||||||
uploadURL := u.endpoint + "/" + escapedKey
|
Name: "Environment Direct OSS",
|
||||||
contentType := strings.TrimSpace(payload.ContentType)
|
Provider: "aliyun_oss",
|
||||||
if contentType == "" {
|
AccessKeyID: u.accessKeyID,
|
||||||
contentType = "application/octet-stream"
|
AccessKeySecret: u.accessKeySecret,
|
||||||
}
|
Priority: 10,
|
||||||
|
Status: "enabled",
|
||||||
var lastErr error
|
Scenes: []string{store.FileStorageSceneUpload, store.FileStorageSceneRequestAsset, store.FileStorageSceneImageResult},
|
||||||
for attempt := 0; attempt < 3; attempt++ {
|
Config: map[string]any{
|
||||||
if attempt > 0 {
|
"endpoint": u.endpoint,
|
||||||
if err := sleepWithContext(ctx, time.Duration(attempt*attempt)*200*time.Millisecond); err != nil {
|
"bucket": u.bucket,
|
||||||
return nil, err
|
"publicBaseUrl": u.publicBaseURL,
|
||||||
}
|
"objectPrefix": u.objectPrefix,
|
||||||
}
|
|
||||||
status, err := u.put(ctx, uploadURL, escapedKey, contentType, payload.Bytes)
|
|
||||||
if err == nil && status >= 200 && status < 300 {
|
|
||||||
publicURL := u.publicBaseURL + "/" + escapedKey
|
|
||||||
return map[string]any{
|
|
||||||
"url": publicURL,
|
|
||||||
"fileName": objectKey,
|
|
||||||
"storageChannel": map[string]any{
|
|
||||||
"channelKey": "environment-direct-oss",
|
|
||||||
"name": "Environment Direct OSS",
|
|
||||||
"provider": "aliyun_oss_direct",
|
|
||||||
},
|
},
|
||||||
"assetStorage": map[string]any{
|
RetryPolicy: map[string]any{
|
||||||
"scene": payload.Scene,
|
"enabled": true,
|
||||||
"source": firstNonEmptyString(payload.Source, "ai-gateway"),
|
"maxRetries": 2,
|
||||||
"strategy": "direct_aliyun_oss",
|
"backoffSeconds": []any{0.25, 1.0},
|
||||||
},
|
},
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
|
||||||
lastErr = err
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
lastErr = fmt.Errorf("HTTP %d", status)
|
|
||||||
if status != http.StatusTooManyRequests && status < 500 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
message := "direct OSS upload failed"
|
|
||||||
if lastErr != nil {
|
|
||||||
message += ": " + lastErr.Error()
|
|
||||||
}
|
|
||||||
return nil, &clients.ClientError{Code: "upload_failed", Message: message, Retryable: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *directOSSUploader) put(ctx context.Context, uploadURL string, escapedKey string, contentType string, payload []byte) (int, error) {
|
|
||||||
date := u.now().UTC().Format(http.TimeFormat)
|
|
||||||
canonicalResource := "/" + u.bucket + "/" + escapedKey
|
|
||||||
stringToSign := "PUT\n\n" + contentType + "\n" + date + "\n" + canonicalResource
|
|
||||||
mac := hmac.New(sha1.New, []byte(u.accessKeySecret))
|
|
||||||
_, _ = mac.Write([]byte(stringToSign))
|
|
||||||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(payload))
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
req.Header.Set("Authorization", "OSS "+u.accessKeyID+":"+signature)
|
|
||||||
req.Header.Set("Content-Type", contentType)
|
|
||||||
req.Header.Set("Date", date)
|
|
||||||
resp, err := u.client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
|
||||||
return resp.StatusCode, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *directOSSUploader) objectKey(payload FileUploadPayload) string {
|
|
||||||
now := u.now().UTC()
|
|
||||||
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
|
|
||||||
baseName := strings.TrimSuffix(path.Base(payload.FileName), path.Ext(payload.FileName))
|
|
||||||
baseName = sanitizeGeneratedAssetNamePart(baseName)
|
|
||||||
if baseName == "" {
|
|
||||||
baseName = "gateway-media"
|
|
||||||
}
|
|
||||||
if len(baseName) > 48 {
|
|
||||||
baseName = baseName[:48]
|
|
||||||
}
|
|
||||||
return fmt.Sprintf(
|
|
||||||
"%s/%s/%04d/%02d/%02d/%s-%s%s",
|
|
||||||
u.objectPrefix,
|
|
||||||
strings.TrimSpace(payload.Scene),
|
|
||||||
now.Year(),
|
|
||||||
now.Month(),
|
|
||||||
now.Day(),
|
|
||||||
baseName,
|
|
||||||
randomHexSuffix(8),
|
|
||||||
extension,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func escapeOSSObjectKey(objectKey string) string {
|
|
||||||
parts := strings.Split(objectKey, "/")
|
|
||||||
for index, part := range parts {
|
|
||||||
parts[index] = url.PathEscape(part)
|
|
||||||
}
|
|
||||||
return strings.Join(parts, "/")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,13 +12,12 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
|
func TestDirectOSSUploadSignsRequestAndReturnsPrivateSignedURL(t *testing.T) {
|
||||||
payload := []byte("production-isomorphic-image")
|
payload := []byte("production-isomorphic-image")
|
||||||
var calls atomic.Int64
|
var calls atomic.Int64
|
||||||
var uploadedPath string
|
var uploadedPath string
|
||||||
@@ -60,9 +59,6 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
|
|||||||
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
|
MediaOSSObjectPrefix: "easyai-ai-gateway/production/media",
|
||||||
}
|
}
|
||||||
uploader := newDirectOSSUploader(cfg)
|
uploader := newDirectOSSUploader(cfg)
|
||||||
uploader.now = func() time.Time {
|
|
||||||
return time.Date(2026, time.July, 30, 12, 0, 0, 0, time.UTC)
|
|
||||||
}
|
|
||||||
service := &Service{cfg: cfg, directOSS: uploader}
|
service := &Service{cfg: cfg, directOSS: uploader}
|
||||||
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
|
uploaded, err := service.UploadFile(context.Background(), FileUploadPayload{
|
||||||
Bytes: payload,
|
Bytes: payload,
|
||||||
@@ -77,29 +73,30 @@ func TestDirectOSSUploadSignsRequestAndReturnsPublicURL(t *testing.T) {
|
|||||||
if calls.Load() != 2 {
|
if calls.Load() != 2 {
|
||||||
t.Fatalf("upload calls=%d, want one retry", calls.Load())
|
t.Fatalf("upload calls=%d, want one retry", calls.Load())
|
||||||
}
|
}
|
||||||
if !strings.HasPrefix(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/2026/07/30/input-") ||
|
if !strings.Contains(uploadedPath, "/easyai-ai-gateway/production/media/request_asset/") ||
|
||||||
!strings.HasSuffix(uploadedPath, ".png") {
|
!strings.HasSuffix(uploadedPath, "/"+sha256Hex(payload)+".png") {
|
||||||
t.Fatalf("unexpected object path=%q", uploadedPath)
|
t.Fatalf("unexpected object path=%q", uploadedPath)
|
||||||
}
|
}
|
||||||
if got := stringFromAny(uploaded["url"]); got != "https://cdn.example.com"+uploadedPath {
|
if got := stringFromAny(uploaded["url"]); !strings.HasPrefix(got, server.URL+uploadedPath+"?") || !strings.Contains(got, "Signature=") {
|
||||||
t.Fatalf("public URL=%q", got)
|
t.Fatalf("private signed URL=%q", got)
|
||||||
|
}
|
||||||
|
if uploaded["accessScope"] != "private" {
|
||||||
|
t.Fatalf("request asset access scope=%v", uploaded["accessScope"])
|
||||||
}
|
}
|
||||||
channel, _ := uploaded["storageChannel"].(map[string]any)
|
channel, _ := uploaded["storageChannel"].(map[string]any)
|
||||||
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
|
if stringFromAny(channel["provider"]) != "aliyun_oss" {
|
||||||
t.Fatalf("storage channel=%+v", channel)
|
t.Fatalf("storage channel=%+v", channel)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
|
func TestDirectOSSPersistsGeneralUploadScene(t *testing.T) {
|
||||||
var calls atomic.Int64
|
var calls atomic.Int64
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
calls.Add(1)
|
calls.Add(1)
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}))
|
}))
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
storageDir := t.TempDir()
|
|
||||||
cfg := config.Config{
|
cfg := config.Config{
|
||||||
LocalUploadedStorageDir: storageDir,
|
|
||||||
MediaOSSDirectEnabled: true,
|
MediaOSSDirectEnabled: true,
|
||||||
MediaOSSEndpoint: server.URL,
|
MediaOSSEndpoint: server.URL,
|
||||||
MediaOSSBucket: "media-bucket",
|
MediaOSSBucket: "media-bucket",
|
||||||
@@ -118,11 +115,11 @@ func TestDirectOSSDoesNotReplaceGeneralUploadScene(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("general upload: %v", err)
|
t.Fatalf("general upload: %v", err)
|
||||||
}
|
}
|
||||||
if calls.Load() != 0 {
|
if calls.Load() != 1 {
|
||||||
t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load())
|
t.Fatalf("direct OSS calls=%d for general upload scene", calls.Load())
|
||||||
}
|
}
|
||||||
channel, _ := uploaded["storageChannel"].(map[string]any)
|
channel, _ := uploaded["storageChannel"].(map[string]any)
|
||||||
if stringFromAny(channel["provider"]) != "local_static" {
|
if stringFromAny(channel["provider"]) != "aliyun_oss" {
|
||||||
t.Fatalf("general upload channel=%+v", channel)
|
t.Fatalf("general upload channel=%+v", channel)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -158,7 +155,7 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
|
|||||||
SourceKey: "b64_json",
|
SourceKey: "b64_json",
|
||||||
},
|
},
|
||||||
0,
|
0,
|
||||||
[]store.FileStorageChannel{{ID: "unused-channel"}},
|
[]store.FileStorageChannel{service.directOSS.fileStorageChannel()},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("direct generated upload: %v", err)
|
t.Fatalf("direct generated upload: %v", err)
|
||||||
@@ -166,11 +163,11 @@ func TestGeneratedInlineAssetUsesDirectOSS(t *testing.T) {
|
|||||||
if calls.Load() != 1 {
|
if calls.Load() != 1 {
|
||||||
t.Fatalf("direct OSS calls=%d, want 1", calls.Load())
|
t.Fatalf("direct OSS calls=%d, want 1", calls.Load())
|
||||||
}
|
}
|
||||||
if contentType != "image/png" || kind != "image" || strategy != "direct_aliyun_oss" {
|
if contentType != "image/png" || kind != "image" || strategy != "upload_inline_media" {
|
||||||
t.Fatalf("generated metadata=%s/%s/%s", contentType, kind, strategy)
|
t.Fatalf("generated metadata=%s/%s/%s", contentType, kind, strategy)
|
||||||
}
|
}
|
||||||
channel, _ := upload["storageChannel"].(map[string]any)
|
channel, _ := upload["storageChannel"].(map[string]any)
|
||||||
if stringFromAny(channel["provider"]) != "aliyun_oss_direct" {
|
if stringFromAny(channel["provider"]) != "aliyun_oss" {
|
||||||
t.Fatalf("storage channel=%+v", channel)
|
t.Fatalf("storage channel=%+v", channel)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,530 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha1"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
objectStorageRequestTimeout = 120 * time.Second
|
||||||
|
objectStorageReadLimit = 256 << 20
|
||||||
|
objectStorageSignedURLTTL = 15 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
var sharedObjectStorageHTTPClient = &http.Client{
|
||||||
|
Timeout: objectStorageRequestTimeout,
|
||||||
|
Transport: &http.Transport{
|
||||||
|
MaxIdleConns: 256,
|
||||||
|
MaxIdleConnsPerHost: 256,
|
||||||
|
MaxConnsPerHost: 256,
|
||||||
|
IdleConnTimeout: 90 * time.Second,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
type objectStorageAdapter struct {
|
||||||
|
channel store.FileStorageChannel
|
||||||
|
client *http.Client
|
||||||
|
now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type FileStorageChannelTestResult struct {
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
PutSucceeded bool `json:"putSucceeded"`
|
||||||
|
HeadSucceeded bool `json:"headSucceeded"`
|
||||||
|
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||||
|
DurationMS int64 `json:"durationMs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFileStorageChannel performs an isolated write, metadata read and cleanup
|
||||||
|
// against one object-storage channel. The random probe avoids overwriting a
|
||||||
|
// content-addressed business object and no object key or credential is exposed.
|
||||||
|
func (s *Service) TestFileStorageChannel(ctx context.Context, channel store.FileStorageChannel) (FileStorageChannelTestResult, error) {
|
||||||
|
startedAt := time.Now()
|
||||||
|
result := FileStorageChannelTestResult{Provider: strings.ToLower(strings.TrimSpace(channel.Provider))}
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
payload := FileUploadPayload{
|
||||||
|
Bytes: []byte("easyai-storage-probe:" + uuid.NewString()),
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
FileName: "probe.bin",
|
||||||
|
Scene: store.FileStorageSceneUpload,
|
||||||
|
Source: "admin-connection-test",
|
||||||
|
}
|
||||||
|
upload, err := adapter.put(ctx, payload)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.PutSucceeded = true
|
||||||
|
objectKey := stringFromAny(upload["objectKey"])
|
||||||
|
if objectKey == "" {
|
||||||
|
return result, storageClientError("storage_write_failed", "object storage probe did not return an object reference", 0, false)
|
||||||
|
}
|
||||||
|
if err := adapter.head(ctx, objectKey); err != nil {
|
||||||
|
_ = adapter.delete(context.WithoutCancel(ctx), objectKey)
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.HeadSucceeded = true
|
||||||
|
if err := adapter.delete(ctx, objectKey); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.DeleteSucceeded = true
|
||||||
|
result.DurationMS = time.Since(startedAt).Milliseconds()
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newObjectStorageAdapter(channel store.FileStorageChannel) (*objectStorageAdapter, error) {
|
||||||
|
provider := strings.ToLower(strings.TrimSpace(channel.Provider))
|
||||||
|
if provider != "aliyun_oss" && provider != "s3" {
|
||||||
|
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported object storage provider", Retryable: false}
|
||||||
|
}
|
||||||
|
if objectStorageConfigString(channel.Config, "endpoint") == "" || objectStorageConfigString(channel.Config, "bucket") == "" {
|
||||||
|
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "object storage endpoint and bucket are required", Retryable: false}
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(channel.AccessKeyID) == "" || strings.TrimSpace(channel.AccessKeySecret) == "" {
|
||||||
|
return nil, &clients.ClientError{Code: "storage_auth_failed", Message: "object storage credentials are not configured", Retryable: false}
|
||||||
|
}
|
||||||
|
return &objectStorageAdapter{
|
||||||
|
channel: channel,
|
||||||
|
client: sharedObjectStorageHTTPClient,
|
||||||
|
now: time.Now,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) put(ctx context.Context, payload FileUploadPayload) (map[string]any, error) {
|
||||||
|
objectKey := a.objectKey(payload)
|
||||||
|
requestURL, err := a.objectURL(objectKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
contentType := strings.TrimSpace(payload.ContentType)
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, bytes.NewReader(payload.Bytes))
|
||||||
|
if err != nil {
|
||||||
|
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", contentType)
|
||||||
|
if err := a.sign(req, sha256Hex(payload.Bytes), a.now().UTC()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := a.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, storageClientError("storage_write_failed", err.Error(), 0, true)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||||
|
if readErr != nil {
|
||||||
|
return nil, storageClientError("storage_write_failed", readErr.Error(), resp.StatusCode, storageStatusRetryable(resp.StatusCode))
|
||||||
|
}
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return nil, objectStorageHTTPError("storage_write_failed", resp.StatusCode, responseBody)
|
||||||
|
}
|
||||||
|
accessScope := objectStorageAccessScope(a.channel, payload.Scene)
|
||||||
|
publicURL := ""
|
||||||
|
if accessScope == "public" {
|
||||||
|
publicURL = a.publicURL(objectKey)
|
||||||
|
}
|
||||||
|
urlExpiresAt := ""
|
||||||
|
if publicURL == "" {
|
||||||
|
publicURL, err = a.presignGet(objectKey, objectStorageSignedURLTTL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
urlExpiresAt = a.now().UTC().Add(objectStorageSignedURLTTL).Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(payload.Bytes)
|
||||||
|
result := map[string]any{
|
||||||
|
"url": publicURL,
|
||||||
|
"fileName": path.Base(objectKey),
|
||||||
|
"objectKey": objectKey,
|
||||||
|
"contentType": contentType,
|
||||||
|
"size": len(payload.Bytes),
|
||||||
|
"sha256": hex.EncodeToString(digest[:]),
|
||||||
|
"accessScope": accessScope,
|
||||||
|
"storageChannel": map[string]any{
|
||||||
|
"id": a.channel.ID,
|
||||||
|
"channelKey": a.channel.ChannelKey,
|
||||||
|
"name": a.channel.Name,
|
||||||
|
"provider": a.channel.Provider,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if urlExpiresAt != "" {
|
||||||
|
result["urlExpiresAt"] = urlExpiresAt
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) get(ctx context.Context, objectKey string) ([]byte, error) {
|
||||||
|
requestURL, err := a.objectURL(objectKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
|
}
|
||||||
|
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := a.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, storageClientError("storage_read_failed", err.Error(), 0, true)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||||
|
return nil, objectStorageHTTPError("storage_read_failed", resp.StatusCode, body)
|
||||||
|
}
|
||||||
|
payload, err := io.ReadAll(io.LimitReader(resp.Body, objectStorageReadLimit+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, storageClientError("storage_read_failed", err.Error(), resp.StatusCode, true)
|
||||||
|
}
|
||||||
|
if len(payload) > objectStorageReadLimit {
|
||||||
|
return nil, storageClientError("storage_read_failed", "stored object exceeds the read limit", resp.StatusCode, false)
|
||||||
|
}
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) head(ctx context.Context, objectKey string) error {
|
||||||
|
return a.emptyObjectRequest(ctx, http.MethodHead, objectKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) delete(ctx context.Context, objectKey string) error {
|
||||||
|
return a.emptyObjectRequest(ctx, http.MethodDelete, objectKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) emptyObjectRequest(ctx context.Context, method string, objectKey string) error {
|
||||||
|
requestURL, err := a.objectURL(objectKey)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, method, requestURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
|
}
|
||||||
|
if err := a.sign(req, sha256Hex(nil), a.now().UTC()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := a.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return storageClientError("storage_read_failed", err.Error(), 0, true)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return objectStorageHTTPError("storage_read_failed", resp.StatusCode, nil)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) presignGet(objectKey string, ttl time.Duration) (string, error) {
|
||||||
|
requestURL, err := a.objectURL(objectKey)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if ttl <= 0 {
|
||||||
|
ttl = objectStorageSignedURLTTL
|
||||||
|
}
|
||||||
|
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||||
|
return a.presignS3Get(requestURL, ttl, a.now().UTC())
|
||||||
|
}
|
||||||
|
return a.presignOSSGet(requestURL, objectKey, ttl, a.now().UTC())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) objectKey(payload FileUploadPayload) string {
|
||||||
|
now := a.now().UTC()
|
||||||
|
digest := sha256.Sum256(payload.Bytes)
|
||||||
|
extension := uploadFileExtension(payload.ContentType, path.Ext(payload.FileName))
|
||||||
|
prefix := strings.Trim(objectStorageConfigString(a.channel.Config, "objectPrefix"), "/")
|
||||||
|
parts := make([]string, 0, 7)
|
||||||
|
if prefix != "" {
|
||||||
|
parts = append(parts, prefix)
|
||||||
|
}
|
||||||
|
parts = append(parts,
|
||||||
|
firstNonEmptyString(strings.TrimSpace(payload.Scene), store.FileStorageSceneUpload),
|
||||||
|
fmt.Sprintf("%04d", now.Year()),
|
||||||
|
fmt.Sprintf("%02d", now.Month()),
|
||||||
|
fmt.Sprintf("%02d", now.Day()),
|
||||||
|
hex.EncodeToString(digest[:])+extension,
|
||||||
|
)
|
||||||
|
return strings.Join(parts, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) objectURL(objectKey string) (string, error) {
|
||||||
|
endpoint, err := url.Parse(strings.TrimRight(objectStorageConfigString(a.channel.Config, "endpoint"), "/"))
|
||||||
|
if err != nil || endpoint.Scheme == "" || endpoint.Host == "" || endpoint.User != nil {
|
||||||
|
return "", storageClientError("storage_config_invalid", "invalid object storage endpoint", 0, false)
|
||||||
|
}
|
||||||
|
bucket := objectStorageConfigString(a.channel.Config, "bucket")
|
||||||
|
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||||
|
if objectStorageConfigBool(a.channel.Config, "forcePathStyle") {
|
||||||
|
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + bucket + "/" + strings.TrimLeft(objectKey, "/")
|
||||||
|
} else {
|
||||||
|
endpoint.Host = bucket + "." + endpoint.Host
|
||||||
|
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
hostname := endpoint.Hostname()
|
||||||
|
forcePathStyle := objectStorageConfigBool(a.channel.Config, "forcePathStyle")
|
||||||
|
if !forcePathStyle && net.ParseIP(hostname) == nil && hostname != "localhost" && !strings.HasPrefix(strings.ToLower(hostname), strings.ToLower(bucket)+".") {
|
||||||
|
if port := endpoint.Port(); port != "" {
|
||||||
|
endpoint.Host = bucket + "." + hostname + ":" + port
|
||||||
|
} else {
|
||||||
|
endpoint.Host = bucket + "." + endpoint.Host
|
||||||
|
}
|
||||||
|
}
|
||||||
|
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + strings.TrimLeft(objectKey, "/")
|
||||||
|
}
|
||||||
|
return endpoint.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) publicURL(objectKey string) string {
|
||||||
|
baseURL := strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseUrl"), "/")
|
||||||
|
if baseURL == "" {
|
||||||
|
baseURL = strings.TrimRight(objectStorageConfigString(a.channel.Config, "publicBaseURL"), "/")
|
||||||
|
}
|
||||||
|
if baseURL == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return baseURL + "/" + escapeObjectKey(objectKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) sign(req *http.Request, payloadHash string, now time.Time) error {
|
||||||
|
if strings.EqualFold(a.channel.Provider, "s3") {
|
||||||
|
a.signS3Request(req, payloadHash, now)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
a.signOSSRequest(req, now)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) signOSSRequest(req *http.Request, now time.Time) {
|
||||||
|
date := now.UTC().Format(http.TimeFormat)
|
||||||
|
contentType := req.Header.Get("Content-Type")
|
||||||
|
canonicalHeaders := ""
|
||||||
|
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||||
|
req.Header.Set("x-oss-security-token", token)
|
||||||
|
canonicalHeaders = "x-oss-security-token:" + token + "\n"
|
||||||
|
}
|
||||||
|
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + req.URL.EscapedPath()
|
||||||
|
stringToSign := req.Method + "\n\n" + contentType + "\n" + date + "\n" + canonicalHeaders + canonicalResource
|
||||||
|
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), stringToSign)
|
||||||
|
req.Header.Set("Authorization", "OSS "+a.channel.AccessKeyID+":"+signature)
|
||||||
|
req.Header.Set("Date", date)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) presignOSSGet(requestURL string, objectKey string, ttl time.Duration, now time.Time) (string, error) {
|
||||||
|
parsed, err := url.Parse(requestURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
|
}
|
||||||
|
expires := strconv.FormatInt(now.Add(ttl).Unix(), 10)
|
||||||
|
canonicalResource := "/" + objectStorageConfigString(a.channel.Config, "bucket") + "/" + strings.TrimLeft(objectKey, "/")
|
||||||
|
signature := hmacSHA1Base64([]byte(a.channel.AccessKeySecret), "GET\n\n\n"+expires+"\n"+canonicalResource)
|
||||||
|
query := parsed.Query()
|
||||||
|
query.Set("OSSAccessKeyId", a.channel.AccessKeyID)
|
||||||
|
query.Set("Expires", expires)
|
||||||
|
query.Set("Signature", signature)
|
||||||
|
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||||
|
query.Set("security-token", token)
|
||||||
|
}
|
||||||
|
parsed.RawQuery = query.Encode()
|
||||||
|
return parsed.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) signS3Request(req *http.Request, payloadHash string, now time.Time) {
|
||||||
|
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
|
||||||
|
amzDate := now.Format("20060102T150405Z")
|
||||||
|
date := now.Format("20060102")
|
||||||
|
req.Header.Set("x-amz-date", amzDate)
|
||||||
|
req.Header.Set("x-amz-content-sha256", payloadHash)
|
||||||
|
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||||
|
req.Header.Set("x-amz-security-token", token)
|
||||||
|
}
|
||||||
|
canonicalHeaders, signedHeaders := s3CanonicalHeaders(req)
|
||||||
|
canonicalRequest := strings.Join([]string{
|
||||||
|
req.Method,
|
||||||
|
s3CanonicalURI(req.URL),
|
||||||
|
s3CanonicalQuery(req.URL.Query()),
|
||||||
|
canonicalHeaders,
|
||||||
|
signedHeaders,
|
||||||
|
payloadHash,
|
||||||
|
}, "\n")
|
||||||
|
scope := date + "/" + region + "/s3/aws4_request"
|
||||||
|
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
|
||||||
|
signature := hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign))
|
||||||
|
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+a.channel.AccessKeyID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+signature)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *objectStorageAdapter) presignS3Get(requestURL string, ttl time.Duration, now time.Time) (string, error) {
|
||||||
|
parsed, err := url.Parse(requestURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", storageClientError("storage_config_invalid", err.Error(), 0, false)
|
||||||
|
}
|
||||||
|
region := firstNonEmptyString(objectStorageConfigString(a.channel.Config, "region"), "us-east-1")
|
||||||
|
amzDate := now.Format("20060102T150405Z")
|
||||||
|
date := now.Format("20060102")
|
||||||
|
scope := date + "/" + region + "/s3/aws4_request"
|
||||||
|
seconds := int64(ttl / time.Second)
|
||||||
|
if seconds < 1 {
|
||||||
|
seconds = 1
|
||||||
|
}
|
||||||
|
if seconds > 7*24*60*60 {
|
||||||
|
seconds = 7 * 24 * 60 * 60
|
||||||
|
}
|
||||||
|
query := parsed.Query()
|
||||||
|
query.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
|
||||||
|
query.Set("X-Amz-Credential", a.channel.AccessKeyID+"/"+scope)
|
||||||
|
query.Set("X-Amz-Date", amzDate)
|
||||||
|
query.Set("X-Amz-Expires", strconv.FormatInt(seconds, 10))
|
||||||
|
query.Set("X-Amz-SignedHeaders", "host")
|
||||||
|
if token := strings.TrimSpace(a.channel.SessionToken); token != "" {
|
||||||
|
query.Set("X-Amz-Security-Token", token)
|
||||||
|
}
|
||||||
|
canonicalRequest := "GET\n" + s3CanonicalURI(parsed) + "\n" + s3CanonicalQuery(query) + "\nhost:" + strings.ToLower(parsed.Host) + "\n\nhost\nUNSIGNED-PAYLOAD"
|
||||||
|
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + sha256Hex([]byte(canonicalRequest))
|
||||||
|
query.Set("X-Amz-Signature", hex.EncodeToString(s3SigningHMAC(a.channel.AccessKeySecret, date, region, stringToSign)))
|
||||||
|
parsed.RawQuery = query.Encode()
|
||||||
|
return parsed.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3CanonicalHeaders(req *http.Request) (string, string) {
|
||||||
|
headers := map[string]string{"host": strings.ToLower(req.URL.Host)}
|
||||||
|
for key, values := range req.Header {
|
||||||
|
lower := strings.ToLower(strings.TrimSpace(key))
|
||||||
|
if lower == "content-type" || strings.HasPrefix(lower, "x-amz-") {
|
||||||
|
headers[lower] = strings.Join(values, ",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keys := make([]string, 0, len(headers))
|
||||||
|
for key := range headers {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
var canonical strings.Builder
|
||||||
|
for _, key := range keys {
|
||||||
|
canonical.WriteString(key)
|
||||||
|
canonical.WriteByte(':')
|
||||||
|
canonical.WriteString(strings.Join(strings.Fields(headers[key]), " "))
|
||||||
|
canonical.WriteByte('\n')
|
||||||
|
}
|
||||||
|
return canonical.String(), strings.Join(keys, ";")
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3CanonicalURI(value *url.URL) string {
|
||||||
|
uri := value.EscapedPath()
|
||||||
|
if uri == "" {
|
||||||
|
return "/"
|
||||||
|
}
|
||||||
|
return uri
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3CanonicalQuery(values url.Values) string {
|
||||||
|
return strings.ReplaceAll(values.Encode(), "+", "%20")
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3SigningHMAC(secret string, date string, region string, stringToSign string) []byte {
|
||||||
|
dateKey := hmacSHA256([]byte("AWS4"+secret), date)
|
||||||
|
regionKey := hmacSHA256(dateKey, region)
|
||||||
|
serviceKey := hmacSHA256(regionKey, "s3")
|
||||||
|
signingKey := hmacSHA256(serviceKey, "aws4_request")
|
||||||
|
return hmacSHA256(signingKey, stringToSign)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hmacSHA256(key []byte, value string) []byte {
|
||||||
|
mac := hmac.New(sha256.New, key)
|
||||||
|
_, _ = mac.Write([]byte(value))
|
||||||
|
return mac.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hmacSHA1Base64(key []byte, value string) string {
|
||||||
|
mac := hmac.New(sha1.New, key)
|
||||||
|
_, _ = mac.Write([]byte(value))
|
||||||
|
return base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
func sha256Hex(value []byte) string {
|
||||||
|
digest := sha256.Sum256(value)
|
||||||
|
return hex.EncodeToString(digest[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageConfigString(config map[string]any, key string) string {
|
||||||
|
if config == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
value, _ := config[key].(string)
|
||||||
|
return strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageConfigBool(config map[string]any, key string) bool {
|
||||||
|
if config == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
value, _ := config[key].(bool)
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageAccessScope(channel store.FileStorageChannel, scene string) string {
|
||||||
|
if scope := strings.ToLower(objectStorageConfigString(channel.Config, "accessScope")); scope == "public" || scope == "private" {
|
||||||
|
return scope
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(scene) == store.FileStorageSceneRequestAsset {
|
||||||
|
return "private"
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseUrl")) != "" || strings.TrimSpace(objectStorageConfigString(channel.Config, "publicBaseURL")) != "" {
|
||||||
|
return "public"
|
||||||
|
}
|
||||||
|
return "private"
|
||||||
|
}
|
||||||
|
|
||||||
|
func escapeObjectKey(objectKey string) string {
|
||||||
|
parts := strings.Split(strings.TrimLeft(objectKey, "/"), "/")
|
||||||
|
for index, part := range parts {
|
||||||
|
parts[index] = url.PathEscape(part)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func storageStatusRetryable(status int) bool {
|
||||||
|
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStorageHTTPError(code string, status int, body []byte) error {
|
||||||
|
if status == http.StatusUnauthorized || status == http.StatusForbidden {
|
||||||
|
code = "storage_auth_failed"
|
||||||
|
}
|
||||||
|
message := http.StatusText(status)
|
||||||
|
if message == "" {
|
||||||
|
message = "object storage request failed"
|
||||||
|
}
|
||||||
|
// The provider response may contain object names, endpoints or credential
|
||||||
|
// diagnostics. Keep it out of the public error chain; channel health stores
|
||||||
|
// only the stable status description as well.
|
||||||
|
return storageClientError(code, message, status, storageStatusRetryable(status))
|
||||||
|
}
|
||||||
|
|
||||||
|
func storageClientError(code string, message string, status int, retryable bool) error {
|
||||||
|
return &clients.ClientError{Code: code, Message: message, StatusCode: status, Retryable: retryable}
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceemulator"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLocalAcceptanceObjectStorageFailoverMatrix(t *testing.T) {
|
||||||
|
emulator := httptest.NewServer(acceptanceemulator.New(acceptanceemulator.Config{}).Handler())
|
||||||
|
defer emulator.Close()
|
||||||
|
service := &Service{}
|
||||||
|
payload := FileUploadPayload{
|
||||||
|
Bytes: []byte("acceptance-object"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset,
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
channels []store.FileStorageChannel
|
||||||
|
winner string
|
||||||
|
wantErrCode string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "oss_auth_to_s3",
|
||||||
|
channels: []store.FileStorageChannel{
|
||||||
|
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-auth/bucket", "oss-auth", false),
|
||||||
|
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3", "s3-ok", false),
|
||||||
|
},
|
||||||
|
winner: "s3-ok",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s3_auth_to_oss",
|
||||||
|
channels: []store.FileStorageChannel{
|
||||||
|
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-auth", "s3-auth", false),
|
||||||
|
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-ok", false),
|
||||||
|
},
|
||||||
|
winner: "oss-ok",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same_channel_transient_retry",
|
||||||
|
channels: []store.FileStorageChannel{
|
||||||
|
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-transient", "s3-transient", true),
|
||||||
|
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss/bucket", "oss-unused", false),
|
||||||
|
},
|
||||||
|
winner: "s3-transient",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "all_channels_failed",
|
||||||
|
channels: []store.FileStorageChannel{
|
||||||
|
acceptanceStorageChannel("aliyun_oss", emulator.URL+"/storage/oss-fail/bucket", "oss-fail", false),
|
||||||
|
acceptanceStorageChannel("s3", emulator.URL+"/storage/s3-fail", "s3-fail", false),
|
||||||
|
},
|
||||||
|
wantErrCode: "storage_write_failed",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
result, err := service.uploadFileWithFailover(t.Context(), payload, test.channels)
|
||||||
|
if test.wantErrCode != "" {
|
||||||
|
if clients.ErrorCode(err) != test.wantErrCode {
|
||||||
|
t.Fatalf("error=%v code=%s", err, clients.ErrorCode(err))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
channel, _ := result["storageChannel"].(map[string]any)
|
||||||
|
if got := stringFromAny(channel["channelKey"]); got != test.winner {
|
||||||
|
t.Fatalf("winner=%q, want %q", got, test.winner)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err := http.Get(emulator.URL + "/report")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
var report acceptanceemulator.Report
|
||||||
|
if err := json.NewDecoder(response.Body).Decode(&report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if report.StoragePuts != 3 || report.StorageFailures != 5 {
|
||||||
|
t.Fatalf("unexpected storage acceptance report: %+v", report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func acceptanceStorageChannel(provider string, endpoint string, key string, retry bool) store.FileStorageChannel {
|
||||||
|
channel := testObjectStorageChannel(provider, endpoint, key)
|
||||||
|
channel.Config["publicBaseUrl"] = ""
|
||||||
|
channel.RetryPolicy = map[string]any{"enabled": retry, "maxRetries": 0}
|
||||||
|
if retry {
|
||||||
|
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
||||||
|
}
|
||||||
|
return channel
|
||||||
|
}
|
||||||
@@ -0,0 +1,314 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type objectStorageMetricEvent struct {
|
||||||
|
event string
|
||||||
|
provider string
|
||||||
|
bytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type objectStorageMetricsRecorder struct {
|
||||||
|
events []objectStorageMetricEvent
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *objectStorageMetricsRecorder) ObserveBillingEvent(string) {}
|
||||||
|
|
||||||
|
func (m *objectStorageMetricsRecorder) ObserveObjectStorage(event string, provider string, bytes int64, _ time.Duration) {
|
||||||
|
m.events = append(m.events, objectStorageMetricEvent{event: event, provider: provider, bytes: bytes})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3ObjectStorageUsesSigV4AndDeterministicObjectKey(t *testing.T) {
|
||||||
|
payload := []byte("same-media-payload")
|
||||||
|
var calls atomic.Int64
|
||||||
|
var requestPath string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
requestPath = r.URL.Path
|
||||||
|
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=access-id/") {
|
||||||
|
t.Errorf("missing SigV4 authorization: %q", r.Header.Get("Authorization"))
|
||||||
|
}
|
||||||
|
if r.Header.Get("x-amz-content-sha256") != sha256Hex(payload) {
|
||||||
|
t.Errorf("payload hash=%q", r.Header.Get("x-amz-content-sha256"))
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
if string(body) != string(payload) {
|
||||||
|
t.Errorf("payload=%q", body)
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
channel := testObjectStorageChannel("s3", server.URL, "s3-primary")
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC) }
|
||||||
|
result, err := adapter.put(t.Context(), FileUploadPayload{Bytes: payload, ContentType: "image/png", FileName: "ignored.jpg", Scene: store.FileStorageSceneImageResult})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantSuffix := "/bucket/media/image_result/2026/08/04/" + sha256Hex(payload) + ".png"
|
||||||
|
if requestPath != wantSuffix {
|
||||||
|
t.Fatalf("request path=%q, want %q", requestPath, wantSuffix)
|
||||||
|
}
|
||||||
|
if stringFromAny(result["objectKey"]) != strings.TrimPrefix(wantSuffix, "/bucket/") {
|
||||||
|
t.Fatalf("object key=%q", result["objectKey"])
|
||||||
|
}
|
||||||
|
if calls.Load() != 1 {
|
||||||
|
t.Fatalf("calls=%d", calls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestAssetUsesPrivateSignedURLEvenWhenPublicBaseURLExists(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
channel := testObjectStorageChannel("s3", server.URL, "s3-private")
|
||||||
|
channel.Config["publicBaseUrl"] = "https://cdn.example"
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adapter.now = func() time.Time { return time.Date(2026, time.August, 4, 0, 0, 0, 0, time.UTC) }
|
||||||
|
|
||||||
|
result, err := adapter.put(t.Context(), FileUploadPayload{
|
||||||
|
Bytes: []byte("private input"), ContentType: "image/png", FileName: "input.png", Scene: store.FileStorageSceneRequestAsset,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result["accessScope"] != "private" {
|
||||||
|
t.Fatalf("accessScope=%v, want private", result["accessScope"])
|
||||||
|
}
|
||||||
|
gotURL := stringFromAny(result["url"])
|
||||||
|
if strings.HasPrefix(gotURL, "https://cdn.example/") || !strings.Contains(gotURL, "X-Amz-Signature=") {
|
||||||
|
t.Fatalf("request asset URL must be signed and private: %s", gotURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAliyunOSSUsesBucketVirtualHostForRegionalEndpoint(t *testing.T) {
|
||||||
|
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
||||||
|
channel.Config["forcePathStyle"] = false
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
value, err := adapter.objectURL("media/request_asset/object.png")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if value != "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/request_asset/object.png" {
|
||||||
|
t.Fatalf("OSS object URL=%q", value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAliyunOSSRequestUsesSignatureV1(t *testing.T) {
|
||||||
|
channel := testObjectStorageChannel("aliyun_oss", "https://oss-cn-hangzhou.aliyuncs.com", "oss-primary")
|
||||||
|
channel.Config["forcePathStyle"] = false
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request, err := http.NewRequest(http.MethodPut, "https://bucket.oss-cn-hangzhou.aliyuncs.com/media/image_result/2026/08/04/OBJECT.png", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request.Header.Set("Content-Type", "image/png")
|
||||||
|
adapter.signOSSRequest(request, time.Date(2026, time.August, 4, 1, 2, 3, 0, time.UTC))
|
||||||
|
if got, want := request.Header.Get("Authorization"), "OSS access-id:fohlh2SG7k7D+cmX48b7Keqa1Ok="; got != want {
|
||||||
|
t.Fatalf("authorization=%q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got := request.Header.Get("Date"); got != "Tue, 04 Aug 2026 01:02:03 GMT" {
|
||||||
|
t.Fatalf("date=%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectStorageRetriesCurrentChannelThenFailsOver(t *testing.T) {
|
||||||
|
var primaryCalls atomic.Int64
|
||||||
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
primaryCalls.Add(1)
|
||||||
|
http.Error(w, "temporary", http.StatusServiceUnavailable)
|
||||||
|
}))
|
||||||
|
defer primary.Close()
|
||||||
|
var secondaryCalls atomic.Int64
|
||||||
|
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
secondaryCalls.Add(1)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer secondary.Close()
|
||||||
|
|
||||||
|
first := testObjectStorageChannel("s3", primary.URL, "s3-primary")
|
||||||
|
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
||||||
|
second := testObjectStorageChannel("aliyun_oss", secondary.URL, "oss-secondary")
|
||||||
|
second.RetryPolicy = map[string]any{"enabled": false}
|
||||||
|
metrics := &objectStorageMetricsRecorder{}
|
||||||
|
service := &Service{billingMetrics: metrics}
|
||||||
|
result, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
channel, _ := result["storageChannel"].(map[string]any)
|
||||||
|
if stringFromAny(channel["channelKey"]) != "oss-secondary" {
|
||||||
|
t.Fatalf("unexpected winning channel: %+v", channel)
|
||||||
|
}
|
||||||
|
if primaryCalls.Load() != 2 || secondaryCalls.Load() != 1 {
|
||||||
|
t.Fatalf("calls primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
|
||||||
|
}
|
||||||
|
assertObjectStorageMetricEvent(t, metrics.events, "retry", "s3", 0)
|
||||||
|
assertObjectStorageMetricEvent(t, metrics.events, "failover", "s3", 0)
|
||||||
|
assertObjectStorageMetricEvent(t, metrics.events, "write_success", "aliyun_oss", int64(len("payload")))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFileStorageChannelConnectionProbeWritesHeadsAndDeletes(t *testing.T) {
|
||||||
|
var methods []string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
methods = append(methods, r.Method)
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodPut, http.MethodHead, http.MethodDelete:
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
default:
|
||||||
|
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
result, err := (&Service{}).TestFileStorageChannel(t.Context(), testObjectStorageChannel("s3", server.URL, "probe"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.PutSucceeded || !result.HeadSucceeded || !result.DeleteSucceeded || result.Provider != "s3" {
|
||||||
|
t.Fatalf("unexpected probe result: %+v", result)
|
||||||
|
}
|
||||||
|
if strings.Join(methods, ",") != "PUT,HEAD,DELETE" {
|
||||||
|
t.Fatalf("probe methods=%v", methods)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectStorageReadUsesChannelRetryPolicy(t *testing.T) {
|
||||||
|
var calls atomic.Int64
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if calls.Add(1) == 1 {
|
||||||
|
http.Error(w, "temporary", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte("stored-payload"))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
channel := testObjectStorageChannel("s3", server.URL, "s3-read")
|
||||||
|
channel.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 1, "backoffSeconds": []any{0.001}}
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
payload, err := readObjectStorageWithRetries(t.Context(), adapter, "media/request_asset/object.png")
|
||||||
|
if err != nil || string(payload) != "stored-payload" || calls.Load() != 2 {
|
||||||
|
t.Fatalf("payload=%q calls=%d err=%v", payload, calls.Load(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectStorageAuthFailureSkipsSameChannelRetry(t *testing.T) {
|
||||||
|
var primaryCalls atomic.Int64
|
||||||
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
primaryCalls.Add(1)
|
||||||
|
http.Error(w, "forbidden details", http.StatusForbidden)
|
||||||
|
}))
|
||||||
|
defer primary.Close()
|
||||||
|
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||||
|
defer secondary.Close()
|
||||||
|
|
||||||
|
first := testObjectStorageChannel("aliyun_oss", primary.URL, "oss-primary")
|
||||||
|
first.RetryPolicy = map[string]any{"enabled": true, "maxRetries": 3, "backoffSeconds": []any{0.001}}
|
||||||
|
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
|
||||||
|
service := &Service{}
|
||||||
|
if _, err := service.uploadFileWithFailover(t.Context(), FileUploadPayload{Bytes: []byte("payload"), ContentType: "image/png", Scene: store.FileStorageSceneRequestAsset}, []store.FileStorageChannel{first, second}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if primaryCalls.Load() != 1 {
|
||||||
|
t.Fatalf("auth failure retried %d times", primaryCalls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestLevelUploadFailureDoesNotSwitchChannels(t *testing.T) {
|
||||||
|
var primaryCalls atomic.Int64
|
||||||
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
primaryCalls.Add(1)
|
||||||
|
http.Error(w, "unsupported media", http.StatusUnsupportedMediaType)
|
||||||
|
}))
|
||||||
|
defer primary.Close()
|
||||||
|
var secondaryCalls atomic.Int64
|
||||||
|
secondary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
secondaryCalls.Add(1)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer secondary.Close()
|
||||||
|
|
||||||
|
first := store.FileStorageChannel{
|
||||||
|
ChannelKey: "server-main-primary", Provider: "server_main_openapi", UploadURL: primary.URL,
|
||||||
|
APIKey: "test-key", RetryPolicy: map[string]any{"enabled": false},
|
||||||
|
}
|
||||||
|
second := testObjectStorageChannel("s3", secondary.URL, "s3-secondary")
|
||||||
|
_, err := (&Service{}).uploadFileWithFailover(t.Context(), FileUploadPayload{
|
||||||
|
Bytes: []byte("payload"), ContentType: "application/x-unsupported", Scene: store.FileStorageSceneUpload,
|
||||||
|
}, []store.FileStorageChannel{first, second})
|
||||||
|
if err == nil || clients.ErrorCode(err) != "upload_failed" {
|
||||||
|
t.Fatalf("unexpected request-level error: %v", err)
|
||||||
|
}
|
||||||
|
if primaryCalls.Load() != 1 || secondaryCalls.Load() != 0 {
|
||||||
|
t.Fatalf("request-level failure switched channels: primary=%d secondary=%d", primaryCalls.Load(), secondaryCalls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAllObjectStorageChannelsFailWithStableError(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "failure", http.StatusBadGateway) }))
|
||||||
|
defer server.Close()
|
||||||
|
channel := testObjectStorageChannel("s3", server.URL, "only")
|
||||||
|
channel.RetryPolicy = map[string]any{"enabled": false}
|
||||||
|
_, err := (&Service{}).uploadFileWithFailover(context.Background(), FileUploadPayload{Bytes: []byte("payload"), Scene: store.FileStorageSceneUpload}, []store.FileStorageChannel{channel})
|
||||||
|
if clients.ErrorCode(err) != "storage_write_failed" || strings.Contains(err.Error(), "failure") {
|
||||||
|
t.Fatalf("unexpected public storage error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testObjectStorageChannel(provider string, endpoint string, channelKey string) store.FileStorageChannel {
|
||||||
|
return store.FileStorageChannel{
|
||||||
|
ChannelKey: channelKey,
|
||||||
|
Name: channelKey,
|
||||||
|
Provider: provider,
|
||||||
|
AccessKeyID: "access-id",
|
||||||
|
AccessKeySecret: "access-secret",
|
||||||
|
Priority: 100,
|
||||||
|
Config: map[string]any{
|
||||||
|
"endpoint": endpoint,
|
||||||
|
"bucket": "bucket",
|
||||||
|
"region": "cn-test-1",
|
||||||
|
"objectPrefix": "media",
|
||||||
|
"publicBaseUrl": "https://cdn.example.com",
|
||||||
|
"forcePathStyle": true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertObjectStorageMetricEvent(t *testing.T, events []objectStorageMetricEvent, event string, provider string, bytes int64) {
|
||||||
|
t.Helper()
|
||||||
|
for _, item := range events {
|
||||||
|
if item.event == event && item.provider == provider && item.bytes == bytes {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("missing metric event %s/%s/%d in %+v", event, provider, bytes, events)
|
||||||
|
}
|
||||||
@@ -148,10 +148,14 @@ func (s *Service) hydrateProviderRequestAssetRef(ctx context.Context, ref map[st
|
|||||||
}
|
}
|
||||||
return base64.StdEncoding.EncodeToString(payload), nil
|
return base64.StdEncoding.EncodeToString(payload), nil
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(asset.URL) == "" {
|
assetURL, err := s.requestAssetAccessURL(ctx, asset)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(assetURL) == "" {
|
||||||
return nil, requestAssetExpiredError(asset)
|
return nil, requestAssetExpiredError(asset)
|
||||||
}
|
}
|
||||||
return asset.URL, nil
|
return assetURL, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value string, path []string, candidate store.RuntimeModelCandidate) (any, error) {
|
func (s *Service) hydrateProviderRequestAssetString(ctx context.Context, value string, path []string, candidate store.RuntimeModelCandidate) (any, error) {
|
||||||
@@ -228,6 +232,10 @@ func (s *Service) resolveRequestAsset(ctx context.Context, ref map[string]any) (
|
|||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
URL: stringFromAny(ref["url"]),
|
URL: stringFromAny(ref["url"]),
|
||||||
StorageProvider: stringFromAny(ref["storageProvider"]),
|
StorageProvider: stringFromAny(ref["storageProvider"]),
|
||||||
|
StorageChannelID: stringFromAny(ref["storageChannelId"]),
|
||||||
|
StorageChannelKey: stringFromAny(ref["storageChannelKey"]),
|
||||||
|
ObjectKey: stringFromAny(ref["objectKey"]),
|
||||||
|
AccessScope: stringFromAny(ref["accessScope"]),
|
||||||
}
|
}
|
||||||
if size := floatFromAny(ref["size"]); size > 0 {
|
if size := floatFromAny(ref["size"]); size > 0 {
|
||||||
asset.ByteSize = int64(size)
|
asset.ByteSize = int64(size)
|
||||||
@@ -254,6 +262,17 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
|
|||||||
if requestAssetIsExpired(asset, time.Now()) {
|
if requestAssetIsExpired(asset, time.Now()) {
|
||||||
return nil, requestAssetExpiredError(asset)
|
return nil, requestAssetExpiredError(asset)
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(asset.ObjectKey) != "" {
|
||||||
|
channel, err := s.fileStorageChannelForAsset(ctx, asset)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return readObjectStorageWithRetries(ctx, adapter, asset.ObjectKey)
|
||||||
|
}
|
||||||
if strings.TrimSpace(asset.LocalPath) != "" {
|
if strings.TrimSpace(asset.LocalPath) != "" {
|
||||||
payload, err := os.ReadFile(asset.LocalPath)
|
payload, err := os.ReadFile(asset.LocalPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -290,6 +309,60 @@ func (s *Service) readRequestAssetBytes(ctx context.Context, asset store.Request
|
|||||||
return nil, requestAssetExpiredError(asset)
|
return nil, requestAssetExpiredError(asset)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func readObjectStorageWithRetries(ctx context.Context, adapter *objectStorageAdapter, objectKey string) ([]byte, error) {
|
||||||
|
maxRetries, delays := uploadRetrySchedule(adapter.channel.RetryPolicy, adapter.channel.Provider)
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||||
|
payload, err := adapter.get(ctx, objectKey)
|
||||||
|
if err == nil {
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
if attempt >= maxRetries || !clients.IsRetryable(err) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err := sleepWithContext(ctx, retryDelayForAttempt(attempt, delays)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) requestAssetAccessURL(ctx context.Context, asset store.RequestAsset) (string, error) {
|
||||||
|
if strings.TrimSpace(asset.ObjectKey) == "" {
|
||||||
|
return asset.URL, nil
|
||||||
|
}
|
||||||
|
channel, err := s.fileStorageChannelForAsset(ctx, asset)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if strings.EqualFold(strings.TrimSpace(asset.AccessScope), "public") {
|
||||||
|
if value := adapter.publicURL(asset.ObjectKey); value != "" {
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return adapter.presignGet(asset.ObjectKey, objectStorageSignedURLTTL)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) fileStorageChannelForAsset(ctx context.Context, asset store.RequestAsset) (store.FileStorageChannel, error) {
|
||||||
|
channelKey := strings.TrimSpace(asset.StorageChannelKey)
|
||||||
|
if channelKey == "environment-direct-oss" && s.directOSS != nil {
|
||||||
|
return s.directOSS.fileStorageChannel(), nil
|
||||||
|
}
|
||||||
|
if s.store == nil || channelKey == "" {
|
||||||
|
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
|
||||||
|
}
|
||||||
|
channel, err := s.store.GetFileStorageChannelByKey(ctx, channelKey)
|
||||||
|
if err != nil {
|
||||||
|
return store.FileStorageChannel{}, &clients.ClientError{Code: "storage_read_failed", Message: "object storage channel is unavailable", Retryable: true}
|
||||||
|
}
|
||||||
|
return channel, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) localPathFromRequestAssetURL(value string) string {
|
func (s *Service) localPathFromRequestAssetURL(value string) string {
|
||||||
raw := strings.TrimSpace(value)
|
raw := strings.TrimSpace(value)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
|
|||||||
@@ -230,6 +230,15 @@ func (s *Service) observeTaskEventSkip(reason string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) observeObjectStorage(event string, provider string, bytes int, duration time.Duration) {
|
||||||
|
observer, ok := s.billingMetrics.(interface {
|
||||||
|
ObserveObjectStorage(string, string, int64, time.Duration)
|
||||||
|
})
|
||||||
|
if ok {
|
||||||
|
observer.ObserveObjectStorage(event, provider, int64(bytes), duration)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
|
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
|
||||||
return s.execute(ctx, task, user, nil)
|
return s.execute(ctx, task, user, nil)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
@@ -74,13 +75,19 @@ func (s *Service) processTaskCallbackBatch(ctx context.Context, client *http.Cli
|
|||||||
}
|
}
|
||||||
|
|
||||||
func deliverTaskCallback(ctx context.Context, client *http.Client, item store.TaskCallbackDelivery, bearerToken string) (int, error) {
|
func deliverTaskCallback(ctx context.Context, client *http.Client, item store.TaskCallbackDelivery, bearerToken string) (int, error) {
|
||||||
body, err := json.Marshal(map[string]any{
|
payload := map[string]any{
|
||||||
"taskId": item.TaskID,
|
"taskId": item.TaskID,
|
||||||
"seq": item.Seq,
|
"seq": item.Seq,
|
||||||
"eventType": item.EventType,
|
"eventType": item.EventType,
|
||||||
"status": item.TaskStatus,
|
"status": item.TaskStatus,
|
||||||
"createdAt": item.CreatedAt.UTC().Format(time.RFC3339Nano),
|
"createdAt": item.CreatedAt.UTC().Format(time.RFC3339Nano),
|
||||||
})
|
}
|
||||||
|
if item.TaskStatus == "failed" || item.TaskStatus == "cancelled" {
|
||||||
|
standard := publicerror.WithIDs(publicerror.FromFields(item.TaskErrorCode, item.TaskErrorMessage, 0, false), item.TaskRequestID, item.TaskID)
|
||||||
|
publicerror.Observe(standard)
|
||||||
|
payload["error"] = standard
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -116,3 +117,33 @@ func TestDeliverTaskCallbackSendsMinimalAuthenticatedBody(t *testing.T) {
|
|||||||
t.Fatalf("callback body=%#v", received)
|
t.Fatalf("callback body=%#v", received)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDeliverFailedTaskCallbackUsesStandardErrorWithoutSocketDetails(t *testing.T) {
|
||||||
|
var received map[string]any
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&received); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
|
||||||
|
status, err := deliverTaskCallback(t.Context(), server.Client(), store.TaskCallbackDelivery{
|
||||||
|
TaskID: "failed-task",
|
||||||
|
Seq: 4,
|
||||||
|
CallbackURL: server.URL,
|
||||||
|
EventType: "task.failed",
|
||||||
|
TaskStatus: "failed",
|
||||||
|
TaskErrorCode: "response_read_error",
|
||||||
|
TaskErrorMessage: raw,
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
}, "")
|
||||||
|
if err != nil || status != http.StatusNoContent {
|
||||||
|
t.Fatalf("status=%d err=%v", status, err)
|
||||||
|
}
|
||||||
|
errorValue, _ := received["error"].(map[string]any)
|
||||||
|
if errorValue["code"] != "upstream_connection_interrupted" || strings.Contains(stringFromAny(errorValue["message"]), "10.42.0.72") {
|
||||||
|
t.Fatalf("unsafe callback error: %#v", errorValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+135
-147
@@ -17,25 +17,18 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/textproto"
|
"net/textproto"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"sort"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultServerMainOpenAPIUploadURL = "http://127.0.0.1:3001/v1/files/upload"
|
const defaultServerMainOpenAPIUploadURL = "http://127.0.0.1:3001/v1/files/upload"
|
||||||
const maxGeneratedAssetFetchBytes = 256 << 20
|
const maxGeneratedAssetFetchBytes = 256 << 20
|
||||||
|
|
||||||
const (
|
|
||||||
localStaticGeneratedPathPrefix = "/static/generated/"
|
|
||||||
localStaticUploadedPathPrefix = "/static/uploaded/"
|
|
||||||
)
|
|
||||||
|
|
||||||
type FileUploadPayload struct {
|
type FileUploadPayload struct {
|
||||||
ContentType string
|
ContentType string
|
||||||
FileName string
|
FileName string
|
||||||
@@ -136,14 +129,6 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
|||||||
return nil, &clients.ClientError{Code: "acceptance_run_inactive", Message: err.Error(), Retryable: false}
|
return nil, &clients.ClientError{Code: "acceptance_run_inactive", Message: err.Error(), Retryable: false}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if policy.LocalizeInlineMedia {
|
|
||||||
next, _, err := s.materializeLocalBinaryResult(ctx, taskID, result)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
redactGeneratedResultRawData(next)
|
|
||||||
return next, nil
|
|
||||||
}
|
|
||||||
if len(data) == 0 && !rawNeedsUpload {
|
if len(data) == 0 && !rawNeedsUpload {
|
||||||
return s.finalizeGeneratedAssets(ctx, taskID, taskKind, result, policy, nil, false, 0)
|
return s.finalizeGeneratedAssets(ctx, taskID, taskKind, result, policy, nil, false, 0)
|
||||||
}
|
}
|
||||||
@@ -179,11 +164,9 @@ func (s *Service) uploadGeneratedAssets(ctx context.Context, taskID string, task
|
|||||||
var channels []store.FileStorageChannel
|
var channels []store.FileStorageChannel
|
||||||
channelsLoaded := false
|
channelsLoaded := false
|
||||||
if needsUpload || rawNeedsUpload {
|
if needsUpload || rawNeedsUpload {
|
||||||
if s.directOSS == nil {
|
|
||||||
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
|
channels, err = s.activeFileStorageChannels(ctx, store.FileStorageSceneImageResult)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "object storage configuration is unavailable", Retryable: true}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
channelsLoaded = true
|
channelsLoaded = true
|
||||||
}
|
}
|
||||||
@@ -323,12 +306,12 @@ func (s *Service) finalizeGeneratedAssets(
|
|||||||
if !TaskResultHasInlineBinary(next) {
|
if !TaskResultHasInlineBinary(next) {
|
||||||
return next, nil
|
return next, nil
|
||||||
}
|
}
|
||||||
persistent, _, err := s.materializeLocalBinaryResult(ctx, taskID, next)
|
return nil, &clients.ClientError{
|
||||||
if err != nil {
|
Code: "storage_write_failed",
|
||||||
return nil, err
|
Message: "generated binary result could not be written to object storage",
|
||||||
|
StatusCode: http.StatusServiceUnavailable,
|
||||||
|
Retryable: true,
|
||||||
}
|
}
|
||||||
redactGeneratedResultRawData(persistent)
|
|
||||||
return persistent, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
|
func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]any) bool {
|
||||||
@@ -355,6 +338,9 @@ func generatedRawValueHasInlineMedia(value any, key string, siblings map[string]
|
|||||||
func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) {
|
func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID string, taskKind string, value any, key string, siblings map[string]any, policy generatedAssetUploadPolicy, channels []store.FileStorageChannel, index *int) (any, bool, error) {
|
||||||
switch typed := value.(type) {
|
switch typed := value.(type) {
|
||||||
case map[string]any:
|
case map[string]any:
|
||||||
|
if payload, contentType, ok := localBufferObjectBytes(typed); ok {
|
||||||
|
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, contentType, firstNonEmptyString(key, "buffer"), siblings, channels, index)
|
||||||
|
}
|
||||||
next := make(map[string]any, len(typed))
|
next := make(map[string]any, len(typed))
|
||||||
changed := false
|
changed := false
|
||||||
for childKey, childValue := range typed {
|
for childKey, childValue := range typed {
|
||||||
@@ -372,6 +358,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
|
|||||||
}
|
}
|
||||||
return value, false, nil
|
return value, false, nil
|
||||||
case []any:
|
case []any:
|
||||||
|
if localBinaryKey(key) {
|
||||||
|
if payload, ok := bytesFromNumberArray(typed); ok {
|
||||||
|
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, payload, mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
|
||||||
|
}
|
||||||
|
}
|
||||||
next := make([]any, len(typed))
|
next := make([]any, len(typed))
|
||||||
changed := false
|
changed := false
|
||||||
for itemIndex, item := range typed {
|
for itemIndex, item := range typed {
|
||||||
@@ -388,6 +379,11 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
|
|||||||
return next, true, nil
|
return next, true, nil
|
||||||
}
|
}
|
||||||
return value, false, nil
|
return value, false, nil
|
||||||
|
case []byte:
|
||||||
|
if len(typed) == 0 {
|
||||||
|
return value, false, nil
|
||||||
|
}
|
||||||
|
return s.uploadGeneratedBinaryValue(ctx, taskID, taskKind, append([]byte(nil), typed...), mediaContentTypeFromItem(siblings), firstNonEmptyString(key, "bytes"), siblings, channels, index)
|
||||||
case string:
|
case string:
|
||||||
asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind)
|
asset, ok := generatedRawInlineMediaAsset(key, typed, siblings, taskKind)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -404,19 +400,36 @@ func (s *Service) uploadGeneratedRawMediaValue(ctx context.Context, taskID strin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) uploadGeneratedBinaryValue(ctx context.Context, taskID string, taskKind string, payload []byte, contentType string, sourceKey string, siblings map[string]any, channels []store.FileStorageChannel, index *int) (any, bool, error) {
|
||||||
|
contentType = firstNonEmptyString(contentType, defaultContentTypeForRawMediaKey(sourceKey))
|
||||||
|
asset := &generatedInlineAsset{
|
||||||
|
Bytes: payload,
|
||||||
|
ContentType: contentType,
|
||||||
|
Kind: mediaKindForAsset(taskKind, siblings, sourceKey, contentType),
|
||||||
|
SourceKey: sourceKey,
|
||||||
|
}
|
||||||
|
upload, resolvedContentType, kind, strategy, err := s.uploadGeneratedAsset(ctx, taskID, asset, *index, channels)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
*index = *index + 1
|
||||||
|
return generatedRawMediaReference(asset, upload, resolvedContentType, kind, strategy), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) {
|
func generatedRawInlineMediaAsset(key string, value string, siblings map[string]any, taskKind string) (*generatedInlineAsset, bool) {
|
||||||
raw := strings.TrimSpace(value)
|
raw := strings.TrimSpace(value)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
|
keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
|
||||||
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
|
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
|
||||||
if !generatedRawDataMediaPayloadKey(key) && !generatedContentTypeIsMedia(contentType) {
|
if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) {
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
if !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
|
if !keyLooksLikeMediaPayload && !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
payload, payloadContentType, ok, err := inlineMediaPayload(raw, generatedRawDataMediaPayloadKey(key))
|
payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload)
|
||||||
if err != nil || !ok || len(payload) == 0 {
|
if err != nil || !ok || len(payload) == 0 {
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
@@ -454,6 +467,18 @@ func generatedRawMediaReference(asset *generatedInlineAsset, upload map[string]a
|
|||||||
if provider := stringFromAny(channel["provider"]); provider != "" {
|
if provider := stringFromAny(channel["provider"]); provider != "" {
|
||||||
ref["storageProvider"] = provider
|
ref["storageProvider"] = provider
|
||||||
}
|
}
|
||||||
|
if id := stringFromAny(channel["id"]); id != "" {
|
||||||
|
ref["storageChannelId"] = id
|
||||||
|
}
|
||||||
|
if key := stringFromAny(channel["channelKey"]); key != "" {
|
||||||
|
ref["storageChannelKey"] = key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if objectKey := stringFromAny(upload["objectKey"]); objectKey != "" {
|
||||||
|
ref["objectKey"] = objectKey
|
||||||
|
}
|
||||||
|
if accessScope := stringFromAny(upload["accessScope"]); accessScope != "" {
|
||||||
|
ref["accessScope"] = accessScope
|
||||||
}
|
}
|
||||||
out := map[string]any{
|
out := map[string]any{
|
||||||
"assetRef": ref,
|
"assetRef": ref,
|
||||||
@@ -650,6 +675,9 @@ func removeASCIIWhitespace(value string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
|
func (s *Service) generatedAssetUploadPolicy(ctx context.Context) (generatedAssetUploadPolicy, error) {
|
||||||
|
if s.store == nil {
|
||||||
|
return defaultGeneratedAssetUploadPolicy(), nil
|
||||||
|
}
|
||||||
settings, err := s.store.GetFileStorageSettings(ctx)
|
settings, err := s.store.GetFileStorageSettings(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if store.IsUndefinedDatabaseObject(err) {
|
if store.IsUndefinedDatabaseObject(err) {
|
||||||
@@ -665,8 +693,6 @@ func generatedAssetUploadPolicyFromName(policyName string) generatedAssetUploadP
|
|||||||
switch policyName {
|
switch policyName {
|
||||||
case store.FileStorageResultUploadPolicyUploadAll:
|
case store.FileStorageResultUploadPolicyUploadAll:
|
||||||
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
|
return generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true}
|
||||||
case store.FileStorageResultUploadPolicyUploadNone:
|
|
||||||
return generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true}
|
|
||||||
default:
|
default:
|
||||||
return defaultGeneratedAssetUploadPolicy()
|
return defaultGeneratedAssetUploadPolicy()
|
||||||
}
|
}
|
||||||
@@ -682,13 +708,8 @@ func (s *Service) uploadGeneratedAsset(ctx context.Context, taskID string, asset
|
|||||||
Scene: store.FileStorageSceneImageResult,
|
Scene: store.FileStorageSceneImageResult,
|
||||||
Source: "ai-gateway",
|
Source: "ai-gateway",
|
||||||
}
|
}
|
||||||
if s.directOSS != nil {
|
|
||||||
upload, err := s.directOSS.upload(ctx, payload)
|
|
||||||
return upload, contentType, kind, "direct_aliyun_oss", err
|
|
||||||
}
|
|
||||||
if len(channels) == 0 {
|
if len(channels) == 0 {
|
||||||
upload, err := s.storeFileLocally(payload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
|
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||||
return upload, contentType, kind, "local_static_inline_media", err
|
|
||||||
}
|
}
|
||||||
upload, err := s.uploadFileWithFailover(ctx, payload, channels)
|
upload, err := s.uploadFileWithFailover(ctx, payload, channels)
|
||||||
return upload, contentType, kind, "upload_inline_media", err
|
return upload, contentType, kind, "upload_inline_media", err
|
||||||
@@ -708,95 +729,13 @@ func (s *Service) uploadGeneratedURLAsset(ctx context.Context, taskID string, as
|
|||||||
Scene: store.FileStorageSceneImageResult,
|
Scene: store.FileStorageSceneImageResult,
|
||||||
Source: "ai-gateway",
|
Source: "ai-gateway",
|
||||||
}
|
}
|
||||||
if s.directOSS != nil {
|
|
||||||
upload, err := s.directOSS.upload(ctx, uploadPayload)
|
|
||||||
return upload, contentType, kind, "direct_aliyun_oss", err
|
|
||||||
}
|
|
||||||
if len(channels) == 0 {
|
if len(channels) == 0 {
|
||||||
upload, err := s.storeFileLocally(uploadPayload, s.cfg.LocalGeneratedStorageDir, config.DefaultLocalGeneratedStorageDir, localStaticGeneratedPathPrefix)
|
return nil, "", "", "", &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||||
return upload, contentType, kind, "local_static_url_media", err
|
|
||||||
}
|
}
|
||||||
upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels)
|
upload, err := s.uploadFileWithFailover(ctx, uploadPayload, channels)
|
||||||
return upload, contentType, kind, "upload_url_media", err
|
return upload, contentType, kind, "upload_url_media", err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) storeFileLocally(payload FileUploadPayload, storageDir string, fallbackStorageDir string, pathPrefix string) (map[string]any, error) {
|
|
||||||
storageDir = strings.TrimSpace(storageDir)
|
|
||||||
if storageDir == "" {
|
|
||||||
storageDir = fallbackStorageDir
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(storageDir, 0o755); err != nil {
|
|
||||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
|
|
||||||
}
|
|
||||||
fileName := filepath.Base(strings.TrimSpace(payload.FileName))
|
|
||||||
if fileName == "" || fileName == "." || fileName == ".." || fileName == string(filepath.Separator) {
|
|
||||||
kind := generatedAssetKindFromContentType("", payload.ContentType)
|
|
||||||
fileName = generatedAssetFileName("generated", 0, payload.ContentType, kind)
|
|
||||||
}
|
|
||||||
targetPath := filepath.Join(storageDir, fileName)
|
|
||||||
file, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: err.Error(), Retryable: true}
|
|
||||||
}
|
|
||||||
_, writeErr := file.Write(payload.Bytes)
|
|
||||||
closeErr := file.Close()
|
|
||||||
if writeErr != nil {
|
|
||||||
_ = os.Remove(targetPath)
|
|
||||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: writeErr.Error(), Retryable: true}
|
|
||||||
}
|
|
||||||
if closeErr != nil {
|
|
||||||
_ = os.Remove(targetPath)
|
|
||||||
return nil, &clients.ClientError{Code: "local_static_store_failed", Message: closeErr.Error(), Retryable: true}
|
|
||||||
}
|
|
||||||
expiresAt := time.Now().Add(time.Duration(s.localStaticAssetTTLHours()) * time.Hour).UTC().Format(time.RFC3339)
|
|
||||||
return map[string]any{
|
|
||||||
"url": s.localStaticFileURL(fileName, pathPrefix),
|
|
||||||
"fileName": fileName,
|
|
||||||
"contentType": payload.ContentType,
|
|
||||||
"size": len(payload.Bytes),
|
|
||||||
"expiresAt": expiresAt,
|
|
||||||
"storageChannel": map[string]any{
|
|
||||||
"id": "local-static",
|
|
||||||
"channelKey": "local-static",
|
|
||||||
"name": "AI Gateway local static storage",
|
|
||||||
"provider": "local_static",
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Service) localStaticAssetTTLHours() int {
|
|
||||||
if s.cfg.LocalTempAssetTTLHours <= 0 {
|
|
||||||
return 24
|
|
||||||
}
|
|
||||||
return s.cfg.LocalTempAssetTTLHours
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Service) localStaticFileURL(fileName string, pathPrefix string) string {
|
|
||||||
if strings.TrimSpace(pathPrefix) == "" {
|
|
||||||
pathPrefix = localStaticUploadedPathPrefix
|
|
||||||
}
|
|
||||||
path := pathPrefix + url.PathEscape(filepath.Base(fileName))
|
|
||||||
baseURL := strings.TrimRight(strings.TrimSpace(s.cfg.PublicBaseURL), "/")
|
|
||||||
if baseURL == "" {
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
return baseURL + path
|
|
||||||
}
|
|
||||||
|
|
||||||
func localStaticUploadFileName(originalName string, contentType string) string {
|
|
||||||
baseName := filepath.Base(strings.TrimSpace(originalName))
|
|
||||||
originalExt := strings.ToLower(filepath.Ext(baseName))
|
|
||||||
namePart := strings.TrimSuffix(baseName, originalExt)
|
|
||||||
namePart = sanitizeGeneratedAssetNamePart(namePart)
|
|
||||||
if namePart == "" {
|
|
||||||
namePart = "gateway-upload"
|
|
||||||
}
|
|
||||||
if len(namePart) > 48 {
|
|
||||||
namePart = namePart[:48]
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s-%s%s", namePart, randomHexSuffix(6), uploadFileExtension(contentType, originalExt))
|
|
||||||
}
|
|
||||||
|
|
||||||
func uploadFileExtension(contentType string, fallbackExt string) string {
|
func uploadFileExtension(contentType string, fallbackExt string) string {
|
||||||
normalized := normalizeGeneratedContentType(contentType)
|
normalized := normalizeGeneratedContentType(contentType)
|
||||||
if generatedContentTypeIsMedia(normalized) {
|
if generatedContentTypeIsMedia(normalized) {
|
||||||
@@ -981,46 +920,41 @@ func (s *Service) UploadFile(ctx context.Context, payload FileUploadPayload) (ma
|
|||||||
if strings.TrimSpace(payload.Scene) == "" {
|
if strings.TrimSpace(payload.Scene) == "" {
|
||||||
payload.Scene = store.FileStorageSceneUpload
|
payload.Scene = store.FileStorageSceneUpload
|
||||||
}
|
}
|
||||||
if s.directOSS != nil && directOSSScene(payload.Scene) {
|
|
||||||
return s.directOSS.upload(ctx, payload)
|
|
||||||
}
|
|
||||||
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
|
channels, err := s.activeFileStorageChannels(ctx, payload.Scene)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
return nil, &clients.ClientError{Code: "upload_config_failed", Message: err.Error(), Retryable: true}
|
||||||
}
|
}
|
||||||
if len(channels) == 0 {
|
if len(channels) == 0 {
|
||||||
payload.FileName = localStaticUploadFileName(payload.FileName, payload.ContentType)
|
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||||
upload, err := s.storeFileLocally(payload, s.cfg.LocalUploadedStorageDir, config.DefaultLocalUploadedStorageDir, localStaticUploadedPathPrefix)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
upload["assetStorage"] = map[string]any{
|
|
||||||
"scene": payload.Scene,
|
|
||||||
"source": firstNonEmptyString(payload.Source, "ai-gateway-openapi"),
|
|
||||||
"strategy": "local_static_upload",
|
|
||||||
}
|
|
||||||
return upload, nil
|
|
||||||
}
|
}
|
||||||
return s.uploadFileWithFailover(ctx, payload, channels)
|
return s.uploadFileWithFailover(ctx, payload, channels)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) activeFileStorageChannels(ctx context.Context, scene string) ([]store.FileStorageChannel, error) {
|
func (s *Service) activeFileStorageChannels(ctx context.Context, scene string) ([]store.FileStorageChannel, error) {
|
||||||
if s.store == nil {
|
channels := make([]store.FileStorageChannel, 0)
|
||||||
return nil, nil
|
if s.directOSS != nil && directOSSScene(scene) {
|
||||||
|
channels = append(channels, s.directOSS.fileStorageChannel())
|
||||||
}
|
}
|
||||||
channels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
|
if s.store == nil {
|
||||||
|
return channels, nil
|
||||||
|
}
|
||||||
|
storedChannels, err := s.store.ListEnabledFileStorageChannelsForScene(ctx, scene)
|
||||||
if err != nil && !store.IsUndefinedDatabaseObject(err) {
|
if err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if len(channels) > 0 {
|
channels = append(channels, storedChannels...)
|
||||||
return channels, nil
|
sort.SliceStable(channels, func(i, j int) bool {
|
||||||
|
if channels[i].Priority != channels[j].Priority {
|
||||||
|
return channels[i].Priority < channels[j].Priority
|
||||||
}
|
}
|
||||||
return nil, nil
|
return channels[i].ChannelKey < channels[j].ChannelKey
|
||||||
|
})
|
||||||
|
return channels, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUploadPayload, channels []store.FileStorageChannel) (map[string]any, error) {
|
func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUploadPayload, channels []store.FileStorageChannel) (map[string]any, error) {
|
||||||
var lastErr error
|
var lastErr error
|
||||||
for _, channel := range channels {
|
for index, channel := range channels {
|
||||||
upload, err := s.uploadWithChannelRetries(ctx, payload, channel)
|
upload, err := s.uploadWithChannelRetries(ctx, payload, channel)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if s.store != nil {
|
if s.store != nil {
|
||||||
@@ -1032,25 +966,59 @@ func (s *Service) uploadFileWithFailover(ctx context.Context, payload FileUpload
|
|||||||
if s.store != nil {
|
if s.store != nil {
|
||||||
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(ctx), channel.ID, err.Error())
|
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(ctx), channel.ID, err.Error())
|
||||||
}
|
}
|
||||||
|
if !storageFailureAllowsFailover(channel, err) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if index+1 < len(channels) {
|
||||||
|
s.observeObjectStorage("failover", channel.Provider, 0, 0)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if lastErr != nil {
|
if lastErr != nil {
|
||||||
return nil, lastErr
|
s.observeObjectStorage("all_failed", "", 0, 0)
|
||||||
|
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "all configured object storage channels failed", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||||
|
}
|
||||||
|
return nil, &clients.ClientError{Code: "storage_write_failed", Message: "no enabled object storage channel", StatusCode: http.StatusServiceUnavailable, Retryable: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
func storageFailureAllowsFailover(channel store.FileStorageChannel, err error) bool {
|
||||||
|
code := strings.ToLower(strings.TrimSpace(clients.ErrorCode(err)))
|
||||||
|
switch code {
|
||||||
|
case "upload_source_too_large", "upload_decode_failed", "invalid_multipart_file", "invalid_multipart_image", "invalid_multipart_audio":
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var clientErr *clients.ClientError
|
||||||
|
if !errors.As(err, &clientErr) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
switch clientErr.StatusCode {
|
||||||
|
case http.StatusRequestEntityTooLarge, http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity:
|
||||||
|
return false
|
||||||
|
case http.StatusBadRequest:
|
||||||
|
// Object-storage 400 responses commonly indicate endpoint/signature
|
||||||
|
// configuration and should move to the next channel. server-main 400 is
|
||||||
|
// the existing upload API's request validation result.
|
||||||
|
return !strings.EqualFold(strings.TrimSpace(channel.Provider), "server_main_openapi")
|
||||||
|
default:
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
return nil, &clients.ClientError{Code: "upload_no_channel", Message: "no enabled file storage channel", Retryable: false}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
||||||
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy)
|
maxRetries, delays := uploadRetrySchedule(channel.RetryPolicy, channel.Provider)
|
||||||
var lastErr error
|
var lastErr error
|
||||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||||
|
startedAt := time.Now()
|
||||||
upload, err := s.uploadOnce(ctx, payload, channel)
|
upload, err := s.uploadOnce(ctx, payload, channel)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
s.observeObjectStorage("write_success", channel.Provider, len(payload.Bytes), time.Since(startedAt))
|
||||||
return upload, nil
|
return upload, nil
|
||||||
}
|
}
|
||||||
|
s.observeObjectStorage("write_failure", channel.Provider, 0, time.Since(startedAt))
|
||||||
lastErr = err
|
lastErr = err
|
||||||
if attempt >= maxRetries || !clients.IsRetryable(err) {
|
if attempt >= maxRetries || !clients.IsRetryable(err) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
s.observeObjectStorage("retry", channel.Provider, 0, 0)
|
||||||
delay := retryDelayForAttempt(attempt, delays)
|
delay := retryDelayForAttempt(attempt, delays)
|
||||||
if err := sleepWithContext(ctx, delay); err != nil {
|
if err := sleepWithContext(ctx, delay); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -1060,9 +1028,21 @@ func (s *Service) uploadWithChannelRetries(ctx context.Context, payload FileUplo
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) uploadOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
func (s *Service) uploadOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
||||||
if strings.ToLower(strings.TrimSpace(channel.Provider)) != "server_main_openapi" {
|
switch strings.ToLower(strings.TrimSpace(channel.Provider)) {
|
||||||
return nil, &clients.ClientError{Code: "upload_unsupported_channel", Message: "unsupported file storage channel: " + channel.Provider, Retryable: false}
|
case "aliyun_oss", "s3":
|
||||||
|
adapter, err := newObjectStorageAdapter(channel)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return adapter.put(ctx, payload)
|
||||||
|
case "server_main_openapi":
|
||||||
|
return s.uploadServerMainOnce(ctx, payload, channel)
|
||||||
|
default:
|
||||||
|
return nil, &clients.ClientError{Code: "storage_config_invalid", Message: "unsupported file storage channel", Retryable: false}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) uploadServerMainOnce(ctx context.Context, payload FileUploadPayload, channel store.FileStorageChannel) (map[string]any, error) {
|
||||||
uploadURL := strings.TrimSpace(channel.UploadURL)
|
uploadURL := strings.TrimSpace(channel.UploadURL)
|
||||||
if uploadURL == "" {
|
if uploadURL == "" {
|
||||||
uploadURL = defaultServerMainOpenAPIUploadURL
|
uploadURL = defaultServerMainOpenAPIUploadURL
|
||||||
@@ -1700,10 +1680,18 @@ func uniqueStringList(values []string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploadRetrySchedule(policy map[string]any) (int, []time.Duration) {
|
func uploadRetrySchedule(policy map[string]any, providers ...string) (int, []time.Duration) {
|
||||||
|
provider := ""
|
||||||
|
if len(providers) > 0 {
|
||||||
|
provider = providers[0]
|
||||||
|
}
|
||||||
if policy == nil {
|
if policy == nil {
|
||||||
|
if strings.EqualFold(provider, "aliyun_oss") || strings.EqualFold(provider, "s3") {
|
||||||
|
policy = map[string]any{"enabled": true, "maxRetries": 2, "backoffSeconds": []any{0.25, 1.0}}
|
||||||
|
} else {
|
||||||
policy = defaultUploadRetryPolicy()
|
policy = defaultUploadRetryPolicy()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
|
if enabled, ok := policy["enabled"].(bool); ok && !enabled {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
@@ -1725,9 +1713,9 @@ func uploadRetryDelays(value any) []time.Duration {
|
|||||||
}
|
}
|
||||||
delays := make([]time.Duration, 0, len(items))
|
delays := make([]time.Duration, 0, len(items))
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
seconds := int(floatFromAny(item))
|
seconds := floatFromAny(item)
|
||||||
if seconds > 0 {
|
if seconds > 0 {
|
||||||
delays = append(delays, time.Duration(seconds)*time.Second)
|
delays = append(delays, time.Duration(seconds*float64(time.Second)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return delays
|
return delays
|
||||||
|
|||||||
@@ -7,11 +7,11 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
@@ -227,38 +227,6 @@ func TestGeneratedAssetDecisionUploadsURLWhenPolicyUploadAll(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGeneratedAssetDecisionPreservesInlineWhenPolicyUploadNone(t *testing.T) {
|
|
||||||
item := map[string]any{
|
|
||||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
|
|
||||||
}
|
|
||||||
|
|
||||||
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if decision.Inline != nil || decision.URL != nil {
|
|
||||||
t.Fatalf("upload_none should not transfer inline payloads: %+v", decision)
|
|
||||||
}
|
|
||||||
if len(decision.StripKeys) != 0 {
|
|
||||||
t.Fatalf("upload_none should preserve b64_json for the caller: %+v", decision.StripKeys)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGeneratedAssetDecisionPreservesInlineAlongsideURLWhenPolicyUploadNone(t *testing.T) {
|
|
||||||
item := map[string]any{
|
|
||||||
"url": "https://cdn.example.com/generated.png",
|
|
||||||
"b64_json": base64.StdEncoding.EncodeToString([]byte("inline image")),
|
|
||||||
}
|
|
||||||
|
|
||||||
decision, err := generatedAssetDecisionForItem("images.generations", item, generatedAssetUploadPolicyFromName(store.FileStorageResultUploadPolicyUploadNone))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if decision.Inline != nil || decision.URL != nil || len(decision.StripKeys) != 0 {
|
|
||||||
t.Fatalf("upload_none should preserve the upstream URL and base64 fields unchanged: %+v", decision)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
|
func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -276,9 +244,9 @@ func TestGeneratedAssetUploadPolicyFromName(t *testing.T) {
|
|||||||
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true, PreserveInlineMedia: false},
|
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: true, PreserveInlineMedia: false},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "upload none",
|
name: "legacy upload none becomes default",
|
||||||
policyName: store.FileStorageResultUploadPolicyUploadNone,
|
policyName: store.FileStorageResultUploadPolicyUploadNone,
|
||||||
want: generatedAssetUploadPolicy{UploadInlineMedia: false, UploadURLMedia: false, PreserveInlineMedia: true, LocalizeInlineMedia: true},
|
want: generatedAssetUploadPolicy{UploadInlineMedia: true, UploadURLMedia: false, PreserveInlineMedia: false},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -330,8 +298,10 @@ func TestAcceptanceGeneratedMediaAllowsOnlyExactEmulatorOrigin(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *testing.T) {
|
func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *testing.T) {
|
||||||
storageDir := t.TempDir()
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
defer server.Close()
|
||||||
|
service := &Service{}
|
||||||
|
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
|
||||||
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
|
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
|
||||||
encoded := base64.StdEncoding.EncodeToString(payload)
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
||||||
result := map[string]any{
|
result := map[string]any{
|
||||||
@@ -348,7 +318,7 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
|
|||||||
"images.edits",
|
"images.edits",
|
||||||
result,
|
result,
|
||||||
defaultGeneratedAssetUploadPolicy(),
|
defaultGeneratedAssetUploadPolicy(),
|
||||||
nil,
|
channels,
|
||||||
true,
|
true,
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
@@ -363,15 +333,8 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
|
|||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested)
|
t.Fatalf("nested binary should be replaced by an asset reference: %+v", nested)
|
||||||
}
|
}
|
||||||
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-nested-binary-01-") {
|
if urlValue := stringFromAny(reference["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") {
|
||||||
t.Fatalf("unexpected local static URL: %s", urlValue)
|
t.Fatalf("unexpected object storage URL: %s", urlValue)
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(storageDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read generated storage: %v", err)
|
|
||||||
}
|
|
||||||
if len(entries) != 1 {
|
|
||||||
t.Fatalf("expected one localized result file, got %d", len(entries))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -405,9 +368,8 @@ func TestGeneratedAssetFileNameIsUniqueAndTyped(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
|
func TestUploadGeneratedAssetFailsWithoutObjectStorage(t *testing.T) {
|
||||||
storageDir := t.TempDir()
|
service := &Service{}
|
||||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
|
||||||
payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
|
payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
|
||||||
asset := &generatedInlineAsset{
|
asset := &generatedInlineAsset{
|
||||||
Bytes: payload,
|
Bytes: payload,
|
||||||
@@ -416,44 +378,14 @@ func TestUploadGeneratedAssetStoresLocalWhenNoChannels(t *testing.T) {
|
|||||||
SourceKey: "b64_json",
|
SourceKey: "b64_json",
|
||||||
}
|
}
|
||||||
|
|
||||||
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
|
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-123", asset, 0, nil)
|
||||||
if err != nil {
|
if clients.ErrorCode(err) != "storage_write_failed" {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
if contentType != "image/png" || kind != "image" || strategy != "local_static_inline_media" {
|
|
||||||
t.Fatalf("unexpected local upload metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
|
|
||||||
}
|
|
||||||
urlValue := stringFromAny(upload["url"])
|
|
||||||
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-123-01-") || !strings.HasSuffix(urlValue, ".png") {
|
|
||||||
t.Fatalf("unexpected local static URL: %s", urlValue)
|
|
||||||
}
|
|
||||||
expiresAt, err := time.Parse(time.RFC3339, stringFromAny(upload["expiresAt"]))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("local static upload should expose expiresAt: %+v", upload)
|
|
||||||
}
|
|
||||||
remaining := time.Until(expiresAt)
|
|
||||||
if remaining < 23*time.Hour+59*time.Minute || remaining > 24*time.Hour+time.Minute {
|
|
||||||
t.Fatalf("local static upload should expire after one day, remaining=%s", remaining)
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(storageDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read local static dir: %v", err)
|
|
||||||
}
|
|
||||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
|
|
||||||
t.Fatalf("expected one PNG file in local static dir, got %+v", entries)
|
|
||||||
}
|
|
||||||
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read local static file: %v", err)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(stored, payload) {
|
|
||||||
t.Fatalf("stored payload does not match source payload")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
|
func TestUploadGeneratedAudioFailsWithoutObjectStorage(t *testing.T) {
|
||||||
storageDir := t.TempDir()
|
service := &Service{}
|
||||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
|
||||||
asset := &generatedInlineAsset{
|
asset := &generatedInlineAsset{
|
||||||
Bytes: []byte("inline audio payload"),
|
Bytes: []byte("inline audio payload"),
|
||||||
ContentType: "audio/mpeg",
|
ContentType: "audio/mpeg",
|
||||||
@@ -461,32 +393,17 @@ func TestUploadGeneratedAssetStoresAudioLocalWhenNoChannels(t *testing.T) {
|
|||||||
SourceKey: "content",
|
SourceKey: "content",
|
||||||
}
|
}
|
||||||
|
|
||||||
upload, contentType, kind, strategy, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
|
_, _, _, _, err := service.uploadGeneratedAsset(context.Background(), "task-tts", asset, 0, nil)
|
||||||
if err != nil {
|
if clients.ErrorCode(err) != "storage_write_failed" {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
if contentType != "audio/mpeg" || kind != "audio" || strategy != "local_static_inline_media" {
|
|
||||||
t.Fatalf("unexpected local audio metadata: contentType=%s kind=%s strategy=%s", contentType, kind, strategy)
|
|
||||||
}
|
|
||||||
urlValue := stringFromAny(upload["url"])
|
|
||||||
if !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-tts-01-") || !strings.HasSuffix(urlValue, ".mp3") {
|
|
||||||
t.Fatalf("unexpected local audio URL: %s", urlValue)
|
|
||||||
}
|
|
||||||
if stringFromAny(upload["expiresAt"]) == "" {
|
|
||||||
t.Fatalf("local audio static upload should expose expiresAt: %+v", upload)
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(storageDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read local static dir: %v", err)
|
|
||||||
}
|
|
||||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".mp3") {
|
|
||||||
t.Fatalf("expected one MP3 file in local static dir, got %+v", entries)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) {
|
func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *testing.T) {
|
||||||
storageDir := t.TempDir()
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||||
service := &Service{cfg: config.Config{LocalGeneratedStorageDir: storageDir}}
|
defer server.Close()
|
||||||
|
service := &Service{}
|
||||||
|
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
|
||||||
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
|
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 160)...)
|
||||||
raw := map[string]any{
|
raw := map[string]any{
|
||||||
"candidates": []any{
|
"candidates": []any{
|
||||||
@@ -506,7 +423,7 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
|
|||||||
}
|
}
|
||||||
index := 0
|
index := 0
|
||||||
|
|
||||||
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), nil, &index)
|
uploaded, changed, err := service.uploadGeneratedRawMediaValue(context.Background(), "task-raw", "chat.completions", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("upload raw media: %v", err)
|
t.Fatalf("upload raw media: %v", err)
|
||||||
}
|
}
|
||||||
@@ -528,22 +445,47 @@ func TestUploadGeneratedRawMediaValueReplacesGeminiInlineDataWithAssetRef(t *tes
|
|||||||
if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) {
|
if ref["sha256"] == "" || ref["contentType"] != "image/png" || ref["size"] != len(payload) {
|
||||||
t.Fatalf("unexpected asset ref: %+v", ref)
|
t.Fatalf("unexpected asset ref: %+v", ref)
|
||||||
}
|
}
|
||||||
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "/static/generated/gateway-result-task-raw-01-") || !strings.HasSuffix(urlValue, ".png") {
|
if urlValue := stringFromAny(data["url"]); !strings.HasPrefix(urlValue, "https://cdn.example.com/media/image_result/") || !strings.HasSuffix(urlValue, ".png") {
|
||||||
t.Fatalf("unexpected raw media URL: %s", urlValue)
|
t.Fatalf("unexpected raw media URL: %s", urlValue)
|
||||||
}
|
}
|
||||||
if inlineData["data"] == base64.StdEncoding.EncodeToString(payload) {
|
if inlineData["data"] == base64.StdEncoding.EncodeToString(payload) {
|
||||||
t.Fatal("raw inlineData still contains base64 payload")
|
t.Fatal("raw inlineData still contains base64 payload")
|
||||||
}
|
}
|
||||||
entries, err := os.ReadDir(storageDir)
|
}
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read generated storage: %v", err)
|
func TestUploadGeneratedRawMediaValueReplacesBufferAndBytesWithAssetRefs(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||||
|
defer server.Close()
|
||||||
|
service := &Service{}
|
||||||
|
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-buffer-result")}
|
||||||
|
raw := map[string]any{
|
||||||
|
"buffer": map[string]any{
|
||||||
|
"type": "Buffer", "mimeType": "image/png", "data": []any{float64(0x89), float64('P'), float64('N'), float64('G')},
|
||||||
|
},
|
||||||
|
"audio_bytes": []any{float64('I'), float64('D'), float64('3')},
|
||||||
|
"direct": []byte("direct bytes"),
|
||||||
}
|
}
|
||||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".png") {
|
index := 0
|
||||||
t.Fatalf("expected one generated PNG, got %+v", entries)
|
uploaded, changed, err := service.uploadGeneratedRawMediaValue(t.Context(), "task-buffer", "images.generations", raw, "", nil, defaultGeneratedAssetUploadPolicy(), channels, &index)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !changed || index != 3 {
|
||||||
|
t.Fatalf("changed=%v uploads=%d", changed, index)
|
||||||
|
}
|
||||||
|
next := uploaded.(map[string]any)
|
||||||
|
for _, key := range []string{"buffer", "audio_bytes", "direct"} {
|
||||||
|
item, ok := next[key].(map[string]any)
|
||||||
|
if !ok || item["assetRef"] == nil || item["upload"] == nil {
|
||||||
|
t.Fatalf("%s was not objectified: %#v", key, next[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if TaskResultHasInlineBinary(next) {
|
||||||
|
t.Fatalf("objectified result still contains inline binary: %#v", next)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
|
func TestUploadFileFailsWithoutObjectStorageAndCreatesNoLocalFile(t *testing.T) {
|
||||||
storageDir := t.TempDir()
|
storageDir := t.TempDir()
|
||||||
service := &Service{cfg: config.Config{
|
service := &Service{cfg: config.Config{
|
||||||
LocalUploadedStorageDir: storageDir,
|
LocalUploadedStorageDir: storageDir,
|
||||||
@@ -552,43 +494,21 @@ func TestUploadFileStoresLocalWhenNoChannels(t *testing.T) {
|
|||||||
}}
|
}}
|
||||||
payload := []byte("%PDF-1.4")
|
payload := []byte("%PDF-1.4")
|
||||||
|
|
||||||
upload, err := service.UploadFile(context.Background(), FileUploadPayload{
|
_, err := service.UploadFile(context.Background(), FileUploadPayload{
|
||||||
Bytes: payload,
|
Bytes: payload,
|
||||||
ContentType: "application/pdf",
|
ContentType: "application/pdf",
|
||||||
FileName: "用户文件.png",
|
FileName: "用户文件.png",
|
||||||
Source: "playground",
|
Source: "playground",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if clients.ErrorCode(err) != "storage_write_failed" {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
urlValue := stringFromAny(upload["url"])
|
|
||||||
if !strings.HasPrefix(urlValue, "/static/uploaded/") || !strings.HasSuffix(urlValue, ".pdf") {
|
|
||||||
t.Fatalf("unexpected uploaded local static URL: %s", urlValue)
|
|
||||||
}
|
|
||||||
if stringFromAny(upload["expiresAt"]) == "" {
|
|
||||||
t.Fatalf("local uploaded static file should expose expiresAt: %+v", upload)
|
|
||||||
}
|
|
||||||
storageChannel, _ := upload["storageChannel"].(map[string]any)
|
|
||||||
if stringFromAny(storageChannel["provider"]) != "local_static" {
|
|
||||||
t.Fatalf("expected local static provider metadata, got %+v", upload["storageChannel"])
|
|
||||||
}
|
|
||||||
assetStorage, _ := upload["assetStorage"].(map[string]any)
|
|
||||||
if stringFromAny(assetStorage["strategy"]) != "local_static_upload" || stringFromAny(assetStorage["scene"]) != store.FileStorageSceneUpload {
|
|
||||||
t.Fatalf("unexpected upload asset storage metadata: %+v", assetStorage)
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(storageDir)
|
entries, err := os.ReadDir(storageDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to read uploaded static dir: %v", err)
|
t.Fatalf("failed to read uploaded static dir: %v", err)
|
||||||
}
|
}
|
||||||
if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".pdf") {
|
if len(entries) != 0 {
|
||||||
t.Fatalf("expected one PDF file in uploaded static dir, got %+v", entries)
|
t.Fatalf("unexpected local files: %+v", entries)
|
||||||
}
|
|
||||||
stored, err := os.ReadFile(filepath.Join(storageDir, entries[0].Name()))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read uploaded static file: %v", err)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(stored, payload) {
|
|
||||||
t.Fatalf("stored uploaded payload does not match source payload")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -86,6 +87,19 @@ type Metrics struct {
|
|||||||
candidateDisabledSkipped atomic.Uint64
|
candidateDisabledSkipped atomic.Uint64
|
||||||
candidateAllFullQueued atomic.Uint64
|
candidateAllFullQueued atomic.Uint64
|
||||||
candidateRoutingOther atomic.Uint64
|
candidateRoutingOther atomic.Uint64
|
||||||
|
storageAliyunWrites atomic.Uint64
|
||||||
|
storageAliyunFailures atomic.Uint64
|
||||||
|
storageAliyunWriteNanos atomic.Uint64
|
||||||
|
storageS3Writes atomic.Uint64
|
||||||
|
storageS3Failures atomic.Uint64
|
||||||
|
storageS3WriteNanos atomic.Uint64
|
||||||
|
storageServerMainWrites atomic.Uint64
|
||||||
|
storageServerMainFailures atomic.Uint64
|
||||||
|
storageServerMainWriteNanos atomic.Uint64
|
||||||
|
storageChannelRetries atomic.Uint64
|
||||||
|
storageChannelFailovers atomic.Uint64
|
||||||
|
storageAllChannelsFailed atomic.Uint64
|
||||||
|
storageObjectifiedBytes atomic.Uint64
|
||||||
taskAdmissionWaitBuckets [11]atomic.Uint64
|
taskAdmissionWaitBuckets [11]atomic.Uint64
|
||||||
taskAdmissionWaitMicros atomic.Uint64
|
taskAdmissionWaitMicros atomic.Uint64
|
||||||
}
|
}
|
||||||
@@ -265,6 +279,40 @@ func (m *Metrics) ObserveCandidateRouting(reason string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Metrics) ObserveObjectStorage(event string, provider string, bytes int64, duration time.Duration) {
|
||||||
|
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||||
|
if duration < 0 {
|
||||||
|
duration = 0
|
||||||
|
}
|
||||||
|
switch event {
|
||||||
|
case "write_success", "write_failure":
|
||||||
|
var writes, failures, nanos *atomic.Uint64
|
||||||
|
switch provider {
|
||||||
|
case "aliyun_oss":
|
||||||
|
writes, failures, nanos = &m.storageAliyunWrites, &m.storageAliyunFailures, &m.storageAliyunWriteNanos
|
||||||
|
case "s3":
|
||||||
|
writes, failures, nanos = &m.storageS3Writes, &m.storageS3Failures, &m.storageS3WriteNanos
|
||||||
|
case "server_main_openapi":
|
||||||
|
writes, failures, nanos = &m.storageServerMainWrites, &m.storageServerMainFailures, &m.storageServerMainWriteNanos
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writes.Add(1)
|
||||||
|
nanos.Add(uint64(duration))
|
||||||
|
if event == "write_failure" {
|
||||||
|
failures.Add(1)
|
||||||
|
} else if bytes > 0 {
|
||||||
|
m.storageObjectifiedBytes.Add(uint64(bytes))
|
||||||
|
}
|
||||||
|
case "retry":
|
||||||
|
m.storageChannelRetries.Add(1)
|
||||||
|
case "failover":
|
||||||
|
m.storageChannelFailovers.Add(1)
|
||||||
|
case "all_failed":
|
||||||
|
m.storageAllChannelsFailed.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) {
|
func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) {
|
||||||
if wait < 0 {
|
if wait < 0 {
|
||||||
wait = 0
|
wait = 0
|
||||||
@@ -471,6 +519,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
|||||||
{"all_full_queued", m.candidateAllFullQueued.Load()},
|
{"all_full_queued", m.candidateAllFullQueued.Load()},
|
||||||
{"other", m.candidateRoutingOther.Load()},
|
{"other", m.candidateRoutingOther.Load()},
|
||||||
})
|
})
|
||||||
|
storageProtocolCounters(w, "easyai_gateway_storage_write_attempts_total", "Object storage write attempts by bounded protocol.",
|
||||||
|
m.storageAliyunWrites.Load(), m.storageS3Writes.Load(), m.storageServerMainWrites.Load())
|
||||||
|
storageProtocolCounters(w, "easyai_gateway_storage_write_failures_total", "Object storage write failures by bounded protocol.",
|
||||||
|
m.storageAliyunFailures.Load(), m.storageS3Failures.Load(), m.storageServerMainFailures.Load())
|
||||||
|
storageProtocolDurationCounters(w,
|
||||||
|
m.storageAliyunWriteNanos.Load(), m.storageS3WriteNanos.Load(), m.storageServerMainWriteNanos.Load())
|
||||||
|
plainCounter(w, "easyai_gateway_storage_channel_retries_total", "Retries performed within the current storage channel.", m.storageChannelRetries.Load())
|
||||||
|
plainCounter(w, "easyai_gateway_storage_channel_failovers_total", "Switches to the next configured storage channel.", m.storageChannelFailovers.Load())
|
||||||
|
plainCounter(w, "easyai_gateway_storage_all_channels_failed_total", "Storage writes where every eligible channel failed.", m.storageAllChannelsFailed.Load())
|
||||||
|
plainCounter(w, "easyai_gateway_storage_objectified_bytes_total", "Binary bytes successfully written through storage channels.", m.storageObjectifiedBytes.Load())
|
||||||
|
publicErrorCounters(w, publicerror.MetricSnapshot())
|
||||||
platformModelRateLimitUtilizationGauges(w, modelRateLimits)
|
platformModelRateLimitUtilizationGauges(w, modelRateLimits)
|
||||||
plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections))
|
plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections))
|
||||||
plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections))
|
plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections))
|
||||||
@@ -567,6 +626,29 @@ func plainFloatGauge(w http.ResponseWriter, name, help string, value float64) {
|
|||||||
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value)
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func storageProtocolCounters(w http.ResponseWriter, name, help string, aliyunOSS, s3, serverMain uint64) {
|
||||||
|
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n", name, help, name)
|
||||||
|
fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %d\n", name, aliyunOSS)
|
||||||
|
fmt.Fprintf(w, "%s{protocol=\"s3\"} %d\n", name, s3)
|
||||||
|
fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %d\n", name, serverMain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func storageProtocolDurationCounters(w http.ResponseWriter, aliyunOSS, s3, serverMain uint64) {
|
||||||
|
const name = "easyai_gateway_storage_write_duration_seconds_total"
|
||||||
|
fmt.Fprintf(w, "# HELP %s Total time spent in object storage write attempts by bounded protocol.\n# TYPE %s counter\n", name, name)
|
||||||
|
fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %.6f\n", name, float64(aliyunOSS)/float64(time.Second))
|
||||||
|
fmt.Fprintf(w, "%s{protocol=\"s3\"} %.6f\n", name, float64(s3)/float64(time.Second))
|
||||||
|
fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %.6f\n", name, float64(serverMain)/float64(time.Second))
|
||||||
|
}
|
||||||
|
|
||||||
|
func publicErrorCounters(w http.ResponseWriter, values []publicerror.MetricCount) {
|
||||||
|
const name = "easyai_gateway_public_errors_total"
|
||||||
|
fmt.Fprintf(w, "# HELP %s Public error responses by bounded stable code.\n# TYPE %s counter\n", name, name)
|
||||||
|
for _, value := range values {
|
||||||
|
fmt.Fprintf(w, "%s{code=\"%s\"} %d\n", name, value.Code, value.Count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func platformModelRateLimitUtilizationGauges(w http.ResponseWriter, statuses []store.ModelRateLimitStatus) {
|
func platformModelRateLimitUtilizationGauges(w http.ResponseWriter, statuses []store.ModelRateLimitStatus) {
|
||||||
const name = "easyai_gateway_platform_model_rate_limit_utilization"
|
const name = "easyai_gateway_platform_model_rate_limit_utilization"
|
||||||
fmt.Fprintf(w, "# HELP %s Cluster-wide platform model quota utilization.\n# TYPE %s gauge\n", name, name)
|
fmt.Fprintf(w, "# HELP %s Cluster-wide platform model quota utilization.\n# TYPE %s gauge\n", name, name)
|
||||||
|
|||||||
@@ -52,6 +52,11 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
|||||||
metrics.ObserveCandidateRouting("cooldown_skipped")
|
metrics.ObserveCandidateRouting("cooldown_skipped")
|
||||||
metrics.ObserveCandidateRouting("disabled_skipped")
|
metrics.ObserveCandidateRouting("disabled_skipped")
|
||||||
metrics.ObserveCandidateRouting("all_full_queued")
|
metrics.ObserveCandidateRouting("all_full_queued")
|
||||||
|
metrics.ObserveObjectStorage("write_success", "aliyun_oss", 128, 25*time.Millisecond)
|
||||||
|
metrics.ObserveObjectStorage("write_failure", "s3", 0, 10*time.Millisecond)
|
||||||
|
metrics.ObserveObjectStorage("retry", "s3", 0, 0)
|
||||||
|
metrics.ObserveObjectStorage("failover", "s3", 0, 0)
|
||||||
|
metrics.ObserveObjectStorage("all_failed", "", 0, 0)
|
||||||
metrics.ObserveAsyncWorkerResize("success")
|
metrics.ObserveAsyncWorkerResize("success")
|
||||||
metrics.ObserveConcurrencyLeaseRenewal("success")
|
metrics.ObserveConcurrencyLeaseRenewal("success")
|
||||||
metrics.ObserveConcurrencyLeaseRenewal("lost")
|
metrics.ObserveConcurrencyLeaseRenewal("lost")
|
||||||
@@ -98,6 +103,13 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
|||||||
`easyai_gateway_candidate_routing_total{outcome="cooldown_skipped"} 1`,
|
`easyai_gateway_candidate_routing_total{outcome="cooldown_skipped"} 1`,
|
||||||
`easyai_gateway_candidate_routing_total{outcome="disabled_skipped"} 1`,
|
`easyai_gateway_candidate_routing_total{outcome="disabled_skipped"} 1`,
|
||||||
`easyai_gateway_candidate_routing_total{outcome="all_full_queued"} 1`,
|
`easyai_gateway_candidate_routing_total{outcome="all_full_queued"} 1`,
|
||||||
|
`easyai_gateway_storage_write_attempts_total{protocol="aliyun_oss"} 1`,
|
||||||
|
`easyai_gateway_storage_write_failures_total{protocol="s3"} 1`,
|
||||||
|
`easyai_gateway_storage_write_duration_seconds_total{protocol="aliyun_oss"} 0.025000`,
|
||||||
|
`easyai_gateway_storage_channel_retries_total 1`,
|
||||||
|
`easyai_gateway_storage_channel_failovers_total 1`,
|
||||||
|
`easyai_gateway_storage_all_channels_failed_total 1`,
|
||||||
|
`easyai_gateway_storage_objectified_bytes_total 128`,
|
||||||
`easyai_gateway_platform_model_rate_limit_utilization{platform_model_id="platform-model-1",metric="concurrent"} 0.750000`,
|
`easyai_gateway_platform_model_rate_limit_utilization{platform_model_id="platform-model-1",metric="concurrent"} 0.750000`,
|
||||||
`easyai_gateway_async_worker_resizes_total{outcome="success"} 1`,
|
`easyai_gateway_async_worker_resizes_total{outcome="success"} 1`,
|
||||||
`easyai_gateway_concurrency_lease_renewals_total{outcome="success"} 1`,
|
`easyai_gateway_concurrency_lease_renewals_total{outcome="success"} 1`,
|
||||||
|
|||||||
@@ -39,6 +39,9 @@ type FileStorageChannel struct {
|
|||||||
Provider string `json:"provider"`
|
Provider string `json:"provider"`
|
||||||
UploadURL string `json:"uploadUrl,omitempty"`
|
UploadURL string `json:"uploadUrl,omitempty"`
|
||||||
APIKey string `json:"-"`
|
APIKey string `json:"-"`
|
||||||
|
AccessKeyID string `json:"-"`
|
||||||
|
AccessKeySecret string `json:"-"`
|
||||||
|
SessionToken string `json:"-"`
|
||||||
CredentialsPreview map[string]any `json:"credentialsPreview,omitempty"`
|
CredentialsPreview map[string]any `json:"credentialsPreview,omitempty"`
|
||||||
Scenes []string `json:"scenes,omitempty"`
|
Scenes []string `json:"scenes,omitempty"`
|
||||||
Config map[string]any `json:"config,omitempty"`
|
Config map[string]any `json:"config,omitempty"`
|
||||||
@@ -58,6 +61,11 @@ type FileStorageChannelInput struct {
|
|||||||
Provider string `json:"provider"`
|
Provider string `json:"provider"`
|
||||||
UploadURL string `json:"uploadUrl"`
|
UploadURL string `json:"uploadUrl"`
|
||||||
APIKey *string `json:"apiKey"`
|
APIKey *string `json:"apiKey"`
|
||||||
|
AccessKey *string `json:"accessKey"`
|
||||||
|
AccessKeyID *string `json:"accessKeyId"`
|
||||||
|
AccessKeySecret *string `json:"accessKeySecret"`
|
||||||
|
SecretKey *string `json:"secretKey"`
|
||||||
|
SessionToken *string `json:"sessionToken"`
|
||||||
Scenes []string `json:"scenes"`
|
Scenes []string `json:"scenes"`
|
||||||
Config map[string]any `json:"config"`
|
Config map[string]any `json:"config"`
|
||||||
RetryPolicy map[string]any `json:"retryPolicy"`
|
RetryPolicy map[string]any `json:"retryPolicy"`
|
||||||
@@ -156,11 +164,18 @@ WHERE id = $1::uuid
|
|||||||
AND deleted_at IS NULL`, id))
|
AND deleted_at IS NULL`, id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Store) GetFileStorageChannelByKey(ctx context.Context, channelKey string) (FileStorageChannel, error) {
|
||||||
|
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
|
||||||
|
SELECT `+fileStorageChannelColumns+`
|
||||||
|
FROM file_storage_channels
|
||||||
|
WHERE channel_key = $1`, strings.TrimSpace(channelKey)))
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) CreateFileStorageChannel(ctx context.Context, input FileStorageChannelInput) (FileStorageChannel, error) {
|
func (s *Store) CreateFileStorageChannel(ctx context.Context, input FileStorageChannelInput) (FileStorageChannel, error) {
|
||||||
input = normalizeFileStorageChannelInput(input)
|
input = normalizeFileStorageChannelInput(input)
|
||||||
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
|
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
|
||||||
config, _ := json.Marshal(configFromFileStorageInput(input))
|
config, _ := json.Marshal(configFromFileStorageInput(input))
|
||||||
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
|
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider))
|
||||||
|
|
||||||
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
|
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
|
||||||
INSERT INTO file_storage_channels (
|
INSERT INTO file_storage_channels (
|
||||||
@@ -182,10 +197,10 @@ RETURNING `+fileStorageChannelColumns,
|
|||||||
|
|
||||||
func (s *Store) UpdateFileStorageChannel(ctx context.Context, id string, input FileStorageChannelInput) (FileStorageChannel, error) {
|
func (s *Store) UpdateFileStorageChannel(ctx context.Context, id string, input FileStorageChannelInput) (FileStorageChannel, error) {
|
||||||
input = normalizeFileStorageChannelInput(input)
|
input = normalizeFileStorageChannelInput(input)
|
||||||
replaceCredentials := input.APIKey != nil
|
replaceCredentials := fileStorageInputReplacesCredentials(input)
|
||||||
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
|
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
|
||||||
config, _ := json.Marshal(configFromFileStorageInput(input))
|
config, _ := json.Marshal(configFromFileStorageInput(input))
|
||||||
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
|
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider))
|
||||||
|
|
||||||
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
|
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
|
||||||
UPDATE file_storage_channels
|
UPDATE file_storage_channels
|
||||||
@@ -193,7 +208,7 @@ SET channel_key = $2,
|
|||||||
name = $3,
|
name = $3,
|
||||||
provider = $4,
|
provider = $4,
|
||||||
upload_url = NULLIF($5, ''),
|
upload_url = NULLIF($5, ''),
|
||||||
credentials = CASE WHEN $6::boolean THEN $7 ELSE credentials END,
|
credentials = CASE WHEN $6::boolean THEN credentials || $7::jsonb ELSE credentials END,
|
||||||
config = $8,
|
config = $8,
|
||||||
retry_policy = $9,
|
retry_policy = $9,
|
||||||
priority = $10,
|
priority = $10,
|
||||||
@@ -307,6 +322,9 @@ func scanFileStorageChannel(scanner fileStorageChannelScanner) (FileStorageChann
|
|||||||
}
|
}
|
||||||
credentialObject := decodeObject(credentials)
|
credentialObject := decodeObject(credentials)
|
||||||
item.APIKey = stringFromObject(credentialObject, "apiKey")
|
item.APIKey = stringFromObject(credentialObject, "apiKey")
|
||||||
|
item.AccessKeyID = stringFromObject(credentialObject, "accessKeyId")
|
||||||
|
item.AccessKeySecret = stringFromObject(credentialObject, "accessKeySecret")
|
||||||
|
item.SessionToken = stringFromObject(credentialObject, "sessionToken")
|
||||||
item.CredentialsPreview = maskCredentialsPreview(credentials)
|
item.CredentialsPreview = maskCredentialsPreview(credentials)
|
||||||
configObject := decodeObject(config)
|
configObject := decodeObject(config)
|
||||||
item.Scenes = fileStorageScenesFromConfig(configObject)
|
item.Scenes = fileStorageScenesFromConfig(configObject)
|
||||||
@@ -324,6 +342,17 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage
|
|||||||
apiKey := strings.TrimSpace(*input.APIKey)
|
apiKey := strings.TrimSpace(*input.APIKey)
|
||||||
input.APIKey = &apiKey
|
input.APIKey = &apiKey
|
||||||
}
|
}
|
||||||
|
if input.AccessKeyID == nil && input.AccessKey != nil {
|
||||||
|
input.AccessKeyID = input.AccessKey
|
||||||
|
}
|
||||||
|
if input.AccessKeySecret == nil && input.SecretKey != nil {
|
||||||
|
input.AccessKeySecret = input.SecretKey
|
||||||
|
}
|
||||||
|
for _, value := range []*string{input.AccessKeyID, input.AccessKeySecret, input.SessionToken} {
|
||||||
|
if value != nil {
|
||||||
|
*value = strings.TrimSpace(*value)
|
||||||
|
}
|
||||||
|
}
|
||||||
input.Scenes = normalizeFileStorageScenes(input.Scenes)
|
input.Scenes = normalizeFileStorageScenes(input.Scenes)
|
||||||
input.Status = strings.ToLower(strings.TrimSpace(input.Status))
|
input.Status = strings.ToLower(strings.TrimSpace(input.Status))
|
||||||
if input.Provider == "" {
|
if input.Provider == "" {
|
||||||
@@ -342,11 +371,24 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage
|
|||||||
}
|
}
|
||||||
|
|
||||||
func credentialsFromFileStorageInput(input FileStorageChannelInput) map[string]any {
|
func credentialsFromFileStorageInput(input FileStorageChannelInput) map[string]any {
|
||||||
apiKey := fileStorageInputAPIKey(input)
|
credentials := map[string]any{}
|
||||||
if apiKey == "" {
|
if input.APIKey != nil {
|
||||||
return map[string]any{}
|
credentials["apiKey"] = fileStorageInputAPIKey(input)
|
||||||
}
|
}
|
||||||
return map[string]any{"apiKey": apiKey}
|
if input.AccessKeyID != nil {
|
||||||
|
credentials["accessKeyId"] = strings.TrimSpace(*input.AccessKeyID)
|
||||||
|
}
|
||||||
|
if input.AccessKeySecret != nil {
|
||||||
|
credentials["accessKeySecret"] = strings.TrimSpace(*input.AccessKeySecret)
|
||||||
|
}
|
||||||
|
if input.SessionToken != nil {
|
||||||
|
credentials["sessionToken"] = strings.TrimSpace(*input.SessionToken)
|
||||||
|
}
|
||||||
|
return credentials
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileStorageInputReplacesCredentials(input FileStorageChannelInput) bool {
|
||||||
|
return input.APIKey != nil || input.AccessKeyID != nil || input.AccessKeySecret != nil || input.AccessKey != nil || input.SecretKey != nil || input.SessionToken != nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fileStorageInputAPIKey(input FileStorageChannelInput) string {
|
func fileStorageInputAPIKey(input FileStorageChannelInput) string {
|
||||||
@@ -549,7 +591,10 @@ func NormalizeFileStorageResultUploadPolicy(policy string) string {
|
|||||||
case "upload_all", "all", "always", "all_upload":
|
case "upload_all", "all", "always", "all_upload":
|
||||||
return FileStorageResultUploadPolicyUploadAll
|
return FileStorageResultUploadPolicyUploadAll
|
||||||
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
|
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
|
||||||
return FileStorageResultUploadPolicyUploadNone
|
// Local result persistence is no longer a supported write path. Preserve
|
||||||
|
// compatibility with historical settings by normalizing them to the safe
|
||||||
|
// object-storage policy.
|
||||||
|
return FileStorageResultUploadPolicyDefault
|
||||||
default:
|
default:
|
||||||
return FileStorageResultUploadPolicyDefault
|
return FileStorageResultUploadPolicyDefault
|
||||||
}
|
}
|
||||||
@@ -601,10 +646,22 @@ func defaultFileStorageScenes() []string {
|
|||||||
return []string{FileStorageSceneUpload, FileStorageSceneImageResult, FileStorageSceneRequestAsset}
|
return []string{FileStorageSceneUpload, FileStorageSceneImageResult, FileStorageSceneRequestAsset}
|
||||||
}
|
}
|
||||||
|
|
||||||
func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any) map[string]any {
|
func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any, providers ...string) map[string]any {
|
||||||
if len(policy) > 0 {
|
if len(policy) > 0 {
|
||||||
return policy
|
return policy
|
||||||
}
|
}
|
||||||
|
provider := ""
|
||||||
|
if len(providers) > 0 {
|
||||||
|
provider = strings.ToLower(strings.TrimSpace(providers[0]))
|
||||||
|
}
|
||||||
|
if provider == "aliyun_oss" || provider == "s3" {
|
||||||
|
return map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"maxRetries": 2,
|
||||||
|
"backoffSeconds": []any{0.25, 1.0},
|
||||||
|
"strategy": "exponential",
|
||||||
|
}
|
||||||
|
}
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"maxRetries": 3,
|
"maxRetries": 3,
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
package store
|
package store
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,3 +17,32 @@ func TestDefaultFileStorageScenesIncludeCrossNodeRequestAssets(t *testing.T) {
|
|||||||
t.Fatalf("default file storage scenes = %#v, want %#v", got, want)
|
t.Fatalf("default file storage scenes = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFileStorageChannelJSONNeverIncludesCredentials(t *testing.T) {
|
||||||
|
channel := FileStorageChannel{
|
||||||
|
Provider: "s3", APIKey: "api-key-secret", AccessKeyID: "access-key-id",
|
||||||
|
AccessKeySecret: "access-key-secret", SessionToken: "session-token",
|
||||||
|
CredentialsPreview: map[string]any{"accessKeyId": "acce…-id", "accessKeySecret": "acce…cret"},
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(channel)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
encoded := string(payload)
|
||||||
|
for _, secret := range []string{"api-key-secret", "access-key-id", "access-key-secret", "session-token"} {
|
||||||
|
if strings.Contains(encoded, secret) {
|
||||||
|
t.Fatalf("channel JSON leaked credential %q: %s", secret, encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(encoded, "credentialsPreview") {
|
||||||
|
t.Fatalf("channel JSON omitted masked preview: %s", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLegacyUploadNoneNormalizesToObjectStorageDefault(t *testing.T) {
|
||||||
|
for _, value := range []string{"upload_none", "none", "never", "skip_all"} {
|
||||||
|
if got := NormalizeFileStorageResultUploadPolicy(value); got != FileStorageResultUploadPolicyDefault {
|
||||||
|
t.Fatalf("policy %q normalized to %q", value, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
"github.com/jackc/pgx/v5/pgconn"
|
"github.com/jackc/pgx/v5/pgconn"
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
@@ -626,6 +627,7 @@ type GatewayTask struct {
|
|||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
ErrorCode string `json:"errorCode,omitempty"`
|
ErrorCode string `json:"errorCode,omitempty"`
|
||||||
ErrorMessage string `json:"errorMessage,omitempty"`
|
ErrorMessage string `json:"errorMessage,omitempty"`
|
||||||
|
PublicError *publicerror.Error `json:"publicError,omitempty"`
|
||||||
CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"`
|
CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"`
|
||||||
CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"`
|
CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"`
|
||||||
CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"`
|
CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"`
|
||||||
@@ -656,7 +658,7 @@ COALESCE(reservation_amount, 0)::float8, COALESCE(execution_token::text, ''),
|
|||||||
COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''),
|
COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''),
|
||||||
COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''),
|
COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''),
|
||||||
COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''),
|
COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''),
|
||||||
COALESCE(error_code, ''), COALESCE(error_message, ''),
|
COALESCE(error_code, ''), COALESCE(error_message, ''), COALESCE(public_error, '{}'::jsonb),
|
||||||
COALESCE(compatibility_protocol, ''), COALESCE(compatibility_public_id, ''),
|
COALESCE(compatibility_protocol, ''), COALESCE(compatibility_public_id, ''),
|
||||||
COALESCE(compatibility_source_protocol, ''), COALESCE(compatibility_submit_http_status, 0),
|
COALESCE(compatibility_source_protocol, ''), COALESCE(compatibility_submit_http_status, 0),
|
||||||
COALESCE(compatibility_submit_headers, '{}'::jsonb), COALESCE(compatibility_submit_body, '{}'::jsonb),
|
COALESCE(compatibility_submit_headers, '{}'::jsonb), COALESCE(compatibility_submit_body, '{}'::jsonb),
|
||||||
@@ -710,6 +712,7 @@ type TaskAttempt struct {
|
|||||||
UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
|
UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
|
||||||
ErrorCode string `json:"errorCode,omitempty"`
|
ErrorCode string `json:"errorCode,omitempty"`
|
||||||
ErrorMessage string `json:"errorMessage,omitempty"`
|
ErrorMessage string `json:"errorMessage,omitempty"`
|
||||||
|
PublicError *publicerror.Error `json:"publicError,omitempty"`
|
||||||
StartedAt time.Time `json:"startedAt"`
|
StartedAt time.Time `json:"startedAt"`
|
||||||
FinishedAt string `json:"finishedAt,omitempty"`
|
FinishedAt string `json:"finishedAt,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -2240,6 +2243,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
|
|||||||
var remoteTaskPayloadBytes []byte
|
var remoteTaskPayloadBytes []byte
|
||||||
var compatibilitySubmitHeadersBytes []byte
|
var compatibilitySubmitHeadersBytes []byte
|
||||||
var compatibilitySubmitBodyBytes []byte
|
var compatibilitySubmitBodyBytes []byte
|
||||||
|
var publicErrorBytes []byte
|
||||||
if err := scanner.Scan(
|
if err := scanner.Scan(
|
||||||
&task.ID,
|
&task.ID,
|
||||||
&task.ExternalTaskID,
|
&task.ExternalTaskID,
|
||||||
@@ -2295,6 +2299,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
|
|||||||
&task.Error,
|
&task.Error,
|
||||||
&task.ErrorCode,
|
&task.ErrorCode,
|
||||||
&task.ErrorMessage,
|
&task.ErrorMessage,
|
||||||
|
&publicErrorBytes,
|
||||||
&task.CompatibilityProtocol,
|
&task.CompatibilityProtocol,
|
||||||
&task.CompatibilityPublicID,
|
&task.CompatibilityPublicID,
|
||||||
&task.CompatibilitySourceProtocol,
|
&task.CompatibilitySourceProtocol,
|
||||||
@@ -2317,9 +2322,25 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
|
|||||||
task.PricingSnapshot = decodeObject(pricingSnapshotBytes)
|
task.PricingSnapshot = decodeObject(pricingSnapshotBytes)
|
||||||
task.CompatibilitySubmitHeaders = decodeObject(compatibilitySubmitHeadersBytes)
|
task.CompatibilitySubmitHeaders = decodeObject(compatibilitySubmitHeadersBytes)
|
||||||
task.CompatibilitySubmitBody = decodeObject(compatibilitySubmitBodyBytes)
|
task.CompatibilitySubmitBody = decodeObject(compatibilitySubmitBodyBytes)
|
||||||
|
if len(publicErrorBytes) > 0 {
|
||||||
|
var snapshot publicerror.Error
|
||||||
|
if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" {
|
||||||
|
task.PublicError = &snapshot
|
||||||
|
}
|
||||||
|
}
|
||||||
return task, nil
|
return task, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func encodePublicErrorSnapshot(code string, message string, status int, retryable bool, requestID string, taskID string) []byte {
|
||||||
|
if strings.TrimSpace(code) == "" && strings.TrimSpace(message) == "" {
|
||||||
|
return []byte("null")
|
||||||
|
}
|
||||||
|
snapshot := publicerror.FromFields(code, message, status, retryable)
|
||||||
|
snapshot = publicerror.WithIDs(snapshot, requestID, taskID)
|
||||||
|
payload, _ := json.Marshal(snapshot)
|
||||||
|
return payload
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) ListTaskEvents(ctx context.Context, taskID string) ([]TaskEvent, error) {
|
func (s *Store) ListTaskEvents(ctx context.Context, taskID string) ([]TaskEvent, error) {
|
||||||
rows, err := s.pool.Query(ctx, `
|
rows, err := s.pool.Query(ctx, `
|
||||||
SELECT id::text, task_id::text, seq, event_type, COALESCE(status, ''), COALESCE(phase, ''),
|
SELECT id::text, task_id::text, seq, event_type, COALESCE(status, ''), COALESCE(phase, ''),
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ type RequestAsset struct {
|
|||||||
ByteSize int64 `json:"byteSize"`
|
ByteSize int64 `json:"byteSize"`
|
||||||
URL string `json:"url"`
|
URL string `json:"url"`
|
||||||
StorageProvider string `json:"storageProvider"`
|
StorageProvider string `json:"storageProvider"`
|
||||||
|
StorageChannelID string `json:"storageChannelId,omitempty"`
|
||||||
|
StorageChannelKey string `json:"storageChannelKey,omitempty"`
|
||||||
|
ObjectKey string `json:"objectKey,omitempty"`
|
||||||
|
AccessScope string `json:"accessScope,omitempty"`
|
||||||
LocalPath string `json:"localPath,omitempty"`
|
LocalPath string `json:"localPath,omitempty"`
|
||||||
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
|
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
|
||||||
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
|
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
|
||||||
@@ -29,6 +33,10 @@ type RequestAssetInput struct {
|
|||||||
ByteSize int64
|
ByteSize int64
|
||||||
URL string
|
URL string
|
||||||
StorageProvider string
|
StorageProvider string
|
||||||
|
StorageChannelID string
|
||||||
|
StorageChannelKey string
|
||||||
|
ObjectKey string
|
||||||
|
AccessScope string
|
||||||
LocalPath string
|
LocalPath string
|
||||||
ExpiresAt *time.Time
|
ExpiresAt *time.Time
|
||||||
}
|
}
|
||||||
@@ -36,6 +44,8 @@ type RequestAssetInput struct {
|
|||||||
func (s *Store) FindRequestAsset(ctx context.Context, sha256 string, contentType string) (RequestAsset, bool, error) {
|
func (s *Store) FindRequestAsset(ctx context.Context, sha256 string, contentType string) (RequestAsset, bool, error) {
|
||||||
asset, err := scanRequestAsset(s.pool.QueryRow(ctx, `
|
asset, err := scanRequestAsset(s.pool.QueryRow(ctx, `
|
||||||
SELECT id::text, sha256, content_type, byte_size, url, storage_provider,
|
SELECT id::text, sha256, content_type, byte_size, url, storage_provider,
|
||||||
|
COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''),
|
||||||
|
COALESCE(object_key, ''), COALESCE(access_scope, 'private'),
|
||||||
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at
|
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at
|
||||||
FROM gateway_request_assets
|
FROM gateway_request_assets
|
||||||
WHERE sha256 = $1 AND content_type = $2`, sha256, contentType))
|
WHERE sha256 = $1 AND content_type = $2`, sha256, contentType))
|
||||||
@@ -51,25 +61,37 @@ WHERE sha256 = $1 AND content_type = $2`, sha256, contentType))
|
|||||||
func (s *Store) UpsertRequestAsset(ctx context.Context, input RequestAssetInput) (RequestAsset, error) {
|
func (s *Store) UpsertRequestAsset(ctx context.Context, input RequestAssetInput) (RequestAsset, error) {
|
||||||
return scanRequestAsset(s.pool.QueryRow(ctx, `
|
return scanRequestAsset(s.pool.QueryRow(ctx, `
|
||||||
INSERT INTO gateway_request_assets (
|
INSERT INTO gateway_request_assets (
|
||||||
sha256, content_type, byte_size, url, storage_provider, local_path, expires_at, expired_at, ref_count
|
sha256, content_type, byte_size, url, storage_provider, storage_channel_id,
|
||||||
|
storage_channel_key, object_key, access_scope, local_path, expires_at, expired_at, ref_count
|
||||||
)
|
)
|
||||||
VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), $7, NULL, 1)
|
VALUES ($1, $2, $3, $4, $5, NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''),
|
||||||
|
COALESCE(NULLIF($9, ''), 'private'), NULLIF($10, ''), $11, NULL, 1)
|
||||||
ON CONFLICT (sha256, content_type) DO UPDATE
|
ON CONFLICT (sha256, content_type) DO UPDATE
|
||||||
SET byte_size = EXCLUDED.byte_size,
|
SET byte_size = EXCLUDED.byte_size,
|
||||||
url = EXCLUDED.url,
|
url = EXCLUDED.url,
|
||||||
storage_provider = EXCLUDED.storage_provider,
|
storage_provider = EXCLUDED.storage_provider,
|
||||||
|
storage_channel_id = EXCLUDED.storage_channel_id,
|
||||||
|
storage_channel_key = EXCLUDED.storage_channel_key,
|
||||||
|
object_key = EXCLUDED.object_key,
|
||||||
|
access_scope = EXCLUDED.access_scope,
|
||||||
local_path = EXCLUDED.local_path,
|
local_path = EXCLUDED.local_path,
|
||||||
expires_at = EXCLUDED.expires_at,
|
expires_at = EXCLUDED.expires_at,
|
||||||
expired_at = NULL,
|
expired_at = NULL,
|
||||||
ref_count = gateway_request_assets.ref_count + 1,
|
ref_count = gateway_request_assets.ref_count + 1,
|
||||||
updated_at = now()
|
updated_at = now()
|
||||||
RETURNING id::text, sha256, content_type, byte_size, url, storage_provider,
|
RETURNING id::text, sha256, content_type, byte_size, url, storage_provider,
|
||||||
|
COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''),
|
||||||
|
COALESCE(object_key, ''), COALESCE(access_scope, 'private'),
|
||||||
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at`,
|
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at`,
|
||||||
input.SHA256,
|
input.SHA256,
|
||||||
input.ContentType,
|
input.ContentType,
|
||||||
input.ByteSize,
|
input.ByteSize,
|
||||||
input.URL,
|
input.URL,
|
||||||
input.StorageProvider,
|
input.StorageProvider,
|
||||||
|
input.StorageChannelID,
|
||||||
|
input.StorageChannelKey,
|
||||||
|
input.ObjectKey,
|
||||||
|
input.AccessScope,
|
||||||
input.LocalPath,
|
input.LocalPath,
|
||||||
input.ExpiresAt,
|
input.ExpiresAt,
|
||||||
))
|
))
|
||||||
@@ -110,6 +132,10 @@ func scanRequestAsset(scanner interface{ Scan(dest ...any) error }) (RequestAsse
|
|||||||
&asset.ByteSize,
|
&asset.ByteSize,
|
||||||
&asset.URL,
|
&asset.URL,
|
||||||
&asset.StorageProvider,
|
&asset.StorageProvider,
|
||||||
|
&asset.StorageChannelID,
|
||||||
|
&asset.StorageChannelKey,
|
||||||
|
&asset.ObjectKey,
|
||||||
|
&asset.AccessScope,
|
||||||
&localPath,
|
&localPath,
|
||||||
&expiresAt,
|
&expiresAt,
|
||||||
&expiredAt,
|
&expiredAt,
|
||||||
|
|||||||
@@ -23,6 +23,9 @@ type TaskCallbackDelivery struct {
|
|||||||
LockToken string
|
LockToken string
|
||||||
EventType string
|
EventType string
|
||||||
TaskStatus string
|
TaskStatus string
|
||||||
|
TaskRequestID string
|
||||||
|
TaskErrorCode string
|
||||||
|
TaskErrorMessage string
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,9 +70,13 @@ func (s *Store) ClaimTaskCallbacks(ctx context.Context, workerID string, limit i
|
|||||||
}
|
}
|
||||||
rows, err := s.pool.Query(ctx, `
|
rows, err := s.pool.Query(ctx, `
|
||||||
WITH picked AS (
|
WITH picked AS (
|
||||||
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at
|
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at,
|
||||||
|
COALESCE(task.request_id, '') AS task_request_id,
|
||||||
|
COALESCE(task.error_code, '') AS task_error_code,
|
||||||
|
COALESCE(task.error_message, task.error, '') AS task_error_message
|
||||||
FROM gateway_task_callback_outbox outbox
|
FROM gateway_task_callback_outbox outbox
|
||||||
JOIN gateway_task_events event ON event.id = outbox.event_id
|
JOIN gateway_task_events event ON event.id = outbox.event_id
|
||||||
|
JOIN gateway_tasks task ON task.id = outbox.task_id
|
||||||
WHERE outbox.created_at >= COALESCE((
|
WHERE outbox.created_at >= COALESCE((
|
||||||
SELECT applied_at
|
SELECT applied_at
|
||||||
FROM schema_migrations
|
FROM schema_migrations
|
||||||
@@ -101,7 +108,7 @@ WHERE outbox.id = picked.id
|
|||||||
RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, ''),
|
RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, ''),
|
||||||
outbox.seq, outbox.callback_url, outbox.status, outbox.attempts,
|
outbox.seq, outbox.callback_url, outbox.status, outbox.attempts,
|
||||||
COALESCE(outbox.lock_token::text, ''), picked.event_type, picked.task_status,
|
COALESCE(outbox.lock_token::text, ''), picked.event_type, picked.task_status,
|
||||||
picked.created_at`,
|
picked.created_at, picked.task_request_id, picked.task_error_code, picked.task_error_message`,
|
||||||
workerID, limit, int(staleAfter/time.Second))
|
workerID, limit, int(staleAfter/time.Second))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -122,6 +129,9 @@ RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text,
|
|||||||
&item.EventType,
|
&item.EventType,
|
||||||
&item.TaskStatus,
|
&item.TaskStatus,
|
||||||
&item.CreatedAt,
|
&item.CreatedAt,
|
||||||
|
&item.TaskRequestID,
|
||||||
|
&item.TaskErrorCode,
|
||||||
|
&item.TaskErrorMessage,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -544,12 +545,14 @@ WHERE id = $1::uuid
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) FailQueuedTask(ctx context.Context, taskID string, code string, message string) (GatewayTask, error) {
|
func (s *Store) FailQueuedTask(ctx context.Context, taskID string, code string, message string) (GatewayTask, error) {
|
||||||
|
publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
|
||||||
tag, err := s.pool.Exec(ctx, `
|
tag, err := s.pool.Exec(ctx, `
|
||||||
UPDATE gateway_tasks
|
UPDATE gateway_tasks
|
||||||
SET status = 'failed',
|
SET status = 'failed',
|
||||||
error = NULL,
|
error = NULL,
|
||||||
error_code = NULLIF($2, ''),
|
error_code = NULLIF($2, ''),
|
||||||
error_message = NULLIF($3, ''),
|
error_message = NULLIF($3, ''),
|
||||||
|
public_error = $4::jsonb,
|
||||||
billing_status = CASE
|
billing_status = CASE
|
||||||
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
|
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
|
||||||
ELSE 'released'
|
ELSE 'released'
|
||||||
@@ -563,7 +566,7 @@ SET status = 'failed',
|
|||||||
finished_at = now(),
|
finished_at = now(),
|
||||||
updated_at = now()
|
updated_at = now()
|
||||||
WHERE id = $1::uuid
|
WHERE id = $1::uuid
|
||||||
AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048))
|
AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048), string(publicErrorJSON))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return GatewayTask{}, err
|
return GatewayTask{}, err
|
||||||
}
|
}
|
||||||
@@ -686,6 +689,7 @@ SET status = 'queued',
|
|||||||
error = NULL,
|
error = NULL,
|
||||||
error_code = NULL,
|
error_code = NULL,
|
||||||
error_message = NULL,
|
error_message = NULL,
|
||||||
|
public_error = NULL,
|
||||||
updated_at = now()
|
updated_at = now()
|
||||||
WHERE id = $1::uuid
|
WHERE id = $1::uuid
|
||||||
AND status IN ('queued', 'running')
|
AND status IN ('queued', 'running')
|
||||||
@@ -722,6 +726,7 @@ SET status = 'queued',
|
|||||||
error = NULL,
|
error = NULL,
|
||||||
error_code = NULL,
|
error_code = NULL,
|
||||||
error_message = NULL,
|
error_message = NULL,
|
||||||
|
public_error = NULL,
|
||||||
updated_at = now()
|
updated_at = now()
|
||||||
WHERE task.id = $1::uuid
|
WHERE task.id = $1::uuid
|
||||||
AND task.status IN ('queued', 'running')
|
AND task.status IN ('queued', 'running')
|
||||||
@@ -1371,6 +1376,7 @@ SELECT a.id::text, a.task_id::text, a.attempt_no,
|
|||||||
a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb),
|
a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb),
|
||||||
COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''),
|
COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''),
|
||||||
COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''),
|
COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''),
|
||||||
|
COALESCE(a.public_error, '{}'::jsonb),
|
||||||
COALESCE(a.pricing_snapshot, '{}'::jsonb), COALESCE(a.request_fingerprint, ''),
|
COALESCE(a.pricing_snapshot, '{}'::jsonb), COALESCE(a.request_fingerprint, ''),
|
||||||
a.upstream_submission_status, COALESCE(a.upstream_submission_updated_at::text, ''),
|
a.upstream_submission_status, COALESCE(a.upstream_submission_updated_at::text, ''),
|
||||||
a.started_at, COALESCE(a.finished_at::text, '')
|
a.started_at, COALESCE(a.finished_at::text, '')
|
||||||
@@ -1403,6 +1409,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
|
|||||||
var requestBytes []byte
|
var requestBytes []byte
|
||||||
var responseBytes []byte
|
var responseBytes []byte
|
||||||
var pricingSnapshotBytes []byte
|
var pricingSnapshotBytes []byte
|
||||||
|
var publicErrorBytes []byte
|
||||||
if err := scanner.Scan(
|
if err := scanner.Scan(
|
||||||
&item.ID,
|
&item.ID,
|
||||||
&item.TaskID,
|
&item.TaskID,
|
||||||
@@ -1430,6 +1437,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
|
|||||||
&item.ResponseDurationMS,
|
&item.ResponseDurationMS,
|
||||||
&item.ErrorCode,
|
&item.ErrorCode,
|
||||||
&item.ErrorMessage,
|
&item.ErrorMessage,
|
||||||
|
&publicErrorBytes,
|
||||||
&pricingSnapshotBytes,
|
&pricingSnapshotBytes,
|
||||||
&item.RequestFingerprint,
|
&item.RequestFingerprint,
|
||||||
&item.UpstreamSubmissionStatus,
|
&item.UpstreamSubmissionStatus,
|
||||||
@@ -1444,6 +1452,12 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
|
|||||||
item.RequestSnapshot = decodeObject(requestBytes)
|
item.RequestSnapshot = decodeObject(requestBytes)
|
||||||
item.ResponseSnapshot = decodeObject(responseBytes)
|
item.ResponseSnapshot = decodeObject(responseBytes)
|
||||||
item.PricingSnapshot = decodeObject(pricingSnapshotBytes)
|
item.PricingSnapshot = decodeObject(pricingSnapshotBytes)
|
||||||
|
if len(publicErrorBytes) > 0 {
|
||||||
|
var snapshot publicerror.Error
|
||||||
|
if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" {
|
||||||
|
item.PublicError = &snapshot
|
||||||
|
}
|
||||||
|
}
|
||||||
enrichTaskAttemptFromMetrics(&item)
|
enrichTaskAttemptFromMetrics(&item)
|
||||||
return item, nil
|
return item, nil
|
||||||
}
|
}
|
||||||
@@ -1534,6 +1548,7 @@ func (s *Store) FinishTaskAttempt(ctx context.Context, input FinishTaskAttemptIn
|
|||||||
}
|
}
|
||||||
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptUsage(input.Usage)))
|
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptUsage(input.Usage)))
|
||||||
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptMetrics(input.Metrics)))
|
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptMetrics(input.Metrics)))
|
||||||
|
publicErrorJSON := encodePublicErrorSnapshot(input.ErrorCode, input.ErrorMessage, statusCode, input.Retryable, input.RequestID, "")
|
||||||
_, err := s.pool.Exec(ctx, `
|
_, err := s.pool.Exec(ctx, `
|
||||||
UPDATE gateway_task_attempts
|
UPDATE gateway_task_attempts
|
||||||
SET status = $2::text,
|
SET status = $2::text,
|
||||||
@@ -1550,6 +1565,7 @@ SET status = $2::text,
|
|||||||
response_duration_ms = $8,
|
response_duration_ms = $8,
|
||||||
error_code = NULLIF($9::text, ''),
|
error_code = NULLIF($9::text, ''),
|
||||||
error_message = NULLIF(left($10::text, 2048), ''),
|
error_message = NULLIF(left($10::text, 2048), ''),
|
||||||
|
public_error = CASE WHEN $2::text = 'failed' THEN $14::jsonb ELSE NULL END,
|
||||||
upstream_submission_status = COALESCE(NULLIF($13::text, ''), upstream_submission_status),
|
upstream_submission_status = COALESCE(NULLIF($13::text, ''), upstream_submission_status),
|
||||||
upstream_submission_updated_at = CASE
|
upstream_submission_updated_at = CASE
|
||||||
WHEN NULLIF($13::text, '') IS NULL THEN upstream_submission_updated_at
|
WHEN NULLIF($13::text, '') IS NULL THEN upstream_submission_updated_at
|
||||||
@@ -1570,6 +1586,7 @@ WHERE id = $1::uuid`,
|
|||||||
usageJSON,
|
usageJSON,
|
||||||
metricsJSON,
|
metricsJSON,
|
||||||
input.UpstreamSubmissionStatus,
|
input.UpstreamSubmissionStatus,
|
||||||
|
string(publicErrorJSON),
|
||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1651,6 +1668,7 @@ SET status = 'succeeded',
|
|||||||
response_duration_ms = $5,
|
response_duration_ms = $5,
|
||||||
error_code = NULL,
|
error_code = NULL,
|
||||||
error_message = NULL,
|
error_message = NULL,
|
||||||
|
public_error = NULL,
|
||||||
finished_at = now()
|
finished_at = now()
|
||||||
WHERE id = $1::uuid`,
|
WHERE id = $1::uuid`,
|
||||||
input.AttemptID, input.RequestID,
|
input.AttemptID, input.RequestID,
|
||||||
@@ -1687,6 +1705,7 @@ SET status = 'succeeded',
|
|||||||
error = NULL,
|
error = NULL,
|
||||||
error_code = NULL,
|
error_code = NULL,
|
||||||
error_message = NULL,
|
error_message = NULL,
|
||||||
|
public_error = NULL,
|
||||||
locked_by = NULL,
|
locked_by = NULL,
|
||||||
locked_at = NULL,
|
locked_at = NULL,
|
||||||
heartbeat_at = NULL,
|
heartbeat_at = NULL,
|
||||||
@@ -2049,6 +2068,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
|
|||||||
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
|
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
|
||||||
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
|
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
|
||||||
message := truncateUTF8Bytes(input.Message, 2048)
|
message := truncateUTF8Bytes(input.Message, 2048)
|
||||||
|
publicErrorJSON := encodePublicErrorSnapshot(input.Code, message, 0, true, input.RequestID, input.TaskID)
|
||||||
finalizedAdmission := false
|
finalizedAdmission := false
|
||||||
err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
|
err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
|
||||||
tag, err := tx.Exec(ctx, `
|
tag, err := tx.Exec(ctx, `
|
||||||
@@ -2057,6 +2077,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
|
|||||||
error = NULL,
|
error = NULL,
|
||||||
error_code = NULLIF($3::text, ''),
|
error_code = NULLIF($3::text, ''),
|
||||||
error_message = NULLIF($2::text, ''),
|
error_message = NULLIF($2::text, ''),
|
||||||
|
public_error = $11::jsonb,
|
||||||
request_id = NULLIF($4::text, ''),
|
request_id = NULLIF($4::text, ''),
|
||||||
metrics = $5::jsonb,
|
metrics = $5::jsonb,
|
||||||
response_started_at = $6::timestamptz,
|
response_started_at = $6::timestamptz,
|
||||||
@@ -2090,6 +2111,7 @@ WHERE id = $1::uuid
|
|||||||
input.ResponseDurationMS,
|
input.ResponseDurationMS,
|
||||||
string(resultJSON),
|
string(resultJSON),
|
||||||
input.ExecutionToken,
|
input.ExecutionToken,
|
||||||
|
string(publicErrorJSON),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
ALTER TABLE gateway_request_assets
|
||||||
|
ADD COLUMN IF NOT EXISTS storage_channel_id uuid REFERENCES file_storage_channels(id) ON DELETE SET NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS storage_channel_key text,
|
||||||
|
ADD COLUMN IF NOT EXISTS object_key text,
|
||||||
|
ADD COLUMN IF NOT EXISTS access_scope text DEFAULT 'private';
|
||||||
|
|
||||||
|
ALTER TABLE gateway_tasks
|
||||||
|
ADD COLUMN IF NOT EXISTS public_error jsonb;
|
||||||
|
|
||||||
|
ALTER TABLE gateway_task_attempts
|
||||||
|
ADD COLUMN IF NOT EXISTS public_error jsonb;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_gateway_request_assets_object
|
||||||
|
ON gateway_request_assets(storage_channel_key, object_key)
|
||||||
|
WHERE object_key IS NOT NULL;
|
||||||
|
|
||||||
|
UPDATE system_settings
|
||||||
|
SET value = jsonb_set(COALESCE(value, '{}'::jsonb), '{resultUploadPolicy}', '"default"'::jsonb, true),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE setting_key = 'file_storage'
|
||||||
|
AND COALESCE(value->>'resultUploadPolicy', '') IN ('upload_none', 'none', 'never', 'disabled', 'no_upload', 'skip', 'skip_all');
|
||||||
@@ -24,7 +24,7 @@
|
|||||||
"outputs": ["{projectRoot}/docs/swagger.json", "{projectRoot}/docs/swagger.yaml"],
|
"outputs": ["{projectRoot}/docs/swagger.json", "{projectRoot}/docs/swagger.yaml"],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "apps/api",
|
"cwd": "apps/api",
|
||||||
"command": "go run github.com/swaggo/swag/cmd/swag@v1.16.4 init --parseInternal -d ./cmd/gateway,./internal/httpapi,./internal/store,./internal/auth,./internal/identity,./internal/runner -g main.go -o docs --outputTypes json,yaml"
|
"command": "go run github.com/swaggo/swag/cmd/swag@v1.16.4 init --parseInternal -d ./cmd/gateway,./internal/httpapi,./internal/store,./internal/auth,./internal/identity,./internal/runner,./internal/publicerror -g main.go -o docs --outputTypes json,yaml"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"test": {
|
"test": {
|
||||||
|
|||||||
@@ -111,6 +111,7 @@ import {
|
|||||||
updateClientCustomizationSettings,
|
updateClientCustomizationSettings,
|
||||||
updateFileStorageChannel,
|
updateFileStorageChannel,
|
||||||
updateFileStorageSettings,
|
updateFileStorageSettings,
|
||||||
|
testFileStorageChannel,
|
||||||
updateGatewayUser,
|
updateGatewayUser,
|
||||||
updatePlatform,
|
updatePlatform,
|
||||||
updatePlatformDynamicPriority,
|
updatePlatformDynamicPriority,
|
||||||
@@ -1182,6 +1183,22 @@ export function App() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function verifyFileStorageChannel(channelId: string) {
|
||||||
|
setCoreState('loading');
|
||||||
|
setCoreMessage('');
|
||||||
|
try {
|
||||||
|
const result = await testFileStorageChannel(token, channelId);
|
||||||
|
const refreshed = await listFileStorageChannels(token);
|
||||||
|
setFileStorageChannels(refreshed.items);
|
||||||
|
setCoreState('ready');
|
||||||
|
setCoreMessage(`对象存储连接测试通过，Put / Head / Delete 均成功，耗时 ${result.durationMs}ms。`);
|
||||||
|
} catch (err) {
|
||||||
|
setCoreState('error');
|
||||||
|
setCoreMessage(err instanceof Error ? err.message : '对象存储连接测试失败');
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function saveClientCustomizationSettings(input: ClientCustomizationSettingsUpdateRequest) {
|
async function saveClientCustomizationSettings(input: ClientCustomizationSettingsUpdateRequest) {
|
||||||
setCoreState('loading');
|
setCoreState('loading');
|
||||||
setCoreMessage('');
|
setCoreMessage('');
|
||||||
@@ -1614,6 +1631,7 @@ export function App() {
|
|||||||
onSaveAccessRule={saveAccessRule}
|
onSaveAccessRule={saveAccessRule}
|
||||||
onSaveFileStorageChannel={saveFileStorageChannel}
|
onSaveFileStorageChannel={saveFileStorageChannel}
|
||||||
onSaveFileStorageSettings={saveFileStorageSettings}
|
onSaveFileStorageSettings={saveFileStorageSettings}
|
||||||
|
onTestFileStorageChannel={verifyFileStorageChannel}
|
||||||
onSaveClientCustomizationSettings={saveClientCustomizationSettings}
|
onSaveClientCustomizationSettings={saveClientCustomizationSettings}
|
||||||
onConnectSecurityEvents={connectSecurityEvents}
|
onConnectSecurityEvents={connectSecurityEvents}
|
||||||
onDisconnectSecurityEvents={disconnectSecurityEvents}
|
onDisconnectSecurityEvents={disconnectSecurityEvents}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import type {
|
|||||||
ClientCustomizationSettingsUpdateRequest,
|
ClientCustomizationSettingsUpdateRequest,
|
||||||
CreatedGatewayApiKey,
|
CreatedGatewayApiKey,
|
||||||
FileStorageChannel,
|
FileStorageChannel,
|
||||||
|
FileStorageChannelTestResult,
|
||||||
FileStorageSettings,
|
FileStorageSettings,
|
||||||
FileStorageSettingsUpdateRequest,
|
FileStorageSettingsUpdateRequest,
|
||||||
FileStorageChannelUpsertRequest,
|
FileStorageChannelUpsertRequest,
|
||||||
@@ -1276,6 +1277,16 @@ export async function deleteFileStorageChannel(token: string, channelId: string)
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function testFileStorageChannel(
|
||||||
|
token: string,
|
||||||
|
channelId: string,
|
||||||
|
): Promise<FileStorageChannelTestResult> {
|
||||||
|
return request<FileStorageChannelTestResult>(`/api/admin/system/file-storage/channels/${channelId}/test`, {
|
||||||
|
method: 'POST',
|
||||||
|
token,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function request<T>(
|
async function request<T>(
|
||||||
path: string,
|
path: string,
|
||||||
options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record<string, string>; signal?: AbortSignal; timeoutMs?: number } = {},
|
options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record<string, string>; signal?: AbortSignal; timeoutMs?: number } = {},
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ export function AdminPage(props: {
|
|||||||
onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise<void>;
|
onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise<void>;
|
||||||
onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise<void>;
|
onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise<void>;
|
||||||
onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise<void>;
|
onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise<void>;
|
||||||
|
onTestFileStorageChannel: (channelId: string) => Promise<void>;
|
||||||
onConnectSecurityEvents: (transmitterIssuer: string) => Promise<void>;
|
onConnectSecurityEvents: (transmitterIssuer: string) => Promise<void>;
|
||||||
onDisconnectSecurityEvents: () => Promise<void>;
|
onDisconnectSecurityEvents: () => Promise<void>;
|
||||||
onRefreshSecurityEvents: () => Promise<void>;
|
onRefreshSecurityEvents: () => Promise<void>;
|
||||||
@@ -214,6 +215,7 @@ export function AdminPage(props: {
|
|||||||
onDeleteFileStorageChannel={props.onDeleteFileStorageChannel}
|
onDeleteFileStorageChannel={props.onDeleteFileStorageChannel}
|
||||||
onSaveFileStorageChannel={props.onSaveFileStorageChannel}
|
onSaveFileStorageChannel={props.onSaveFileStorageChannel}
|
||||||
onSaveFileStorageSettings={props.onSaveFileStorageSettings}
|
onSaveFileStorageSettings={props.onSaveFileStorageSettings}
|
||||||
|
onTestFileStorageChannel={props.onTestFileStorageChannel}
|
||||||
onSaveClientCustomizationSettings={props.onSaveClientCustomizationSettings}
|
onSaveClientCustomizationSettings={props.onSaveClientCustomizationSettings}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ type ClientCustomizationForm = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
type FileStorageChannelForm = {
|
type FileStorageChannelForm = {
|
||||||
|
accessKeyId: string;
|
||||||
|
accessKeyIdPreview: string;
|
||||||
|
accessKeySecret: string;
|
||||||
|
accessKeySecretPreview: string;
|
||||||
apiKey: string;
|
apiKey: string;
|
||||||
apiKeyPreview: string;
|
apiKeyPreview: string;
|
||||||
channelKey: string;
|
channelKey: string;
|
||||||
@@ -23,13 +27,22 @@ type FileStorageChannelForm = {
|
|||||||
priority: string;
|
priority: string;
|
||||||
provider: string;
|
provider: string;
|
||||||
retryPolicyJson: string;
|
retryPolicyJson: string;
|
||||||
|
sessionToken: string;
|
||||||
|
sessionTokenPreview: string;
|
||||||
scenes: string[];
|
scenes: string[];
|
||||||
status: string;
|
status: string;
|
||||||
uploadUrl: string;
|
uploadUrl: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
const defaultUploadUrl = 'http://127.0.0.1:3001/v1/files/upload';
|
const defaultUploadUrl = 'http://127.0.0.1:3001/v1/files/upload';
|
||||||
const defaultRetryPolicy = {
|
const defaultObjectStorageRetryPolicy = {
|
||||||
|
enabled: true,
|
||||||
|
maxRetries: 2,
|
||||||
|
backoffSeconds: [0.25, 1],
|
||||||
|
strategy: 'exponential',
|
||||||
|
};
|
||||||
|
|
||||||
|
const defaultServerMainRetryPolicy = {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
maxRetries: 3,
|
maxRetries: 3,
|
||||||
backoffSeconds: [60, 120, 180],
|
backoffSeconds: [60, 120, 180],
|
||||||
@@ -39,7 +52,7 @@ const defaultRetryPolicy = {
|
|||||||
const providerOptions = [
|
const providerOptions = [
|
||||||
{ value: 'server_main_openapi', label: 'server-main OpenAPI' },
|
{ value: 'server_main_openapi', label: 'server-main OpenAPI' },
|
||||||
{ value: 'aliyun_oss', label: '阿里云 OSS' },
|
{ value: 'aliyun_oss', label: '阿里云 OSS' },
|
||||||
{ value: 'tencent_cos', label: '腾讯云 COS' },
|
{ value: 's3', label: 'S3 / S3 兼容存储' },
|
||||||
];
|
];
|
||||||
|
|
||||||
const defaultScenes = ['upload', 'image_result', 'request_asset'];
|
const defaultScenes = ['upload', 'image_result', 'request_asset'];
|
||||||
@@ -52,7 +65,6 @@ const sceneOptions = [
|
|||||||
const resultUploadPolicyOptions = [
|
const resultUploadPolicyOptions = [
|
||||||
{ value: 'default', label: '默认：仅非链接资源转存', description: 'URL 结果直接保存；base64 / buffer 等结果转存后保存 URL' },
|
{ value: 'default', label: '默认：仅非链接资源转存', description: 'URL 结果直接保存；base64 / buffer 等结果转存后保存 URL' },
|
||||||
{ value: 'upload_all', label: '全部转存', description: 'URL、base64、buffer 等生成媒体结果都会转存到当前文件渠道' },
|
{ value: 'upload_all', label: '全部转存', description: 'URL、base64、buffer 等生成媒体结果都会转存到当前文件渠道' },
|
||||||
{ value: 'upload_none', label: '不做外部转存', description: 'base64 / buffer 临时写入本地静态文件；数据库仅保存占位符，默认 24 小时内按需恢复' },
|
|
||||||
];
|
];
|
||||||
|
|
||||||
export function SystemSettingsPanel(props: {
|
export function SystemSettingsPanel(props: {
|
||||||
@@ -66,6 +78,7 @@ export function SystemSettingsPanel(props: {
|
|||||||
onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise<void>;
|
onSaveClientCustomizationSettings: (input: ClientCustomizationSettingsUpdateRequest) => Promise<void>;
|
||||||
onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise<void>;
|
onSaveFileStorageChannel: (input: FileStorageChannelUpsertRequest, channelId?: string) => Promise<void>;
|
||||||
onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise<void>;
|
onSaveFileStorageSettings: (input: FileStorageSettingsUpdateRequest) => Promise<void>;
|
||||||
|
onTestFileStorageChannel: (channelId: string) => Promise<void>;
|
||||||
}) {
|
}) {
|
||||||
const [activeTab, setActiveTab] = useState<SystemSettingsTab>('fileStorage');
|
const [activeTab, setActiveTab] = useState<SystemSettingsTab>('fileStorage');
|
||||||
const [dialogOpen, setDialogOpen] = useState(false);
|
const [dialogOpen, setDialogOpen] = useState(false);
|
||||||
@@ -75,6 +88,7 @@ export function SystemSettingsPanel(props: {
|
|||||||
const [clientCustomizationForm, setClientCustomizationForm] = useState<ClientCustomizationForm>(() => clientCustomizationSettingsToForm(props.clientCustomizationSettings));
|
const [clientCustomizationForm, setClientCustomizationForm] = useState<ClientCustomizationForm>(() => clientCustomizationSettingsToForm(props.clientCustomizationSettings));
|
||||||
const [settingsPolicy, setSettingsPolicy] = useState(() => normalizeResultUploadPolicy(props.settings?.resultUploadPolicy));
|
const [settingsPolicy, setSettingsPolicy] = useState(() => normalizeResultUploadPolicy(props.settings?.resultUploadPolicy));
|
||||||
const [localError, setLocalError] = useState('');
|
const [localError, setLocalError] = useState('');
|
||||||
|
const [testingChannelId, setTestingChannelId] = useState('');
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setSettingsPolicy(normalizeResultUploadPolicy(props.settings?.resultUploadPolicy));
|
setSettingsPolicy(normalizeResultUploadPolicy(props.settings?.resultUploadPolicy));
|
||||||
@@ -130,6 +144,18 @@ export function SystemSettingsPanel(props: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function testChannel(channel: FileStorageChannel) {
|
||||||
|
setLocalError('');
|
||||||
|
setTestingChannelId(channel.id);
|
||||||
|
try {
|
||||||
|
await props.onTestFileStorageChannel(channel.id);
|
||||||
|
} catch (err) {
|
||||||
|
setLocalError(err instanceof Error ? err.message : '对象存储连接测试失败');
|
||||||
|
} finally {
|
||||||
|
setTestingChannelId('');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function saveSettings() {
|
async function saveSettings() {
|
||||||
setLocalError('');
|
setLocalError('');
|
||||||
try {
|
try {
|
||||||
@@ -154,7 +180,7 @@ export function SystemSettingsPanel(props: {
|
|||||||
<CardHeader>
|
<CardHeader>
|
||||||
<div>
|
<div>
|
||||||
<CardTitle>系统设置</CardTitle>
|
<CardTitle>系统设置</CardTitle>
|
||||||
<p className="mutedText">集中维护网关级配置；文件存储渠道按优先级轮转，单渠道使用 60/120/180 秒退避重试。</p>
|
<p className="mutedText">集中维护网关级配置；文件存储渠道按优先级轮转，对象存储默认使用 250ms / 1s 退避。</p>
|
||||||
</div>
|
</div>
|
||||||
<Badge variant="secondary">{props.channels.length} 个文件渠道</Badge>
|
<Badge variant="secondary">{props.channels.length} 个文件渠道</Badge>
|
||||||
</CardHeader>
|
</CardHeader>
|
||||||
@@ -177,7 +203,7 @@ export function SystemSettingsPanel(props: {
|
|||||||
<div className="fileStorageSettingsCard">
|
<div className="fileStorageSettingsCard">
|
||||||
<div>
|
<div>
|
||||||
<strong>全局生成媒体转存策略</strong>
|
<strong>全局生成媒体转存策略</strong>
|
||||||
<span>对图片、音频和视频生成结果统一生效；启用转存但没有可用渠道时，回退到保留 24 小时的网关本地静态资源。</span>
|
<span>对图片、音频和视频生成结果统一生效；没有可用对象存储时任务会安全失败，不再写入网关本机目录。</span>
|
||||||
</div>
|
</div>
|
||||||
<Label>
|
<Label>
|
||||||
策略
|
策略
|
||||||
@@ -195,7 +221,7 @@ export function SystemSettingsPanel(props: {
|
|||||||
<div className="fileStorageToolbar">
|
<div className="fileStorageToolbar">
|
||||||
<div>
|
<div>
|
||||||
<strong>文件存储渠道</strong>
|
<strong>文件存储渠道</strong>
|
||||||
<span>server-main OpenAPI 渠道只需要上传路由和 API Key。</span>
|
<span>支持 server-main OpenAPI、阿里云 OSS 与 S3 协议；按优先级重试下一个渠道。</span>
|
||||||
</div>
|
</div>
|
||||||
<Button type="button" onClick={openCreateDialog}>
|
<Button type="button" onClick={openCreateDialog}>
|
||||||
<Plus size={15} />
|
<Plus size={15} />
|
||||||
@@ -224,6 +250,12 @@ export function SystemSettingsPanel(props: {
|
|||||||
{channel.lastError && <span>最近错误: {channel.lastError}</span>}
|
{channel.lastError && <span>最近错误: {channel.lastError}</span>}
|
||||||
</div>
|
</div>
|
||||||
<footer>
|
<footer>
|
||||||
|
{channel.provider !== 'server_main_openapi' && (
|
||||||
|
<Button type="button" variant="outline" size="sm" disabled={testingChannelId === channel.id} onClick={() => testChannel(channel)}>
|
||||||
|
<ShieldCheck size={14} />
|
||||||
|
{testingChannelId === channel.id ? '测试中…' : '连接测试'}
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
<Button type="button" variant="outline" size="sm" onClick={() => editChannel(channel)}>
|
<Button type="button" variant="outline" size="sm" onClick={() => editChannel(channel)}>
|
||||||
<Pencil size={14} />
|
<Pencil size={14} />
|
||||||
修改
|
修改
|
||||||
@@ -312,7 +344,16 @@ export function SystemSettingsPanel(props: {
|
|||||||
</Label>
|
</Label>
|
||||||
<Label>
|
<Label>
|
||||||
渠道类型
|
渠道类型
|
||||||
<Select value={form.provider} onChange={(event) => setForm({ ...form, provider: event.target.value })}>
|
<Select value={form.provider} onChange={(event) => {
|
||||||
|
const provider = event.target.value;
|
||||||
|
setForm({
|
||||||
|
...form,
|
||||||
|
provider,
|
||||||
|
retryPolicyJson: editingChannel
|
||||||
|
? form.retryPolicyJson
|
||||||
|
: stringifyJson(defaultRetryPolicyForProvider(provider)),
|
||||||
|
});
|
||||||
|
}}>
|
||||||
{providerOptions.map((item) => <option value={item.value} key={item.value}>{item.label}</option>)}
|
{providerOptions.map((item) => <option value={item.value} key={item.value}>{item.label}</option>)}
|
||||||
</Select>
|
</Select>
|
||||||
</Label>
|
</Label>
|
||||||
@@ -337,15 +378,29 @@ export function SystemSettingsPanel(props: {
|
|||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
</Label>
|
</Label>
|
||||||
<Label className="spanTwo">
|
{form.provider === 'server_main_openapi' && <Label className="spanTwo">
|
||||||
上传路由
|
上传路由
|
||||||
<Input value={form.uploadUrl} onChange={(event) => setForm({ ...form, uploadUrl: event.target.value })} placeholder={defaultUploadUrl} />
|
<Input value={form.uploadUrl} onChange={(event) => setForm({ ...form, uploadUrl: event.target.value })} placeholder={defaultUploadUrl} />
|
||||||
</Label>
|
</Label>}
|
||||||
<Label className="platformCredentialField">
|
{form.provider === 'server_main_openapi' && <Label className="platformCredentialField">
|
||||||
API Key
|
API Key
|
||||||
<Input value={form.apiKey} onChange={(event) => setForm({ ...form, apiKey: event.target.value })} placeholder={credentialInputPlaceholder(form.apiKeyPreview)} />
|
<Input value={form.apiKey} onChange={(event) => setForm({ ...form, apiKey: event.target.value })} placeholder={credentialInputPlaceholder(form.apiKeyPreview)} />
|
||||||
<small>保持脱敏值不变表示不修改；填写新值表示覆盖；清空后保存表示清除。</small>
|
<small>保持脱敏值不变表示不修改；填写新值表示覆盖；清空后保存表示清除。</small>
|
||||||
|
</Label>}
|
||||||
|
{form.provider !== 'server_main_openapi' && <>
|
||||||
|
<Label>
|
||||||
|
Access Key ID
|
||||||
|
<Input value={form.accessKeyId} onChange={(event) => setForm({ ...form, accessKeyId: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeyIdPreview)} />
|
||||||
</Label>
|
</Label>
|
||||||
|
<Label>
|
||||||
|
Access Key Secret
|
||||||
|
<Input type="password" value={form.accessKeySecret} onChange={(event) => setForm({ ...form, accessKeySecret: event.target.value })} placeholder={credentialInputPlaceholder(form.accessKeySecretPreview)} />
|
||||||
|
</Label>
|
||||||
|
<Label className="spanTwo">
|
||||||
|
Session Token（可选）
|
||||||
|
<Input type="password" value={form.sessionToken} onChange={(event) => setForm({ ...form, sessionToken: event.target.value })} placeholder={credentialInputPlaceholder(form.sessionTokenPreview)} />
|
||||||
|
</Label>
|
||||||
|
</>}
|
||||||
<Label>
|
<Label>
|
||||||
优先级
|
优先级
|
||||||
<Input type="number" min={1} value={form.priority} onChange={(event) => setForm({ ...form, priority: event.target.value })} />
|
<Input type="number" min={1} value={form.priority} onChange={(event) => setForm({ ...form, priority: event.target.value })} />
|
||||||
@@ -403,6 +458,10 @@ function clientCustomizationFormToPayload(form: ClientCustomizationForm): Client
|
|||||||
|
|
||||||
function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
|
function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
|
||||||
return {
|
return {
|
||||||
|
accessKeyId: '',
|
||||||
|
accessKeyIdPreview: '',
|
||||||
|
accessKeySecret: '',
|
||||||
|
accessKeySecretPreview: '',
|
||||||
apiKey: '',
|
apiKey: '',
|
||||||
apiKeyPreview: '',
|
apiKeyPreview: '',
|
||||||
channelKey,
|
channelKey,
|
||||||
@@ -410,7 +469,9 @@ function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
|
|||||||
name: 'server-main OpenAPI',
|
name: 'server-main OpenAPI',
|
||||||
priority: '100',
|
priority: '100',
|
||||||
provider: 'server_main_openapi',
|
provider: 'server_main_openapi',
|
||||||
retryPolicyJson: stringifyJson(defaultRetryPolicy),
|
retryPolicyJson: stringifyJson(defaultServerMainRetryPolicy),
|
||||||
|
sessionToken: '',
|
||||||
|
sessionTokenPreview: '',
|
||||||
scenes: defaultScenes,
|
scenes: defaultScenes,
|
||||||
status: 'disabled',
|
status: 'disabled',
|
||||||
uploadUrl: defaultUploadUrl,
|
uploadUrl: defaultUploadUrl,
|
||||||
@@ -420,6 +481,10 @@ function defaultChannelForm(channelKey = ''): FileStorageChannelForm {
|
|||||||
function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
||||||
const preview = apiKeyPreview(channel);
|
const preview = apiKeyPreview(channel);
|
||||||
return {
|
return {
|
||||||
|
accessKeyId: credentialPreview(channel, 'accessKeyId'),
|
||||||
|
accessKeyIdPreview: credentialPreview(channel, 'accessKeyId'),
|
||||||
|
accessKeySecret: credentialPreview(channel, 'accessKeySecret'),
|
||||||
|
accessKeySecretPreview: credentialPreview(channel, 'accessKeySecret'),
|
||||||
apiKey: preview,
|
apiKey: preview,
|
||||||
apiKeyPreview: preview,
|
apiKeyPreview: preview,
|
||||||
channelKey: channel.channelKey,
|
channelKey: channel.channelKey,
|
||||||
@@ -427,7 +492,9 @@ function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
|||||||
name: channel.name,
|
name: channel.name,
|
||||||
priority: String(channel.priority || 100),
|
priority: String(channel.priority || 100),
|
||||||
provider: channel.provider || 'server_main_openapi',
|
provider: channel.provider || 'server_main_openapi',
|
||||||
retryPolicyJson: stringifyJson(channel.retryPolicy ?? defaultRetryPolicy),
|
retryPolicyJson: stringifyJson(channel.retryPolicy ?? defaultRetryPolicyForProvider(channel.provider)),
|
||||||
|
sessionToken: credentialPreview(channel, 'sessionToken'),
|
||||||
|
sessionTokenPreview: credentialPreview(channel, 'sessionToken'),
|
||||||
scenes: normalizeScenes(channel.scenes),
|
scenes: normalizeScenes(channel.scenes),
|
||||||
status: channel.status || 'disabled',
|
status: channel.status || 'disabled',
|
||||||
uploadUrl: channel.uploadUrl || defaultUploadUrl,
|
uploadUrl: channel.uploadUrl || defaultUploadUrl,
|
||||||
@@ -436,6 +503,8 @@ function channelToForm(channel: FileStorageChannel): FileStorageChannelForm {
|
|||||||
|
|
||||||
function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRequest {
|
function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRequest {
|
||||||
return {
|
return {
|
||||||
|
accessKeyId: credentialPayloadValue(form.accessKeyId, form.accessKeyIdPreview),
|
||||||
|
accessKeySecret: credentialPayloadValue(form.accessKeySecret, form.accessKeySecretPreview),
|
||||||
apiKey: apiKeyPayloadValue(form),
|
apiKey: apiKeyPayloadValue(form),
|
||||||
channelKey: form.channelKey.trim(),
|
channelKey: form.channelKey.trim(),
|
||||||
config: parseJsonObject(form.configJson, '扩展配置 JSON'),
|
config: parseJsonObject(form.configJson, '扩展配置 JSON'),
|
||||||
@@ -443,6 +512,7 @@ function formToPayload(form: FileStorageChannelForm): FileStorageChannelUpsertRe
|
|||||||
priority: Number(form.priority) || 100,
|
priority: Number(form.priority) || 100,
|
||||||
provider: form.provider,
|
provider: form.provider,
|
||||||
retryPolicy: parseJsonObject(form.retryPolicyJson, '重试策略 JSON'),
|
retryPolicy: parseJsonObject(form.retryPolicyJson, '重试策略 JSON'),
|
||||||
|
sessionToken: credentialPayloadValue(form.sessionToken, form.sessionTokenPreview),
|
||||||
scenes: normalizeScenes(form.scenes),
|
scenes: normalizeScenes(form.scenes),
|
||||||
status: form.status,
|
status: form.status,
|
||||||
uploadUrl: form.uploadUrl.trim(),
|
uploadUrl: form.uploadUrl.trim(),
|
||||||
@@ -495,8 +565,8 @@ function nextScenes(current: string[], scene: string, checked: boolean) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function retryPolicySummary(policy?: Record<string, unknown>) {
|
function retryPolicySummary(policy?: Record<string, unknown>) {
|
||||||
const maxRetries = numberFromUnknown(policy?.maxRetries) || 3;
|
const maxRetries = numberFromUnknown(policy?.maxRetries) || 2;
|
||||||
const backoff = Array.isArray(policy?.backoffSeconds) ? policy?.backoffSeconds.join('/') : '60/120/180';
|
const backoff = Array.isArray(policy?.backoffSeconds) ? policy?.backoffSeconds.join('/') : '0.25/1';
|
||||||
return `${maxRetries} 次 · ${backoff}s`;
|
return `${maxRetries} 次 · ${backoff}s`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -506,9 +576,22 @@ function apiKeyPreview(channel: FileStorageChannel) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function apiKeyPayloadValue(form: FileStorageChannelForm) {
|
function apiKeyPayloadValue(form: FileStorageChannelForm) {
|
||||||
const value = form.apiKey.trim();
|
return credentialPayloadValue(form.apiKey, form.apiKeyPreview);
|
||||||
if (form.apiKeyPreview && value === form.apiKeyPreview) return undefined;
|
}
|
||||||
return value || (form.apiKeyPreview ? '' : undefined);
|
|
||||||
|
function credentialPreview(channel: FileStorageChannel, key: string) {
|
||||||
|
const value = channel.credentialsPreview?.[key];
|
||||||
|
return typeof value === 'string' ? value : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function credentialPayloadValue(value: string, preview: string) {
|
||||||
|
const normalized = value.trim();
|
||||||
|
if (preview && normalized === preview) return undefined;
|
||||||
|
return normalized || (preview ? '' : undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultRetryPolicyForProvider(provider: string) {
|
||||||
|
return provider === 'server_main_openapi' ? defaultServerMainRetryPolicy : defaultObjectStorageRetryPolicy;
|
||||||
}
|
}
|
||||||
|
|
||||||
function credentialInputPlaceholder(preview: string) {
|
function credentialInputPlaceholder(preview: string) {
|
||||||
|
|||||||
@@ -1039,7 +1039,7 @@ export interface FileStorageChannel {
|
|||||||
id: string;
|
id: string;
|
||||||
channelKey: string;
|
channelKey: string;
|
||||||
name: string;
|
name: string;
|
||||||
provider: 'server_main_openapi' | 'aliyun_oss' | 'tencent_cos' | string;
|
provider: 'server_main_openapi' | 'aliyun_oss' | 's3' | string;
|
||||||
uploadUrl?: string;
|
uploadUrl?: string;
|
||||||
credentialsPreview?: Record<string, unknown>;
|
credentialsPreview?: Record<string, unknown>;
|
||||||
scenes?: string[];
|
scenes?: string[];
|
||||||
@@ -1057,9 +1057,14 @@ export interface FileStorageChannel {
|
|||||||
export interface FileStorageChannelUpsertRequest {
|
export interface FileStorageChannelUpsertRequest {
|
||||||
channelKey: string;
|
channelKey: string;
|
||||||
name: string;
|
name: string;
|
||||||
provider?: 'server_main_openapi' | 'aliyun_oss' | 'tencent_cos' | string;
|
provider?: 'server_main_openapi' | 'aliyun_oss' | 's3' | string;
|
||||||
uploadUrl?: string;
|
uploadUrl?: string;
|
||||||
apiKey?: string;
|
apiKey?: string;
|
||||||
|
accessKey?: string;
|
||||||
|
accessKeyId?: string;
|
||||||
|
accessKeySecret?: string;
|
||||||
|
secretKey?: string;
|
||||||
|
sessionToken?: string;
|
||||||
scenes?: string[];
|
scenes?: string[];
|
||||||
config?: Record<string, unknown>;
|
config?: Record<string, unknown>;
|
||||||
retryPolicy?: Record<string, unknown>;
|
retryPolicy?: Record<string, unknown>;
|
||||||
@@ -1067,12 +1072,20 @@ export interface FileStorageChannelUpsertRequest {
|
|||||||
status?: 'enabled' | 'disabled' | string;
|
status?: 'enabled' | 'disabled' | string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface FileStorageChannelTestResult {
|
||||||
|
provider: 'aliyun_oss' | 's3' | string;
|
||||||
|
putSucceeded: boolean;
|
||||||
|
headSucceeded: boolean;
|
||||||
|
deleteSucceeded: boolean;
|
||||||
|
durationMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface FileStorageSettings {
|
export interface FileStorageSettings {
|
||||||
resultUploadPolicy: 'default' | 'upload_all' | 'upload_none' | string;
|
resultUploadPolicy: 'default' | 'upload_all' | string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface FileStorageSettingsUpdateRequest {
|
export interface FileStorageSettingsUpdateRequest {
|
||||||
resultUploadPolicy: 'default' | 'upload_all' | 'upload_none' | string;
|
resultUploadPolicy: 'default' | 'upload_all' | string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ClientCustomizationSettings {
|
export interface ClientCustomizationSettings {
|
||||||
@@ -1238,6 +1251,19 @@ export interface IdentityRevisionPolicyUpdate {
|
|||||||
sessionRefreshSeconds?: number;
|
sessionRefreshSeconds?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface PublicErrorV1 {
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
category: string;
|
||||||
|
httpStatus: number;
|
||||||
|
retryable: boolean;
|
||||||
|
action: string;
|
||||||
|
retryAfterSeconds?: number;
|
||||||
|
requestId?: string;
|
||||||
|
taskId?: string;
|
||||||
|
version: 'v1' | string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface GatewayTask {
|
export interface GatewayTask {
|
||||||
id: string;
|
id: string;
|
||||||
kind: string;
|
kind: string;
|
||||||
@@ -1284,6 +1310,7 @@ export interface GatewayTask {
|
|||||||
error?: string;
|
error?: string;
|
||||||
errorCode?: string;
|
errorCode?: string;
|
||||||
errorMessage?: string;
|
errorMessage?: string;
|
||||||
|
publicError?: PublicErrorV1;
|
||||||
attempts?: GatewayTaskAttempt[];
|
attempts?: GatewayTaskAttempt[];
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
@@ -1367,6 +1394,7 @@ export interface GatewayTaskAttempt {
|
|||||||
upstreamSubmissionUpdatedAt?: string;
|
upstreamSubmissionUpdatedAt?: string;
|
||||||
errorCode?: string;
|
errorCode?: string;
|
||||||
errorMessage?: string;
|
errorMessage?: string;
|
||||||
|
publicError?: PublicErrorV1;
|
||||||
startedAt: string;
|
startedAt: string;
|
||||||
finishedAt?: string;
|
finishedAt?: string;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user