feat(storage): 统一二进制对象存储与公开错误

新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换,保留 server-main 兼容与环境 OSS 内存通道。

将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化,生产路径不再写入本机静态目录,历史本地资源仅保留只读兼容。

引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出,同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。

验证:go test ./... -count=1;go vet ./...;pnpm lint;pnpm test;pnpm build;pnpm openapi;tests/ci/migrations-test.sh。
This commit is contained in:
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
+14 -8
View File
@@ -9,6 +9,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -97,11 +98,16 @@ func writeWireResponse(w http.ResponseWriter, wire *clients.WireResponse) {
}
func writeProtocolError(w http.ResponseWriter, protocol string, status int, message string, details map[string]any, code string) {
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
standard = publicErrorWithRetryAfter(standard, details)
publicerror.Observe(standard)
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
details = safePublicErrorDetails(details, standard, true)
switch protocol {
case clients.ProtocolGeminiGenerateContent:
writeGeminiError(w, status, message, details, code)
case clients.ProtocolVolcesContents:
writeVolcesError(w, status, message, code)
writeVolcesPublicError(w, standard)
case clients.ProtocolKlingV1Omni, clients.ProtocolKlingV2Omni:
writeKelingCompatError(w, "", newKelingCompatError(status, kelingCompatBusinessCode(code, message), message))
default:
@@ -185,11 +191,11 @@ func googleRPCStatus(status int, code string) string {
}
func writeVolcesError(w http.ResponseWriter, status int, message string, code string) {
if strings.TrimSpace(code) == "" {
code = http.StatusText(status)
}
writeJSON(w, status, map[string]any{"error": map[string]any{
"code": code,
"message": message,
}})
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
publicerror.Observe(standard)
writeVolcesPublicError(w, standard)
}
func writeVolcesPublicError(w http.ResponseWriter, standard publicerror.Error) {
writeJSON(w, standard.HTTPStatus, map[string]any{"error": standard})
}
@@ -47,7 +47,7 @@ func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) {
}
}
func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
func TestProtocolErrorsUseCompatibleShapesWithStandardPublicErrors(t *testing.T) {
tests := []struct {
name string
protocol string
@@ -78,10 +78,10 @@ func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusBadGateway,
assertBody: func(t *testing.T, body map[string]any) {
errorBody := requireObject(t, body["error"])
if errorBody["code"] != "upstream_submission_unknown" {
if errorBody["code"] != "upstream_submission_unknown" || errorBody["httpStatus"] != float64(http.StatusBadGateway) || errorBody["retryable"] != true {
t.Fatalf("unexpected Volces error: %+v", body)
}
assertNoKeys(t, errorBody, "status", "retryable", "taskId", "gateway_status")
assertNoKeys(t, errorBody, "status", "taskId", "gateway_status")
},
},
}
@@ -135,6 +135,44 @@ func TestWireResponsePassthroughPreservesStatusUnknownFieldsAndAllowedHeaders(t
}
}
func TestCompatibilityErrorWritersNeverExposeTransportDetails(t *testing.T) {
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
tests := []struct {
name string
write func(http.ResponseWriter)
}{
{name: "openai", write: func(w http.ResponseWriter) {
writeProtocolError(w, clients.ProtocolOpenAIResponses, http.StatusOK, raw, map[string]any{
"provider": "secret-provider", "endpoint": "https://private.example.invalid", "bucket": "private-bucket",
}, "response_read_error")
}},
{name: "volces", write: func(w http.ResponseWriter) {
writeVolcesError(w, http.StatusOK, raw, "response_read_error")
}},
{name: "kling", write: func(w http.ResponseWriter) {
writeKlingCompatError(w, http.StatusOK, raw, "response_read_error")
}},
{name: "keling", write: func(w http.ResponseWriter) {
writeKelingCompatError(w, "request-1", newKelingCompatError(http.StatusOK, 5001, raw))
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
recorder := httptest.NewRecorder()
test.write(recorder)
if recorder.Code != http.StatusBadGateway {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
body := recorder.Body.String()
if strings.Contains(body, "10.42.0.72") || strings.Contains(body, "47.77.191.126") ||
strings.Contains(body, "secret-provider") || strings.Contains(body, "private.example.invalid") || strings.Contains(body, "private-bucket") ||
!strings.Contains(body, "upstream_connection_interrupted") {
t.Fatalf("transport details were not standardized: %s", body)
}
})
}
}
func TestCompatibilityStatusMappings(t *testing.T) {
for internal, want := range map[string]string{
"queued": "queued", "running": "running", "succeeded": "succeeded", "failed": "failed", "cancelled": "cancelled",
+22 -8
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -58,14 +59,21 @@ func writeEasyAIAsyncError(w http.ResponseWriter, status int, message string, de
if len(codes) > 0 {
code = strings.TrimSpace(codes[0])
}
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
standard = publicErrorWithRetryAfter(standard, details)
publicerror.Observe(standard)
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
errorPayload := map[string]any{
"message": message,
"status": status,
"message": message,
"status": status,
"code": code,
"category": standard.Category,
"httpStatus": standard.HTTPStatus,
"retryable": standard.Retryable,
"action": standard.Action,
"version": standard.Version,
}
if code != "" {
errorPayload["code"] = code
}
for key, value := range details {
for key, value := range safePublicErrorDetails(details, standard, false) {
errorPayload[key] = value
}
response := map[string]any{
@@ -158,8 +166,14 @@ func easyAITaskResultResponse(task store.GatewayTask) map[string]any {
if message != "" {
response["message"] = message
}
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" {
response["code"] = code
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" || status == "failed" {
standard := publicTaskError(task)
if code != "" && task.ErrorCode == "" {
standard = publicerror.WithIDs(publicerror.FromFields(code, message, 0, false), task.RequestID, task.ID)
}
response["code"] = standard.Code
response["message"] = standard.Message
response["error"] = standard
}
return response
}
@@ -13,7 +13,7 @@ const maxGatewayUploadBytes = 256 << 20
// uploadFile godoc
// @Summary 上传文件
// @Description 上传文件到配置的文件存储通道;没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。
// @Description 上传文件到配置的对象存储通道;所有通道失败时返回标准化存储错误,不写入本机静态目录。单文件最大 256MiB。
// @Tags files
// @Accept multipart/form-data
// @Produce json
@@ -55,11 +55,8 @@ func (s *Server) uploadFile(w http.ResponseWriter, r *http.Request) {
})
if err != nil {
s.logger.Error("upload file failed", "error", err)
status := http.StatusBadGateway
if clients.ErrorCode(err) == "upload_no_channel" {
status = http.StatusServiceUnavailable
}
writeError(w, status, err.Error())
status := statusFromRunError(err)
writeError(w, status, err.Error(), clients.ErrorCode(err))
return
}
writeJSON(w, http.StatusOK, easyAIFileUploadResponse(upload))
+8 -11
View File
@@ -17,6 +17,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -159,7 +160,7 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
if err != nil {
s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err)
status := http.StatusBadRequest
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
status = http.StatusBadGateway
}
writeGeminiTaskError(status, err.Error(), nil, clients.ErrorCode(err))
@@ -318,14 +319,13 @@ func (s *Server) writeGeminiGenerateContentStream(runCtx context.Context, w http
}
applyRunErrorHeaders(w, runErr)
if !nativePassthrough && !convertedFrames {
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, clients.ProtocolGeminiGenerateContent) {
writeWireResponse(w, wire)
return
}
writeGeminiTaskError(statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
} else if !nativePassthrough {
status := statusFromRunError(runErr)
writeGeminiSSEFrame(w, geminiErrorEnvelope(status, runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr)))
standard := publicerror.FromFields(runErrorCode(runErr), runErrorMessage(runErr), statusFromRunError(runErr), clients.IsRetryable(runErr))
standard = publicErrorWithRetryAfter(standard, runErrorDetails(runErr))
publicerror.Observe(standard)
details := safePublicErrorDetails(runErrorDetails(runErr), standard, true)
writeGeminiSSEFrame(w, geminiErrorEnvelope(standard.HTTPStatus, standard.Message, details, standard.Code))
if flusher != nil {
flusher.Flush()
}
@@ -683,10 +683,7 @@ func (s *Server) finalizeGeminiFilesUpload(w http.ResponseWriter, r *http.Reques
})
if err != nil {
s.logger.Error("Gemini files upload failed", "error", err)
status := http.StatusBadGateway
if clients.ErrorCode(err) == "upload_no_channel" {
status = http.StatusServiceUnavailable
}
status := statusFromRunError(err)
writeProtocolError(w, clients.ProtocolGeminiGenerateContent, status, err.Error(), nil, clients.ErrorCode(err))
return
}
+6 -8
View File
@@ -1154,7 +1154,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind)
if err != nil {
status := http.StatusBadRequest
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
status = http.StatusBadGateway
}
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
@@ -1195,7 +1195,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
if err != nil {
s.logger.Warn("prepare task request failed", "kind", kind, "error", err)
status := http.StatusBadRequest
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
status = http.StatusBadGateway
}
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
@@ -1556,10 +1556,6 @@ func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.Response
return
}
applyRunErrorHeaders(w, runErr)
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, targetProtocol) {
writeWireResponse(w, wire)
return
}
if targetProtocol != "" {
writeProtocolError(w, targetProtocol, statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
return
@@ -1673,6 +1669,8 @@ func modelNameFromValue(value any) string {
func statusFromRunError(err error) int {
switch {
case clients.ErrorCode(err) == "storage_write_failed" || clients.ErrorCode(err) == "storage_read_failed" || clients.ErrorCode(err) == "storage_config_invalid" || clients.ErrorCode(err) == "storage_auth_failed":
return http.StatusServiceUnavailable
case clients.ErrorCode(err) == "binary_result_expired":
return http.StatusGone
case clients.ErrorCode(err) == "binary_result_corrupted" || clients.ErrorCode(err) == "result_binary_not_materialized":
@@ -1949,7 +1947,7 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) {
return
}
writeJSON(w, http.StatusOK, map[string]any{
"items": result.Items,
"items": publicTaskList(result.Items),
"total": result.Total,
"page": result.Page,
"pageSize": result.PageSize,
@@ -2034,7 +2032,7 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
writeStoredBinaryResultError(w, err)
return
}
writeJSON(w, http.StatusOK, task)
writeJSON(w, http.StatusOK, publicGatewayTask(task))
return
}
if store.IsNotFound(err) {
@@ -15,6 +15,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -66,17 +67,19 @@ type KelingOmniWatermarkInfo struct {
}
type KelingCompatibleEnvelope struct {
Code int `json:"code" example:"0"`
Message string `json:"message" example:"SUCCEED"`
RequestID string `json:"request_id"`
Data any `json:"data,omitempty"`
Code int `json:"code" example:"0"`
Message string `json:"message" example:"SUCCEED"`
RequestID string `json:"request_id"`
Data any `json:"data,omitempty"`
Error *publicerror.Error `json:"error,omitempty"`
}
type kelingCompatError struct {
HTTPStatus int
Code int
Message string
RequestID string
HTTPStatus int
Code int
Message string
RequestID string
PublicError *publicerror.Error
}
func (e *kelingCompatError) Error() string {
@@ -200,10 +203,6 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
}
task, createErr = s.waitForCompatibilitySubmission(r, task)
if createErr != nil {
if wire := clients.ErrorWireResponse(createErr); wireResponseMatches(wire, clients.ProtocolKlingV1Omni) {
writeWireResponse(w, wire)
return
}
writeKelingCompatError(w, requestID, kelingCompatGatewayError(createErr))
return
}
@@ -690,11 +689,13 @@ func kelingCompatTaskData(task store.GatewayTask) map[string]any {
"enabled": kelingCompatTaskWatermark(task.Request),
},
}
if message := kelingCompatTaskMessage(task); message != "" {
data["task_status_msg"] = message
}
if kelingCompatTaskStatus(task.Status) == "failed" {
data["task_status_code"] = kelingCompatBusinessCode(task.ErrorCode, kelingCompatTaskMessage(task))
standard := publicTaskError(task)
data["task_status_msg"] = standard.Message
data["task_status_code"] = kelingCompatBusinessCode(standard.Code, standard.Message)
data["error"] = standard
} else if message := kelingCompatTaskMessage(task); message != "" {
data["task_status_msg"] = message
}
if videos := kelingCompatTaskVideos(task.Result); len(videos) > 0 {
data["task_result"] = map[string]any{"videos": videos}
@@ -770,6 +771,9 @@ func firstKelingCompatValue(values ...any) any {
}
func kelingCompatTaskMessage(task store.GatewayTask) string {
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" || task.Error != "" {
return publicTaskError(task).Message
}
return strings.TrimSpace(firstNonEmpty(task.ErrorMessage, task.Error, task.Message))
}
@@ -825,8 +829,11 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
return newKelingCompatError(http.StatusInternalServerError, 5000, "unknown gateway error")
}
codeText := clients.ErrorCode(err)
businessCode := kelingCompatBusinessCode(codeText, err.Error())
status := http.StatusInternalServerError
status := statusFromRunError(err)
standard := publicerror.FromFields(codeText, err.Error(), status, clients.IsRetryable(err))
publicerror.Observe(standard)
businessCode := kelingCompatBusinessCode(standard.Code, standard.Message)
status = standard.HTTPStatus
switch businessCode {
case 1101:
status = http.StatusPaymentRequired
@@ -841,7 +848,9 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
case 5001:
status = http.StatusBadGateway
}
return newKelingCompatError(status, businessCode, err.Error())
result := newKelingCompatError(status, businessCode, standard.Message)
result.PublicError = &standard
return result
}
func kelingCompatBusinessCode(errorCode string, message string) int {
@@ -923,9 +932,18 @@ func writeKelingCompatError(w http.ResponseWriter, requestID string, err *keling
if status == 0 {
status = http.StatusInternalServerError
}
standard := err.PublicError
if standard == nil {
value := publicerror.FromFields("", err.Message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
value = publicerror.WithIDs(value, requestID, "")
publicerror.Observe(value)
standard = &value
}
status = standard.HTTPStatus
writeJSON(w, status, KelingCompatibleEnvelope{
Code: err.Code,
Message: err.Message,
Message: standard.Message,
RequestID: requestID,
Error: standard,
})
}
+15 -14
View File
@@ -14,6 +14,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -191,10 +192,6 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
}
task, err = s.waitForCompatibilitySubmission(r, task)
if err != nil {
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, targetProtocol) {
writeWireResponse(w, wire)
return
}
writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
return
}
@@ -667,7 +664,9 @@ func klingV1TaskData(task store.GatewayTask) map[string]any {
"created_at": task.CreatedAt.UnixMilli(), "updated_at": task.UpdatedAt.UnixMilli(),
}
if task.ErrorMessage != "" || task.Error != "" {
data["task_status_msg"] = firstNonEmpty(task.ErrorMessage, task.Error)
standard := publicTaskError(task)
data["task_status_msg"] = standard.Message
data["error"] = standard
}
if watermarkInfo, ok := task.Request["watermark_info"].(map[string]any); ok {
data["watermark_info"] = watermarkInfo
@@ -698,7 +697,9 @@ func klingV2TaskData(task store.GatewayTask) map[string]any {
"external_id": task.ExternalTaskID,
}
if message := firstNonEmpty(task.ErrorMessage, task.Error); message != "" {
data["message"] = message
standard := publicTaskError(task)
data["message"] = standard.Message
data["error"] = standard
}
if outputs := klingV2Outputs(task); len(outputs) > 0 {
data["outputs"] = outputs
@@ -827,14 +828,14 @@ func decodeKlingJSON(r *http.Request, target any) error {
}
func writeKlingCompatError(w http.ResponseWriter, status int, message string, code string) {
if strings.TrimSpace(code) == "" {
code = "invalid_request"
}
writeJSON(w, status, map[string]any{
"code": klingCompatErrorCode(status),
"message": message,
"request_id": "",
"error": code,
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
publicerror.Observe(standard)
writeJSON(w, standard.HTTPStatus, map[string]any{
"code": klingCompatErrorCode(standard.HTTPStatus),
"message": standard.Message,
"request_id": "",
"error": standard.Code,
"public_error": standard,
})
}
+28 -15
View File
@@ -2,6 +2,7 @@ package httpapi
import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -32,11 +33,19 @@ type ErrorEnvelope struct {
}
type ErrorPayload struct {
Message string `json:"message" example:"invalid json body"`
Status int `json:"status" example:"400"`
Code string `json:"code,omitempty" example:"rate_limit"`
Type string `json:"type,omitempty" example:"invalid_request_error"`
Param any `json:"param,omitempty"`
Message string `json:"message" example:"invalid json body"`
Status int `json:"status" example:"400"`
Code string `json:"code,omitempty" example:"rate_limit"`
Category string `json:"category,omitempty" example:"rate_limit"`
HTTPStatus int `json:"httpStatus,omitempty" example:"429"`
Retryable bool `json:"retryable" example:"true"`
Action string `json:"action,omitempty" example:"retry_after"`
Version string `json:"version,omitempty" example:"v1"`
RetryAfterSeconds int `json:"retryAfterSeconds,omitempty" example:"2"`
RequestID string `json:"requestId,omitempty"`
TaskID string `json:"taskId,omitempty"`
Type string `json:"type,omitempty" example:"invalid_request_error"`
Param any `json:"param,omitempty"`
}
type OpenAIErrorEnvelope struct {
@@ -62,12 +71,7 @@ type GeminiErrorStatus struct {
}
type VolcesErrorEnvelope struct {
Error VolcesErrorPayload `json:"error"`
}
type VolcesErrorPayload struct {
Code string `json:"code" example:"invalid_parameter"`
Message string `json:"message" example:"model is required"`
Error publicerror.Error `json:"error"`
}
type VolcesContentsGenerationTaskResponse struct {
@@ -82,10 +86,11 @@ type VolcesContentsGenerationTaskResponse struct {
}
type KlingErrorEnvelope struct {
Code int `json:"code" example:"1001"`
Message string `json:"message" example:"invalid parameter"`
RequestID string `json:"request_id"`
Error string `json:"error,omitempty" example:"invalid_parameter"`
Code int `json:"code" example:"1001"`
Message string `json:"message" example:"invalid parameter"`
RequestID string `json:"request_id"`
Error string `json:"error,omitempty" example:"invalid_parameter"`
PublicError *publicerror.Error `json:"public_error,omitempty"`
}
type AuthResponse struct {
@@ -232,6 +237,14 @@ type FileStorageChannelListResponse struct {
Items []store.FileStorageChannel `json:"items"`
}
type FileStorageChannelTestResponse struct {
Provider string `json:"provider"`
PutSucceeded bool `json:"putSucceeded"`
HeadSucceeded bool `json:"headSucceeded"`
DeleteSucceeded bool `json:"deleteSucceeded"`
DurationMS int64 `json:"durationMs"`
}
type FileUploadResponse struct {
ID string `json:"id,omitempty" example:"file_abc123"`
URL string `json:"url,omitempty" example:"/static/uploaded/upload-abc123.png"`
+120
View File
@@ -0,0 +1,120 @@
package httpapi
import (
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func publicGatewayTask(task store.GatewayTask) store.GatewayTask {
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" {
value := publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error), storedTaskErrorStatus(task.ErrorCode), false)
if task.PublicError != nil && task.PublicError.Code != "" {
value = *task.PublicError
}
value = publicerror.WithIDs(value, task.RequestID, task.ID)
publicerror.Observe(value)
task.PublicError = &value
task.ErrorCode = value.Code
task.ErrorMessage = value.Message
task.Error = value.Message
}
for index := range task.Attempts {
attempt := &task.Attempts[index]
if attempt.ErrorCode == "" && attempt.ErrorMessage == "" {
continue
}
status := attempt.StatusCode
if status <= 0 {
status = http.StatusBadGateway
}
value := publicerror.FromFields(attempt.ErrorCode, attempt.ErrorMessage, status, attempt.Retryable)
if attempt.PublicError != nil && attempt.PublicError.Code != "" {
value = *attempt.PublicError
}
value = publicerror.WithIDs(value, attempt.RequestID, task.ID)
attempt.PublicError = &value
attempt.ErrorCode = value.Code
attempt.ErrorMessage = value.Message
}
return task
}
func publicTaskError(task store.GatewayTask) publicerror.Error {
if task.PublicError != nil && task.PublicError.Code != "" {
return publicerror.WithIDs(*task.PublicError, task.RequestID, task.ID)
}
status := storedTaskErrorStatus(task.ErrorCode)
if status <= 0 {
status = http.StatusBadGateway
}
return publicerror.WithIDs(publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error, task.Message), status, false), task.RequestID, task.ID)
}
func publicTaskList(items []store.GatewayTask) []store.GatewayTask {
out := make([]store.GatewayTask, len(items))
for index, task := range items {
out[index] = publicGatewayTask(task)
}
return out
}
func publicErrorMap(value publicerror.Error) map[string]any {
out := map[string]any{
"code": value.Code, "message": value.Message, "category": value.Category,
"httpStatus": value.HTTPStatus, "retryable": value.Retryable, "action": value.Action,
"version": value.Version,
}
if value.RetryAfterSeconds > 0 {
out["retryAfterSeconds"] = value.RetryAfterSeconds
}
if requestID := strings.TrimSpace(value.RequestID); requestID != "" {
out["requestId"] = requestID
}
if taskID := strings.TrimSpace(value.TaskID); taskID != "" {
out["taskId"] = taskID
}
return out
}
func publicErrorWithRetryAfter(value publicerror.Error, details map[string]any) publicerror.Error {
if value.RetryAfterSeconds > 0 || details == nil {
return value
}
switch typed := details["retryAfterSeconds"].(type) {
case int:
value.RetryAfterSeconds = typed
case int32:
value.RetryAfterSeconds = int(typed)
case int64:
value.RetryAfterSeconds = int(typed)
case float64:
value.RetryAfterSeconds = int(typed)
}
if value.RetryAfterSeconds < 0 {
value.RetryAfterSeconds = 0
}
return value
}
func safePublicErrorDetails(details map[string]any, value publicerror.Error, includePublicError bool) map[string]any {
out := map[string]any{}
for _, key := range []string{"param", "retryAfterSeconds", "recoveryAt", "rateLimit", "pricing"} {
if item, ok := details[key]; ok && item != nil {
out[key] = item
}
}
if value.Category == "request" {
for _, key := range []string{"reason", "diagnosticId"} {
if item, ok := details[key]; ok && item != nil {
out[key] = item
}
}
}
if includePublicError {
out["publicError"] = value
}
return out
}
+135 -72
View File
@@ -12,7 +12,6 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -38,12 +37,6 @@ type decodedRequestAsset struct {
ContentType string
}
type requestAssetOptions struct {
RequirePublicURL bool
UploadScene string
Source string
}
type requestAssetLock struct {
mu sync.Mutex
refs int
@@ -115,6 +108,19 @@ func (s *Server) prepareRequestAssetRefs(ctx context.Context, body map[string]an
}
func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path []string, siblings map[string]any) (any, error) {
if decoded, ok, err := requestAssetFromBinaryValue(requestAssetPathKey(path), path, value, siblings); err != nil {
return nil, err
} else if ok {
if err := s.acquireMediaRequestSlot(ctx); err != nil {
return nil, err
}
defer s.releaseMediaRequestSlot()
ref, err := s.ensureRequestAsset(ctx, decoded)
if err != nil {
return nil, err
}
return requestAssetWrapper(ref), nil
}
switch typed := value.(type) {
case map[string]any:
if typed["assetRef"] != nil {
@@ -156,6 +162,83 @@ func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path [
}
}
func requestAssetFromBinaryValue(key string, path []string, value any, siblings map[string]any) (decodedRequestAsset, bool, error) {
var payload []byte
contentType := ""
switch typed := value.(type) {
case []byte:
payload = append([]byte(nil), typed...)
case map[string]any:
if !strings.EqualFold(strings.TrimSpace(stringFromRequestAny(typed["type"])), "buffer") {
return decodedRequestAsset{}, false, nil
}
contentType = firstNonEmptyRequestString(typed, "contentType", "mimeType", "mime_type")
switch data := typed["data"].(type) {
case []byte:
payload = append([]byte(nil), data...)
case []any:
var ok bool
payload, ok = requestBytesFromNumberArray(data)
if !ok {
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data must contain byte values"))
}
default:
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data is required"))
}
case []any:
if !strictRequestBinaryArrayField(key, path) {
return decodedRequestAsset{}, false, nil
}
var ok bool
payload, ok = requestBytesFromNumberArray(typed)
if !ok {
return decodedRequestAsset{}, false, nil
}
default:
return decodedRequestAsset{}, false, nil
}
if len(payload) == 0 {
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("binary media payload is empty"))
}
return decodedRequestAsset{
Bytes: payload,
ContentType: requestAssetContentType(contentType, payload, key, path, siblings),
}, true, nil
}
func requestAssetPathKey(path []string) string {
for index := len(path) - 1; index >= 0; index-- {
value := strings.TrimSpace(path[index])
if value != "" && !strings.HasPrefix(value, "[") {
return value
}
}
return ""
}
func strictRequestBinaryArrayField(key string, path []string) bool {
value := strings.ToLower(strings.TrimSpace(key))
if value == "bytes" || value == "buffer" || strings.Contains(value, "binary") || strings.Contains(value, "buffer") {
return true
}
return requestMediaKind(key, path, nil) != "" && (value == "data" || value == "content")
}
func requestBytesFromNumberArray(values []any) ([]byte, bool) {
if len(values) == 0 {
return nil, false
}
payload := make([]byte, len(values))
for index, value := range values {
number, ok := value.(float64)
if !ok || number < 0 || number > 255 || number != float64(byte(number)) {
return nil, false
}
payload[index] = byte(number)
}
return payload, true
}
func (s *Server) prepareRequestAssetField(ctx context.Context, key string, path []string, value any, siblings map[string]any) (map[string]any, bool, error) {
text, ok := value.(string)
if !ok {
@@ -291,68 +374,43 @@ func requestAssetFromValue(key string, path []string, value any, siblings map[st
}
func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
UploadScene: store.FileStorageSceneRequestAsset,
Source: "ai-gateway-request",
})
}
func (s *Server) ensurePublicRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
RequirePublicURL: true,
UploadScene: store.FileStorageSceneUpload,
Source: "ai-gateway-form-data",
})
}
func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded decodedRequestAsset, options requestAssetOptions) (map[string]any, error) {
sum := sha256.Sum256(decoded.Bytes)
sha := hex.EncodeToString(sum[:])
contentType := strings.TrimSpace(decoded.ContentType)
if contentType == "" {
contentType = "application/octet-stream"
}
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + options.UploadScene + "\x00" + strconv.FormatBool(options.RequirePublicURL))
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + store.FileStorageSceneRequestAsset)
defer release()
now := time.Now()
if existing, ok, err := s.store.FindRequestAsset(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
} else if ok && requestAssetStillUsable(existing, now) {
ref := requestAssetRef(existing)
if !options.RequirePublicURL || requestAssetRefHasPublicURL(ref) {
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
}
return ref, nil
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
return nil, err
}
return ref, nil
}
uploadScene := strings.TrimSpace(options.UploadScene)
if uploadScene == "" {
uploadScene = store.FileStorageSceneRequestAsset
}
source := strings.TrimSpace(options.Source)
if source == "" {
source = "ai-gateway-request"
}
upload, err := s.runner.UploadFile(ctx, runner.FileUploadPayload{
Bytes: decoded.Bytes,
ContentType: contentType,
FileName: requestAssetFileName(sha, contentType),
Scene: uploadScene,
Source: source,
Scene: store.FileStorageSceneRequestAsset,
Source: "ai-gateway-request",
})
if err != nil {
return nil, err
}
storageProvider := requestAssetStorageProvider(upload)
storageChannelID, storageChannelKey := requestAssetStorageChannel(upload)
objectKey := stringFromRequestAny(upload["objectKey"])
accessScope := firstNonEmpty(stringFromRequestAny(upload["accessScope"]), "private")
url := stringFromRequestAny(upload["url"])
if url == "" {
return nil, &clients.ClientError{Code: "request_asset_upload_failed", Message: "file storage response did not include url", Retryable: false}
}
if options.RequirePublicURL && !requestAssetURLIsPublic(storageProvider, url) {
return nil, &clients.ClientError{Code: "request_asset_public_url_required", Message: "multipart image assets require a public file storage URL; enable a non-local file storage channel for uploads", Retryable: false}
}
var expiresAt *time.Time
localPath := ""
if storageProvider == "local_static" {
@@ -361,23 +419,31 @@ func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded deco
localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
}
asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
SHA256: sha,
ContentType: contentType,
ByteSize: int64(len(decoded.Bytes)),
URL: url,
StorageProvider: storageProvider,
LocalPath: localPath,
ExpiresAt: expiresAt,
SHA256: sha,
ContentType: contentType,
ByteSize: int64(len(decoded.Bytes)),
URL: url,
StorageProvider: storageProvider,
StorageChannelID: storageChannelID,
StorageChannelKey: storageChannelKey,
ObjectKey: objectKey,
AccessScope: accessScope,
LocalPath: localPath,
ExpiresAt: expiresAt,
})
if err != nil {
if store.IsUndefinedDatabaseObject(err) {
return map[string]any{
"sha256": sha,
"url": url,
"contentType": contentType,
"size": len(decoded.Bytes),
"storageProvider": storageProvider,
"expiresAt": timePtrToRFC3339(expiresAt),
"sha256": sha,
"url": url,
"contentType": contentType,
"size": len(decoded.Bytes),
"storageProvider": storageProvider,
"storageChannelId": storageChannelID,
"storageChannelKey": storageChannelKey,
"objectKey": objectKey,
"accessScope": accessScope,
"expiresAt": timePtrToRFC3339(expiresAt),
}, nil
}
return nil, err
@@ -493,27 +559,19 @@ func requestAssetWrapper(ref map[string]any) map[string]any {
func requestAssetRef(asset store.RequestAsset) map[string]any {
return map[string]any{
"sha256": asset.SHA256,
"url": asset.URL,
"contentType": asset.ContentType,
"size": asset.ByteSize,
"storageProvider": asset.StorageProvider,
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
"sha256": asset.SHA256,
"url": asset.URL,
"contentType": asset.ContentType,
"size": asset.ByteSize,
"storageProvider": asset.StorageProvider,
"storageChannelId": asset.StorageChannelID,
"storageChannelKey": asset.StorageChannelKey,
"objectKey": asset.ObjectKey,
"accessScope": asset.AccessScope,
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
}
}
func requestAssetRefHasPublicURL(ref map[string]any) bool {
return requestAssetURLIsPublic(stringFromRequestAny(ref["storageProvider"]), stringFromRequestAny(ref["url"]))
}
func requestAssetURLIsPublic(storageProvider string, url string) bool {
if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") {
return false
}
lower := strings.ToLower(strings.TrimSpace(url))
return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
}
func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
if asset.ExpiredAt != nil {
return false
@@ -543,6 +601,11 @@ func requestAssetStorageProvider(upload map[string]any) string {
return "unknown"
}
func requestAssetStorageChannel(upload map[string]any) (string, string) {
channel, _ := upload["storageChannel"].(map[string]any)
return stringFromRequestAny(channel["id"]), stringFromRequestAny(channel["channelKey"])
}
func requestAssetLocalPath(storageDir string, fileName string) string {
if strings.TrimSpace(storageDir) == "" {
storageDir = config.DefaultLocalUploadedStorageDir
@@ -41,6 +41,33 @@ func TestRequestAssetFromValueDetectsDataURLAndRawBase64(t *testing.T) {
}
}
func TestRequestAssetFromBinaryValueDetectsBufferAndByteArray(t *testing.T) {
png := []any{float64(0x89), float64('P'), float64('N'), float64('G')}
decoded, ok, err := requestAssetFromBinaryValue("image", []string{"input", "image"}, map[string]any{
"type": "Buffer",
"data": png,
"mimeType": "image/png",
}, nil)
if err != nil {
t.Fatal(err)
}
if !ok || decoded.ContentType != "image/png" || len(decoded.Bytes) != 4 || decoded.Bytes[0] != 0x89 {
t.Fatalf("unexpected Buffer asset: ok=%v decoded=%+v", ok, decoded)
}
decoded, ok, err = requestAssetFromBinaryValue("bytes", []string{"input_audio", "bytes"}, png, map[string]any{"format": "mp3"})
if err != nil {
t.Fatal(err)
}
if !ok || len(decoded.Bytes) != 4 {
t.Fatalf("unexpected byte array asset: ok=%v decoded=%+v", ok, decoded)
}
if _, ok, err := requestAssetFromBinaryValue("values", []string{"embedding", "values"}, png, nil); err != nil || ok {
t.Fatalf("ordinary numeric arrays must remain JSON: ok=%v err=%v", ok, err)
}
}
func TestMediaRequestBodySlotLimitsPreAuthWorkAndCanReleaseEarly(t *testing.T) {
server := &Server{mediaRequestBodySlots: make(chan struct{}, 2)}
var critical atomic.Int64
+24 -13
View File
@@ -5,6 +5,8 @@ import (
"fmt"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
)
func writeJSON(w http.ResponseWriter, status int, value any) {
@@ -18,22 +20,31 @@ func writeError(w http.ResponseWriter, status int, message string, codes ...stri
}
func writeErrorWithDetails(w http.ResponseWriter, status int, message string, details map[string]any, codes ...string) {
errorPayload := map[string]any{
"message": message,
"status": status,
}
code := ""
if len(codes) > 0 {
if code := strings.TrimSpace(codes[0]); code != "" {
errorPayload["code"] = code
if code == "invalid_parameter" || code == "unsupported_response_parameter" {
errorPayload["type"] = "invalid_request_error"
if _, ok := details["param"]; !ok {
errorPayload["param"] = nil
}
}
code = strings.TrimSpace(codes[0])
}
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
standard = publicErrorWithRetryAfter(standard, details)
publicerror.Observe(standard)
status = standard.HTTPStatus
errorPayload := map[string]any{
"message": standard.Message,
"status": status,
"code": standard.Code,
"category": standard.Category,
"httpStatus": standard.HTTPStatus,
"retryable": standard.Retryable,
"action": standard.Action,
"version": standard.Version,
}
if standard.Code == "invalid_parameter" || standard.Code == "unsupported_response_parameter" {
errorPayload["type"] = "invalid_request_error"
if _, ok := details["param"]; !ok {
errorPayload["param"] = nil
}
}
for key, value := range details {
for key, value := range safePublicErrorDetails(details, standard, false) {
errorPayload[key] = value
}
writeJSON(w, status, map[string]any{"error": errorPayload})
+1
View File
@@ -286,6 +286,7 @@ func NewServerWithStores(
mux.Handle("GET /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listFileStorageChannels)))
mux.Handle("POST /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createFileStorageChannel)))
mux.Handle("PATCH /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageChannel)))
mux.Handle("POST /api/admin/system/file-storage/channels/{channelID}/test", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.testFileStorageChannel)))
mux.Handle("DELETE /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteFileStorageChannel)))
mux.Handle("GET /api/admin/platforms", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listPlatforms)))
mux.Handle("POST /api/admin/platforms", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createPlatform)))
@@ -1,10 +1,13 @@
package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/url"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
@@ -74,6 +77,10 @@ func (s *Server) updateFileStorageSettings(w http.ResponseWriter, r *http.Reques
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
if legacyLocalStoragePolicy(input.ResultUploadPolicy) {
writeError(w, http.StatusBadRequest, "upload_none is no longer supported; configure an object storage channel")
return
}
settings, err := s.store.UpdateFileStorageSettings(r.Context(), input)
if err != nil {
s.logger.Error("update file storage settings failed", "error", err)
@@ -155,7 +162,7 @@ func (s *Server) updateClientCustomizationSettings(w http.ResponseWriter, r *htt
// createFileStorageChannel godoc
// @Summary 创建文件存储通道
// @Description 创建文件存储通道,当前主要用于配置 server-main OpenAPI 上传通道。
// @Description 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。
// @Tags system
// @Accept json
// @Produce json
@@ -271,6 +278,46 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request
w.WriteHeader(http.StatusNoContent)
}
// testFileStorageChannel godoc
// @Summary 测试对象存储通道
// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。
// @Tags system
// @Produce json
// @Security BearerAuth
// @Param channelID path string true "文件存储通道 ID"
// @Success 200 {object} FileStorageChannelTestResponse
// @Failure 400 {object} ErrorEnvelope
// @Failure 401 {object} ErrorEnvelope
// @Failure 403 {object} ErrorEnvelope
// @Failure 404 {object} ErrorEnvelope
// @Failure 503 {object} ErrorEnvelope
// @Router /api/admin/system/file-storage/channels/{channelID}/test [post]
func (s *Server) testFileStorageChannel(w http.ResponseWriter, r *http.Request) {
channel, err := s.store.GetFileStorageChannel(r.Context(), r.PathValue("channelID"))
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusNotFound, "file storage channel not found")
return
}
s.logger.Error("get file storage channel for test failed", "error", err)
writeError(w, http.StatusInternalServerError, "get file storage channel failed")
return
}
if channel.Provider != "aliyun_oss" && channel.Provider != "s3" {
writeError(w, http.StatusBadRequest, "connection test is supported for aliyun_oss and s3 channels", "invalid_parameter")
return
}
result, err := s.runner.TestFileStorageChannel(r.Context(), channel)
if err != nil {
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(r.Context()), channel.ID, err.Error())
s.logger.Warn("file storage channel test failed", "channel_id", channel.ID, "provider", channel.Provider, "error", err)
writeError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
return
}
_ = s.store.MarkFileStorageChannelSuccess(context.WithoutCancel(r.Context()), channel.ID)
writeJSON(w, http.StatusOK, result)
}
func validateFileStorageChannelInput(input store.FileStorageChannelInput, existing *store.FileStorageChannel) string {
provider := strings.ToLower(strings.TrimSpace(input.Provider))
if provider == "" {
@@ -286,6 +333,9 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
if status != "enabled" && status != "disabled" {
return "status must be enabled or disabled"
}
if provider != "server_main_openapi" && provider != "aliyun_oss" && provider != "s3" {
return "provider must be server_main_openapi, aliyun_oss or s3"
}
if provider == "server_main_openapi" {
hasAPIKey := false
if input.APIKey != nil {
@@ -297,5 +347,83 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
return "server-main OpenAPI channel requires API key before enabling"
}
}
if provider == "aliyun_oss" || provider == "s3" {
if fileStorageConfigContainsCredential(input.Config) {
return "object storage credentials must use the write-only credential fields, not config"
}
endpoint := fileStorageConfigString(input.Config, "endpoint")
if endpoint == "" || fileStorageConfigString(input.Config, "region") == "" || fileStorageConfigString(input.Config, "bucket") == "" {
return "object storage channel requires config.endpoint, config.region and config.bucket"
}
if !validFileStorageBaseURL(endpoint) {
return "object storage config.endpoint must be an http or https URL without embedded credentials"
}
if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
}
accessKeyID := input.AccessKeyID
if accessKeyID == nil {
accessKeyID = input.AccessKey
}
accessKeySecret := input.AccessKeySecret
if accessKeySecret == nil {
accessKeySecret = input.SecretKey
}
hasAccessKeyID := fileStorageCredentialPresent(accessKeyID, existing, func(item *store.FileStorageChannel) string { return item.AccessKeyID })
hasAccessKeySecret := fileStorageCredentialPresent(accessKeySecret, existing, func(item *store.FileStorageChannel) string { return item.AccessKeySecret })
if status == "enabled" && (!hasAccessKeyID || !hasAccessKeySecret) {
return "object storage channel requires accessKeyId and accessKeySecret before enabling"
}
}
return ""
}
func validFileStorageBaseURL(value string) bool {
parsed, err := url.Parse(strings.TrimSpace(value))
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
}
func fileStorageConfigContainsCredential(value any) bool {
switch typed := value.(type) {
case map[string]any:
for key, item := range typed {
normalized := strings.NewReplacer("_", "", "-", "", ".", "").Replace(strings.ToLower(strings.TrimSpace(key)))
switch normalized {
case "apikey", "accesskey", "accesskeyid", "accesskeysecret", "secretkey", "sessiontoken", "ststoken", "password", "credential", "credentials", "authorization":
return true
}
if fileStorageConfigContainsCredential(item) {
return true
}
}
case []any:
for _, item := range typed {
if fileStorageConfigContainsCredential(item) {
return true
}
}
}
return false
}
func fileStorageConfigString(config map[string]any, key string) string {
value, _ := config[key].(string)
return strings.TrimSpace(value)
}
func fileStorageCredentialPresent(input *string, existing *store.FileStorageChannel, current func(*store.FileStorageChannel) string) bool {
if input != nil {
return strings.TrimSpace(*input) != ""
}
return existing != nil && strings.TrimSpace(current(existing)) != ""
}
func legacyLocalStoragePolicy(value string) bool {
normalized := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(value)), "-", "_")
switch normalized {
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
return true
default:
return false
}
}
@@ -0,0 +1,55 @@
package httpapi
import (
"strings"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestValidateObjectStorageChannelAcceptsS3CredentialAliases(t *testing.T) {
accessKey := "access"
secretKey := "secret"
input := store.FileStorageChannelInput{
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
AccessKey: &accessKey, SecretKey: &secretKey,
Config: map[string]any{
"endpoint": "https://s3.example.com", "region": "us-east-1", "bucket": "media",
},
}
if message := validateFileStorageChannelInput(input, nil); message != "" {
t.Fatalf("valid S3 channel rejected: %s", message)
}
}
func TestValidateObjectStorageChannelRejectsCredentialsInConfig(t *testing.T) {
accessKeyID := "access"
accessKeySecret := "secret"
input := store.FileStorageChannelInput{
ChannelKey: "oss-primary", Name: "OSS primary", Provider: "aliyun_oss", Status: "enabled",
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
Config: map[string]any{
"endpoint": "https://oss-cn-hangzhou.aliyuncs.com", "region": "cn-hangzhou", "bucket": "media",
"nested": map[string]any{"session_token": "must-not-be-public"},
},
}
message := validateFileStorageChannelInput(input, nil)
if !strings.Contains(message, "write-only credential fields") {
t.Fatalf("credential-bearing config was accepted: %q", message)
}
}
func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) {
accessKeyID := "access"
accessKeySecret := "secret"
input := store.FileStorageChannelInput{
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
Config: map[string]any{
"endpoint": "https://user:password@s3.example.com", "region": "us-east-1", "bucket": "media",
},
}
if message := validateFileStorageChannelInput(input, nil); !strings.Contains(message, "without embedded credentials") {
t.Fatalf("credentialed endpoint was accepted: %q", message)
}
}
@@ -69,6 +69,10 @@ func writeIdempotentTaskReplay(w http.ResponseWriter, task store.GatewayTask, co
func storedTaskErrorStatus(code string) int {
switch strings.TrimSpace(code) {
case "storage_write_failed", "storage_read_failed", "storage_config_invalid", "storage_auth_failed":
return http.StatusServiceUnavailable
case "binary_result_expired", "result_expired", "result_unavailable":
return http.StatusGone
case "pricing_unavailable", "response_chain_unavailable", "billing_hold":
return http.StatusServiceUnavailable
case "insufficient_balance":
+1 -1
View File
@@ -255,7 +255,7 @@ func (s *Server) uploadImageEditMultipartAsset(ctx context.Context, field string
if !strings.HasPrefix(strings.ToLower(contentType), "image/") {
return nil, &clients.ClientError{Code: "invalid_multipart_image", Message: "image edit multipart files must be images", Retryable: false}
}
ref, err := s.ensurePublicRequestAsset(ctx, decodedRequestAsset{
ref, err := s.ensureRequestAsset(ctx, decodedRequestAsset{
Bytes: payload,
ContentType: contentType,
})
@@ -352,7 +352,7 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
response["usage"] = legacyUsage
}
if task.Status == "failed" || task.Status == "cancelled" {
response["error"] = map[string]any{"code": firstNonEmpty(task.ErrorCode, strings.ToUpper(task.Status)), "message": firstNonEmpty(task.ErrorMessage, task.Error, task.Message)}
response["error"] = publicErrorMap(publicTaskError(task))
}
return response
}
@@ -431,10 +431,6 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
} else if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
status = http.StatusBadRequest
}
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, clients.ProtocolVolcesContents) {
writeWireResponse(w, wire)
return
}
writeVolcesError(w, status, err.Error(), clients.ErrorCode(err))
}