feat(storage): 统一二进制对象存储与公开错误
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换,保留 server-main 兼容与环境 OSS 内存通道。 将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化,生产路径不再写入本机静态目录,历史本地资源仅保留只读兼容。 引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出,同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。 验证:go test ./... -count=1;go vet ./...;pnpm lint;pnpm test;pnpm build;pnpm openapi;tests/ci/migrations-test.sh。
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
@@ -97,11 +98,16 @@ func writeWireResponse(w http.ResponseWriter, wire *clients.WireResponse) {
|
||||
}
|
||||
|
||||
func writeProtocolError(w http.ResponseWriter, protocol string, status int, message string, details map[string]any, code string) {
|
||||
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||
standard = publicErrorWithRetryAfter(standard, details)
|
||||
publicerror.Observe(standard)
|
||||
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
|
||||
details = safePublicErrorDetails(details, standard, true)
|
||||
switch protocol {
|
||||
case clients.ProtocolGeminiGenerateContent:
|
||||
writeGeminiError(w, status, message, details, code)
|
||||
case clients.ProtocolVolcesContents:
|
||||
writeVolcesError(w, status, message, code)
|
||||
writeVolcesPublicError(w, standard)
|
||||
case clients.ProtocolKlingV1Omni, clients.ProtocolKlingV2Omni:
|
||||
writeKelingCompatError(w, "", newKelingCompatError(status, kelingCompatBusinessCode(code, message), message))
|
||||
default:
|
||||
@@ -185,11 +191,11 @@ func googleRPCStatus(status int, code string) string {
|
||||
}
|
||||
|
||||
func writeVolcesError(w http.ResponseWriter, status int, message string, code string) {
|
||||
if strings.TrimSpace(code) == "" {
|
||||
code = http.StatusText(status)
|
||||
}
|
||||
writeJSON(w, status, map[string]any{"error": map[string]any{
|
||||
"code": code,
|
||||
"message": message,
|
||||
}})
|
||||
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||
publicerror.Observe(standard)
|
||||
writeVolcesPublicError(w, standard)
|
||||
}
|
||||
|
||||
func writeVolcesPublicError(w http.ResponseWriter, standard publicerror.Error) {
|
||||
writeJSON(w, standard.HTTPStatus, map[string]any{"error": standard})
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
|
||||
func TestProtocolErrorsUseCompatibleShapesWithStandardPublicErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
protocol string
|
||||
@@ -78,10 +78,10 @@ func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
|
||||
name: "volces", protocol: clients.ProtocolVolcesContents, status: http.StatusBadGateway,
|
||||
assertBody: func(t *testing.T, body map[string]any) {
|
||||
errorBody := requireObject(t, body["error"])
|
||||
if errorBody["code"] != "upstream_submission_unknown" {
|
||||
if errorBody["code"] != "upstream_submission_unknown" || errorBody["httpStatus"] != float64(http.StatusBadGateway) || errorBody["retryable"] != true {
|
||||
t.Fatalf("unexpected Volces error: %+v", body)
|
||||
}
|
||||
assertNoKeys(t, errorBody, "status", "retryable", "taskId", "gateway_status")
|
||||
assertNoKeys(t, errorBody, "status", "taskId", "gateway_status")
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -135,6 +135,44 @@ func TestWireResponsePassthroughPreservesStatusUnknownFieldsAndAllowedHeaders(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompatibilityErrorWritersNeverExposeTransportDetails(t *testing.T) {
|
||||
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
|
||||
tests := []struct {
|
||||
name string
|
||||
write func(http.ResponseWriter)
|
||||
}{
|
||||
{name: "openai", write: func(w http.ResponseWriter) {
|
||||
writeProtocolError(w, clients.ProtocolOpenAIResponses, http.StatusOK, raw, map[string]any{
|
||||
"provider": "secret-provider", "endpoint": "https://private.example.invalid", "bucket": "private-bucket",
|
||||
}, "response_read_error")
|
||||
}},
|
||||
{name: "volces", write: func(w http.ResponseWriter) {
|
||||
writeVolcesError(w, http.StatusOK, raw, "response_read_error")
|
||||
}},
|
||||
{name: "kling", write: func(w http.ResponseWriter) {
|
||||
writeKlingCompatError(w, http.StatusOK, raw, "response_read_error")
|
||||
}},
|
||||
{name: "keling", write: func(w http.ResponseWriter) {
|
||||
writeKelingCompatError(w, "request-1", newKelingCompatError(http.StatusOK, 5001, raw))
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
recorder := httptest.NewRecorder()
|
||||
test.write(recorder)
|
||||
if recorder.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
body := recorder.Body.String()
|
||||
if strings.Contains(body, "10.42.0.72") || strings.Contains(body, "47.77.191.126") ||
|
||||
strings.Contains(body, "secret-provider") || strings.Contains(body, "private.example.invalid") || strings.Contains(body, "private-bucket") ||
|
||||
!strings.Contains(body, "upstream_connection_interrupted") {
|
||||
t.Fatalf("transport details were not standardized: %s", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompatibilityStatusMappings(t *testing.T) {
|
||||
for internal, want := range map[string]string{
|
||||
"queued": "queued", "running": "running", "succeeded": "succeeded", "failed": "failed", "cancelled": "cancelled",
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
@@ -58,14 +59,21 @@ func writeEasyAIAsyncError(w http.ResponseWriter, status int, message string, de
|
||||
if len(codes) > 0 {
|
||||
code = strings.TrimSpace(codes[0])
|
||||
}
|
||||
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||
standard = publicErrorWithRetryAfter(standard, details)
|
||||
publicerror.Observe(standard)
|
||||
status, message, code = standard.HTTPStatus, standard.Message, standard.Code
|
||||
errorPayload := map[string]any{
|
||||
"message": message,
|
||||
"status": status,
|
||||
"message": message,
|
||||
"status": status,
|
||||
"code": code,
|
||||
"category": standard.Category,
|
||||
"httpStatus": standard.HTTPStatus,
|
||||
"retryable": standard.Retryable,
|
||||
"action": standard.Action,
|
||||
"version": standard.Version,
|
||||
}
|
||||
if code != "" {
|
||||
errorPayload["code"] = code
|
||||
}
|
||||
for key, value := range details {
|
||||
for key, value := range safePublicErrorDetails(details, standard, false) {
|
||||
errorPayload[key] = value
|
||||
}
|
||||
response := map[string]any{
|
||||
@@ -158,8 +166,14 @@ func easyAITaskResultResponse(task store.GatewayTask) map[string]any {
|
||||
if message != "" {
|
||||
response["message"] = message
|
||||
}
|
||||
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" {
|
||||
response["code"] = code
|
||||
if code := firstNonEmpty(task.ErrorCode, easyAIString(cleanResult["code"])); code != "" || status == "failed" {
|
||||
standard := publicTaskError(task)
|
||||
if code != "" && task.ErrorCode == "" {
|
||||
standard = publicerror.WithIDs(publicerror.FromFields(code, message, 0, false), task.RequestID, task.ID)
|
||||
}
|
||||
response["code"] = standard.Code
|
||||
response["message"] = standard.Message
|
||||
response["error"] = standard
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ const maxGatewayUploadBytes = 256 << 20
|
||||
|
||||
// uploadFile godoc
|
||||
// @Summary 上传文件
|
||||
// @Description 上传文件到配置的文件存储通道;没有启用通道时回退到本地静态上传目录。单文件最大 256MiB。
|
||||
// @Description 上传文件到配置的对象存储通道;所有通道失败时返回标准化存储错误,不写入本机静态目录。单文件最大 256MiB。
|
||||
// @Tags files
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
@@ -55,11 +55,8 @@ func (s *Server) uploadFile(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Error("upload file failed", "error", err)
|
||||
status := http.StatusBadGateway
|
||||
if clients.ErrorCode(err) == "upload_no_channel" {
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
writeError(w, status, err.Error())
|
||||
status := statusFromRunError(err)
|
||||
writeError(w, status, err.Error(), clients.ErrorCode(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, easyAIFileUploadResponse(upload))
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
@@ -159,7 +160,7 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
|
||||
if err != nil {
|
||||
s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err)
|
||||
status := http.StatusBadRequest
|
||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
|
||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
writeGeminiTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
||||
@@ -318,14 +319,13 @@ func (s *Server) writeGeminiGenerateContentStream(runCtx context.Context, w http
|
||||
}
|
||||
applyRunErrorHeaders(w, runErr)
|
||||
if !nativePassthrough && !convertedFrames {
|
||||
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, clients.ProtocolGeminiGenerateContent) {
|
||||
writeWireResponse(w, wire)
|
||||
return
|
||||
}
|
||||
writeGeminiTaskError(statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
|
||||
} else if !nativePassthrough {
|
||||
status := statusFromRunError(runErr)
|
||||
writeGeminiSSEFrame(w, geminiErrorEnvelope(status, runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr)))
|
||||
standard := publicerror.FromFields(runErrorCode(runErr), runErrorMessage(runErr), statusFromRunError(runErr), clients.IsRetryable(runErr))
|
||||
standard = publicErrorWithRetryAfter(standard, runErrorDetails(runErr))
|
||||
publicerror.Observe(standard)
|
||||
details := safePublicErrorDetails(runErrorDetails(runErr), standard, true)
|
||||
writeGeminiSSEFrame(w, geminiErrorEnvelope(standard.HTTPStatus, standard.Message, details, standard.Code))
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
@@ -683,10 +683,7 @@ func (s *Server) finalizeGeminiFilesUpload(w http.ResponseWriter, r *http.Reques
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Error("Gemini files upload failed", "error", err)
|
||||
status := http.StatusBadGateway
|
||||
if clients.ErrorCode(err) == "upload_no_channel" {
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
status := statusFromRunError(err)
|
||||
writeProtocolError(w, clients.ProtocolGeminiGenerateContent, status, err.Error(), nil, clients.ErrorCode(err))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1154,7 +1154,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
|
||||
body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind)
|
||||
if err != nil {
|
||||
status := http.StatusBadRequest
|
||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
|
||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" || code == "request_asset_public_url_required" {
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
||||
@@ -1195,7 +1195,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
|
||||
if err != nil {
|
||||
s.logger.Warn("prepare task request failed", "kind", kind, "error", err)
|
||||
status := http.StatusBadRequest
|
||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || code == "request_asset_upload_failed" {
|
||||
if code := clients.ErrorCode(err); strings.HasPrefix(code, "upload_") || strings.HasPrefix(code, "storage_") || code == "request_asset_upload_failed" {
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
writeTaskError(status, err.Error(), nil, clients.ErrorCode(err))
|
||||
@@ -1556,10 +1556,6 @@ func writeProtocolCompatibleTaskResponse(runCtx context.Context, w http.Response
|
||||
return
|
||||
}
|
||||
applyRunErrorHeaders(w, runErr)
|
||||
if wire := clients.ErrorWireResponse(runErr); wireResponseMatches(wire, targetProtocol) {
|
||||
writeWireResponse(w, wire)
|
||||
return
|
||||
}
|
||||
if targetProtocol != "" {
|
||||
writeProtocolError(w, targetProtocol, statusFromRunError(runErr), runErrorMessage(runErr), runErrorDetails(runErr), runErrorCode(runErr))
|
||||
return
|
||||
@@ -1673,6 +1669,8 @@ func modelNameFromValue(value any) string {
|
||||
|
||||
func statusFromRunError(err error) int {
|
||||
switch {
|
||||
case clients.ErrorCode(err) == "storage_write_failed" || clients.ErrorCode(err) == "storage_read_failed" || clients.ErrorCode(err) == "storage_config_invalid" || clients.ErrorCode(err) == "storage_auth_failed":
|
||||
return http.StatusServiceUnavailable
|
||||
case clients.ErrorCode(err) == "binary_result_expired":
|
||||
return http.StatusGone
|
||||
case clients.ErrorCode(err) == "binary_result_corrupted" || clients.ErrorCode(err) == "result_binary_not_materialized":
|
||||
@@ -1949,7 +1947,7 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"items": result.Items,
|
||||
"items": publicTaskList(result.Items),
|
||||
"total": result.Total,
|
||||
"page": result.Page,
|
||||
"pageSize": result.PageSize,
|
||||
@@ -2034,7 +2032,7 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
|
||||
writeStoredBinaryResultError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, task)
|
||||
writeJSON(w, http.StatusOK, publicGatewayTask(task))
|
||||
return
|
||||
}
|
||||
if store.IsNotFound(err) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
@@ -66,17 +67,19 @@ type KelingOmniWatermarkInfo struct {
|
||||
}
|
||||
|
||||
type KelingCompatibleEnvelope struct {
|
||||
Code int `json:"code" example:"0"`
|
||||
Message string `json:"message" example:"SUCCEED"`
|
||||
RequestID string `json:"request_id"`
|
||||
Data any `json:"data,omitempty"`
|
||||
Code int `json:"code" example:"0"`
|
||||
Message string `json:"message" example:"SUCCEED"`
|
||||
RequestID string `json:"request_id"`
|
||||
Data any `json:"data,omitempty"`
|
||||
Error *publicerror.Error `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type kelingCompatError struct {
|
||||
HTTPStatus int
|
||||
Code int
|
||||
Message string
|
||||
RequestID string
|
||||
HTTPStatus int
|
||||
Code int
|
||||
Message string
|
||||
RequestID string
|
||||
PublicError *publicerror.Error
|
||||
}
|
||||
|
||||
func (e *kelingCompatError) Error() string {
|
||||
@@ -200,10 +203,6 @@ func (s *Server) createKelingOmniVideo(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
task, createErr = s.waitForCompatibilitySubmission(r, task)
|
||||
if createErr != nil {
|
||||
if wire := clients.ErrorWireResponse(createErr); wireResponseMatches(wire, clients.ProtocolKlingV1Omni) {
|
||||
writeWireResponse(w, wire)
|
||||
return
|
||||
}
|
||||
writeKelingCompatError(w, requestID, kelingCompatGatewayError(createErr))
|
||||
return
|
||||
}
|
||||
@@ -690,11 +689,13 @@ func kelingCompatTaskData(task store.GatewayTask) map[string]any {
|
||||
"enabled": kelingCompatTaskWatermark(task.Request),
|
||||
},
|
||||
}
|
||||
if message := kelingCompatTaskMessage(task); message != "" {
|
||||
data["task_status_msg"] = message
|
||||
}
|
||||
if kelingCompatTaskStatus(task.Status) == "failed" {
|
||||
data["task_status_code"] = kelingCompatBusinessCode(task.ErrorCode, kelingCompatTaskMessage(task))
|
||||
standard := publicTaskError(task)
|
||||
data["task_status_msg"] = standard.Message
|
||||
data["task_status_code"] = kelingCompatBusinessCode(standard.Code, standard.Message)
|
||||
data["error"] = standard
|
||||
} else if message := kelingCompatTaskMessage(task); message != "" {
|
||||
data["task_status_msg"] = message
|
||||
}
|
||||
if videos := kelingCompatTaskVideos(task.Result); len(videos) > 0 {
|
||||
data["task_result"] = map[string]any{"videos": videos}
|
||||
@@ -770,6 +771,9 @@ func firstKelingCompatValue(values ...any) any {
|
||||
}
|
||||
|
||||
func kelingCompatTaskMessage(task store.GatewayTask) string {
|
||||
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" || task.Error != "" {
|
||||
return publicTaskError(task).Message
|
||||
}
|
||||
return strings.TrimSpace(firstNonEmpty(task.ErrorMessage, task.Error, task.Message))
|
||||
}
|
||||
|
||||
@@ -825,8 +829,11 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
|
||||
return newKelingCompatError(http.StatusInternalServerError, 5000, "unknown gateway error")
|
||||
}
|
||||
codeText := clients.ErrorCode(err)
|
||||
businessCode := kelingCompatBusinessCode(codeText, err.Error())
|
||||
status := http.StatusInternalServerError
|
||||
status := statusFromRunError(err)
|
||||
standard := publicerror.FromFields(codeText, err.Error(), status, clients.IsRetryable(err))
|
||||
publicerror.Observe(standard)
|
||||
businessCode := kelingCompatBusinessCode(standard.Code, standard.Message)
|
||||
status = standard.HTTPStatus
|
||||
switch businessCode {
|
||||
case 1101:
|
||||
status = http.StatusPaymentRequired
|
||||
@@ -841,7 +848,9 @@ func kelingCompatGatewayError(err error) *kelingCompatError {
|
||||
case 5001:
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
return newKelingCompatError(status, businessCode, err.Error())
|
||||
result := newKelingCompatError(status, businessCode, standard.Message)
|
||||
result.PublicError = &standard
|
||||
return result
|
||||
}
|
||||
|
||||
func kelingCompatBusinessCode(errorCode string, message string) int {
|
||||
@@ -923,9 +932,18 @@ func writeKelingCompatError(w http.ResponseWriter, requestID string, err *keling
|
||||
if status == 0 {
|
||||
status = http.StatusInternalServerError
|
||||
}
|
||||
standard := err.PublicError
|
||||
if standard == nil {
|
||||
value := publicerror.FromFields("", err.Message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||
value = publicerror.WithIDs(value, requestID, "")
|
||||
publicerror.Observe(value)
|
||||
standard = &value
|
||||
}
|
||||
status = standard.HTTPStatus
|
||||
writeJSON(w, status, KelingCompatibleEnvelope{
|
||||
Code: err.Code,
|
||||
Message: err.Message,
|
||||
Message: standard.Message,
|
||||
RequestID: requestID,
|
||||
Error: standard,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
@@ -191,10 +192,6 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
|
||||
}
|
||||
task, err = s.waitForCompatibilitySubmission(r, task)
|
||||
if err != nil {
|
||||
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, targetProtocol) {
|
||||
writeWireResponse(w, wire)
|
||||
return
|
||||
}
|
||||
writeKlingCompatError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
|
||||
return
|
||||
}
|
||||
@@ -667,7 +664,9 @@ func klingV1TaskData(task store.GatewayTask) map[string]any {
|
||||
"created_at": task.CreatedAt.UnixMilli(), "updated_at": task.UpdatedAt.UnixMilli(),
|
||||
}
|
||||
if task.ErrorMessage != "" || task.Error != "" {
|
||||
data["task_status_msg"] = firstNonEmpty(task.ErrorMessage, task.Error)
|
||||
standard := publicTaskError(task)
|
||||
data["task_status_msg"] = standard.Message
|
||||
data["error"] = standard
|
||||
}
|
||||
if watermarkInfo, ok := task.Request["watermark_info"].(map[string]any); ok {
|
||||
data["watermark_info"] = watermarkInfo
|
||||
@@ -698,7 +697,9 @@ func klingV2TaskData(task store.GatewayTask) map[string]any {
|
||||
"external_id": task.ExternalTaskID,
|
||||
}
|
||||
if message := firstNonEmpty(task.ErrorMessage, task.Error); message != "" {
|
||||
data["message"] = message
|
||||
standard := publicTaskError(task)
|
||||
data["message"] = standard.Message
|
||||
data["error"] = standard
|
||||
}
|
||||
if outputs := klingV2Outputs(task); len(outputs) > 0 {
|
||||
data["outputs"] = outputs
|
||||
@@ -827,14 +828,14 @@ func decodeKlingJSON(r *http.Request, target any) error {
|
||||
}
|
||||
|
||||
func writeKlingCompatError(w http.ResponseWriter, status int, message string, code string) {
|
||||
if strings.TrimSpace(code) == "" {
|
||||
code = "invalid_request"
|
||||
}
|
||||
writeJSON(w, status, map[string]any{
|
||||
"code": klingCompatErrorCode(status),
|
||||
"message": message,
|
||||
"request_id": "",
|
||||
"error": code,
|
||||
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||
publicerror.Observe(standard)
|
||||
writeJSON(w, standard.HTTPStatus, map[string]any{
|
||||
"code": klingCompatErrorCode(standard.HTTPStatus),
|
||||
"message": standard.Message,
|
||||
"request_id": "",
|
||||
"error": standard.Code,
|
||||
"public_error": standard,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package httpapi
|
||||
|
||||
import (
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
@@ -32,11 +33,19 @@ type ErrorEnvelope struct {
|
||||
}
|
||||
|
||||
type ErrorPayload struct {
|
||||
Message string `json:"message" example:"invalid json body"`
|
||||
Status int `json:"status" example:"400"`
|
||||
Code string `json:"code,omitempty" example:"rate_limit"`
|
||||
Type string `json:"type,omitempty" example:"invalid_request_error"`
|
||||
Param any `json:"param,omitempty"`
|
||||
Message string `json:"message" example:"invalid json body"`
|
||||
Status int `json:"status" example:"400"`
|
||||
Code string `json:"code,omitempty" example:"rate_limit"`
|
||||
Category string `json:"category,omitempty" example:"rate_limit"`
|
||||
HTTPStatus int `json:"httpStatus,omitempty" example:"429"`
|
||||
Retryable bool `json:"retryable" example:"true"`
|
||||
Action string `json:"action,omitempty" example:"retry_after"`
|
||||
Version string `json:"version,omitempty" example:"v1"`
|
||||
RetryAfterSeconds int `json:"retryAfterSeconds,omitempty" example:"2"`
|
||||
RequestID string `json:"requestId,omitempty"`
|
||||
TaskID string `json:"taskId,omitempty"`
|
||||
Type string `json:"type,omitempty" example:"invalid_request_error"`
|
||||
Param any `json:"param,omitempty"`
|
||||
}
|
||||
|
||||
type OpenAIErrorEnvelope struct {
|
||||
@@ -62,12 +71,7 @@ type GeminiErrorStatus struct {
|
||||
}
|
||||
|
||||
type VolcesErrorEnvelope struct {
|
||||
Error VolcesErrorPayload `json:"error"`
|
||||
}
|
||||
|
||||
type VolcesErrorPayload struct {
|
||||
Code string `json:"code" example:"invalid_parameter"`
|
||||
Message string `json:"message" example:"model is required"`
|
||||
Error publicerror.Error `json:"error"`
|
||||
}
|
||||
|
||||
type VolcesContentsGenerationTaskResponse struct {
|
||||
@@ -82,10 +86,11 @@ type VolcesContentsGenerationTaskResponse struct {
|
||||
}
|
||||
|
||||
type KlingErrorEnvelope struct {
|
||||
Code int `json:"code" example:"1001"`
|
||||
Message string `json:"message" example:"invalid parameter"`
|
||||
RequestID string `json:"request_id"`
|
||||
Error string `json:"error,omitempty" example:"invalid_parameter"`
|
||||
Code int `json:"code" example:"1001"`
|
||||
Message string `json:"message" example:"invalid parameter"`
|
||||
RequestID string `json:"request_id"`
|
||||
Error string `json:"error,omitempty" example:"invalid_parameter"`
|
||||
PublicError *publicerror.Error `json:"public_error,omitempty"`
|
||||
}
|
||||
|
||||
type AuthResponse struct {
|
||||
@@ -232,6 +237,14 @@ type FileStorageChannelListResponse struct {
|
||||
Items []store.FileStorageChannel `json:"items"`
|
||||
}
|
||||
|
||||
type FileStorageChannelTestResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
PutSucceeded bool `json:"putSucceeded"`
|
||||
HeadSucceeded bool `json:"headSucceeded"`
|
||||
DeleteSucceeded bool `json:"deleteSucceeded"`
|
||||
DurationMS int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
type FileUploadResponse struct {
|
||||
ID string `json:"id,omitempty" example:"file_abc123"`
|
||||
URL string `json:"url,omitempty" example:"/static/uploaded/upload-abc123.png"`
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func publicGatewayTask(task store.GatewayTask) store.GatewayTask {
|
||||
if task.Status == "failed" || task.Status == "cancelled" || task.ErrorCode != "" || task.ErrorMessage != "" {
|
||||
value := publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error), storedTaskErrorStatus(task.ErrorCode), false)
|
||||
if task.PublicError != nil && task.PublicError.Code != "" {
|
||||
value = *task.PublicError
|
||||
}
|
||||
value = publicerror.WithIDs(value, task.RequestID, task.ID)
|
||||
publicerror.Observe(value)
|
||||
task.PublicError = &value
|
||||
task.ErrorCode = value.Code
|
||||
task.ErrorMessage = value.Message
|
||||
task.Error = value.Message
|
||||
}
|
||||
for index := range task.Attempts {
|
||||
attempt := &task.Attempts[index]
|
||||
if attempt.ErrorCode == "" && attempt.ErrorMessage == "" {
|
||||
continue
|
||||
}
|
||||
status := attempt.StatusCode
|
||||
if status <= 0 {
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
value := publicerror.FromFields(attempt.ErrorCode, attempt.ErrorMessage, status, attempt.Retryable)
|
||||
if attempt.PublicError != nil && attempt.PublicError.Code != "" {
|
||||
value = *attempt.PublicError
|
||||
}
|
||||
value = publicerror.WithIDs(value, attempt.RequestID, task.ID)
|
||||
attempt.PublicError = &value
|
||||
attempt.ErrorCode = value.Code
|
||||
attempt.ErrorMessage = value.Message
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
func publicTaskError(task store.GatewayTask) publicerror.Error {
|
||||
if task.PublicError != nil && task.PublicError.Code != "" {
|
||||
return publicerror.WithIDs(*task.PublicError, task.RequestID, task.ID)
|
||||
}
|
||||
status := storedTaskErrorStatus(task.ErrorCode)
|
||||
if status <= 0 {
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
return publicerror.WithIDs(publicerror.FromFields(task.ErrorCode, firstNonEmpty(task.ErrorMessage, task.Error, task.Message), status, false), task.RequestID, task.ID)
|
||||
}
|
||||
|
||||
func publicTaskList(items []store.GatewayTask) []store.GatewayTask {
|
||||
out := make([]store.GatewayTask, len(items))
|
||||
for index, task := range items {
|
||||
out[index] = publicGatewayTask(task)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func publicErrorMap(value publicerror.Error) map[string]any {
|
||||
out := map[string]any{
|
||||
"code": value.Code, "message": value.Message, "category": value.Category,
|
||||
"httpStatus": value.HTTPStatus, "retryable": value.Retryable, "action": value.Action,
|
||||
"version": value.Version,
|
||||
}
|
||||
if value.RetryAfterSeconds > 0 {
|
||||
out["retryAfterSeconds"] = value.RetryAfterSeconds
|
||||
}
|
||||
if requestID := strings.TrimSpace(value.RequestID); requestID != "" {
|
||||
out["requestId"] = requestID
|
||||
}
|
||||
if taskID := strings.TrimSpace(value.TaskID); taskID != "" {
|
||||
out["taskId"] = taskID
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func publicErrorWithRetryAfter(value publicerror.Error, details map[string]any) publicerror.Error {
|
||||
if value.RetryAfterSeconds > 0 || details == nil {
|
||||
return value
|
||||
}
|
||||
switch typed := details["retryAfterSeconds"].(type) {
|
||||
case int:
|
||||
value.RetryAfterSeconds = typed
|
||||
case int32:
|
||||
value.RetryAfterSeconds = int(typed)
|
||||
case int64:
|
||||
value.RetryAfterSeconds = int(typed)
|
||||
case float64:
|
||||
value.RetryAfterSeconds = int(typed)
|
||||
}
|
||||
if value.RetryAfterSeconds < 0 {
|
||||
value.RetryAfterSeconds = 0
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func safePublicErrorDetails(details map[string]any, value publicerror.Error, includePublicError bool) map[string]any {
|
||||
out := map[string]any{}
|
||||
for _, key := range []string{"param", "retryAfterSeconds", "recoveryAt", "rateLimit", "pricing"} {
|
||||
if item, ok := details[key]; ok && item != nil {
|
||||
out[key] = item
|
||||
}
|
||||
}
|
||||
if value.Category == "request" {
|
||||
for _, key := range []string{"reason", "diagnosticId"} {
|
||||
if item, ok := details[key]; ok && item != nil {
|
||||
out[key] = item
|
||||
}
|
||||
}
|
||||
}
|
||||
if includePublicError {
|
||||
out["publicError"] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -38,12 +37,6 @@ type decodedRequestAsset struct {
|
||||
ContentType string
|
||||
}
|
||||
|
||||
type requestAssetOptions struct {
|
||||
RequirePublicURL bool
|
||||
UploadScene string
|
||||
Source string
|
||||
}
|
||||
|
||||
type requestAssetLock struct {
|
||||
mu sync.Mutex
|
||||
refs int
|
||||
@@ -115,6 +108,19 @@ func (s *Server) prepareRequestAssetRefs(ctx context.Context, body map[string]an
|
||||
}
|
||||
|
||||
func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path []string, siblings map[string]any) (any, error) {
|
||||
if decoded, ok, err := requestAssetFromBinaryValue(requestAssetPathKey(path), path, value, siblings); err != nil {
|
||||
return nil, err
|
||||
} else if ok {
|
||||
if err := s.acquireMediaRequestSlot(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer s.releaseMediaRequestSlot()
|
||||
ref, err := s.ensureRequestAsset(ctx, decoded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return requestAssetWrapper(ref), nil
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
if typed["assetRef"] != nil {
|
||||
@@ -156,6 +162,83 @@ func (s *Server) prepareRequestAssetValue(ctx context.Context, value any, path [
|
||||
}
|
||||
}
|
||||
|
||||
func requestAssetFromBinaryValue(key string, path []string, value any, siblings map[string]any) (decodedRequestAsset, bool, error) {
|
||||
var payload []byte
|
||||
contentType := ""
|
||||
switch typed := value.(type) {
|
||||
case []byte:
|
||||
payload = append([]byte(nil), typed...)
|
||||
case map[string]any:
|
||||
if !strings.EqualFold(strings.TrimSpace(stringFromRequestAny(typed["type"])), "buffer") {
|
||||
return decodedRequestAsset{}, false, nil
|
||||
}
|
||||
contentType = firstNonEmptyRequestString(typed, "contentType", "mimeType", "mime_type")
|
||||
switch data := typed["data"].(type) {
|
||||
case []byte:
|
||||
payload = append([]byte(nil), data...)
|
||||
case []any:
|
||||
var ok bool
|
||||
payload, ok = requestBytesFromNumberArray(data)
|
||||
if !ok {
|
||||
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data must contain byte values"))
|
||||
}
|
||||
default:
|
||||
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("Buffer data is required"))
|
||||
}
|
||||
case []any:
|
||||
if !strictRequestBinaryArrayField(key, path) {
|
||||
return decodedRequestAsset{}, false, nil
|
||||
}
|
||||
var ok bool
|
||||
payload, ok = requestBytesFromNumberArray(typed)
|
||||
if !ok {
|
||||
return decodedRequestAsset{}, false, nil
|
||||
}
|
||||
default:
|
||||
return decodedRequestAsset{}, false, nil
|
||||
}
|
||||
if len(payload) == 0 {
|
||||
return decodedRequestAsset{}, false, requestAssetDecodeError(fmt.Errorf("binary media payload is empty"))
|
||||
}
|
||||
return decodedRequestAsset{
|
||||
Bytes: payload,
|
||||
ContentType: requestAssetContentType(contentType, payload, key, path, siblings),
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func requestAssetPathKey(path []string) string {
|
||||
for index := len(path) - 1; index >= 0; index-- {
|
||||
value := strings.TrimSpace(path[index])
|
||||
if value != "" && !strings.HasPrefix(value, "[") {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func strictRequestBinaryArrayField(key string, path []string) bool {
|
||||
value := strings.ToLower(strings.TrimSpace(key))
|
||||
if value == "bytes" || value == "buffer" || strings.Contains(value, "binary") || strings.Contains(value, "buffer") {
|
||||
return true
|
||||
}
|
||||
return requestMediaKind(key, path, nil) != "" && (value == "data" || value == "content")
|
||||
}
|
||||
|
||||
func requestBytesFromNumberArray(values []any) ([]byte, bool) {
|
||||
if len(values) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
payload := make([]byte, len(values))
|
||||
for index, value := range values {
|
||||
number, ok := value.(float64)
|
||||
if !ok || number < 0 || number > 255 || number != float64(byte(number)) {
|
||||
return nil, false
|
||||
}
|
||||
payload[index] = byte(number)
|
||||
}
|
||||
return payload, true
|
||||
}
|
||||
|
||||
func (s *Server) prepareRequestAssetField(ctx context.Context, key string, path []string, value any, siblings map[string]any) (map[string]any, bool, error) {
|
||||
text, ok := value.(string)
|
||||
if !ok {
|
||||
@@ -291,68 +374,43 @@ func requestAssetFromValue(key string, path []string, value any, siblings map[st
|
||||
}
|
||||
|
||||
func (s *Server) ensureRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
|
||||
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
|
||||
UploadScene: store.FileStorageSceneRequestAsset,
|
||||
Source: "ai-gateway-request",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) ensurePublicRequestAsset(ctx context.Context, decoded decodedRequestAsset) (map[string]any, error) {
|
||||
return s.ensureRequestAssetWithOptions(ctx, decoded, requestAssetOptions{
|
||||
RequirePublicURL: true,
|
||||
UploadScene: store.FileStorageSceneUpload,
|
||||
Source: "ai-gateway-form-data",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded decodedRequestAsset, options requestAssetOptions) (map[string]any, error) {
|
||||
sum := sha256.Sum256(decoded.Bytes)
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
contentType := strings.TrimSpace(decoded.ContentType)
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + options.UploadScene + "\x00" + strconv.FormatBool(options.RequirePublicURL))
|
||||
release := s.acquireRequestAssetLock(sha + "\x00" + contentType + "\x00" + store.FileStorageSceneRequestAsset)
|
||||
defer release()
|
||||
now := time.Now()
|
||||
if existing, ok, err := s.store.FindRequestAsset(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||
return nil, err
|
||||
} else if ok && requestAssetStillUsable(existing, now) {
|
||||
ref := requestAssetRef(existing)
|
||||
if !options.RequirePublicURL || requestAssetRefHasPublicURL(ref) {
|
||||
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||
return nil, err
|
||||
}
|
||||
return ref, nil
|
||||
if err := s.store.IncrementRequestAssetRefCount(ctx, sha, contentType); err != nil && !store.IsUndefinedDatabaseObject(err) {
|
||||
return nil, err
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
uploadScene := strings.TrimSpace(options.UploadScene)
|
||||
if uploadScene == "" {
|
||||
uploadScene = store.FileStorageSceneRequestAsset
|
||||
}
|
||||
source := strings.TrimSpace(options.Source)
|
||||
if source == "" {
|
||||
source = "ai-gateway-request"
|
||||
}
|
||||
upload, err := s.runner.UploadFile(ctx, runner.FileUploadPayload{
|
||||
Bytes: decoded.Bytes,
|
||||
ContentType: contentType,
|
||||
FileName: requestAssetFileName(sha, contentType),
|
||||
Scene: uploadScene,
|
||||
Source: source,
|
||||
Scene: store.FileStorageSceneRequestAsset,
|
||||
Source: "ai-gateway-request",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storageProvider := requestAssetStorageProvider(upload)
|
||||
storageChannelID, storageChannelKey := requestAssetStorageChannel(upload)
|
||||
objectKey := stringFromRequestAny(upload["objectKey"])
|
||||
accessScope := firstNonEmpty(stringFromRequestAny(upload["accessScope"]), "private")
|
||||
url := stringFromRequestAny(upload["url"])
|
||||
if url == "" {
|
||||
return nil, &clients.ClientError{Code: "request_asset_upload_failed", Message: "file storage response did not include url", Retryable: false}
|
||||
}
|
||||
if options.RequirePublicURL && !requestAssetURLIsPublic(storageProvider, url) {
|
||||
return nil, &clients.ClientError{Code: "request_asset_public_url_required", Message: "multipart image assets require a public file storage URL; enable a non-local file storage channel for uploads", Retryable: false}
|
||||
}
|
||||
var expiresAt *time.Time
|
||||
localPath := ""
|
||||
if storageProvider == "local_static" {
|
||||
@@ -361,23 +419,31 @@ func (s *Server) ensureRequestAssetWithOptions(ctx context.Context, decoded deco
|
||||
localPath = requestAssetLocalPath(s.cfg.LocalUploadedStorageDir, stringFromRequestAny(upload["fileName"]))
|
||||
}
|
||||
asset, err := s.store.UpsertRequestAsset(ctx, store.RequestAssetInput{
|
||||
SHA256: sha,
|
||||
ContentType: contentType,
|
||||
ByteSize: int64(len(decoded.Bytes)),
|
||||
URL: url,
|
||||
StorageProvider: storageProvider,
|
||||
LocalPath: localPath,
|
||||
ExpiresAt: expiresAt,
|
||||
SHA256: sha,
|
||||
ContentType: contentType,
|
||||
ByteSize: int64(len(decoded.Bytes)),
|
||||
URL: url,
|
||||
StorageProvider: storageProvider,
|
||||
StorageChannelID: storageChannelID,
|
||||
StorageChannelKey: storageChannelKey,
|
||||
ObjectKey: objectKey,
|
||||
AccessScope: accessScope,
|
||||
LocalPath: localPath,
|
||||
ExpiresAt: expiresAt,
|
||||
})
|
||||
if err != nil {
|
||||
if store.IsUndefinedDatabaseObject(err) {
|
||||
return map[string]any{
|
||||
"sha256": sha,
|
||||
"url": url,
|
||||
"contentType": contentType,
|
||||
"size": len(decoded.Bytes),
|
||||
"storageProvider": storageProvider,
|
||||
"expiresAt": timePtrToRFC3339(expiresAt),
|
||||
"sha256": sha,
|
||||
"url": url,
|
||||
"contentType": contentType,
|
||||
"size": len(decoded.Bytes),
|
||||
"storageProvider": storageProvider,
|
||||
"storageChannelId": storageChannelID,
|
||||
"storageChannelKey": storageChannelKey,
|
||||
"objectKey": objectKey,
|
||||
"accessScope": accessScope,
|
||||
"expiresAt": timePtrToRFC3339(expiresAt),
|
||||
}, nil
|
||||
}
|
||||
return nil, err
|
||||
@@ -493,27 +559,19 @@ func requestAssetWrapper(ref map[string]any) map[string]any {
|
||||
|
||||
func requestAssetRef(asset store.RequestAsset) map[string]any {
|
||||
return map[string]any{
|
||||
"sha256": asset.SHA256,
|
||||
"url": asset.URL,
|
||||
"contentType": asset.ContentType,
|
||||
"size": asset.ByteSize,
|
||||
"storageProvider": asset.StorageProvider,
|
||||
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
|
||||
"sha256": asset.SHA256,
|
||||
"url": asset.URL,
|
||||
"contentType": asset.ContentType,
|
||||
"size": asset.ByteSize,
|
||||
"storageProvider": asset.StorageProvider,
|
||||
"storageChannelId": asset.StorageChannelID,
|
||||
"storageChannelKey": asset.StorageChannelKey,
|
||||
"objectKey": asset.ObjectKey,
|
||||
"accessScope": asset.AccessScope,
|
||||
"expiresAt": timePtrToRFC3339(asset.ExpiresAt),
|
||||
}
|
||||
}
|
||||
|
||||
func requestAssetRefHasPublicURL(ref map[string]any) bool {
|
||||
return requestAssetURLIsPublic(stringFromRequestAny(ref["storageProvider"]), stringFromRequestAny(ref["url"]))
|
||||
}
|
||||
|
||||
func requestAssetURLIsPublic(storageProvider string, url string) bool {
|
||||
if strings.EqualFold(strings.TrimSpace(storageProvider), "local_static") {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(strings.TrimSpace(url))
|
||||
return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
|
||||
}
|
||||
|
||||
func requestAssetStillUsable(asset store.RequestAsset, now time.Time) bool {
|
||||
if asset.ExpiredAt != nil {
|
||||
return false
|
||||
@@ -543,6 +601,11 @@ func requestAssetStorageProvider(upload map[string]any) string {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func requestAssetStorageChannel(upload map[string]any) (string, string) {
|
||||
channel, _ := upload["storageChannel"].(map[string]any)
|
||||
return stringFromRequestAny(channel["id"]), stringFromRequestAny(channel["channelKey"])
|
||||
}
|
||||
|
||||
func requestAssetLocalPath(storageDir string, fileName string) string {
|
||||
if strings.TrimSpace(storageDir) == "" {
|
||||
storageDir = config.DefaultLocalUploadedStorageDir
|
||||
|
||||
@@ -41,6 +41,33 @@ func TestRequestAssetFromValueDetectsDataURLAndRawBase64(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAssetFromBinaryValueDetectsBufferAndByteArray(t *testing.T) {
|
||||
png := []any{float64(0x89), float64('P'), float64('N'), float64('G')}
|
||||
decoded, ok, err := requestAssetFromBinaryValue("image", []string{"input", "image"}, map[string]any{
|
||||
"type": "Buffer",
|
||||
"data": png,
|
||||
"mimeType": "image/png",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ok || decoded.ContentType != "image/png" || len(decoded.Bytes) != 4 || decoded.Bytes[0] != 0x89 {
|
||||
t.Fatalf("unexpected Buffer asset: ok=%v decoded=%+v", ok, decoded)
|
||||
}
|
||||
|
||||
decoded, ok, err = requestAssetFromBinaryValue("bytes", []string{"input_audio", "bytes"}, png, map[string]any{"format": "mp3"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ok || len(decoded.Bytes) != 4 {
|
||||
t.Fatalf("unexpected byte array asset: ok=%v decoded=%+v", ok, decoded)
|
||||
}
|
||||
|
||||
if _, ok, err := requestAssetFromBinaryValue("values", []string{"embedding", "values"}, png, nil); err != nil || ok {
|
||||
t.Fatalf("ordinary numeric arrays must remain JSON: ok=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaRequestBodySlotLimitsPreAuthWorkAndCanReleaseEarly(t *testing.T) {
|
||||
server := &Server{mediaRequestBodySlots: make(chan struct{}, 2)}
|
||||
var critical atomic.Int64
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
)
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
@@ -18,22 +20,31 @@ func writeError(w http.ResponseWriter, status int, message string, codes ...stri
|
||||
}
|
||||
|
||||
func writeErrorWithDetails(w http.ResponseWriter, status int, message string, details map[string]any, codes ...string) {
|
||||
errorPayload := map[string]any{
|
||||
"message": message,
|
||||
"status": status,
|
||||
}
|
||||
code := ""
|
||||
if len(codes) > 0 {
|
||||
if code := strings.TrimSpace(codes[0]); code != "" {
|
||||
errorPayload["code"] = code
|
||||
if code == "invalid_parameter" || code == "unsupported_response_parameter" {
|
||||
errorPayload["type"] = "invalid_request_error"
|
||||
if _, ok := details["param"]; !ok {
|
||||
errorPayload["param"] = nil
|
||||
}
|
||||
}
|
||||
code = strings.TrimSpace(codes[0])
|
||||
}
|
||||
standard := publicerror.FromFields(code, message, status, status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500)
|
||||
standard = publicErrorWithRetryAfter(standard, details)
|
||||
publicerror.Observe(standard)
|
||||
status = standard.HTTPStatus
|
||||
errorPayload := map[string]any{
|
||||
"message": standard.Message,
|
||||
"status": status,
|
||||
"code": standard.Code,
|
||||
"category": standard.Category,
|
||||
"httpStatus": standard.HTTPStatus,
|
||||
"retryable": standard.Retryable,
|
||||
"action": standard.Action,
|
||||
"version": standard.Version,
|
||||
}
|
||||
if standard.Code == "invalid_parameter" || standard.Code == "unsupported_response_parameter" {
|
||||
errorPayload["type"] = "invalid_request_error"
|
||||
if _, ok := details["param"]; !ok {
|
||||
errorPayload["param"] = nil
|
||||
}
|
||||
}
|
||||
for key, value := range details {
|
||||
for key, value := range safePublicErrorDetails(details, standard, false) {
|
||||
errorPayload[key] = value
|
||||
}
|
||||
writeJSON(w, status, map[string]any{"error": errorPayload})
|
||||
|
||||
@@ -286,6 +286,7 @@ func NewServerWithStores(
|
||||
mux.Handle("GET /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listFileStorageChannels)))
|
||||
mux.Handle("POST /api/admin/system/file-storage/channels", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createFileStorageChannel)))
|
||||
mux.Handle("PATCH /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageChannel)))
|
||||
mux.Handle("POST /api/admin/system/file-storage/channels/{channelID}/test", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.testFileStorageChannel)))
|
||||
mux.Handle("DELETE /api/admin/system/file-storage/channels/{channelID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteFileStorageChannel)))
|
||||
mux.Handle("GET /api/admin/platforms", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listPlatforms)))
|
||||
mux.Handle("POST /api/admin/platforms", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createPlatform)))
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
@@ -74,6 +77,10 @@ func (s *Server) updateFileStorageSettings(w http.ResponseWriter, r *http.Reques
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
if legacyLocalStoragePolicy(input.ResultUploadPolicy) {
|
||||
writeError(w, http.StatusBadRequest, "upload_none is no longer supported; configure an object storage channel")
|
||||
return
|
||||
}
|
||||
settings, err := s.store.UpdateFileStorageSettings(r.Context(), input)
|
||||
if err != nil {
|
||||
s.logger.Error("update file storage settings failed", "error", err)
|
||||
@@ -155,7 +162,7 @@ func (s *Server) updateClientCustomizationSettings(w http.ResponseWriter, r *htt
|
||||
|
||||
// createFileStorageChannel godoc
|
||||
// @Summary 创建文件存储通道
|
||||
// @Description 创建文件存储通道,当前主要用于配置 server-main OpenAPI 上传通道。
|
||||
// @Description 创建 server-main OpenAPI、阿里云 OSS 或 S3 兼容文件存储通道。
|
||||
// @Tags system
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
@@ -271,6 +278,46 @@ func (s *Server) deleteFileStorageChannel(w http.ResponseWriter, r *http.Request
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// testFileStorageChannel godoc
|
||||
// @Summary 测试对象存储通道
|
||||
// @Description 对指定 OSS 或 S3 通道执行隔离的 Put、Head、Delete 探针,不返回凭据或对象键。
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param channelID path string true "文件存储通道 ID"
|
||||
// @Success 200 {object} FileStorageChannelTestResponse
|
||||
// @Failure 400 {object} ErrorEnvelope
|
||||
// @Failure 401 {object} ErrorEnvelope
|
||||
// @Failure 403 {object} ErrorEnvelope
|
||||
// @Failure 404 {object} ErrorEnvelope
|
||||
// @Failure 503 {object} ErrorEnvelope
|
||||
// @Router /api/admin/system/file-storage/channels/{channelID}/test [post]
|
||||
func (s *Server) testFileStorageChannel(w http.ResponseWriter, r *http.Request) {
|
||||
channel, err := s.store.GetFileStorageChannel(r.Context(), r.PathValue("channelID"))
|
||||
if err != nil {
|
||||
if store.IsNotFound(err) {
|
||||
writeError(w, http.StatusNotFound, "file storage channel not found")
|
||||
return
|
||||
}
|
||||
s.logger.Error("get file storage channel for test failed", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "get file storage channel failed")
|
||||
return
|
||||
}
|
||||
if channel.Provider != "aliyun_oss" && channel.Provider != "s3" {
|
||||
writeError(w, http.StatusBadRequest, "connection test is supported for aliyun_oss and s3 channels", "invalid_parameter")
|
||||
return
|
||||
}
|
||||
result, err := s.runner.TestFileStorageChannel(r.Context(), channel)
|
||||
if err != nil {
|
||||
_ = s.store.MarkFileStorageChannelFailure(context.WithoutCancel(r.Context()), channel.ID, err.Error())
|
||||
s.logger.Warn("file storage channel test failed", "channel_id", channel.ID, "provider", channel.Provider, "error", err)
|
||||
writeError(w, statusFromRunError(err), err.Error(), clients.ErrorCode(err))
|
||||
return
|
||||
}
|
||||
_ = s.store.MarkFileStorageChannelSuccess(context.WithoutCancel(r.Context()), channel.ID)
|
||||
writeJSON(w, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func validateFileStorageChannelInput(input store.FileStorageChannelInput, existing *store.FileStorageChannel) string {
|
||||
provider := strings.ToLower(strings.TrimSpace(input.Provider))
|
||||
if provider == "" {
|
||||
@@ -286,6 +333,9 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
||||
if status != "enabled" && status != "disabled" {
|
||||
return "status must be enabled or disabled"
|
||||
}
|
||||
if provider != "server_main_openapi" && provider != "aliyun_oss" && provider != "s3" {
|
||||
return "provider must be server_main_openapi, aliyun_oss or s3"
|
||||
}
|
||||
if provider == "server_main_openapi" {
|
||||
hasAPIKey := false
|
||||
if input.APIKey != nil {
|
||||
@@ -297,5 +347,83 @@ func validateFileStorageChannelInput(input store.FileStorageChannelInput, existi
|
||||
return "server-main OpenAPI channel requires API key before enabling"
|
||||
}
|
||||
}
|
||||
if provider == "aliyun_oss" || provider == "s3" {
|
||||
if fileStorageConfigContainsCredential(input.Config) {
|
||||
return "object storage credentials must use the write-only credential fields, not config"
|
||||
}
|
||||
endpoint := fileStorageConfigString(input.Config, "endpoint")
|
||||
if endpoint == "" || fileStorageConfigString(input.Config, "region") == "" || fileStorageConfigString(input.Config, "bucket") == "" {
|
||||
return "object storage channel requires config.endpoint, config.region and config.bucket"
|
||||
}
|
||||
if !validFileStorageBaseURL(endpoint) {
|
||||
return "object storage config.endpoint must be an http or https URL without embedded credentials"
|
||||
}
|
||||
if publicBaseURL := firstNonEmpty(fileStorageConfigString(input.Config, "publicBaseUrl"), fileStorageConfigString(input.Config, "publicBaseURL")); publicBaseURL != "" && !validFileStorageBaseURL(publicBaseURL) {
|
||||
return "object storage config.publicBaseUrl must be an http or https URL without embedded credentials"
|
||||
}
|
||||
accessKeyID := input.AccessKeyID
|
||||
if accessKeyID == nil {
|
||||
accessKeyID = input.AccessKey
|
||||
}
|
||||
accessKeySecret := input.AccessKeySecret
|
||||
if accessKeySecret == nil {
|
||||
accessKeySecret = input.SecretKey
|
||||
}
|
||||
hasAccessKeyID := fileStorageCredentialPresent(accessKeyID, existing, func(item *store.FileStorageChannel) string { return item.AccessKeyID })
|
||||
hasAccessKeySecret := fileStorageCredentialPresent(accessKeySecret, existing, func(item *store.FileStorageChannel) string { return item.AccessKeySecret })
|
||||
if status == "enabled" && (!hasAccessKeyID || !hasAccessKeySecret) {
|
||||
return "object storage channel requires accessKeyId and accessKeySecret before enabling"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func validFileStorageBaseURL(value string) bool {
|
||||
parsed, err := url.Parse(strings.TrimSpace(value))
|
||||
return err == nil && parsed.User == nil && parsed.Host != "" && (parsed.Scheme == "http" || parsed.Scheme == "https")
|
||||
}
|
||||
|
||||
func fileStorageConfigContainsCredential(value any) bool {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for key, item := range typed {
|
||||
normalized := strings.NewReplacer("_", "", "-", "", ".", "").Replace(strings.ToLower(strings.TrimSpace(key)))
|
||||
switch normalized {
|
||||
case "apikey", "accesskey", "accesskeyid", "accesskeysecret", "secretkey", "sessiontoken", "ststoken", "password", "credential", "credentials", "authorization":
|
||||
return true
|
||||
}
|
||||
if fileStorageConfigContainsCredential(item) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, item := range typed {
|
||||
if fileStorageConfigContainsCredential(item) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fileStorageConfigString(config map[string]any, key string) string {
|
||||
value, _ := config[key].(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func fileStorageCredentialPresent(input *string, existing *store.FileStorageChannel, current func(*store.FileStorageChannel) string) bool {
|
||||
if input != nil {
|
||||
return strings.TrimSpace(*input) != ""
|
||||
}
|
||||
return existing != nil && strings.TrimSpace(current(existing)) != ""
|
||||
}
|
||||
|
||||
func legacyLocalStoragePolicy(value string) bool {
|
||||
normalized := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(value)), "-", "_")
|
||||
switch normalized {
|
||||
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestValidateObjectStorageChannelAcceptsS3CredentialAliases(t *testing.T) {
|
||||
accessKey := "access"
|
||||
secretKey := "secret"
|
||||
input := store.FileStorageChannelInput{
|
||||
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
|
||||
AccessKey: &accessKey, SecretKey: &secretKey,
|
||||
Config: map[string]any{
|
||||
"endpoint": "https://s3.example.com", "region": "us-east-1", "bucket": "media",
|
||||
},
|
||||
}
|
||||
if message := validateFileStorageChannelInput(input, nil); message != "" {
|
||||
t.Fatalf("valid S3 channel rejected: %s", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateObjectStorageChannelRejectsCredentialsInConfig(t *testing.T) {
|
||||
accessKeyID := "access"
|
||||
accessKeySecret := "secret"
|
||||
input := store.FileStorageChannelInput{
|
||||
ChannelKey: "oss-primary", Name: "OSS primary", Provider: "aliyun_oss", Status: "enabled",
|
||||
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
|
||||
Config: map[string]any{
|
||||
"endpoint": "https://oss-cn-hangzhou.aliyuncs.com", "region": "cn-hangzhou", "bucket": "media",
|
||||
"nested": map[string]any{"session_token": "must-not-be-public"},
|
||||
},
|
||||
}
|
||||
message := validateFileStorageChannelInput(input, nil)
|
||||
if !strings.Contains(message, "write-only credential fields") {
|
||||
t.Fatalf("credential-bearing config was accepted: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateObjectStorageChannelRejectsCredentialedEndpoint(t *testing.T) {
|
||||
accessKeyID := "access"
|
||||
accessKeySecret := "secret"
|
||||
input := store.FileStorageChannelInput{
|
||||
ChannelKey: "s3-primary", Name: "S3 primary", Provider: "s3", Status: "enabled",
|
||||
AccessKeyID: &accessKeyID, AccessKeySecret: &accessKeySecret,
|
||||
Config: map[string]any{
|
||||
"endpoint": "https://user:password@s3.example.com", "region": "us-east-1", "bucket": "media",
|
||||
},
|
||||
}
|
||||
if message := validateFileStorageChannelInput(input, nil); !strings.Contains(message, "without embedded credentials") {
|
||||
t.Fatalf("credentialed endpoint was accepted: %q", message)
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,10 @@ func writeIdempotentTaskReplay(w http.ResponseWriter, task store.GatewayTask, co
|
||||
|
||||
func storedTaskErrorStatus(code string) int {
|
||||
switch strings.TrimSpace(code) {
|
||||
case "storage_write_failed", "storage_read_failed", "storage_config_invalid", "storage_auth_failed":
|
||||
return http.StatusServiceUnavailable
|
||||
case "binary_result_expired", "result_expired", "result_unavailable":
|
||||
return http.StatusGone
|
||||
case "pricing_unavailable", "response_chain_unavailable", "billing_hold":
|
||||
return http.StatusServiceUnavailable
|
||||
case "insufficient_balance":
|
||||
|
||||
@@ -255,7 +255,7 @@ func (s *Server) uploadImageEditMultipartAsset(ctx context.Context, field string
|
||||
if !strings.HasPrefix(strings.ToLower(contentType), "image/") {
|
||||
return nil, &clients.ClientError{Code: "invalid_multipart_image", Message: "image edit multipart files must be images", Retryable: false}
|
||||
}
|
||||
ref, err := s.ensurePublicRequestAsset(ctx, decodedRequestAsset{
|
||||
ref, err := s.ensureRequestAsset(ctx, decodedRequestAsset{
|
||||
Bytes: payload,
|
||||
ContentType: contentType,
|
||||
})
|
||||
|
||||
@@ -352,7 +352,7 @@ func volcesCompatibleTask(task store.GatewayTask) map[string]any {
|
||||
response["usage"] = legacyUsage
|
||||
}
|
||||
if task.Status == "failed" || task.Status == "cancelled" {
|
||||
response["error"] = map[string]any{"code": firstNonEmpty(task.ErrorCode, strings.ToUpper(task.Status)), "message": firstNonEmpty(task.ErrorMessage, task.Error, task.Message)}
|
||||
response["error"] = publicErrorMap(publicTaskError(task))
|
||||
}
|
||||
return response
|
||||
}
|
||||
@@ -431,10 +431,6 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
|
||||
} else if errors.Is(err, store.ErrTaskRequestBinaryNotMaterialized) {
|
||||
status = http.StatusBadRequest
|
||||
}
|
||||
if wire := clients.ErrorWireResponse(err); wireResponseMatches(wire, clients.ProtocolVolcesContents) {
|
||||
writeWireResponse(w, wire)
|
||||
return
|
||||
}
|
||||
writeVolcesError(w, status, err.Error(), clients.ErrorCode(err))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user