feat(storage): 统一二进制对象存储与公开错误
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。 将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。 引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。 验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
package publicerror
|
||||
|
||||
import "sync/atomic"
|
||||
|
||||
type MetricCount struct {
|
||||
Code string
|
||||
Count uint64
|
||||
}
|
||||
|
||||
var observed = struct {
|
||||
upstreamConnectionInterrupted atomic.Uint64
|
||||
upstreamTimeout atomic.Uint64
|
||||
upstreamRateLimited atomic.Uint64
|
||||
upstreamUnavailable atomic.Uint64
|
||||
upstreamInvalidResponse atomic.Uint64
|
||||
upstreamAuthFailed atomic.Uint64
|
||||
upstreamRequestRejected atomic.Uint64
|
||||
storageWriteFailed atomic.Uint64
|
||||
storageReadFailed atomic.Uint64
|
||||
resultExpired atomic.Uint64
|
||||
resultUnavailable atomic.Uint64
|
||||
invalidParameter atomic.Uint64
|
||||
gatewayError atomic.Uint64
|
||||
}{}
|
||||
|
||||
func Observe(value Error) {
|
||||
switch value.Code {
|
||||
case "upstream_connection_interrupted":
|
||||
observed.upstreamConnectionInterrupted.Add(1)
|
||||
case "upstream_timeout":
|
||||
observed.upstreamTimeout.Add(1)
|
||||
case "upstream_rate_limited":
|
||||
observed.upstreamRateLimited.Add(1)
|
||||
case "upstream_unavailable":
|
||||
observed.upstreamUnavailable.Add(1)
|
||||
case "upstream_invalid_response":
|
||||
observed.upstreamInvalidResponse.Add(1)
|
||||
case "upstream_auth_failed":
|
||||
observed.upstreamAuthFailed.Add(1)
|
||||
case "upstream_request_rejected":
|
||||
observed.upstreamRequestRejected.Add(1)
|
||||
case "storage_write_failed":
|
||||
observed.storageWriteFailed.Add(1)
|
||||
case "storage_read_failed":
|
||||
observed.storageReadFailed.Add(1)
|
||||
case "result_expired":
|
||||
observed.resultExpired.Add(1)
|
||||
case "result_unavailable":
|
||||
observed.resultUnavailable.Add(1)
|
||||
case "invalid_parameter":
|
||||
observed.invalidParameter.Add(1)
|
||||
default:
|
||||
observed.gatewayError.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func MetricSnapshot() []MetricCount {
|
||||
return []MetricCount{
|
||||
{Code: "upstream_connection_interrupted", Count: observed.upstreamConnectionInterrupted.Load()},
|
||||
{Code: "upstream_timeout", Count: observed.upstreamTimeout.Load()},
|
||||
{Code: "upstream_rate_limited", Count: observed.upstreamRateLimited.Load()},
|
||||
{Code: "upstream_unavailable", Count: observed.upstreamUnavailable.Load()},
|
||||
{Code: "upstream_invalid_response", Count: observed.upstreamInvalidResponse.Load()},
|
||||
{Code: "upstream_auth_failed", Count: observed.upstreamAuthFailed.Load()},
|
||||
{Code: "upstream_request_rejected", Count: observed.upstreamRequestRejected.Load()},
|
||||
{Code: "storage_write_failed", Count: observed.storageWriteFailed.Load()},
|
||||
{Code: "storage_read_failed", Count: observed.storageReadFailed.Load()},
|
||||
{Code: "result_expired", Count: observed.resultExpired.Load()},
|
||||
{Code: "result_unavailable", Count: observed.resultUnavailable.Load()},
|
||||
{Code: "invalid_parameter", Count: observed.invalidParameter.Load()},
|
||||
{Code: "gateway_error", Count: observed.gatewayError.Load()},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package publicerror
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const Version = "v1"
|
||||
|
||||
type Error struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Category string `json:"category"`
|
||||
HTTPStatus int `json:"httpStatus"`
|
||||
Retryable bool `json:"retryable"`
|
||||
Action string `json:"action"`
|
||||
RetryAfterSeconds int `json:"retryAfterSeconds,omitempty"`
|
||||
RequestID string `json:"requestId,omitempty"`
|
||||
TaskID string `json:"taskId,omitempty"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
func FromFields(code string, message string, status int, retryable bool) Error {
|
||||
originalCode := strings.TrimSpace(code)
|
||||
code = strings.ToLower(originalCode)
|
||||
message = strings.TrimSpace(message)
|
||||
if status <= 0 {
|
||||
status = statusFromCode(code)
|
||||
}
|
||||
if strings.HasPrefix(originalCode, "GATEWAY_") {
|
||||
return newError(originalCode, safeRequestMessage(message), "gateway", status, retryable, retryAction(retryable))
|
||||
}
|
||||
if mapped, ok := mappedError(code, message, status, retryable); ok {
|
||||
return mapped
|
||||
}
|
||||
if sensitiveTransportMessage(message) {
|
||||
return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry")
|
||||
}
|
||||
if code == "" {
|
||||
code = defaultCodeForStatus(status)
|
||||
} else {
|
||||
code = originalCode
|
||||
}
|
||||
if message == "" {
|
||||
message = defaultMessageForStatus(status)
|
||||
}
|
||||
category := "gateway"
|
||||
action := "none"
|
||||
if status == http.StatusTooManyRequests {
|
||||
category, action, retryable = "rate_limit", "retry_after", true
|
||||
} else if status >= 500 {
|
||||
category = "upstream"
|
||||
if retryable {
|
||||
action = "retry"
|
||||
}
|
||||
} else if status >= 400 {
|
||||
category, action = "request", "fix_request"
|
||||
}
|
||||
return newError(code, message, category, status, retryable, action)
|
||||
}
|
||||
|
||||
func WithIDs(value Error, requestID string, taskID string) Error {
|
||||
value.RequestID = strings.TrimSpace(requestID)
|
||||
value.TaskID = strings.TrimSpace(taskID)
|
||||
return value
|
||||
}
|
||||
|
||||
func mappedError(code string, message string, status int, retryable bool) (Error, bool) {
|
||||
switch code {
|
||||
case "bad_request", "invalid_request", "invalid_parameter", "unsupported_kind", "unsupported_operation", "unsupported_response_parameter":
|
||||
return newError(code, safeRequestMessage(message), "request", http.StatusBadRequest, false, "fix_request"), true
|
||||
case "cancelled", "task_cancelled":
|
||||
return newError(code, safeRequestMessage(message), "request", http.StatusConflict, false, "none"), true
|
||||
case "validation_in_progress":
|
||||
return newError(code, firstNonEmptyMessage(message, "New production tasks are paused while validation is running."), "gateway", http.StatusServiceUnavailable, true, "retry_after"), true
|
||||
case "traffic_gate_unavailable":
|
||||
return newError(code, "The gateway traffic admission service is temporarily unavailable.", "gateway", http.StatusServiceUnavailable, true, "retry"), true
|
||||
case "response_read_error", "stream_read_error", "network", "connection_reset", "upstream_connection_interrupted", "unexpected_eof", "http2_stream_closed", "request_asset_fetch_failed":
|
||||
return newError("upstream_connection_interrupted", "The upstream connection was interrupted before a complete response was received.", "upstream", http.StatusBadGateway, true, "retry"), true
|
||||
case "upstream_timeout", "timeout", "context_deadline_exceeded":
|
||||
return newError("upstream_timeout", "The upstream service did not respond in time.", "upstream", http.StatusGatewayTimeout, true, "retry"), true
|
||||
case "rate_limit", "upstream_rate_limited", "too_many_requests":
|
||||
return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", http.StatusTooManyRequests, true, "retry_after"), true
|
||||
case "upstream_unavailable", "upstream_overloaded", "service_unavailable", "bad_gateway", "server_error", "provider_failed":
|
||||
if status > 0 && status < 500 {
|
||||
break
|
||||
}
|
||||
return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true
|
||||
case "upload_invalid_response", "invalid_response", "response_too_large", "invalid_upstream_response", "upstream_invalid_response":
|
||||
return newError("upstream_invalid_response", "The upstream service returned an invalid or incomplete response.", "upstream", http.StatusBadGateway, true, "retry"), true
|
||||
case "invalid_api_key", "authentication_error", "auth_failed", "missing_credentials", "upstream_auth_failed":
|
||||
return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true
|
||||
case "storage_write_failed", "storage_config_invalid", "storage_auth_failed", "upload_config_failed", "upload_no_channel", "upload_network", "upload_read_failed", "upload_failed", "local_result_storage_unavailable", "local_static_store_failed", "request_asset_upload_failed":
|
||||
return newError("storage_write_failed", "The media asset could not be written to object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true
|
||||
case "storage_read_failed", "upload_source_fetch_failed", "upload_source_read_failed":
|
||||
return newError("storage_read_failed", "The media asset could not be read from object storage.", "storage", http.StatusServiceUnavailable, true, "retry"), true
|
||||
case "binary_result_expired", "result_expired":
|
||||
return newError("result_expired", "The generated result has expired and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true
|
||||
case "result_unavailable", "request_asset_expired":
|
||||
return newError("result_unavailable", "The generated result is no longer available and must be submitted again.", "storage", http.StatusGone, false, "resubmit"), true
|
||||
case "binary_result_corrupted", "result_binary_not_materialized":
|
||||
return newError("result_corrupted", "The stored result failed an integrity check.", "storage", http.StatusInternalServerError, false, "contact_support"), true
|
||||
}
|
||||
if status == http.StatusTooManyRequests {
|
||||
return newError("upstream_rate_limited", "The upstream service rate limit was reached.", "rate_limit", status, true, "retry_after"), true
|
||||
}
|
||||
if (status == http.StatusUnauthorized || status == http.StatusForbidden) && (strings.HasPrefix(code, "http_") || strings.Contains(code, "provider")) {
|
||||
return newError("upstream_auth_failed", "The upstream service rejected the configured credentials.", "upstream", http.StatusBadGateway, false, "contact_support"), true
|
||||
}
|
||||
if strings.HasPrefix(code, "http_") || (code == "provider_failed" && status > 0 && status < 500) {
|
||||
return newError("upstream_request_rejected", "The upstream service rejected the request.", "upstream", http.StatusBadRequest, false, "fix_request"), true
|
||||
}
|
||||
if status >= 500 && code != "upstream_submission_unknown" && (strings.Contains(code, "upstream") || strings.Contains(code, "provider")) {
|
||||
return newError("upstream_unavailable", "The upstream service is temporarily unavailable.", "upstream", http.StatusServiceUnavailable, true, "retry"), true
|
||||
}
|
||||
if status >= 500 && code != "upstream_submission_unknown" {
|
||||
return newError("gateway_error", defaultMessageForStatus(status), "gateway", status, retryable, retryAction(retryable)), true
|
||||
}
|
||||
return Error{}, false
|
||||
}
|
||||
|
||||
func safeRequestMessage(message string) string {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" || sensitiveTransportMessage(message) {
|
||||
return "The request parameters are invalid."
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func firstNonEmptyMessage(message string, fallback string) string {
|
||||
if message = strings.TrimSpace(message); message != "" && !sensitiveTransportMessage(message) {
|
||||
return message
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func retryAction(retryable bool) string {
|
||||
if retryable {
|
||||
return "retry"
|
||||
}
|
||||
return "contact_support"
|
||||
}
|
||||
|
||||
func newError(code string, message string, category string, status int, retryable bool, action string) Error {
|
||||
return Error{Code: code, Message: message, Category: category, HTTPStatus: status, Retryable: retryable, Action: action, Version: Version}
|
||||
}
|
||||
|
||||
func sensitiveTransportMessage(message string) bool {
|
||||
value := strings.ToLower(message)
|
||||
for _, marker := range []string{"read tcp ", "write tcp ", "dial tcp ", "connection reset by peer", "unexpected eof", "http2:", "stream error", "context deadline exceeded"} {
|
||||
if strings.Contains(value, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func statusFromCode(code string) int {
|
||||
switch code {
|
||||
case "rate_limit", "upstream_rate_limited", "too_many_requests":
|
||||
return http.StatusTooManyRequests
|
||||
case "upstream_timeout", "timeout", "context_deadline_exceeded":
|
||||
return http.StatusGatewayTimeout
|
||||
case "binary_result_expired", "result_expired", "result_unavailable", "request_asset_expired":
|
||||
return http.StatusGone
|
||||
case "storage_write_failed", "storage_read_failed", "local_result_storage_unavailable":
|
||||
return http.StatusServiceUnavailable
|
||||
default:
|
||||
return http.StatusBadGateway
|
||||
}
|
||||
}
|
||||
|
||||
func defaultCodeForStatus(status int) string {
|
||||
switch status {
|
||||
case http.StatusBadRequest:
|
||||
return "invalid_request"
|
||||
case http.StatusUnauthorized:
|
||||
return "unauthorized"
|
||||
case http.StatusForbidden:
|
||||
return "forbidden"
|
||||
case http.StatusNotFound:
|
||||
return "not_found"
|
||||
case http.StatusConflict:
|
||||
return "conflict"
|
||||
case http.StatusTooManyRequests:
|
||||
return "rate_limited"
|
||||
default:
|
||||
if status >= 500 {
|
||||
return "gateway_error"
|
||||
}
|
||||
return "request_failed"
|
||||
}
|
||||
}
|
||||
|
||||
func defaultMessageForStatus(status int) string {
|
||||
if value := http.StatusText(status); value != "" {
|
||||
return value
|
||||
}
|
||||
return "Request failed."
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package publicerror
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTransportErrorsNeverExposeSocketDetails(t *testing.T) {
|
||||
raw := "read tcp 10.42.0.72:54960->47.77.191.126:443: read: connection reset by peer"
|
||||
got := FromFields("response_read_error", raw, http.StatusOK, true)
|
||||
if got.Code != "upstream_connection_interrupted" || got.HTTPStatus != http.StatusBadGateway || !got.Retryable {
|
||||
t.Fatalf("unexpected public error: %+v", got)
|
||||
}
|
||||
if strings.Contains(got.Message, "10.42.0.72") || strings.Contains(got.Message, "47.77.191.126") {
|
||||
t.Fatalf("public error leaked socket details: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageAndExpiredErrorsHaveStableActions(t *testing.T) {
|
||||
storage := FromFields("storage_write_failed", "secret endpoint", 0, true)
|
||||
if storage.Category != "storage" || storage.Action != "retry" || storage.HTTPStatus != http.StatusServiceUnavailable {
|
||||
t.Fatalf("unexpected storage error: %+v", storage)
|
||||
}
|
||||
expired := FromFields("binary_result_expired", "local binary result has expired", 0, false)
|
||||
if expired.Code != "result_expired" || expired.Action != "resubmit" || expired.HTTPStatus != http.StatusGone {
|
||||
t.Fatalf("unexpected expired error: %+v", expired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownServerErrorDoesNotExposeProviderBody(t *testing.T) {
|
||||
raw := `provider rejected request: {"account":"secret-project","detail":"internal"}`
|
||||
got := FromFields("vendor_opaque_error", raw, http.StatusBadGateway, true)
|
||||
if got.Code != "gateway_error" || got.Message == raw || strings.Contains(got.Message, "secret-project") {
|
||||
t.Fatalf("unknown provider error was exposed: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderHTTPErrorDoesNotExposeProviderBody(t *testing.T) {
|
||||
raw := `{"error":{"message":"bucket private-a rejected secret-project"}}`
|
||||
for _, test := range []struct {
|
||||
code string
|
||||
status int
|
||||
wantCode string
|
||||
wantStatus int
|
||||
}{
|
||||
{code: "http_400", status: http.StatusBadRequest, wantCode: "upstream_request_rejected", wantStatus: http.StatusBadRequest},
|
||||
{code: "auth_failed", status: http.StatusUnauthorized, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway},
|
||||
{code: "provider_failed", status: http.StatusForbidden, wantCode: "upstream_auth_failed", wantStatus: http.StatusBadGateway},
|
||||
{code: "provider_failed", status: http.StatusTooManyRequests, wantCode: "upstream_rate_limited", wantStatus: http.StatusTooManyRequests},
|
||||
{code: "server_error", status: http.StatusBadGateway, wantCode: "upstream_unavailable", wantStatus: http.StatusServiceUnavailable},
|
||||
{code: "invalid_response", status: http.StatusOK, wantCode: "upstream_invalid_response", wantStatus: http.StatusBadGateway},
|
||||
} {
|
||||
got := FromFields(test.code, raw, test.status, true)
|
||||
if got.Code != test.wantCode || got.HTTPStatus != test.wantStatus {
|
||||
t.Fatalf("%s: unexpected public error: %+v", test.code, got)
|
||||
}
|
||||
if strings.Contains(got.Message, "private-a") || strings.Contains(got.Message, "secret-project") {
|
||||
t.Fatalf("%s: provider body leaked: %+v", test.code, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationGateKeepsStablePublicCode(t *testing.T) {
|
||||
got := FromFields("validation_in_progress", "new production tasks are paused while validation is running", http.StatusServiceUnavailable, true)
|
||||
if got.Code != "validation_in_progress" || got.HTTPStatus != http.StatusServiceUnavailable || !got.Retryable {
|
||||
t.Fatalf("unexpected validation gate error: %+v", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user