feat(storage): 统一二进制对象存储与公开错误
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。 将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。 引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。 验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
@@ -86,6 +87,19 @@ type Metrics struct {
|
||||
candidateDisabledSkipped atomic.Uint64
|
||||
candidateAllFullQueued atomic.Uint64
|
||||
candidateRoutingOther atomic.Uint64
|
||||
storageAliyunWrites atomic.Uint64
|
||||
storageAliyunFailures atomic.Uint64
|
||||
storageAliyunWriteNanos atomic.Uint64
|
||||
storageS3Writes atomic.Uint64
|
||||
storageS3Failures atomic.Uint64
|
||||
storageS3WriteNanos atomic.Uint64
|
||||
storageServerMainWrites atomic.Uint64
|
||||
storageServerMainFailures atomic.Uint64
|
||||
storageServerMainWriteNanos atomic.Uint64
|
||||
storageChannelRetries atomic.Uint64
|
||||
storageChannelFailovers atomic.Uint64
|
||||
storageAllChannelsFailed atomic.Uint64
|
||||
storageObjectifiedBytes atomic.Uint64
|
||||
taskAdmissionWaitBuckets [11]atomic.Uint64
|
||||
taskAdmissionWaitMicros atomic.Uint64
|
||||
}
|
||||
@@ -265,6 +279,40 @@ func (m *Metrics) ObserveCandidateRouting(reason string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveObjectStorage(event string, provider string, bytes int64, duration time.Duration) {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
switch event {
|
||||
case "write_success", "write_failure":
|
||||
var writes, failures, nanos *atomic.Uint64
|
||||
switch provider {
|
||||
case "aliyun_oss":
|
||||
writes, failures, nanos = &m.storageAliyunWrites, &m.storageAliyunFailures, &m.storageAliyunWriteNanos
|
||||
case "s3":
|
||||
writes, failures, nanos = &m.storageS3Writes, &m.storageS3Failures, &m.storageS3WriteNanos
|
||||
case "server_main_openapi":
|
||||
writes, failures, nanos = &m.storageServerMainWrites, &m.storageServerMainFailures, &m.storageServerMainWriteNanos
|
||||
default:
|
||||
return
|
||||
}
|
||||
writes.Add(1)
|
||||
nanos.Add(uint64(duration))
|
||||
if event == "write_failure" {
|
||||
failures.Add(1)
|
||||
} else if bytes > 0 {
|
||||
m.storageObjectifiedBytes.Add(uint64(bytes))
|
||||
}
|
||||
case "retry":
|
||||
m.storageChannelRetries.Add(1)
|
||||
case "failover":
|
||||
m.storageChannelFailovers.Add(1)
|
||||
case "all_failed":
|
||||
m.storageAllChannelsFailed.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveTaskAdmissionWait(wait time.Duration) {
|
||||
if wait < 0 {
|
||||
wait = 0
|
||||
@@ -471,6 +519,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
|
||||
{"all_full_queued", m.candidateAllFullQueued.Load()},
|
||||
{"other", m.candidateRoutingOther.Load()},
|
||||
})
|
||||
storageProtocolCounters(w, "easyai_gateway_storage_write_attempts_total", "Object storage write attempts by bounded protocol.",
|
||||
m.storageAliyunWrites.Load(), m.storageS3Writes.Load(), m.storageServerMainWrites.Load())
|
||||
storageProtocolCounters(w, "easyai_gateway_storage_write_failures_total", "Object storage write failures by bounded protocol.",
|
||||
m.storageAliyunFailures.Load(), m.storageS3Failures.Load(), m.storageServerMainFailures.Load())
|
||||
storageProtocolDurationCounters(w,
|
||||
m.storageAliyunWriteNanos.Load(), m.storageS3WriteNanos.Load(), m.storageServerMainWriteNanos.Load())
|
||||
plainCounter(w, "easyai_gateway_storage_channel_retries_total", "Retries performed within the current storage channel.", m.storageChannelRetries.Load())
|
||||
plainCounter(w, "easyai_gateway_storage_channel_failovers_total", "Switches to the next configured storage channel.", m.storageChannelFailovers.Load())
|
||||
plainCounter(w, "easyai_gateway_storage_all_channels_failed_total", "Storage writes where every eligible channel failed.", m.storageAllChannelsFailed.Load())
|
||||
plainCounter(w, "easyai_gateway_storage_objectified_bytes_total", "Binary bytes successfully written through storage channels.", m.storageObjectifiedBytes.Load())
|
||||
publicErrorCounters(w, publicerror.MetricSnapshot())
|
||||
platformModelRateLimitUtilizationGauges(w, modelRateLimits)
|
||||
plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections))
|
||||
plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections))
|
||||
@@ -567,6 +626,29 @@ func plainFloatGauge(w http.ResponseWriter, name, help string, value float64) {
|
||||
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %.6f\n", name, help, name, name, value)
|
||||
}
|
||||
|
||||
func storageProtocolCounters(w http.ResponseWriter, name, help string, aliyunOSS, s3, serverMain uint64) {
|
||||
fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s counter\n", name, help, name)
|
||||
fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %d\n", name, aliyunOSS)
|
||||
fmt.Fprintf(w, "%s{protocol=\"s3\"} %d\n", name, s3)
|
||||
fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %d\n", name, serverMain)
|
||||
}
|
||||
|
||||
func storageProtocolDurationCounters(w http.ResponseWriter, aliyunOSS, s3, serverMain uint64) {
|
||||
const name = "easyai_gateway_storage_write_duration_seconds_total"
|
||||
fmt.Fprintf(w, "# HELP %s Total time spent in object storage write attempts by bounded protocol.\n# TYPE %s counter\n", name, name)
|
||||
fmt.Fprintf(w, "%s{protocol=\"aliyun_oss\"} %.6f\n", name, float64(aliyunOSS)/float64(time.Second))
|
||||
fmt.Fprintf(w, "%s{protocol=\"s3\"} %.6f\n", name, float64(s3)/float64(time.Second))
|
||||
fmt.Fprintf(w, "%s{protocol=\"server_main_openapi\"} %.6f\n", name, float64(serverMain)/float64(time.Second))
|
||||
}
|
||||
|
||||
func publicErrorCounters(w http.ResponseWriter, values []publicerror.MetricCount) {
|
||||
const name = "easyai_gateway_public_errors_total"
|
||||
fmt.Fprintf(w, "# HELP %s Public error responses by bounded stable code.\n# TYPE %s counter\n", name, name)
|
||||
for _, value := range values {
|
||||
fmt.Fprintf(w, "%s{code=\"%s\"} %d\n", name, value.Code, value.Count)
|
||||
}
|
||||
}
|
||||
|
||||
func platformModelRateLimitUtilizationGauges(w http.ResponseWriter, statuses []store.ModelRateLimitStatus) {
|
||||
const name = "easyai_gateway_platform_model_rate_limit_utilization"
|
||||
fmt.Fprintf(w, "# HELP %s Cluster-wide platform model quota utilization.\n# TYPE %s gauge\n", name, name)
|
||||
|
||||
@@ -52,6 +52,11 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
||||
metrics.ObserveCandidateRouting("cooldown_skipped")
|
||||
metrics.ObserveCandidateRouting("disabled_skipped")
|
||||
metrics.ObserveCandidateRouting("all_full_queued")
|
||||
metrics.ObserveObjectStorage("write_success", "aliyun_oss", 128, 25*time.Millisecond)
|
||||
metrics.ObserveObjectStorage("write_failure", "s3", 0, 10*time.Millisecond)
|
||||
metrics.ObserveObjectStorage("retry", "s3", 0, 0)
|
||||
metrics.ObserveObjectStorage("failover", "s3", 0, 0)
|
||||
metrics.ObserveObjectStorage("all_failed", "", 0, 0)
|
||||
metrics.ObserveAsyncWorkerResize("success")
|
||||
metrics.ObserveConcurrencyLeaseRenewal("success")
|
||||
metrics.ObserveConcurrencyLeaseRenewal("lost")
|
||||
@@ -98,6 +103,13 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
|
||||
`easyai_gateway_candidate_routing_total{outcome="cooldown_skipped"} 1`,
|
||||
`easyai_gateway_candidate_routing_total{outcome="disabled_skipped"} 1`,
|
||||
`easyai_gateway_candidate_routing_total{outcome="all_full_queued"} 1`,
|
||||
`easyai_gateway_storage_write_attempts_total{protocol="aliyun_oss"} 1`,
|
||||
`easyai_gateway_storage_write_failures_total{protocol="s3"} 1`,
|
||||
`easyai_gateway_storage_write_duration_seconds_total{protocol="aliyun_oss"} 0.025000`,
|
||||
`easyai_gateway_storage_channel_retries_total 1`,
|
||||
`easyai_gateway_storage_channel_failovers_total 1`,
|
||||
`easyai_gateway_storage_all_channels_failed_total 1`,
|
||||
`easyai_gateway_storage_objectified_bytes_total 128`,
|
||||
`easyai_gateway_platform_model_rate_limit_utilization{platform_model_id="platform-model-1",metric="concurrent"} 0.750000`,
|
||||
`easyai_gateway_async_worker_resizes_total{outcome="success"} 1`,
|
||||
`easyai_gateway_concurrency_lease_renewals_total{outcome="success"} 1`,
|
||||
|
||||
Reference in New Issue
Block a user