feat(acceptance): 增加生产同构媒体压力验收模式

引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器,覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。

新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排;Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。

验证:Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。
This commit is contained in:
2026-07-30 23:06:19 +08:00
parent f33d6d64e0
commit 3053ba4925
45 changed files with 5214 additions and 45 deletions
@@ -279,6 +279,12 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
return
}
}
postgresPool := store.PostgresPoolMetricsSnapshot{}
if poolProvider, ok := provider.(interface {
PostgresPoolMetrics() store.PostgresPoolMetricsSnapshot
}); ok {
postgresPool = poolProvider.PostgresPoolMetrics()
}
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{
{"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()},
@@ -350,6 +356,12 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
plainGauge(w, "easyai_gateway_worker_active_instances", "Active distributed worker instances with a fresh heartbeat.", m.asyncWorkerActiveInstances.Load())
plainGauge(w, "easyai_gateway_worker_global_capacity", "Global asynchronous execution capacity before instance allocation.", m.asyncWorkerGlobalCapacity.Load())
plainGauge(w, "easyai_gateway_worker_allocated_capacity", "Asynchronous execution capacity allocated to this worker instance.", m.asyncWorkerAllocated.Load())
plainGauge(w, "easyai_gateway_postgres_pool_max_connections", "Maximum PostgreSQL connections in this process pool.", int64(postgresPool.MaxConnections))
plainGauge(w, "easyai_gateway_postgres_pool_total_connections", "Current PostgreSQL connections in this process pool.", int64(postgresPool.TotalConnections))
plainGauge(w, "easyai_gateway_postgres_pool_acquired_connections", "Currently acquired PostgreSQL connections in this process pool.", int64(postgresPool.AcquiredConnections))
plainGauge(w, "easyai_gateway_postgres_pool_idle_connections", "Currently idle PostgreSQL connections in this process pool.", int64(postgresPool.IdleConnections))
plainCounter(w, "easyai_gateway_postgres_pool_empty_acquire_total", "Pool acquires that had to wait for a PostgreSQL connection.", uint64(max(postgresPool.EmptyAcquireCount, 0)))
plainCounter(w, "easyai_gateway_postgres_pool_canceled_acquire_total", "Canceled PostgreSQL pool acquires.", uint64(max(postgresPool.CanceledAcquireCount, 0)))
outcomeCounters(w, "easyai_gateway_async_worker_resizes_total", "Asynchronous worker resize attempts by bounded outcome.", []outcomeValue{
{"success", m.asyncWorkerResizeSuccess.Load()},
{"refresh_failed", m.asyncWorkerRefreshFailed.Load()},
@@ -7,17 +7,24 @@ import (
"strings"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
type metricsSnapshot struct {
mode string
last time.Time
pool store.PostgresPoolMetricsSnapshot
}
func (m metricsSnapshot) SecurityEventMetrics(context.Context, string, string) (string, *time.Time, error) {
return m.mode, &m.last, nil
}
func (m metricsSnapshot) PostgresPoolMetrics() store.PostgresPoolMetricsSnapshot {
return m.pool
}
func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
metrics := &Metrics{}
metrics.ObserveReceipt("accepted", 20*time.Millisecond, 2)
@@ -34,7 +41,14 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
metrics.ObserveTaskEventSkip("budget_exceeded")
recorder := httptest.NewRecorder()
metrics.Handler(metricsSnapshot{mode: "push_healthy", last: time.Now().Add(-time.Minute)}, "issuer", "audience", true).
metrics.Handler(metricsSnapshot{
mode: "push_healthy",
last: time.Now().Add(-time.Minute),
pool: store.PostgresPoolMetricsSnapshot{
MaxConnections: 32, TotalConnections: 18, AcquiredConnections: 12,
IdleConnections: 6, EmptyAcquireCount: 3, CanceledAcquireCount: 1,
},
}, "issuer", "audience", true).
ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("metrics status=%d", recorder.Code)
@@ -56,6 +70,12 @@ func TestMetricsExposeBoundedOutcomesAndState(t *testing.T) {
`easyai_gateway_concurrency_lease_renewals_total{outcome="lost"} 1`,
`easyai_gateway_task_events_skipped_total{outcome="duplicate"} 1`,
`easyai_gateway_task_events_skipped_total{outcome="budget_exceeded"} 1`,
`easyai_gateway_postgres_pool_max_connections 32`,
`easyai_gateway_postgres_pool_total_connections 18`,
`easyai_gateway_postgres_pool_acquired_connections 12`,
`easyai_gateway_postgres_pool_idle_connections 6`,
`easyai_gateway_postgres_pool_empty_acquire_total 3`,
`easyai_gateway_postgres_pool_canceled_acquire_total 1`,
} {
if !strings.Contains(recorder.Body.String(), expected) {
t.Fatalf("missing metric %q in:\n%s", expected, recorder.Body.String())