feat(acceptance): 建立同构验收与弹性容量体系

实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护,并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
+74 -14
View File
@@ -10,6 +10,7 @@ import (
"syscall"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/capacitycontroller"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/httpapi"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
@@ -37,8 +38,15 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
executionMaxConns := cfg.DatabaseMaxConns
if cfg.DatabaseCriticalMaxConns > 0 {
executionMaxConns -= cfg.DatabaseCriticalMaxConns
}
if cfg.DatabaseRiverMaxConns > 0 {
executionMaxConns -= cfg.DatabaseRiverMaxConns
}
db, err := store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
MaxConns: cfg.DatabaseMaxConns,
MaxConns: executionMaxConns,
MinIdleConns: cfg.DatabaseMinIdleConns,
MaxConnIdleTime: time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
@@ -50,26 +58,78 @@ func main() {
os.Exit(1)
}
defer db.Close()
coordinationDB := db
if cfg.DatabaseCriticalMaxConns > 0 {
coordinationDB, err = store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
MaxConns: cfg.DatabaseCriticalMaxConns,
MinIdleConns: min(cfg.DatabaseCriticalMaxConns, 1),
MaxConnIdleTime: time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,
})
if err != nil {
stop()
logger.Error("connect critical postgres pool failed", "error", err)
os.Exit(1)
}
defer coordinationDB.Close()
}
riverDB := db
if cfg.DatabaseRiverMaxConns > 0 {
riverDB, err = store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
MaxConns: cfg.DatabaseRiverMaxConns,
MinIdleConns: min(cfg.DatabaseRiverMaxConns, 1),
MaxConnIdleTime: time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,
})
if err != nil {
stop()
logger.Error("connect River postgres pool failed", "error", err)
os.Exit(1)
}
defer riverDB.Close()
}
// Cancel background LISTEN and worker goroutines before closing the pool.
// This also prevents a startup panic from waiting forever in pgxpool.Close.
defer stop()
if recovery, err := db.RecoverInterruptedRuntimeState(ctx); err != nil {
logger.Error("recover interrupted runtime state failed", "error", err)
os.Exit(1)
} else if recovery.ReleasedConcurrencyLeases > 0 || recovery.ReleasedRateReservations > 0 || recovery.FailedAttempts > 0 || recovery.FailedTasks > 0 || recovery.RequeuedAsyncTasks > 0 || recovery.CleanedTaskAdmissions > 0 {
logger.Warn("interrupted runtime state recovered",
"releasedConcurrencyLeases", recovery.ReleasedConcurrencyLeases,
"releasedRateReservations", recovery.ReleasedRateReservations,
"failedAttempts", recovery.FailedAttempts,
"failedTasks", recovery.FailedTasks,
"requeuedAsyncTasks", recovery.RequeuedAsyncTasks,
"cleanedTaskAdmissions", recovery.CleanedTaskAdmissions,
)
if cfg.RunsBackgroundWorkers() {
if recovery, recoveryErr := coordinationDB.RecoverInterruptedRuntimeState(ctx); recoveryErr != nil {
logger.Error("recover interrupted runtime state failed", "error", recoveryErr)
os.Exit(1)
} else if recovery.ReleasedConcurrencyLeases > 0 || recovery.ReleasedRateReservations > 0 || recovery.FailedAttempts > 0 || recovery.FailedTasks > 0 || recovery.RequeuedAsyncTasks > 0 || recovery.CleanedTaskAdmissions > 0 {
logger.Warn("interrupted runtime state recovered",
"releasedConcurrencyLeases", recovery.ReleasedConcurrencyLeases,
"releasedRateReservations", recovery.ReleasedRateReservations,
"failedAttempts", recovery.FailedAttempts,
"failedTasks", recovery.FailedTasks,
"requeuedAsyncTasks", recovery.RequeuedAsyncTasks,
"cleanedTaskAdmissions", recovery.CleanedTaskAdmissions,
)
}
}
var handler http.Handler
if cfg.RunsCapacityController() {
kubernetes, kubernetesErr := capacitycontroller.NewKubernetesClient(capacitycontroller.KubernetesConfig{
Namespace: cfg.CapacityControllerNamespace,
APIServer: cfg.CapacityControllerAPIServer,
TokenFile: cfg.CapacityControllerTokenFile,
CAFile: cfg.CapacityControllerCAFile,
})
if kubernetesErr != nil {
logger.Error("initialize capacity controller Kubernetes client failed", "error", kubernetesErr)
os.Exit(1)
}
controller := capacitycontroller.New(cfg, coordinationDB, kubernetes, logger)
go controller.Run(ctx)
handler = controller.Handler()
} else {
handler = httpapi.NewServerWithStores(ctx, cfg, db, coordinationDB, riverDB, logger)
}
server := &http.Server{
Addr: cfg.HTTPAddr,
Handler: httpapi.NewServerWithContext(ctx, cfg, db, logger),
Handler: handler,
ReadHeaderTimeout: 10 * time.Second,
}