feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护,并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/capacitycontroller"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/httpapi"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
@@ -37,8 +38,15 @@ func main() {
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
executionMaxConns := cfg.DatabaseMaxConns
|
||||
if cfg.DatabaseCriticalMaxConns > 0 {
|
||||
executionMaxConns -= cfg.DatabaseCriticalMaxConns
|
||||
}
|
||||
if cfg.DatabaseRiverMaxConns > 0 {
|
||||
executionMaxConns -= cfg.DatabaseRiverMaxConns
|
||||
}
|
||||
db, err := store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
|
||||
MaxConns: cfg.DatabaseMaxConns,
|
||||
MaxConns: executionMaxConns,
|
||||
MinIdleConns: cfg.DatabaseMinIdleConns,
|
||||
MaxConnIdleTime: time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
|
||||
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
|
||||
@@ -50,26 +58,78 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
coordinationDB := db
|
||||
if cfg.DatabaseCriticalMaxConns > 0 {
|
||||
coordinationDB, err = store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
|
||||
MaxConns: cfg.DatabaseCriticalMaxConns,
|
||||
MinIdleConns: min(cfg.DatabaseCriticalMaxConns, 1),
|
||||
MaxConnIdleTime: time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
|
||||
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
|
||||
LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
stop()
|
||||
logger.Error("connect critical postgres pool failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer coordinationDB.Close()
|
||||
}
|
||||
riverDB := db
|
||||
if cfg.DatabaseRiverMaxConns > 0 {
|
||||
riverDB, err = store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
|
||||
MaxConns: cfg.DatabaseRiverMaxConns,
|
||||
MinIdleConns: min(cfg.DatabaseRiverMaxConns, 1),
|
||||
MaxConnIdleTime: time.Duration(cfg.DatabaseMaxConnIdleSeconds) * time.Second,
|
||||
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
|
||||
LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
stop()
|
||||
logger.Error("connect River postgres pool failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer riverDB.Close()
|
||||
}
|
||||
// Cancel background LISTEN and worker goroutines before closing the pool.
|
||||
// This also prevents a startup panic from waiting forever in pgxpool.Close.
|
||||
defer stop()
|
||||
if recovery, err := db.RecoverInterruptedRuntimeState(ctx); err != nil {
|
||||
logger.Error("recover interrupted runtime state failed", "error", err)
|
||||
os.Exit(1)
|
||||
} else if recovery.ReleasedConcurrencyLeases > 0 || recovery.ReleasedRateReservations > 0 || recovery.FailedAttempts > 0 || recovery.FailedTasks > 0 || recovery.RequeuedAsyncTasks > 0 || recovery.CleanedTaskAdmissions > 0 {
|
||||
logger.Warn("interrupted runtime state recovered",
|
||||
"releasedConcurrencyLeases", recovery.ReleasedConcurrencyLeases,
|
||||
"releasedRateReservations", recovery.ReleasedRateReservations,
|
||||
"failedAttempts", recovery.FailedAttempts,
|
||||
"failedTasks", recovery.FailedTasks,
|
||||
"requeuedAsyncTasks", recovery.RequeuedAsyncTasks,
|
||||
"cleanedTaskAdmissions", recovery.CleanedTaskAdmissions,
|
||||
)
|
||||
if cfg.RunsBackgroundWorkers() {
|
||||
if recovery, recoveryErr := coordinationDB.RecoverInterruptedRuntimeState(ctx); recoveryErr != nil {
|
||||
logger.Error("recover interrupted runtime state failed", "error", recoveryErr)
|
||||
os.Exit(1)
|
||||
} else if recovery.ReleasedConcurrencyLeases > 0 || recovery.ReleasedRateReservations > 0 || recovery.FailedAttempts > 0 || recovery.FailedTasks > 0 || recovery.RequeuedAsyncTasks > 0 || recovery.CleanedTaskAdmissions > 0 {
|
||||
logger.Warn("interrupted runtime state recovered",
|
||||
"releasedConcurrencyLeases", recovery.ReleasedConcurrencyLeases,
|
||||
"releasedRateReservations", recovery.ReleasedRateReservations,
|
||||
"failedAttempts", recovery.FailedAttempts,
|
||||
"failedTasks", recovery.FailedTasks,
|
||||
"requeuedAsyncTasks", recovery.RequeuedAsyncTasks,
|
||||
"cleanedTaskAdmissions", recovery.CleanedTaskAdmissions,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
var handler http.Handler
|
||||
if cfg.RunsCapacityController() {
|
||||
kubernetes, kubernetesErr := capacitycontroller.NewKubernetesClient(capacitycontroller.KubernetesConfig{
|
||||
Namespace: cfg.CapacityControllerNamespace,
|
||||
APIServer: cfg.CapacityControllerAPIServer,
|
||||
TokenFile: cfg.CapacityControllerTokenFile,
|
||||
CAFile: cfg.CapacityControllerCAFile,
|
||||
})
|
||||
if kubernetesErr != nil {
|
||||
logger.Error("initialize capacity controller Kubernetes client failed", "error", kubernetesErr)
|
||||
os.Exit(1)
|
||||
}
|
||||
controller := capacitycontroller.New(cfg, coordinationDB, kubernetes, logger)
|
||||
go controller.Run(ctx)
|
||||
handler = controller.Handler()
|
||||
} else {
|
||||
handler = httpapi.NewServerWithStores(ctx, cfg, db, coordinationDB, riverDB, logger)
|
||||
}
|
||||
server := &http.Server{
|
||||
Addr: cfg.HTTPAddr,
|
||||
Handler: httpapi.NewServerWithContext(ctx, cfg, db, logger),
|
||||
Handler: handler,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user