perf(postgres): 分离 API 池并降低连接预热
生产同构千并发请求在干净队列上复现 API Pool 31/32、idle=1 且 canceled acquire 持续增长,Worker Pool 仅使用少量连接,确认瓶颈位于 API 数据库连接预算而非 Worker 执行槽。 新增可配置的 AI_GATEWAY_DATABASE_MIN_IDLE_CONNS,生产仅预热 4 条连接;API Pool 独立固定为 48,Worker Pool 继续跟随 P24/P28/P32 的 32/36/40 档位。发布工具、Kubernetes 清单、验收门禁和文档同步更新。 验证:Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。
This commit is contained in:
@@ -164,7 +164,7 @@ AI_GATEWAY_DATABASE_URL=postgresql://easyai:easyai2025@localhost:5432/easyai_ai_
|
|||||||
|
|
||||||
如果现有 `easyai-pgvector` 没有把 `5432` 映射到宿主机,就需要补端口映射,或者把 AI Gateway 后端容器化后接入同一个 `easyai` Docker network。
|
如果现有 `easyai-pgvector` 没有把 `5432` 映射到宿主机,就需要补端口映射,或者把 AI Gateway 后端容器化后接入同一个 `easyai` Docker network。
|
||||||
|
|
||||||
异步队列 worker 不使用固定业务并发。服务分别汇总启用平台模型和活跃用户组的有效 `concurrent` 策略,采用两者中更严格的集群容量,默认每 5 秒在线调整 River 执行容量。策略解析兼容历史 `platformLimits/modelLimits.max_concurrent_requests`,运行时统一转换为 `rules`。`AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT` 默认 `2048`,限制策略推导出的集群目标;`AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT` 默认 `32`,按单 Worker 的内存安全容量限制实例分配,即使其他实例失活也不会突破。Worker 的 `AI_GATEWAY_DATABASE_MAX_CONNS` 必须高于实例执行容量,为心跳、选主、租约续期和健康检查保留连接;生产环境按每实例执行容量 `24` 配置连接池 `32`。平台模型和用户组的 PostgreSQL concurrency lease 仍是业务并发真值。可通过 `AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS` 调整刷新周期。
|
异步队列 worker 不使用固定业务并发。服务分别汇总启用平台模型和活跃用户组的有效 `concurrent` 策略,采用两者中更严格的集群容量,默认每 5 秒在线调整 River 执行容量。策略解析兼容历史 `platformLimits/modelLimits.max_concurrent_requests`,运行时统一转换为 `rules`。`AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT` 默认 `2048`,限制策略推导出的集群目标;`AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT` 默认 `32`,按单 Worker 的内存安全容量限制实例分配,即使其他实例失活也不会突破。Worker 的 `AI_GATEWAY_DATABASE_MAX_CONNS` 必须高于实例执行容量,为心跳、选主、租约续期和健康检查保留连接;生产环境按每实例执行容量 `24` 配置连接池上限 `32`。`AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 控制启动预热连接数,默认 `0`,生产 K3s 配置为 `4`,避免把连接池上限全部常驻占用。平台模型和用户组的 PostgreSQL concurrency lease 仍是业务并发真值。可通过 `AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS` 调整刷新周期。
|
||||||
|
|
||||||
## 迁移原则
|
## 迁移原则
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ func main() {
|
|||||||
|
|
||||||
db, err := store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
|
db, err := store.ConnectWithPoolOptions(ctx, cfg.DatabaseURL, store.PostgresPoolOptions{
|
||||||
MaxConns: cfg.DatabaseMaxConns,
|
MaxConns: cfg.DatabaseMaxConns,
|
||||||
|
MinIdleConns: cfg.DatabaseMinIdleConns,
|
||||||
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
|
IdleInTransactionTimeout: time.Duration(cfg.DatabaseIdleInTransactionTimeoutSeconds) * time.Second,
|
||||||
LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,
|
LockTimeout: time.Duration(cfg.DatabaseLockTimeoutSeconds) * time.Second,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ type Config struct {
|
|||||||
BillingEngineMode string
|
BillingEngineMode string
|
||||||
ProcessRole string
|
ProcessRole string
|
||||||
DatabaseMaxConns int
|
DatabaseMaxConns int
|
||||||
|
DatabaseMinIdleConns int
|
||||||
DatabaseIdleInTransactionTimeoutSeconds int
|
DatabaseIdleInTransactionTimeoutSeconds int
|
||||||
DatabaseLockTimeoutSeconds int
|
DatabaseLockTimeoutSeconds int
|
||||||
MediaRequestConcurrency int
|
MediaRequestConcurrency int
|
||||||
@@ -122,6 +123,7 @@ func Load() Config {
|
|||||||
BillingEngineMode: strings.ToLower(env("BILLING_ENGINE_MODE", "observe")),
|
BillingEngineMode: strings.ToLower(env("BILLING_ENGINE_MODE", "observe")),
|
||||||
ProcessRole: strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_PROCESS_ROLE", "all"))),
|
ProcessRole: strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_PROCESS_ROLE", "all"))),
|
||||||
DatabaseMaxConns: envInt("AI_GATEWAY_DATABASE_MAX_CONNS", 0),
|
DatabaseMaxConns: envInt("AI_GATEWAY_DATABASE_MAX_CONNS", 0),
|
||||||
|
DatabaseMinIdleConns: envOptionalIntValidated("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", 0),
|
||||||
|
|
||||||
DatabaseIdleInTransactionTimeoutSeconds: envOptionalIntValidated(
|
DatabaseIdleInTransactionTimeoutSeconds: envOptionalIntValidated(
|
||||||
"AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS",
|
"AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS",
|
||||||
@@ -147,6 +149,10 @@ func (c Config) Validate() error {
|
|||||||
if c.DatabaseMaxConns < 0 || c.DatabaseMaxConns > 1000 {
|
if c.DatabaseMaxConns < 0 || c.DatabaseMaxConns > 1000 {
|
||||||
return errors.New("AI_GATEWAY_DATABASE_MAX_CONNS must be between 1 and 1000 when configured")
|
return errors.New("AI_GATEWAY_DATABASE_MAX_CONNS must be between 1 and 1000 when configured")
|
||||||
}
|
}
|
||||||
|
if c.DatabaseMinIdleConns < 0 || c.DatabaseMinIdleConns > 1000 ||
|
||||||
|
(c.DatabaseMaxConns > 0 && c.DatabaseMinIdleConns > c.DatabaseMaxConns) {
|
||||||
|
return errors.New("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS must be between 0 and AI_GATEWAY_DATABASE_MAX_CONNS")
|
||||||
|
}
|
||||||
if c.DatabaseIdleInTransactionTimeoutSeconds < 0 || c.DatabaseIdleInTransactionTimeoutSeconds > 3600 {
|
if c.DatabaseIdleInTransactionTimeoutSeconds < 0 || c.DatabaseIdleInTransactionTimeoutSeconds > 3600 {
|
||||||
return errors.New("AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS must be between 0 and 3600")
|
return errors.New("AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS must be between 0 and 3600")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -144,9 +144,12 @@ func TestProcessRolePrecedenceAndCompatibility(t *testing.T) {
|
|||||||
|
|
||||||
func TestValidateProcessRoleAndDatabasePool(t *testing.T) {
|
func TestValidateProcessRoleAndDatabasePool(t *testing.T) {
|
||||||
cfg := Load()
|
cfg := Load()
|
||||||
if cfg.DatabaseIdleInTransactionTimeoutSeconds != 60 || cfg.DatabaseLockTimeoutSeconds != 30 {
|
if cfg.DatabaseMinIdleConns != 0 ||
|
||||||
|
cfg.DatabaseIdleInTransactionTimeoutSeconds != 60 ||
|
||||||
|
cfg.DatabaseLockTimeoutSeconds != 30 {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"database transaction timeouts = %d/%d, want 60/30",
|
"database pool minimum/transaction timeouts = %d/%d/%d, want 0/60/30",
|
||||||
|
cfg.DatabaseMinIdleConns,
|
||||||
cfg.DatabaseIdleInTransactionTimeoutSeconds,
|
cfg.DatabaseIdleInTransactionTimeoutSeconds,
|
||||||
cfg.DatabaseLockTimeoutSeconds,
|
cfg.DatabaseLockTimeoutSeconds,
|
||||||
)
|
)
|
||||||
@@ -161,6 +164,16 @@ func TestValidateProcessRoleAndDatabasePool(t *testing.T) {
|
|||||||
t.Fatalf("Validate() error = %v, want invalid database max conns", err)
|
t.Fatalf("Validate() error = %v, want invalid database max conns", err)
|
||||||
}
|
}
|
||||||
cfg.DatabaseMaxConns = 16
|
cfg.DatabaseMaxConns = 16
|
||||||
|
cfg.DatabaseMinIdleConns = 17
|
||||||
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MIN_IDLE_CONNS") {
|
||||||
|
t.Fatalf("Validate() error = %v, want invalid database min idle conns", err)
|
||||||
|
}
|
||||||
|
cfg.DatabaseMinIdleConns = 4
|
||||||
|
t.Setenv("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", "not-an-integer")
|
||||||
|
if err := Load().Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MIN_IDLE_CONNS") {
|
||||||
|
t.Fatalf("Validate() error = %v, want invalid non-integer database min idle conns", err)
|
||||||
|
}
|
||||||
|
t.Setenv("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", "4")
|
||||||
cfg.DatabaseIdleInTransactionTimeoutSeconds = 3601
|
cfg.DatabaseIdleInTransactionTimeoutSeconds = 3601
|
||||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDLE_IN_TRANSACTION") {
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDLE_IN_TRANSACTION") {
|
||||||
t.Fatalf("Validate() error = %v, want invalid idle transaction timeout", err)
|
t.Fatalf("Validate() error = %v, want invalid idle transaction timeout", err)
|
||||||
|
|||||||
@@ -27,11 +27,11 @@ type Store struct {
|
|||||||
const (
|
const (
|
||||||
postgresApplicationName = "easyai-ai-gateway"
|
postgresApplicationName = "easyai-ai-gateway"
|
||||||
postgresConnectTimeout = 5 * time.Second
|
postgresConnectTimeout = 5 * time.Second
|
||||||
postgresPoolWarmLimit = 64
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type PostgresPoolOptions struct {
|
type PostgresPoolOptions struct {
|
||||||
MaxConns int
|
MaxConns int
|
||||||
|
MinIdleConns int
|
||||||
IdleInTransactionTimeout time.Duration
|
IdleInTransactionTimeout time.Duration
|
||||||
LockTimeout time.Duration
|
LockTimeout time.Duration
|
||||||
}
|
}
|
||||||
@@ -130,8 +130,8 @@ func postgresPoolConfigWithOptions(databaseURL string, options PostgresPoolOptio
|
|||||||
}
|
}
|
||||||
if options.MaxConns > 0 {
|
if options.MaxConns > 0 {
|
||||||
config.MaxConns = int32(options.MaxConns)
|
config.MaxConns = int32(options.MaxConns)
|
||||||
config.MinIdleConns = int32(min(options.MaxConns, postgresPoolWarmLimit))
|
|
||||||
}
|
}
|
||||||
|
config.MinIdleConns = int32(min(max(options.MinIdleConns, 0), int(config.MaxConns)))
|
||||||
return config, nil
|
return config, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestPostgresPoolConfigRejectsMalformedURL(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPostgresPoolConfigWarmsBoundedIdleConnections(t *testing.T) {
|
func TestPostgresPoolConfigUsesExplicitMinimumIdleConnections(t *testing.T) {
|
||||||
config, err := postgresPoolConfig(
|
config, err := postgresPoolConfig(
|
||||||
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
||||||
24,
|
24,
|
||||||
@@ -35,23 +35,23 @@ func TestPostgresPoolConfigWarmsBoundedIdleConnections(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("parse PostgreSQL pool config: %v", err)
|
t.Fatalf("parse PostgreSQL pool config: %v", err)
|
||||||
}
|
}
|
||||||
if config.MaxConns != 24 || config.MinIdleConns != 24 {
|
if config.MaxConns != 24 || config.MinIdleConns != 0 {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"pool bounds max=%d minIdle=%d, want 24/24",
|
"pool bounds max=%d minIdle=%d, want 24/0",
|
||||||
config.MaxConns,
|
config.MaxConns,
|
||||||
config.MinIdleConns,
|
config.MinIdleConns,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
small, err := postgresPoolConfig(
|
warmed, err := postgresPoolConfigWithOptions(
|
||||||
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
||||||
4,
|
PostgresPoolOptions{MaxConns: 24, MinIdleConns: 4},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("parse small PostgreSQL pool config: %v", err)
|
t.Fatalf("parse warmed PostgreSQL pool config: %v", err)
|
||||||
}
|
}
|
||||||
if small.MinIdleConns != 4 {
|
if warmed.MinIdleConns != 4 {
|
||||||
t.Fatalf("small pool minIdle=%d, want 4", small.MinIdleConns)
|
t.Fatalf("warmed pool minIdle=%d, want 4", warmed.MinIdleConns)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,6 +60,7 @@ func TestPostgresPoolConfigSetsBoundedTransactionTimeouts(t *testing.T) {
|
|||||||
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
"postgresql://gateway:password@localhost:5432/gateway?sslmode=disable",
|
||||||
PostgresPoolOptions{
|
PostgresPoolOptions{
|
||||||
MaxConns: 32,
|
MaxConns: 32,
|
||||||
|
MinIdleConns: 4,
|
||||||
IdleInTransactionTimeout: 60 * time.Second,
|
IdleInTransactionTimeout: 60 * time.Second,
|
||||||
LockTimeout: 30 * time.Second,
|
LockTimeout: 30 * time.Second,
|
||||||
},
|
},
|
||||||
@@ -73,8 +74,8 @@ func TestPostgresPoolConfigSetsBoundedTransactionTimeouts(t *testing.T) {
|
|||||||
if got := config.ConnConfig.RuntimeParams["lock_timeout"]; got != "30000ms" {
|
if got := config.ConnConfig.RuntimeParams["lock_timeout"]; got != "30000ms" {
|
||||||
t.Fatalf("lock timeout = %q, want 30000ms", got)
|
t.Fatalf("lock timeout = %q, want 30000ms", got)
|
||||||
}
|
}
|
||||||
if config.MaxConns != 32 || config.MinIdleConns != 32 {
|
if config.MaxConns != 32 || config.MinIdleConns != 4 {
|
||||||
t.Fatalf("pool bounds max=%d minIdle=%d, want 32/32", config.MaxConns, config.MinIdleConns)
|
t.Fatalf("pool bounds max=%d minIdle=%d, want 32/4", config.MaxConns, config.MinIdleConns)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ source "$config_file"
|
|||||||
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
||||||
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
||||||
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
|
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
|
||||||
|
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=48}"
|
||||||
|
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
|
||||||
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
||||||
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
||||||
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
||||||
@@ -31,6 +33,7 @@ for capacity_value in \
|
|||||||
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||||
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||||
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||||
|
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||||
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||||
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
|
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
|
||||||
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
|
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
|
||||||
@@ -38,11 +41,18 @@ for capacity_value in \
|
|||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
done
|
done
|
||||||
|
[[ $AI_GATEWAY_DATABASE_MIN_IDLE_CONNS =~ ^[0-9]+$ ]] || {
|
||||||
|
echo 'database minimum idle connection configuration must be a non-negative integer' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
|
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
|
||||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
|
||||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
|
||||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
|
||||||
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
|
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
|
||||||
|
AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
|
||||||
|
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
|
||||||
|
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
|
||||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
|
||||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
|
||||||
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
||||||
@@ -127,6 +137,7 @@ rollout_worker_site() {
|
|||||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||||
|
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
|
||||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
|
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
|
||||||
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||||
@@ -146,7 +157,8 @@ rollout_api_capacity_site() {
|
|||||||
local site=$1
|
local site=$1
|
||||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||||
|
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
|
||||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,7 +170,7 @@ apply_capacity_config() {
|
|||||||
verify_site hongkong
|
verify_site hongkong
|
||||||
verify_site ningbo
|
verify_site ningbo
|
||||||
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
||||||
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
||||||
}
|
}
|
||||||
|
|
||||||
rollout_site() {
|
rollout_site() {
|
||||||
@@ -170,7 +182,8 @@ rollout_site() {
|
|||||||
if [[ $api_changed == true ]]; then
|
if [[ $api_changed == true ]]; then
|
||||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS"
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
|
||||||
|
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS"
|
||||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||||
"api=$api_image"
|
"api=$api_image"
|
||||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||||
|
|||||||
@@ -10,5 +10,7 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
|||||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
||||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
||||||
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
||||||
|
AI_GATEWAY_API_DATABASE_MAX_CONNS=48
|
||||||
|
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4
|
||||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
||||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
||||||
|
|||||||
@@ -53,7 +53,9 @@ spec:
|
|||||||
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
||||||
value: "false"
|
value: "false"
|
||||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||||
value: "32"
|
value: "48"
|
||||||
|
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||||
|
value: "4"
|
||||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||||
value: "16"
|
value: "16"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
@@ -182,7 +184,9 @@ spec:
|
|||||||
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
|
||||||
value: "false"
|
value: "false"
|
||||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||||
value: "32"
|
value: "48"
|
||||||
|
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||||
|
value: "4"
|
||||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||||
value: "16"
|
value: "16"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
@@ -312,6 +316,8 @@ spec:
|
|||||||
value: "true"
|
value: "true"
|
||||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||||
value: "32"
|
value: "32"
|
||||||
|
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||||
|
value: "4"
|
||||||
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
||||||
value: "24"
|
value: "24"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
@@ -443,6 +449,8 @@ spec:
|
|||||||
value: "true"
|
value: "true"
|
||||||
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
- name: AI_GATEWAY_DATABASE_MAX_CONNS
|
||||||
value: "32"
|
value: "32"
|
||||||
|
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||||
|
value: "4"
|
||||||
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
- name: AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
||||||
value: "24"
|
value: "24"
|
||||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ WebP 和 4K 图片。128 组输入组合避免只命中相同缓存;每四个
|
|||||||
```text
|
```text
|
||||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT
|
||||||
AI_GATEWAY_DATABASE_MAX_CONNS
|
AI_GATEWAY_DATABASE_MAX_CONNS
|
||||||
|
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
|
||||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO
|
||||||
@@ -79,6 +80,11 @@ AI_GATEWAY_WORKER_REPLICAS_HONGKONG
|
|||||||
| P28 | 28 | 36 | 28 | 56 |
|
| P28 | 28 | 36 | 28 | 56 |
|
||||||
| P32 | 32 | 40 | 32 | 64 |
|
| P32 | 32 | 40 | 32 | 64 |
|
||||||
|
|
||||||
|
API Pod 使用独立发布配置 `AI_GATEWAY_API_DATABASE_MAX_CONNS`,生产同构验收固定为 `48`。
|
||||||
|
所有 API/Worker Pool 的 `AI_GATEWAY_DATABASE_MIN_IDLE_CONNS` 固定为 `4`,仅预热必要连接,
|
||||||
|
其余连接按负载弹性创建,避免四个 Pod 在空闲时就占满 PostgreSQL 连接预算。各档表中的
|
||||||
|
数据库池仍表示单个 Worker 的最大连接数。
|
||||||
|
|
||||||
每档依次运行全部五个模拟 Profile 三次。失败时恢复上一个已完整通过的档位,流量保持
|
每档依次运行全部五个模拟 Profile 三次。失败时恢复上一个已完整通过的档位,流量保持
|
||||||
`validation`,不会自动放开正式请求。
|
`validation`,不会自动放开正式请求。
|
||||||
|
|
||||||
|
|||||||
@@ -667,6 +667,8 @@ mark_run_failed() {
|
|||||||
apply_capacity_profile() {
|
apply_capacity_profile() {
|
||||||
local profile=$1
|
local profile=$1
|
||||||
local slots pool media global
|
local slots pool media global
|
||||||
|
local api_pool=48
|
||||||
|
local min_idle=4
|
||||||
case $profile in
|
case $profile in
|
||||||
P24) slots=24; pool=32; media=24; global=48 ;;
|
P24) slots=24; pool=32; media=24; global=48 ;;
|
||||||
P28) slots=28; pool=36; media=28; global=56 ;;
|
P28) slots=28; pool=36; media=28; global=56 ;;
|
||||||
@@ -679,13 +681,15 @@ apply_capacity_profile() {
|
|||||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$slots" \
|
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$slots" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
||||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$pool" \
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$pool" \
|
||||||
|
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \
|
||||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$media" \
|
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$media" \
|
||||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$media" >/dev/null
|
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$media" >/dev/null
|
||||||
remote_kubectl rollout status "deployment/easyai-worker-$site" \
|
remote_kubectl rollout status "deployment/easyai-worker-$site" \
|
||||||
-n "$namespace" --timeout=300s
|
-n "$namespace" --timeout=300s
|
||||||
remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \
|
remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \
|
||||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
||||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$pool" >/dev/null
|
"AI_GATEWAY_DATABASE_MAX_CONNS=$api_pool" \
|
||||||
|
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" >/dev/null
|
||||||
remote_kubectl rollout status "deployment/easyai-api-$site" \
|
remote_kubectl rollout status "deployment/easyai-api-$site" \
|
||||||
-n "$namespace" --timeout=300s
|
-n "$namespace" --timeout=300s
|
||||||
done
|
done
|
||||||
@@ -895,6 +899,7 @@ wait_for_memory_recovery() {
|
|||||||
|
|
||||||
verify_runtime_gates() {
|
verify_runtime_gates() {
|
||||||
local expected_slots expected_global expected_pool
|
local expected_slots expected_global expected_pool
|
||||||
|
local expected_api_pool=48
|
||||||
case $active_profile in
|
case $active_profile in
|
||||||
P24) expected_slots=24; expected_pool=32; expected_global=48 ;;
|
P24) expected_slots=24; expected_pool=32; expected_global=48 ;;
|
||||||
P28) expected_slots=28; expected_pool=36; expected_global=56 ;;
|
P28) expected_slots=28; expected_pool=36; expected_global=56 ;;
|
||||||
@@ -1055,8 +1060,8 @@ WHERE status='active'
|
|||||||
pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics")
|
pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics")
|
||||||
acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics")
|
acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics")
|
||||||
idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics")
|
idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics")
|
||||||
[[ ${pool_max%.*} == "$expected_pool" ]]
|
[[ ${pool_max%.*} == "$expected_api_pool" ]]
|
||||||
if [[ ${acquired%.*} == "$expected_pool" && ${idle%.*} == 0 ]]; then
|
if [[ ${acquired%.*} == "$expected_api_pool" && ${idle%.*} == 0 ]]; then
|
||||||
echo "API connection pool saturated for site $site" >&2
|
echo "API connection pool saturated for site $site" >&2
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
Reference in New Issue
Block a user