perf(postgres): 回收突发连接并等待验收副作用
失败 Run 的未提交任务已取消,但退款 outbox 尚未完成时下一轮会继承 API 连接高水位和 canceled acquire 增量,污染容量验收。 新增 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS,生产配置 30 秒回收突发空闲连接;验收中止后等待旧任务、退款、回调和数据库连接全部收敛后才允许新负载启动。 验证:Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。
This commit is contained in:
@@ -226,21 +226,41 @@ wait_for_existing_tasks_to_drain() {
|
||||
|
||||
wait_for_terminal_acceptance_tasks_to_drain() {
|
||||
local deadline=$((SECONDS + 300))
|
||||
local active
|
||||
local state active side_effects connections max_connections
|
||||
while (( SECONDS < deadline )); do
|
||||
active=$(database_query "
|
||||
SELECT count(*)
|
||||
FROM gateway_tasks task
|
||||
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
|
||||
WHERE run.status IN ('failed', 'aborted')
|
||||
AND task.status IN ('queued', 'running');")
|
||||
if [[ $active == 0 ]]; then
|
||||
state=$(database_query "
|
||||
SELECT
|
||||
(SELECT count(*)
|
||||
FROM gateway_tasks task
|
||||
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
|
||||
WHERE run.status IN ('failed', 'aborted')
|
||||
AND task.status IN ('queued', 'running'))||','||
|
||||
(
|
||||
(SELECT count(*)
|
||||
FROM settlement_outbox settlement
|
||||
JOIN gateway_tasks task ON task.id=settlement.task_id
|
||||
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
|
||||
WHERE run.status IN ('failed', 'aborted')
|
||||
AND settlement.status IN ('pending', 'processing', 'retryable_failed'))
|
||||
+
|
||||
(SELECT count(*)
|
||||
FROM gateway_task_callback_outbox callback
|
||||
JOIN gateway_tasks task ON task.id=callback.task_id
|
||||
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
|
||||
WHERE run.status IN ('failed', 'aborted')
|
||||
AND callback.status <> 'delivered')
|
||||
)||','||
|
||||
(SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
|
||||
(SELECT setting FROM pg_settings WHERE name='max_connections');")
|
||||
IFS=',' read -r active side_effects connections max_connections <<<"$state"
|
||||
if [[ $active == 0 && $side_effects == 0 ]] &&
|
||||
(( connections * 2 < max_connections )); then
|
||||
return 0
|
||||
fi
|
||||
echo "waiting_for_terminal_acceptance_tasks=$active"
|
||||
echo "waiting_for_terminal_acceptance_tasks=$active side_effects=$side_effects database_connections=$connections"
|
||||
sleep 2
|
||||
done
|
||||
echo "terminal acceptance tasks did not drain within 5 minutes: active=$active" >&2
|
||||
echo "terminal acceptance state did not drain within 5 minutes: active=$active side_effects=$side_effects database_connections=$connections" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -675,6 +695,7 @@ apply_capacity_profile() {
|
||||
local slots pool media global
|
||||
local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
|
||||
local min_idle=4
|
||||
local max_idle_seconds=30
|
||||
case $profile in
|
||||
P24) slots=24; pool=32; media=24; global=48 ;;
|
||||
P28) slots=28; pool=36; media=28; global=56 ;;
|
||||
@@ -688,6 +709,7 @@ apply_capacity_profile() {
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$pool" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$max_idle_seconds" \
|
||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$media" \
|
||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$media" >/dev/null
|
||||
remote_kubectl rollout status "deployment/easyai-worker-$site" \
|
||||
@@ -695,7 +717,8 @@ apply_capacity_profile() {
|
||||
remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$api_pool" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" >/dev/null
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$max_idle_seconds" >/dev/null
|
||||
remote_kubectl rollout status "deployment/easyai-api-$site" \
|
||||
-n "$namespace" --timeout=300s
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user