perf(postgres): 回收突发连接并等待验收副作用

失败 Run 的未提交任务已取消,但退款 outbox 尚未完成时下一轮会继承 API 连接高水位和 canceled acquire 增量,污染容量验收。

新增 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS,生产配置 30 秒回收突发空闲连接;验收中止后等待旧任务、退款、回调和数据库连接全部收敛后才允许新负载启动。

验证:Go 全量测试、gofmt、bash -n、ShellCheck、kubectl kustomize 和 git diff --check 通过。
This commit is contained in:
2026-07-31 03:46:41 +08:00
parent 75a56b21bc
commit 36546bd5c4
11 changed files with 80 additions and 18 deletions
+34 -11
View File
@@ -226,21 +226,41 @@ wait_for_existing_tasks_to_drain() {
wait_for_terminal_acceptance_tasks_to_drain() {
local deadline=$((SECONDS + 300))
local active
local state active side_effects connections max_connections
while (( SECONDS < deadline )); do
active=$(database_query "
SELECT count(*)
FROM gateway_tasks task
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
WHERE run.status IN ('failed', 'aborted')
AND task.status IN ('queued', 'running');")
if [[ $active == 0 ]]; then
state=$(database_query "
SELECT
(SELECT count(*)
FROM gateway_tasks task
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
WHERE run.status IN ('failed', 'aborted')
AND task.status IN ('queued', 'running'))||','||
(
(SELECT count(*)
FROM settlement_outbox settlement
JOIN gateway_tasks task ON task.id=settlement.task_id
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
WHERE run.status IN ('failed', 'aborted')
AND settlement.status IN ('pending', 'processing', 'retryable_failed'))
+
(SELECT count(*)
FROM gateway_task_callback_outbox callback
JOIN gateway_tasks task ON task.id=callback.task_id
JOIN gateway_acceptance_runs run ON run.id=task.acceptance_run_id
WHERE run.status IN ('failed', 'aborted')
AND callback.status <> 'delivered')
)||','||
(SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
(SELECT setting FROM pg_settings WHERE name='max_connections');")
IFS=',' read -r active side_effects connections max_connections <<<"$state"
if [[ $active == 0 && $side_effects == 0 ]] &&
(( connections * 2 < max_connections )); then
return 0
fi
echo "waiting_for_terminal_acceptance_tasks=$active"
echo "waiting_for_terminal_acceptance_tasks=$active side_effects=$side_effects database_connections=$connections"
sleep 2
done
echo "terminal acceptance tasks did not drain within 5 minutes: active=$active" >&2
echo "terminal acceptance state did not drain within 5 minutes: active=$active side_effects=$side_effects database_connections=$connections" >&2
return 1
}
@@ -675,6 +695,7 @@ apply_capacity_profile() {
local slots pool media global
local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
local min_idle=4
local max_idle_seconds=30
case $profile in
P24) slots=24; pool=32; media=24; global=48 ;;
P28) slots=28; pool=36; media=28; global=56 ;;
@@ -688,6 +709,7 @@ apply_capacity_profile() {
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$pool" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$max_idle_seconds" \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$media" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$media" >/dev/null
remote_kubectl rollout status "deployment/easyai-worker-$site" \
@@ -695,7 +717,8 @@ apply_capacity_profile() {
remote_kubectl set env "deployment/easyai-api-$site" -n "$namespace" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$global" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$api_pool" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" >/dev/null
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$min_idle" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$max_idle_seconds" >/dev/null
remote_kubectl rollout status "deployment/easyai-api-$site" \
-n "$namespace" --timeout=300s
done