BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(acceptance): 实时执行压力资源硬门禁

Browse Source 
压力采样一旦发现节点内存、数据库连接、Pod RSS、租约或排队等待越线，立即联动停止负载，避免失败档继续堆积任务。\n\n同时保证连接池零值写入报告，并增加边界回归测试。已通过 bash -n、ShellCheck、production acceptance 脚本测试和人工发布脚本测试。
This commit is contained in:
wangbo
2026-08-01 15:44:57 +08:00
parent 3514c13b0d
commit 8ca3e2dbc2
2 changed files with 42 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cluster/run-production-acceptance.sh
+23 -9
View File
@@ -2590,6 +2590,15 @@ run_with_pressure_monitor() {
return "$load_status"
}
pressure_row_passes_live_gates() {
local pressure_row=$1
awk -F',' '
$4 > 900 || $6 <= 0 || $5 * 4 >= $6 * 3 ||
$7 >= 80 || $8 >= 1536 || $14 > 0 || $15 > 0 ||
$16 < 1 || $16 > 2 || $28 >= 1536 {exit 1}
' <<<"$pressure_row"
}
current_max_pod_memory_mib() {
remote_kubectl top pods -n "$namespace" \
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
@@ -2888,7 +2897,7 @@ sample_pressure() {
local output=$1
local stop_file=$2
local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics
local consecutive_failures=0 missing_fields metrics_status
local consecutive_failures=0 missing_fields metrics_status pressure_row
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output"
while [[ ! -f $stop_file ]]; do
if ! database_state=$(database_query "
@@ -2923,18 +2932,18 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
(( metrics_status == 0 )) || exit 1
awk -v workload="$workload" '
BEGIN {idle=999999; critical_idle=999999; river_idle=999999; active=999999}
$1=="easyai_gateway_postgres_pool_acquired_connections" && $2>acquired {acquired=$2}
$1=="easyai_gateway_postgres_pool_max_connections" && $2>pool_max {pool_max=$2}
$1=="easyai_gateway_postgres_pool_acquired_connections" && (!acquired_seen || $2>acquired) {acquired=$2; acquired_seen=1}
$1=="easyai_gateway_postgres_pool_max_connections" && (!pool_max_seen || $2>pool_max) {pool_max=$2; pool_max_seen=1}
$1=="easyai_gateway_postgres_pool_idle_connections" && $2<idle {idle=$2}
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty+=$2}
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled+=$2}
$1=="easyai_gateway_postgres_critical_pool_acquired_connections" && $2>critical_acquired {critical_acquired=$2}
$1=="easyai_gateway_postgres_critical_pool_max_connections" && $2>critical_max {critical_max=$2}
$1=="easyai_gateway_postgres_critical_pool_acquired_connections" && (!critical_acquired_seen || $2>critical_acquired) {critical_acquired=$2; critical_acquired_seen=1}
$1=="easyai_gateway_postgres_critical_pool_max_connections" && (!critical_max_seen || $2>critical_max) {critical_max=$2; critical_max_seen=1}
$1=="easyai_gateway_postgres_critical_pool_idle_connections" && $2<critical_idle {critical_idle=$2}
$1=="easyai_gateway_postgres_critical_pool_empty_acquire_total" {critical_empty+=$2}
$1=="easyai_gateway_postgres_critical_pool_canceled_acquire_total" {critical_canceled+=$2}
$1=="easyai_gateway_postgres_river_pool_acquired_connections" && $2>river_acquired {river_acquired=$2}
$1=="easyai_gateway_postgres_river_pool_max_connections" && $2>river_max {river_max=$2}
$1=="easyai_gateway_postgres_river_pool_acquired_connections" && (!river_acquired_seen || $2>river_acquired) {river_acquired=$2; river_acquired_seen=1}
$1=="easyai_gateway_postgres_river_pool_max_connections" && (!river_max_seen || $2>river_max) {river_max=$2; river_max_seen=1}
$1=="easyai_gateway_postgres_river_pool_idle_connections" && $2<river_idle {river_idle=$2}
$1=="easyai_gateway_postgres_river_pool_empty_acquire_total" {river_empty+=$2}
$1=="easyai_gateway_postgres_river_pool_canceled_acquire_total" {river_canceled+=$2}
@@ -3014,9 +3023,14 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
continue
fi
consecutive_failures=0
printf '%s,%s,%s,%s,%s,%s\n' \
printf -v pressure_row '%s,%s,%s,%s,%s,%s' \
"$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$database_state" "$node_memory_percent" \
"$pod_memory_mib" "$pool_state" "$worker_resources" >>"$output"
"$pod_memory_mib" "$pool_state" "$worker_resources"
printf '%s\n' "$pressure_row" >>"$output"
if ! pressure_row_passes_live_gates "$pressure_row"; then
echo 'pressure sampling crossed a live resource or lease hard gate' >&2
return 1
fi
sleep 5
done
}
tests/release/production-acceptance-script-test.sh
+19
View File
@@ -93,6 +93,25 @@ grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics"
[[ $(metric_max test_metric <<<"$metrics") == 1 ]]
[[ $(metric_sum test_metric <<<"$metrics") == 2 ]]
passing_pressure_row='2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100'
pressure_row_passes_live_gates "$passing_pressure_row"
for failed_pressure_row in \
'2026-08-01T00:00:00Z,0,0,901,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,150,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,149,200,80,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1536,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,1,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,1,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,0,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,3,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1536,100'; do
if pressure_row_passes_live_gates "$failed_pressure_row"; then
echo "unsafe pressure row passed live gates: $failed_pressure_row" >&2
exit 1
fi
done
long_running_load() {
while :; do
sleep 1