From 8ca3e2dbc2de7024728a86755034855398c04959 Mon Sep 17 00:00:00 2001 From: wangbo Date: Sat, 1 Aug 2026 15:44:57 +0800 Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E5=AE=9E=E6=97=B6=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E5=8E=8B=E5=8A=9B=E8=B5=84=E6=BA=90=E7=A1=AC=E9=97=A8?= =?UTF-8?q?=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 压力采样一旦发现节点内存、数据库连接、Pod RSS、租约或排队等待越线，立即联动停止负载，避免失败档继续堆积任务。\n\n同时保证连接池零值写入报告，并增加边界回归测试。已通过 bash -n、ShellCheck、production acceptance 脚本测试和人工发布脚本测试。 --- scripts/cluster/run-production-acceptance.sh | 32 +++++++++++++------ .../production-acceptance-script-test.sh | 19 +++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 1f12bec..2917111 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -2590,6 +2590,15 @@ run_with_pressure_monitor() { return "$load_status" } +pressure_row_passes_live_gates() { + local pressure_row=$1 + awk -F',' ' + $4 > 900 || $6 <= 0 || $5 * 4 >= $6 * 3 || + $7 >= 80 || $8 >= 1536 || $14 > 0 || $15 > 0 || + $16 < 1 || $16 > 2 || $28 >= 1536 {exit 1} + ' <<<"$pressure_row" +} + current_max_pod_memory_mib() { remote_kubectl top pods -n "$namespace" \ -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers | @@ -2888,7 +2897,7 @@ sample_pressure() { local output=$1 local stop_file=$2 local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics - local consecutive_failures=0 missing_fields metrics_status + local consecutive_failures=0 missing_fields metrics_status pressure_row echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output" while [[ ! -f $stop_file ]]; do if ! database_state=$(database_query " @@ -2923,18 +2932,18 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then (( metrics_status == 0 )) || exit 1 awk -v workload="$workload" ' BEGIN {idle=999999; critical_idle=999999; river_idle=999999; active=999999} - $1=="easyai_gateway_postgres_pool_acquired_connections" && $2>acquired {acquired=$2} - $1=="easyai_gateway_postgres_pool_max_connections" && $2>pool_max {pool_max=$2} + $1=="easyai_gateway_postgres_pool_acquired_connections" && (!acquired_seen || $2>acquired) {acquired=$2; acquired_seen=1} + $1=="easyai_gateway_postgres_pool_max_connections" && (!pool_max_seen || $2>pool_max) {pool_max=$2; pool_max_seen=1} $1=="easyai_gateway_postgres_pool_idle_connections" && $2critical_acquired {critical_acquired=$2} - $1=="easyai_gateway_postgres_critical_pool_max_connections" && $2>critical_max {critical_max=$2} + $1=="easyai_gateway_postgres_critical_pool_acquired_connections" && (!critical_acquired_seen || $2>critical_acquired) {critical_acquired=$2; critical_acquired_seen=1} + $1=="easyai_gateway_postgres_critical_pool_max_connections" && (!critical_max_seen || $2>critical_max) {critical_max=$2; critical_max_seen=1} $1=="easyai_gateway_postgres_critical_pool_idle_connections" && $2river_acquired {river_acquired=$2} - $1=="easyai_gateway_postgres_river_pool_max_connections" && $2>river_max {river_max=$2} + $1=="easyai_gateway_postgres_river_pool_acquired_connections" && (!river_acquired_seen || $2>river_acquired) {river_acquired=$2; river_acquired_seen=1} + $1=="easyai_gateway_postgres_river_pool_max_connections" && (!river_max_seen || $2>river_max) {river_max=$2; river_max_seen=1} $1=="easyai_gateway_postgres_river_pool_idle_connections" && $2>"$output" + "$pod_memory_mib" "$pool_state" "$worker_resources" + printf '%s\n' "$pressure_row" >>"$output" + if ! pressure_row_passes_live_gates "$pressure_row"; then + echo 'pressure sampling crossed a live resource or lease hard gate' >&2 + return 1 + fi sleep 5 done } diff --git a/tests/release/production-acceptance-script-test.sh b/tests/release/production-acceptance-script-test.sh index 2e9b78f..9d35dea 100755 --- a/tests/release/production-acceptance-script-test.sh +++ b/tests/release/production-acceptance-script-test.sh @@ -93,6 +93,25 @@ grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics" [[ $(metric_max test_metric <<<"$metrics") == 1 ]] [[ $(metric_sum test_metric <<<"$metrics") == 2 ]] +passing_pressure_row='2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' +pressure_row_passes_live_gates "$passing_pressure_row" + +for failed_pressure_row in \ + '2026-08-01T00:00:00Z,0,0,901,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,150,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,149,200,80,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,149,200,79,1536,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,1,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,1,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,0,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,3,48,0,8,8,0,0,0,16,16,0,0,1535,100' \ + '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1536,100'; do + if pressure_row_passes_live_gates "$failed_pressure_row"; then + echo "unsafe pressure row passed live gates: $failed_pressure_row" >&2 + exit 1 + fi +done + long_running_load() { while :; do sleep 1