From 53fc79691a7bebeccb0d7b60a6edeaee0365e804 Mon Sep 17 00:00:00 2001 From: wangbo Date: Sat, 1 Aug 2026 14:37:45 +0800 Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E8=81=94=E5=8A=A8=E5=8E=8B?= =?UTF-8?q?=E5=8A=9B=E9=87=87=E6=A0=B7=E4=B8=8E=E8=B4=9F=E8=BD=BD=E5=81=9C?= =?UTF-8?q?=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 零副本 Worker 站点没有 metrics，应从聚合采样中跳过；有副本的站点采样失败仍保持硬失败。补充缺失字段诊断，避免只输出无上下文的采样失败。\n\n容量轮次和混合长稳压测现在持续监视压力采样进程；采样器提前退出时立即终止当前负载并走失败收口，避免在资源监控失效后继续造任务。\n\n验证：bash -n、ShellCheck、零副本站点测试、采样失败联动停止测试、manual-release-test。 --- scripts/cluster/run-production-acceptance.sh | 73 ++++++++++++++++--- .../production-acceptance-script-test.sh | 45 ++++++++++++ 2 files changed, 108 insertions(+), 10 deletions(-) diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 39d23dc..98fde55 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -2074,12 +2074,12 @@ WHERE acceptance_run_id IS NULL sample_pressure "$pressure_report" "$pressure_stop" & pressure_pid=$! active_pressure_pid=$pressure_pid - run_mixed_load_profile \ + run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \ mixed-soak "$AI_GATEWAY_ACCEPTANCE_SOAK_DURATION" \ "$soak_image_rate" "$soak_video_rate" \ "$report_root/mixed-soak.json" 3h || status=$? if (( status == 0 )); then - run_mixed_load_profile \ + run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \ mixed-overload "$AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION" \ "$overload_image_rate" "$overload_video_rate" \ "$report_root/mixed-overload.json" 30m || status=$? @@ -2506,9 +2506,15 @@ verify_control_plane_and_recent_logs() { workload_metrics_for_site() { local workload=$1 local site=$2 - local host pod_ip + local host pod_ip replicas [[ $workload == api || $workload == worker ]] [[ $site == ningbo || $site == hongkong ]] + if [[ $workload == worker ]]; then + replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \ + -o 'jsonpath={.spec.replicas}') || return 1 + [[ $replicas =~ ^[0-9]+$ ]] || return 1 + (( replicas > 0 )) || return 2 + fi if [[ $site == ningbo ]]; then host=$CLUSTER_NINGBO_HOST else @@ -2524,6 +2530,28 @@ workload_metrics_for_site() { cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics" } +run_with_pressure_monitor() { + local pressure_pid=$1 + shift + local load_pid load_status=0 + "$@" & + load_pid=$! + active_load_pid=$load_pid + while kill -0 "$load_pid" >/dev/null 2>&1; do + if ! kill -0 "$pressure_pid" >/dev/null 2>&1; then + kill "$load_pid" >/dev/null 2>&1 || true + wait "$load_pid" >/dev/null 2>&1 || true + active_load_pid= + echo 'pressure sampler exited before the active load completed' >&2 + return 1 + fi + sleep "${AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS:-2}" + done + wait "$load_pid" || load_status=$? + active_load_pid= + return "$load_status" +} + current_max_pod_memory_mib() { remote_kubectl top pods -n "$namespace" \ -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers | @@ -2801,7 +2829,7 @@ sample_pressure() { local output=$1 local stop_file=$2 local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics - local consecutive_failures=0 + local consecutive_failures=0 missing_fields metrics_status echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output" while [[ ! -f $stop_file ]]; do if ! database_state=$(database_query " @@ -2828,7 +2856,12 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then if ! pool_state=$( for workload in api worker; do for site in ningbo hongkong; do - metrics=$(workload_metrics_for_site "$workload" "$site") || exit + metrics_status=0 + metrics=$(workload_metrics_for_site "$workload" "$site") || metrics_status=$? + if (( metrics_status == 2 )); then + continue + fi + (( metrics_status == 0 )) || exit 1 awk -v workload="$workload" ' $1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2} $1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2} @@ -2900,9 +2933,16 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then ) 2>/dev/null; then pool_state= fi + missing_fields= + [[ -n $database_state ]] || missing_fields+=database_state, + [[ -n $node_memory_percent ]] || missing_fields+=node_memory_percent, + [[ -n $pod_memory_mib ]] || missing_fields+=pod_memory_mib, + [[ -n $worker_resources ]] || missing_fields+=worker_resources, + [[ -n $pool_state ]] || missing_fields+=pool_state, if [[ -z $database_state || -z $node_memory_percent || -z $pod_memory_mib || -z $worker_resources || -z $pool_state ]]; then ((consecutive_failures += 1)) + echo "pressure sampling unavailable: attempt=$consecutive_failures fields=${missing_fields%,}" >&2 if (( consecutive_failures >= 3 )); then echo 'pressure sampling failed three consecutive times' >&2 return 1 @@ -3053,11 +3093,24 @@ run_capacity_round() { sample_pressure "$pressure_report" "$pressure_stop" & pressure_pid=$! active_pressure_pid=$pressure_pid - run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$? - if (( status == 0 )); then run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?; fi - if (( status == 0 )); then run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?; fi - if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi - if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi + run_with_pressure_monitor "$pressure_pid" \ + run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$? + if (( status == 0 )); then + run_with_pressure_monitor "$pressure_pid" \ + run_load_profile gemini-large "$prefix-gemini-large.json" || status=$? + fi + if (( status == 0 )); then + run_with_pressure_monitor "$pressure_pid" \ + run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$? + fi + if (( status == 0 )); then + run_with_pressure_monitor "$pressure_pid" \ + run_load_profile video-throughput "$prefix-video-throughput.json" || status=$? + fi + if (( status == 0 )); then + run_with_pressure_monitor "$pressure_pid" \ + run_recovery_profile "$prefix-video-recovery.json" || status=$? + fi touch "$pressure_stop" wait "$pressure_pid" || pressure_status=$? active_pressure_pid= diff --git a/tests/release/production-acceptance-script-test.sh b/tests/release/production-acceptance-script-test.sh index f816a87..94ffbdd 100755 --- a/tests/release/production-acceptance-script-test.sh +++ b/tests/release/production-acceptance-script-test.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2034 # Variables are consumed by dynamically sourced functions. set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) @@ -15,6 +16,15 @@ awk ' # shellcheck source=/dev/null source "$tmp/is-release-ancestor.sh" +awk ' + /^workload_metrics_for_site\(\)/ { capture = 1 } + capture && /^current_max_pod_memory_mib\(\)/ { exit } + capture { print } +' "$script" >"$tmp/pressure-monitor.sh" + +# shellcheck source=/dev/null +source "$tmp/pressure-monitor.sh" + cluster_root=$tmp/repository git init -q "$cluster_root" git -C "$cluster_root" config user.name 'Acceptance Script Test' @@ -35,4 +45,39 @@ if is_release_ancestor invalid "$third"; then exit 1 fi +remote_kubectl() { + printf 0 +} + +cluster_ssh() { + echo 'zero-replica Worker attempted to collect metrics' >&2 + return 1 +} + +namespace=easyai +if workload_metrics_for_site worker ningbo; then + echo 'zero-replica Worker site returned metrics success' >&2 + exit 1 +else + metrics_status=$? +fi +[[ $metrics_status == 2 ]] + +long_running_load() { + while :; do + sleep 1 + done +} + +(sleep 1; exit 1) & +pressure_pid=$! +AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS=1 +active_load_pid= +if run_with_pressure_monitor "$pressure_pid" long_running_load; then + echo 'load continued after the pressure sampler failed' >&2 + exit 1 +fi +wait "$pressure_pid" >/dev/null 2>&1 || true +[[ -z $active_load_pid ]] + echo 'production_acceptance_script_tests=PASS'