diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 2da29a7..c441cf7 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -1078,9 +1078,10 @@ sample_pressure() { local output=$1 local stop_file=$2 local database_state node_memory_percent pod_memory_mib pool_state pod metrics + local consecutive_failures=0 echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max' >"$output" while [[ ! -f $stop_file ]]; do - database_state=$(database_query " + if ! database_state=$(database_query " SELECT count(*) FILTER (WHERE status='queued')||','|| count(*) FILTER (WHERE status='running')||','|| @@ -1088,11 +1089,17 @@ SELECT (SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','|| (SELECT setting FROM pg_settings WHERE name='max_connections') FROM gateway_tasks -WHERE acceptance_run_id='$run_id'::uuid;") || break - node_memory_percent=$(remote_kubectl top nodes --no-headers | - awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break - pod_memory_mib=$(current_max_pod_memory_mib) || break - pool_state=$( +WHERE acceptance_run_id='$run_id'::uuid;"); then + database_state= + fi + if ! node_memory_percent=$(remote_kubectl top nodes --no-headers 2>/dev/null | + awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}'); then + node_memory_percent= + fi + if ! pod_memory_mib=$(current_max_pod_memory_mib 2>/dev/null); then + pod_memory_mib= + fi + if ! pool_state=$( for workload in api worker; do for site in ningbo hongkong; do pod=$(remote_kubectl get pods -n "$namespace" \ @@ -1144,7 +1151,20 @@ WHERE acceptance_run_id='$run_id'::uuid;") || break } END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated} ' - ) || break + ) 2>/dev/null; then + pool_state= + fi + if [[ -z $database_state || -z $node_memory_percent || + -z $pod_memory_mib || -z $pool_state ]]; then + ((consecutive_failures += 1)) + if (( consecutive_failures >= 3 )); then + echo 'pressure sampling failed three consecutive times' >&2 + return 1 + fi + sleep 2 + continue + fi + consecutive_failures=0 printf '%s,%s,%s,%s,%s\n' \ "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$database_state" "$node_memory_percent" "$pod_memory_mib" "$pool_state" >>"$output" sleep 5 @@ -1201,7 +1221,7 @@ run_capacity_round() { local prefix=$report_root/$profile_slug-$repetition local pressure_report=$prefix-pressure.csv local pressure_stop=$temporary_root/pressure-stop - local pressure_pid status=0 + local pressure_pid status=0 pressure_status=0 local baseline_memory_mib local expected_slots case $profile in @@ -1219,8 +1239,11 @@ run_capacity_round() { if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi touch "$pressure_stop" - wait "$pressure_pid" || true - (( status == 0 )) || return "$status" + wait "$pressure_pid" || pressure_status=$? + (( status == 0 && pressure_status == 0 )) || { + (( status != 0 )) && return "$status" + return "$pressure_status" + } verify_pressure_report "$pressure_report" "$expected_slots" verify_runtime_gates wait_for_memory_recovery "$baseline_memory_mib"