fix(acceptance): 重试瞬时压力采样失败

Pod 滚动后的首次 metrics 读取可能短暂失败,原采样器会直接退出且主流程忽略后台状态,导致验收缺少资源曲线。\n\n采样器现在允许两次瞬时失败,连续三次失败才中止;主流程同时检查采样进程状态,确保任何曲线中断都会明确阻断验收。\n\n验证:bash -n、ShellCheck、git diff --check。
This commit is contained in:
2026-07-31 03:03:11 +08:00
parent 78005aaf85
commit 1bd9e618ca
+33 -10
View File
@@ -1078,9 +1078,10 @@ sample_pressure() {
local output=$1
local stop_file=$2
local database_state node_memory_percent pod_memory_mib pool_state pod metrics
local consecutive_failures=0
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max' >"$output"
while [[ ! -f $stop_file ]]; do
database_state=$(database_query "
if ! database_state=$(database_query "
SELECT
count(*) FILTER (WHERE status='queued')||','||
count(*) FILTER (WHERE status='running')||','||
@@ -1088,11 +1089,17 @@ SELECT
(SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
(SELECT setting FROM pg_settings WHERE name='max_connections')
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid;") || break
node_memory_percent=$(remote_kubectl top nodes --no-headers |
awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break
pod_memory_mib=$(current_max_pod_memory_mib) || break
pool_state=$(
WHERE acceptance_run_id='$run_id'::uuid;"); then
database_state=
fi
if ! node_memory_percent=$(remote_kubectl top nodes --no-headers 2>/dev/null |
awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}'); then
node_memory_percent=
fi
if ! pod_memory_mib=$(current_max_pod_memory_mib 2>/dev/null); then
pod_memory_mib=
fi
if ! pool_state=$(
for workload in api worker; do
for site in ningbo hongkong; do
pod=$(remote_kubectl get pods -n "$namespace" \
@@ -1144,7 +1151,20 @@ WHERE acceptance_run_id='$run_id'::uuid;") || break
}
END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated}
'
) || break
) 2>/dev/null; then
pool_state=
fi
if [[ -z $database_state || -z $node_memory_percent ||
-z $pod_memory_mib || -z $pool_state ]]; then
((consecutive_failures += 1))
if (( consecutive_failures >= 3 )); then
echo 'pressure sampling failed three consecutive times' >&2
return 1
fi
sleep 2
continue
fi
consecutive_failures=0
printf '%s,%s,%s,%s,%s\n' \
"$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$database_state" "$node_memory_percent" "$pod_memory_mib" "$pool_state" >>"$output"
sleep 5
@@ -1201,7 +1221,7 @@ run_capacity_round() {
local prefix=$report_root/$profile_slug-$repetition
local pressure_report=$prefix-pressure.csv
local pressure_stop=$temporary_root/pressure-stop
local pressure_pid status=0
local pressure_pid status=0 pressure_status=0
local baseline_memory_mib
local expected_slots
case $profile in
@@ -1219,8 +1239,11 @@ run_capacity_round() {
if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi
if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi
touch "$pressure_stop"
wait "$pressure_pid" || true
(( status == 0 )) || return "$status"
wait "$pressure_pid" || pressure_status=$?
(( status == 0 && pressure_status == 0 )) || {
(( status != 0 )) && return "$status"
return "$pressure_status"
}
verify_pressure_report "$pressure_report" "$expected_slots"
verify_runtime_gates
wait_for_memory_recovery "$baseline_memory_mib"