fix(acceptance): 重试瞬时压力采样失败
Pod 滚动后的首次 metrics 读取可能短暂失败,原采样器会直接退出且主流程忽略后台状态,导致验收缺少资源曲线。\n\n采样器现在允许两次瞬时失败,连续三次失败才中止;主流程同时检查采样进程状态,确保任何曲线中断都会明确阻断验收。\n\n验证:bash -n、ShellCheck、git diff --check。
This commit is contained in:
@@ -1078,9 +1078,10 @@ sample_pressure() {
|
||||
local output=$1
|
||||
local stop_file=$2
|
||||
local database_state node_memory_percent pod_memory_mib pool_state pod metrics
|
||||
local consecutive_failures=0
|
||||
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max' >"$output"
|
||||
while [[ ! -f $stop_file ]]; do
|
||||
database_state=$(database_query "
|
||||
if ! database_state=$(database_query "
|
||||
SELECT
|
||||
count(*) FILTER (WHERE status='queued')||','||
|
||||
count(*) FILTER (WHERE status='running')||','||
|
||||
@@ -1088,11 +1089,17 @@ SELECT
|
||||
(SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
|
||||
(SELECT setting FROM pg_settings WHERE name='max_connections')
|
||||
FROM gateway_tasks
|
||||
WHERE acceptance_run_id='$run_id'::uuid;") || break
|
||||
node_memory_percent=$(remote_kubectl top nodes --no-headers |
|
||||
awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}') || break
|
||||
pod_memory_mib=$(current_max_pod_memory_mib) || break
|
||||
pool_state=$(
|
||||
WHERE acceptance_run_id='$run_id'::uuid;"); then
|
||||
database_state=
|
||||
fi
|
||||
if ! node_memory_percent=$(remote_kubectl top nodes --no-headers 2>/dev/null |
|
||||
awk '{value=$5; sub(/%$/, "", value); if (value>max) max=value} END {print max+0}'); then
|
||||
node_memory_percent=
|
||||
fi
|
||||
if ! pod_memory_mib=$(current_max_pod_memory_mib 2>/dev/null); then
|
||||
pod_memory_mib=
|
||||
fi
|
||||
if ! pool_state=$(
|
||||
for workload in api worker; do
|
||||
for site in ningbo hongkong; do
|
||||
pod=$(remote_kubectl get pods -n "$namespace" \
|
||||
@@ -1144,7 +1151,20 @@ WHERE acceptance_run_id='$run_id'::uuid;") || break
|
||||
}
|
||||
END {print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated}
|
||||
'
|
||||
) || break
|
||||
) 2>/dev/null; then
|
||||
pool_state=
|
||||
fi
|
||||
if [[ -z $database_state || -z $node_memory_percent ||
|
||||
-z $pod_memory_mib || -z $pool_state ]]; then
|
||||
((consecutive_failures += 1))
|
||||
if (( consecutive_failures >= 3 )); then
|
||||
echo 'pressure sampling failed three consecutive times' >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
continue
|
||||
fi
|
||||
consecutive_failures=0
|
||||
printf '%s,%s,%s,%s,%s\n' \
|
||||
"$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$database_state" "$node_memory_percent" "$pod_memory_mib" "$pool_state" >>"$output"
|
||||
sleep 5
|
||||
@@ -1201,7 +1221,7 @@ run_capacity_round() {
|
||||
local prefix=$report_root/$profile_slug-$repetition
|
||||
local pressure_report=$prefix-pressure.csv
|
||||
local pressure_stop=$temporary_root/pressure-stop
|
||||
local pressure_pid status=0
|
||||
local pressure_pid status=0 pressure_status=0
|
||||
local baseline_memory_mib
|
||||
local expected_slots
|
||||
case $profile in
|
||||
@@ -1219,8 +1239,11 @@ run_capacity_round() {
|
||||
if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi
|
||||
if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi
|
||||
touch "$pressure_stop"
|
||||
wait "$pressure_pid" || true
|
||||
(( status == 0 )) || return "$status"
|
||||
wait "$pressure_pid" || pressure_status=$?
|
||||
(( status == 0 && pressure_status == 0 )) || {
|
||||
(( status != 0 )) && return "$status"
|
||||
return "$pressure_status"
|
||||
}
|
||||
verify_pressure_report "$pressure_report" "$expected_slots"
|
||||
verify_runtime_gates
|
||||
wait_for_memory_recovery "$baseline_memory_mib"
|
||||
|
||||
Reference in New Issue
Block a user