fix(acceptance): 联动压力采样与负载停止

零副本 Worker 站点没有 metrics,应从聚合采样中跳过;有副本的站点采样失败仍保持硬失败。补充缺失字段诊断,避免只输出无上下文的采样失败。\n\n容量轮次和混合长稳压测现在持续监视压力采样进程;采样器提前退出时立即终止当前负载并走失败收口,避免在资源监控失效后继续造任务。\n\n验证:bash -n、ShellCheck、零副本站点测试、采样失败联动停止测试、manual-release-test。
This commit is contained in:
2026-08-01 14:37:45 +08:00
parent 8749f38eb3
commit 53fc79691a
2 changed files with 108 additions and 10 deletions
+63 -10
View File
@@ -2074,12 +2074,12 @@ WHERE acceptance_run_id IS NULL
sample_pressure "$pressure_report" "$pressure_stop" &
pressure_pid=$!
active_pressure_pid=$pressure_pid
run_mixed_load_profile \
run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
mixed-soak "$AI_GATEWAY_ACCEPTANCE_SOAK_DURATION" \
"$soak_image_rate" "$soak_video_rate" \
"$report_root/mixed-soak.json" 3h || status=$?
if (( status == 0 )); then
run_mixed_load_profile \
run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
mixed-overload "$AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION" \
"$overload_image_rate" "$overload_video_rate" \
"$report_root/mixed-overload.json" 30m || status=$?
@@ -2506,9 +2506,15 @@ verify_control_plane_and_recent_logs() {
workload_metrics_for_site() {
local workload=$1
local site=$2
local host pod_ip
local host pod_ip replicas
[[ $workload == api || $workload == worker ]]
[[ $site == ningbo || $site == hongkong ]]
if [[ $workload == worker ]]; then
replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \
-o 'jsonpath={.spec.replicas}') || return 1
[[ $replicas =~ ^[0-9]+$ ]] || return 1
(( replicas > 0 )) || return 2
fi
if [[ $site == ningbo ]]; then
host=$CLUSTER_NINGBO_HOST
else
@@ -2524,6 +2530,28 @@ workload_metrics_for_site() {
cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics"
}
run_with_pressure_monitor() {
local pressure_pid=$1
shift
local load_pid load_status=0
"$@" &
load_pid=$!
active_load_pid=$load_pid
while kill -0 "$load_pid" >/dev/null 2>&1; do
if ! kill -0 "$pressure_pid" >/dev/null 2>&1; then
kill "$load_pid" >/dev/null 2>&1 || true
wait "$load_pid" >/dev/null 2>&1 || true
active_load_pid=
echo 'pressure sampler exited before the active load completed' >&2
return 1
fi
sleep "${AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS:-2}"
done
wait "$load_pid" || load_status=$?
active_load_pid=
return "$load_status"
}
current_max_pod_memory_mib() {
remote_kubectl top pods -n "$namespace" \
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
@@ -2801,7 +2829,7 @@ sample_pressure() {
local output=$1
local stop_file=$2
local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics
local consecutive_failures=0
local consecutive_failures=0 missing_fields metrics_status
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output"
while [[ ! -f $stop_file ]]; do
if ! database_state=$(database_query "
@@ -2828,7 +2856,12 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
if ! pool_state=$(
for workload in api worker; do
for site in ningbo hongkong; do
metrics=$(workload_metrics_for_site "$workload" "$site") || exit
metrics_status=0
metrics=$(workload_metrics_for_site "$workload" "$site") || metrics_status=$?
if (( metrics_status == 2 )); then
continue
fi
(( metrics_status == 0 )) || exit 1
awk -v workload="$workload" '
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
@@ -2900,9 +2933,16 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
) 2>/dev/null; then
pool_state=
fi
missing_fields=
[[ -n $database_state ]] || missing_fields+=database_state,
[[ -n $node_memory_percent ]] || missing_fields+=node_memory_percent,
[[ -n $pod_memory_mib ]] || missing_fields+=pod_memory_mib,
[[ -n $worker_resources ]] || missing_fields+=worker_resources,
[[ -n $pool_state ]] || missing_fields+=pool_state,
if [[ -z $database_state || -z $node_memory_percent ||
-z $pod_memory_mib || -z $worker_resources || -z $pool_state ]]; then
((consecutive_failures += 1))
echo "pressure sampling unavailable: attempt=$consecutive_failures fields=${missing_fields%,}" >&2
if (( consecutive_failures >= 3 )); then
echo 'pressure sampling failed three consecutive times' >&2
return 1
@@ -3053,11 +3093,24 @@ run_capacity_round() {
sample_pressure "$pressure_report" "$pressure_stop" &
pressure_pid=$!
active_pressure_pid=$pressure_pid
run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$?
if (( status == 0 )); then run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?; fi
if (( status == 0 )); then run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?; fi
if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi
if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi
run_with_pressure_monitor "$pressure_pid" \
run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$?
if (( status == 0 )); then
run_with_pressure_monitor "$pressure_pid" \
run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?
fi
if (( status == 0 )); then
run_with_pressure_monitor "$pressure_pid" \
run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?
fi
if (( status == 0 )); then
run_with_pressure_monitor "$pressure_pid" \
run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?
fi
if (( status == 0 )); then
run_with_pressure_monitor "$pressure_pid" \
run_recovery_profile "$prefix-video-recovery.json" || status=$?
fi
touch "$pressure_stop"
wait "$pressure_pid" || pressure_status=$?
active_pressure_pid=