fix(acceptance): 联动压力采样与负载停止
零副本 Worker 站点没有 metrics,应从聚合采样中跳过;有副本的站点采样失败仍保持硬失败。补充缺失字段诊断,避免只输出无上下文的采样失败。\n\n容量轮次和混合长稳压测现在持续监视压力采样进程;采样器提前退出时立即终止当前负载并走失败收口,避免在资源监控失效后继续造任务。\n\n验证:bash -n、ShellCheck、零副本站点测试、采样失败联动停止测试、manual-release-test。
This commit is contained in:
@@ -2074,12 +2074,12 @@ WHERE acceptance_run_id IS NULL
|
||||
sample_pressure "$pressure_report" "$pressure_stop" &
|
||||
pressure_pid=$!
|
||||
active_pressure_pid=$pressure_pid
|
||||
run_mixed_load_profile \
|
||||
run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
|
||||
mixed-soak "$AI_GATEWAY_ACCEPTANCE_SOAK_DURATION" \
|
||||
"$soak_image_rate" "$soak_video_rate" \
|
||||
"$report_root/mixed-soak.json" 3h || status=$?
|
||||
if (( status == 0 )); then
|
||||
run_mixed_load_profile \
|
||||
run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
|
||||
mixed-overload "$AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION" \
|
||||
"$overload_image_rate" "$overload_video_rate" \
|
||||
"$report_root/mixed-overload.json" 30m || status=$?
|
||||
@@ -2506,9 +2506,15 @@ verify_control_plane_and_recent_logs() {
|
||||
workload_metrics_for_site() {
|
||||
local workload=$1
|
||||
local site=$2
|
||||
local host pod_ip
|
||||
local host pod_ip replicas
|
||||
[[ $workload == api || $workload == worker ]]
|
||||
[[ $site == ningbo || $site == hongkong ]]
|
||||
if [[ $workload == worker ]]; then
|
||||
replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') || return 1
|
||||
[[ $replicas =~ ^[0-9]+$ ]] || return 1
|
||||
(( replicas > 0 )) || return 2
|
||||
fi
|
||||
if [[ $site == ningbo ]]; then
|
||||
host=$CLUSTER_NINGBO_HOST
|
||||
else
|
||||
@@ -2524,6 +2530,28 @@ workload_metrics_for_site() {
|
||||
cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics"
|
||||
}
|
||||
|
||||
run_with_pressure_monitor() {
|
||||
local pressure_pid=$1
|
||||
shift
|
||||
local load_pid load_status=0
|
||||
"$@" &
|
||||
load_pid=$!
|
||||
active_load_pid=$load_pid
|
||||
while kill -0 "$load_pid" >/dev/null 2>&1; do
|
||||
if ! kill -0 "$pressure_pid" >/dev/null 2>&1; then
|
||||
kill "$load_pid" >/dev/null 2>&1 || true
|
||||
wait "$load_pid" >/dev/null 2>&1 || true
|
||||
active_load_pid=
|
||||
echo 'pressure sampler exited before the active load completed' >&2
|
||||
return 1
|
||||
fi
|
||||
sleep "${AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS:-2}"
|
||||
done
|
||||
wait "$load_pid" || load_status=$?
|
||||
active_load_pid=
|
||||
return "$load_status"
|
||||
}
|
||||
|
||||
current_max_pod_memory_mib() {
|
||||
remote_kubectl top pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
|
||||
@@ -2801,7 +2829,7 @@ sample_pressure() {
|
||||
local output=$1
|
||||
local stop_file=$2
|
||||
local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics
|
||||
local consecutive_failures=0
|
||||
local consecutive_failures=0 missing_fields metrics_status
|
||||
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output"
|
||||
while [[ ! -f $stop_file ]]; do
|
||||
if ! database_state=$(database_query "
|
||||
@@ -2828,7 +2856,12 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
|
||||
if ! pool_state=$(
|
||||
for workload in api worker; do
|
||||
for site in ningbo hongkong; do
|
||||
metrics=$(workload_metrics_for_site "$workload" "$site") || exit
|
||||
metrics_status=0
|
||||
metrics=$(workload_metrics_for_site "$workload" "$site") || metrics_status=$?
|
||||
if (( metrics_status == 2 )); then
|
||||
continue
|
||||
fi
|
||||
(( metrics_status == 0 )) || exit 1
|
||||
awk -v workload="$workload" '
|
||||
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
|
||||
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
|
||||
@@ -2900,9 +2933,16 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
|
||||
) 2>/dev/null; then
|
||||
pool_state=
|
||||
fi
|
||||
missing_fields=
|
||||
[[ -n $database_state ]] || missing_fields+=database_state,
|
||||
[[ -n $node_memory_percent ]] || missing_fields+=node_memory_percent,
|
||||
[[ -n $pod_memory_mib ]] || missing_fields+=pod_memory_mib,
|
||||
[[ -n $worker_resources ]] || missing_fields+=worker_resources,
|
||||
[[ -n $pool_state ]] || missing_fields+=pool_state,
|
||||
if [[ -z $database_state || -z $node_memory_percent ||
|
||||
-z $pod_memory_mib || -z $worker_resources || -z $pool_state ]]; then
|
||||
((consecutive_failures += 1))
|
||||
echo "pressure sampling unavailable: attempt=$consecutive_failures fields=${missing_fields%,}" >&2
|
||||
if (( consecutive_failures >= 3 )); then
|
||||
echo 'pressure sampling failed three consecutive times' >&2
|
||||
return 1
|
||||
@@ -3053,11 +3093,24 @@ run_capacity_round() {
|
||||
sample_pressure "$pressure_report" "$pressure_stop" &
|
||||
pressure_pid=$!
|
||||
active_pressure_pid=$pressure_pid
|
||||
run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$?
|
||||
if (( status == 0 )); then run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?; fi
|
||||
if (( status == 0 )); then run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?; fi
|
||||
if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi
|
||||
if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi
|
||||
run_with_pressure_monitor "$pressure_pid" \
|
||||
run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$?
|
||||
if (( status == 0 )); then
|
||||
run_with_pressure_monitor "$pressure_pid" \
|
||||
run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?
|
||||
fi
|
||||
if (( status == 0 )); then
|
||||
run_with_pressure_monitor "$pressure_pid" \
|
||||
run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?
|
||||
fi
|
||||
if (( status == 0 )); then
|
||||
run_with_pressure_monitor "$pressure_pid" \
|
||||
run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?
|
||||
fi
|
||||
if (( status == 0 )); then
|
||||
run_with_pressure_monitor "$pressure_pid" \
|
||||
run_recovery_profile "$prefix-video-recovery.json" || status=$?
|
||||
fi
|
||||
touch "$pressure_stop"
|
||||
wait "$pressure_pid" || pressure_status=$?
|
||||
active_pressure_pid=
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC2034 # Variables are consumed by dynamically sourced functions.
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
@@ -15,6 +16,15 @@ awk '
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/is-release-ancestor.sh"
|
||||
|
||||
awk '
|
||||
/^workload_metrics_for_site\(\)/ { capture = 1 }
|
||||
capture && /^current_max_pod_memory_mib\(\)/ { exit }
|
||||
capture { print }
|
||||
' "$script" >"$tmp/pressure-monitor.sh"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/pressure-monitor.sh"
|
||||
|
||||
cluster_root=$tmp/repository
|
||||
git init -q "$cluster_root"
|
||||
git -C "$cluster_root" config user.name 'Acceptance Script Test'
|
||||
@@ -35,4 +45,39 @@ if is_release_ancestor invalid "$third"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
remote_kubectl() {
|
||||
printf 0
|
||||
}
|
||||
|
||||
cluster_ssh() {
|
||||
echo 'zero-replica Worker attempted to collect metrics' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
namespace=easyai
|
||||
if workload_metrics_for_site worker ningbo; then
|
||||
echo 'zero-replica Worker site returned metrics success' >&2
|
||||
exit 1
|
||||
else
|
||||
metrics_status=$?
|
||||
fi
|
||||
[[ $metrics_status == 2 ]]
|
||||
|
||||
long_running_load() {
|
||||
while :; do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
(sleep 1; exit 1) &
|
||||
pressure_pid=$!
|
||||
AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS=1
|
||||
active_load_pid=
|
||||
if run_with_pressure_monitor "$pressure_pid" long_running_load; then
|
||||
echo 'load continued after the pressure sampler failed' >&2
|
||||
exit 1
|
||||
fi
|
||||
wait "$pressure_pid" >/dev/null 2>&1 || true
|
||||
[[ -z $active_load_pid ]]
|
||||
|
||||
echo 'production_acceptance_script_tests=PASS'
|
||||
|
||||
Reference in New Issue
Block a user