fix(acceptance): 联动压力采样与负载停止
零副本 Worker 站点没有 metrics,应从聚合采样中跳过;有副本的站点采样失败仍保持硬失败。补充缺失字段诊断,避免只输出无上下文的采样失败。\n\n容量轮次和混合长稳压测现在持续监视压力采样进程;采样器提前退出时立即终止当前负载并走失败收口,避免在资源监控失效后继续造任务。\n\n验证:bash -n、ShellCheck、零副本站点测试、采样失败联动停止测试、manual-release-test。
This commit is contained in:
@@ -2074,12 +2074,12 @@ WHERE acceptance_run_id IS NULL
|
|||||||
sample_pressure "$pressure_report" "$pressure_stop" &
|
sample_pressure "$pressure_report" "$pressure_stop" &
|
||||||
pressure_pid=$!
|
pressure_pid=$!
|
||||||
active_pressure_pid=$pressure_pid
|
active_pressure_pid=$pressure_pid
|
||||||
run_mixed_load_profile \
|
run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
|
||||||
mixed-soak "$AI_GATEWAY_ACCEPTANCE_SOAK_DURATION" \
|
mixed-soak "$AI_GATEWAY_ACCEPTANCE_SOAK_DURATION" \
|
||||||
"$soak_image_rate" "$soak_video_rate" \
|
"$soak_image_rate" "$soak_video_rate" \
|
||||||
"$report_root/mixed-soak.json" 3h || status=$?
|
"$report_root/mixed-soak.json" 3h || status=$?
|
||||||
if (( status == 0 )); then
|
if (( status == 0 )); then
|
||||||
run_mixed_load_profile \
|
run_with_pressure_monitor "$pressure_pid" run_mixed_load_profile \
|
||||||
mixed-overload "$AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION" \
|
mixed-overload "$AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION" \
|
||||||
"$overload_image_rate" "$overload_video_rate" \
|
"$overload_image_rate" "$overload_video_rate" \
|
||||||
"$report_root/mixed-overload.json" 30m || status=$?
|
"$report_root/mixed-overload.json" 30m || status=$?
|
||||||
@@ -2506,9 +2506,15 @@ verify_control_plane_and_recent_logs() {
|
|||||||
workload_metrics_for_site() {
|
workload_metrics_for_site() {
|
||||||
local workload=$1
|
local workload=$1
|
||||||
local site=$2
|
local site=$2
|
||||||
local host pod_ip
|
local host pod_ip replicas
|
||||||
[[ $workload == api || $workload == worker ]]
|
[[ $workload == api || $workload == worker ]]
|
||||||
[[ $site == ningbo || $site == hongkong ]]
|
[[ $site == ningbo || $site == hongkong ]]
|
||||||
|
if [[ $workload == worker ]]; then
|
||||||
|
replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \
|
||||||
|
-o 'jsonpath={.spec.replicas}') || return 1
|
||||||
|
[[ $replicas =~ ^[0-9]+$ ]] || return 1
|
||||||
|
(( replicas > 0 )) || return 2
|
||||||
|
fi
|
||||||
if [[ $site == ningbo ]]; then
|
if [[ $site == ningbo ]]; then
|
||||||
host=$CLUSTER_NINGBO_HOST
|
host=$CLUSTER_NINGBO_HOST
|
||||||
else
|
else
|
||||||
@@ -2524,6 +2530,28 @@ workload_metrics_for_site() {
|
|||||||
cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics"
|
cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
run_with_pressure_monitor() {
|
||||||
|
local pressure_pid=$1
|
||||||
|
shift
|
||||||
|
local load_pid load_status=0
|
||||||
|
"$@" &
|
||||||
|
load_pid=$!
|
||||||
|
active_load_pid=$load_pid
|
||||||
|
while kill -0 "$load_pid" >/dev/null 2>&1; do
|
||||||
|
if ! kill -0 "$pressure_pid" >/dev/null 2>&1; then
|
||||||
|
kill "$load_pid" >/dev/null 2>&1 || true
|
||||||
|
wait "$load_pid" >/dev/null 2>&1 || true
|
||||||
|
active_load_pid=
|
||||||
|
echo 'pressure sampler exited before the active load completed' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep "${AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS:-2}"
|
||||||
|
done
|
||||||
|
wait "$load_pid" || load_status=$?
|
||||||
|
active_load_pid=
|
||||||
|
return "$load_status"
|
||||||
|
}
|
||||||
|
|
||||||
current_max_pod_memory_mib() {
|
current_max_pod_memory_mib() {
|
||||||
remote_kubectl top pods -n "$namespace" \
|
remote_kubectl top pods -n "$namespace" \
|
||||||
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
|
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
|
||||||
@@ -2801,7 +2829,7 @@ sample_pressure() {
|
|||||||
local output=$1
|
local output=$1
|
||||||
local stop_file=$2
|
local stop_file=$2
|
||||||
local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics
|
local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics
|
||||||
local consecutive_failures=0
|
local consecutive_failures=0 missing_fields metrics_status
|
||||||
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output"
|
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output"
|
||||||
while [[ ! -f $stop_file ]]; do
|
while [[ ! -f $stop_file ]]; do
|
||||||
if ! database_state=$(database_query "
|
if ! database_state=$(database_query "
|
||||||
@@ -2828,7 +2856,12 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
|
|||||||
if ! pool_state=$(
|
if ! pool_state=$(
|
||||||
for workload in api worker; do
|
for workload in api worker; do
|
||||||
for site in ningbo hongkong; do
|
for site in ningbo hongkong; do
|
||||||
metrics=$(workload_metrics_for_site "$workload" "$site") || exit
|
metrics_status=0
|
||||||
|
metrics=$(workload_metrics_for_site "$workload" "$site") || metrics_status=$?
|
||||||
|
if (( metrics_status == 2 )); then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
(( metrics_status == 0 )) || exit 1
|
||||||
awk -v workload="$workload" '
|
awk -v workload="$workload" '
|
||||||
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
|
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
|
||||||
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
|
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
|
||||||
@@ -2900,9 +2933,16 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
|
|||||||
) 2>/dev/null; then
|
) 2>/dev/null; then
|
||||||
pool_state=
|
pool_state=
|
||||||
fi
|
fi
|
||||||
|
missing_fields=
|
||||||
|
[[ -n $database_state ]] || missing_fields+=database_state,
|
||||||
|
[[ -n $node_memory_percent ]] || missing_fields+=node_memory_percent,
|
||||||
|
[[ -n $pod_memory_mib ]] || missing_fields+=pod_memory_mib,
|
||||||
|
[[ -n $worker_resources ]] || missing_fields+=worker_resources,
|
||||||
|
[[ -n $pool_state ]] || missing_fields+=pool_state,
|
||||||
if [[ -z $database_state || -z $node_memory_percent ||
|
if [[ -z $database_state || -z $node_memory_percent ||
|
||||||
-z $pod_memory_mib || -z $worker_resources || -z $pool_state ]]; then
|
-z $pod_memory_mib || -z $worker_resources || -z $pool_state ]]; then
|
||||||
((consecutive_failures += 1))
|
((consecutive_failures += 1))
|
||||||
|
echo "pressure sampling unavailable: attempt=$consecutive_failures fields=${missing_fields%,}" >&2
|
||||||
if (( consecutive_failures >= 3 )); then
|
if (( consecutive_failures >= 3 )); then
|
||||||
echo 'pressure sampling failed three consecutive times' >&2
|
echo 'pressure sampling failed three consecutive times' >&2
|
||||||
return 1
|
return 1
|
||||||
@@ -3053,11 +3093,24 @@ run_capacity_round() {
|
|||||||
sample_pressure "$pressure_report" "$pressure_stop" &
|
sample_pressure "$pressure_report" "$pressure_stop" &
|
||||||
pressure_pid=$!
|
pressure_pid=$!
|
||||||
active_pressure_pid=$pressure_pid
|
active_pressure_pid=$pressure_pid
|
||||||
run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$?
|
run_with_pressure_monitor "$pressure_pid" \
|
||||||
if (( status == 0 )); then run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?; fi
|
run_load_profile gemini-baseline "$prefix-gemini-baseline.json" || status=$?
|
||||||
if (( status == 0 )); then run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?; fi
|
if (( status == 0 )); then
|
||||||
if (( status == 0 )); then run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?; fi
|
run_with_pressure_monitor "$pressure_pid" \
|
||||||
if (( status == 0 )); then run_recovery_profile "$prefix-video-recovery.json" || status=$?; fi
|
run_load_profile gemini-large "$prefix-gemini-large.json" || status=$?
|
||||||
|
fi
|
||||||
|
if (( status == 0 )); then
|
||||||
|
run_with_pressure_monitor "$pressure_pid" \
|
||||||
|
run_load_profile gemini-peak "$prefix-gemini-peak.json" || status=$?
|
||||||
|
fi
|
||||||
|
if (( status == 0 )); then
|
||||||
|
run_with_pressure_monitor "$pressure_pid" \
|
||||||
|
run_load_profile video-throughput "$prefix-video-throughput.json" || status=$?
|
||||||
|
fi
|
||||||
|
if (( status == 0 )); then
|
||||||
|
run_with_pressure_monitor "$pressure_pid" \
|
||||||
|
run_recovery_profile "$prefix-video-recovery.json" || status=$?
|
||||||
|
fi
|
||||||
touch "$pressure_stop"
|
touch "$pressure_stop"
|
||||||
wait "$pressure_pid" || pressure_status=$?
|
wait "$pressure_pid" || pressure_status=$?
|
||||||
active_pressure_pid=
|
active_pressure_pid=
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
# shellcheck disable=SC2034 # Variables are consumed by dynamically sourced functions.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||||
@@ -15,6 +16,15 @@ awk '
|
|||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "$tmp/is-release-ancestor.sh"
|
source "$tmp/is-release-ancestor.sh"
|
||||||
|
|
||||||
|
awk '
|
||||||
|
/^workload_metrics_for_site\(\)/ { capture = 1 }
|
||||||
|
capture && /^current_max_pod_memory_mib\(\)/ { exit }
|
||||||
|
capture { print }
|
||||||
|
' "$script" >"$tmp/pressure-monitor.sh"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "$tmp/pressure-monitor.sh"
|
||||||
|
|
||||||
cluster_root=$tmp/repository
|
cluster_root=$tmp/repository
|
||||||
git init -q "$cluster_root"
|
git init -q "$cluster_root"
|
||||||
git -C "$cluster_root" config user.name 'Acceptance Script Test'
|
git -C "$cluster_root" config user.name 'Acceptance Script Test'
|
||||||
@@ -35,4 +45,39 @@ if is_release_ancestor invalid "$third"; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
remote_kubectl() {
|
||||||
|
printf 0
|
||||||
|
}
|
||||||
|
|
||||||
|
cluster_ssh() {
|
||||||
|
echo 'zero-replica Worker attempted to collect metrics' >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace=easyai
|
||||||
|
if workload_metrics_for_site worker ningbo; then
|
||||||
|
echo 'zero-replica Worker site returned metrics success' >&2
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
metrics_status=$?
|
||||||
|
fi
|
||||||
|
[[ $metrics_status == 2 ]]
|
||||||
|
|
||||||
|
long_running_load() {
|
||||||
|
while :; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
(sleep 1; exit 1) &
|
||||||
|
pressure_pid=$!
|
||||||
|
AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS=1
|
||||||
|
active_load_pid=
|
||||||
|
if run_with_pressure_monitor "$pressure_pid" long_running_load; then
|
||||||
|
echo 'load continued after the pressure sampler failed' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
wait "$pressure_pid" >/dev/null 2>&1 || true
|
||||||
|
[[ -z $active_load_pid ]]
|
||||||
|
|
||||||
echo 'production_acceptance_script_tests=PASS'
|
echo 'production_acceptance_script_tests=PASS'
|
||||||
|
|||||||
Reference in New Issue
Block a user