BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(acceptance): 按实际节点聚合多 Worker 指标

Browse Source 
同站点存在多个物理节点时，按每个 Ready Pod 的 nodeName 映射真实 SSH 主机采集 metrics，并隔离 ssh 标准输入，避免吞掉后续 Pod 目标。所有 Worker Pod 指标按容量最小/最大、连接池最小/最大和计数器总和聚合，不再随机漏采一个实例。\n\n运行时门禁根据已启用的第四节点计算 Ready 节点数，并跳过零副本站点，同时继续逐 Pod 校验 P24/P28/P32 容量、连接池和租约。\n\n验证：bash -n、ShellCheck、双节点 metrics 映射测试、生产只读双 Worker metrics 探针、manual-release-test。
This commit is contained in:
wangbo
2026-08-01 15:14:39 +08:00
parent 53fc79691a
commit 3514c13b0d
2 changed files with 154 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cluster/run-production-acceptance.sh
+120 -57
View File
@@ -2506,28 +2506,66 @@ verify_control_plane_and_recent_logs() {
workload_metrics_for_site() { workload_metrics_for_site() {
local workload=$1 local workload=$1
local site=$2 local site=$2
local host pod_ip replicas local host pod_ip node_name replicas targets
[[ $workload == api || $workload == worker ]] [[ $workload == api || $workload == worker ]]
[[ $site == ningbo || $site == hongkong ]] [[ $site == ningbo || $site == hongkong ]]
if [[ $workload == worker ]]; then targets=$(remote_kubectl get pods -n "$namespace" \
-l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json |
jq -r '.items[]
| select(.metadata.deletionTimestamp == null)
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
| [.status.podIP,.spec.nodeName]
| @tsv') || return 1
if [[ -z $targets && $workload == worker ]]; then
replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \ replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \
-o 'jsonpath={.spec.replicas}') || return 1 -o 'jsonpath={.spec.replicas}') || return 1
[[ $replicas =~ ^[0-9]+$ ]] || return 1 [[ $replicas =~ ^[0-9]+$ ]] || return 1
(( replicas > 0 )) || return 2 (( replicas > 0 )) || return 2
fi fi
if [[ $site == ningbo ]]; then [[ -n $targets ]] || return 1
host=$CLUSTER_NINGBO_HOST while IFS=$'\t' read -r pod_ip node_name; do
[[ $pod_ip =~ ^[0-9a-fA-F:.]+$ && $node_name =~ ^[a-z0-9-]+$ ]] || return 1
case $node_name in
easyai-ningbo) host=$CLUSTER_NINGBO_HOST ;;
easyai-hongkong) host=$CLUSTER_HONGKONG_HOST ;;
easyai-los-angeles) host=$CLUSTER_LOS_ANGELES_HOST ;;
*)
if [[ -n $CLUSTER_WORKER_NODE_4_NAME &&
$node_name == "$CLUSTER_WORKER_NODE_4_NAME" ]]; then
host=$CLUSTER_WORKER_NODE_4_HOST
else else
host=$CLUSTER_HONGKONG_HOST echo "metrics target node has no SSH mapping: node=$node_name" >&2
return 1
fi fi
pod_ip=$(remote_kubectl get pods -n "$namespace" \ ;;
-l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json | esac
jq -r '[.items[] cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics" \
| select(.metadata.deletionTimestamp == null) </dev/null || return 1
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True")) done <<<"$targets"
| .status.podIP][0] // empty') }
[[ $pod_ip =~ ^[0-9a-fA-F:.]+$ ]] || return 1
cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics" metric_min() {
local metric=$1
awk -v metric="$metric" '
$1==metric && (!seen || $2<value) {value=$2; seen=1}
END {if (seen) print value}
'
}
metric_max() {
local metric=$1
awk -v metric="$metric" '
$1==metric && (!seen || $2>value) {value=$2; seen=1}
END {if (seen) print value}
'
}
metric_sum() {
local metric=$1
awk -v metric="$metric" '
$1==metric {value+=$2; seen=1}
END {if (seen) print value}
'
} }
run_with_pressure_monitor() { run_with_pressure_monitor() {
@@ -2599,6 +2637,7 @@ wait_for_memory_recovery() {
verify_runtime_gates() { verify_runtime_gates() {
local expected_slots expected_global expected_pool expected_worker_execution_pool expected_api_execution_pool local expected_slots expected_global expected_pool expected_worker_execution_pool expected_api_execution_pool
local expected_ready_nodes=3
local expected_critical_pool=4 local expected_critical_pool=4
local expected_api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS local expected_api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
local expected_worker_river_pool=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS local expected_worker_river_pool=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS
@@ -2610,8 +2649,11 @@ verify_runtime_gates() {
esac esac
expected_worker_execution_pool=$((expected_pool - expected_critical_pool - expected_worker_river_pool)) expected_worker_execution_pool=$((expected_pool - expected_critical_pool - expected_worker_river_pool))
expected_api_execution_pool=$((expected_api_pool - expected_critical_pool - expected_api_river_pool)) expected_api_execution_pool=$((expected_api_pool - expected_critical_pool - expected_api_river_pool))
if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
expected_ready_nodes=4
fi
[[ $(remote_kubectl get nodes -o json | [[ $(remote_kubectl get nodes -o json |
jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == 3 ]] jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == "$expected_ready_nodes" ]]
[[ $(remote_kubectl get nodes -o json | [[ $(remote_kubectl get nodes -o json |
jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]]
[[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.readyInstances}') == 2 ]] [[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.readyInstances}') == 2 ]]
@@ -2755,8 +2797,9 @@ WHERE status='active'
(( connections * 4 < max_connections * 3 )) (( connections * 4 < max_connections * 3 ))
local pod metrics desired current global allocated active_instances pool_max acquired idle local pod metrics desired current global allocated active_instances pool_max acquired idle
local critical_pool_max critical_acquired critical_idle local desired_max current_max global_max pool_max_min metrics_status
local river_pool_max river_acquired river_idle local critical_pool_max critical_pool_min critical_acquired critical_idle
local river_pool_max river_pool_min river_acquired river_idle
local lease_failures lease_lost local lease_failures lease_lost
for site in ningbo hongkong; do for site in ningbo hongkong; do
metrics=$(workload_metrics_for_site api "$site") metrics=$(workload_metrics_for_site api "$site")
@@ -2781,29 +2824,45 @@ WHERE status='active'
done done
for _ in 1 2 3 4 5 6; do for _ in 1 2 3 4 5 6; do
for site in ningbo hongkong; do for site in ningbo hongkong; do
metrics=$(workload_metrics_for_site worker "$site") metrics_status=0
desired=$(awk '$1=="easyai_gateway_async_worker_desired_capacity" {print $2}' <<<"$metrics") metrics=$(workload_metrics_for_site worker "$site") || metrics_status=$?
current=$(awk '$1=="easyai_gateway_async_worker_capacity" {print $2}' <<<"$metrics") if (( metrics_status == 2 )); then
global=$(awk '$1=="easyai_gateway_worker_global_capacity" {print $2}' <<<"$metrics") continue
allocated=$(awk '$1=="easyai_gateway_worker_allocated_capacity" {print $2}' <<<"$metrics") fi
active_instances=$(awk '$1=="easyai_gateway_worker_active_instances" {print $2}' <<<"$metrics") (( metrics_status == 0 )) || return 1
pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics") desired=$(metric_min easyai_gateway_async_worker_desired_capacity <<<"$metrics")
acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics") desired_max=$(metric_max easyai_gateway_async_worker_desired_capacity <<<"$metrics")
idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics") current=$(metric_min easyai_gateway_async_worker_capacity <<<"$metrics")
lease_failures=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {print $2}' <<<"$metrics") current_max=$(metric_max easyai_gateway_async_worker_capacity <<<"$metrics")
lease_lost=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {print $2}' <<<"$metrics") global=$(metric_min easyai_gateway_worker_global_capacity <<<"$metrics")
[[ ${desired%.*} == "$expected_global" && ${current%.*} == "$expected_slots" ]] global_max=$(metric_max easyai_gateway_worker_global_capacity <<<"$metrics")
[[ ${global%.*} == "$expected_global" && ${allocated%.*} -le "$expected_slots" ]] allocated=$(metric_max easyai_gateway_worker_allocated_capacity <<<"$metrics")
active_instances=$(metric_min easyai_gateway_worker_active_instances <<<"$metrics")
pool_max=$(metric_max easyai_gateway_postgres_pool_max_connections <<<"$metrics")
pool_max_min=$(metric_min easyai_gateway_postgres_pool_max_connections <<<"$metrics")
acquired=$(metric_max easyai_gateway_postgres_pool_acquired_connections <<<"$metrics")
idle=$(metric_min easyai_gateway_postgres_pool_idle_connections <<<"$metrics")
lease_failures=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="failure"}' <<<"$metrics")
lease_lost=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="lost"}' <<<"$metrics")
[[ ${desired%.*} == "$expected_global" && ${desired_max%.*} == "$expected_global" ]]
[[ ${current%.*} == "$expected_slots" && ${current_max%.*} == "$expected_slots" ]]
[[ ${global%.*} == "$expected_global" && ${global_max%.*} == "$expected_global" ]]
[[ ${allocated%.*} -le "$expected_slots" ]]
[[ ${active_instances%.*} == 2 ]] [[ ${active_instances%.*} == 2 ]]
[[ ${pool_max%.*} == "$expected_worker_execution_pool" ]] [[ ${pool_max%.*} == "$expected_worker_execution_pool" &&
critical_pool_max=$(awk '$1=="easyai_gateway_postgres_critical_pool_max_connections" {print $2}' <<<"$metrics") ${pool_max_min%.*} == "$expected_worker_execution_pool" ]]
critical_acquired=$(awk '$1=="easyai_gateway_postgres_critical_pool_acquired_connections" {print $2}' <<<"$metrics") critical_pool_max=$(metric_max easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics")
critical_idle=$(awk '$1=="easyai_gateway_postgres_critical_pool_idle_connections" {print $2}' <<<"$metrics") critical_pool_min=$(metric_min easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics")
river_pool_max=$(awk '$1=="easyai_gateway_postgres_river_pool_max_connections" {print $2}' <<<"$metrics") critical_acquired=$(metric_max easyai_gateway_postgres_critical_pool_acquired_connections <<<"$metrics")
river_acquired=$(awk '$1=="easyai_gateway_postgres_river_pool_acquired_connections" {print $2}' <<<"$metrics") critical_idle=$(metric_min easyai_gateway_postgres_critical_pool_idle_connections <<<"$metrics")
river_idle=$(awk '$1=="easyai_gateway_postgres_river_pool_idle_connections" {print $2}' <<<"$metrics") river_pool_max=$(metric_max easyai_gateway_postgres_river_pool_max_connections <<<"$metrics")
[[ ${critical_pool_max%.*} == "$expected_critical_pool" ]] river_pool_min=$(metric_min easyai_gateway_postgres_river_pool_max_connections <<<"$metrics")
[[ ${river_pool_max%.*} == "$expected_worker_river_pool" ]] river_acquired=$(metric_max easyai_gateway_postgres_river_pool_acquired_connections <<<"$metrics")
river_idle=$(metric_min easyai_gateway_postgres_river_pool_idle_connections <<<"$metrics")
[[ ${critical_pool_max%.*} == "$expected_critical_pool" &&
${critical_pool_min%.*} == "$expected_critical_pool" ]]
[[ ${river_pool_max%.*} == "$expected_worker_river_pool" &&
${river_pool_min%.*} == "$expected_worker_river_pool" ]]
[[ ${lease_failures%.*} == 0 && ${lease_lost%.*} == 0 ]] [[ ${lease_failures%.*} == 0 && ${lease_lost%.*} == 0 ]]
if [[ ${acquired%.*} == "$expected_worker_execution_pool" && ${idle%.*} == 0 ]]; then if [[ ${acquired%.*} == "$expected_worker_execution_pool" && ${idle%.*} == 0 ]]; then
echo "Worker connection pool saturated for site $site" >&2 echo "Worker connection pool saturated for site $site" >&2
@@ -2863,25 +2922,26 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
fi fi
(( metrics_status == 0 )) || exit 1 (( metrics_status == 0 )) || exit 1
awk -v workload="$workload" ' awk -v workload="$workload" '
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2} BEGIN {idle=999999; critical_idle=999999; river_idle=999999; active=999999}
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2} $1=="easyai_gateway_postgres_pool_acquired_connections" && $2>acquired {acquired=$2}
$1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2} $1=="easyai_gateway_postgres_pool_max_connections" && $2>pool_max {pool_max=$2}
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2} $1=="easyai_gateway_postgres_pool_idle_connections" && $2<idle {idle=$2}
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2} $1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty+=$2}
$1=="easyai_gateway_postgres_critical_pool_acquired_connections" {critical_acquired=$2} $1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled+=$2}
$1=="easyai_gateway_postgres_critical_pool_max_connections" {critical_max=$2} $1=="easyai_gateway_postgres_critical_pool_acquired_connections" && $2>critical_acquired {critical_acquired=$2}
$1=="easyai_gateway_postgres_critical_pool_idle_connections" {critical_idle=$2} $1=="easyai_gateway_postgres_critical_pool_max_connections" && $2>critical_max {critical_max=$2}
$1=="easyai_gateway_postgres_critical_pool_empty_acquire_total" {critical_empty=$2} $1=="easyai_gateway_postgres_critical_pool_idle_connections" && $2<critical_idle {critical_idle=$2}
$1=="easyai_gateway_postgres_critical_pool_canceled_acquire_total" {critical_canceled=$2} $1=="easyai_gateway_postgres_critical_pool_empty_acquire_total" {critical_empty+=$2}
$1=="easyai_gateway_postgres_river_pool_acquired_connections" {river_acquired=$2} $1=="easyai_gateway_postgres_critical_pool_canceled_acquire_total" {critical_canceled+=$2}
$1=="easyai_gateway_postgres_river_pool_max_connections" {river_max=$2} $1=="easyai_gateway_postgres_river_pool_acquired_connections" && $2>river_acquired {river_acquired=$2}
$1=="easyai_gateway_postgres_river_pool_idle_connections" {river_idle=$2} $1=="easyai_gateway_postgres_river_pool_max_connections" && $2>river_max {river_max=$2}
$1=="easyai_gateway_postgres_river_pool_empty_acquire_total" {river_empty=$2} $1=="easyai_gateway_postgres_river_pool_idle_connections" && $2<river_idle {river_idle=$2}
$1=="easyai_gateway_postgres_river_pool_canceled_acquire_total" {river_canceled=$2} $1=="easyai_gateway_postgres_river_pool_empty_acquire_total" {river_empty+=$2}
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2} $1=="easyai_gateway_postgres_river_pool_canceled_acquire_total" {river_canceled+=$2}
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2} index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure+=$2}
$1=="easyai_gateway_worker_active_instances" {active=$2} index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost+=$2}
$1=="easyai_gateway_worker_allocated_capacity" {allocated=$2} $1=="easyai_gateway_worker_active_instances" && $2<active {active=$2}
$1=="easyai_gateway_worker_allocated_capacity" && $2>allocated {allocated=$2}
END { END {
if (workload=="api") { if (workload=="api") {
failure=0 failure=0
@@ -2889,6 +2949,9 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
active=999999 active=999999
allocated=0 allocated=0
} }
if (idle==999999) idle=0
if (critical_idle==999999) critical_idle=0
if (river_idle==999999) river_idle=0
print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated "," critical_acquired "," critical_max "," critical_idle "," critical_empty "," critical_canceled "," river_acquired "," river_max "," river_idle "," river_empty "," river_canceled print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated "," critical_acquired "," critical_max "," critical_idle "," critical_empty "," critical_canceled "," river_acquired "," river_max "," river_idle "," river_empty "," river_canceled
} }
' <<<"$metrics" ' <<<"$metrics"
tests/release/production-acceptance-script-test.sh
+31 -1
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# shellcheck disable=SC2034 # Variables are consumed by dynamically sourced functions. # shellcheck disable=SC2034,SC2329 # Symbols are consumed by dynamically sourced functions.
set -euo pipefail set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
@@ -46,7 +46,11 @@ if is_release_ancestor invalid "$third"; then
fi fi
remote_kubectl() { remote_kubectl() {
if [[ $* == *'get pods'* ]]; then
printf '{"items":[]}'
else
printf 0 printf 0
fi
} }
cluster_ssh() { cluster_ssh() {
@@ -63,6 +67,32 @@ else
fi fi
[[ $metrics_status == 2 ]] [[ $metrics_status == 2 ]]
remote_kubectl() {
printf '%s\n' '{"items":[
{"metadata":{},"spec":{"nodeName":"easyai-hongkong"},"status":{"podIP":"10.42.3.10","conditions":[{"type":"Ready","status":"True"}]}},
{"metadata":{},"spec":{"nodeName":"easyai-hongkong-worker-2"},"status":{"podIP":"10.42.5.10","conditions":[{"type":"Ready","status":"True"}]}}
]}'
}
cluster_ssh() {
while IFS= read -r _; do :; done
printf '# target_host %s\n' "$1"
printf 'test_metric 1\n'
}
CLUSTER_NINGBO_HOST=root@ningbo.invalid
CLUSTER_HONGKONG_HOST=root@hongkong.invalid
CLUSTER_LOS_ANGELES_HOST=root@losangeles.invalid
CLUSTER_WORKER_NODE_4_NAME=easyai-hongkong-worker-2
CLUSTER_WORKER_NODE_4_HOST=root@hongkong-worker-2.invalid
metrics=$(workload_metrics_for_site worker hongkong)
grep -Fq '# target_host root@hongkong.invalid' <<<"$metrics"
grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics"
[[ $(grep -c '^test_metric 1$' <<<"$metrics") == 2 ]]
[[ $(metric_min test_metric <<<"$metrics") == 1 ]]
[[ $(metric_max test_metric <<<"$metrics") == 1 ]]
[[ $(metric_sum test_metric <<<"$metrics") == 2 ]]
long_running_load() { long_running_load() {
while :; do while :; do
sleep 1 sleep 1