BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(acceptance): 按实际节点聚合多 Worker 指标

Browse Source 
同站点存在多个物理节点时，按每个 Ready Pod 的 nodeName 映射真实 SSH 主机采集 metrics，并隔离 ssh 标准输入，避免吞掉后续 Pod 目标。所有 Worker Pod 指标按容量最小/最大、连接池最小/最大和计数器总和聚合，不再随机漏采一个实例。\n\n运行时门禁根据已启用的第四节点计算 Ready 节点数，并跳过零副本站点，同时继续逐 Pod 校验 P24/P28/P32 容量、连接池和租约。\n\n验证：bash -n、ShellCheck、双节点 metrics 映射测试、生产只读双 Worker metrics 探针、manual-release-test。
This commit is contained in:
wangbo
2026-08-01 15:14:39 +08:00
parent 53fc79691a
commit 3514c13b0d
2 changed files with 154 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cluster/run-production-acceptance.sh
+122 -59
View File
@@ -2506,28 +2506,66 @@ verify_control_plane_and_recent_logs() {
workload_metrics_for_site() {
local workload=$1
local site=$2
local host pod_ip replicas
local host pod_ip node_name replicas targets
[[ $workload == api || $workload == worker ]]
[[ $site == ningbo || $site == hongkong ]]
if [[ $workload == worker ]]; then
targets=$(remote_kubectl get pods -n "$namespace" \
-l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json |
jq -r '.items[]
| select(.metadata.deletionTimestamp == null)
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
| [.status.podIP,.spec.nodeName]
| @tsv') || return 1
if [[ -z $targets && $workload == worker ]]; then
replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \
-o 'jsonpath={.spec.replicas}') || return 1
[[ $replicas =~ ^[0-9]+$ ]] || return 1
(( replicas > 0 )) || return 2
fi
if [[ $site == ningbo ]]; then
host=$CLUSTER_NINGBO_HOST
else
host=$CLUSTER_HONGKONG_HOST
fi
pod_ip=$(remote_kubectl get pods -n "$namespace" \
-l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json |
jq -r '[.items[]
| select(.metadata.deletionTimestamp == null)
| select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))
| .status.podIP][0] // empty')
[[ $pod_ip =~ ^[0-9a-fA-F:.]+$ ]] || return 1
cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics"
[[ -n $targets ]] || return 1
while IFS=$'\t' read -r pod_ip node_name; do
[[ $pod_ip =~ ^[0-9a-fA-F:.]+$ && $node_name =~ ^[a-z0-9-]+$ ]] || return 1
case $node_name in
easyai-ningbo) host=$CLUSTER_NINGBO_HOST ;;
easyai-hongkong) host=$CLUSTER_HONGKONG_HOST ;;
easyai-los-angeles) host=$CLUSTER_LOS_ANGELES_HOST ;;
*)
if [[ -n $CLUSTER_WORKER_NODE_4_NAME &&
$node_name == "$CLUSTER_WORKER_NODE_4_NAME" ]]; then
host=$CLUSTER_WORKER_NODE_4_HOST
else
echo "metrics target node has no SSH mapping: node=$node_name" >&2
return 1
fi
;;
esac
cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics" \
</dev/null || return 1
done <<<"$targets"
}
metric_min() {
local metric=$1
awk -v metric="$metric" '
$1==metric && (!seen || $2<value) {value=$2; seen=1}
END {if (seen) print value}
'
}
metric_max() {
local metric=$1
awk -v metric="$metric" '
$1==metric && (!seen || $2>value) {value=$2; seen=1}
END {if (seen) print value}
'
}
metric_sum() {
local metric=$1
awk -v metric="$metric" '
$1==metric {value+=$2; seen=1}
END {if (seen) print value}
'
}
run_with_pressure_monitor() {
@@ -2599,6 +2637,7 @@ wait_for_memory_recovery() {
verify_runtime_gates() {
local expected_slots expected_global expected_pool expected_worker_execution_pool expected_api_execution_pool
local expected_ready_nodes=3
local expected_critical_pool=4
local expected_api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
local expected_worker_river_pool=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS
@@ -2610,8 +2649,11 @@ verify_runtime_gates() {
esac
expected_worker_execution_pool=$((expected_pool - expected_critical_pool - expected_worker_river_pool))
expected_api_execution_pool=$((expected_api_pool - expected_critical_pool - expected_api_river_pool))
if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
expected_ready_nodes=4
fi
[[ $(remote_kubectl get nodes -o json |
jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == 3 ]]
jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == "$expected_ready_nodes" ]]
[[ $(remote_kubectl get nodes -o json |
jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]]
[[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.readyInstances}') == 2 ]]
@@ -2755,8 +2797,9 @@ WHERE status='active'
(( connections * 4 < max_connections * 3 ))
local pod metrics desired current global allocated active_instances pool_max acquired idle
local critical_pool_max critical_acquired critical_idle
local river_pool_max river_acquired river_idle
local desired_max current_max global_max pool_max_min metrics_status
local critical_pool_max critical_pool_min critical_acquired critical_idle
local river_pool_max river_pool_min river_acquired river_idle
local lease_failures lease_lost
for site in ningbo hongkong; do
metrics=$(workload_metrics_for_site api "$site")
@@ -2781,29 +2824,45 @@ WHERE status='active'
done
for _ in 1 2 3 4 5 6; do
for site in ningbo hongkong; do
metrics=$(workload_metrics_for_site worker "$site")
desired=$(awk '$1=="easyai_gateway_async_worker_desired_capacity" {print $2}' <<<"$metrics")
current=$(awk '$1=="easyai_gateway_async_worker_capacity" {print $2}' <<<"$metrics")
global=$(awk '$1=="easyai_gateway_worker_global_capacity" {print $2}' <<<"$metrics")
allocated=$(awk '$1=="easyai_gateway_worker_allocated_capacity" {print $2}' <<<"$metrics")
active_instances=$(awk '$1=="easyai_gateway_worker_active_instances" {print $2}' <<<"$metrics")
pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics")
acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics")
idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics")
lease_failures=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {print $2}' <<<"$metrics")
lease_lost=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {print $2}' <<<"$metrics")
[[ ${desired%.*} == "$expected_global" && ${current%.*} == "$expected_slots" ]]
[[ ${global%.*} == "$expected_global" && ${allocated%.*} -le "$expected_slots" ]]
metrics_status=0
metrics=$(workload_metrics_for_site worker "$site") || metrics_status=$?
if (( metrics_status == 2 )); then
continue
fi
(( metrics_status == 0 )) || return 1
desired=$(metric_min easyai_gateway_async_worker_desired_capacity <<<"$metrics")
desired_max=$(metric_max easyai_gateway_async_worker_desired_capacity <<<"$metrics")
current=$(metric_min easyai_gateway_async_worker_capacity <<<"$metrics")
current_max=$(metric_max easyai_gateway_async_worker_capacity <<<"$metrics")
global=$(metric_min easyai_gateway_worker_global_capacity <<<"$metrics")
global_max=$(metric_max easyai_gateway_worker_global_capacity <<<"$metrics")
allocated=$(metric_max easyai_gateway_worker_allocated_capacity <<<"$metrics")
active_instances=$(metric_min easyai_gateway_worker_active_instances <<<"$metrics")
pool_max=$(metric_max easyai_gateway_postgres_pool_max_connections <<<"$metrics")
pool_max_min=$(metric_min easyai_gateway_postgres_pool_max_connections <<<"$metrics")
acquired=$(metric_max easyai_gateway_postgres_pool_acquired_connections <<<"$metrics")
idle=$(metric_min easyai_gateway_postgres_pool_idle_connections <<<"$metrics")
lease_failures=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="failure"}' <<<"$metrics")
lease_lost=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="lost"}' <<<"$metrics")
[[ ${desired%.*} == "$expected_global" && ${desired_max%.*} == "$expected_global" ]]
[[ ${current%.*} == "$expected_slots" && ${current_max%.*} == "$expected_slots" ]]
[[ ${global%.*} == "$expected_global" && ${global_max%.*} == "$expected_global" ]]
[[ ${allocated%.*} -le "$expected_slots" ]]
[[ ${active_instances%.*} == 2 ]]
[[ ${pool_max%.*} == "$expected_worker_execution_pool" ]]
critical_pool_max=$(awk '$1=="easyai_gateway_postgres_critical_pool_max_connections" {print $2}' <<<"$metrics")
critical_acquired=$(awk '$1=="easyai_gateway_postgres_critical_pool_acquired_connections" {print $2}' <<<"$metrics")
critical_idle=$(awk '$1=="easyai_gateway_postgres_critical_pool_idle_connections" {print $2}' <<<"$metrics")
river_pool_max=$(awk '$1=="easyai_gateway_postgres_river_pool_max_connections" {print $2}' <<<"$metrics")
river_acquired=$(awk '$1=="easyai_gateway_postgres_river_pool_acquired_connections" {print $2}' <<<"$metrics")
river_idle=$(awk '$1=="easyai_gateway_postgres_river_pool_idle_connections" {print $2}' <<<"$metrics")
[[ ${critical_pool_max%.*} == "$expected_critical_pool" ]]
[[ ${river_pool_max%.*} == "$expected_worker_river_pool" ]]
[[ ${pool_max%.*} == "$expected_worker_execution_pool" &&
${pool_max_min%.*} == "$expected_worker_execution_pool" ]]
critical_pool_max=$(metric_max easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics")
critical_pool_min=$(metric_min easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics")
critical_acquired=$(metric_max easyai_gateway_postgres_critical_pool_acquired_connections <<<"$metrics")
critical_idle=$(metric_min easyai_gateway_postgres_critical_pool_idle_connections <<<"$metrics")
river_pool_max=$(metric_max easyai_gateway_postgres_river_pool_max_connections <<<"$metrics")
river_pool_min=$(metric_min easyai_gateway_postgres_river_pool_max_connections <<<"$metrics")
river_acquired=$(metric_max easyai_gateway_postgres_river_pool_acquired_connections <<<"$metrics")
river_idle=$(metric_min easyai_gateway_postgres_river_pool_idle_connections <<<"$metrics")
[[ ${critical_pool_max%.*} == "$expected_critical_pool" &&
${critical_pool_min%.*} == "$expected_critical_pool" ]]
[[ ${river_pool_max%.*} == "$expected_worker_river_pool" &&
${river_pool_min%.*} == "$expected_worker_river_pool" ]]
[[ ${lease_failures%.*} == 0 && ${lease_lost%.*} == 0 ]]
if [[ ${acquired%.*} == "$expected_worker_execution_pool" && ${idle%.*} == 0 ]]; then
echo "Worker connection pool saturated for site $site" >&2
@@ -2863,25 +2922,26 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
fi
(( metrics_status == 0 )) || exit 1
awk -v workload="$workload" '
$1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2}
$1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2}
$1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2}
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2}
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2}
$1=="easyai_gateway_postgres_critical_pool_acquired_connections" {critical_acquired=$2}
$1=="easyai_gateway_postgres_critical_pool_max_connections" {critical_max=$2}
$1=="easyai_gateway_postgres_critical_pool_idle_connections" {critical_idle=$2}
$1=="easyai_gateway_postgres_critical_pool_empty_acquire_total" {critical_empty=$2}
$1=="easyai_gateway_postgres_critical_pool_canceled_acquire_total" {critical_canceled=$2}
$1=="easyai_gateway_postgres_river_pool_acquired_connections" {river_acquired=$2}
$1=="easyai_gateway_postgres_river_pool_max_connections" {river_max=$2}
$1=="easyai_gateway_postgres_river_pool_idle_connections" {river_idle=$2}
$1=="easyai_gateway_postgres_river_pool_empty_acquire_total" {river_empty=$2}
$1=="easyai_gateway_postgres_river_pool_canceled_acquire_total" {river_canceled=$2}
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2}
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2}
$1=="easyai_gateway_worker_active_instances" {active=$2}
$1=="easyai_gateway_worker_allocated_capacity" {allocated=$2}
BEGIN {idle=999999; critical_idle=999999; river_idle=999999; active=999999}
$1=="easyai_gateway_postgres_pool_acquired_connections" && $2>acquired {acquired=$2}
$1=="easyai_gateway_postgres_pool_max_connections" && $2>pool_max {pool_max=$2}
$1=="easyai_gateway_postgres_pool_idle_connections" && $2<idle {idle=$2}
$1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty+=$2}
$1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled+=$2}
$1=="easyai_gateway_postgres_critical_pool_acquired_connections" && $2>critical_acquired {critical_acquired=$2}
$1=="easyai_gateway_postgres_critical_pool_max_connections" && $2>critical_max {critical_max=$2}
$1=="easyai_gateway_postgres_critical_pool_idle_connections" && $2<critical_idle {critical_idle=$2}
$1=="easyai_gateway_postgres_critical_pool_empty_acquire_total" {critical_empty+=$2}
$1=="easyai_gateway_postgres_critical_pool_canceled_acquire_total" {critical_canceled+=$2}
$1=="easyai_gateway_postgres_river_pool_acquired_connections" && $2>river_acquired {river_acquired=$2}
$1=="easyai_gateway_postgres_river_pool_max_connections" && $2>river_max {river_max=$2}
$1=="easyai_gateway_postgres_river_pool_idle_connections" && $2<river_idle {river_idle=$2}
$1=="easyai_gateway_postgres_river_pool_empty_acquire_total" {river_empty+=$2}
$1=="easyai_gateway_postgres_river_pool_canceled_acquire_total" {river_canceled+=$2}
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure+=$2}
index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost+=$2}
$1=="easyai_gateway_worker_active_instances" && $2<active {active=$2}
$1=="easyai_gateway_worker_allocated_capacity" && $2>allocated {allocated=$2}
END {
if (workload=="api") {
failure=0
@@ -2889,6 +2949,9 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
active=999999
allocated=0
}
if (idle==999999) idle=0
if (critical_idle==999999) critical_idle=0
if (river_idle==999999) river_idle=0
print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated "," critical_acquired "," critical_max "," critical_idle "," critical_empty "," critical_canceled "," river_acquired "," river_max "," river_idle "," river_empty "," river_canceled
}
' <<<"$metrics"