diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 98fde55..1f12bec 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -2506,28 +2506,66 @@ verify_control_plane_and_recent_logs() { workload_metrics_for_site() { local workload=$1 local site=$2 - local host pod_ip replicas + local host pod_ip node_name replicas targets [[ $workload == api || $workload == worker ]] [[ $site == ningbo || $site == hongkong ]] - if [[ $workload == worker ]]; then + targets=$(remote_kubectl get pods -n "$namespace" \ + -l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json | + jq -r '.items[] + | select(.metadata.deletionTimestamp == null) + | select(any(.status.conditions[]?; .type=="Ready" and .status=="True")) + | [.status.podIP,.spec.nodeName] + | @tsv') || return 1 + if [[ -z $targets && $workload == worker ]]; then replicas=$(remote_kubectl get deployment "easyai-worker-$site" -n "$namespace" \ -o 'jsonpath={.spec.replicas}') || return 1 [[ $replicas =~ ^[0-9]+$ ]] || return 1 (( replicas > 0 )) || return 2 fi - if [[ $site == ningbo ]]; then - host=$CLUSTER_NINGBO_HOST - else - host=$CLUSTER_HONGKONG_HOST - fi - pod_ip=$(remote_kubectl get pods -n "$namespace" \ - -l "app.kubernetes.io/name=easyai-$workload,easyai.io/site=$site" -o json | - jq -r '[.items[] - | select(.metadata.deletionTimestamp == null) - | select(any(.status.conditions[]?; .type=="Ready" and .status=="True")) - | .status.podIP][0] // empty') - [[ $pod_ip =~ ^[0-9a-fA-F:.]+$ ]] || return 1 - cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics" + [[ -n $targets ]] || return 1 + while IFS=$'\t' read -r pod_ip node_name; do + [[ $pod_ip =~ ^[0-9a-fA-F:.]+$ && $node_name =~ ^[a-z0-9-]+$ ]] || return 1 + case $node_name in + easyai-ningbo) host=$CLUSTER_NINGBO_HOST ;; + easyai-hongkong) host=$CLUSTER_HONGKONG_HOST ;; + easyai-los-angeles) host=$CLUSTER_LOS_ANGELES_HOST ;; + *) + if [[ -n $CLUSTER_WORKER_NODE_4_NAME && + $node_name == "$CLUSTER_WORKER_NODE_4_NAME" ]]; then + host=$CLUSTER_WORKER_NODE_4_HOST + else + echo "metrics target node has no SSH mapping: node=$node_name" >&2 + return 1 + fi + ;; + esac + cluster_ssh "$host" "curl -fsS --max-time 5 http://$pod_ip:8088/metrics" \ + value) {value=$2; seen=1} + END {if (seen) print value} + ' +} + +metric_sum() { + local metric=$1 + awk -v metric="$metric" ' + $1==metric {value+=$2; seen=1} + END {if (seen) print value} + ' } run_with_pressure_monitor() { @@ -2599,6 +2637,7 @@ wait_for_memory_recovery() { verify_runtime_gates() { local expected_slots expected_global expected_pool expected_worker_execution_pool expected_api_execution_pool + local expected_ready_nodes=3 local expected_critical_pool=4 local expected_api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS local expected_worker_river_pool=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS @@ -2610,8 +2649,11 @@ verify_runtime_gates() { esac expected_worker_execution_pool=$((expected_pool - expected_critical_pool - expected_worker_river_pool)) expected_api_execution_pool=$((expected_api_pool - expected_critical_pool - expected_api_river_pool)) + if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then + expected_ready_nodes=4 + fi [[ $(remote_kubectl get nodes -o json | - jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == 3 ]] + jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') == "$expected_ready_nodes" ]] [[ $(remote_kubectl get nodes -o json | jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] [[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" -o 'jsonpath={.status.readyInstances}') == 2 ]] @@ -2755,8 +2797,9 @@ WHERE status='active' (( connections * 4 < max_connections * 3 )) local pod metrics desired current global allocated active_instances pool_max acquired idle - local critical_pool_max critical_acquired critical_idle - local river_pool_max river_acquired river_idle + local desired_max current_max global_max pool_max_min metrics_status + local critical_pool_max critical_pool_min critical_acquired critical_idle + local river_pool_max river_pool_min river_acquired river_idle local lease_failures lease_lost for site in ningbo hongkong; do metrics=$(workload_metrics_for_site api "$site") @@ -2781,29 +2824,45 @@ WHERE status='active' done for _ in 1 2 3 4 5 6; do for site in ningbo hongkong; do - metrics=$(workload_metrics_for_site worker "$site") - desired=$(awk '$1=="easyai_gateway_async_worker_desired_capacity" {print $2}' <<<"$metrics") - current=$(awk '$1=="easyai_gateway_async_worker_capacity" {print $2}' <<<"$metrics") - global=$(awk '$1=="easyai_gateway_worker_global_capacity" {print $2}' <<<"$metrics") - allocated=$(awk '$1=="easyai_gateway_worker_allocated_capacity" {print $2}' <<<"$metrics") - active_instances=$(awk '$1=="easyai_gateway_worker_active_instances" {print $2}' <<<"$metrics") - pool_max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print $2}' <<<"$metrics") - acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print $2}' <<<"$metrics") - idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print $2}' <<<"$metrics") - lease_failures=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {print $2}' <<<"$metrics") - lease_lost=$(awk 'index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {print $2}' <<<"$metrics") - [[ ${desired%.*} == "$expected_global" && ${current%.*} == "$expected_slots" ]] - [[ ${global%.*} == "$expected_global" && ${allocated%.*} -le "$expected_slots" ]] + metrics_status=0 + metrics=$(workload_metrics_for_site worker "$site") || metrics_status=$? + if (( metrics_status == 2 )); then + continue + fi + (( metrics_status == 0 )) || return 1 + desired=$(metric_min easyai_gateway_async_worker_desired_capacity <<<"$metrics") + desired_max=$(metric_max easyai_gateway_async_worker_desired_capacity <<<"$metrics") + current=$(metric_min easyai_gateway_async_worker_capacity <<<"$metrics") + current_max=$(metric_max easyai_gateway_async_worker_capacity <<<"$metrics") + global=$(metric_min easyai_gateway_worker_global_capacity <<<"$metrics") + global_max=$(metric_max easyai_gateway_worker_global_capacity <<<"$metrics") + allocated=$(metric_max easyai_gateway_worker_allocated_capacity <<<"$metrics") + active_instances=$(metric_min easyai_gateway_worker_active_instances <<<"$metrics") + pool_max=$(metric_max easyai_gateway_postgres_pool_max_connections <<<"$metrics") + pool_max_min=$(metric_min easyai_gateway_postgres_pool_max_connections <<<"$metrics") + acquired=$(metric_max easyai_gateway_postgres_pool_acquired_connections <<<"$metrics") + idle=$(metric_min easyai_gateway_postgres_pool_idle_connections <<<"$metrics") + lease_failures=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="failure"}' <<<"$metrics") + lease_lost=$(metric_sum 'easyai_gateway_concurrency_lease_renewals_total{outcome="lost"}' <<<"$metrics") + [[ ${desired%.*} == "$expected_global" && ${desired_max%.*} == "$expected_global" ]] + [[ ${current%.*} == "$expected_slots" && ${current_max%.*} == "$expected_slots" ]] + [[ ${global%.*} == "$expected_global" && ${global_max%.*} == "$expected_global" ]] + [[ ${allocated%.*} -le "$expected_slots" ]] [[ ${active_instances%.*} == 2 ]] - [[ ${pool_max%.*} == "$expected_worker_execution_pool" ]] - critical_pool_max=$(awk '$1=="easyai_gateway_postgres_critical_pool_max_connections" {print $2}' <<<"$metrics") - critical_acquired=$(awk '$1=="easyai_gateway_postgres_critical_pool_acquired_connections" {print $2}' <<<"$metrics") - critical_idle=$(awk '$1=="easyai_gateway_postgres_critical_pool_idle_connections" {print $2}' <<<"$metrics") - river_pool_max=$(awk '$1=="easyai_gateway_postgres_river_pool_max_connections" {print $2}' <<<"$metrics") - river_acquired=$(awk '$1=="easyai_gateway_postgres_river_pool_acquired_connections" {print $2}' <<<"$metrics") - river_idle=$(awk '$1=="easyai_gateway_postgres_river_pool_idle_connections" {print $2}' <<<"$metrics") - [[ ${critical_pool_max%.*} == "$expected_critical_pool" ]] - [[ ${river_pool_max%.*} == "$expected_worker_river_pool" ]] + [[ ${pool_max%.*} == "$expected_worker_execution_pool" && + ${pool_max_min%.*} == "$expected_worker_execution_pool" ]] + critical_pool_max=$(metric_max easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics") + critical_pool_min=$(metric_min easyai_gateway_postgres_critical_pool_max_connections <<<"$metrics") + critical_acquired=$(metric_max easyai_gateway_postgres_critical_pool_acquired_connections <<<"$metrics") + critical_idle=$(metric_min easyai_gateway_postgres_critical_pool_idle_connections <<<"$metrics") + river_pool_max=$(metric_max easyai_gateway_postgres_river_pool_max_connections <<<"$metrics") + river_pool_min=$(metric_min easyai_gateway_postgres_river_pool_max_connections <<<"$metrics") + river_acquired=$(metric_max easyai_gateway_postgres_river_pool_acquired_connections <<<"$metrics") + river_idle=$(metric_min easyai_gateway_postgres_river_pool_idle_connections <<<"$metrics") + [[ ${critical_pool_max%.*} == "$expected_critical_pool" && + ${critical_pool_min%.*} == "$expected_critical_pool" ]] + [[ ${river_pool_max%.*} == "$expected_worker_river_pool" && + ${river_pool_min%.*} == "$expected_worker_river_pool" ]] [[ ${lease_failures%.*} == 0 && ${lease_lost%.*} == 0 ]] if [[ ${acquired%.*} == "$expected_worker_execution_pool" && ${idle%.*} == 0 ]]; then echo "Worker connection pool saturated for site $site" >&2 @@ -2863,25 +2922,26 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then fi (( metrics_status == 0 )) || exit 1 awk -v workload="$workload" ' - $1=="easyai_gateway_postgres_pool_acquired_connections" {acquired=$2} - $1=="easyai_gateway_postgres_pool_max_connections" {pool_max=$2} - $1=="easyai_gateway_postgres_pool_idle_connections" {idle=$2} - $1=="easyai_gateway_postgres_pool_empty_acquire_total" {empty=$2} - $1=="easyai_gateway_postgres_pool_canceled_acquire_total" {canceled=$2} - $1=="easyai_gateway_postgres_critical_pool_acquired_connections" {critical_acquired=$2} - $1=="easyai_gateway_postgres_critical_pool_max_connections" {critical_max=$2} - $1=="easyai_gateway_postgres_critical_pool_idle_connections" {critical_idle=$2} - $1=="easyai_gateway_postgres_critical_pool_empty_acquire_total" {critical_empty=$2} - $1=="easyai_gateway_postgres_critical_pool_canceled_acquire_total" {critical_canceled=$2} - $1=="easyai_gateway_postgres_river_pool_acquired_connections" {river_acquired=$2} - $1=="easyai_gateway_postgres_river_pool_max_connections" {river_max=$2} - $1=="easyai_gateway_postgres_river_pool_idle_connections" {river_idle=$2} - $1=="easyai_gateway_postgres_river_pool_empty_acquire_total" {river_empty=$2} - $1=="easyai_gateway_postgres_river_pool_canceled_acquire_total" {river_canceled=$2} - index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"failure\"}")==1 {failure=$2} - index($1,"easyai_gateway_concurrency_lease_renewals_total{outcome=\"lost\"}")==1 {lost=$2} - $1=="easyai_gateway_worker_active_instances" {active=$2} - $1=="easyai_gateway_worker_allocated_capacity" {allocated=$2} + BEGIN {idle=999999; critical_idle=999999; river_idle=999999; active=999999} + $1=="easyai_gateway_postgres_pool_acquired_connections" && $2>acquired {acquired=$2} + $1=="easyai_gateway_postgres_pool_max_connections" && $2>pool_max {pool_max=$2} + $1=="easyai_gateway_postgres_pool_idle_connections" && $2critical_acquired {critical_acquired=$2} + $1=="easyai_gateway_postgres_critical_pool_max_connections" && $2>critical_max {critical_max=$2} + $1=="easyai_gateway_postgres_critical_pool_idle_connections" && $2river_acquired {river_acquired=$2} + $1=="easyai_gateway_postgres_river_pool_max_connections" && $2>river_max {river_max=$2} + $1=="easyai_gateway_postgres_river_pool_idle_connections" && $2allocated {allocated=$2} END { if (workload=="api") { failure=0 @@ -2889,6 +2949,9 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then active=999999 allocated=0 } + if (idle==999999) idle=0 + if (critical_idle==999999) critical_idle=0 + if (river_idle==999999) river_idle=0 print acquired "," pool_max "," idle "," empty "," canceled "," failure "," lost "," active "," allocated "," critical_acquired "," critical_max "," critical_idle "," critical_empty "," critical_canceled "," river_acquired "," river_max "," river_idle "," river_empty "," river_canceled } ' <<<"$metrics" diff --git a/tests/release/production-acceptance-script-test.sh b/tests/release/production-acceptance-script-test.sh index 94ffbdd..2e9b78f 100755 --- a/tests/release/production-acceptance-script-test.sh +++ b/tests/release/production-acceptance-script-test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# shellcheck disable=SC2034 # Variables are consumed by dynamically sourced functions. +# shellcheck disable=SC2034,SC2329 # Symbols are consumed by dynamically sourced functions. set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) @@ -46,7 +46,11 @@ if is_release_ancestor invalid "$third"; then fi remote_kubectl() { - printf 0 + if [[ $* == *'get pods'* ]]; then + printf '{"items":[]}' + else + printf 0 + fi } cluster_ssh() { @@ -63,6 +67,32 @@ else fi [[ $metrics_status == 2 ]] +remote_kubectl() { + printf '%s\n' '{"items":[ + {"metadata":{},"spec":{"nodeName":"easyai-hongkong"},"status":{"podIP":"10.42.3.10","conditions":[{"type":"Ready","status":"True"}]}}, + {"metadata":{},"spec":{"nodeName":"easyai-hongkong-worker-2"},"status":{"podIP":"10.42.5.10","conditions":[{"type":"Ready","status":"True"}]}} + ]}' +} + +cluster_ssh() { + while IFS= read -r _; do :; done + printf '# target_host %s\n' "$1" + printf 'test_metric 1\n' +} + +CLUSTER_NINGBO_HOST=root@ningbo.invalid +CLUSTER_HONGKONG_HOST=root@hongkong.invalid +CLUSTER_LOS_ANGELES_HOST=root@losangeles.invalid +CLUSTER_WORKER_NODE_4_NAME=easyai-hongkong-worker-2 +CLUSTER_WORKER_NODE_4_HOST=root@hongkong-worker-2.invalid +metrics=$(workload_metrics_for_site worker hongkong) +grep -Fq '# target_host root@hongkong.invalid' <<<"$metrics" +grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics" +[[ $(grep -c '^test_metric 1$' <<<"$metrics") == 2 ]] +[[ $(metric_min test_metric <<<"$metrics") == 1 ]] +[[ $(metric_max test_metric <<<"$metrics") == 1 ]] +[[ $(metric_sum test_metric <<<"$metrics") == 2 ]] + long_running_load() { while :; do sleep 1