wangbo
3514c13b0d
同站点存在多个物理节点时，按每个 Ready Pod 的 nodeName 映射真实 SSH 主机采集 metrics，并隔离 ssh 标准输入，避免吞掉后续 Pod 目标。所有 Worker Pod 指标按容量最小/最大、连接池最小/最大和计数器总和聚合，不再随机漏采一个实例。\n\n运行时门禁根据已启用的第四节点计算 Ready 节点数，并跳过零副本站点，同时继续逐 Pod 校验 P24/P28/P32 容量、连接池和租约。\n\n验证：bash -n、ShellCheck、双节点 metrics 映射测试、生产只读双 Worker metrics 探针、manual-release-test。
114 lines
3.3 KiB
Bash
Executable File
114 lines
3.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2034,SC2329 # Symbols are consumed by dynamically sourced functions.
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
|
|
script=$root/scripts/cluster/run-production-acceptance.sh
|
|
awk '
|
|
/^is_release_ancestor\(\)/ { capture = 1 }
|
|
capture && /^create_and_activate_run\(\)/ { exit }
|
|
capture { print }
|
|
' "$script" >"$tmp/is-release-ancestor.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/is-release-ancestor.sh"
|
|
|
|
awk '
|
|
/^workload_metrics_for_site\(\)/ { capture = 1 }
|
|
capture && /^current_max_pod_memory_mib\(\)/ { exit }
|
|
capture { print }
|
|
' "$script" >"$tmp/pressure-monitor.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/pressure-monitor.sh"
|
|
|
|
cluster_root=$tmp/repository
|
|
git init -q "$cluster_root"
|
|
git -C "$cluster_root" config user.name 'Acceptance Script Test'
|
|
git -C "$cluster_root" config user.email 'acceptance-script-test@example.invalid'
|
|
git -C "$cluster_root" commit --allow-empty -qm first
|
|
first=$(git -C "$cluster_root" rev-parse HEAD)
|
|
git -C "$cluster_root" commit --allow-empty -qm second
|
|
git -C "$cluster_root" commit --allow-empty -qm third
|
|
third=$(git -C "$cluster_root" rev-parse HEAD)
|
|
|
|
is_release_ancestor "$first" "$third"
|
|
if is_release_ancestor "$third" "$first"; then
|
|
echo 'descendant release was accepted as an ancestor' >&2
|
|
exit 1
|
|
fi
|
|
if is_release_ancestor invalid "$third"; then
|
|
echo 'invalid release identifier was accepted as an ancestor' >&2
|
|
exit 1
|
|
fi
|
|
|
|
remote_kubectl() {
|
|
if [[ $* == *'get pods'* ]]; then
|
|
printf '{"items":[]}'
|
|
else
|
|
printf 0
|
|
fi
|
|
}
|
|
|
|
cluster_ssh() {
|
|
echo 'zero-replica Worker attempted to collect metrics' >&2
|
|
return 1
|
|
}
|
|
|
|
namespace=easyai
|
|
if workload_metrics_for_site worker ningbo; then
|
|
echo 'zero-replica Worker site returned metrics success' >&2
|
|
exit 1
|
|
else
|
|
metrics_status=$?
|
|
fi
|
|
[[ $metrics_status == 2 ]]
|
|
|
|
remote_kubectl() {
|
|
printf '%s\n' '{"items":[
|
|
{"metadata":{},"spec":{"nodeName":"easyai-hongkong"},"status":{"podIP":"10.42.3.10","conditions":[{"type":"Ready","status":"True"}]}},
|
|
{"metadata":{},"spec":{"nodeName":"easyai-hongkong-worker-2"},"status":{"podIP":"10.42.5.10","conditions":[{"type":"Ready","status":"True"}]}}
|
|
]}'
|
|
}
|
|
|
|
cluster_ssh() {
|
|
while IFS= read -r _; do :; done
|
|
printf '# target_host %s\n' "$1"
|
|
printf 'test_metric 1\n'
|
|
}
|
|
|
|
CLUSTER_NINGBO_HOST=root@ningbo.invalid
|
|
CLUSTER_HONGKONG_HOST=root@hongkong.invalid
|
|
CLUSTER_LOS_ANGELES_HOST=root@losangeles.invalid
|
|
CLUSTER_WORKER_NODE_4_NAME=easyai-hongkong-worker-2
|
|
CLUSTER_WORKER_NODE_4_HOST=root@hongkong-worker-2.invalid
|
|
metrics=$(workload_metrics_for_site worker hongkong)
|
|
grep -Fq '# target_host root@hongkong.invalid' <<<"$metrics"
|
|
grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics"
|
|
[[ $(grep -c '^test_metric 1$' <<<"$metrics") == 2 ]]
|
|
[[ $(metric_min test_metric <<<"$metrics") == 1 ]]
|
|
[[ $(metric_max test_metric <<<"$metrics") == 1 ]]
|
|
[[ $(metric_sum test_metric <<<"$metrics") == 2 ]]
|
|
|
|
long_running_load() {
|
|
while :; do
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
(sleep 1; exit 1) &
|
|
pressure_pid=$!
|
|
AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS=1
|
|
active_load_pid=
|
|
if run_with_pressure_monitor "$pressure_pid" long_running_load; then
|
|
echo 'load continued after the pressure sampler failed' >&2
|
|
exit 1
|
|
fi
|
|
wait "$pressure_pid" >/dev/null 2>&1 || true
|
|
[[ -z $active_load_pid ]]
|
|
|
|
echo 'production_acceptance_script_tests=PASS'
|