wangbo
c55e52af11
原因：资源前置检查中的一次瞬时 SSH 中断会直接终止 Run，失败收尾又可能因 partial 报告已存在而无法完成。\n\n影响：节点内存采集增加有界重试并区分传输故障；验收管理 GET 与 finish 支持香港本地入口安全回退，并对响应丢失后的 409 反查 Run 终态；已有 partial 报告会验证复用。\n\n验证：bash -n、ShellCheck、production-acceptance-script-test.sh、manual-release-test.sh 通过。
264 lines
8.9 KiB
Bash
Executable File
264 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2034,SC2329 # Symbols are consumed by dynamically sourced functions.
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
|
|
script=$root/scripts/cluster/run-production-acceptance.sh
|
|
|
|
# shellcheck source=scripts/cluster/common.sh
|
|
source "$root/scripts/cluster/common.sh"
|
|
ssh() {
|
|
printf '%s\n' "$@" >"$tmp/ssh-arguments"
|
|
}
|
|
cluster_ssh_key=$tmp/test-key
|
|
cluster_ssh_control_dir=$tmp/ssh-control
|
|
cluster_ssh root@cluster.invalid true
|
|
grep -Fxq 'ConnectionAttempts=3' "$tmp/ssh-arguments"
|
|
grep -Fxq 'ControlMaster=auto' "$tmp/ssh-arguments"
|
|
grep -Fxq 'ControlPersist=120' "$tmp/ssh-arguments"
|
|
grep -Fxq "ControlPath=$cluster_ssh_control_dir/%C" "$tmp/ssh-arguments"
|
|
[[ $(stat -f '%Lp' "$cluster_ssh_control_dir") == 700 ]]
|
|
|
|
awk '
|
|
/^is_release_ancestor\(\)/ { capture = 1 }
|
|
capture && /^create_and_activate_run\(\)/ { exit }
|
|
capture { print }
|
|
' "$script" >"$tmp/is-release-ancestor.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/is-release-ancestor.sh"
|
|
|
|
awk '
|
|
/^workload_metrics_for_site\(\)/ { capture = 1 }
|
|
capture && /^current_max_pod_memory_mib\(\)/ { exit }
|
|
capture { print }
|
|
' "$script" >"$tmp/pressure-monitor.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/pressure-monitor.sh"
|
|
|
|
awk '
|
|
/^read_node_memory_percent\(\)/ { capture = 1 }
|
|
capture && /^verify_resource_preconditions\(\)/ { exit }
|
|
capture { print }
|
|
' "$script" >"$tmp/resource-precondition.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/resource-precondition.sh"
|
|
|
|
awk '
|
|
/^merge_partial_acceptance_report\(\)/ { capture = 1 }
|
|
capture && /^mark_run_failed\(\)/ { exit }
|
|
capture { print }
|
|
' "$script" >"$tmp/partial-report.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/partial-report.sh"
|
|
|
|
awk '
|
|
/^admin_request\(\)/ { capture = 1 }
|
|
capture && /^verify_release_cas\(\)/ { exit }
|
|
capture { print }
|
|
' "$script" >"$tmp/admin-request.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/admin-request.sh"
|
|
|
|
cluster_root=$tmp/repository
|
|
git init -q "$cluster_root"
|
|
git -C "$cluster_root" config user.name 'Acceptance Script Test'
|
|
git -C "$cluster_root" config user.email 'acceptance-script-test@example.invalid'
|
|
git -C "$cluster_root" commit --allow-empty -qm first
|
|
first=$(git -C "$cluster_root" rev-parse HEAD)
|
|
git -C "$cluster_root" commit --allow-empty -qm second
|
|
git -C "$cluster_root" commit --allow-empty -qm third
|
|
third=$(git -C "$cluster_root" rev-parse HEAD)
|
|
|
|
is_release_ancestor "$first" "$third"
|
|
if is_release_ancestor "$third" "$first"; then
|
|
echo 'descendant release was accepted as an ancestor' >&2
|
|
exit 1
|
|
fi
|
|
if is_release_ancestor invalid "$third"; then
|
|
echo 'invalid release identifier was accepted as an ancestor' >&2
|
|
exit 1
|
|
fi
|
|
|
|
remote_kubectl() {
|
|
if [[ $* == *'get pods'* ]]; then
|
|
printf '{"items":[]}'
|
|
else
|
|
printf 0
|
|
fi
|
|
}
|
|
|
|
cluster_ssh() {
|
|
echo 'zero-replica Worker attempted to collect metrics' >&2
|
|
return 1
|
|
}
|
|
|
|
namespace=easyai
|
|
if workload_metrics_for_site worker ningbo; then
|
|
echo 'zero-replica Worker site returned metrics success' >&2
|
|
exit 1
|
|
else
|
|
metrics_status=$?
|
|
fi
|
|
[[ $metrics_status == 2 ]]
|
|
|
|
remote_kubectl() {
|
|
printf '%s\n' '{"items":[
|
|
{"metadata":{},"spec":{"nodeName":"easyai-hongkong"},"status":{"podIP":"10.42.3.10","conditions":[{"type":"Ready","status":"True"}]}},
|
|
{"metadata":{},"spec":{"nodeName":"easyai-hongkong-worker-2"},"status":{"podIP":"10.42.5.10","conditions":[{"type":"Ready","status":"True"}]}}
|
|
]}'
|
|
}
|
|
|
|
cluster_ssh() {
|
|
while IFS= read -r _; do :; done
|
|
printf '# target_host %s\n' "$1"
|
|
printf 'test_metric 1\n'
|
|
}
|
|
|
|
CLUSTER_NINGBO_HOST=root@ningbo.invalid
|
|
CLUSTER_HONGKONG_HOST=root@hongkong.invalid
|
|
CLUSTER_LOS_ANGELES_HOST=root@losangeles.invalid
|
|
CLUSTER_WORKER_NODE_4_NAME=easyai-hongkong-worker-2
|
|
CLUSTER_WORKER_NODE_4_HOST=root@hongkong-worker-2.invalid
|
|
metrics=$(workload_metrics_for_site worker hongkong)
|
|
grep -Fq '# target_host root@hongkong.invalid' <<<"$metrics"
|
|
grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics"
|
|
[[ $(grep -c '^test_metric 1$' <<<"$metrics") == 2 ]]
|
|
[[ $(metric_min test_metric <<<"$metrics") == 1 ]]
|
|
[[ $(metric_max test_metric <<<"$metrics") == 1 ]]
|
|
[[ $(metric_sum test_metric <<<"$metrics") == 2 ]]
|
|
|
|
memory_attempts=$tmp/memory-attempts
|
|
printf '0\n' >"$memory_attempts"
|
|
remote_kubectl() {
|
|
local attempts
|
|
attempts=$(<"$memory_attempts")
|
|
attempts=$((attempts + 1))
|
|
printf '%s\n' "$attempts" >"$memory_attempts"
|
|
if (( attempts == 1 )); then
|
|
return 1
|
|
fi
|
|
printf 'easyai-hongkong 100m 5%% 1024Mi 64%%\n'
|
|
}
|
|
sleep() { :; }
|
|
[[ $(read_node_memory_percent easyai-hongkong) == 64 ]]
|
|
[[ $(<"$memory_attempts") == 2 ]]
|
|
|
|
remote_kubectl() { return 1; }
|
|
if read_node_memory_percent easyai-hongkong >/dev/null 2>&1; then
|
|
echo 'unavailable node metrics passed resource precondition collection' >&2
|
|
exit 1
|
|
else
|
|
resource_status=$?
|
|
fi
|
|
[[ $resource_status == 2 ]]
|
|
|
|
cluster_root=$tmp/repository
|
|
local_acceptance_report=$tmp/local-report.json
|
|
summary=$tmp/summary.partial.json
|
|
partial=$tmp/acceptance-report.partial.json
|
|
printf '{}\n' >"$local_acceptance_report"
|
|
printf '{}\n' >"$summary"
|
|
printf '{}\n' >"$partial"
|
|
node_calls=$tmp/node-calls
|
|
node() { printf '%s\n' "$*" >>"$node_calls"; }
|
|
merge_partial_acceptance_report "$summary" "$partial"
|
|
[[ $(wc -l <"$node_calls" | tr -d ' ') == 1 ]]
|
|
grep -Fq ' validate --input ' "$node_calls"
|
|
unlink "$partial"
|
|
merge_partial_acceptance_report "$summary" "$partial"
|
|
[[ $(wc -l <"$node_calls" | tr -d ' ') == 2 ]]
|
|
grep -Fq ' merge-production --local-report ' "$node_calls"
|
|
|
|
acceptance_admin_base=http://10.43.1.1:8088
|
|
acceptance_admin_fallback_base=http://10.43.2.2:8088
|
|
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=test-token
|
|
CLUSTER_NINGBO_HOST=root@ningbo.invalid
|
|
CLUSTER_HONGKONG_HOST=root@hongkong.invalid
|
|
admin_attempts=$tmp/admin-attempts
|
|
printf '0\n' >"$admin_attempts"
|
|
cluster_ssh() {
|
|
local attempts
|
|
attempts=$(<"$admin_attempts")
|
|
attempts=$((attempts + 1))
|
|
printf '%s\n' "$attempts" >"$admin_attempts"
|
|
if (( attempts == 1 )); then
|
|
return 1
|
|
fi
|
|
printf '{"status":"running"}\n200\n'
|
|
}
|
|
admin_output=$tmp/admin-output.json
|
|
admin_request GET /api/admin/system/acceptance/runs/test-run '' "$admin_output"
|
|
[[ $(jq -r '.status' "$admin_output") == running ]]
|
|
[[ $(<"$admin_attempts") == 2 ]]
|
|
|
|
printf '0\n' >"$admin_attempts"
|
|
cluster_ssh() {
|
|
local attempts
|
|
attempts=$(<"$admin_attempts")
|
|
attempts=$((attempts + 1))
|
|
printf '%s\n' "$attempts" >"$admin_attempts"
|
|
case $attempts in
|
|
1) return 1 ;;
|
|
2) printf '{"error":"acceptance run is not running"}\n409\n' ;;
|
|
3) printf '{"status":"failed"}\n200\n' ;;
|
|
esac
|
|
}
|
|
admin_body='{"passed":false}'
|
|
admin_request POST /api/admin/system/acceptance/runs/test-run/finish "$admin_body" "$admin_output"
|
|
[[ $(jq -r '.status' "$admin_output") == failed ]]
|
|
[[ $(<"$admin_attempts") == 3 ]]
|
|
|
|
passing_pressure_row='2026-08-01T00:00:00Z,0,0,899,149,200,84,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100'
|
|
pressure_row_passes_live_gates "$passing_pressure_row"
|
|
[[ $(pressure_row_live_gate_reason '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100') == \
|
|
'oldest_wait_seconds=900 limit=900' ]]
|
|
[[ $(pressure_row_live_gate_reason '2026-08-01T00:00:00Z,0,0,899,149,200,85,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100') == \
|
|
'node_memory_percent=85 hard_limit=85' ]]
|
|
|
|
for failed_pressure_row in \
|
|
'2026-08-01T00:00:00Z,0,0,901,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,150,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,149,200,80,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1536,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,1,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,1,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,0,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,3,48,0,8,8,0,0,0,16,16,0,0,1535,100' \
|
|
'2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1536,100'; do
|
|
if pressure_row_passes_live_gates "$failed_pressure_row"; then
|
|
echo "unsafe pressure row passed live gates: $failed_pressure_row" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
long_running_load() {
|
|
while :; do
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
(sleep 1; exit 1) &
|
|
pressure_pid=$!
|
|
AI_GATEWAY_ACCEPTANCE_PROCESS_WATCH_SECONDS=1
|
|
active_load_pid=
|
|
if run_with_pressure_monitor "$pressure_pid" long_running_load; then
|
|
echo 'load continued after the pressure sampler failed' >&2
|
|
exit 1
|
|
fi
|
|
wait "$pressure_pid" >/dev/null 2>&1 || true
|
|
[[ -z $active_load_pid ]]
|
|
|
|
deploy_call_line=$(grep -n '^deploy_protocol_emulator$' "$script" | cut -d: -f1)
|
|
activate_call_line=$(grep -n '^create_and_activate_run$' "$script" | cut -d: -f1)
|
|
[[ -n $deploy_call_line && -n $activate_call_line && $deploy_call_line -lt $activate_call_line ]]
|
|
|
|
echo 'production_acceptance_script_tests=PASS'
|