fix(acceptance): 增强生产验收控制面容错
原因:资源前置检查中的一次瞬时 SSH 中断会直接终止 Run,失败收尾又可能因 partial 报告已存在而无法完成。\n\n影响:节点内存采集增加有界重试并区分传输故障;验收管理 GET 与 finish 支持香港本地入口安全回退,并对响应丢失后的 409 反查 Run 终态;已有 partial 报告会验证复用。\n\n验证:bash -n、ShellCheck、production-acceptance-script-test.sh、manual-release-test.sh 通过。
This commit is contained in:
@@ -40,6 +40,33 @@ awk '
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/pressure-monitor.sh"
|
||||
|
||||
awk '
|
||||
/^read_node_memory_percent\(\)/ { capture = 1 }
|
||||
capture && /^verify_resource_preconditions\(\)/ { exit }
|
||||
capture { print }
|
||||
' "$script" >"$tmp/resource-precondition.sh"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/resource-precondition.sh"
|
||||
|
||||
awk '
|
||||
/^merge_partial_acceptance_report\(\)/ { capture = 1 }
|
||||
capture && /^mark_run_failed\(\)/ { exit }
|
||||
capture { print }
|
||||
' "$script" >"$tmp/partial-report.sh"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/partial-report.sh"
|
||||
|
||||
awk '
|
||||
/^admin_request\(\)/ { capture = 1 }
|
||||
capture && /^verify_release_cas\(\)/ { exit }
|
||||
capture { print }
|
||||
' "$script" >"$tmp/admin-request.sh"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/admin-request.sh"
|
||||
|
||||
cluster_root=$tmp/repository
|
||||
git init -q "$cluster_root"
|
||||
git -C "$cluster_root" config user.name 'Acceptance Script Test'
|
||||
@@ -108,6 +135,87 @@ grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics"
|
||||
[[ $(metric_max test_metric <<<"$metrics") == 1 ]]
|
||||
[[ $(metric_sum test_metric <<<"$metrics") == 2 ]]
|
||||
|
||||
memory_attempts=$tmp/memory-attempts
|
||||
printf '0\n' >"$memory_attempts"
|
||||
remote_kubectl() {
|
||||
local attempts
|
||||
attempts=$(<"$memory_attempts")
|
||||
attempts=$((attempts + 1))
|
||||
printf '%s\n' "$attempts" >"$memory_attempts"
|
||||
if (( attempts == 1 )); then
|
||||
return 1
|
||||
fi
|
||||
printf 'easyai-hongkong 100m 5%% 1024Mi 64%%\n'
|
||||
}
|
||||
sleep() { :; }
|
||||
[[ $(read_node_memory_percent easyai-hongkong) == 64 ]]
|
||||
[[ $(<"$memory_attempts") == 2 ]]
|
||||
|
||||
remote_kubectl() { return 1; }
|
||||
if read_node_memory_percent easyai-hongkong >/dev/null 2>&1; then
|
||||
echo 'unavailable node metrics passed resource precondition collection' >&2
|
||||
exit 1
|
||||
else
|
||||
resource_status=$?
|
||||
fi
|
||||
[[ $resource_status == 2 ]]
|
||||
|
||||
cluster_root=$tmp/repository
|
||||
local_acceptance_report=$tmp/local-report.json
|
||||
summary=$tmp/summary.partial.json
|
||||
partial=$tmp/acceptance-report.partial.json
|
||||
printf '{}\n' >"$local_acceptance_report"
|
||||
printf '{}\n' >"$summary"
|
||||
printf '{}\n' >"$partial"
|
||||
node_calls=$tmp/node-calls
|
||||
node() { printf '%s\n' "$*" >>"$node_calls"; }
|
||||
merge_partial_acceptance_report "$summary" "$partial"
|
||||
[[ $(wc -l <"$node_calls" | tr -d ' ') == 1 ]]
|
||||
grep -Fq ' validate --input ' "$node_calls"
|
||||
unlink "$partial"
|
||||
merge_partial_acceptance_report "$summary" "$partial"
|
||||
[[ $(wc -l <"$node_calls" | tr -d ' ') == 2 ]]
|
||||
grep -Fq ' merge-production --local-report ' "$node_calls"
|
||||
|
||||
acceptance_admin_base=http://10.43.1.1:8088
|
||||
acceptance_admin_fallback_base=http://10.43.2.2:8088
|
||||
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=test-token
|
||||
CLUSTER_NINGBO_HOST=root@ningbo.invalid
|
||||
CLUSTER_HONGKONG_HOST=root@hongkong.invalid
|
||||
admin_attempts=$tmp/admin-attempts
|
||||
printf '0\n' >"$admin_attempts"
|
||||
cluster_ssh() {
|
||||
local attempts
|
||||
attempts=$(<"$admin_attempts")
|
||||
attempts=$((attempts + 1))
|
||||
printf '%s\n' "$attempts" >"$admin_attempts"
|
||||
if (( attempts == 1 )); then
|
||||
return 1
|
||||
fi
|
||||
printf '{"status":"running"}\n200\n'
|
||||
}
|
||||
admin_output=$tmp/admin-output.json
|
||||
admin_request GET /api/admin/system/acceptance/runs/test-run '' "$admin_output"
|
||||
[[ $(jq -r '.status' "$admin_output") == running ]]
|
||||
[[ $(<"$admin_attempts") == 2 ]]
|
||||
|
||||
printf '0\n' >"$admin_attempts"
|
||||
cluster_ssh() {
|
||||
local attempts
|
||||
attempts=$(<"$admin_attempts")
|
||||
attempts=$((attempts + 1))
|
||||
printf '%s\n' "$attempts" >"$admin_attempts"
|
||||
case $attempts in
|
||||
1) return 1 ;;
|
||||
2) printf '{"error":"acceptance run is not running"}\n409\n' ;;
|
||||
3) printf '{"status":"failed"}\n200\n' ;;
|
||||
esac
|
||||
}
|
||||
admin_body='{"passed":false}'
|
||||
admin_request POST /api/admin/system/acceptance/runs/test-run/finish "$admin_body" "$admin_output"
|
||||
[[ $(jq -r '.status' "$admin_output") == failed ]]
|
||||
[[ $(<"$admin_attempts") == 3 ]]
|
||||
|
||||
passing_pressure_row='2026-08-01T00:00:00Z,0,0,899,149,200,84,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100'
|
||||
pressure_row_passes_live_gates "$passing_pressure_row"
|
||||
[[ $(pressure_row_live_gate_reason '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100') == \
|
||||
|
||||
Reference in New Issue
Block a user