From c55e52af11df68645e29c4556cb80b8a3280fb48 Mon Sep 17 00:00:00 2001 From: wangbo Date: Sun, 2 Aug 2026 10:13:35 +0800 Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E5=A2=9E=E5=BC=BA=E7=94=9F?= =?UTF-8?q?=E4=BA=A7=E9=AA=8C=E6=94=B6=E6=8E=A7=E5=88=B6=E9=9D=A2=E5=AE=B9?= =?UTF-8?q?=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因：资源前置检查中的一次瞬时 SSH 中断会直接终止 Run，失败收尾又可能因 partial 报告已存在而无法完成。\n\n影响：节点内存采集增加有界重试并区分传输故障；验收管理 GET 与 finish 支持香港本地入口安全回退，并对响应丢失后的 409 反查 Run 终态；已有 partial 报告会验证复用。\n\n验证：bash -n、ShellCheck、production-acceptance-script-test.sh、manual-release-test.sh 通过。 --- scripts/cluster/run-production-acceptance.sh | 108 +++++++++++++++--- .../production-acceptance-script-test.sh | 108 ++++++++++++++++++ 2 files changed, 200 insertions(+), 16 deletions(-) diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 7fec103..4a03539 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -266,6 +266,7 @@ failure_reason= failure_gate_id= failure_recorded=false acceptance_admin_base= +acceptance_admin_fallback_base= acceptance_group_id= acceptance_participants_json='[]' AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY @@ -612,17 +613,24 @@ admin_request() { local body=${3:-} local output=$4 local status response payload token_encoded body_encoded has_body=false - local service_ip service_port remote_script remote_command + local service_ip service_port fallback_service_ip fallback_service_port + local remote_script remote_command fallback_used=false if [[ -z $acceptance_admin_base ]]; then service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \ -o 'jsonpath={.spec.clusterIP}') service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \ -o 'jsonpath={.spec.ports[0].port}') - [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ ]] || { + fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \ + -o 'jsonpath={.spec.clusterIP}') + fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \ + -o 'jsonpath={.spec.ports[0].port}') + [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ && + $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || { echo 'acceptance admin Service has an invalid cluster address' >&2 return 1 } acceptance_admin_base=http://$service_ip:$service_port + acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port fi token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | openssl base64 -A) @@ -651,11 +659,38 @@ fi "$remote_script" "$acceptance_admin_base" "$method" "$path" \ "$token_encoded" "$body_encoded" "$has_body" if ! response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_command"); then - echo "acceptance control API transport failed: method=$method path=$path" >&2 - return 1 + if [[ $method != GET && $path != /api/admin/system/acceptance/runs/*/finish ]]; then + echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2 + return 1 + fi + printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \ + "$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \ + "$token_encoded" "$body_encoded" "$has_body" + if ! response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then + echo "acceptance control API transport failed on both sites: method=$method path=$path" >&2 + return 1 + fi + fallback_used=true fi status=${response##*$'\n'} payload=${response%$'\n'*} + if [[ $fallback_used == true && $status == 409 && + $path == /api/admin/system/acceptance/runs/*/finish ]]; then + local expected_status verification_path verification_response verification_status + expected_status=$(jq -r 'if .passed == true then "passed" else "failed" end' <<<"$body") + verification_path=${path%/finish} + printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \ + "$remote_script" "$acceptance_admin_fallback_base" GET "$verification_path" \ + "$token_encoded" '' false + if verification_response=$(cluster_ssh "$CLUSTER_HONGKONG_HOST" "$remote_command"); then + verification_status=${verification_response##*$'\n'} + payload=${verification_response%$'\n'*} + if [[ $verification_status == 200 && + $(jq -r '.status // empty' <<<"$payload") == "$expected_status" ]]; then + status=200 + fi + fi + fi printf '%s' "$payload" >"$output" [[ $status =~ ^2[0-9][0-9]$ ]] || { echo "acceptance control API failed: method=$method path=$path status=$status" >&2 @@ -721,6 +756,23 @@ verify_release_cas() { fi } +read_node_memory_percent() { + local node=$1 + local attempt output memory_percent + for attempt in 1 2 3; do + if output=$(remote_kubectl top node "$node" --no-headers 2>/dev/null); then + memory_percent=$(awk '{value=$5; sub(/%$/, "", value); print value}' <<<"$output") + if [[ $memory_percent =~ ^[0-9]+$ ]]; then + printf '%s\n' "$memory_percent" + return 0 + fi + fi + (( attempt == 3 )) || sleep 2 + done + echo "Worker node memory collection failed after retries: node=$node attempts=3" >&2 + return 2 +} + verify_resource_preconditions() { local report=$temporary_root/resource-preconditions.csv local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads @@ -772,11 +824,11 @@ verify_resource_preconditions() { for sample in $(seq 1 90); do while read -r node; do [[ -n $node ]] || continue - memory_percent=$(remote_kubectl top node "$node" --no-headers | - awk '{value=$5; sub(/%$/, "", value); print value}') - [[ $memory_percent =~ ^[0-9]+$ && - $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || { - echo "Worker node memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2 + if ! memory_percent=$(read_node_memory_percent "$node"); then + return 2 + fi + [[ $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || { + echo "Worker node memory precondition failed: node=$node memory_percent=$memory_percent limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2 return 1 } printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \ @@ -1446,6 +1498,24 @@ create_and_activate_run() { $(jq -r '.runId' "$activate_response") == "$run_id" ]] } +merge_partial_acceptance_report() { + local summary=$1 + local output=$2 + if [[ -e $output ]]; then + [[ -f $output && ! -L $output ]] || { + echo "existing partial acceptance report is not a regular file: $output" >&2 + return 1 + } + node "$cluster_root/scripts/acceptance/report.mjs" validate \ + --input "$output" >/dev/null + return 0 + fi + node "$cluster_root/scripts/acceptance/report.mjs" merge-production \ + --local-report "$local_acceptance_report" \ + --production-summary "$summary" \ + --output "$output" >/dev/null +} + mark_run_failed() { local reason=$1 local response=$temporary_root/failed-run.json @@ -1518,10 +1588,9 @@ mark_run_failed() { passed:false }' >"$report_root/summary.partial.json" chmod 0600 "$report_root/summary.partial.json" - node "$cluster_root/scripts/acceptance/report.mjs" merge-production \ - --local-report "$local_acceptance_report" \ - --production-summary "$report_root/summary.partial.json" \ - --output "$report_root/acceptance-report.partial.json" >/dev/null || true + merge_partial_acceptance_report \ + "$report_root/summary.partial.json" \ + "$report_root/acceptance-report.partial.json" || true if admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$body" "$response"; then failure_recorded=true if ! wait_for_terminal_acceptance_tasks_to_drain; then @@ -3602,9 +3671,16 @@ if ! wait_for_post_rollout_stability P24; then exit 1 fi verify_release_cas -if ! verify_resource_preconditions; then - failure_gate_id=resource_preconditions - failure_reason='production resource preconditions failed' +resource_precondition_status=0 +verify_resource_preconditions || resource_precondition_status=$? +if (( resource_precondition_status != 0 )); then + if (( resource_precondition_status == 2 )); then + failure_gate_id=resource_precondition_transport + failure_reason='production resource precondition metrics could not be collected after bounded retries' + else + failure_gate_id=resource_preconditions + failure_reason='production resource preconditions failed' + fi mark_run_failed "$failure_reason" exit 1 fi diff --git a/tests/release/production-acceptance-script-test.sh b/tests/release/production-acceptance-script-test.sh index a6ff006..b4ef8c8 100755 --- a/tests/release/production-acceptance-script-test.sh +++ b/tests/release/production-acceptance-script-test.sh @@ -40,6 +40,33 @@ awk ' # shellcheck source=/dev/null source "$tmp/pressure-monitor.sh" +awk ' + /^read_node_memory_percent\(\)/ { capture = 1 } + capture && /^verify_resource_preconditions\(\)/ { exit } + capture { print } +' "$script" >"$tmp/resource-precondition.sh" + +# shellcheck source=/dev/null +source "$tmp/resource-precondition.sh" + +awk ' + /^merge_partial_acceptance_report\(\)/ { capture = 1 } + capture && /^mark_run_failed\(\)/ { exit } + capture { print } +' "$script" >"$tmp/partial-report.sh" + +# shellcheck source=/dev/null +source "$tmp/partial-report.sh" + +awk ' + /^admin_request\(\)/ { capture = 1 } + capture && /^verify_release_cas\(\)/ { exit } + capture { print } +' "$script" >"$tmp/admin-request.sh" + +# shellcheck source=/dev/null +source "$tmp/admin-request.sh" + cluster_root=$tmp/repository git init -q "$cluster_root" git -C "$cluster_root" config user.name 'Acceptance Script Test' @@ -108,6 +135,87 @@ grep -Fq '# target_host root@hongkong-worker-2.invalid' <<<"$metrics" [[ $(metric_max test_metric <<<"$metrics") == 1 ]] [[ $(metric_sum test_metric <<<"$metrics") == 2 ]] +memory_attempts=$tmp/memory-attempts +printf '0\n' >"$memory_attempts" +remote_kubectl() { + local attempts + attempts=$(<"$memory_attempts") + attempts=$((attempts + 1)) + printf '%s\n' "$attempts" >"$memory_attempts" + if (( attempts == 1 )); then + return 1 + fi + printf 'easyai-hongkong 100m 5%% 1024Mi 64%%\n' +} +sleep() { :; } +[[ $(read_node_memory_percent easyai-hongkong) == 64 ]] +[[ $(<"$memory_attempts") == 2 ]] + +remote_kubectl() { return 1; } +if read_node_memory_percent easyai-hongkong >/dev/null 2>&1; then + echo 'unavailable node metrics passed resource precondition collection' >&2 + exit 1 +else + resource_status=$? +fi +[[ $resource_status == 2 ]] + +cluster_root=$tmp/repository +local_acceptance_report=$tmp/local-report.json +summary=$tmp/summary.partial.json +partial=$tmp/acceptance-report.partial.json +printf '{}\n' >"$local_acceptance_report" +printf '{}\n' >"$summary" +printf '{}\n' >"$partial" +node_calls=$tmp/node-calls +node() { printf '%s\n' "$*" >>"$node_calls"; } +merge_partial_acceptance_report "$summary" "$partial" +[[ $(wc -l <"$node_calls" | tr -d ' ') == 1 ]] +grep -Fq ' validate --input ' "$node_calls" +unlink "$partial" +merge_partial_acceptance_report "$summary" "$partial" +[[ $(wc -l <"$node_calls" | tr -d ' ') == 2 ]] +grep -Fq ' merge-production --local-report ' "$node_calls" + +acceptance_admin_base=http://10.43.1.1:8088 +acceptance_admin_fallback_base=http://10.43.2.2:8088 +AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN=test-token +CLUSTER_NINGBO_HOST=root@ningbo.invalid +CLUSTER_HONGKONG_HOST=root@hongkong.invalid +admin_attempts=$tmp/admin-attempts +printf '0\n' >"$admin_attempts" +cluster_ssh() { + local attempts + attempts=$(<"$admin_attempts") + attempts=$((attempts + 1)) + printf '%s\n' "$attempts" >"$admin_attempts" + if (( attempts == 1 )); then + return 1 + fi + printf '{"status":"running"}\n200\n' +} +admin_output=$tmp/admin-output.json +admin_request GET /api/admin/system/acceptance/runs/test-run '' "$admin_output" +[[ $(jq -r '.status' "$admin_output") == running ]] +[[ $(<"$admin_attempts") == 2 ]] + +printf '0\n' >"$admin_attempts" +cluster_ssh() { + local attempts + attempts=$(<"$admin_attempts") + attempts=$((attempts + 1)) + printf '%s\n' "$attempts" >"$admin_attempts" + case $attempts in + 1) return 1 ;; + 2) printf '{"error":"acceptance run is not running"}\n409\n' ;; + 3) printf '{"status":"failed"}\n200\n' ;; + esac +} +admin_body='{"passed":false}' +admin_request POST /api/admin/system/acceptance/runs/test-run/finish "$admin_body" "$admin_output" +[[ $(jq -r '.status' "$admin_output") == failed ]] +[[ $(<"$admin_attempts") == 3 ]] + passing_pressure_row='2026-08-01T00:00:00Z,0,0,899,149,200,84,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100' pressure_row_passes_live_gates "$passing_pressure_row" [[ $(pressure_row_live_gate_reason '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100') == \