diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 63f6677..bd511ca 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -333,6 +333,7 @@ report_root= stable_profile=P24 active_profile=P24 single_node_stable_slots=4 +single_node_finish_status= failure_reason= failure_gate_id= failure_recorded=false @@ -3758,6 +3759,17 @@ finish_single_node_diagnostic() { task_state=$(database_query " SELECT count(*)||':'||count(*) FILTER (WHERE status='succeeded')||':'||count(*) FILTER (WHERE status IN ('queued','running')) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;") + local task_total task_succeeded task_active task_state_extra + IFS=':' read -r task_total task_succeeded task_active task_state_extra <<<"$task_state" + if [[ $diagnostic_status == completed ]] && + { [[ -n $task_state_extra ]] || + [[ ! $task_total =~ ^[0-9]+$ ]] || + [[ ! $task_succeeded =~ ^[0-9]+$ ]] || + [[ ! $task_active =~ ^[0-9]+$ ]] || + (( task_total <= 0 || task_succeeded != task_total || task_active != 0 )); }; then + diagnostic_status=failed + reason="single-node task terminal gate failed: state=$task_state" + fi local results_json=$temporary_root/single-node-results.json local -a result_files=() shopt -s nullglob @@ -3794,12 +3806,13 @@ FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;") chmod 0600 "$report_root/single-node-summary.json" local finish_body finish_body=$(jq -cn \ - --arg reason "single-node diagnostic completed; real upstream and certification intentionally skipped" \ + --arg reason "$reason" \ --arg status "$diagnostic_status" \ --argjson stableSlots "$single_node_stable_slots" \ '{passed:false,failureReason:$reason,report:{diagnosticStatus:$status,site:"ningbo",realUpstreamRequests:0,certification:false,stableVideoSlots:$stableSlots}}') admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response" failure_recorded=true + single_node_finish_status=$diagnostic_status cleanup_single_node_emulator cleanup_single_node_kubernetes_resources } @@ -3816,16 +3829,21 @@ run_single_node_acceptance() { fi local slots requests for slots in "${single_node_video_slots[@]}"; do - apply_single_node_capacity "$slots" || break + if ! apply_single_node_capacity "$slots"; then + finish_single_node_diagnostic failed "video capacity did not converge at slots=$slots" + return 1 + fi requests=$((slots * 6)) - (( requests < 64 )) && requests=64 + (( requests < 24 )) && requests=24 if ! run_single_node_profile image-video video-capacity "$slots" "$requests"; then - break + finish_single_node_diagnostic failed "video capacity diagnostic failed at slots=$slots" + return 1 fi single_node_stable_slots=$slots done apply_single_node_capacity "$single_node_stable_slots" finish_single_node_diagnostic completed 'single-node emulator-only capacity ladder completed' + [[ $single_node_finish_status == completed ]] || return 1 echo "single_node_acceptance=PASS_DIAGNOSTIC run_id=$run_id stable_video_slots=$single_node_stable_slots traffic_mode=validation certification=false report=$report_root/single-node-summary.json" }