fix(acceptance): 阻断本地控制面漂移污染验收

本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。

scripts/acceptance/run-local-acceptance.sh

@@ -7,13 +7,18 @@ private_root="$repository_root/.local-secrets/acceptance"
 state_root="$private_root/state"
 context=k3d-easyai-acceptance-local
 namespace=easyai
+cluster_name=easyai-acceptance-local
 runtime_path_file="$state_root/current-runtime-path"
 snapshot="$state_root/snapshot.json"
 load_binary="$state_root/easyai-ai-gateway-acceptance-load"
+identity_file="$state_root/control-plane-identity.json"
 active_load_pid=
 netem_active=false
 current_phase=initialization
 stable_profile=
+failure_gate_id=local_execution_incomplete
+# shellcheck source=scripts/acceptance/local-control-plane.sh
+source "$script_dir/local-control-plane.sh"
 
 usage() {
   cat <<'EOF'
@@ -51,12 +56,16 @@ cleanup() {
   fi
   if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then
     restore_profile_best_effort "${stable_profile:-P24}"
+    if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then
+      failure_gate_id=$(jq -r '.id' "$failure_gate_file" 2>/dev/null || printf '%s' "$failure_gate_id")
+    fi
     local partial_args=(
       "$script_dir/report.mjs" build-local-partial
       --runtime "$runtime" \
       --snapshot "$snapshot" \
       --reports "$report_root" \
       --failure-phase "$current_phase" \
+      --failure-gate "$failure_gate_id" \
       --output "$report_root/acceptance-report.partial.json"
     )
     if [[ -n $stable_profile ]]; then
@@ -89,6 +98,7 @@ require_local_cluster() {
   release_sha=$(jq -r '.releaseSha' "$runtime")
   report_root="$repository_root/dist/acceptance/local/$run_id"
   install -d -m 0700 "$report_root"
+  failure_gate_file="$report_root/failure-gate.json"
   (
     cd "$repository_root/apps/api"
     env -u AI_GATEWAY_TEST_DATABASE_URL go build -trimpath \
@@ -96,6 +106,27 @@ require_local_cluster() {
   )
 }
 
+record_failure_gate() {
+  local id=$1 detail=$2 temporary
+  [[ $id =~ ^[a-z0-9_]+$ ]]
+  failure_gate_id=$id
+  [[ -n ${failure_gate_file:-} ]] || return 0
+  if [[ ! -e $failure_gate_file ]]; then
+    temporary="${failure_gate_file}.tmp.$$"
+    jq -n --arg id "$id" --arg detail "$detail" \
+      '{schemaVersion:"acceptance-failure-gate/v1",id:$id,detail:$detail}' >"$temporary"
+    chmod 0600 "$temporary"
+    mv "$temporary" "$failure_gate_file"
+  fi
+}
+
+fail_gate() {
+  local id=$1 detail=$2
+  record_failure_gate "$id" "$detail"
+  echo "gate=$id $detail" >&2
+  return 1
+}
+
 database_query() {
   local sql=$1 primary
   primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
@@ -115,6 +146,12 @@ load_environment() {
 run_load() {
   local profile=$1 report_path=$2
   shift 2
+  local stdout_path="$report_path.stdout" stdout_temporary="${report_path}.stdout.tmp.$$"
+  local load_pid load_status=0 unavailable_samples=0 infrastructure_failure=false
+  [[ ! -e $report_path && ! -e $stdout_path && ! -e $stdout_temporary ]] ||
+    fail_gate acceptance_report_exists "refusing to overwrite an existing load report"
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
+    fail_gate local_control_plane_restarted "local K3s server identity changed before load"
   AI_GATEWAY_ACCEPTANCE_GATEWAYS='https://127.0.0.1:18443,https://127.0.0.1:19443' \
   AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=gateway.easyai.local \
   AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE="$state_root/ca.crt" \
@@ -125,8 +162,49 @@ run_load() {
   AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
   AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
     "$load_binary" -profile "$profile" -report "$report_path" "$@" \
-      >"$report_path.stdout"
-  chmod 0600 "$report_path" "$report_path.stdout"
+      >"$stdout_temporary" &
+  load_pid=$!
+  active_load_pid=$load_pid
+  while kill -0 "$load_pid" >/dev/null 2>&1; do
+    if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
+      record_failure_gate local_control_plane_restarted "local K3s server restarted during load"
+      infrastructure_failure=true
+      kill "$load_pid" >/dev/null 2>&1 || true
+      break
+    fi
+    if verify_local_apiserver_ready "$context"; then
+      unavailable_samples=0
+    else
+      unavailable_samples=$((unavailable_samples + 1))
+      if ((unavailable_samples >= 3)); then
+        record_failure_gate local_control_plane_unavailable "Kubernetes API readiness failed for three consecutive samples"
+        infrastructure_failure=true
+        kill "$load_pid" >/dev/null 2>&1 || true
+        break
+      fi
+    fi
+    sleep 2
+  done
+  if wait "$load_pid"; then
+    load_status=0
+  else
+    load_status=$?
+  fi
+  active_load_pid=
+  if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
+    record_failure_gate local_control_plane_restarted "local K3s server identity changed at load completion"
+    infrastructure_failure=true
+  fi
+  [[ ! -e $stdout_path ]] || fail_gate acceptance_report_exists "load stdout report already exists"
+  mv "$stdout_temporary" "$stdout_path"
+  chmod 0600 "$stdout_path"
+  if [[ -f $report_path && ! -L $report_path ]]; then
+    chmod 0600 "$report_path"
+  fi
+  if [[ $infrastructure_failure == true ]]; then
+    return 1
+  fi
+  ((load_status == 0)) || return "$load_status"
   jq -e '.schemaVersion == "acceptance-load-report/v1" and .secretSafe == true and .passed == true' \
     "$report_path" >/dev/null
 }
@@ -147,15 +225,26 @@ sample_resources() {
 
 verify_hard_gates() {
   local unhealthy ready sync_state connections max_connections queue duplicate_remote duplicate_billing
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
+    fail_gate local_control_plane_restarted "local K3s server identity changed"
+  verify_local_apiserver_ready "$context" ||
+    fail_gate local_control_plane_unavailable "Kubernetes API readiness failed"
+  verify_local_node_capacity "$context" ||
+    fail_gate local_node_capacity_mismatch "node allocatable resources drifted from the acceptance envelope"
+  verify_local_etcd_runtime_logs "$cluster_name" "$identity_file" ||
+    fail_gate local_etcd_latency "etcd emitted a severe latency or timeout signal during the run"
   ready=$(kubectl --context "$context" get nodes -o json |
     jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length')
-  [[ $ready == 3 ]]
+  [[ $ready == 3 ]] || fail_gate local_nodes_not_ready "not all three local K3s nodes are Ready"
   [[ $(kubectl --context "$context" get nodes -o json |
-    jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]]
+    jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] ||
+    fail_gate local_node_memory_pressure "a local K3s node reports MemoryPressure"
   [[ $(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
-    -o 'jsonpath={.status.readyInstances}') == 2 ]]
+    -o 'jsonpath={.status.readyInstances}') == 2 ]] ||
+    fail_gate local_postgres_not_ready "local PostgreSQL is not 2/2 Ready"
   sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
-  [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]]
+  [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] ||
+    fail_gate local_postgres_not_synchronous "local PostgreSQL lost its synchronous replica"
   unhealthy=$(kubectl --context "$context" -n "$namespace" get pods -o json |
     jq '[.items[]
       | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
@@ -163,11 +252,11 @@ verify_hard_gates() {
       | .status.containerStatuses[]?
       | select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or
                .state.terminated.reason == "OOMKilled")] | length')
-  [[ $unhealthy == 0 ]]
+  [[ $unhealthy == 0 ]] || fail_gate gateway_pod_restarted "an API or Worker container restarted or was OOMKilled"
   while read -r _node _cpu _cpu_percent _memory memory_percent; do
     memory_percent=${memory_percent%\%}
     [[ $memory_percent =~ ^[0-9]+$ ]]
-    (( memory_percent < 80 ))
+    (( memory_percent < 80 )) || fail_gate local_node_memory_target "local node memory reached the 80 percent target"
   done < <(kubectl --context "$context" top nodes --no-headers)
   while read -r _pod _cpu memory; do
     case $memory in
@@ -176,20 +265,23 @@ verify_hard_gates() {
       *Ki) memory=$(awk -v value="${memory%Ki}" 'BEGIN {printf "%.0f", value/1024}') ;;
       *) return 1 ;;
     esac
-    (( memory < 1536 ))
+    (( memory < 1536 )) || fail_gate gateway_pod_memory_limit "an API or Worker Pod reached 1.5 GiB RSS"
   done < <(kubectl --context "$context" -n "$namespace" top pods \
     -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers)
   connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
   max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
-  (( connections < 150 && connections * 4 < max_connections * 3 ))
+  (( connections < 150 && connections * 4 < max_connections * 3 )) ||
+    fail_gate postgres_connection_budget "PostgreSQL client sessions exceeded the acceptance budget"
   queue=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
-  [[ $queue == 0:0 ]]
+  [[ $queue == 0:0 ]] || fail_gate acceptance_queue_not_drained "acceptance queue did not return to zero"
   duplicate_remote=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) d;")
   duplicate_billing=$(database_query "SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) d;")
-  [[ $duplicate_remote == 0 && $duplicate_billing == 0 ]]
+  [[ $duplicate_remote == 0 ]] || fail_gate duplicate_upstream_submission "duplicate remote task IDs were detected"
+  [[ $duplicate_billing == 0 ]] || fail_gate duplicate_billing "duplicate billing transactions were detected"
   kubectl --context "$context" -n "$namespace" exec deployment/easyai-acceptance-callback-collector -- \
     wget -qO- http://127.0.0.1:8091/report |
-    jq -e '.duplicates == 0 and .invalid == 0' >/dev/null
+    jq -e '.duplicates == 0 and .invalid == 0' >/dev/null ||
+    fail_gate callback_validation_failed "callback collector detected duplicate or invalid callbacks"
 }
 
 apply_profile() {
@@ -218,7 +310,8 @@ apply_profile() {
       AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
       AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
       AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
-      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null
+      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
+      AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
     kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
   done
   for deployment in easyai-api-ningbo easyai-api-hongkong easyai-capacity-controller; do
@@ -254,7 +347,8 @@ restore_profile_best_effort() {
       AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
       AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
       AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
-      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null 2>&1 || true
+      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
+      AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null 2>&1 || true
   done
 }
 
@@ -514,23 +608,29 @@ full() {
 
 command=${1:-}
 shift || true
-require_local_cluster
-load_environment
 case $command in
   quick)
     [[ $# -eq 0 ]] || { usage >&2; exit 64; }
-    quick
     ;;
-  artifact-smoke)
+  artifact-smoke|full)
     [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
-    artifact_smoke "$2"
-    ;;
-  full)
-    [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
-    full "$2"
     ;;
   *)
     usage >&2
     exit 64
     ;;
 esac
+"$script_dir/local-cluster.sh" new-run
+require_local_cluster
+load_environment
+case $command in
+  quick)
+    quick
+    ;;
+  artifact-smoke)
+    artifact_smoke "$2"
+    ;;
+  full)
+    full "$2"
+    ;;
+esac