fix(acceptance): 阻断本地控制面漂移污染验收

本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
2026-07-31 23:07:19 +08:00
parent 015ff8ea6c
commit a95184b5b6
20 changed files with 970 additions and 85 deletions
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+# Shared read-only guards for the local three-server K3s acceptance cluster.
+# The caller owns shell options and failure reporting.
+
+capture_local_control_plane_identity() {
+  local context=$1 cluster_name=$2 containers nodes
+  containers=$(docker inspect \
+    "k3d-${cluster_name}-server-0" \
+    "k3d-${cluster_name}-server-1" \
+    "k3d-${cluster_name}-server-2" |
+    jq -c --arg network "k3d-$cluster_name" '[.[] | {
+      name:(.Name | ltrimstr("/")),
+      id:.Id,
+      restartCount:.RestartCount,
+      startedAt:.State.StartedAt,
+      ip:.NetworkSettings.Networks[$network].IPAddress
+    }] | sort_by(.name)') || return 1
+  nodes=$(kubectl --context "$context" get nodes -o json |
+    jq -c '[.items[] | {
+      name:.metadata.name,
+      uid:.metadata.uid,
+      site:.metadata.labels["easyai.io/site"],
+      cpu:.status.allocatable.cpu,
+      memory:.status.allocatable.memory
+    }] | sort_by(.name)') || return 1
+  jq -cn --argjson containers "$containers" --argjson nodes "$nodes" \
+    '{containers:$containers,nodes:$nodes}'
+}
+
+record_local_control_plane_identity() {
+  local context=$1 cluster_name=$2 output=$3 current temporary
+  current=$(capture_local_control_plane_identity "$context" "$cluster_name") || return 1
+  temporary="${output}.tmp.$$"
+  jq -cn \
+    --arg recordedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+    --argjson current "$current" \
+    '{schemaVersion:"local-control-plane-identity/v1",recordedAt:$recordedAt} + $current' \
+    >"$temporary"
+  chmod 0600 "$temporary"
+  mv "$temporary" "$output"
+}
+
+verify_local_control_plane_identity() {
+  local context=$1 cluster_name=$2 baseline=$3 expected current
+  [[ -f $baseline && ! -L $baseline ]] || return 1
+  expected=$(jq -cS '{containers,nodes}' "$baseline") || return 1
+  current=$(capture_local_control_plane_identity "$context" "$cluster_name" |
+    jq -cS '{containers,nodes}') || return 1
+  [[ $current == "$expected" ]]
+}
+
+verify_local_node_capacity() {
+  local context=$1
+  kubectl --context "$context" get nodes -o json | jq -e '
+    def cpu_m:
+      if endswith("m") then rtrimstr("m") | tonumber else tonumber * 1000 end;
+    def memory_mi:
+      if endswith("Ki") then rtrimstr("Ki") | tonumber / 1024
+      elif endswith("Mi") then rtrimstr("Mi") | tonumber
+      elif endswith("Gi") then rtrimstr("Gi") | tonumber * 1024
+      else 0 end;
+    ([.items[] | select(.metadata.labels["easyai.io/workload"] == "true") |
+      select((.status.allocatable.cpu | cpu_m) == 3000 and
+             (.status.allocatable.memory | memory_mi) == 6656)] | length) == 2 and
+    ([.items[] | select(.metadata.labels["easyai.io/site"] == "los-angeles") |
+      select((.status.allocatable.cpu | cpu_m) == 1000 and
+             (.status.allocatable.memory | memory_mi) == 2560)] | length) == 1 and
+    ([.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length) == 3 and
+    ([.items[] | select(any(.status.conditions[]; .type == "MemoryPressure" and .status == "True"))] | length) == 0
+  ' >/dev/null
+}
+
+verify_local_apiserver_ready() {
+  local context=$1
+  kubectl --context "$context" get --raw='/readyz?verbose' 2>/dev/null |
+    grep -Fq 'readyz check passed'
+}
+
+verify_local_etcd_runtime_logs() {
+  local cluster_name=$1 baseline=$2 recorded_at name
+  [[ -f $baseline && ! -L $baseline ]] || return 1
+  recorded_at=$(jq -r '.recordedAt' "$baseline") || return 1
+  [[ $recorded_at == ????-??-??T*Z ]]
+  for name in \
+    "k3d-${cluster_name}-server-0" \
+    "k3d-${cluster_name}-server-1" \
+    "k3d-${cluster_name}-server-2"; do
+    if docker logs --since "$recorded_at" "$name" 2>&1 |
+      grep -Eqi 'heartbeat interval is too long|apply request took too long|slow fdatasync|failed to check local etcd status|etcdserver: request timed out|waiting for ReadIndex response took too long'; then
+      return 1
+    fi
+  done
+}