fix(acceptance): 阻断本地控制面漂移污染验收

本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。

scripts/acceptance/local-cluster.sh

@@ -12,6 +12,11 @@ media_root="$private_root/media"
 cluster_name=easyai-acceptance-local
 context="k3d-$cluster_name"
 namespace=easyai
+cluster_network="k3d-$cluster_name"
+cluster_subnet=172.30.0.0/16
+identity_file="$state_root/control-plane-identity.json"
+# shellcheck source=scripts/acceptance/local-control-plane.sh
+source "$script_dir/local-control-plane.sh"
 
 usage() {
   cat <<'EOF'
@@ -19,11 +24,13 @@ Usage:
   scripts/acceptance/local-cluster.sh install-tools
   scripts/acceptance/local-cluster.sh preflight
   scripts/acceptance/local-cluster.sh up --snapshot <acceptance-snapshot-v1.json>
+  scripts/acceptance/local-cluster.sh new-run
   scripts/acceptance/local-cluster.sh status
   scripts/acceptance/local-cluster.sh down --confirm
 
-The full cluster requires Docker Desktop memory >= 24 GiB. Failed `up` runs are
-kept intact for diagnosis. `down` is the only destructive lifecycle action.
+The full cluster requires Docker Desktop memory >= 24 GiB and host free disk
+>= 30 GiB. Failed `up` runs are kept intact for diagnosis. `down` is the only
+destructive lifecycle action.
 EOF
 }
 
@@ -34,7 +41,9 @@ load_lock() {
   }
   # shellcheck source=/dev/null
   source "$lock_file"
-  : "${K3D_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
+  : "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
+  : "${CNPG_CONTROLLER_IMAGE:?}" "${CNPG_POSTGRES_IMAGE:?}" "${K3S_COREDNS_IMAGE:?}"
+  : "${K3S_METRICS_SERVER_IMAGE:?}" "${K3S_LOCAL_PATH_IMAGE:?}"
 }
 
 require_command() {
@@ -60,6 +69,7 @@ install_tools() {
   load_lock
   require_command curl
   require_command shasum
+  require_command go
   install -d -m 0700 "$tools_root"
   local os arch checksum url temporary
   os=$(uname -s | tr '[:upper:]' '[:lower:]')
@@ -91,7 +101,9 @@ install_tools() {
   }
   chmod 0555 "$temporary"
   mv "$temporary" "$tools_root/k3d"
-  echo "local_acceptance_tools=PASS k3d=$K3D_VERSION path=$tools_root/k3d"
+  GOBIN="$tools_root" go install "github.com/google/go-containerregistry/cmd/crane@$CRANE_VERSION"
+  chmod 0555 "$tools_root/crane"
+  echo "local_acceptance_tools=PASS k3d=$K3D_VERSION crane=$CRANE_VERSION path=$tools_root"
 }
 
 k3d_binary() {
@@ -106,6 +118,10 @@ docker_memory_bytes() {
   docker info --format '{{.MemTotal}}'
 }
 
+host_available_disk_bytes() {
+  df -Pk "$repository_root" | awk 'NR == 2 {printf "%.0f\n", $4 * 1024}'
+}
+
 preflight() {
   load_lock
   require_command docker
@@ -117,7 +133,7 @@ preflight() {
   require_command go
   require_command node
   require_command curl
-  local k3d memory_bytes architecture
+  local k3d memory_bytes architecture cpu_count available_disk_bytes available_disk_gib
   k3d=$(k3d_binary) || {
     echo 'k3d is missing; run install-tools' >&2
     exit 1
@@ -126,6 +142,10 @@ preflight() {
     echo "k3d must be pinned to $K3D_VERSION" >&2
     exit 1
   }
+  if [[ ! -x $tools_root/crane ]] || ! "$tools_root/crane" version | grep -Fq "${CRANE_VERSION#v}"; then
+    echo "crane must be pinned to $CRANE_VERSION; run install-tools" >&2
+    exit 1
+  fi
   docker info >/dev/null
   memory_bytes=$(docker_memory_bytes)
   [[ $memory_bytes =~ ^[0-9]+$ && $memory_bytes -ge 25769803776 ]] || {
@@ -139,7 +159,18 @@ preflight() {
     exit 1
   }
   docker buildx version >/dev/null
-  echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes architecture=$architecture k3d=$K3D_VERSION"
+  cpu_count=$(docker info --format '{{.NCPU}}')
+  [[ $cpu_count =~ ^[0-9]+$ && $cpu_count -ge 10 ]] || {
+    echo "Docker Desktop CPUs=${cpu_count:-unknown}; local acceptance requires at least 10" >&2
+    exit 1
+  }
+  available_disk_bytes=$(host_available_disk_bytes)
+  [[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge 32212254720 ]] || {
+    available_disk_gib=$(awk -v bytes="${available_disk_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
+    echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least 30 GiB" >&2
+    exit 1
+  }
+  echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes architecture=$architecture k3d=$K3D_VERSION"
 }
 
 ensure_private_material() {
@@ -185,9 +216,174 @@ generate_tls() {
 }
 
 render_k3d_config() {
-  local output=$1
+  local output=$1 host_cpu host_memory_bytes host_memory_mib
+  local site_system_cpu witness_system_cpu site_system_memory witness_system_memory
   [[ $media_root != *'|'* ]]
-  sed "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" "$manifest_root/k3d.yaml" >"$output"
+  host_cpu=$(docker info --format '{{.NCPU}}')
+  host_memory_bytes=$(docker_memory_bytes)
+  host_memory_mib=$((host_memory_bytes / 1024 / 1024))
+  site_system_cpu="$(((host_cpu - 4) * 1000 + 750))m"
+  witness_system_cpu="$(((host_cpu - 2) * 1000 + 750))m"
+  site_system_memory="$((host_memory_mib - 8192 + 1024))Mi"
+  witness_system_memory="$((host_memory_mib - 4096 + 1024))Mi"
+  sed \
+    -e "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" \
+    -e "s|EASYAI_ACCEPTANCE_K3S_IMAGE|${K3S_IMAGE%%@*}|g" \
+    -e "s|EASYAI_ACCEPTANCE_CLUSTER_SUBNET|$cluster_subnet|g" \
+    -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_CPU|$site_system_cpu|g" \
+    -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_CPU|$witness_system_cpu|g" \
+    -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_MEMORY|$site_system_memory|g" \
+    -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_MEMORY|$witness_system_memory|g" \
+    "$manifest_root/k3d.yaml" >"$output"
+}
+
+cluster_volumes() {
+  printf '%s\n' \
+    easyai-acceptance-local-server-0-etcd \
+    easyai-acceptance-local-server-1-etcd \
+    easyai-acceptance-local-server-2-etcd \
+    easyai-acceptance-local-server-0-storage \
+    easyai-acceptance-local-server-1-storage
+}
+
+create_cluster_volumes() {
+  local volume
+  while read -r volume; do
+    if docker volume inspect "$volume" >/dev/null 2>&1; then
+      echo "stale local acceptance volume exists: $volume; run explicit down --confirm" >&2
+      return 1
+    fi
+  done < <(cluster_volumes)
+  while read -r volume; do
+    docker volume create --label easyai.io/environment=local-acceptance "$volume" >/dev/null
+  done < <(cluster_volumes)
+}
+
+remove_cluster_volumes() {
+  local volume
+  while read -r volume; do
+    docker volume rm "$volume" >/dev/null 2>&1 || true
+  done < <(cluster_volumes)
+}
+
+reconcile_cluster_network() {
+  docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
+  local actual_subnet attached_containers
+  actual_subnet=$(docker network inspect "$cluster_network" --format '{{(index .IPAM.Config 0).Subnet}}')
+  attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
+  if [[ $actual_subnet == "$cluster_subnet" ]]; then
+    return 0
+  fi
+  if ((attached_containers != 0)); then
+    echo "stale local acceptance network has active containers: network=$cluster_network expected_subnet=$cluster_subnet actual_subnet=$actual_subnet" >&2
+    return 1
+  fi
+  docker network rm "$cluster_network" >/dev/null
+}
+
+remove_cluster_network() {
+  docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
+  local attached_containers
+  attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
+  if ((attached_containers != 0)); then
+    echo "refusing to remove local acceptance network with active containers: network=$cluster_network containers=$attached_containers" >&2
+    return 1
+  fi
+  docker network rm "$cluster_network" >/dev/null
+}
+
+pull_dependency_images() {
+  local image
+  for image in "$K3S_IMAGE" "$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
+    "$K3S_COREDNS_IMAGE" "$K3S_METRICS_SERVER_IMAGE" "$K3S_LOCAL_PATH_IMAGE"; do
+    pull_dependency_image "$image"
+  done
+}
+
+docker_pull_with_deadline() {
+  local image=$1 pid deadline
+  docker pull "$image" >/dev/null 2>&1 &
+  pid=$!
+  deadline=$((SECONDS + 10))
+  while kill -0 "$pid" >/dev/null 2>&1; do
+    if ((SECONDS >= deadline)); then
+      kill "$pid" >/dev/null 2>&1 || true
+      wait "$pid" >/dev/null 2>&1 || true
+      return 1
+    fi
+    sleep 1
+  done
+  wait "$pid"
+}
+
+docker_start_with_deadline() {
+  local container=$1 pid deadline
+  docker start "$container" >/dev/null 2>&1 &
+  pid=$!
+  deadline=$((SECONDS + 10))
+  while kill -0 "$pid" >/dev/null 2>&1; do
+    if ((SECONDS >= deadline)); then
+      kill "$pid" >/dev/null 2>&1 || true
+      wait "$pid" >/dev/null 2>&1 || true
+      return 1
+    fi
+    sleep 1
+  done
+  wait "$pid"
+}
+
+verify_docker_container_start() {
+  local image=$1 container="easyai-acceptance-start-probe-$$" exit_code
+  docker create --name "$container" --network none --entrypoint /bin/true "$image" >/dev/null
+  if ! docker_start_with_deadline "$container"; then
+    docker rm -f "$container" >/dev/null 2>&1 || true
+    echo 'gate=local_docker_container_start_unavailable Docker cannot start a minimal acceptance container within 10 seconds' >&2
+    return 1
+  fi
+  exit_code=$(docker inspect "$container" --format '{{.State.ExitCode}}')
+  docker rm -f "$container" >/dev/null
+  [[ $exit_code == 0 ]] || {
+    echo "gate=local_docker_container_start_failed minimal acceptance container exit_code=$exit_code" >&2
+    return 1
+  }
+}
+
+pull_dependency_image() {
+  local image=$1 name_tag repository digest cache_tag native_arch archive source_tag
+  name_tag=${image%%@*}
+  repository=${name_tag%:*}
+  digest=${image##*@sha256:}
+  [[ $digest =~ ^[0-9a-f]{64}$ ]]
+  cache_tag="$repository:easyai-lock-${digest:0:20}"
+  if docker image inspect "$image" >/dev/null 2>&1; then
+    docker tag "$image" "$name_tag"
+    docker tag "$image" "$cache_tag"
+    return
+  fi
+  if docker image inspect "$cache_tag" >/dev/null 2>&1; then
+    docker tag "$cache_tag" "$name_tag"
+    return
+  fi
+  if docker_pull_with_deadline "$image"; then
+    docker tag "$image" "$name_tag"
+    docker tag "$name_tag" "$cache_tag"
+    return
+  fi
+  native_arch=$(docker info --format '{{.Architecture}}')
+  case $native_arch in
+    aarch64|arm64) native_arch=arm64 ;;
+    x86_64) native_arch=amd64 ;;
+    *) return 1 ;;
+  esac
+  archive="$state_root/dependency-image.tar"
+  [[ ! -L $archive ]]
+  umask 077
+  "$tools_root/crane" pull --platform "linux/$native_arch" "$image" "$archive"
+  docker load -i "$archive" >/dev/null
+  : >"$archive"
+  source_tag="$repository:i-was-a-digest"
+  docker tag "$source_tag" "$name_tag"
+  docker tag "$source_tag" "$cache_tag"
 }
 
 wait_rollout() {
@@ -430,6 +626,15 @@ render_and_apply_application() {
   done
   kubectl --context "$context" -n "$namespace" scale \
     deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null
+  kubectl --context "$context" -n "$namespace" set resources deployment/easyai-api-ningbo \
+    deployment/easyai-api-hongkong --containers=api \
+    --requests=cpu=250m,memory=512Mi --limits=cpu=750m,memory=2Gi >/dev/null
+  kubectl --context "$context" -n "$namespace" set resources deployment/easyai-worker-ningbo \
+    deployment/easyai-worker-hongkong --containers=worker \
+    --requests=cpu=750m,memory=1536Mi --limits=cpu=1500m,memory=2Gi >/dev/null
+  kubectl --context "$context" -n "$namespace" set env deployment/easyai-worker-ningbo \
+    deployment/easyai-worker-hongkong \
+    AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
 }
 
 bootstrap_runtime() {
@@ -439,7 +644,7 @@ bootstrap_runtime() {
   local snapshot_hash release_sha runtime_file
   snapshot_hash=$(jq -r '.snapshotSha256' "$snapshot")
   release_sha=$(git -C "$repository_root" rev-parse HEAD)
-  runtime_file="$state_root/runtime-$snapshot_hash.json"
+  runtime_file="$state_root/runtime-$snapshot_hash-$(date -u '+%Y%m%dT%H%M%SZ')-$$.json"
   kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
     --ignore-not-found --wait=true >/dev/null
   cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
@@ -561,6 +766,8 @@ up_cluster() {
   api_image="easyai-acceptance-api:$release_sha"
   web_image="easyai-acceptance-web:$release_sha"
   netem_image="easyai-acceptance-netem:$release_sha"
+  pull_dependency_images
+  verify_docker_container_start "${K3S_IMAGE%%@*}"
   docker buildx build --load --platform "linux/$native_arch" --target api \
     -t "$api_image" "$repository_root"
   docker buildx build --load --platform "linux/$native_arch" --target web \
@@ -569,7 +776,10 @@ up_cluster() {
     -t "$netem_image" "$repository_root"
   api_digest=$(docker image inspect "$api_image" --format '{{.Id}}')
   [[ $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
+  verify_docker_container_start "$netem_image"
 
+  reconcile_cluster_network
+  create_cluster_volumes
   config="$state_root/k3d.rendered.yaml"
   render_k3d_config "$config"
   EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config"
@@ -577,7 +787,14 @@ up_cluster() {
   docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null
   docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null
   docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null
-  "$k3d" image import -c "$cluster_name" "$api_image" "$web_image" "$netem_image"
+  verify_local_node_capacity "$context" || {
+    echo 'local K3s node allocatable capacity does not match 4/8, 4/8, 2/4 resource envelope' >&2
+    exit 1
+  }
+  "$k3d" image import -c "$cluster_name" \
+    "$api_image" "$web_image" "$netem_image" \
+    "${CNPG_CONTROLLER_IMAGE%%@*}" "${CNPG_POSTGRES_IMAGE%%@*}" \
+    "${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"
 
   local cnpg_manifest="$state_root/cnpg.yaml"
   curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest"
@@ -590,7 +807,8 @@ up_cluster() {
   kubectl --context "$context" -n cnpg-system rollout status \
     deployment/cnpg-controller-manager --timeout=10m
   apply_runtime_secrets
-  kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/database.yaml" >/dev/null
+  sed "s|imageName: .*|imageName: ${CNPG_POSTGRES_IMAGE%%@*}|" "$manifest_root/database.yaml" |
+    kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
   kubectl --context "$context" -n "$namespace" wait --for=condition=Ready \
     cluster/easyai-postgres --timeout=15m >/dev/null
   run_migrations "$api_image"
@@ -619,9 +837,58 @@ up_cluster() {
   bootstrap_runtime "$api_image" "$api_digest" "$snapshot"
   wait_internal_gateway
   "$script_dir/network-fault.sh" baseline
+  sleep 1
+  record_local_control_plane_identity "$context" "$cluster_name" "$identity_file"
   echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt"
 }
 
+new_run() {
+  load_lock
+  preflight
+  [[ -z $(git -C "$repository_root" status --short) ]] || {
+    echo 'local acceptance requires a clean source tree for a fresh Run ID' >&2
+    exit 1
+  }
+  if ! private_file "$state_root/current-runtime-path" || ! private_file "$state_root/snapshot.json"; then
+    echo 'local acceptance runtime or snapshot is unavailable' >&2
+    exit 1
+  fi
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
+    echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild the cluster' >&2
+    exit 1
+  }
+  verify_local_node_capacity "$context" || {
+    echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
+    exit 1
+  }
+  verify_local_apiserver_ready "$context" || {
+    echo 'gate=local_control_plane_unavailable local Kubernetes API is not ready' >&2
+    exit 1
+  }
+  local previous_runtime api_image api_digest previous_release current_release
+  previous_runtime=$(<"$state_root/current-runtime-path")
+  private_file "$previous_runtime" || {
+    echo 'local acceptance current runtime is unavailable' >&2
+    exit 1
+  }
+  previous_release=$(jq -r '.releaseSha' "$previous_runtime")
+  current_release=$(git -C "$repository_root" rev-parse HEAD)
+  [[ $previous_release == "$current_release" ]] || {
+    echo 'local acceptance source changed after cluster creation; rebuild the cluster' >&2
+    exit 1
+  }
+  api_digest=$(jq -r '.apiImageDigest' "$previous_runtime")
+  api_image=$(kubectl --context "$context" -n "$namespace" get deployment easyai-api-ningbo \
+    -o 'jsonpath={.spec.template.spec.containers[?(@.name=="api")].image}')
+  [[ -n $api_image && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
+  bootstrap_runtime "$api_image" "$api_digest" "$state_root/snapshot.json"
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
+    echo 'gate=local_control_plane_restarted local K3s server changed while creating a fresh run' >&2
+    exit 1
+  }
+  echo "local_acceptance_new_run=PASS runtime_path=$state_root/current-runtime-path"
+}
+
 status_cluster() {
   local k3d
   k3d=$(k3d_binary) || {
@@ -634,6 +901,14 @@ status_cluster() {
   }
   kubectl --context "$context" get nodes -o wide
   kubectl --context "$context" -n "$namespace" get cluster,pod,deployment
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
+    echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild required' >&2
+    exit 1
+  }
+  verify_local_node_capacity "$context" || {
+    echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
+    exit 1
+  }
   echo "local_acceptance_status=PASS context=$context"
 }
 
@@ -644,7 +919,11 @@ down_cluster() {
   }
   local k3d
   k3d=$(k3d_binary)
-  "$k3d" cluster delete "$cluster_name"
+  if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then
+    "$k3d" cluster delete "$cluster_name"
+  fi
+  remove_cluster_volumes
+  remove_cluster_network
   echo "local_acceptance_down=PASS preserved_private_root=$private_root"
 }
 
@@ -661,6 +940,10 @@ case ${1:-} in
     shift
     up_cluster "$@"
     ;;
+  new-run)
+    [[ $# -eq 1 ]] || { usage >&2; exit 64; }
+    new_run
+    ;;
   status)
     [[ $# -eq 1 ]] || { usage >&2; exit 64; }
     status_cluster