fix(acceptance): 阻断本地控制面漂移污染验收

本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。

scripts/acceptance/local-cluster.sh

@@ -12,6 +12,11 @@ media_root="$private_root/media"
 cluster_name=easyai-acceptance-local
 context="k3d-$cluster_name"
 namespace=easyai
+cluster_network="k3d-$cluster_name"
+cluster_subnet=172.30.0.0/16
+identity_file="$state_root/control-plane-identity.json"
+# shellcheck source=scripts/acceptance/local-control-plane.sh
+source "$script_dir/local-control-plane.sh"
 
 usage() {
   cat <<'EOF'
@@ -19,11 +24,13 @@ Usage:
   scripts/acceptance/local-cluster.sh install-tools
   scripts/acceptance/local-cluster.sh preflight
   scripts/acceptance/local-cluster.sh up --snapshot <acceptance-snapshot-v1.json>
+  scripts/acceptance/local-cluster.sh new-run
   scripts/acceptance/local-cluster.sh status
   scripts/acceptance/local-cluster.sh down --confirm
 
-The full cluster requires Docker Desktop memory >= 24 GiB. Failed `up` runs are
-kept intact for diagnosis. `down` is the only destructive lifecycle action.
+The full cluster requires Docker Desktop memory >= 24 GiB and host free disk
+>= 30 GiB. Failed `up` runs are kept intact for diagnosis. `down` is the only
+destructive lifecycle action.
 EOF
 }
 
@@ -34,7 +41,9 @@ load_lock() {
   }
   # shellcheck source=/dev/null
   source "$lock_file"
-  : "${K3D_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
+  : "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
+  : "${CNPG_CONTROLLER_IMAGE:?}" "${CNPG_POSTGRES_IMAGE:?}" "${K3S_COREDNS_IMAGE:?}"
+  : "${K3S_METRICS_SERVER_IMAGE:?}" "${K3S_LOCAL_PATH_IMAGE:?}"
 }
 
 require_command() {
@@ -60,6 +69,7 @@ install_tools() {
   load_lock
   require_command curl
   require_command shasum
+  require_command go
   install -d -m 0700 "$tools_root"
   local os arch checksum url temporary
   os=$(uname -s | tr '[:upper:]' '[:lower:]')
@@ -91,7 +101,9 @@ install_tools() {
   }
   chmod 0555 "$temporary"
   mv "$temporary" "$tools_root/k3d"
-  echo "local_acceptance_tools=PASS k3d=$K3D_VERSION path=$tools_root/k3d"
+  GOBIN="$tools_root" go install "github.com/google/go-containerregistry/cmd/crane@$CRANE_VERSION"
+  chmod 0555 "$tools_root/crane"
+  echo "local_acceptance_tools=PASS k3d=$K3D_VERSION crane=$CRANE_VERSION path=$tools_root"
 }
 
 k3d_binary() {
@@ -106,6 +118,10 @@ docker_memory_bytes() {
   docker info --format '{{.MemTotal}}'
 }
 
+host_available_disk_bytes() {
+  df -Pk "$repository_root" | awk 'NR == 2 {printf "%.0f\n", $4 * 1024}'
+}
+
 preflight() {
   load_lock
   require_command docker
@@ -117,7 +133,7 @@ preflight() {
   require_command go
   require_command node
   require_command curl
-  local k3d memory_bytes architecture
+  local k3d memory_bytes architecture cpu_count available_disk_bytes available_disk_gib
   k3d=$(k3d_binary) || {
     echo 'k3d is missing; run install-tools' >&2
     exit 1
@@ -126,6 +142,10 @@ preflight() {
     echo "k3d must be pinned to $K3D_VERSION" >&2
     exit 1
   }
+  if [[ ! -x $tools_root/crane ]] || ! "$tools_root/crane" version | grep -Fq "${CRANE_VERSION#v}"; then
+    echo "crane must be pinned to $CRANE_VERSION; run install-tools" >&2
+    exit 1
+  fi
   docker info >/dev/null
   memory_bytes=$(docker_memory_bytes)
   [[ $memory_bytes =~ ^[0-9]+$ && $memory_bytes -ge 25769803776 ]] || {
@@ -139,7 +159,18 @@ preflight() {
     exit 1
   }
   docker buildx version >/dev/null
-  echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes architecture=$architecture k3d=$K3D_VERSION"
+  cpu_count=$(docker info --format '{{.NCPU}}')
+  [[ $cpu_count =~ ^[0-9]+$ && $cpu_count -ge 10 ]] || {
+    echo "Docker Desktop CPUs=${cpu_count:-unknown}; local acceptance requires at least 10" >&2
+    exit 1
+  }
+  available_disk_bytes=$(host_available_disk_bytes)
+  [[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge 32212254720 ]] || {
+    available_disk_gib=$(awk -v bytes="${available_disk_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
+    echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least 30 GiB" >&2
+    exit 1
+  }
+  echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes architecture=$architecture k3d=$K3D_VERSION"
 }
 
 ensure_private_material() {
@@ -185,9 +216,174 @@ generate_tls() {
 }
 
 render_k3d_config() {
-  local output=$1
+  local output=$1 host_cpu host_memory_bytes host_memory_mib
+  local site_system_cpu witness_system_cpu site_system_memory witness_system_memory
   [[ $media_root != *'|'* ]]
-  sed "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" "$manifest_root/k3d.yaml" >"$output"
+  host_cpu=$(docker info --format '{{.NCPU}}')
+  host_memory_bytes=$(docker_memory_bytes)
+  host_memory_mib=$((host_memory_bytes / 1024 / 1024))
+  site_system_cpu="$(((host_cpu - 4) * 1000 + 750))m"
+  witness_system_cpu="$(((host_cpu - 2) * 1000 + 750))m"
+  site_system_memory="$((host_memory_mib - 8192 + 1024))Mi"
+  witness_system_memory="$((host_memory_mib - 4096 + 1024))Mi"
+  sed \
+    -e "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" \
+    -e "s|EASYAI_ACCEPTANCE_K3S_IMAGE|${K3S_IMAGE%%@*}|g" \
+    -e "s|EASYAI_ACCEPTANCE_CLUSTER_SUBNET|$cluster_subnet|g" \
+    -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_CPU|$site_system_cpu|g" \
+    -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_CPU|$witness_system_cpu|g" \
+    -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_MEMORY|$site_system_memory|g" \
+    -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_MEMORY|$witness_system_memory|g" \
+    "$manifest_root/k3d.yaml" >"$output"
+}
+
+cluster_volumes() {
+  printf '%s\n' \
+    easyai-acceptance-local-server-0-etcd \
+    easyai-acceptance-local-server-1-etcd \
+    easyai-acceptance-local-server-2-etcd \
+    easyai-acceptance-local-server-0-storage \
+    easyai-acceptance-local-server-1-storage
+}
+
+create_cluster_volumes() {
+  local volume
+  while read -r volume; do
+    if docker volume inspect "$volume" >/dev/null 2>&1; then
+      echo "stale local acceptance volume exists: $volume; run explicit down --confirm" >&2
+      return 1
+    fi
+  done < <(cluster_volumes)
+  while read -r volume; do
+    docker volume create --label easyai.io/environment=local-acceptance "$volume" >/dev/null
+  done < <(cluster_volumes)
+}
+
+remove_cluster_volumes() {
+  local volume
+  while read -r volume; do
+    docker volume rm "$volume" >/dev/null 2>&1 || true
+  done < <(cluster_volumes)
+}
+
+reconcile_cluster_network() {
+  docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
+  local actual_subnet attached_containers
+  actual_subnet=$(docker network inspect "$cluster_network" --format '{{(index .IPAM.Config 0).Subnet}}')
+  attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
+  if [[ $actual_subnet == "$cluster_subnet" ]]; then
+    return 0
+  fi
+  if ((attached_containers != 0)); then
+    echo "stale local acceptance network has active containers: network=$cluster_network expected_subnet=$cluster_subnet actual_subnet=$actual_subnet" >&2
+    return 1
+  fi
+  docker network rm "$cluster_network" >/dev/null
+}
+
+remove_cluster_network() {
+  docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
+  local attached_containers
+  attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
+  if ((attached_containers != 0)); then
+    echo "refusing to remove local acceptance network with active containers: network=$cluster_network containers=$attached_containers" >&2
+    return 1
+  fi
+  docker network rm "$cluster_network" >/dev/null
+}
+
+pull_dependency_images() {
+  local image
+  for image in "$K3S_IMAGE" "$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
+    "$K3S_COREDNS_IMAGE" "$K3S_METRICS_SERVER_IMAGE" "$K3S_LOCAL_PATH_IMAGE"; do
+    pull_dependency_image "$image"
+  done
+}
+
+docker_pull_with_deadline() {
+  local image=$1 pid deadline
+  docker pull "$image" >/dev/null 2>&1 &
+  pid=$!
+  deadline=$((SECONDS + 10))
+  while kill -0 "$pid" >/dev/null 2>&1; do
+    if ((SECONDS >= deadline)); then
+      kill "$pid" >/dev/null 2>&1 || true
+      wait "$pid" >/dev/null 2>&1 || true
+      return 1
+    fi
+    sleep 1
+  done
+  wait "$pid"
+}
+
+docker_start_with_deadline() {
+  local container=$1 pid deadline
+  docker start "$container" >/dev/null 2>&1 &
+  pid=$!
+  deadline=$((SECONDS + 10))
+  while kill -0 "$pid" >/dev/null 2>&1; do
+    if ((SECONDS >= deadline)); then
+      kill "$pid" >/dev/null 2>&1 || true
+      wait "$pid" >/dev/null 2>&1 || true
+      return 1
+    fi
+    sleep 1
+  done
+  wait "$pid"
+}
+
+verify_docker_container_start() {
+  local image=$1 container="easyai-acceptance-start-probe-$$" exit_code
+  docker create --name "$container" --network none --entrypoint /bin/true "$image" >/dev/null
+  if ! docker_start_with_deadline "$container"; then
+    docker rm -f "$container" >/dev/null 2>&1 || true
+    echo 'gate=local_docker_container_start_unavailable Docker cannot start a minimal acceptance container within 10 seconds' >&2
+    return 1
+  fi
+  exit_code=$(docker inspect "$container" --format '{{.State.ExitCode}}')
+  docker rm -f "$container" >/dev/null
+  [[ $exit_code == 0 ]] || {
+    echo "gate=local_docker_container_start_failed minimal acceptance container exit_code=$exit_code" >&2
+    return 1
+  }
+}
+
+pull_dependency_image() {
+  local image=$1 name_tag repository digest cache_tag native_arch archive source_tag
+  name_tag=${image%%@*}
+  repository=${name_tag%:*}
+  digest=${image##*@sha256:}
+  [[ $digest =~ ^[0-9a-f]{64}$ ]]
+  cache_tag="$repository:easyai-lock-${digest:0:20}"
+  if docker image inspect "$image" >/dev/null 2>&1; then
+    docker tag "$image" "$name_tag"
+    docker tag "$image" "$cache_tag"
+    return
+  fi
+  if docker image inspect "$cache_tag" >/dev/null 2>&1; then
+    docker tag "$cache_tag" "$name_tag"
+    return
+  fi
+  if docker_pull_with_deadline "$image"; then
+    docker tag "$image" "$name_tag"
+    docker tag "$name_tag" "$cache_tag"
+    return
+  fi
+  native_arch=$(docker info --format '{{.Architecture}}')
+  case $native_arch in
+    aarch64|arm64) native_arch=arm64 ;;
+    x86_64) native_arch=amd64 ;;
+    *) return 1 ;;
+  esac
+  archive="$state_root/dependency-image.tar"
+  [[ ! -L $archive ]]
+  umask 077
+  "$tools_root/crane" pull --platform "linux/$native_arch" "$image" "$archive"
+  docker load -i "$archive" >/dev/null
+  : >"$archive"
+  source_tag="$repository:i-was-a-digest"
+  docker tag "$source_tag" "$name_tag"
+  docker tag "$source_tag" "$cache_tag"
 }
 
 wait_rollout() {
@@ -430,6 +626,15 @@ render_and_apply_application() {
   done
   kubectl --context "$context" -n "$namespace" scale \
     deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null
+  kubectl --context "$context" -n "$namespace" set resources deployment/easyai-api-ningbo \
+    deployment/easyai-api-hongkong --containers=api \
+    --requests=cpu=250m,memory=512Mi --limits=cpu=750m,memory=2Gi >/dev/null
+  kubectl --context "$context" -n "$namespace" set resources deployment/easyai-worker-ningbo \
+    deployment/easyai-worker-hongkong --containers=worker \
+    --requests=cpu=750m,memory=1536Mi --limits=cpu=1500m,memory=2Gi >/dev/null
+  kubectl --context "$context" -n "$namespace" set env deployment/easyai-worker-ningbo \
+    deployment/easyai-worker-hongkong \
+    AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
 }
 
 bootstrap_runtime() {
@@ -439,7 +644,7 @@ bootstrap_runtime() {
   local snapshot_hash release_sha runtime_file
   snapshot_hash=$(jq -r '.snapshotSha256' "$snapshot")
   release_sha=$(git -C "$repository_root" rev-parse HEAD)
-  runtime_file="$state_root/runtime-$snapshot_hash.json"
+  runtime_file="$state_root/runtime-$snapshot_hash-$(date -u '+%Y%m%dT%H%M%SZ')-$$.json"
   kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
     --ignore-not-found --wait=true >/dev/null
   cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
@@ -561,6 +766,8 @@ up_cluster() {
   api_image="easyai-acceptance-api:$release_sha"
   web_image="easyai-acceptance-web:$release_sha"
   netem_image="easyai-acceptance-netem:$release_sha"
+  pull_dependency_images
+  verify_docker_container_start "${K3S_IMAGE%%@*}"
   docker buildx build --load --platform "linux/$native_arch" --target api \
     -t "$api_image" "$repository_root"
   docker buildx build --load --platform "linux/$native_arch" --target web \
@@ -569,7 +776,10 @@ up_cluster() {
     -t "$netem_image" "$repository_root"
   api_digest=$(docker image inspect "$api_image" --format '{{.Id}}')
   [[ $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
+  verify_docker_container_start "$netem_image"
 
+  reconcile_cluster_network
+  create_cluster_volumes
   config="$state_root/k3d.rendered.yaml"
   render_k3d_config "$config"
   EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config"
@@ -577,7 +787,14 @@ up_cluster() {
   docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null
   docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null
   docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null
-  "$k3d" image import -c "$cluster_name" "$api_image" "$web_image" "$netem_image"
+  verify_local_node_capacity "$context" || {
+    echo 'local K3s node allocatable capacity does not match 4/8, 4/8, 2/4 resource envelope' >&2
+    exit 1
+  }
+  "$k3d" image import -c "$cluster_name" \
+    "$api_image" "$web_image" "$netem_image" \
+    "${CNPG_CONTROLLER_IMAGE%%@*}" "${CNPG_POSTGRES_IMAGE%%@*}" \
+    "${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"
 
   local cnpg_manifest="$state_root/cnpg.yaml"
   curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest"
@@ -590,7 +807,8 @@ up_cluster() {
   kubectl --context "$context" -n cnpg-system rollout status \
     deployment/cnpg-controller-manager --timeout=10m
   apply_runtime_secrets
-  kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/database.yaml" >/dev/null
+  sed "s|imageName: .*|imageName: ${CNPG_POSTGRES_IMAGE%%@*}|" "$manifest_root/database.yaml" |
+    kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
   kubectl --context "$context" -n "$namespace" wait --for=condition=Ready \
     cluster/easyai-postgres --timeout=15m >/dev/null
   run_migrations "$api_image"
@@ -619,9 +837,58 @@ up_cluster() {
   bootstrap_runtime "$api_image" "$api_digest" "$snapshot"
   wait_internal_gateway
   "$script_dir/network-fault.sh" baseline
+  sleep 1
+  record_local_control_plane_identity "$context" "$cluster_name" "$identity_file"
   echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt"
 }
 
+new_run() {
+  load_lock
+  preflight
+  [[ -z $(git -C "$repository_root" status --short) ]] || {
+    echo 'local acceptance requires a clean source tree for a fresh Run ID' >&2
+    exit 1
+  }
+  if ! private_file "$state_root/current-runtime-path" || ! private_file "$state_root/snapshot.json"; then
+    echo 'local acceptance runtime or snapshot is unavailable' >&2
+    exit 1
+  fi
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
+    echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild the cluster' >&2
+    exit 1
+  }
+  verify_local_node_capacity "$context" || {
+    echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
+    exit 1
+  }
+  verify_local_apiserver_ready "$context" || {
+    echo 'gate=local_control_plane_unavailable local Kubernetes API is not ready' >&2
+    exit 1
+  }
+  local previous_runtime api_image api_digest previous_release current_release
+  previous_runtime=$(<"$state_root/current-runtime-path")
+  private_file "$previous_runtime" || {
+    echo 'local acceptance current runtime is unavailable' >&2
+    exit 1
+  }
+  previous_release=$(jq -r '.releaseSha' "$previous_runtime")
+  current_release=$(git -C "$repository_root" rev-parse HEAD)
+  [[ $previous_release == "$current_release" ]] || {
+    echo 'local acceptance source changed after cluster creation; rebuild the cluster' >&2
+    exit 1
+  }
+  api_digest=$(jq -r '.apiImageDigest' "$previous_runtime")
+  api_image=$(kubectl --context "$context" -n "$namespace" get deployment easyai-api-ningbo \
+    -o 'jsonpath={.spec.template.spec.containers[?(@.name=="api")].image}')
+  [[ -n $api_image && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
+  bootstrap_runtime "$api_image" "$api_digest" "$state_root/snapshot.json"
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
+    echo 'gate=local_control_plane_restarted local K3s server changed while creating a fresh run' >&2
+    exit 1
+  }
+  echo "local_acceptance_new_run=PASS runtime_path=$state_root/current-runtime-path"
+}
+
 status_cluster() {
   local k3d
   k3d=$(k3d_binary) || {
@@ -634,6 +901,14 @@ status_cluster() {
   }
   kubectl --context "$context" get nodes -o wide
   kubectl --context "$context" -n "$namespace" get cluster,pod,deployment
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
+    echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild required' >&2
+    exit 1
+  }
+  verify_local_node_capacity "$context" || {
+    echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
+    exit 1
+  }
   echo "local_acceptance_status=PASS context=$context"
 }
 
@@ -644,7 +919,11 @@ down_cluster() {
   }
   local k3d
   k3d=$(k3d_binary)
-  "$k3d" cluster delete "$cluster_name"
+  if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then
+    "$k3d" cluster delete "$cluster_name"
+  fi
+  remove_cluster_volumes
+  remove_cluster_network
   echo "local_acceptance_down=PASS preserved_private_root=$private_root"
 }
 
@@ -661,6 +940,10 @@ case ${1:-} in
     shift
     up_cluster "$@"
     ;;
+  new-run)
+    [[ $# -eq 1 ]] || { usage >&2; exit 64; }
+    new_run
+    ;;
   status)
     [[ $# -eq 1 ]] || { usage >&2; exit 64; }
     status_cluster

scripts/acceptance/local-control-plane.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+# Shared read-only guards for the local three-server K3s acceptance cluster.
+# The caller owns shell options and failure reporting.
+
+capture_local_control_plane_identity() {
+  local context=$1 cluster_name=$2 containers nodes
+  containers=$(docker inspect \
+    "k3d-${cluster_name}-server-0" \
+    "k3d-${cluster_name}-server-1" \
+    "k3d-${cluster_name}-server-2" |
+    jq -c --arg network "k3d-$cluster_name" '[.[] | {
+      name:(.Name | ltrimstr("/")),
+      id:.Id,
+      restartCount:.RestartCount,
+      startedAt:.State.StartedAt,
+      ip:.NetworkSettings.Networks[$network].IPAddress
+    }] | sort_by(.name)') || return 1
+  nodes=$(kubectl --context "$context" get nodes -o json |
+    jq -c '[.items[] | {
+      name:.metadata.name,
+      uid:.metadata.uid,
+      site:.metadata.labels["easyai.io/site"],
+      cpu:.status.allocatable.cpu,
+      memory:.status.allocatable.memory
+    }] | sort_by(.name)') || return 1
+  jq -cn --argjson containers "$containers" --argjson nodes "$nodes" \
+    '{containers:$containers,nodes:$nodes}'
+}
+
+record_local_control_plane_identity() {
+  local context=$1 cluster_name=$2 output=$3 current temporary
+  current=$(capture_local_control_plane_identity "$context" "$cluster_name") || return 1
+  temporary="${output}.tmp.$$"
+  jq -cn \
+    --arg recordedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+    --argjson current "$current" \
+    '{schemaVersion:"local-control-plane-identity/v1",recordedAt:$recordedAt} + $current' \
+    >"$temporary"
+  chmod 0600 "$temporary"
+  mv "$temporary" "$output"
+}
+
+verify_local_control_plane_identity() {
+  local context=$1 cluster_name=$2 baseline=$3 expected current
+  [[ -f $baseline && ! -L $baseline ]] || return 1
+  expected=$(jq -cS '{containers,nodes}' "$baseline") || return 1
+  current=$(capture_local_control_plane_identity "$context" "$cluster_name" |
+    jq -cS '{containers,nodes}') || return 1
+  [[ $current == "$expected" ]]
+}
+
+verify_local_node_capacity() {
+  local context=$1
+  kubectl --context "$context" get nodes -o json | jq -e '
+    def cpu_m:
+      if endswith("m") then rtrimstr("m") | tonumber else tonumber * 1000 end;
+    def memory_mi:
+      if endswith("Ki") then rtrimstr("Ki") | tonumber / 1024
+      elif endswith("Mi") then rtrimstr("Mi") | tonumber
+      elif endswith("Gi") then rtrimstr("Gi") | tonumber * 1024
+      else 0 end;
+    ([.items[] | select(.metadata.labels["easyai.io/workload"] == "true") |
+      select((.status.allocatable.cpu | cpu_m) == 3000 and
+             (.status.allocatable.memory | memory_mi) == 6656)] | length) == 2 and
+    ([.items[] | select(.metadata.labels["easyai.io/site"] == "los-angeles") |
+      select((.status.allocatable.cpu | cpu_m) == 1000 and
+             (.status.allocatable.memory | memory_mi) == 2560)] | length) == 1 and
+    ([.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length) == 3 and
+    ([.items[] | select(any(.status.conditions[]; .type == "MemoryPressure" and .status == "True"))] | length) == 0
+  ' >/dev/null
+}
+
+verify_local_apiserver_ready() {
+  local context=$1
+  kubectl --context "$context" get --raw='/readyz?verbose' 2>/dev/null |
+    grep -Fq 'readyz check passed'
+}
+
+verify_local_etcd_runtime_logs() {
+  local cluster_name=$1 baseline=$2 recorded_at name
+  [[ -f $baseline && ! -L $baseline ]] || return 1
+  recorded_at=$(jq -r '.recordedAt' "$baseline") || return 1
+  [[ $recorded_at == ????-??-??T*Z ]]
+  for name in \
+    "k3d-${cluster_name}-server-0" \
+    "k3d-${cluster_name}-server-1" \
+    "k3d-${cluster_name}-server-2"; do
+    if docker logs --since "$recorded_at" "$name" 2>&1 |
+      grep -Eqi 'heartbeat interval is too long|apply request took too long|slow fdatasync|failed to check local etcd status|etcdserver: request timed out|waiting for ReadIndex response took too long'; then
+      return 1
+    fi
+  done
+}

scripts/acceptance/report.mjs

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { createHash } from 'node:crypto';
-import { lstat, mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
+import { lstat, mkdir, readFile, readdir, rename, unlink, writeFile } from 'node:fs/promises';
 import { dirname, resolve } from 'node:path';
 
 const shaPattern = /^[0-9a-f]{40}$/;
@@ -26,6 +26,29 @@ async function regularJSON(path) {
   return JSON.parse(await readFile(absolute, 'utf8'));
 }
 
+async function writeNewJSON(path, value) {
+  const absolute = resolve(path);
+  await mkdir(dirname(absolute), { recursive: true, mode: 0o700 });
+  await writeFile(absolute, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx' });
+}
+
+async function replaceJSONAtomically(input, output, value) {
+  const inputPath = resolve(input);
+  const outputPath = resolve(output);
+  if (inputPath !== outputPath) {
+    await writeNewJSON(outputPath, value);
+    return;
+  }
+  const temporary = `${outputPath}.tmp-${process.pid}-${Date.now()}`;
+  try {
+    await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx' });
+    await rename(temporary, outputPath);
+  } catch (error) {
+    await unlink(temporary).catch(() => {});
+    throw error;
+  }
+}
+
 function assertSecretSafe(value, path = '$') {
   if (Array.isArray(value)) {
     value.forEach((item, index) => assertSecretSafe(item, `${path}[${index}]`));
@@ -85,6 +108,8 @@ async function loadReports(directory) {
         profile: report.profile,
         passed: report.passed,
         phases: report.phases,
+        failure: report.failure,
+        failureOperation: report.failureOperation,
         startedAt: report.startedAt,
         finishedAt: report.finishedAt
       });
@@ -150,8 +175,7 @@ async function buildLocal(values) {
     secretSafe: true
   };
   validate(report);
-  await mkdir(dirname(resolve(values.output)), { recursive: true, mode: 0o700 });
-  await writeFile(resolve(values.output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
+  await writeNewJSON(values.output, report);
   process.stdout.write(`acceptance_report_local=PASS sha256=${createHash('sha256').update(JSON.stringify(report)).digest('hex')}\n`);
 }
 
@@ -172,6 +196,10 @@ async function buildLocalPartial(values) {
     throw new Error('partial local report inputs have incompatible CAS identity');
   }
   const completedLoadsPassed = loads.every((item) => item.passed === true);
+  const failureGate = values['failure-gate'] ?? 'local_execution_incomplete';
+  if (!/^[a-z0-9_]+$/.test(failureGate)) {
+    throw new Error('--failure-gate must be a stable lowercase gate ID');
+  }
   const report = {
     schemaVersion: 'acceptance-report/v1',
     runId: runtime.runId,
@@ -202,7 +230,7 @@ async function buildLocalPartial(values) {
     } : null,
     gates: [
       {
-        id: 'local_execution_complete',
+        id: failureGate,
         passed: false,
         detail: `stopped during ${values['failure-phase']}`
       },
@@ -221,8 +249,7 @@ async function buildLocalPartial(values) {
     secretSafe: true
   };
   validate(report);
-  await mkdir(dirname(resolve(values.output)), { recursive: true, mode: 0o700 });
-  await writeFile(resolve(values.output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
+  await writeNewJSON(values.output, report);
   process.stdout.write('acceptance_report_local_partial=PASS\n');
 }
 
@@ -259,8 +286,7 @@ async function mergeProduction(values) {
   };
   local.createdAt = new Date().toISOString();
   validate(local);
-  await mkdir(dirname(resolve(values.output)), { recursive: true, mode: 0o700 });
-  await writeFile(resolve(values.output), `${JSON.stringify(local, null, 2)}\n`, { mode: 0o600 });
+  await writeNewJSON(values.output, local);
   process.stdout.write(`acceptance_report_production=PASS promotion_ready=${local.promotion.ready}\n`);
 }
 
@@ -285,7 +311,7 @@ if (command === 'validate') {
   }
   report.promotion.trafficMode = 'live';
   report.promotion.promotedAt = new Date().toISOString();
-  await writeFile(resolve(output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
+  await replaceJSONAtomically(input, output, report);
   process.stdout.write('acceptance_report_promoted=PASS\n');
 } else if (command === 'attach-monitor') {
   const input = values.input ?? '';
@@ -309,7 +335,7 @@ if (command === 'validate') {
     report.promotion.ready = false;
     report.promotion.trafficMode = 'validation';
   }
-  await writeFile(resolve(output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
+  await replaceJSONAtomically(input, output, report);
   process.stdout.write(`acceptance_report_monitor=PASS monitor_passed=${monitor.passed === true}\n`);
 } else {
   throw new Error('usage: report.mjs {validate|build-local|build-local-partial|merge-production|mark-promoted|attach-monitor} [options]');

scripts/acceptance/run-local-acceptance.sh

@@ -7,13 +7,18 @@ private_root="$repository_root/.local-secrets/acceptance"
 state_root="$private_root/state"
 context=k3d-easyai-acceptance-local
 namespace=easyai
+cluster_name=easyai-acceptance-local
 runtime_path_file="$state_root/current-runtime-path"
 snapshot="$state_root/snapshot.json"
 load_binary="$state_root/easyai-ai-gateway-acceptance-load"
+identity_file="$state_root/control-plane-identity.json"
 active_load_pid=
 netem_active=false
 current_phase=initialization
 stable_profile=
+failure_gate_id=local_execution_incomplete
+# shellcheck source=scripts/acceptance/local-control-plane.sh
+source "$script_dir/local-control-plane.sh"
 
 usage() {
   cat <<'EOF'
@@ -51,12 +56,16 @@ cleanup() {
   fi
   if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then
     restore_profile_best_effort "${stable_profile:-P24}"
+    if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then
+      failure_gate_id=$(jq -r '.id' "$failure_gate_file" 2>/dev/null || printf '%s' "$failure_gate_id")
+    fi
     local partial_args=(
       "$script_dir/report.mjs" build-local-partial
       --runtime "$runtime" \
       --snapshot "$snapshot" \
       --reports "$report_root" \
       --failure-phase "$current_phase" \
+      --failure-gate "$failure_gate_id" \
       --output "$report_root/acceptance-report.partial.json"
     )
     if [[ -n $stable_profile ]]; then
@@ -89,6 +98,7 @@ require_local_cluster() {
   release_sha=$(jq -r '.releaseSha' "$runtime")
   report_root="$repository_root/dist/acceptance/local/$run_id"
   install -d -m 0700 "$report_root"
+  failure_gate_file="$report_root/failure-gate.json"
   (
     cd "$repository_root/apps/api"
     env -u AI_GATEWAY_TEST_DATABASE_URL go build -trimpath \
@@ -96,6 +106,27 @@ require_local_cluster() {
   )
 }
 
+record_failure_gate() {
+  local id=$1 detail=$2 temporary
+  [[ $id =~ ^[a-z0-9_]+$ ]]
+  failure_gate_id=$id
+  [[ -n ${failure_gate_file:-} ]] || return 0
+  if [[ ! -e $failure_gate_file ]]; then
+    temporary="${failure_gate_file}.tmp.$$"
+    jq -n --arg id "$id" --arg detail "$detail" \
+      '{schemaVersion:"acceptance-failure-gate/v1",id:$id,detail:$detail}' >"$temporary"
+    chmod 0600 "$temporary"
+    mv "$temporary" "$failure_gate_file"
+  fi
+}
+
+fail_gate() {
+  local id=$1 detail=$2
+  record_failure_gate "$id" "$detail"
+  echo "gate=$id $detail" >&2
+  return 1
+}
+
 database_query() {
   local sql=$1 primary
   primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
@@ -115,6 +146,12 @@ load_environment() {
 run_load() {
   local profile=$1 report_path=$2
   shift 2
+  local stdout_path="$report_path.stdout" stdout_temporary="${report_path}.stdout.tmp.$$"
+  local load_pid load_status=0 unavailable_samples=0 infrastructure_failure=false
+  [[ ! -e $report_path && ! -e $stdout_path && ! -e $stdout_temporary ]] ||
+    fail_gate acceptance_report_exists "refusing to overwrite an existing load report"
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
+    fail_gate local_control_plane_restarted "local K3s server identity changed before load"
   AI_GATEWAY_ACCEPTANCE_GATEWAYS='https://127.0.0.1:18443,https://127.0.0.1:19443' \
   AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=gateway.easyai.local \
   AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE="$state_root/ca.crt" \
@@ -125,8 +162,49 @@ run_load() {
   AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
   AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
     "$load_binary" -profile "$profile" -report "$report_path" "$@" \
-      >"$report_path.stdout"
-  chmod 0600 "$report_path" "$report_path.stdout"
+      >"$stdout_temporary" &
+  load_pid=$!
+  active_load_pid=$load_pid
+  while kill -0 "$load_pid" >/dev/null 2>&1; do
+    if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
+      record_failure_gate local_control_plane_restarted "local K3s server restarted during load"
+      infrastructure_failure=true
+      kill "$load_pid" >/dev/null 2>&1 || true
+      break
+    fi
+    if verify_local_apiserver_ready "$context"; then
+      unavailable_samples=0
+    else
+      unavailable_samples=$((unavailable_samples + 1))
+      if ((unavailable_samples >= 3)); then
+        record_failure_gate local_control_plane_unavailable "Kubernetes API readiness failed for three consecutive samples"
+        infrastructure_failure=true
+        kill "$load_pid" >/dev/null 2>&1 || true
+        break
+      fi
+    fi
+    sleep 2
+  done
+  if wait "$load_pid"; then
+    load_status=0
+  else
+    load_status=$?
+  fi
+  active_load_pid=
+  if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
+    record_failure_gate local_control_plane_restarted "local K3s server identity changed at load completion"
+    infrastructure_failure=true
+  fi
+  [[ ! -e $stdout_path ]] || fail_gate acceptance_report_exists "load stdout report already exists"
+  mv "$stdout_temporary" "$stdout_path"
+  chmod 0600 "$stdout_path"
+  if [[ -f $report_path && ! -L $report_path ]]; then
+    chmod 0600 "$report_path"
+  fi
+  if [[ $infrastructure_failure == true ]]; then
+    return 1
+  fi
+  ((load_status == 0)) || return "$load_status"
   jq -e '.schemaVersion == "acceptance-load-report/v1" and .secretSafe == true and .passed == true' \
     "$report_path" >/dev/null
 }
@@ -147,15 +225,26 @@ sample_resources() {
 
 verify_hard_gates() {
   local unhealthy ready sync_state connections max_connections queue duplicate_remote duplicate_billing
+  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
+    fail_gate local_control_plane_restarted "local K3s server identity changed"
+  verify_local_apiserver_ready "$context" ||
+    fail_gate local_control_plane_unavailable "Kubernetes API readiness failed"
+  verify_local_node_capacity "$context" ||
+    fail_gate local_node_capacity_mismatch "node allocatable resources drifted from the acceptance envelope"
+  verify_local_etcd_runtime_logs "$cluster_name" "$identity_file" ||
+    fail_gate local_etcd_latency "etcd emitted a severe latency or timeout signal during the run"
   ready=$(kubectl --context "$context" get nodes -o json |
     jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length')
-  [[ $ready == 3 ]]
+  [[ $ready == 3 ]] || fail_gate local_nodes_not_ready "not all three local K3s nodes are Ready"
   [[ $(kubectl --context "$context" get nodes -o json |
-    jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]]
+    jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] ||
+    fail_gate local_node_memory_pressure "a local K3s node reports MemoryPressure"
   [[ $(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
-    -o 'jsonpath={.status.readyInstances}') == 2 ]]
+    -o 'jsonpath={.status.readyInstances}') == 2 ]] ||
+    fail_gate local_postgres_not_ready "local PostgreSQL is not 2/2 Ready"
   sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
-  [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]]
+  [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] ||
+    fail_gate local_postgres_not_synchronous "local PostgreSQL lost its synchronous replica"
   unhealthy=$(kubectl --context "$context" -n "$namespace" get pods -o json |
     jq '[.items[]
       | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
@@ -163,11 +252,11 @@ verify_hard_gates() {
       | .status.containerStatuses[]?
       | select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or
                .state.terminated.reason == "OOMKilled")] | length')
-  [[ $unhealthy == 0 ]]
+  [[ $unhealthy == 0 ]] || fail_gate gateway_pod_restarted "an API or Worker container restarted or was OOMKilled"
   while read -r _node _cpu _cpu_percent _memory memory_percent; do
     memory_percent=${memory_percent%\%}
     [[ $memory_percent =~ ^[0-9]+$ ]]
-    (( memory_percent < 80 ))
+    (( memory_percent < 80 )) || fail_gate local_node_memory_target "local node memory reached the 80 percent target"
   done < <(kubectl --context "$context" top nodes --no-headers)
   while read -r _pod _cpu memory; do
     case $memory in
@@ -176,20 +265,23 @@ verify_hard_gates() {
       *Ki) memory=$(awk -v value="${memory%Ki}" 'BEGIN {printf "%.0f", value/1024}') ;;
       *) return 1 ;;
     esac
-    (( memory < 1536 ))
+    (( memory < 1536 )) || fail_gate gateway_pod_memory_limit "an API or Worker Pod reached 1.5 GiB RSS"
   done < <(kubectl --context "$context" -n "$namespace" top pods \
     -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers)
   connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
   max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
-  (( connections < 150 && connections * 4 < max_connections * 3 ))
+  (( connections < 150 && connections * 4 < max_connections * 3 )) ||
+    fail_gate postgres_connection_budget "PostgreSQL client sessions exceeded the acceptance budget"
   queue=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
-  [[ $queue == 0:0 ]]
+  [[ $queue == 0:0 ]] || fail_gate acceptance_queue_not_drained "acceptance queue did not return to zero"
   duplicate_remote=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) d;")
   duplicate_billing=$(database_query "SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) d;")
-  [[ $duplicate_remote == 0 && $duplicate_billing == 0 ]]
+  [[ $duplicate_remote == 0 ]] || fail_gate duplicate_upstream_submission "duplicate remote task IDs were detected"
+  [[ $duplicate_billing == 0 ]] || fail_gate duplicate_billing "duplicate billing transactions were detected"
   kubectl --context "$context" -n "$namespace" exec deployment/easyai-acceptance-callback-collector -- \
     wget -qO- http://127.0.0.1:8091/report |
-    jq -e '.duplicates == 0 and .invalid == 0' >/dev/null
+    jq -e '.duplicates == 0 and .invalid == 0' >/dev/null ||
+    fail_gate callback_validation_failed "callback collector detected duplicate or invalid callbacks"
 }
 
 apply_profile() {
@@ -218,7 +310,8 @@ apply_profile() {
       AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
       AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
       AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
-      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null
+      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
+      AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
     kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
   done
   for deployment in easyai-api-ningbo easyai-api-hongkong easyai-capacity-controller; do
@@ -254,7 +347,8 @@ restore_profile_best_effort() {
       AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
       AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
       AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
-      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null 2>&1 || true
+      AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
+      AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null 2>&1 || true
   done
 }
 
@@ -514,23 +608,29 @@ full() {
 
 command=${1:-}
 shift || true
-require_local_cluster
-load_environment
 case $command in
   quick)
     [[ $# -eq 0 ]] || { usage >&2; exit 64; }
-    quick
     ;;
-  artifact-smoke)
+  artifact-smoke|full)
     [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
-    artifact_smoke "$2"
-    ;;
-  full)
-    [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
-    full "$2"
     ;;
   *)
     usage >&2
     exit 64
     ;;
 esac
+"$script_dir/local-cluster.sh" new-run
+require_local_cluster
+load_environment
+case $command in
+  quick)
+    quick
+    ;;
+  artifact-smoke)
+    artifact_smoke "$2"
+    ;;
+  full)
+    full "$2"
+    ;;
+esac