fix(release): 修正 Worker 滚动与失败传播

单站点 Worker 启用硬反亲和后，maxSurge=1 会让新 Pod 无法调度并造成滚动超时。将 Worker 改为先下线旧 Pod 再创建新 Pod，同时保持 API 的零中断滚动策略。\n\n发布辅助脚本为关键 kubectl 操作补充显式失败返回，避免函数位于条件表达式时 Bash 忽略 errexit，进而把失败发布误报为成功。新增回归测试模拟 rollout status 失败且旧 Pod 仍 Ready 的场景。\n\n验证：bash -n、ShellCheck、cluster-release-helper-test、manual-release-test、生产 API Server dry-run。
2026-08-01 13:43:33 +08:00
parent 80801b7bbb
commit 4d84210344
4 changed files with 121 additions and 22 deletions
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2034 # Variables are consumed by the dynamically sourced function.
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+
+helper=$root/deploy/kubernetes/easyai-ai-gateway-cluster-release
+application=$root/deploy/kubernetes/production/application.yaml
+
+awk '
+  /^rollout_worker_site\(\)/ { capture = 1 }
+  capture && /^rollout_capacity_controller\(\)/ { exit }
+  capture { print }
+' "$helper" >"$tmp/rollout-worker-site.sh"
+
+# shellcheck source=/dev/null
+source "$tmp/rollout-worker-site.sh"
+
+cat >"$tmp/kubectl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ${1:-} == rollout && ${2:-} == status ]]; then
+  exit 42
+fi
+if [[ ${1:-} == get && $* == *jsonpath* ]]; then
+  printf worker
+  exit 0
+fi
+if [[ ${1:-} == get && $* == *'-o json'* ]]; then
+  printf '{"status":{"readyReplicas":1}}\n'
+  exit 0
+fi
+exit 0
+EOF
+chmod +x "$tmp/kubectl"
+
+kubectl=("$tmp/kubectl")
+NAMESPACE=easyai
+ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
+AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
+AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
+AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
+AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
+AI_GATEWAY_DATABASE_MAX_CONNS=32
+AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4
+AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=4
+AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=2
+AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
+AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
+AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
+AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
+AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
+AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
+AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
+AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
+AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
+AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
+AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
+
+# activate_manifest invokes this function inside an && condition. A failed
+# rollout must still propagate instead of falling through to a stale Ready Pod.
+if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then
+  echo 'failed Worker rollout was reported as successful' >&2
+  exit 1
+fi
+
+deployment_strategy() {
+  local deployment=$1
+  awk -v deployment="$deployment" '
+    $0 == "  name: " deployment { capture = 1 }
+    capture && /^---$/ { exit }
+    capture { print }
+  ' "$application"
+}
+
+for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
+  strategy=$(deployment_strategy "$deployment")
+  grep -Fq '      maxSurge: 0' <<<"$strategy"
+  grep -Fq '      maxUnavailable: 1' <<<"$strategy"
+done
+
+for deployment in easyai-api-ningbo easyai-api-hongkong; do
+  strategy=$(deployment_strategy "$deployment")
+  grep -Fq '      maxSurge: 1' <<<"$strategy"
+  grep -Fq '      maxUnavailable: 0' <<<"$strategy"
+done
+
+echo 'cluster_release_helper_tests=PASS'
@@ -306,4 +306,6 @@ if grep -Fq 'archive_timeout: 60s' "$root/deploy/kubernetes/production/database.
  exit 1
 fi

+"$root/tests/release/cluster-release-helper-test.sh" >/dev/null
+
 echo 'manual_release_tests=PASS'