From 4d84210344f04eeabed47d1b0e13f9252eb0eb70 Mon Sep 17 00:00:00 2001
From: wangbo <wangbolhr@163.com>
Date: Sat, 1 Aug 2026 13:43:33 +0800
Subject: [PATCH] =?UTF-8?q?fix(release):=20=E4=BF=AE=E6=AD=A3=20Worker=20?=
 =?UTF-8?q?=E6=BB=9A=E5=8A=A8=E4=B8=8E=E5=A4=B1=E8=B4=A5=E4=BC=A0=E6=92=AD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

单站点 Worker 启用硬反亲和后，maxSurge=1 会让新 Pod 无法调度并造成滚动超时。将 Worker 改为先下线旧 Pod 再创建新 Pod，同时保持 API 的零中断滚动策略。\n\n发布辅助脚本为关键 kubectl 操作补充显式失败返回，避免函数位于条件表达式时 Bash 忽略 errexit，进而把失败发布误报为成功。新增回归测试模拟 rollout status 失败且旧 Pod 仍 Ready 的场景。\n\n验证：bash -n、ShellCheck、cluster-release-helper-test、manual-release-test、生产 API Server dry-run。
---
 .../easyai-ai-gateway-cluster-release         | 41 +++++----
 deploy/kubernetes/production/application.yaml |  8 +-
 tests/release/cluster-release-helper-test.sh  | 92 +++++++++++++++++++
 tests/release/manual-release-test.sh          |  2 +
 4 files changed, 121 insertions(+), 22 deletions(-)
 create mode 100755 tests/release/cluster-release-helper-test.sh

diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release
index a4762ff..302d494 100755
--- a/deploy/kubernetes/easyai-ai-gateway-cluster-release
+++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release
@@ -364,23 +364,24 @@ rollout_worker_site() {
     "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
     "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
     "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
-    "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA"
+    "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
   "${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
     --containers=worker \
     --requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
-    --limits='cpu=2,memory=2Gi'
+    --limits='cpu=2,memory=2Gi' || return 1
   "${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
-    --replicas="$replicas"
+    --replicas="$replicas" || return 1
   if [[ -n $api_image ]]; then
     "${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
-      "worker=$api_image"
+      "worker=$api_image" || return 1
   fi
-  "${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
+  "${kubectl[@]}" rollout status "deployment/easyai-worker-$site" \
+    -n "$NAMESPACE" --timeout=300s || return 1
   [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-    -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
+    -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]] || return 1
   ready_replicas=$("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-    -o json | jq -r '.status.readyReplicas // 0')
-  [[ $ready_replicas == "$replicas" ]]
+    -o json | jq -r '.status.readyReplicas // 0') || return 1
+  [[ $ready_replicas == "$replicas" ]] || return 1
 }
 
 rollout_capacity_controller() {
@@ -405,13 +406,15 @@ rollout_capacity_controller() {
     "AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
     "AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
     "AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
-    "AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
+    "AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" || return 1
   if [[ -n $api_image ]]; then
     "${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
-      "capacity-controller=$api_image"
+      "capacity-controller=$api_image" || return 1
   fi
-  "${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=2
-  "${kubectl[@]}" rollout status deployment/easyai-capacity-controller -n "$NAMESPACE" --timeout=300s
+  "${kubectl[@]}" scale deployment/easyai-capacity-controller \
+    -n "$NAMESPACE" --replicas=2 || return 1
+  "${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
+    -n "$NAMESPACE" --timeout=300s || return 1
 }
 
 run_capacity_controller_preflight() {
@@ -575,17 +578,19 @@ rollout_site() {
       "AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
       "AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
       "AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
-      "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
+      "AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" || return 1
     "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
-      "api=$api_image"
-    "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
+      "api=$api_image" || return 1
+    "${kubectl[@]}" rollout status "deployment/easyai-api-$site" \
+      -n "$NAMESPACE" --timeout=300s || return 1
   fi
   if [[ $web_changed == true ]]; then
     "${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
-      "web=$web_image"
-    "${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
+      "web=$web_image" || return 1
+    "${kubectl[@]}" rollout status "deployment/easyai-web-$site" \
+      -n "$NAMESPACE" --timeout=300s || return 1
   fi
-  verify_site "$site"
+  verify_site "$site" || return 1
 }
 
 run_backup() {
diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml
index 49a1261..e479f0f 100644
--- a/deploy/kubernetes/production/application.yaml
+++ b/deploy/kubernetes/production/application.yaml
@@ -438,8 +438,8 @@ spec:
   strategy:
     type: RollingUpdate
     rollingUpdate:
-      maxSurge: 1
-      maxUnavailable: 0
+      maxSurge: 0
+      maxUnavailable: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: easyai-worker
@@ -611,8 +611,8 @@ spec:
   strategy:
     type: RollingUpdate
     rollingUpdate:
-      maxSurge: 1
-      maxUnavailable: 0
+      maxSurge: 0
+      maxUnavailable: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: easyai-worker
diff --git a/tests/release/cluster-release-helper-test.sh b/tests/release/cluster-release-helper-test.sh
new file mode 100755
index 0000000..6b81fb6
--- /dev/null
+++ b/tests/release/cluster-release-helper-test.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2034 # Variables are consumed by the dynamically sourced function.
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+
+helper=$root/deploy/kubernetes/easyai-ai-gateway-cluster-release
+application=$root/deploy/kubernetes/production/application.yaml
+
+awk '
+  /^rollout_worker_site\(\)/ { capture = 1 }
+  capture && /^rollout_capacity_controller\(\)/ { exit }
+  capture { print }
+' "$helper" >"$tmp/rollout-worker-site.sh"
+
+# shellcheck source=/dev/null
+source "$tmp/rollout-worker-site.sh"
+
+cat >"$tmp/kubectl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ${1:-} == rollout && ${2:-} == status ]]; then
+  exit 42
+fi
+if [[ ${1:-} == get && $* == *jsonpath* ]]; then
+  printf worker
+  exit 0
+fi
+if [[ ${1:-} == get && $* == *'-o json'* ]]; then
+  printf '{"status":{"readyReplicas":1}}\n'
+  exit 0
+fi
+exit 0
+EOF
+chmod +x "$tmp/kubectl"
+
+kubectl=("$tmp/kubectl")
+NAMESPACE=easyai
+ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
+AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
+AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
+AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
+AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
+AI_GATEWAY_DATABASE_MAX_CONNS=32
+AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4
+AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=4
+AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=2
+AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
+AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
+AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
+AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
+AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
+AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
+AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
+AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
+AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
+AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
+AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
+
+# activate_manifest invokes this function inside an && condition. A failed
+# rollout must still propagate instead of falling through to a stale Ready Pod.
+if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then
+  echo 'failed Worker rollout was reported as successful' >&2
+  exit 1
+fi
+
+deployment_strategy() {
+  local deployment=$1
+  awk -v deployment="$deployment" '
+    $0 == "  name: " deployment { capture = 1 }
+    capture && /^---$/ { exit }
+    capture { print }
+  ' "$application"
+}
+
+for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
+  strategy=$(deployment_strategy "$deployment")
+  grep -Fq '      maxSurge: 0' <<<"$strategy"
+  grep -Fq '      maxUnavailable: 1' <<<"$strategy"
+done
+
+for deployment in easyai-api-ningbo easyai-api-hongkong; do
+  strategy=$(deployment_strategy "$deployment")
+  grep -Fq '      maxSurge: 1' <<<"$strategy"
+  grep -Fq '      maxUnavailable: 0' <<<"$strategy"
+done
+
+echo 'cluster_release_helper_tests=PASS'
diff --git a/tests/release/manual-release-test.sh b/tests/release/manual-release-test.sh
index eeb3e85..84d2078 100755
--- a/tests/release/manual-release-test.sh
+++ b/tests/release/manual-release-test.sh
@@ -306,4 +306,6 @@ if grep -Fq 'archive_timeout: 60s' "$root/deploy/kubernetes/production/database.
   exit 1
 fi
 
+"$root/tests/release/cluster-release-helper-test.sh" >/dev/null
+
 echo 'manual_release_tests=PASS'