fix(release): 修正 Worker 滚动与失败传播

单站点 Worker 启用硬反亲和后,maxSurge=1 会让新 Pod 无法调度并造成滚动超时。将 Worker 改为先下线旧 Pod 再创建新 Pod,同时保持 API 的零中断滚动策略。\n\n发布辅助脚本为关键 kubectl 操作补充显式失败返回,避免函数位于条件表达式时 Bash 忽略 errexit,进而把失败发布误报为成功。新增回归测试模拟 rollout status 失败且旧 Pod 仍 Ready 的场景。\n\n验证:bash -n、ShellCheck、cluster-release-helper-test、manual-release-test、生产 API Server dry-run。
This commit is contained in:
2026-08-01 13:43:33 +08:00
parent 80801b7bbb
commit 4d84210344
4 changed files with 121 additions and 22 deletions
@@ -364,23 +364,24 @@ rollout_worker_site() {
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA"
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--containers=worker \
--requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
--limits='cpu=2,memory=2Gi'
--limits='cpu=2,memory=2Gi' || return 1
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--replicas="$replicas"
--replicas="$replicas" || return 1
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"worker=$api_image"
"worker=$api_image" || return 1
fi
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" \
-n "$NAMESPACE" --timeout=300s || return 1
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]] || return 1
ready_replicas=$("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o json | jq -r '.status.readyReplicas // 0')
[[ $ready_replicas == "$replicas" ]]
-o json | jq -r '.status.readyReplicas // 0') || return 1
[[ $ready_replicas == "$replicas" ]] || return 1
}
rollout_capacity_controller() {
@@ -405,13 +406,15 @@ rollout_capacity_controller() {
"AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" || return 1
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
"capacity-controller=$api_image"
"capacity-controller=$api_image" || return 1
fi
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=2
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller -n "$NAMESPACE" --timeout=300s
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
-n "$NAMESPACE" --replicas=2 || return 1
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
-n "$NAMESPACE" --timeout=300s || return 1
}
run_capacity_controller_preflight() {
@@ -575,17 +578,19 @@ rollout_site() {
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" || return 1
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
"api=$api_image"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
"api=$api_image" || return 1
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" \
-n "$NAMESPACE" --timeout=300s || return 1
fi
if [[ $web_changed == true ]]; then
"${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
"web=$web_image"
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
"web=$web_image" || return 1
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" \
-n "$NAMESPACE" --timeout=300s || return 1
fi
verify_site "$site"
verify_site "$site" || return 1
}
run_backup() {
@@ -438,8 +438,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-worker
@@ -611,8 +611,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-worker