fix(release): 修正 Worker 滚动与失败传播
单站点 Worker 启用硬反亲和后,maxSurge=1 会让新 Pod 无法调度并造成滚动超时。将 Worker 改为先下线旧 Pod 再创建新 Pod,同时保持 API 的零中断滚动策略。\n\n发布辅助脚本为关键 kubectl 操作补充显式失败返回,避免函数位于条件表达式时 Bash 忽略 errexit,进而把失败发布误报为成功。新增回归测试模拟 rollout status 失败且旧 Pod 仍 Ready 的场景。\n\n验证:bash -n、ShellCheck、cluster-release-helper-test、manual-release-test、生产 API Server dry-run。
This commit is contained in:
@@ -364,23 +364,24 @@ rollout_worker_site() {
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA"
|
||||
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
|
||||
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
--containers=worker \
|
||||
--requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
|
||||
--limits='cpu=2,memory=2Gi'
|
||||
--limits='cpu=2,memory=2Gi' || return 1
|
||||
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
--replicas="$replicas"
|
||||
--replicas="$replicas" || return 1
|
||||
if [[ -n $api_image ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"worker=$api_image"
|
||||
"worker=$api_image" || return 1
|
||||
fi
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]] || return 1
|
||||
ready_replicas=$("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o json | jq -r '.status.readyReplicas // 0')
|
||||
[[ $ready_replicas == "$replicas" ]]
|
||||
-o json | jq -r '.status.readyReplicas // 0') || return 1
|
||||
[[ $ready_replicas == "$replicas" ]] || return 1
|
||||
}
|
||||
|
||||
rollout_capacity_controller() {
|
||||
@@ -405,13 +406,15 @@ rollout_capacity_controller() {
|
||||
"AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
|
||||
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
|
||||
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" || return 1
|
||||
if [[ -n $api_image ]]; then
|
||||
"${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
|
||||
"capacity-controller=$api_image"
|
||||
"capacity-controller=$api_image" || return 1
|
||||
fi
|
||||
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=2
|
||||
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller -n "$NAMESPACE" --timeout=300s
|
||||
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
|
||||
-n "$NAMESPACE" --replicas=2 || return 1
|
||||
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
}
|
||||
|
||||
run_capacity_controller_preflight() {
|
||||
@@ -575,17 +578,19 @@ rollout_site() {
|
||||
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" || return 1
|
||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"api=$api_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
"api=$api_image" || return 1
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
fi
|
||||
if [[ $web_changed == true ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
|
||||
"web=$web_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
|
||||
"web=$web_image" || return 1
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
fi
|
||||
verify_site "$site"
|
||||
verify_site "$site" || return 1
|
||||
}
|
||||
|
||||
run_backup() {
|
||||
|
||||
@@ -438,8 +438,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: easyai-worker
|
||||
@@ -611,8 +611,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: easyai-worker
|
||||
|
||||
Reference in New Issue
Block a user