fix(release): 修正 Worker 滚动与失败传播
单站点 Worker 启用硬反亲和后，maxSurge=1 会让新 Pod 无法调度并造成滚动超时。将 Worker 改为先下线旧 Pod 再创建新 Pod，同时保持 API 的零中断滚动策略。\n\n发布辅助脚本为关键 kubectl 操作补充显式失败返回，避免函数位于条件表达式时 Bash 忽略 errexit，进而把失败发布误报为成功。新增回归测试模拟 rollout status 失败且旧 Pod 仍 Ready 的场景。\n\n验证：bash -n、ShellCheck、cluster-release-helper-test、manual-release-test、生产 API Server dry-run。
This commit is contained in:
@@ -364,23 +364,24 @@ rollout_worker_site() {
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA"
|
||||
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
|
||||
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
--containers=worker \
|
||||
--requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
|
||||
--limits='cpu=2,memory=2Gi'
|
||||
--limits='cpu=2,memory=2Gi' || return 1
|
||||
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
--replicas="$replicas"
|
||||
--replicas="$replicas" || return 1
|
||||
if [[ -n $api_image ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"worker=$api_image"
|
||||
"worker=$api_image" || return 1
|
||||
fi
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]] || return 1
|
||||
ready_replicas=$("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o json | jq -r '.status.readyReplicas // 0')
|
||||
[[ $ready_replicas == "$replicas" ]]
|
||||
-o json | jq -r '.status.readyReplicas // 0') || return 1
|
||||
[[ $ready_replicas == "$replicas" ]] || return 1
|
||||
}
|
||||
|
||||
rollout_capacity_controller() {
|
||||
@@ -405,13 +406,15 @@ rollout_capacity_controller() {
|
||||
"AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
|
||||
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
|
||||
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" || return 1
|
||||
if [[ -n $api_image ]]; then
|
||||
"${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
|
||||
"capacity-controller=$api_image"
|
||||
"capacity-controller=$api_image" || return 1
|
||||
fi
|
||||
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=2
|
||||
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller -n "$NAMESPACE" --timeout=300s
|
||||
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
|
||||
-n "$NAMESPACE" --replicas=2 || return 1
|
||||
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
}
|
||||
|
||||
run_capacity_controller_preflight() {
|
||||
@@ -575,17 +578,19 @@ rollout_site() {
|
||||
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
|
||||
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" || return 1
|
||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"api=$api_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
"api=$api_image" || return 1
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
fi
|
||||
if [[ $web_changed == true ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
|
||||
"web=$web_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
|
||||
"web=$web_image" || return 1
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
fi
|
||||
verify_site "$site"
|
||||
verify_site "$site" || return 1
|
||||
}
|
||||
|
||||
run_backup() {
|
||||
|
||||
@@ -438,8 +438,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: easyai-worker
|
||||
@@ -611,8 +611,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: easyai-worker
|
||||
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC2034 # Variables are consumed by the dynamically sourced function.
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
helper=$root/deploy/kubernetes/easyai-ai-gateway-cluster-release
|
||||
application=$root/deploy/kubernetes/production/application.yaml
|
||||
|
||||
awk '
|
||||
/^rollout_worker_site\(\)/ { capture = 1 }
|
||||
capture && /^rollout_capacity_controller\(\)/ { exit }
|
||||
capture { print }
|
||||
' "$helper" >"$tmp/rollout-worker-site.sh"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/rollout-worker-site.sh"
|
||||
|
||||
cat >"$tmp/kubectl" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ ${1:-} == rollout && ${2:-} == status ]]; then
|
||||
exit 42
|
||||
fi
|
||||
if [[ ${1:-} == get && $* == *jsonpath* ]]; then
|
||||
printf worker
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${1:-} == get && $* == *'-o json'* ]]; then
|
||||
printf '{"status":{"readyReplicas":1}}\n'
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$tmp/kubectl"
|
||||
|
||||
kubectl=("$tmp/kubectl")
|
||||
NAMESPACE=easyai
|
||||
ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
||||
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4
|
||||
AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=4
|
||||
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=2
|
||||
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
|
||||
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
|
||||
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
|
||||
|
||||
# activate_manifest invokes this function inside an && condition. A failed
|
||||
# rollout must still propagate instead of falling through to a stale Ready Pod.
|
||||
if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then
|
||||
echo 'failed Worker rollout was reported as successful' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
deployment_strategy() {
|
||||
local deployment=$1
|
||||
awk -v deployment="$deployment" '
|
||||
$0 == " name: " deployment { capture = 1 }
|
||||
capture && /^---$/ { exit }
|
||||
capture { print }
|
||||
' "$application"
|
||||
}
|
||||
|
||||
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
strategy=$(deployment_strategy "$deployment")
|
||||
grep -Fq ' maxSurge: 0' <<<"$strategy"
|
||||
grep -Fq ' maxUnavailable: 1' <<<"$strategy"
|
||||
done
|
||||
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong; do
|
||||
strategy=$(deployment_strategy "$deployment")
|
||||
grep -Fq ' maxSurge: 1' <<<"$strategy"
|
||||
grep -Fq ' maxUnavailable: 0' <<<"$strategy"
|
||||
done
|
||||
|
||||
echo 'cluster_release_helper_tests=PASS'
|
||||
@@ -306,4 +306,6 @@ if grep -Fq 'archive_timeout: 60s' "$root/deploy/kubernetes/production/database.
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"$root/tests/release/cluster-release-helper-test.sh" >/dev/null
|
||||
|
||||
echo 'manual_release_tests=PASS'
|
||||
|
||||
Reference in New Issue
Block a user