wangbo
4d84210344
单站点 Worker 启用硬反亲和后，maxSurge=1 会让新 Pod 无法调度并造成滚动超时。将 Worker 改为先下线旧 Pod 再创建新 Pod，同时保持 API 的零中断滚动策略。\n\n发布辅助脚本为关键 kubectl 操作补充显式失败返回，避免函数位于条件表达式时 Bash 忽略 errexit，进而把失败发布误报为成功。新增回归测试模拟 rollout status 失败且旧 Pod 仍 Ready 的场景。\n\n验证：bash -n、ShellCheck、cluster-release-helper-test、manual-release-test、生产 API Server dry-run。
93 lines
2.8 KiB
Bash
Executable File
93 lines
2.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2034 # Variables are consumed by the dynamically sourced function.
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
|
|
helper=$root/deploy/kubernetes/easyai-ai-gateway-cluster-release
|
|
application=$root/deploy/kubernetes/production/application.yaml
|
|
|
|
awk '
|
|
/^rollout_worker_site\(\)/ { capture = 1 }
|
|
capture && /^rollout_capacity_controller\(\)/ { exit }
|
|
capture { print }
|
|
' "$helper" >"$tmp/rollout-worker-site.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/rollout-worker-site.sh"
|
|
|
|
cat >"$tmp/kubectl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
if [[ ${1:-} == rollout && ${2:-} == status ]]; then
|
|
exit 42
|
|
fi
|
|
if [[ ${1:-} == get && $* == *jsonpath* ]]; then
|
|
printf worker
|
|
exit 0
|
|
fi
|
|
if [[ ${1:-} == get && $* == *'-o json'* ]]; then
|
|
printf '{"status":{"readyReplicas":1}}\n'
|
|
exit 0
|
|
fi
|
|
exit 0
|
|
EOF
|
|
chmod +x "$tmp/kubectl"
|
|
|
|
kubectl=("$tmp/kubectl")
|
|
NAMESPACE=easyai
|
|
ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
|
|
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
|
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4
|
|
AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=4
|
|
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=2
|
|
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
|
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
|
|
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
|
|
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
|
|
|
|
# activate_manifest invokes this function inside an && condition. A failed
|
|
# rollout must still propagate instead of falling through to a stale Ready Pod.
|
|
if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then
|
|
echo 'failed Worker rollout was reported as successful' >&2
|
|
exit 1
|
|
fi
|
|
|
|
deployment_strategy() {
|
|
local deployment=$1
|
|
awk -v deployment="$deployment" '
|
|
$0 == " name: " deployment { capture = 1 }
|
|
capture && /^---$/ { exit }
|
|
capture { print }
|
|
' "$application"
|
|
}
|
|
|
|
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
|
strategy=$(deployment_strategy "$deployment")
|
|
grep -Fq ' maxSurge: 0' <<<"$strategy"
|
|
grep -Fq ' maxUnavailable: 1' <<<"$strategy"
|
|
done
|
|
|
|
for deployment in easyai-api-ningbo easyai-api-hongkong; do
|
|
strategy=$(deployment_strategy "$deployment")
|
|
grep -Fq ' maxSurge: 1' <<<"$strategy"
|
|
grep -Fq ' maxUnavailable: 0' <<<"$strategy"
|
|
done
|
|
|
|
echo 'cluster_release_helper_tests=PASS'
|