wangbo
a5dd7f36f5
生产容量控制器原 Role 仅允许访问宁波 Worker，香港 Worker 启用后导致 reconciliation 返回 403。补齐香港 Deployment 及 scale 子资源权限。 同时将失败发布的 Deployment 恢复从合并式 apply 改为基于资源版本的精确 replace，并优先删除发布前不存在的站点，避免新增环境变量残留导致回滚卡死。 验证：bash -n、ShellCheck、cluster release helper、manual release 与差异检查均通过。
121 lines
4.2 KiB
Bash
Executable File
121 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2034 # Variables are consumed by the dynamically sourced function.
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
|
|
helper=$root/deploy/kubernetes/easyai-ai-gateway-cluster-release
|
|
application=$root/deploy/kubernetes/production/application.yaml
|
|
|
|
awk '
|
|
/^rollout_worker_site\(\)/ { capture = 1 }
|
|
capture && /^rollout_capacity_controller\(\)/ { exit }
|
|
capture { print }
|
|
' "$helper" >"$tmp/rollout-worker-site.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/rollout-worker-site.sh"
|
|
|
|
cat >"$tmp/kubectl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
if [[ ${1:-} == rollout && ${2:-} == status ]]; then
|
|
exit 42
|
|
fi
|
|
if [[ ${1:-} == get && $* == *jsonpath* ]]; then
|
|
printf worker
|
|
exit 0
|
|
fi
|
|
if [[ ${1:-} == get && $* == *'-o json'* ]]; then
|
|
printf '{"status":{"readyReplicas":1}}\n'
|
|
exit 0
|
|
fi
|
|
exit 0
|
|
EOF
|
|
chmod +x "$tmp/kubectl"
|
|
|
|
kubectl=("$tmp/kubectl")
|
|
NAMESPACE=easyai
|
|
ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
|
|
AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=8
|
|
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
|
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4
|
|
AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=4
|
|
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=2
|
|
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
|
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
|
|
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
|
|
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
|
|
|
|
grep -Fq "\"AI_GATEWAY_REVISION=\${ACTIVE_RELEASE_SHA:-}\"" "$helper"
|
|
|
|
# activate_manifest invokes this function inside an && condition. A failed
|
|
# rollout must still propagate instead of falling through to a stale Ready Pod.
|
|
if rollout_worker_site ningbo 'registry.example.invalid/api@sha256:deadbeef' && true; then
|
|
echo 'failed Worker rollout was reported as successful' >&2
|
|
exit 1
|
|
fi
|
|
if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then
|
|
echo 'failed Hong Kong Worker rollout was reported as successful' >&2
|
|
exit 1
|
|
fi
|
|
|
|
deployment_strategy() {
|
|
local deployment=$1
|
|
awk -v deployment="$deployment" '
|
|
$0 == " name: " deployment { capture = 1 }
|
|
capture && /^---$/ { exit }
|
|
capture { print }
|
|
' "$application"
|
|
}
|
|
|
|
strategy=$(deployment_strategy easyai-worker-ningbo)
|
|
grep -Fq ' maxSurge: 0' <<<"$strategy"
|
|
grep -Fq ' maxUnavailable: 1' <<<"$strategy"
|
|
|
|
strategy=$(deployment_strategy easyai-api-ningbo)
|
|
grep -Fq ' maxSurge: 1' <<<"$strategy"
|
|
grep -Fq ' maxUnavailable: 0' <<<"$strategy"
|
|
|
|
grep -Fq 'name: easyai-api-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
|
|
grep -Fq 'name: easyai-web-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
|
|
if grep -Fq 'name: easyai-worker-hongkong' \
|
|
"$root/deploy/kubernetes/production/kustomization.yaml"; then
|
|
echo 'Hong Kong Worker is still deleted from the production desired state' >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1' \
|
|
"$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"
|
|
grep -Fq "hongkong) replicas=\$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" "$helper"
|
|
grep -Fq "\"\${kubectl[@]}\" replace -f -" "$helper"
|
|
if grep -Fq "\"\${kubectl[@]}\" apply -f \"\$snapshot\"" "$helper"; then
|
|
echo 'deployment rollback still uses merge semantics instead of exact restore' >&2
|
|
exit 1
|
|
fi
|
|
if [[ $(grep -Fc 'resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]' \
|
|
"$root/deploy/kubernetes/production/service-account-rbac.yaml") -ne 2 ]]; then
|
|
echo 'capacity controller RBAC does not cover both Worker deployments' >&2
|
|
exit 1
|
|
fi
|
|
|
|
hongkong_strategy=$(deployment_strategy easyai-worker-hongkong)
|
|
grep -Fq ' replicas: 1' <<<"$hongkong_strategy"
|
|
grep -Fq 'easyai.io/worker-only' <<<"$hongkong_strategy"
|
|
|
|
echo 'cluster_release_helper_tests=PASS'
|