fix(release): 补齐香港 Worker 权限与精确回滚
生产容量控制器原 Role 仅允许访问宁波 Worker,香港 Worker 启用后导致 reconciliation 返回 403。补齐香港 Deployment 及 scale 子资源权限。 同时将失败发布的 Deployment 恢复从合并式 apply 改为基于资源版本的精确 replace,并优先删除发布前不存在的站点,避免新增环境变量残留导致回滚卡死。 验证:bash -n、ShellCheck、cluster release helper、manual release 与差异检查均通过。
This commit is contained in:
@@ -375,22 +375,42 @@ snapshot_application_deployments() {
|
|||||||
|
|
||||||
restore_application_deployments() {
|
restore_application_deployments() {
|
||||||
local snapshot=$1
|
local snapshot=$1
|
||||||
local deployment
|
local deployment deployment_json resource_version
|
||||||
[[ -s $snapshot ]] || return 1
|
[[ -s $snapshot ]] || return 1
|
||||||
"${kubectl[@]}" apply -f "$snapshot" >/dev/null
|
|
||||||
|
# Remove deployments that did not exist before the release first. This keeps
|
||||||
|
# a newly introduced site from affecting the restored controller while its
|
||||||
|
# previous environment is still being reinstated.
|
||||||
for deployment in \
|
for deployment in \
|
||||||
easyai-api-ningbo easyai-api-hongkong \
|
easyai-api-ningbo easyai-api-hongkong \
|
||||||
easyai-worker-ningbo easyai-worker-hongkong \
|
easyai-worker-ningbo easyai-worker-hongkong \
|
||||||
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong; do
|
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong; do
|
||||||
if jq -e --arg name "$deployment" \
|
if ! jq -e --arg name "$deployment" \
|
||||||
'any(.items[]; .metadata.name == $name)' "$snapshot" >/dev/null; then
|
'any(.items[]; .metadata.name == $name)' "$snapshot" >/dev/null; then
|
||||||
"${kubectl[@]}" rollout status "deployment/$deployment" \
|
|
||||||
-n "$NAMESPACE" --timeout=300s || true
|
|
||||||
else
|
|
||||||
"${kubectl[@]}" delete deployment "$deployment" -n "$NAMESPACE" \
|
"${kubectl[@]}" delete deployment "$deployment" -n "$NAMESPACE" \
|
||||||
--ignore-not-found=true --wait=true >/dev/null || true
|
--ignore-not-found=true --wait=true >/dev/null || true
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# kubectl apply merges list fields and would retain env entries added by a
|
||||||
|
# failed rollout. Replace each existing Deployment with the captured spec so
|
||||||
|
# the recovery is exact while preserving the live resource identity.
|
||||||
|
while IFS= read -r deployment_json; do
|
||||||
|
deployment=$(jq -r '.metadata.name' <<<"$deployment_json")
|
||||||
|
if resource_version=$("${kubectl[@]}" get deployment "$deployment" \
|
||||||
|
-n "$NAMESPACE" -o jsonpath='{.metadata.resourceVersion}' 2>/dev/null); then
|
||||||
|
jq --arg resourceVersion "$resource_version" \
|
||||||
|
'.metadata.resourceVersion = $resourceVersion' <<<"$deployment_json" |
|
||||||
|
"${kubectl[@]}" replace -f - >/dev/null
|
||||||
|
else
|
||||||
|
"${kubectl[@]}" create -f - <<<"$deployment_json" >/dev/null
|
||||||
|
fi
|
||||||
|
done < <(jq -c '.items[]' "$snapshot")
|
||||||
|
|
||||||
|
while IFS= read -r deployment; do
|
||||||
|
"${kubectl[@]}" rollout status "deployment/$deployment" \
|
||||||
|
-n "$NAMESPACE" --timeout=300s || true
|
||||||
|
done < <(jq -r '.items[].metadata.name' "$snapshot")
|
||||||
}
|
}
|
||||||
|
|
||||||
rollout_worker_site() {
|
rollout_worker_site() {
|
||||||
|
|||||||
@@ -48,11 +48,11 @@ metadata:
|
|||||||
rules:
|
rules:
|
||||||
- apiGroups: ["apps"]
|
- apiGroups: ["apps"]
|
||||||
resources: ["deployments"]
|
resources: ["deployments"]
|
||||||
resourceNames: ["easyai-worker-ningbo"]
|
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
|
||||||
verbs: ["get"]
|
verbs: ["get"]
|
||||||
- apiGroups: ["apps"]
|
- apiGroups: ["apps"]
|
||||||
resources: ["deployments/scale"]
|
resources: ["deployments/scale"]
|
||||||
resourceNames: ["easyai-worker-ningbo"]
|
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
|
||||||
verbs: ["get", "update", "patch"]
|
verbs: ["get", "update", "patch"]
|
||||||
- apiGroups: [""]
|
- apiGroups: [""]
|
||||||
resources: ["pods"]
|
resources: ["pods"]
|
||||||
|
|||||||
@@ -102,6 +102,16 @@ fi
|
|||||||
grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1' \
|
grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1' \
|
||||||
"$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"
|
"$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"
|
||||||
grep -Fq "hongkong) replicas=\$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" "$helper"
|
grep -Fq "hongkong) replicas=\$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" "$helper"
|
||||||
|
grep -Fq "\"\${kubectl[@]}\" replace -f -" "$helper"
|
||||||
|
if grep -Fq "\"\${kubectl[@]}\" apply -f \"\$snapshot\"" "$helper"; then
|
||||||
|
echo 'deployment rollback still uses merge semantics instead of exact restore' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ $(grep -Fc 'resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]' \
|
||||||
|
"$root/deploy/kubernetes/production/service-account-rbac.yaml") -ne 2 ]]; then
|
||||||
|
echo 'capacity controller RBAC does not cover both Worker deployments' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
hongkong_strategy=$(deployment_strategy easyai-worker-hongkong)
|
hongkong_strategy=$(deployment_strategy easyai-worker-hongkong)
|
||||||
grep -Fq ' replicas: 1' <<<"$hongkong_strategy"
|
grep -Fq ' replicas: 1' <<<"$hongkong_strategy"
|
||||||
|
|||||||
Reference in New Issue
Block a user