diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 2d5421d..f2b9bf6 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -375,22 +375,42 @@ snapshot_application_deployments() { restore_application_deployments() { local snapshot=$1 - local deployment + local deployment deployment_json resource_version [[ -s $snapshot ]] || return 1 - "${kubectl[@]}" apply -f "$snapshot" >/dev/null + + # Remove deployments that did not exist before the release first. This keeps + # a newly introduced site from affecting the restored controller while its + # previous environment is still being reinstated. for deployment in \ easyai-api-ningbo easyai-api-hongkong \ easyai-worker-ningbo easyai-worker-hongkong \ easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong; do - if jq -e --arg name "$deployment" \ + if ! jq -e --arg name "$deployment" \ 'any(.items[]; .metadata.name == $name)' "$snapshot" >/dev/null; then - "${kubectl[@]}" rollout status "deployment/$deployment" \ - -n "$NAMESPACE" --timeout=300s || true - else "${kubectl[@]}" delete deployment "$deployment" -n "$NAMESPACE" \ --ignore-not-found=true --wait=true >/dev/null || true fi done + + # kubectl apply merges list fields and would retain env entries added by a + # failed rollout. Replace each existing Deployment with the captured spec so + # the recovery is exact while preserving the live resource identity. + while IFS= read -r deployment_json; do + deployment=$(jq -r '.metadata.name' <<<"$deployment_json") + if resource_version=$("${kubectl[@]}" get deployment "$deployment" \ + -n "$NAMESPACE" -o jsonpath='{.metadata.resourceVersion}' 2>/dev/null); then + jq --arg resourceVersion "$resource_version" \ + '.metadata.resourceVersion = $resourceVersion' <<<"$deployment_json" | + "${kubectl[@]}" replace -f - >/dev/null + else + "${kubectl[@]}" create -f - <<<"$deployment_json" >/dev/null + fi + done < <(jq -c '.items[]' "$snapshot") + + while IFS= read -r deployment; do + "${kubectl[@]}" rollout status "deployment/$deployment" \ + -n "$NAMESPACE" --timeout=300s || true + done < <(jq -r '.items[].metadata.name' "$snapshot") } rollout_worker_site() { diff --git a/deploy/kubernetes/production/service-account-rbac.yaml b/deploy/kubernetes/production/service-account-rbac.yaml index 7430eac..c8da50d 100644 --- a/deploy/kubernetes/production/service-account-rbac.yaml +++ b/deploy/kubernetes/production/service-account-rbac.yaml @@ -48,11 +48,11 @@ metadata: rules: - apiGroups: ["apps"] resources: ["deployments"] - resourceNames: ["easyai-worker-ningbo"] + resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"] verbs: ["get"] - apiGroups: ["apps"] resources: ["deployments/scale"] - resourceNames: ["easyai-worker-ningbo"] + resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"] verbs: ["get", "update", "patch"] - apiGroups: [""] resources: ["pods"] diff --git a/tests/release/cluster-release-helper-test.sh b/tests/release/cluster-release-helper-test.sh index ac9c465..0167a25 100755 --- a/tests/release/cluster-release-helper-test.sh +++ b/tests/release/cluster-release-helper-test.sh @@ -102,6 +102,16 @@ fi grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1' \ "$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example" grep -Fq "hongkong) replicas=\$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" "$helper" +grep -Fq "\"\${kubectl[@]}\" replace -f -" "$helper" +if grep -Fq "\"\${kubectl[@]}\" apply -f \"\$snapshot\"" "$helper"; then + echo 'deployment rollback still uses merge semantics instead of exact restore' >&2 + exit 1 +fi +if [[ $(grep -Fc 'resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]' \ + "$root/deploy/kubernetes/production/service-account-rbac.yaml") -ne 2 ]]; then + echo 'capacity controller RBAC does not cover both Worker deployments' >&2 + exit 1 +fi hongkong_strategy=$(deployment_strategy easyai-worker-hongkong) grep -Fq ' replicas: 1' <<<"$hongkong_strategy"