From 0b2bd1d51209da1ffd45136d69b195a8a1e7d036 Mon Sep 17 00:00:00 2001 From: wangbo Date: Tue, 4 Aug 2026 20:33:06 +0800 Subject: [PATCH] =?UTF-8?q?feat(release):=20=E5=9B=BA=E5=AE=9A=E5=90=AF?= =?UTF-8?q?=E7=94=A8=E9=A6=99=E6=B8=AF=E5=BC=82=E6=AD=A5=20Worker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将生产 Worker 拓扑固定为宁波 1 个、香港 1 个，并让发布、容量配置和回滚流程持续保留香港 Worker。当前仍只使用宁波 PostgreSQL，关闭横向自动扩缩容，避免依赖数据库同步副本。 验证：API 全量 go test、迁移安全检查、cluster release helper、manual release、bash -n、ShellCheck、Kustomize 渲染均通过。 --- .../easyai-ai-gateway-cluster-release | 49 +++++++++++++------ ...ai-ai-gateway-cluster-release.conf.example | 3 ++ deploy/kubernetes/production/application.yaml | 2 +- .../kubernetes/production/kustomization.yaml | 9 ---- tests/release/cluster-release-helper-test.sh | 21 ++++++-- 5 files changed, 55 insertions(+), 29 deletions(-) diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 161be85..2d5421d 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -16,6 +16,7 @@ source "$config_file" : "${K3S_BIN:=/usr/local/bin/k3s}" : "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}" : "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}" +: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}" : "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}" : "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}" : "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}" @@ -34,6 +35,8 @@ source "$config_file" : "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}" : "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}" : "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}" +: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}" +: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=1}" : "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}" : "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}" : "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}" @@ -56,7 +59,10 @@ source "$config_file" for replica_value in \ "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO"; do + "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ + "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG"; do [[ $replica_value =~ ^[0-9]+$ ]] || { echo 'Worker replica configuration must contain non-negative integers' >&2 exit 1 @@ -127,13 +133,16 @@ done AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO && AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO && AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 && + AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG && + AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG && + AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 && AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT && AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 && AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 && AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 && AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET && AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET + - AI_GATEWAY_WORKER_REPLICAS_NINGBO * + (AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) * AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || { echo 'worker capacity configuration exceeds the release safety bounds' >&2 exit 1 @@ -159,6 +168,7 @@ manifest_get() { capacity_config_hash() { printf '%s\n' \ "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ + "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \ @@ -177,6 +187,8 @@ capacity_config_hash() { "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ + "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ + "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \ "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \ "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \ @@ -196,6 +208,9 @@ record_capacity_config() { --arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ --arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ --arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ + --arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ + --arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ + --arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ --arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ --arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ --arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \ @@ -213,11 +228,11 @@ record_capacity_config() { --arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \ '{data:{ AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo, - AI_GATEWAY_WORKER_REPLICAS_HONGKONG:null, + AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong, AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo, - AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:null, + AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong, AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo, - AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:null, + AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong, AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling, AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit, AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit, @@ -245,6 +260,7 @@ record_capacity_config() { for deployment in \ easyai-api-ningbo \ easyai-worker-ningbo \ + easyai-worker-hongkong \ easyai-capacity-controller; do "${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \ "easyai.io/config-hash=$config_hash" --overwrite >/dev/null @@ -295,9 +311,9 @@ verify_site() { wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway' } -retire_hongkong_site() { +retire_hongkong_edges() { "${kubectl[@]}" delete deployment \ - easyai-api-hongkong easyai-worker-hongkong easyai-web-hongkong \ + easyai-api-hongkong easyai-web-hongkong \ -n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null "${kubectl[@]}" delete service \ api-hongkong-edge web-hongkong-edge \ @@ -383,6 +399,7 @@ rollout_worker_site() { local replicas ready_replicas case $site in ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;; + hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;; *) return 1 ;; esac "${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \ @@ -403,9 +420,9 @@ rollout_worker_site() { "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG-' \ - 'AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG-' \ - 'AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG-' \ + "AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ + "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ + "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1 "${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \ --containers=worker \ @@ -435,9 +452,9 @@ rollout_capacity_controller() { "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG-' \ - 'AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG-' \ - 'AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG-' \ + "AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ + "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ + "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \ "AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \ "AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \ @@ -604,6 +621,7 @@ apply_capacity_config() { return 1 } rollout_worker_site ningbo "" + rollout_worker_site hongkong "" rollout_api_capacity_site ningbo if [[ $capacity_controller_enabled == true ]]; then rollout_capacity_controller "" @@ -613,7 +631,7 @@ apply_capacity_config() { verify_site ningbo wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' record_capacity_config - echo "production_capacity=PASS config_hash=$(capacity_config_hash) site=ningbo replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" + echo "production_capacity=PASS config_hash=$(capacity_config_hash) replicas_ningbo=$AI_GATEWAY_WORKER_REPLICAS_NINGBO replicas_hongkong=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" } rollout_site() { @@ -924,6 +942,7 @@ activate_manifest() { { rollout_worker_site ningbo "$api_image" && { [[ $action == rollback ]] || rollout_capacity_controller "$api_image"; }; }; } && + rollout_worker_site hongkong "$api_image" && rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" && wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then : @@ -936,7 +955,7 @@ activate_manifest() { return 1 fi - retire_hongkong_site + retire_hongkong_edges record_capacity_config record_current "$manifest" "$action" "$started_at" rm -f -- "$deployment_snapshot" "$current_file" diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example index 59afa5c..1acc25d 100644 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example @@ -7,6 +7,7 @@ DESIRED_STATE_DIR=/usr/local/share/easyai-ai-gateway-release/production # Worker 容量只需修改本发布配置并执行获授权的 `capacity`，不再修改 Go 代码。 # 副本数属于 Deployment 配置，不能由容器内部环境变量改变；发布工具读取这两个变量并写入 replicas。 AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 +AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=8 AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 @@ -25,6 +26,8 @@ AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 +AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 +AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48 AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index 0bd77c0..f745469 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -602,7 +602,7 @@ metadata: app.kubernetes.io/component: worker app.kubernetes.io/part-of: easyai-ai-gateway spec: - replicas: 2 + replicas: 1 strategy: type: RollingUpdate rollingUpdate: diff --git a/deploy/kubernetes/production/kustomization.yaml b/deploy/kubernetes/production/kustomization.yaml index 17eff9c..68e1e60 100644 --- a/deploy/kubernetes/production/kustomization.yaml +++ b/deploy/kubernetes/production/kustomization.yaml @@ -18,15 +18,6 @@ patches: kind: Deployment metadata: name: easyai-api-hongkong - - target: - kind: Deployment - name: easyai-worker-hongkong - patch: |- - $patch: delete - apiVersion: apps/v1 - kind: Deployment - metadata: - name: easyai-worker-hongkong - target: kind: Deployment name: easyai-web-hongkong diff --git a/tests/release/cluster-release-helper-test.sh b/tests/release/cluster-release-helper-test.sh index 9d76e22..ac9c465 100755 --- a/tests/release/cluster-release-helper-test.sh +++ b/tests/release/cluster-release-helper-test.sh @@ -41,6 +41,7 @@ kubectl=("$tmp/kubectl") NAMESPACE=easyai ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 +AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48 @@ -55,6 +56,8 @@ AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 +AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 +AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48 AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500 AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512 @@ -67,6 +70,10 @@ if rollout_worker_site ningbo 'registry.example.invalid/api@sha256:deadbeef' && echo 'failed Worker rollout was reported as successful' >&2 exit 1 fi +if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then + echo 'failed Hong Kong Worker rollout was reported as successful' >&2 + exit 1 +fi deployment_strategy() { local deployment=$1 @@ -86,12 +93,18 @@ grep -Fq ' maxSurge: 1' <<<"$strategy" grep -Fq ' maxUnavailable: 0' <<<"$strategy" grep -Fq 'name: easyai-api-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml" -grep -Fq 'name: easyai-worker-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml" grep -Fq 'name: easyai-web-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml" -if grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=' \ - "$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"; then - echo 'Hong Kong Worker capacity remains in the single-node release environment' >&2 +if grep -Fq 'name: easyai-worker-hongkong' \ + "$root/deploy/kubernetes/production/kustomization.yaml"; then + echo 'Hong Kong Worker is still deleted from the production desired state' >&2 exit 1 fi +grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1' \ + "$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example" +grep -Fq "hongkong) replicas=\$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" "$helper" + +hongkong_strategy=$(deployment_strategy easyai-worker-hongkong) +grep -Fq ' replicas: 1' <<<"$hongkong_strategy" +grep -Fq 'easyai.io/worker-only' <<<"$hongkong_strategy" echo 'cluster_release_helper_tests=PASS'