diff --git a/apps/api/internal/capacitycontroller/controller.go b/apps/api/internal/capacitycontroller/controller.go index 9c9897f..7d9877e 100644 --- a/apps/api/internal/capacitycontroller/controller.go +++ b/apps/api/internal/capacitycontroller/controller.go @@ -140,12 +140,15 @@ func (controller *Controller) reconcile(ctx context.Context) error { sites := make([]SiteResources, 0, 2) kubernetesStates := make(map[string]KubernetesSiteState, 2) for _, site := range []string{"ningbo", "hongkong"} { + minReplicas, maxReplicas := controller.siteReplicaBounds(site) + if maxReplicas == 0 { + continue + } state, stateErr := controller.kubernetes.SiteState(ctx, site) if stateErr != nil { return stateErr } kubernetesStates[site] = state - minReplicas, maxReplicas := controller.siteReplicaBounds(site) sites = append(sites, SiteResources{ Site: site, CurrentReplicas: state.CurrentReplicas, MinReplicas: minReplicas, MaxReplicas: maxReplicas, diff --git a/apps/api/internal/config/config.go b/apps/api/internal/config/config.go index d3d5be4..133d123 100644 --- a/apps/api/internal/config/config.go +++ b/apps/api/internal/config/config.go @@ -198,11 +198,11 @@ func Load() Config { ) != "", WorkerAutoscalingEnabled: env("AI_GATEWAY_WORKER_AUTOSCALING_ENABLED", "false") == "true", WorkerReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_NINGBO", 1), - WorkerReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", 1), + WorkerReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", 0), WorkerMinReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO", 1), - WorkerMinReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", 1), + WorkerMinReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", 0), WorkerMaxReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO", 1), - WorkerMaxReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", 1), + WorkerMaxReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", 0), WorkerTargetOutstandingPerReplica: envOptionalIntValidated("AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA", 0), WorkerScaleUpWindowSeconds: envIntValidated("AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS", 20), WorkerScaleDownStabilizationSeconds: envIntValidated( diff --git a/apps/api/internal/config/config_test.go b/apps/api/internal/config/config_test.go index bb43ddc..b613e37 100644 --- a/apps/api/internal/config/config_test.go +++ b/apps/api/internal/config/config_test.go @@ -217,6 +217,26 @@ func TestWorkerAutoscalingConfiguration(t *testing.T) { } } +func TestWorkerTopologyDefaultsToNingboSingleNode(t *testing.T) { + for _, name := range []string{ + "AI_GATEWAY_WORKER_REPLICAS_NINGBO", + "AI_GATEWAY_WORKER_REPLICAS_HONGKONG", + "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO", + "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", + "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO", + "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", + } { + t.Setenv(name, "") + } + cfg := Load() + if cfg.WorkerReplicasNingbo != 1 || cfg.WorkerMinReplicasNingbo != 1 || cfg.WorkerMaxReplicasNingbo != 1 { + t.Fatalf("Ningbo Worker defaults=%d/%d/%d, want 1/1/1", cfg.WorkerReplicasNingbo, cfg.WorkerMinReplicasNingbo, cfg.WorkerMaxReplicasNingbo) + } + if cfg.WorkerReplicasHongkong != 0 || cfg.WorkerMinReplicasHongkong != 0 || cfg.WorkerMaxReplicasHongkong != 0 { + t.Fatalf("Hong Kong Worker defaults=%d/%d/%d, want disabled", cfg.WorkerReplicasHongkong, cfg.WorkerMinReplicasHongkong, cfg.WorkerMaxReplicasHongkong) + } +} + func TestLoadPrefersWorkerGlobalHardLimitAlias(t *testing.T) { t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "48") t.Setenv("AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT", "96") diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index dd21cbe..7142a58 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -15,8 +15,7 @@ source "$config_file" : "${PUBLIC_BASE_URL:=https://ai.51easyai.com}" : "${K3S_BIN:=/usr/local/bin/k3s}" : "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}" -: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=0}" -: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=2}" +: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}" : "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}" : "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}" : "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}" @@ -33,10 +32,8 @@ source "$config_file" : "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}" : "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}" : "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}" -: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=0}" -: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}" -: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=0}" -: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=2}" +: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}" +: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}" : "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}" : "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}" : "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}" @@ -45,7 +42,7 @@ source "$config_file" : "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}" : "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}" : "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}" -: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}" +: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS + 4))}" [[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || { echo 'invalid cluster release configuration' >&2 exit 1 @@ -58,11 +55,8 @@ source "$config_file" } for replica_value in \ "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ - "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG"; do + "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO"; do [[ $replica_value =~ ^[0-9]+$ ]] || { echo 'Worker replica configuration must contain non-negative integers' >&2 exit 1 @@ -110,8 +104,7 @@ done exit 1 } (( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 && - AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 && - AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG >= 1 && + AI_GATEWAY_WORKER_REPLICAS_NINGBO >= 1 && AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 && AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 && AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 && @@ -133,17 +126,14 @@ done AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 && AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO && AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO && - AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG && - AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG && AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 && - AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 && AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT && AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 && AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 && AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 && AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET && AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET + - (AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) * + AI_GATEWAY_WORKER_REPLICAS_NINGBO * AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || { echo 'worker capacity configuration exceeds the release safety bounds' >&2 exit 1 @@ -169,7 +159,6 @@ manifest_get() { capacity_config_hash() { printf '%s\n' \ "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \ @@ -187,9 +176,7 @@ capacity_config_hash() { "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \ "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \ "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \ "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \ @@ -207,11 +194,8 @@ record_capacity_config() { local runtime_patch runtime_patch=$(jq -nc \ --arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ - --arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ --arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ - --arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ --arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - --arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ --arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ --arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ --arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \ @@ -229,11 +213,11 @@ record_capacity_config() { --arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \ '{data:{ AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo, - AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong, + AI_GATEWAY_WORKER_REPLICAS_HONGKONG:null, AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo, - AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong, + AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:null, AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo, - AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong, + AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:null, AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling, AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit, AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit, @@ -259,8 +243,8 @@ record_capacity_config() { "${kubectl[@]}" annotate namespace "$NAMESPACE" \ "easyai.io/config-hash=$config_hash" --overwrite >/dev/null for deployment in \ - easyai-api-ningbo easyai-api-hongkong \ - easyai-worker-ningbo easyai-worker-hongkong \ + easyai-api-ningbo \ + easyai-worker-ningbo \ easyai-capacity-controller; do "${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \ "easyai.io/config-hash=$config_hash" --overwrite >/dev/null @@ -299,11 +283,6 @@ verify_site() { local api_port local web_port case $site in - hongkong) - address=10.77.0.2 - api_port=31089 - web_port=31179 - ;; ningbo) address=10.77.0.1 api_port=31088 @@ -316,10 +295,19 @@ verify_site() { wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway' } +retire_hongkong_site() { + "${kubectl[@]}" delete deployment \ + easyai-api-hongkong easyai-worker-hongkong easyai-web-hongkong \ + -n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null + "${kubectl[@]}" delete service \ + api-hongkong-edge web-hongkong-edge \ + -n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null +} + apply_desired_state() { local api_image=$1 local web_image=$2 - local capacity_controller_replicas=${3:-2} + local capacity_controller_replicas=${3:-1} local api_repository=${api_image%@*} local api_digest=${api_image#*@} local web_repository=${web_image%@*} @@ -333,7 +321,7 @@ apply_desired_state() { "$working_directory/kustomization.yaml" if [[ $capacity_controller_replicas == 0 ]]; then sed -i \ - '/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 2$/ replicas: 0/' \ + '/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 1$/ replicas: 0/' \ "$working_directory/application.yaml" fi "${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \ @@ -395,7 +383,6 @@ rollout_worker_site() { local replicas ready_replicas case $site in ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;; - hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;; *) return 1 ;; esac "${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \ @@ -414,11 +401,8 @@ rollout_worker_site() { "AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \ "AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ - "AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ - "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1 "${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \ --containers=worker \ @@ -446,11 +430,8 @@ rollout_capacity_controller() { "${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \ "AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ - "AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ - "AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ - "AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \ "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \ "AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \ "AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \ @@ -469,7 +450,7 @@ rollout_capacity_controller() { "capacity-controller=$api_image" || return 1 fi "${kubectl[@]}" scale deployment/easyai-capacity-controller \ - -n "$NAMESPACE" --replicas=2 || return 1 + -n "$NAMESPACE" --replicas=1 || return 1 "${kubectl[@]}" rollout status deployment/easyai-capacity-controller \ -n "$NAMESPACE" --timeout=300s || return 1 } @@ -493,6 +474,7 @@ spec: restartPolicy: Never serviceAccountName: easyai-capacity-controller nodeSelector: + easyai.io/site: ningbo easyai.io/workload: "true" securityContext: runAsNonRoot: true @@ -615,20 +597,17 @@ apply_capacity_config() { echo 'current release SHA is unavailable for capacity update' >&2 return 1 } - rollout_worker_site hongkong "" rollout_worker_site ningbo "" - rollout_api_capacity_site hongkong rollout_api_capacity_site ningbo if [[ $capacity_controller_enabled == true ]]; then rollout_capacity_controller "" else "${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0 fi - verify_site hongkong verify_site ningbo wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' record_capacity_config - echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" + echo "production_capacity=PASS config_hash=$(capacity_config_hash) site=ningbo replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" } rollout_site() { @@ -786,10 +765,10 @@ record_current() { "easyai.io/config-hash=$(capacity_config_hash)" \ --overwrite >/dev/null for deployment in \ - easyai-api-ningbo easyai-api-hongkong \ - easyai-worker-ningbo easyai-worker-hongkong \ + easyai-api-ningbo \ + easyai-worker-ningbo \ easyai-capacity-controller \ - easyai-web-ningbo easyai-web-hongkong; do + easyai-web-ningbo; do "${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \ "easyai.io/release=$source_sha" \ "easyai.io/config-hash=$(capacity_config_hash)" \ @@ -936,11 +915,9 @@ activate_manifest() { fi if { [[ $api_changed != true ]] || - { rollout_worker_site hongkong "$api_image" && - rollout_worker_site ningbo "$api_image" && + { rollout_worker_site ningbo "$api_image" && { [[ $action == rollback ]] || rollout_capacity_controller "$api_image"; }; }; } && - rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" && rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" && wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then : @@ -953,6 +930,7 @@ activate_manifest() { return 1 fi + retire_hongkong_site record_capacity_config record_current "$manifest" "$action" "$started_at" rm -f -- "$deployment_snapshot" "$current_file" @@ -972,10 +950,8 @@ case ${1:-} in node "$manifest_tool" validate "$2" >/dev/null api_image=$(manifest_get "$2" images.api) web_image=$(manifest_get "$2" images.web) - for site in ningbo hongkong; do - [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \ - -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]] - done + [[ $("${kubectl[@]}" get deployment easyai-worker-ningbo -n "$NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]] [[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \ -o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]] [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \ diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example index c44f663..59afa5c 100644 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example @@ -6,8 +6,7 @@ DESIRED_STATE_DIR=/usr/local/share/easyai-ai-gateway-release/production # Worker 容量只需修改本发布配置并执行获授权的 `capacity`,不再修改 Go 代码。 # 副本数属于 Deployment 配置,不能由容器内部环境变量改变;发布工具读取这两个变量并写入 replicas。 -AI_GATEWAY_WORKER_REPLICAS_NINGBO=0 -AI_GATEWAY_WORKER_REPLICAS_HONGKONG=2 +AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=8 AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 @@ -24,10 +23,8 @@ AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24 AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false -AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=0 -AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 -AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=0 -AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=2 +AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 +AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48 AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 @@ -36,5 +33,5 @@ AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=75 AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=85 AI_GATEWAY_NODE_CPU_TARGET_PERCENT=70 AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=150 -# 2 个 API 池(2×32)加 2 个 capacity-controller 池(2×4)。 -AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=72 +# 1 个 API 池(32)加 1 个 capacity-controller 池(4)。 +AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=36 diff --git a/deploy/kubernetes/production/application-config.yaml b/deploy/kubernetes/production/application-config.yaml index 5a1f4a5..a52bdf6 100644 --- a/deploy/kubernetes/production/application-config.yaml +++ b/deploy/kubernetes/production/application-config.yaml @@ -46,12 +46,9 @@ data: AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48" AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5" AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "false" - AI_GATEWAY_WORKER_REPLICAS_NINGBO: "0" - AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "2" - AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "0" - AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG: "1" - AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "0" - AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "2" + AI_GATEWAY_WORKER_REPLICAS_NINGBO: "1" + AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "1" + AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "1" AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48" AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20" AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600" diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index 447d1c2..0bd77c0 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -158,7 +158,7 @@ metadata: app.kubernetes.io/component: capacity-controller app.kubernetes.io/part-of: easyai-ai-gateway spec: - replicas: 2 + replicas: 1 strategy: type: RollingUpdate rollingUpdate: @@ -176,22 +176,9 @@ spec: spec: serviceAccountName: easyai-capacity-controller terminationGracePeriodSeconds: 30 - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: easyai.io/site - labelSelector: - matchLabels: - app.kubernetes.io/name: easyai-capacity-controller nodeSelector: + easyai.io/site: ningbo easyai.io/workload: "true" - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: easyai.io/site - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app.kubernetes.io/name: easyai-capacity-controller securityContext: runAsNonRoot: true runAsUser: 10001 @@ -225,7 +212,7 @@ spec: - name: AI_GATEWAY_WORKER_DATABASE_MAX_CONNS value: "32" - name: AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET - value: "72" + value: "36" - name: POD_NAMESPACE valueFrom: fieldRef: @@ -438,7 +425,7 @@ metadata: app.kubernetes.io/component: worker app.kubernetes.io/part-of: easyai-ai-gateway spec: - replicas: 0 + replicas: 1 strategy: type: RollingUpdate rollingUpdate: diff --git a/deploy/kubernetes/production/kustomization.yaml b/deploy/kubernetes/production/kustomization.yaml index 294c559..17eff9c 100644 --- a/deploy/kubernetes/production/kustomization.yaml +++ b/deploy/kubernetes/production/kustomization.yaml @@ -8,6 +8,52 @@ resources: - application.yaml - database.yaml - network-policy.yaml +patches: + - target: + kind: Deployment + name: easyai-api-hongkong + patch: |- + $patch: delete + apiVersion: apps/v1 + kind: Deployment + metadata: + name: easyai-api-hongkong + - target: + kind: Deployment + name: easyai-worker-hongkong + patch: |- + $patch: delete + apiVersion: apps/v1 + kind: Deployment + metadata: + name: easyai-worker-hongkong + - target: + kind: Deployment + name: easyai-web-hongkong + patch: |- + $patch: delete + apiVersion: apps/v1 + kind: Deployment + metadata: + name: easyai-web-hongkong + - target: + kind: Service + name: api-hongkong-edge + patch: |- + $patch: delete + apiVersion: v1 + kind: Service + metadata: + name: api-hongkong-edge + - target: + kind: Service + name: web-hongkong-edge + patch: |- + $patch: delete + apiVersion: v1 + kind: Service + metadata: + name: web-hongkong-edge images: # Deployment tooling must replace both sentinel digests with release-manifest # digests before server-side apply. diff --git a/deploy/kubernetes/production/service-account-rbac.yaml b/deploy/kubernetes/production/service-account-rbac.yaml index c8da50d..7430eac 100644 --- a/deploy/kubernetes/production/service-account-rbac.yaml +++ b/deploy/kubernetes/production/service-account-rbac.yaml @@ -48,11 +48,11 @@ metadata: rules: - apiGroups: ["apps"] resources: ["deployments"] - resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"] + resourceNames: ["easyai-worker-ningbo"] verbs: ["get"] - apiGroups: ["apps"] resources: ["deployments/scale"] - resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"] + resourceNames: ["easyai-worker-ningbo"] verbs: ["get", "update", "patch"] - apiGroups: [""] resources: ["pods"] diff --git a/scripts/cluster/common.sh b/scripts/cluster/common.sh index f929c3b..9f4ef94 100755 --- a/scripts/cluster/common.sh +++ b/scripts/cluster/common.sh @@ -24,16 +24,14 @@ load_cluster_env() { cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key} : "${AI_GATEWAY_DEPLOY_HOST:?}" - : "${AI_GATEWAY_DEPLOY_HOST_2:?}" - : "${AI_GATEWAY_DEPLOY_HOST_3:?}" : "${ALI_KEY:?}" : "${ALI_SECRET:?}" : "${ALI_REGION:?}" : "${ALI_BUCKET:?}" CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST} - CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2} - CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3} + CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-${AI_GATEWAY_DEPLOY_HOST_2:-}} + CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-${AI_GATEWAY_DEPLOY_HOST_3:-}} CLUSTER_WORKER_NODE_4_HOST=${AI_GATEWAY_WORKER_NODE_4_HOST:-} CLUSTER_WORKER_NODE_4_NAME=${AI_GATEWAY_WORKER_NODE_4_NAME:-} CLUSTER_WORKER_NODE_4_SITE=${AI_GATEWAY_WORKER_NODE_4_SITE:-} @@ -46,6 +44,7 @@ load_cluster_env() { export CLUSTER_WORKER_NODE_4_ENABLED for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do + [[ -n $host ]] || continue [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || { echo "cluster hosts must use root@host syntax" >&2 return 1 diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 846fe2b..f11ebd6 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -705,17 +705,22 @@ admin_request() { -o 'jsonpath={.spec.clusterIP}') service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \ -o 'jsonpath={.spec.ports[0].port}') - fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \ - -o 'jsonpath={.spec.clusterIP}') - fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \ - -o 'jsonpath={.spec.ports[0].port}') - [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ && - $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || { + [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ ]] || { echo 'acceptance admin Service has an invalid cluster address' >&2 return 1 } acceptance_admin_base=http://$service_ip:$service_port - acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port + if [[ ${single_node_acceptance:-false} != true ]]; then + fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \ + -o 'jsonpath={.spec.clusterIP}') + fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \ + -o 'jsonpath={.spec.ports[0].port}') + [[ $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || { + echo 'acceptance fallback admin Service has an invalid cluster address' >&2 + return 1 + } + acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port + fi fi token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | openssl base64 -A) @@ -748,6 +753,10 @@ fi echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2 return 1 fi + if [[ ${single_node_acceptance:-false} == true ]]; then + echo "acceptance control API transport failed on Ningbo: method=$method path=$path" >&2 + return 1 + fi printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \ "$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \ "$token_encoded" "$body_encoded" "$has_body" @@ -794,15 +803,13 @@ verify_release_cas() { echo "production release CAS mismatch: expected=$release_sha current=$current_sha" >&2 return 1 } - for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do + for deployment in easyai-api-ningbo easyai-worker-ningbo; do [[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \ -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]] done [[ $verify_capacity == true ]] || return 0 [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \ -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]] - [[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \ - -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]] if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then local expected_replicas eligible_nodes eligible_count pod_report local running_count invalid_count node4_count used_nodes @@ -943,9 +950,7 @@ snapshot_pre_acceptance_capacity() { local output=$report_root/pre-acceptance-capacity.json remote_kubectl get \ deployment/easyai-api-ningbo \ - deployment/easyai-api-hongkong \ deployment/easyai-worker-ningbo \ - deployment/easyai-worker-hongkong \ deployment/easyai-capacity-controller \ configmap/easyai-ai-gateway-config \ configmap/easyai-gateway-release \ @@ -3466,7 +3471,6 @@ apply_single_node_capacity() { }}') remote_kubectl label node easyai-ningbo easyai.io/worker=true --overwrite >/dev/null remote_kubectl scale deployment/easyai-capacity-controller -n "$namespace" --replicas=0 >/dev/null - remote_kubectl scale deployment/easyai-worker-hongkong -n "$namespace" --replicas=0 >/dev/null remote_kubectl patch configmap easyai-ai-gateway-config -n "$namespace" \ --type=merge -p "$config_patch" >/dev/null remote_kubectl set env deployment/easyai-worker-ningbo -n "$namespace" \ diff --git a/tests/release/cluster-release-helper-test.sh b/tests/release/cluster-release-helper-test.sh index 2a06b49..9d76e22 100755 --- a/tests/release/cluster-release-helper-test.sh +++ b/tests/release/cluster-release-helper-test.sh @@ -41,7 +41,6 @@ kubectl=("$tmp/kubectl") NAMESPACE=easyai ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 -AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48 @@ -55,9 +54,7 @@ AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24 AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 -AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 -AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48 AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500 AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512 @@ -66,7 +63,7 @@ grep -Fq "\"AI_GATEWAY_REVISION=\${ACTIVE_RELEASE_SHA:-}\"" "$helper" # activate_manifest invokes this function inside an && condition. A failed # rollout must still propagate instead of falling through to a stale Ready Pod. -if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then +if rollout_worker_site ningbo 'registry.example.invalid/api@sha256:deadbeef' && true; then echo 'failed Worker rollout was reported as successful' >&2 exit 1 fi @@ -80,16 +77,21 @@ deployment_strategy() { ' "$application" } -for deployment in easyai-worker-ningbo easyai-worker-hongkong; do - strategy=$(deployment_strategy "$deployment") - grep -Fq ' maxSurge: 0' <<<"$strategy" - grep -Fq ' maxUnavailable: 1' <<<"$strategy" -done +strategy=$(deployment_strategy easyai-worker-ningbo) +grep -Fq ' maxSurge: 0' <<<"$strategy" +grep -Fq ' maxUnavailable: 1' <<<"$strategy" -for deployment in easyai-api-ningbo easyai-api-hongkong; do - strategy=$(deployment_strategy "$deployment") - grep -Fq ' maxSurge: 1' <<<"$strategy" - grep -Fq ' maxUnavailable: 0' <<<"$strategy" -done +strategy=$(deployment_strategy easyai-api-ningbo) +grep -Fq ' maxSurge: 1' <<<"$strategy" +grep -Fq ' maxUnavailable: 0' <<<"$strategy" + +grep -Fq 'name: easyai-api-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml" +grep -Fq 'name: easyai-worker-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml" +grep -Fq 'name: easyai-web-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml" +if grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=' \ + "$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"; then + echo 'Hong Kong Worker capacity remains in the single-node release environment' >&2 + exit 1 +fi echo 'cluster_release_helper_tests=PASS'