feat(deploy): 切换宁波单节点生产拓扑
移除已停用的香港 API、Web、Worker 和边缘 Service 渲染结果,将 Worker 与容量控制器固定到宁波单实例,并在发布成功后清理旧香港资源。\n\n容量控制器忽略最大副本为零的站点,集群环境允许不再声明香港主机;单节点 production acceptance 不再依赖香港 Service、Deployment 或 SSH 回退。\n\n验证:Go 全量测试与 go vet;迁移安全检查;Kustomize 单节点渲染;ShellCheck;cluster release、production acceptance 和 manual release 测试。
This commit is contained in:
@@ -140,12 +140,15 @@ func (controller *Controller) reconcile(ctx context.Context) error {
|
||||
sites := make([]SiteResources, 0, 2)
|
||||
kubernetesStates := make(map[string]KubernetesSiteState, 2)
|
||||
for _, site := range []string{"ningbo", "hongkong"} {
|
||||
minReplicas, maxReplicas := controller.siteReplicaBounds(site)
|
||||
if maxReplicas == 0 {
|
||||
continue
|
||||
}
|
||||
state, stateErr := controller.kubernetes.SiteState(ctx, site)
|
||||
if stateErr != nil {
|
||||
return stateErr
|
||||
}
|
||||
kubernetesStates[site] = state
|
||||
minReplicas, maxReplicas := controller.siteReplicaBounds(site)
|
||||
sites = append(sites, SiteResources{
|
||||
Site: site, CurrentReplicas: state.CurrentReplicas,
|
||||
MinReplicas: minReplicas, MaxReplicas: maxReplicas,
|
||||
|
||||
@@ -198,11 +198,11 @@ func Load() Config {
|
||||
) != "",
|
||||
WorkerAutoscalingEnabled: env("AI_GATEWAY_WORKER_AUTOSCALING_ENABLED", "false") == "true",
|
||||
WorkerReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_NINGBO", 1),
|
||||
WorkerReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", 1),
|
||||
WorkerReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", 0),
|
||||
WorkerMinReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO", 1),
|
||||
WorkerMinReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", 1),
|
||||
WorkerMinReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", 0),
|
||||
WorkerMaxReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO", 1),
|
||||
WorkerMaxReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", 1),
|
||||
WorkerMaxReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", 0),
|
||||
WorkerTargetOutstandingPerReplica: envOptionalIntValidated("AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA", 0),
|
||||
WorkerScaleUpWindowSeconds: envIntValidated("AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS", 20),
|
||||
WorkerScaleDownStabilizationSeconds: envIntValidated(
|
||||
|
||||
@@ -217,6 +217,26 @@ func TestWorkerAutoscalingConfiguration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerTopologyDefaultsToNingboSingleNode(t *testing.T) {
|
||||
for _, name := range []string{
|
||||
"AI_GATEWAY_WORKER_REPLICAS_NINGBO",
|
||||
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG",
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO",
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG",
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO",
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG",
|
||||
} {
|
||||
t.Setenv(name, "")
|
||||
}
|
||||
cfg := Load()
|
||||
if cfg.WorkerReplicasNingbo != 1 || cfg.WorkerMinReplicasNingbo != 1 || cfg.WorkerMaxReplicasNingbo != 1 {
|
||||
t.Fatalf("Ningbo Worker defaults=%d/%d/%d, want 1/1/1", cfg.WorkerReplicasNingbo, cfg.WorkerMinReplicasNingbo, cfg.WorkerMaxReplicasNingbo)
|
||||
}
|
||||
if cfg.WorkerReplicasHongkong != 0 || cfg.WorkerMinReplicasHongkong != 0 || cfg.WorkerMaxReplicasHongkong != 0 {
|
||||
t.Fatalf("Hong Kong Worker defaults=%d/%d/%d, want disabled", cfg.WorkerReplicasHongkong, cfg.WorkerMinReplicasHongkong, cfg.WorkerMaxReplicasHongkong)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPrefersWorkerGlobalHardLimitAlias(t *testing.T) {
|
||||
t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "48")
|
||||
t.Setenv("AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT", "96")
|
||||
|
||||
@@ -15,8 +15,7 @@ source "$config_file"
|
||||
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
|
||||
: "${K3S_BIN:=/usr/local/bin/k3s}"
|
||||
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
|
||||
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=0}"
|
||||
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=2}"
|
||||
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
|
||||
@@ -33,10 +32,8 @@ source "$config_file"
|
||||
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
||||
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
||||
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
|
||||
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=0}"
|
||||
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
|
||||
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=0}"
|
||||
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=2}"
|
||||
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
|
||||
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
|
||||
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
|
||||
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
|
||||
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
|
||||
@@ -45,7 +42,7 @@ source "$config_file"
|
||||
: "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}"
|
||||
: "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}"
|
||||
: "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}"
|
||||
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}"
|
||||
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS + 4))}"
|
||||
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
||||
echo 'invalid cluster release configuration' >&2
|
||||
exit 1
|
||||
@@ -58,11 +55,8 @@ source "$config_file"
|
||||
}
|
||||
for replica_value in \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG"; do
|
||||
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO"; do
|
||||
[[ $replica_value =~ ^[0-9]+$ ]] || {
|
||||
echo 'Worker replica configuration must contain non-negative integers' >&2
|
||||
exit 1
|
||||
@@ -110,8 +104,7 @@ done
|
||||
exit 1
|
||||
}
|
||||
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG >= 1 &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO >= 1 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 &&
|
||||
@@ -133,17 +126,14 @@ done
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 &&
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO &&
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG &&
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 &&
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 &&
|
||||
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT &&
|
||||
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 &&
|
||||
AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 &&
|
||||
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 &&
|
||||
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET &&
|
||||
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET +
|
||||
(AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) *
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO *
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || {
|
||||
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
||||
exit 1
|
||||
@@ -169,7 +159,6 @@ manifest_get() {
|
||||
capacity_config_hash() {
|
||||
printf '%s\n' \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
|
||||
@@ -187,9 +176,7 @@ capacity_config_hash() {
|
||||
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
|
||||
"$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
||||
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
|
||||
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
|
||||
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
|
||||
@@ -207,11 +194,8 @@ record_capacity_config() {
|
||||
local runtime_patch
|
||||
runtime_patch=$(jq -nc \
|
||||
--arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
||||
--arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
||||
--arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
||||
--arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
||||
--arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
||||
--arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
||||
--arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
||||
--arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
--arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
|
||||
@@ -229,11 +213,11 @@ record_capacity_config() {
|
||||
--arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
|
||||
'{data:{
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo,
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong,
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:null,
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo,
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong,
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:null,
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo,
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong,
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:null,
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling,
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit,
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit,
|
||||
@@ -259,8 +243,8 @@ record_capacity_config() {
|
||||
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
|
||||
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
|
||||
for deployment in \
|
||||
easyai-api-ningbo easyai-api-hongkong \
|
||||
easyai-worker-ningbo easyai-worker-hongkong \
|
||||
easyai-api-ningbo \
|
||||
easyai-worker-ningbo \
|
||||
easyai-capacity-controller; do
|
||||
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
|
||||
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
|
||||
@@ -299,11 +283,6 @@ verify_site() {
|
||||
local api_port
|
||||
local web_port
|
||||
case $site in
|
||||
hongkong)
|
||||
address=10.77.0.2
|
||||
api_port=31089
|
||||
web_port=31179
|
||||
;;
|
||||
ningbo)
|
||||
address=10.77.0.1
|
||||
api_port=31088
|
||||
@@ -316,10 +295,19 @@ verify_site() {
|
||||
wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
|
||||
}
|
||||
|
||||
retire_hongkong_site() {
|
||||
"${kubectl[@]}" delete deployment \
|
||||
easyai-api-hongkong easyai-worker-hongkong easyai-web-hongkong \
|
||||
-n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null
|
||||
"${kubectl[@]}" delete service \
|
||||
api-hongkong-edge web-hongkong-edge \
|
||||
-n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null
|
||||
}
|
||||
|
||||
apply_desired_state() {
|
||||
local api_image=$1
|
||||
local web_image=$2
|
||||
local capacity_controller_replicas=${3:-2}
|
||||
local capacity_controller_replicas=${3:-1}
|
||||
local api_repository=${api_image%@*}
|
||||
local api_digest=${api_image#*@}
|
||||
local web_repository=${web_image%@*}
|
||||
@@ -333,7 +321,7 @@ apply_desired_state() {
|
||||
"$working_directory/kustomization.yaml"
|
||||
if [[ $capacity_controller_replicas == 0 ]]; then
|
||||
sed -i \
|
||||
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 2$/ replicas: 0/' \
|
||||
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 1$/ replicas: 0/' \
|
||||
"$working_directory/application.yaml"
|
||||
fi
|
||||
"${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \
|
||||
@@ -395,7 +383,6 @@ rollout_worker_site() {
|
||||
local replicas ready_replicas
|
||||
case $site in
|
||||
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
|
||||
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
@@ -414,11 +401,8 @@ rollout_worker_site() {
|
||||
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \
|
||||
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
||||
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
|
||||
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
--containers=worker \
|
||||
@@ -446,11 +430,8 @@ rollout_capacity_controller() {
|
||||
"${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
||||
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
||||
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
||||
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
|
||||
"AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
|
||||
"AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
|
||||
@@ -469,7 +450,7 @@ rollout_capacity_controller() {
|
||||
"capacity-controller=$api_image" || return 1
|
||||
fi
|
||||
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
|
||||
-n "$NAMESPACE" --replicas=2 || return 1
|
||||
-n "$NAMESPACE" --replicas=1 || return 1
|
||||
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
|
||||
-n "$NAMESPACE" --timeout=300s || return 1
|
||||
}
|
||||
@@ -493,6 +474,7 @@ spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: easyai-capacity-controller
|
||||
nodeSelector:
|
||||
easyai.io/site: ningbo
|
||||
easyai.io/workload: "true"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
@@ -615,20 +597,17 @@ apply_capacity_config() {
|
||||
echo 'current release SHA is unavailable for capacity update' >&2
|
||||
return 1
|
||||
}
|
||||
rollout_worker_site hongkong ""
|
||||
rollout_worker_site ningbo ""
|
||||
rollout_api_capacity_site hongkong
|
||||
rollout_api_capacity_site ningbo
|
||||
if [[ $capacity_controller_enabled == true ]]; then
|
||||
rollout_capacity_controller ""
|
||||
else
|
||||
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0
|
||||
fi
|
||||
verify_site hongkong
|
||||
verify_site ningbo
|
||||
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
||||
record_capacity_config
|
||||
echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
|
||||
echo "production_capacity=PASS config_hash=$(capacity_config_hash) site=ningbo replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
|
||||
}
|
||||
|
||||
rollout_site() {
|
||||
@@ -786,10 +765,10 @@ record_current() {
|
||||
"easyai.io/config-hash=$(capacity_config_hash)" \
|
||||
--overwrite >/dev/null
|
||||
for deployment in \
|
||||
easyai-api-ningbo easyai-api-hongkong \
|
||||
easyai-worker-ningbo easyai-worker-hongkong \
|
||||
easyai-api-ningbo \
|
||||
easyai-worker-ningbo \
|
||||
easyai-capacity-controller \
|
||||
easyai-web-ningbo easyai-web-hongkong; do
|
||||
easyai-web-ningbo; do
|
||||
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
|
||||
"easyai.io/release=$source_sha" \
|
||||
"easyai.io/config-hash=$(capacity_config_hash)" \
|
||||
@@ -936,11 +915,9 @@ activate_manifest() {
|
||||
fi
|
||||
|
||||
if { [[ $api_changed != true ]] ||
|
||||
{ rollout_worker_site hongkong "$api_image" &&
|
||||
rollout_worker_site ningbo "$api_image" &&
|
||||
{ rollout_worker_site ningbo "$api_image" &&
|
||||
{ [[ $action == rollback ]] ||
|
||||
rollout_capacity_controller "$api_image"; }; }; } &&
|
||||
rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
|
||||
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
|
||||
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
|
||||
:
|
||||
@@ -953,6 +930,7 @@ activate_manifest() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
retire_hongkong_site
|
||||
record_capacity_config
|
||||
record_current "$manifest" "$action" "$started_at"
|
||||
rm -f -- "$deployment_snapshot" "$current_file"
|
||||
@@ -972,10 +950,8 @@ case ${1:-} in
|
||||
node "$manifest_tool" validate "$2" >/dev/null
|
||||
api_image=$(manifest_get "$2" images.api)
|
||||
web_image=$(manifest_get "$2" images.web)
|
||||
for site in ningbo hongkong; do
|
||||
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
|
||||
done
|
||||
[[ $("${kubectl[@]}" get deployment easyai-worker-ningbo -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
|
||||
[[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]]
|
||||
[[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
|
||||
|
||||
@@ -6,8 +6,7 @@ DESIRED_STATE_DIR=/usr/local/share/easyai-ai-gateway-release/production
|
||||
|
||||
# Worker 容量只需修改本发布配置并执行获授权的 `capacity`,不再修改 Go 代码。
|
||||
# 副本数属于 Deployment 配置,不能由容器内部环境变量改变;发布工具读取这两个变量并写入 replicas。
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO=0
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=2
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
||||
AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=8
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
||||
@@ -24,10 +23,8 @@ AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=0
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=0
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=2
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
|
||||
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20
|
||||
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600
|
||||
@@ -36,5 +33,5 @@ AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=75
|
||||
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=85
|
||||
AI_GATEWAY_NODE_CPU_TARGET_PERCENT=70
|
||||
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=150
|
||||
# 2 个 API 池(2×32)加 2 个 capacity-controller 池(2×4)。
|
||||
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=72
|
||||
# 1 个 API 池(32)加 1 个 capacity-controller 池(4)。
|
||||
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=36
|
||||
|
||||
@@ -46,12 +46,9 @@ data:
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48"
|
||||
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "false"
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO: "0"
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "2"
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "0"
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG: "1"
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "0"
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "2"
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO: "1"
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "1"
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "1"
|
||||
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48"
|
||||
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20"
|
||||
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600"
|
||||
|
||||
@@ -158,7 +158,7 @@ metadata:
|
||||
app.kubernetes.io/component: capacity-controller
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
spec:
|
||||
replicas: 2
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
@@ -176,22 +176,9 @@ spec:
|
||||
spec:
|
||||
serviceAccountName: easyai-capacity-controller
|
||||
terminationGracePeriodSeconds: 30
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- topologyKey: easyai.io/site
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: easyai-capacity-controller
|
||||
nodeSelector:
|
||||
easyai.io/site: ningbo
|
||||
easyai.io/workload: "true"
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: easyai.io/site
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: easyai-capacity-controller
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
@@ -225,7 +212,7 @@ spec:
|
||||
- name: AI_GATEWAY_WORKER_DATABASE_MAX_CONNS
|
||||
value: "32"
|
||||
- name: AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET
|
||||
value: "72"
|
||||
value: "36"
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
@@ -438,7 +425,7 @@ metadata:
|
||||
app.kubernetes.io/component: worker
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
spec:
|
||||
replicas: 0
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
|
||||
@@ -8,6 +8,52 @@ resources:
|
||||
- application.yaml
|
||||
- database.yaml
|
||||
- network-policy.yaml
|
||||
patches:
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: easyai-api-hongkong
|
||||
patch: |-
|
||||
$patch: delete
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: easyai-api-hongkong
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: easyai-worker-hongkong
|
||||
patch: |-
|
||||
$patch: delete
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: easyai-worker-hongkong
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: easyai-web-hongkong
|
||||
patch: |-
|
||||
$patch: delete
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: easyai-web-hongkong
|
||||
- target:
|
||||
kind: Service
|
||||
name: api-hongkong-edge
|
||||
patch: |-
|
||||
$patch: delete
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api-hongkong-edge
|
||||
- target:
|
||||
kind: Service
|
||||
name: web-hongkong-edge
|
||||
patch: |-
|
||||
$patch: delete
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: web-hongkong-edge
|
||||
images:
|
||||
# Deployment tooling must replace both sentinel digests with release-manifest
|
||||
# digests before server-side apply.
|
||||
|
||||
@@ -48,11 +48,11 @@ metadata:
|
||||
rules:
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
|
||||
resourceNames: ["easyai-worker-ningbo"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments/scale"]
|
||||
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
|
||||
resourceNames: ["easyai-worker-ningbo"]
|
||||
verbs: ["get", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
|
||||
@@ -24,16 +24,14 @@ load_cluster_env() {
|
||||
cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key}
|
||||
|
||||
: "${AI_GATEWAY_DEPLOY_HOST:?}"
|
||||
: "${AI_GATEWAY_DEPLOY_HOST_2:?}"
|
||||
: "${AI_GATEWAY_DEPLOY_HOST_3:?}"
|
||||
: "${ALI_KEY:?}"
|
||||
: "${ALI_SECRET:?}"
|
||||
: "${ALI_REGION:?}"
|
||||
: "${ALI_BUCKET:?}"
|
||||
|
||||
CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
|
||||
CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2}
|
||||
CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3}
|
||||
CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-${AI_GATEWAY_DEPLOY_HOST_2:-}}
|
||||
CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-${AI_GATEWAY_DEPLOY_HOST_3:-}}
|
||||
CLUSTER_WORKER_NODE_4_HOST=${AI_GATEWAY_WORKER_NODE_4_HOST:-}
|
||||
CLUSTER_WORKER_NODE_4_NAME=${AI_GATEWAY_WORKER_NODE_4_NAME:-}
|
||||
CLUSTER_WORKER_NODE_4_SITE=${AI_GATEWAY_WORKER_NODE_4_SITE:-}
|
||||
@@ -46,6 +44,7 @@ load_cluster_env() {
|
||||
export CLUSTER_WORKER_NODE_4_ENABLED
|
||||
|
||||
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
|
||||
[[ -n $host ]] || continue
|
||||
[[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
|
||||
echo "cluster hosts must use root@host syntax" >&2
|
||||
return 1
|
||||
|
||||
@@ -705,17 +705,22 @@ admin_request() {
|
||||
-o 'jsonpath={.spec.clusterIP}')
|
||||
service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.ports[0].port}')
|
||||
fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.clusterIP}')
|
||||
fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.ports[0].port}')
|
||||
[[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ &&
|
||||
$fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || {
|
||||
[[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ ]] || {
|
||||
echo 'acceptance admin Service has an invalid cluster address' >&2
|
||||
return 1
|
||||
}
|
||||
acceptance_admin_base=http://$service_ip:$service_port
|
||||
acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port
|
||||
if [[ ${single_node_acceptance:-false} != true ]]; then
|
||||
fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.clusterIP}')
|
||||
fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.ports[0].port}')
|
||||
[[ $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || {
|
||||
echo 'acceptance fallback admin Service has an invalid cluster address' >&2
|
||||
return 1
|
||||
}
|
||||
acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port
|
||||
fi
|
||||
fi
|
||||
token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" |
|
||||
openssl base64 -A)
|
||||
@@ -748,6 +753,10 @@ fi
|
||||
echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ${single_node_acceptance:-false} == true ]]; then
|
||||
echo "acceptance control API transport failed on Ningbo: method=$method path=$path" >&2
|
||||
return 1
|
||||
fi
|
||||
printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
|
||||
"$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \
|
||||
"$token_encoded" "$body_encoded" "$has_body"
|
||||
@@ -794,15 +803,13 @@ verify_release_cas() {
|
||||
echo "production release CAS mismatch: expected=$release_sha current=$current_sha" >&2
|
||||
return 1
|
||||
}
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
for deployment in easyai-api-ningbo easyai-worker-ningbo; do
|
||||
[[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \
|
||||
-o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
|
||||
done
|
||||
[[ $verify_capacity == true ]] || return 0
|
||||
[[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
|
||||
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
|
||||
if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
|
||||
local expected_replicas eligible_nodes eligible_count pod_report
|
||||
local running_count invalid_count node4_count used_nodes
|
||||
@@ -943,9 +950,7 @@ snapshot_pre_acceptance_capacity() {
|
||||
local output=$report_root/pre-acceptance-capacity.json
|
||||
remote_kubectl get \
|
||||
deployment/easyai-api-ningbo \
|
||||
deployment/easyai-api-hongkong \
|
||||
deployment/easyai-worker-ningbo \
|
||||
deployment/easyai-worker-hongkong \
|
||||
deployment/easyai-capacity-controller \
|
||||
configmap/easyai-ai-gateway-config \
|
||||
configmap/easyai-gateway-release \
|
||||
@@ -3466,7 +3471,6 @@ apply_single_node_capacity() {
|
||||
}}')
|
||||
remote_kubectl label node easyai-ningbo easyai.io/worker=true --overwrite >/dev/null
|
||||
remote_kubectl scale deployment/easyai-capacity-controller -n "$namespace" --replicas=0 >/dev/null
|
||||
remote_kubectl scale deployment/easyai-worker-hongkong -n "$namespace" --replicas=0 >/dev/null
|
||||
remote_kubectl patch configmap easyai-ai-gateway-config -n "$namespace" \
|
||||
--type=merge -p "$config_patch" >/dev/null
|
||||
remote_kubectl set env deployment/easyai-worker-ningbo -n "$namespace" \
|
||||
|
||||
@@ -41,7 +41,6 @@ kubectl=("$tmp/kubectl")
|
||||
NAMESPACE=easyai
|
||||
ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
|
||||
@@ -55,9 +54,7 @@ AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
|
||||
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
|
||||
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
|
||||
@@ -66,7 +63,7 @@ grep -Fq "\"AI_GATEWAY_REVISION=\${ACTIVE_RELEASE_SHA:-}\"" "$helper"
|
||||
|
||||
# activate_manifest invokes this function inside an && condition. A failed
|
||||
# rollout must still propagate instead of falling through to a stale Ready Pod.
|
||||
if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then
|
||||
if rollout_worker_site ningbo 'registry.example.invalid/api@sha256:deadbeef' && true; then
|
||||
echo 'failed Worker rollout was reported as successful' >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -80,16 +77,21 @@ deployment_strategy() {
|
||||
' "$application"
|
||||
}
|
||||
|
||||
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
strategy=$(deployment_strategy "$deployment")
|
||||
grep -Fq ' maxSurge: 0' <<<"$strategy"
|
||||
grep -Fq ' maxUnavailable: 1' <<<"$strategy"
|
||||
done
|
||||
strategy=$(deployment_strategy easyai-worker-ningbo)
|
||||
grep -Fq ' maxSurge: 0' <<<"$strategy"
|
||||
grep -Fq ' maxUnavailable: 1' <<<"$strategy"
|
||||
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong; do
|
||||
strategy=$(deployment_strategy "$deployment")
|
||||
grep -Fq ' maxSurge: 1' <<<"$strategy"
|
||||
grep -Fq ' maxUnavailable: 0' <<<"$strategy"
|
||||
done
|
||||
strategy=$(deployment_strategy easyai-api-ningbo)
|
||||
grep -Fq ' maxSurge: 1' <<<"$strategy"
|
||||
grep -Fq ' maxUnavailable: 0' <<<"$strategy"
|
||||
|
||||
grep -Fq 'name: easyai-api-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
|
||||
grep -Fq 'name: easyai-worker-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
|
||||
grep -Fq 'name: easyai-web-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
|
||||
if grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=' \
|
||||
"$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"; then
|
||||
echo 'Hong Kong Worker capacity remains in the single-node release environment' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo 'cluster_release_helper_tests=PASS'
|
||||
|
||||
Reference in New Issue
Block a user