feat(deploy): 切换宁波单节点生产拓扑

移除已停用的香港 API、Web、Worker 和边缘 Service 渲染结果,将 Worker 与容量控制器固定到宁波单实例,并在发布成功后清理旧香港资源。\n\n容量控制器忽略最大副本为零的站点,集群环境允许不再声明香港主机;单节点 production acceptance 不再依赖香港 Service、Deployment 或 SSH 回退。\n\n验证:Go 全量测试与 go vet;迁移安全检查;Kustomize 单节点渲染;ShellCheck;cluster release、production acceptance 和 manual release 测试。
This commit is contained in:
2026-08-03 09:59:34 +08:00
parent 5e9a7888ae
commit 4ff85d96e0
12 changed files with 156 additions and 125 deletions
@@ -140,12 +140,15 @@ func (controller *Controller) reconcile(ctx context.Context) error {
sites := make([]SiteResources, 0, 2) sites := make([]SiteResources, 0, 2)
kubernetesStates := make(map[string]KubernetesSiteState, 2) kubernetesStates := make(map[string]KubernetesSiteState, 2)
for _, site := range []string{"ningbo", "hongkong"} { for _, site := range []string{"ningbo", "hongkong"} {
minReplicas, maxReplicas := controller.siteReplicaBounds(site)
if maxReplicas == 0 {
continue
}
state, stateErr := controller.kubernetes.SiteState(ctx, site) state, stateErr := controller.kubernetes.SiteState(ctx, site)
if stateErr != nil { if stateErr != nil {
return stateErr return stateErr
} }
kubernetesStates[site] = state kubernetesStates[site] = state
minReplicas, maxReplicas := controller.siteReplicaBounds(site)
sites = append(sites, SiteResources{ sites = append(sites, SiteResources{
Site: site, CurrentReplicas: state.CurrentReplicas, Site: site, CurrentReplicas: state.CurrentReplicas,
MinReplicas: minReplicas, MaxReplicas: maxReplicas, MinReplicas: minReplicas, MaxReplicas: maxReplicas,
+3 -3
View File
@@ -198,11 +198,11 @@ func Load() Config {
) != "", ) != "",
WorkerAutoscalingEnabled: env("AI_GATEWAY_WORKER_AUTOSCALING_ENABLED", "false") == "true", WorkerAutoscalingEnabled: env("AI_GATEWAY_WORKER_AUTOSCALING_ENABLED", "false") == "true",
WorkerReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_NINGBO", 1), WorkerReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_NINGBO", 1),
WorkerReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", 1), WorkerReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", 0),
WorkerMinReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO", 1), WorkerMinReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO", 1),
WorkerMinReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", 1), WorkerMinReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", 0),
WorkerMaxReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO", 1), WorkerMaxReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO", 1),
WorkerMaxReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", 1), WorkerMaxReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", 0),
WorkerTargetOutstandingPerReplica: envOptionalIntValidated("AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA", 0), WorkerTargetOutstandingPerReplica: envOptionalIntValidated("AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA", 0),
WorkerScaleUpWindowSeconds: envIntValidated("AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS", 20), WorkerScaleUpWindowSeconds: envIntValidated("AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS", 20),
WorkerScaleDownStabilizationSeconds: envIntValidated( WorkerScaleDownStabilizationSeconds: envIntValidated(
+20
View File
@@ -217,6 +217,26 @@ func TestWorkerAutoscalingConfiguration(t *testing.T) {
} }
} }
func TestWorkerTopologyDefaultsToNingboSingleNode(t *testing.T) {
for _, name := range []string{
"AI_GATEWAY_WORKER_REPLICAS_NINGBO",
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG",
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO",
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG",
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO",
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG",
} {
t.Setenv(name, "")
}
cfg := Load()
if cfg.WorkerReplicasNingbo != 1 || cfg.WorkerMinReplicasNingbo != 1 || cfg.WorkerMaxReplicasNingbo != 1 {
t.Fatalf("Ningbo Worker defaults=%d/%d/%d, want 1/1/1", cfg.WorkerReplicasNingbo, cfg.WorkerMinReplicasNingbo, cfg.WorkerMaxReplicasNingbo)
}
if cfg.WorkerReplicasHongkong != 0 || cfg.WorkerMinReplicasHongkong != 0 || cfg.WorkerMaxReplicasHongkong != 0 {
t.Fatalf("Hong Kong Worker defaults=%d/%d/%d, want disabled", cfg.WorkerReplicasHongkong, cfg.WorkerMinReplicasHongkong, cfg.WorkerMaxReplicasHongkong)
}
}
func TestLoadPrefersWorkerGlobalHardLimitAlias(t *testing.T) { func TestLoadPrefersWorkerGlobalHardLimitAlias(t *testing.T) {
t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "48") t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "48")
t.Setenv("AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT", "96") t.Setenv("AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT", "96")
@@ -15,8 +15,7 @@ source "$config_file"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}" : "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}" : "${K3S_BIN:=/usr/local/bin/k3s}"
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}" : "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=0}" : "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}" : "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}" : "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}" : "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
@@ -33,10 +32,8 @@ source "$config_file"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}" : "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}" : "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}" : "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=0}" : "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}" : "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=0}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}" : "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}" : "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}" : "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
@@ -45,7 +42,7 @@ source "$config_file"
: "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}" : "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}"
: "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}" : "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}"
: "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}" : "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}"
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}" : "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS + 4))}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || { [[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
echo 'invalid cluster release configuration' >&2 echo 'invalid cluster release configuration' >&2
exit 1 exit 1
@@ -58,11 +55,8 @@ source "$config_file"
} }
for replica_value in \ for replica_value in \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \ "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO"; do
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG"; do
[[ $replica_value =~ ^[0-9]+$ ]] || { [[ $replica_value =~ ^[0-9]+$ ]] || {
echo 'Worker replica configuration must contain non-negative integers' >&2 echo 'Worker replica configuration must contain non-negative integers' >&2
exit 1 exit 1
@@ -110,8 +104,7 @@ done
exit 1 exit 1
} }
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 && (( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 && AI_GATEWAY_WORKER_REPLICAS_NINGBO >= 1 &&
AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG >= 1 &&
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 && AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 && AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 && AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 &&
@@ -133,17 +126,14 @@ done
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 && AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 &&
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO && AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO && AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 && AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT && AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT &&
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 && AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 &&
AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 && AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 &&
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 && AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET && AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET + AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET +
(AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) * AI_GATEWAY_WORKER_REPLICAS_NINGBO *
AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || { AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || {
echo 'worker capacity configuration exceeds the release safety bounds' >&2 echo 'worker capacity configuration exceeds the release safety bounds' >&2
exit 1 exit 1
@@ -169,7 +159,6 @@ manifest_get() {
capacity_config_hash() { capacity_config_hash() {
printf '%s\n' \ printf '%s\n' \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \ "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
@@ -187,9 +176,7 @@ capacity_config_hash() {
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \ "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \ "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \ "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \ "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
@@ -207,11 +194,8 @@ record_capacity_config() {
local runtime_patch local runtime_patch
runtime_patch=$(jq -nc \ runtime_patch=$(jq -nc \
--arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ --arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
--arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
--arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ --arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
--arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
--arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ --arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
--arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
--arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ --arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
--arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \ --arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
--arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \ --arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
@@ -229,11 +213,11 @@ record_capacity_config() {
--arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \ --arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
'{data:{ '{data:{
AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo, AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo,
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong, AI_GATEWAY_WORKER_REPLICAS_HONGKONG:null,
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo, AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo,
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong, AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:null,
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo, AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo,
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong, AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:null,
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling, AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling,
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit, AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit,
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit, AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit,
@@ -259,8 +243,8 @@ record_capacity_config() {
"${kubectl[@]}" annotate namespace "$NAMESPACE" \ "${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null "easyai.io/config-hash=$config_hash" --overwrite >/dev/null
for deployment in \ for deployment in \
easyai-api-ningbo easyai-api-hongkong \ easyai-api-ningbo \
easyai-worker-ningbo easyai-worker-hongkong \ easyai-worker-ningbo \
easyai-capacity-controller; do easyai-capacity-controller; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \ "${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null "easyai.io/config-hash=$config_hash" --overwrite >/dev/null
@@ -299,11 +283,6 @@ verify_site() {
local api_port local api_port
local web_port local web_port
case $site in case $site in
hongkong)
address=10.77.0.2
api_port=31089
web_port=31179
;;
ningbo) ningbo)
address=10.77.0.1 address=10.77.0.1
api_port=31088 api_port=31088
@@ -316,10 +295,19 @@ verify_site() {
wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway' wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
} }
retire_hongkong_site() {
"${kubectl[@]}" delete deployment \
easyai-api-hongkong easyai-worker-hongkong easyai-web-hongkong \
-n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null
"${kubectl[@]}" delete service \
api-hongkong-edge web-hongkong-edge \
-n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null
}
apply_desired_state() { apply_desired_state() {
local api_image=$1 local api_image=$1
local web_image=$2 local web_image=$2
local capacity_controller_replicas=${3:-2} local capacity_controller_replicas=${3:-1}
local api_repository=${api_image%@*} local api_repository=${api_image%@*}
local api_digest=${api_image#*@} local api_digest=${api_image#*@}
local web_repository=${web_image%@*} local web_repository=${web_image%@*}
@@ -333,7 +321,7 @@ apply_desired_state() {
"$working_directory/kustomization.yaml" "$working_directory/kustomization.yaml"
if [[ $capacity_controller_replicas == 0 ]]; then if [[ $capacity_controller_replicas == 0 ]]; then
sed -i \ sed -i \
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 2$/ replicas: 0/' \ '/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 1$/ replicas: 0/' \
"$working_directory/application.yaml" "$working_directory/application.yaml"
fi fi
"${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \ "${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \
@@ -395,7 +383,6 @@ rollout_worker_site() {
local replicas ready_replicas local replicas ready_replicas
case $site in case $site in
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;; ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
*) return 1 ;; *) return 1 ;;
esac esac
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \ "${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
@@ -414,11 +401,8 @@ rollout_worker_site() {
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \ "AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ "AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1 "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \ "${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--containers=worker \ --containers=worker \
@@ -446,11 +430,8 @@ rollout_capacity_controller() {
"${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \ "${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \ "AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \ "AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \ "AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \ "AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \ "AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
@@ -469,7 +450,7 @@ rollout_capacity_controller() {
"capacity-controller=$api_image" || return 1 "capacity-controller=$api_image" || return 1
fi fi
"${kubectl[@]}" scale deployment/easyai-capacity-controller \ "${kubectl[@]}" scale deployment/easyai-capacity-controller \
-n "$NAMESPACE" --replicas=2 || return 1 -n "$NAMESPACE" --replicas=1 || return 1
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller \ "${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
-n "$NAMESPACE" --timeout=300s || return 1 -n "$NAMESPACE" --timeout=300s || return 1
} }
@@ -493,6 +474,7 @@ spec:
restartPolicy: Never restartPolicy: Never
serviceAccountName: easyai-capacity-controller serviceAccountName: easyai-capacity-controller
nodeSelector: nodeSelector:
easyai.io/site: ningbo
easyai.io/workload: "true" easyai.io/workload: "true"
securityContext: securityContext:
runAsNonRoot: true runAsNonRoot: true
@@ -615,20 +597,17 @@ apply_capacity_config() {
echo 'current release SHA is unavailable for capacity update' >&2 echo 'current release SHA is unavailable for capacity update' >&2
return 1 return 1
} }
rollout_worker_site hongkong ""
rollout_worker_site ningbo "" rollout_worker_site ningbo ""
rollout_api_capacity_site hongkong
rollout_api_capacity_site ningbo rollout_api_capacity_site ningbo
if [[ $capacity_controller_enabled == true ]]; then if [[ $capacity_controller_enabled == true ]]; then
rollout_capacity_controller "" rollout_capacity_controller ""
else else
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0 "${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0
fi fi
verify_site hongkong
verify_site ningbo verify_site ningbo
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
record_capacity_config record_capacity_config
echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" echo "production_capacity=PASS config_hash=$(capacity_config_hash) site=ningbo replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
} }
rollout_site() { rollout_site() {
@@ -786,10 +765,10 @@ record_current() {
"easyai.io/config-hash=$(capacity_config_hash)" \ "easyai.io/config-hash=$(capacity_config_hash)" \
--overwrite >/dev/null --overwrite >/dev/null
for deployment in \ for deployment in \
easyai-api-ningbo easyai-api-hongkong \ easyai-api-ningbo \
easyai-worker-ningbo easyai-worker-hongkong \ easyai-worker-ningbo \
easyai-capacity-controller \ easyai-capacity-controller \
easyai-web-ningbo easyai-web-hongkong; do easyai-web-ningbo; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \ "${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/release=$source_sha" \ "easyai.io/release=$source_sha" \
"easyai.io/config-hash=$(capacity_config_hash)" \ "easyai.io/config-hash=$(capacity_config_hash)" \
@@ -936,11 +915,9 @@ activate_manifest() {
fi fi
if { [[ $api_changed != true ]] || if { [[ $api_changed != true ]] ||
{ rollout_worker_site hongkong "$api_image" && { rollout_worker_site ningbo "$api_image" &&
rollout_worker_site ningbo "$api_image" &&
{ [[ $action == rollback ]] || { [[ $action == rollback ]] ||
rollout_capacity_controller "$api_image"; }; }; } && rollout_capacity_controller "$api_image"; }; }; } &&
rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" && rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
: :
@@ -953,6 +930,7 @@ activate_manifest() {
return 1 return 1
fi fi
retire_hongkong_site
record_capacity_config record_capacity_config
record_current "$manifest" "$action" "$started_at" record_current "$manifest" "$action" "$started_at"
rm -f -- "$deployment_snapshot" "$current_file" rm -f -- "$deployment_snapshot" "$current_file"
@@ -972,10 +950,8 @@ case ${1:-} in
node "$manifest_tool" validate "$2" >/dev/null node "$manifest_tool" validate "$2" >/dev/null
api_image=$(manifest_get "$2" images.api) api_image=$(manifest_get "$2" images.api)
web_image=$(manifest_get "$2" images.web) web_image=$(manifest_get "$2" images.web)
for site in ningbo hongkong; do [[ $("${kubectl[@]}" get deployment easyai-worker-ningbo -n "$NAMESPACE" \
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]] -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
done
[[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \ [[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]] -o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]]
[[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \ [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
@@ -6,8 +6,7 @@ DESIRED_STATE_DIR=/usr/local/share/easyai-ai-gateway-release/production
# Worker 容量只需修改本发布配置并执行获授权的 `capacity`,不再修改 Go 代码。 # Worker 容量只需修改本发布配置并执行获授权的 `capacity`,不再修改 Go 代码。
# 副本数属于 Deployment 配置,不能由容器内部环境变量改变;发布工具读取这两个变量并写入 replicas。 # 副本数属于 Deployment 配置,不能由容器内部环境变量改变;发布工具读取这两个变量并写入 replicas。
AI_GATEWAY_WORKER_REPLICAS_NINGBO=0 AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=2
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=8 AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=8
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
@@ -24,10 +23,8 @@ AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=0 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=0
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=2
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48 AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600
@@ -36,5 +33,5 @@ AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=75
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=85 AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=85
AI_GATEWAY_NODE_CPU_TARGET_PERCENT=70 AI_GATEWAY_NODE_CPU_TARGET_PERCENT=70
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=150 AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=150
# 2 个 API 池(2×32)加 2 个 capacity-controller 池(2×4)。 # 1 个 API 池(32)加 1 个 capacity-controller 池(4)。
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=72 AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=36
@@ -46,12 +46,9 @@ data:
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48" AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48"
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5" AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "false" AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "false"
AI_GATEWAY_WORKER_REPLICAS_NINGBO: "0" AI_GATEWAY_WORKER_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "2" AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "0" AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "0"
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "2"
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48" AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48"
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20" AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20"
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600" AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600"
+4 -17
View File
@@ -158,7 +158,7 @@ metadata:
app.kubernetes.io/component: capacity-controller app.kubernetes.io/component: capacity-controller
app.kubernetes.io/part-of: easyai-ai-gateway app.kubernetes.io/part-of: easyai-ai-gateway
spec: spec:
replicas: 2 replicas: 1
strategy: strategy:
type: RollingUpdate type: RollingUpdate
rollingUpdate: rollingUpdate:
@@ -176,22 +176,9 @@ spec:
spec: spec:
serviceAccountName: easyai-capacity-controller serviceAccountName: easyai-capacity-controller
terminationGracePeriodSeconds: 30 terminationGracePeriodSeconds: 30
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: easyai.io/site
labelSelector:
matchLabels:
app.kubernetes.io/name: easyai-capacity-controller
nodeSelector: nodeSelector:
easyai.io/site: ningbo
easyai.io/workload: "true" easyai.io/workload: "true"
topologySpreadConstraints:
- maxSkew: 1
topologyKey: easyai.io/site
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: easyai-capacity-controller
securityContext: securityContext:
runAsNonRoot: true runAsNonRoot: true
runAsUser: 10001 runAsUser: 10001
@@ -225,7 +212,7 @@ spec:
- name: AI_GATEWAY_WORKER_DATABASE_MAX_CONNS - name: AI_GATEWAY_WORKER_DATABASE_MAX_CONNS
value: "32" value: "32"
- name: AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET - name: AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET
value: "72" value: "36"
- name: POD_NAMESPACE - name: POD_NAMESPACE
valueFrom: valueFrom:
fieldRef: fieldRef:
@@ -438,7 +425,7 @@ metadata:
app.kubernetes.io/component: worker app.kubernetes.io/component: worker
app.kubernetes.io/part-of: easyai-ai-gateway app.kubernetes.io/part-of: easyai-ai-gateway
spec: spec:
replicas: 0 replicas: 1
strategy: strategy:
type: RollingUpdate type: RollingUpdate
rollingUpdate: rollingUpdate:
@@ -8,6 +8,52 @@ resources:
- application.yaml - application.yaml
- database.yaml - database.yaml
- network-policy.yaml - network-policy.yaml
patches:
- target:
kind: Deployment
name: easyai-api-hongkong
patch: |-
$patch: delete
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-api-hongkong
- target:
kind: Deployment
name: easyai-worker-hongkong
patch: |-
$patch: delete
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-worker-hongkong
- target:
kind: Deployment
name: easyai-web-hongkong
patch: |-
$patch: delete
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-web-hongkong
- target:
kind: Service
name: api-hongkong-edge
patch: |-
$patch: delete
apiVersion: v1
kind: Service
metadata:
name: api-hongkong-edge
- target:
kind: Service
name: web-hongkong-edge
patch: |-
$patch: delete
apiVersion: v1
kind: Service
metadata:
name: web-hongkong-edge
images: images:
# Deployment tooling must replace both sentinel digests with release-manifest # Deployment tooling must replace both sentinel digests with release-manifest
# digests before server-side apply. # digests before server-side apply.
@@ -48,11 +48,11 @@ metadata:
rules: rules:
- apiGroups: ["apps"] - apiGroups: ["apps"]
resources: ["deployments"] resources: ["deployments"]
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"] resourceNames: ["easyai-worker-ningbo"]
verbs: ["get"] verbs: ["get"]
- apiGroups: ["apps"] - apiGroups: ["apps"]
resources: ["deployments/scale"] resources: ["deployments/scale"]
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"] resourceNames: ["easyai-worker-ningbo"]
verbs: ["get", "update", "patch"] verbs: ["get", "update", "patch"]
- apiGroups: [""] - apiGroups: [""]
resources: ["pods"] resources: ["pods"]
+3 -4
View File
@@ -24,16 +24,14 @@ load_cluster_env() {
cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key} cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key}
: "${AI_GATEWAY_DEPLOY_HOST:?}" : "${AI_GATEWAY_DEPLOY_HOST:?}"
: "${AI_GATEWAY_DEPLOY_HOST_2:?}"
: "${AI_GATEWAY_DEPLOY_HOST_3:?}"
: "${ALI_KEY:?}" : "${ALI_KEY:?}"
: "${ALI_SECRET:?}" : "${ALI_SECRET:?}"
: "${ALI_REGION:?}" : "${ALI_REGION:?}"
: "${ALI_BUCKET:?}" : "${ALI_BUCKET:?}"
CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST} CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2} CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-${AI_GATEWAY_DEPLOY_HOST_2:-}}
CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3} CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-${AI_GATEWAY_DEPLOY_HOST_3:-}}
CLUSTER_WORKER_NODE_4_HOST=${AI_GATEWAY_WORKER_NODE_4_HOST:-} CLUSTER_WORKER_NODE_4_HOST=${AI_GATEWAY_WORKER_NODE_4_HOST:-}
CLUSTER_WORKER_NODE_4_NAME=${AI_GATEWAY_WORKER_NODE_4_NAME:-} CLUSTER_WORKER_NODE_4_NAME=${AI_GATEWAY_WORKER_NODE_4_NAME:-}
CLUSTER_WORKER_NODE_4_SITE=${AI_GATEWAY_WORKER_NODE_4_SITE:-} CLUSTER_WORKER_NODE_4_SITE=${AI_GATEWAY_WORKER_NODE_4_SITE:-}
@@ -46,6 +44,7 @@ load_cluster_env() {
export CLUSTER_WORKER_NODE_4_ENABLED export CLUSTER_WORKER_NODE_4_ENABLED
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
[[ -n $host ]] || continue
[[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || { [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
echo "cluster hosts must use root@host syntax" >&2 echo "cluster hosts must use root@host syntax" >&2
return 1 return 1
+16 -12
View File
@@ -705,18 +705,23 @@ admin_request() {
-o 'jsonpath={.spec.clusterIP}') -o 'jsonpath={.spec.clusterIP}')
service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \ service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
-o 'jsonpath={.spec.ports[0].port}') -o 'jsonpath={.spec.ports[0].port}')
fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \ [[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ ]] || {
-o 'jsonpath={.spec.clusterIP}')
fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
-o 'jsonpath={.spec.ports[0].port}')
[[ $service_ip =~ ^[0-9.]+$ && $service_port =~ ^[0-9]+$ &&
$fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || {
echo 'acceptance admin Service has an invalid cluster address' >&2 echo 'acceptance admin Service has an invalid cluster address' >&2
return 1 return 1
} }
acceptance_admin_base=http://$service_ip:$service_port acceptance_admin_base=http://$service_ip:$service_port
if [[ ${single_node_acceptance:-false} != true ]]; then
fallback_service_ip=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
-o 'jsonpath={.spec.clusterIP}')
fallback_service_port=$(remote_kubectl get service api-hongkong-edge -n "$namespace" \
-o 'jsonpath={.spec.ports[0].port}')
[[ $fallback_service_ip =~ ^[0-9.]+$ && $fallback_service_port =~ ^[0-9]+$ ]] || {
echo 'acceptance fallback admin Service has an invalid cluster address' >&2
return 1
}
acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port acceptance_admin_fallback_base=http://$fallback_service_ip:$fallback_service_port
fi fi
fi
token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" |
openssl base64 -A) openssl base64 -A)
body_encoded=$(printf '%s' "$body" | openssl base64 -A) body_encoded=$(printf '%s' "$body" | openssl base64 -A)
@@ -748,6 +753,10 @@ fi
echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2 echo "acceptance control API transport failed without safe retry: method=$method path=$path" >&2
return 1 return 1
fi fi
if [[ ${single_node_acceptance:-false} == true ]]; then
echo "acceptance control API transport failed on Ningbo: method=$method path=$path" >&2
return 1
fi
printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \ printf -v remote_command 'bash -c %q -- %q %q %q %q %q %q' \
"$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \ "$remote_script" "$acceptance_admin_fallback_base" "$method" "$path" \
"$token_encoded" "$body_encoded" "$has_body" "$token_encoded" "$body_encoded" "$has_body"
@@ -794,15 +803,13 @@ verify_release_cas() {
echo "production release CAS mismatch: expected=$release_sha current=$current_sha" >&2 echo "production release CAS mismatch: expected=$release_sha current=$current_sha" >&2
return 1 return 1
} }
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do for deployment in easyai-api-ningbo easyai-worker-ningbo; do
[[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \ [[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \
-o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]] -o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
done done
[[ $verify_capacity == true ]] || return 0 [[ $verify_capacity == true ]] || return 0
[[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \ [[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]] -o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then if [[ $CLUSTER_WORKER_NODE_4_ENABLED == true ]]; then
local expected_replicas eligible_nodes eligible_count pod_report local expected_replicas eligible_nodes eligible_count pod_report
local running_count invalid_count node4_count used_nodes local running_count invalid_count node4_count used_nodes
@@ -943,9 +950,7 @@ snapshot_pre_acceptance_capacity() {
local output=$report_root/pre-acceptance-capacity.json local output=$report_root/pre-acceptance-capacity.json
remote_kubectl get \ remote_kubectl get \
deployment/easyai-api-ningbo \ deployment/easyai-api-ningbo \
deployment/easyai-api-hongkong \
deployment/easyai-worker-ningbo \ deployment/easyai-worker-ningbo \
deployment/easyai-worker-hongkong \
deployment/easyai-capacity-controller \ deployment/easyai-capacity-controller \
configmap/easyai-ai-gateway-config \ configmap/easyai-ai-gateway-config \
configmap/easyai-gateway-release \ configmap/easyai-gateway-release \
@@ -3466,7 +3471,6 @@ apply_single_node_capacity() {
}}') }}')
remote_kubectl label node easyai-ningbo easyai.io/worker=true --overwrite >/dev/null remote_kubectl label node easyai-ningbo easyai.io/worker=true --overwrite >/dev/null
remote_kubectl scale deployment/easyai-capacity-controller -n "$namespace" --replicas=0 >/dev/null remote_kubectl scale deployment/easyai-capacity-controller -n "$namespace" --replicas=0 >/dev/null
remote_kubectl scale deployment/easyai-worker-hongkong -n "$namespace" --replicas=0 >/dev/null
remote_kubectl patch configmap easyai-ai-gateway-config -n "$namespace" \ remote_kubectl patch configmap easyai-ai-gateway-config -n "$namespace" \
--type=merge -p "$config_patch" >/dev/null --type=merge -p "$config_patch" >/dev/null
remote_kubectl set env deployment/easyai-worker-ningbo -n "$namespace" \ remote_kubectl set env deployment/easyai-worker-ningbo -n "$namespace" \
+16 -14
View File
@@ -41,7 +41,6 @@ kubectl=("$tmp/kubectl")
NAMESPACE=easyai NAMESPACE=easyai
ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48 AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48 AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
@@ -55,9 +54,7 @@ AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24 AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48 AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500 AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512 AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
@@ -66,7 +63,7 @@ grep -Fq "\"AI_GATEWAY_REVISION=\${ACTIVE_RELEASE_SHA:-}\"" "$helper"
# activate_manifest invokes this function inside an && condition. A failed # activate_manifest invokes this function inside an && condition. A failed
# rollout must still propagate instead of falling through to a stale Ready Pod. # rollout must still propagate instead of falling through to a stale Ready Pod.
if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then if rollout_worker_site ningbo 'registry.example.invalid/api@sha256:deadbeef' && true; then
echo 'failed Worker rollout was reported as successful' >&2 echo 'failed Worker rollout was reported as successful' >&2
exit 1 exit 1
fi fi
@@ -80,16 +77,21 @@ deployment_strategy() {
' "$application" ' "$application"
} }
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do strategy=$(deployment_strategy easyai-worker-ningbo)
strategy=$(deployment_strategy "$deployment") grep -Fq ' maxSurge: 0' <<<"$strategy"
grep -Fq ' maxSurge: 0' <<<"$strategy" grep -Fq ' maxUnavailable: 1' <<<"$strategy"
grep -Fq ' maxUnavailable: 1' <<<"$strategy"
done
for deployment in easyai-api-ningbo easyai-api-hongkong; do strategy=$(deployment_strategy easyai-api-ningbo)
strategy=$(deployment_strategy "$deployment") grep -Fq ' maxSurge: 1' <<<"$strategy"
grep -Fq ' maxSurge: 1' <<<"$strategy" grep -Fq ' maxUnavailable: 0' <<<"$strategy"
grep -Fq ' maxUnavailable: 0' <<<"$strategy"
done grep -Fq 'name: easyai-api-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
grep -Fq 'name: easyai-worker-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
grep -Fq 'name: easyai-web-hongkong' "$root/deploy/kubernetes/production/kustomization.yaml"
if grep -Fq 'AI_GATEWAY_WORKER_REPLICAS_HONGKONG=' \
"$root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example"; then
echo 'Hong Kong Worker capacity remains in the single-node release environment' >&2
exit 1
fi
echo 'cluster_release_helper_tests=PASS' echo 'cluster_release_helper_tests=PASS'