feat(cluster): 接入深圳专用 Worker 节点

新增深圳 K3s agent 与四节点 WireGuard 全互联接入脚本,将深圳归入香港逻辑 Worker 池并用污点限制为 Worker 专用。\n\n生产验收支持宁波 0、香港逻辑池 2 的双物理节点基线,补充副本拓扑分散、容量档位、链路巡检和零副本站点校验。\n\n已通过 Go 全量测试、go vet、govulncheck、迁移安全检查、ShellCheck、发布脚本测试、前后端 lint/test/build、Compose 与 Kubernetes 渲染校验。
This commit is contained in:
2026-08-01 09:38:10 +08:00
parent 132cda35d8
commit 70b0ffb9ae
12 changed files with 539 additions and 87 deletions
+124 -34
View File
@@ -99,6 +99,10 @@ require_commands curl git go jq node openssl sed shasum
: "${AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS:=15}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG}"
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
@@ -167,11 +171,20 @@ fi
echo 'etcd latency gates must be positive integer milliseconds' >&2
exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ &&
[[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO =~ ^[0-9]+$ &&
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO =~ ^[0-9]+$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[0-9]+$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG)) -eq 2 &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG -le 16 ]] || {
echo 'acceptance autoscaling replica caps must be between 1 and 16' >&2
echo 'acceptance requires exactly two baseline Workers and ordered 0..16 per-site autoscaling bounds' >&2
exit 1
}
@@ -263,8 +276,8 @@ image_stable_throughput=
image_admitted_throughput=
video_stable_throughput=
video_admitted_throughput=
certified_max_replicas_ningbo=1
certified_max_replicas_hongkong=1
certified_max_replicas_ningbo=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
runtime_observation_started_at=
cleanup() {
@@ -636,21 +649,45 @@ verify_release_cas() {
-o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
done
[[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
-o 'jsonpath={.spec.replicas}') == 1 ]]
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
-o 'jsonpath={.spec.replicas}') == 1 ]]
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) &&
[[ -n $CLUSTER_SHENZHEN_HOST ]]; then
local worker_node pod_count
for worker_node in easyai-hongkong easyai-shenzhen; do
pod_count=$(remote_kubectl get pods -n "$namespace" \
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \
--field-selector "spec.nodeName=$worker_node" -o json | jq \
'[.items[] | select(.status.phase=="Running")] | length')
[[ $pod_count == 1 ]] || {
echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2
return 1
}
done
fi
}
verify_resource_preconditions() {
local report=$temporary_root/resource-preconditions.csv
local ningbo_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
ningbo_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
worker_nodes=
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
worker_nodes+=$'\n'
fi
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
fi
worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u)
los_angeles_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=los-angeles' \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
[[ -n $ningbo_nodes && -n $los_angeles_nodes ]]
[[ -n $worker_nodes && -n $los_angeles_nodes ]]
invalid_mixed_workloads=$(remote_kubectl get pods -A -o json |
jq --argjson nodes "$(jq -Rn --arg value "$ningbo_nodes" '$value | split("\n") | map(select(length>0))')" '
jq --argjson nodes "$(jq -Rn --arg value "$worker_nodes" '$value | split("\n") | map(select(length>0))')" '
[.items[]
| select(.spec.nodeName as $name | $nodes | index($name))
| select(.metadata.name | test("staging|runner|(^|-)ci(-|$)"; "i"))
@@ -661,7 +698,7 @@ verify_resource_preconditions() {
(.resources.limits.cpu // "") == ""))]
| length')
[[ $invalid_mixed_workloads == 0 ]] || {
echo 'Ningbo staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2
echo 'Worker-site staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2
return 1
}
witness_workloads=$(remote_kubectl get pods -A -o json |
@@ -687,12 +724,12 @@ verify_resource_preconditions() {
awk '{value=$5; sub(/%$/, "", value); print value}')
[[ $memory_percent =~ ^[0-9]+$ &&
$memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
echo "Worker node memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
return 1
}
printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
"$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
done <<<"$ningbo_nodes"
done <<<"$worker_nodes"
(( sample == 90 )) || sleep 10
done
cp "$report" "$report_root/resource-preconditions.csv"
@@ -1440,19 +1477,27 @@ fail_acceptance_gate() {
apply_capacity_profile() {
local profile=$1
local slots pool media global
local slots pool media global baseline_replicas
local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
local min_idle=4
local max_idle_seconds=30
case $profile in
P24) slots=24; pool=32; media=24; global=48 ;;
P28) slots=28; pool=36; media=28; global=56 ;;
P32) slots=32; pool=40; media=32; global=64 ;;
P24) slots=24; pool=32; media=24 ;;
P28) slots=28; pool=36; media=28 ;;
P32) slots=32; pool=40; media=32 ;;
*) return 1 ;;
esac
baseline_replicas=$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG))
global=$((slots * baseline_replicas))
local command_text capacity_output
printf -v command_text \
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
"$slots" "$global" "$global" "$pool" "$api_pool" \
"$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" "$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS" \
"$min_idle" "$max_idle_seconds" \
@@ -1605,8 +1650,9 @@ apply_autoscaling_profile() {
local max_replicas_ningbo=${2:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO}
local max_replicas_hongkong=${3:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG}
local slots pool media global
[[ $max_replicas_ningbo =~ ^[1-9][0-9]*$ &&
$max_replicas_hongkong =~ ^[1-9][0-9]*$ ]] || return 1
[[ $max_replicas_ningbo =~ ^[0-9]+$ &&
$max_replicas_hongkong =~ ^[0-9]+$ &&
$((max_replicas_ningbo + max_replicas_hongkong)) -ge 1 ]] || return 1
(( max_replicas_ningbo <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
max_replicas_hongkong <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG )) || return 1
case $profile in
@@ -1619,7 +1665,11 @@ apply_autoscaling_profile() {
global=$((slots * (max_replicas_ningbo + max_replicas_hongkong)))
local command_text capacity_output
printf -v command_text \
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
"$max_replicas_ningbo" \
"$max_replicas_hongkong" \
"$slots" "$global" "$global" "$pool" \
@@ -1697,14 +1747,15 @@ run_autoscaling_topology_phase() {
"$(jq -r '.plan.rawDesired // 0' <<<"${status:-{}}")" \
"$(jq -r '.plan.desiredTotal // 0' <<<"${status:-{}}")" \
"$(jq -r '.plan.frozenReason // ""' <<<"${status:-{}}")" "$resource_max" >>"$observations"
if (( ningbo_replicas == 1 && hongkong_replicas == 1 )); then
if (( ningbo_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO &&
hongkong_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG )); then
drained=true
break
fi
sleep 10
done
[[ $drained == true ]] || {
echo "capacity controller did not safely drain $phase to topology 1+1" >&2
echo "capacity controller did not safely drain $phase to configured minimum topology" >&2
return 1
}
local unsafe_residue
@@ -1734,15 +1785,18 @@ run_autoscaling_acceptance() {
local report=$report_root/autoscaling.json
local observations=$report_root/autoscaling-observations.csv
local status deadline resource_max_hongkong resource_max_ningbo
local tested_1_plus_2=false tested_2_plus_2=false
local tested_south_pool_scale_out=false tested_1_plus_2=false tested_2_plus_2=false
local requested_hongkong_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG
local requested_ningbo_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO
(( requested_hongkong_max > 2 )) && requested_hongkong_max=2
(( requested_ningbo_max > 2 )) && requested_ningbo_max=2
autoscaling_phase_observed_max=2
autoscaling_phase_observed_max=$((
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO +
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
))
echo 'timestamp,phase,ningbo_replicas,hongkong_replicas,raw_desired,desired_total,frozen_reason,resource_max_total' >"$observations"
apply_autoscaling_profile "$stable_profile" 1 "$requested_hongkong_max"
apply_autoscaling_profile "$stable_profile" "$requested_ningbo_max" "$requested_hongkong_max"
deadline=$((SECONDS + 180))
while (( SECONDS < deadline )); do
status=$(capacity_controller_status || true)
@@ -1755,7 +1809,17 @@ run_autoscaling_acceptance() {
}
resource_max_hongkong=$(jq -r \
'[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
if (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG < AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
requested_hongkong_max >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
resource_max_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG )); then
run_autoscaling_topology_phase south-pool-scale-out \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" "$observations"
tested_south_pool_scale_out=true
certified_max_replicas_ningbo=0
certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
elif (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then
run_autoscaling_topology_phase 1-plus-2 1 2 "$observations"
tested_1_plus_2=true
certified_max_replicas_hongkong=2
@@ -1785,6 +1849,9 @@ run_autoscaling_acceptance() {
jq -n \
--arg profile "$stable_profile" \
--argjson baselineNingbo "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
--argjson baselineHongkong "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
--argjson testedSouthPoolScaleOut "$tested_south_pool_scale_out" \
--argjson testedOnePlusTwo "$tested_1_plus_2" \
--argjson testedTwoPlusTwo "$tested_2_plus_2" \
--argjson observedMaxReplicas "$autoscaling_phase_observed_max" \
@@ -1792,8 +1859,10 @@ run_autoscaling_acceptance() {
passed:true,
drained:true,
capacityProfile:$profile,
baseline:{ningbo:$baselineNingbo,hongkong:$baselineHongkong},
topologies:{
"1+1":true,
"1+1":($baselineNingbo == 1 and $baselineHongkong == 1),
"0+1_to_0+2":$testedSouthPoolScaleOut,
"1+2":$testedOnePlusTwo,
"2+2":$testedTwoPlusTwo
},
@@ -2002,11 +2071,17 @@ WHERE acceptance_run_id IS NULL
}
activate_certified_autoscaling() {
local deadline status resource_max
local deadline status resource_max test_min_ningbo test_min_hongkong
test_min_ningbo=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO
test_min_hongkong=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
apply_autoscaling_profile \
"$stable_profile" \
"$certified_max_replicas_ningbo" \
"$certified_max_replicas_hongkong"
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$test_min_ningbo
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$test_min_hongkong
deadline=$((SECONDS + 180))
while (( SECONDS < deadline )); do
status=$(capacity_controller_status || true)
@@ -2023,7 +2098,9 @@ activate_certified_autoscaling() {
'[.plan.sites[] | select(.site == "ningbo") | .resourceMax][0] // 0' <<<"$status")
certified_max_replicas_hongkong=$(jq -r \
'[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
(( certified_max_replicas_ningbo >= 1 && certified_max_replicas_hongkong >= 1 ))
(( certified_max_replicas_ningbo >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
certified_max_replicas_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
certified_max_replicas_ningbo + certified_max_replicas_hongkong >= 2 ))
jq -n \
--arg profile "$stable_profile" \
--argjson ningbo "$certified_max_replicas_ningbo" \
@@ -2035,9 +2112,9 @@ activate_certified_autoscaling() {
resourceMaxTotal:$total
}' >"$report_root/certified-site-capacity.json"
[[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
-o 'jsonpath={.spec.replicas}') == 1 ]]
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
-o 'jsonpath={.spec.replicas}') == 1 ]]
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
stage_capacity_profiles
verify_runtime_gates
}
@@ -2332,6 +2409,14 @@ verify_cluster_links() {
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen'
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo'
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen'
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong'
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen'
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles'
fi
}
verify_control_plane_and_recent_logs() {
@@ -2349,6 +2434,11 @@ verify_control_plane_and_recent_logs() {
[[ $error_count == 0 ]] || return 1
fi
done
if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then
error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
"journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
[[ $error_count == 0 ]] || return 1
fi
for workload in api worker; do
while read -r pod; do
error_count=$(remote_kubectl logs "$pod" -n "$namespace" "$log_window" |