feat(cluster): 接入深圳专用 Worker 节点
新增深圳 K3s agent 与四节点 WireGuard 全互联接入脚本,将深圳归入香港逻辑 Worker 池并用污点限制为 Worker 专用。\n\n生产验收支持宁波 0、香港逻辑池 2 的双物理节点基线,补充副本拓扑分散、容量档位、链路巡检和零副本站点校验。\n\n已通过 Go 全量测试、go vet、govulncheck、迁移安全检查、ShellCheck、发布脚本测试、前后端 lint/test/build、Compose 与 Kubernetes 渲染校验。
This commit is contained in:
@@ -99,6 +99,10 @@ require_commands curl git go jq node openssl sed shasum
|
||||
: "${AI_GATEWAY_ACCEPTANCE_ETCD_WAL_FSYNC_MAX_MS:=15}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO:=1}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG:=1}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG:=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
|
||||
@@ -167,11 +171,20 @@ fi
|
||||
echo 'etcd latency gates must be positive integer milliseconds' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ &&
|
||||
[[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
|
||||
$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG)) -eq 2 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
|
||||
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
|
||||
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG -le 16 ]] || {
|
||||
echo 'acceptance autoscaling replica caps must be between 1 and 16' >&2
|
||||
echo 'acceptance requires exactly two baseline Workers and ordered 0..16 per-site autoscaling bounds' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -263,8 +276,8 @@ image_stable_throughput=
|
||||
image_admitted_throughput=
|
||||
video_stable_throughput=
|
||||
video_admitted_throughput=
|
||||
certified_max_replicas_ningbo=1
|
||||
certified_max_replicas_hongkong=1
|
||||
certified_max_replicas_ningbo=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
|
||||
certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
|
||||
runtime_observation_started_at=
|
||||
|
||||
cleanup() {
|
||||
@@ -636,21 +649,45 @@ verify_release_cas() {
|
||||
-o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
|
||||
done
|
||||
[[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') == 1 ]]
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
|
||||
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') == 1 ]]
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
|
||||
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
|
||||
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) &&
|
||||
[[ -n $CLUSTER_SHENZHEN_HOST ]]; then
|
||||
local worker_node pod_count
|
||||
for worker_node in easyai-hongkong easyai-shenzhen; do
|
||||
pod_count=$(remote_kubectl get pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong' \
|
||||
--field-selector "spec.nodeName=$worker_node" -o json | jq \
|
||||
'[.items[] | select(.status.phase=="Running")] | length')
|
||||
[[ $pod_count == 1 ]] || {
|
||||
echo "Hong Kong logical Worker pool is not spread across both physical nodes: node=$worker_node running_pods=$pod_count" >&2
|
||||
return 1
|
||||
}
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
verify_resource_preconditions() {
|
||||
local report=$temporary_root/resource-preconditions.csv
|
||||
local ningbo_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
|
||||
ningbo_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
local worker_nodes los_angeles_nodes invalid_mixed_workloads witness_workloads
|
||||
worker_nodes=
|
||||
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO > 0 )); then
|
||||
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=ningbo' \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
worker_nodes+=$'\n'
|
||||
fi
|
||||
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG > 0 )); then
|
||||
worker_nodes+=$(remote_kubectl get nodes -l 'easyai.io/site=hongkong' \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
fi
|
||||
worker_nodes=$(printf '%s\n' "$worker_nodes" | sed '/^$/d' | LC_ALL=C sort -u)
|
||||
los_angeles_nodes=$(remote_kubectl get nodes -l 'easyai.io/site=los-angeles' \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
[[ -n $ningbo_nodes && -n $los_angeles_nodes ]]
|
||||
[[ -n $worker_nodes && -n $los_angeles_nodes ]]
|
||||
invalid_mixed_workloads=$(remote_kubectl get pods -A -o json |
|
||||
jq --argjson nodes "$(jq -Rn --arg value "$ningbo_nodes" '$value | split("\n") | map(select(length>0))')" '
|
||||
jq --argjson nodes "$(jq -Rn --arg value "$worker_nodes" '$value | split("\n") | map(select(length>0))')" '
|
||||
[.items[]
|
||||
| select(.spec.nodeName as $name | $nodes | index($name))
|
||||
| select(.metadata.name | test("staging|runner|(^|-)ci(-|$)"; "i"))
|
||||
@@ -661,7 +698,7 @@ verify_resource_preconditions() {
|
||||
(.resources.limits.cpu // "") == ""))]
|
||||
| length')
|
||||
[[ $invalid_mixed_workloads == 0 ]] || {
|
||||
echo 'Ningbo staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2
|
||||
echo 'Worker-site staging/CI workloads must have explicit CPU and memory requests/limits before acceptance' >&2
|
||||
return 1
|
||||
}
|
||||
witness_workloads=$(remote_kubectl get pods -A -o json |
|
||||
@@ -687,12 +724,12 @@ verify_resource_preconditions() {
|
||||
awk '{value=$5; sub(/%$/, "", value); print value}')
|
||||
[[ $memory_percent =~ ^[0-9]+$ &&
|
||||
$memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
|
||||
echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
|
||||
echo "Worker node memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
|
||||
return 1
|
||||
}
|
||||
printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
|
||||
"$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
|
||||
done <<<"$ningbo_nodes"
|
||||
done <<<"$worker_nodes"
|
||||
(( sample == 90 )) || sleep 10
|
||||
done
|
||||
cp "$report" "$report_root/resource-preconditions.csv"
|
||||
@@ -1440,19 +1477,27 @@ fail_acceptance_gate() {
|
||||
|
||||
apply_capacity_profile() {
|
||||
local profile=$1
|
||||
local slots pool media global
|
||||
local slots pool media global baseline_replicas
|
||||
local api_pool=$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS
|
||||
local min_idle=4
|
||||
local max_idle_seconds=30
|
||||
case $profile in
|
||||
P24) slots=24; pool=32; media=24; global=48 ;;
|
||||
P28) slots=28; pool=36; media=28; global=56 ;;
|
||||
P32) slots=32; pool=40; media=32; global=64 ;;
|
||||
P24) slots=24; pool=32; media=24 ;;
|
||||
P28) slots=28; pool=36; media=28 ;;
|
||||
P32) slots=32; pool=40; media=32 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
baseline_replicas=$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG))
|
||||
global=$((slots * baseline_replicas))
|
||||
local command_text capacity_output
|
||||
printf -v command_text \
|
||||
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
|
||||
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=%q AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=%q AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
|
||||
"$slots" "$global" "$global" "$pool" "$api_pool" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" "$AI_GATEWAY_ACCEPTANCE_API_RIVER_MAX_CONNS" \
|
||||
"$min_idle" "$max_idle_seconds" \
|
||||
@@ -1605,8 +1650,9 @@ apply_autoscaling_profile() {
|
||||
local max_replicas_ningbo=${2:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO}
|
||||
local max_replicas_hongkong=${3:-$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG}
|
||||
local slots pool media global
|
||||
[[ $max_replicas_ningbo =~ ^[1-9][0-9]*$ &&
|
||||
$max_replicas_hongkong =~ ^[1-9][0-9]*$ ]] || return 1
|
||||
[[ $max_replicas_ningbo =~ ^[0-9]+$ &&
|
||||
$max_replicas_hongkong =~ ^[0-9]+$ &&
|
||||
$((max_replicas_ningbo + max_replicas_hongkong)) -ge 1 ]] || return 1
|
||||
(( max_replicas_ningbo <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
|
||||
max_replicas_hongkong <= AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG )) || return 1
|
||||
case $profile in
|
||||
@@ -1619,7 +1665,11 @@ apply_autoscaling_profile() {
|
||||
global=$((slots * (max_replicas_ningbo + max_replicas_hongkong)))
|
||||
local command_text capacity_output
|
||||
printf -v command_text \
|
||||
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1 AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1 AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
|
||||
'AI_GATEWAY_WORKER_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=%q AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=%q AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=true AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20 AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600 AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600 AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=%q AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=%q AI_GATEWAY_DATABASE_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_MAX_CONNS=%q AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4 AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=%q AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4 AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30 AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=%q AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=%q AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=%q AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=%q AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=%q %q capacity' \
|
||||
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG" \
|
||||
"$max_replicas_ningbo" \
|
||||
"$max_replicas_hongkong" \
|
||||
"$slots" "$global" "$global" "$pool" \
|
||||
@@ -1697,14 +1747,15 @@ run_autoscaling_topology_phase() {
|
||||
"$(jq -r '.plan.rawDesired // 0' <<<"${status:-{}}")" \
|
||||
"$(jq -r '.plan.desiredTotal // 0' <<<"${status:-{}}")" \
|
||||
"$(jq -r '.plan.frozenReason // ""' <<<"${status:-{}}")" "$resource_max" >>"$observations"
|
||||
if (( ningbo_replicas == 1 && hongkong_replicas == 1 )); then
|
||||
if (( ningbo_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO &&
|
||||
hongkong_replicas == AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG )); then
|
||||
drained=true
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
[[ $drained == true ]] || {
|
||||
echo "capacity controller did not safely drain $phase to topology 1+1" >&2
|
||||
echo "capacity controller did not safely drain $phase to configured minimum topology" >&2
|
||||
return 1
|
||||
}
|
||||
local unsafe_residue
|
||||
@@ -1734,15 +1785,18 @@ run_autoscaling_acceptance() {
|
||||
local report=$report_root/autoscaling.json
|
||||
local observations=$report_root/autoscaling-observations.csv
|
||||
local status deadline resource_max_hongkong resource_max_ningbo
|
||||
local tested_1_plus_2=false tested_2_plus_2=false
|
||||
local tested_south_pool_scale_out=false tested_1_plus_2=false tested_2_plus_2=false
|
||||
local requested_hongkong_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG
|
||||
local requested_ningbo_max=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO
|
||||
(( requested_hongkong_max > 2 )) && requested_hongkong_max=2
|
||||
(( requested_ningbo_max > 2 )) && requested_ningbo_max=2
|
||||
autoscaling_phase_observed_max=2
|
||||
autoscaling_phase_observed_max=$((
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO +
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
|
||||
))
|
||||
echo 'timestamp,phase,ningbo_replicas,hongkong_replicas,raw_desired,desired_total,frozen_reason,resource_max_total' >"$observations"
|
||||
|
||||
apply_autoscaling_profile "$stable_profile" 1 "$requested_hongkong_max"
|
||||
apply_autoscaling_profile "$stable_profile" "$requested_ningbo_max" "$requested_hongkong_max"
|
||||
deadline=$((SECONDS + 180))
|
||||
while (( SECONDS < deadline )); do
|
||||
status=$(capacity_controller_status || true)
|
||||
@@ -1755,7 +1809,17 @@ run_autoscaling_acceptance() {
|
||||
}
|
||||
resource_max_hongkong=$(jq -r \
|
||||
'[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
|
||||
if (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then
|
||||
if (( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 0 &&
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG < AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
|
||||
requested_hongkong_max >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
|
||||
resource_max_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG )); then
|
||||
run_autoscaling_topology_phase south-pool-scale-out \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" "$observations"
|
||||
tested_south_pool_scale_out=true
|
||||
certified_max_replicas_ningbo=0
|
||||
certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
|
||||
elif (( requested_hongkong_max >= 2 && resource_max_hongkong >= 2 )); then
|
||||
run_autoscaling_topology_phase 1-plus-2 1 2 "$observations"
|
||||
tested_1_plus_2=true
|
||||
certified_max_replicas_hongkong=2
|
||||
@@ -1785,6 +1849,9 @@ run_autoscaling_acceptance() {
|
||||
|
||||
jq -n \
|
||||
--arg profile "$stable_profile" \
|
||||
--argjson baselineNingbo "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" \
|
||||
--argjson baselineHongkong "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" \
|
||||
--argjson testedSouthPoolScaleOut "$tested_south_pool_scale_out" \
|
||||
--argjson testedOnePlusTwo "$tested_1_plus_2" \
|
||||
--argjson testedTwoPlusTwo "$tested_2_plus_2" \
|
||||
--argjson observedMaxReplicas "$autoscaling_phase_observed_max" \
|
||||
@@ -1792,8 +1859,10 @@ run_autoscaling_acceptance() {
|
||||
passed:true,
|
||||
drained:true,
|
||||
capacityProfile:$profile,
|
||||
baseline:{ningbo:$baselineNingbo,hongkong:$baselineHongkong},
|
||||
topologies:{
|
||||
"1+1":true,
|
||||
"1+1":($baselineNingbo == 1 and $baselineHongkong == 1),
|
||||
"0+1_to_0+2":$testedSouthPoolScaleOut,
|
||||
"1+2":$testedOnePlusTwo,
|
||||
"2+2":$testedTwoPlusTwo
|
||||
},
|
||||
@@ -2002,11 +2071,17 @@ WHERE acceptance_run_id IS NULL
|
||||
}
|
||||
|
||||
activate_certified_autoscaling() {
|
||||
local deadline status resource_max
|
||||
local deadline status resource_max test_min_ningbo test_min_hongkong
|
||||
test_min_ningbo=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO
|
||||
test_min_hongkong=$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
|
||||
apply_autoscaling_profile \
|
||||
"$stable_profile" \
|
||||
"$certified_max_replicas_ningbo" \
|
||||
"$certified_max_replicas_hongkong"
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=$test_min_ningbo
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=$test_min_hongkong
|
||||
deadline=$((SECONDS + 180))
|
||||
while (( SECONDS < deadline )); do
|
||||
status=$(capacity_controller_status || true)
|
||||
@@ -2023,7 +2098,9 @@ activate_certified_autoscaling() {
|
||||
'[.plan.sites[] | select(.site == "ningbo") | .resourceMax][0] // 0' <<<"$status")
|
||||
certified_max_replicas_hongkong=$(jq -r \
|
||||
'[.plan.sites[] | select(.site == "hongkong") | .resourceMax][0] // 0' <<<"$status")
|
||||
(( certified_max_replicas_ningbo >= 1 && certified_max_replicas_hongkong >= 1 ))
|
||||
(( certified_max_replicas_ningbo >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
|
||||
certified_max_replicas_hongkong >= AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
|
||||
certified_max_replicas_ningbo + certified_max_replicas_hongkong >= 2 ))
|
||||
jq -n \
|
||||
--arg profile "$stable_profile" \
|
||||
--argjson ningbo "$certified_max_replicas_ningbo" \
|
||||
@@ -2035,9 +2112,9 @@ activate_certified_autoscaling() {
|
||||
resourceMaxTotal:$total
|
||||
}' >"$report_root/certified-site-capacity.json"
|
||||
[[ $(remote_kubectl get deployment easyai-worker-ningbo -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') == 1 ]]
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO" ]]
|
||||
[[ $(remote_kubectl get deployment easyai-worker-hongkong -n "$namespace" \
|
||||
-o 'jsonpath={.spec.replicas}') == 1 ]]
|
||||
-o 'jsonpath={.spec.replicas}') == "$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG" ]]
|
||||
stage_capacity_profiles
|
||||
verify_runtime_gates
|
||||
}
|
||||
@@ -2332,6 +2409,14 @@ verify_cluster_links() {
|
||||
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.1 300 'los_angeles_to_ningbo'
|
||||
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.3 300 'hongkong_to_los_angeles'
|
||||
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.2 300 'los_angeles_to_hongkong'
|
||||
if [[ -n $CLUSTER_SHENZHEN_HOST ]]; then
|
||||
verify_wireguard "$CLUSTER_NINGBO_HOST" 10.77.0.4 80 'ningbo_to_shenzhen'
|
||||
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.1 80 'shenzhen_to_ningbo'
|
||||
verify_wireguard "$CLUSTER_HONGKONG_HOST" 10.77.0.4 80 'hongkong_to_shenzhen'
|
||||
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.2 80 'shenzhen_to_hongkong'
|
||||
verify_wireguard "$CLUSTER_LOS_ANGELES_HOST" 10.77.0.4 300 'los_angeles_to_shenzhen'
|
||||
verify_wireguard "$CLUSTER_SHENZHEN_HOST" 10.77.0.3 300 'shenzhen_to_los_angeles'
|
||||
fi
|
||||
}
|
||||
|
||||
verify_control_plane_and_recent_logs() {
|
||||
@@ -2349,6 +2434,11 @@ verify_control_plane_and_recent_logs() {
|
||||
[[ $error_count == 0 ]] || return 1
|
||||
fi
|
||||
done
|
||||
if [[ -n $CLUSTER_SHENZHEN_HOST && -n $runtime_observation_started_at ]]; then
|
||||
error_count=$(cluster_ssh "$CLUSTER_SHENZHEN_HOST" \
|
||||
"journalctl -u k3s-agent --since '$runtime_observation_started_at' --no-pager 2>/dev/null | awk 'BEGIN {IGNORECASE=1} /connection refused/ || /i\/o timeout/ || /node.*not ready/ || /failed to sync/ {count++} END {print count+0}'")
|
||||
[[ $error_count == 0 ]] || return 1
|
||||
fi
|
||||
for workload in api worker; do
|
||||
while read -r pod; do
|
||||
error_count=$(remote_kubectl logs "$pod" -n "$namespace" "$log_window" |
|
||||
|
||||
Reference in New Issue
Block a user