BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(acceptance): 配置化资源预热门槛

Browse Source 
宁波节点混部 legacy 生产服务，固定 65% 预热门槛无法反映当前可用资源，也无法通过停止 staging/CI 实质改善。

预热内存门槛现在可在 50–79% 显式配置并写入报告；80%运行目标、85%硬上限和 15 分钟稳定窗口保持不变。

验证：bash -n、ShellCheck、验收报告测试。
This commit is contained in:
wangbo
2026-08-01 00:15:05 +08:00
parent 3a01a5c93e
commit 9fd9c267d6
2 changed files with 17 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cluster/run-production-acceptance.sh
+14 -4
View File
@@ -42,6 +42,7 @@ Optional model overrides (otherwise selected from current production candidates)
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME
AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS
AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY
AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT
EOF
}
@@ -88,6 +89,7 @@ require_commands curl git go jq node openssl sed
: "${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:=32}"
: "${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB:=1536}"
: "${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES:=500}"
: "${AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT:=65}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
@@ -138,6 +140,12 @@ fi
echo 'AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES must be between 100 and 1900' >&2
exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -ge 50 &&
$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -lt 80 ]] || {
echo 'AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT must be between 50 and 79' >&2
exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
@@ -641,18 +649,20 @@ verify_resource_preconditions() {
echo 'Los Angeles witness node is carrying a Worker, controller, or acceptance emulator' >&2
return 1
}
echo 'timestamp,node,memory_percent' >"$report"
echo 'timestamp,node,memory_percent,precondition_percent' >"$report"
local sample node memory_percent
for sample in $(seq 1 90); do
while read -r node; do
[[ -n $node ]] || continue
memory_percent=$(remote_kubectl top node "$node" --no-headers |
awk '{value=$5; sub(/%$/, "", value); print value}')
[[ $memory_percent =~ ^[0-9]+$ && $memory_percent -lt 65 ]] || {
echo "Ningbo memory must remain below 65% for 15 minutes before acceptance: node=$node memory_percent=${memory_percent:-unknown}" >&2
[[ $memory_percent =~ ^[0-9]+$ &&
$memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
return 1
}
printf '%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" "$memory_percent" >>"$report"
printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
"$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
done <<<"$ningbo_nodes"
(( sample == 90 )) || sleep 10
done