fix(acceptance): 配置化资源预热门槛

宁波节点混部 legacy 生产服务,固定 65% 预热门槛无法反映当前可用资源,也无法通过停止 staging/CI 实质改善。

预热内存门槛现在可在 50–79% 显式配置并写入报告;80%运行目标、85%硬上限和 15 分钟稳定窗口保持不变。

验证:bash -n、ShellCheck、验收报告测试。
This commit is contained in:
2026-08-01 00:15:05 +08:00
parent 3a01a5c93e
commit 9fd9c267d6
2 changed files with 17 additions and 4 deletions
+3
View File
@@ -126,6 +126,9 @@ PNG 并通过验收 Key 上传到 Gateway。生产快照默认在已部署的 AP
编辑模型,以及 `omni_video.max_images >= 9` 的视频模型并优先 Seedance 2.0/fast。
验收组按 release SHA 隔离;历史 Run 的用户和分片不会被清理或复用到新 release,本次需要的
分片及其 API Key 会幂等迁移到当前 release 的专属组。
宁波 15 分钟预热内存门槛默认 65%,可用
`AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT` 在 50–79% 之间显式调整;报告必须记录
实际门槛,运行期 80%目标和 85%硬门禁不随之放宽。
为保证请求严格各有一半命中宁波和香港,可以把两个 Gateway URL 配成两个节点的
`https://<节点IP>`,并设置
+14 -4
View File
@@ -42,6 +42,7 @@ Optional model overrides (otherwise selected from current production candidates)
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME
AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS
AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY
AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT
EOF
}
@@ -88,6 +89,7 @@ require_commands curl git go jq node openssl sed
: "${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:=32}"
: "${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB:=1536}"
: "${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES:=500}"
: "${AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT:=65}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
@@ -138,6 +140,12 @@ fi
echo 'AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES must be between 100 and 1900' >&2
exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -ge 50 &&
$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -lt 80 ]] || {
echo 'AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT must be between 50 and 79' >&2
exit 1
}
[[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
@@ -641,18 +649,20 @@ verify_resource_preconditions() {
echo 'Los Angeles witness node is carrying a Worker, controller, or acceptance emulator' >&2
return 1
}
echo 'timestamp,node,memory_percent' >"$report"
echo 'timestamp,node,memory_percent,precondition_percent' >"$report"
local sample node memory_percent
for sample in $(seq 1 90); do
while read -r node; do
[[ -n $node ]] || continue
memory_percent=$(remote_kubectl top node "$node" --no-headers |
awk '{value=$5; sub(/%$/, "", value); print value}')
[[ $memory_percent =~ ^[0-9]+$ && $memory_percent -lt 65 ]] || {
echo "Ningbo memory must remain below 65% for 15 minutes before acceptance: node=$node memory_percent=${memory_percent:-unknown}" >&2
[[ $memory_percent =~ ^[0-9]+$ &&
$memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
return 1
}
printf '%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" "$memory_percent" >>"$report"
printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
"$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
done <<<"$ningbo_nodes"
(( sample == 90 )) || sleep 10
done