fix(acceptance): 配置化资源预热门槛
宁波节点混部 legacy 生产服务,固定 65% 预热门槛无法反映当前可用资源,也无法通过停止 staging/CI 实质改善。 预热内存门槛现在可在 50–79% 显式配置并写入报告;80%运行目标、85%硬上限和 15 分钟稳定窗口保持不变。 验证:bash -n、ShellCheck、验收报告测试。
This commit is contained in:
@@ -126,6 +126,9 @@ PNG 并通过验收 Key 上传到 Gateway。生产快照默认在已部署的 AP
|
||||
编辑模型,以及 `omni_video.max_images >= 9` 的视频模型并优先 Seedance 2.0/fast。
|
||||
验收组按 release SHA 隔离;历史 Run 的用户和分片不会被清理或复用到新 release,本次需要的
|
||||
分片及其 API Key 会幂等迁移到当前 release 的专属组。
|
||||
宁波 15 分钟预热内存门槛默认 65%,可用
|
||||
`AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT` 在 50–79% 之间显式调整;报告必须记录
|
||||
实际门槛,运行期 80%目标和 85%硬门禁不随之放宽。
|
||||
|
||||
为保证请求严格各有一半命中宁波和香港,可以把两个 Gateway URL 配成两个节点的
|
||||
`https://<节点IP>`,并设置
|
||||
|
||||
@@ -42,6 +42,7 @@ Optional model overrides (otherwise selected from current production candidates)
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME
|
||||
AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS
|
||||
AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY
|
||||
AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -88,6 +89,7 @@ require_commands curl git go jq node openssl sed
|
||||
: "${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:=32}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB:=1536}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES:=500}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT:=65}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
|
||||
@@ -138,6 +140,12 @@ fi
|
||||
echo 'AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES must be between 100 and 1900' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -ge 50 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -lt 80 ]] || {
|
||||
echo 'AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT must be between 50 and 79' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
|
||||
@@ -641,18 +649,20 @@ verify_resource_preconditions() {
|
||||
echo 'Los Angeles witness node is carrying a Worker, controller, or acceptance emulator' >&2
|
||||
return 1
|
||||
}
|
||||
echo 'timestamp,node,memory_percent' >"$report"
|
||||
echo 'timestamp,node,memory_percent,precondition_percent' >"$report"
|
||||
local sample node memory_percent
|
||||
for sample in $(seq 1 90); do
|
||||
while read -r node; do
|
||||
[[ -n $node ]] || continue
|
||||
memory_percent=$(remote_kubectl top node "$node" --no-headers |
|
||||
awk '{value=$5; sub(/%$/, "", value); print value}')
|
||||
[[ $memory_percent =~ ^[0-9]+$ && $memory_percent -lt 65 ]] || {
|
||||
echo "Ningbo memory must remain below 65% for 15 minutes before acceptance: node=$node memory_percent=${memory_percent:-unknown}" >&2
|
||||
[[ $memory_percent =~ ^[0-9]+$ &&
|
||||
$memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
|
||||
echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
|
||||
return 1
|
||||
}
|
||||
printf '%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" "$memory_percent" >>"$report"
|
||||
printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
|
||||
"$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
|
||||
done <<<"$ningbo_nodes"
|
||||
(( sample == 90 )) || sleep 10
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user