From 9fd9c267d6d8bc5da55332ec4c212731ac60e84b Mon Sep 17 00:00:00 2001
From: wangbo <wangbolhr@163.com>
Date: Sat, 1 Aug 2026 00:15:05 +0800
Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E9=85=8D=E7=BD=AE=E5=8C=96?=
 =?UTF-8?q?=E8=B5=84=E6=BA=90=E9=A2=84=E7=83=AD=E9=97=A8=E6=A7=9B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

宁波节点混部 legacy 生产服务，固定 65% 预热门槛无法反映当前可用资源，也无法通过停止 staging/CI 实质改善。

预热内存门槛现在可在 50–79% 显式配置并写入报告；80%运行目标、85%硬上限和 15 分钟稳定窗口保持不变。

验证：bash -n、ShellCheck、验收报告测试。
---
 docs/operations/production-acceptance.md     |  3 +++
 scripts/cluster/run-production-acceptance.sh | 18 ++++++++++++++----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md
index 1589514..bc0221c 100644
--- a/docs/operations/production-acceptance.md
+++ b/docs/operations/production-acceptance.md
@@ -126,6 +126,9 @@ PNG 并通过验收 Key 上传到 Gateway。生产快照默认在已部署的 AP
 编辑模型，以及 `omni_video.max_images >= 9` 的视频模型并优先 Seedance 2.0/fast。
 验收组按 release SHA 隔离；历史 Run 的用户和分片不会被清理或复用到新 release，本次需要的
 分片及其 API Key 会幂等迁移到当前 release 的专属组。
+宁波 15 分钟预热内存门槛默认 65%，可用
+`AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT` 在 50–79% 之间显式调整；报告必须记录
+实际门槛，运行期 80%目标和 85%硬门禁不随之放宽。
 
 为保证请求严格各有一半命中宁波和香港，可以把两个 Gateway URL 配成两个节点的
 `https://<节点IP>`，并设置
diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh
index fb9546b..72a22af 100755
--- a/scripts/cluster/run-production-acceptance.sh
+++ b/scripts/cluster/run-production-acceptance.sh
@@ -42,6 +42,7 @@ Optional model overrides (otherwise selected from current production candidates)
   AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME
   AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS
   AI_GATEWAY_ACCEPTANCE_API_MEDIA_REQUEST_CONCURRENCY
+  AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT
 EOF
 }
 
@@ -88,6 +89,7 @@ require_commands curl git go jq node openssl sed
 : "${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:=32}"
 : "${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB:=1536}"
 : "${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES:=500}"
+: "${AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT:=65}"
 : "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO:=2}"
 : "${AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG:=2}"
 : "${AI_GATEWAY_ACCEPTANCE_SOAK_DURATION:=2h}"
@@ -138,6 +140,12 @@ fi
   echo 'AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES must be between 100 and 1900' >&2
   exit 1
 }
+[[ $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT =~ ^[1-9][0-9]*$ &&
+   $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -ge 50 &&
+   $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT -lt 80 ]] || {
+  echo 'AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT must be between 50 and 79' >&2
+  exit 1
+}
 [[ $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[1-9][0-9]*$ &&
    $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[1-9][0-9]*$ &&
    $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
@@ -641,18 +649,20 @@ verify_resource_preconditions() {
     echo 'Los Angeles witness node is carrying a Worker, controller, or acceptance emulator' >&2
     return 1
   }
-  echo 'timestamp,node,memory_percent' >"$report"
+  echo 'timestamp,node,memory_percent,precondition_percent' >"$report"
   local sample node memory_percent
   for sample in $(seq 1 90); do
     while read -r node; do
       [[ -n $node ]] || continue
       memory_percent=$(remote_kubectl top node "$node" --no-headers |
         awk '{value=$5; sub(/%$/, "", value); print value}')
-      [[ $memory_percent =~ ^[0-9]+$ && $memory_percent -lt 65 ]] || {
-        echo "Ningbo memory must remain below 65% for 15 minutes before acceptance: node=$node memory_percent=${memory_percent:-unknown}" >&2
+      [[ $memory_percent =~ ^[0-9]+$ &&
+         $memory_percent -lt $AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT ]] || {
+        echo "Ningbo memory precondition failed: node=$node memory_percent=${memory_percent:-unknown} limit_percent=$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT required_window_minutes=15" >&2
         return 1
       }
-      printf '%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" "$memory_percent" >>"$report"
+      printf '%s,%s,%s,%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$node" \
+        "$memory_percent" "$AI_GATEWAY_ACCEPTANCE_NODE_MEMORY_PRECONDITION_PERCENT" >>"$report"
     done <<<"$ningbo_nodes"
     (( sample == 90 )) || sleep 10
   done