fix(acceptance): 按自适应容量执行单节点压测
修正单节点验收将 hard limit 等同空闲分配容量的问题，从冷启动容量 4 开始逐级加压，并要求实际分配容量和运行任务峰值均达到目标后才计入稳定并发。\n\n异常退出时同步清理模拟器和回调收集器资源。已执行 bash -n、ShellCheck 和 production acceptance 脚本测试。
This commit is contained in:
@@ -132,10 +132,10 @@ fi
|
||||
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=8}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=8}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=4}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=4}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS:=96}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=8 12 16 24 32 40 48}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=4 8 12 16 24 32 40 48}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}"
|
||||
: "${AI_GATEWAY_ONLINE_ACCOUNT:=}"
|
||||
@@ -383,6 +383,9 @@ cleanup() {
|
||||
if [[ $single_node_emulator_installed == true ]]; then
|
||||
cleanup_single_node_emulator >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
cleanup_single_node_kubernetes_resources >/dev/null 2>&1 || true
|
||||
fi
|
||||
if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
|
||||
set +e
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
@@ -3500,10 +3503,14 @@ apply_single_node_capacity() {
|
||||
local deadline=$((SECONDS + 90)) allocation
|
||||
while (( SECONDS < deadline )); do
|
||||
allocation=$(database_query "
|
||||
SELECT count(*)||':'||COALESCE(max(allocated_capacity),0)
|
||||
SELECT count(*)||':'||
|
||||
COALESCE(max(hard_capacity_limit),0)||':'||
|
||||
COALESCE(max(allocated_capacity),0)||':'||
|
||||
COALESCE(max(safe_capacity),0)||':'||
|
||||
COALESCE(max(pressure_state),'unknown')
|
||||
FROM gateway_worker_instances
|
||||
WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';")
|
||||
if [[ $allocation == "1:$slots" ]]; then
|
||||
if single_node_capacity_ready "$allocation" "$slots"; then
|
||||
local metrics_ready=false
|
||||
for _ in {1..30}; do
|
||||
if remote_kubectl top node easyai-ningbo --no-headers >/dev/null 2>&1 &&
|
||||
@@ -3520,15 +3527,30 @@ WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';")
|
||||
return 1
|
||||
}
|
||||
active_profile=S$slots
|
||||
echo "single_node_capacity=PASS slots=$slots worker_pool=$worker_pool active_instances=1"
|
||||
local active_instances hard_limit allocated safe_capacity pressure_state
|
||||
IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state <<<"$allocation"
|
||||
echo "single_node_capacity=PASS hard_limit=$slots allocated=$allocated safe_capacity=$safe_capacity pressure_state=$pressure_state worker_pool=$worker_pool active_instances=1"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "single-node Worker allocation did not converge: expected=1:$slots actual=${allocation:-missing}" >&2
|
||||
echo "single-node Worker capacity did not converge: expected=active:1 hard_limit:$slots allocated_and_safe:1..$slots pressure:not-critical actual=${allocation:-missing}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
single_node_capacity_ready() {
|
||||
local state=$1
|
||||
local expected_hard_limit=$2
|
||||
local active_instances hard_limit allocated safe_capacity pressure_state extra
|
||||
IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state extra <<<"$state"
|
||||
[[ -z $extra &&
|
||||
$active_instances =~ ^[0-9]+$ && $active_instances -eq 1 &&
|
||||
$hard_limit =~ ^[0-9]+$ && $hard_limit -eq $expected_hard_limit &&
|
||||
$allocated =~ ^[0-9]+$ && $allocated -ge 1 && $allocated -le $expected_hard_limit &&
|
||||
$safe_capacity =~ ^[0-9]+$ && $safe_capacity -ge 1 && $safe_capacity -le $expected_hard_limit &&
|
||||
$pressure_state != critical ]]
|
||||
}
|
||||
|
||||
install_single_node_load_driver() {
|
||||
[[ $remote_load_drivers_installed == false ]] || return 0
|
||||
local binary_path env_file remote_env local_digest remote_digest
|
||||
@@ -3676,6 +3698,17 @@ record_single_node_result() {
|
||||
load:$load[0]
|
||||
}' >"$summary"
|
||||
chmod 0600 "$summary"
|
||||
if (( peak_allocated < slots || peak_running < slots )); then
|
||||
echo "single-node target concurrency was not observed: configured=$slots peak_allocated=$peak_allocated peak_running=$peak_running" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_single_node_kubernetes_resources() {
|
||||
remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
}
|
||||
|
||||
run_single_node_profile() {
|
||||
@@ -3768,10 +3801,7 @@ FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
|
||||
admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response"
|
||||
failure_recorded=true
|
||||
cleanup_single_node_emulator
|
||||
remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
cleanup_single_node_kubernetes_resources
|
||||
}
|
||||
|
||||
run_single_node_acceptance() {
|
||||
|
||||
@@ -67,6 +67,15 @@ awk '
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/admin-request.sh"
|
||||
|
||||
awk '
|
||||
/^single_node_capacity_ready\(\)/ { capture = 1 }
|
||||
capture && /^install_single_node_load_driver\(\)/ { exit }
|
||||
capture { print }
|
||||
' "$script" >"$tmp/single-node-capacity.sh"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$tmp/single-node-capacity.sh"
|
||||
|
||||
cluster_root=$tmp/repository
|
||||
git init -q "$cluster_root"
|
||||
git -C "$cluster_root" config user.name 'Acceptance Script Test'
|
||||
@@ -216,6 +225,23 @@ admin_request POST /api/admin/system/acceptance/runs/test-run/finish "$admin_bod
|
||||
[[ $(jq -r '.status' "$admin_output") == failed ]]
|
||||
[[ $(<"$admin_attempts") == 3 ]]
|
||||
|
||||
single_node_capacity_ready '1:8:4:4:normal' 8
|
||||
single_node_capacity_ready '1:8:8:8:busy' 8
|
||||
for invalid_capacity_state in \
|
||||
'0:8:4:4:normal' \
|
||||
'1:7:4:4:normal' \
|
||||
'1:8:0:4:normal' \
|
||||
'1:8:9:4:normal' \
|
||||
'1:8:4:0:normal' \
|
||||
'1:8:4:9:normal' \
|
||||
'1:8:4:4:critical' \
|
||||
'1:8:4:4:normal:extra'; do
|
||||
if single_node_capacity_ready "$invalid_capacity_state" 8; then
|
||||
echo "unsafe single-node capacity state passed: $invalid_capacity_state" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
passing_pressure_row='2026-08-01T00:00:00Z,0,0,899,149,200,84,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100'
|
||||
pressure_row_passes_live_gates "$passing_pressure_row"
|
||||
[[ $(pressure_row_live_gate_reason '2026-08-01T00:00:00Z,0,0,900,149,200,79,1535,0,32,32,0,0,0,0,2,48,0,8,8,0,0,0,16,16,0,0,1535,100') == \
|
||||
|
||||
Reference in New Issue
Block a user