fix(acceptance): 按自适应容量执行单节点压测
修正单节点验收将 hard limit 等同空闲分配容量的问题,从冷启动容量 4 开始逐级加压,并要求实际分配容量和运行任务峰值均达到目标后才计入稳定并发。\n\n异常退出时同步清理模拟器和回调收集器资源。已执行 bash -n、ShellCheck 和 production acceptance 脚本测试。
This commit is contained in:
@@ -132,10 +132,10 @@ fi
|
||||
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=8}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=8}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=4}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=4}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS:=96}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=8 12 16 24 32 40 48}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=4 8 12 16 24 32 40 48}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}"
|
||||
: "${AI_GATEWAY_ONLINE_ACCOUNT:=}"
|
||||
@@ -383,6 +383,9 @@ cleanup() {
|
||||
if [[ $single_node_emulator_installed == true ]]; then
|
||||
cleanup_single_node_emulator >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
cleanup_single_node_kubernetes_resources >/dev/null 2>&1 || true
|
||||
fi
|
||||
if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
|
||||
set +e
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
@@ -3500,10 +3503,14 @@ apply_single_node_capacity() {
|
||||
local deadline=$((SECONDS + 90)) allocation
|
||||
while (( SECONDS < deadline )); do
|
||||
allocation=$(database_query "
|
||||
SELECT count(*)||':'||COALESCE(max(allocated_capacity),0)
|
||||
SELECT count(*)||':'||
|
||||
COALESCE(max(hard_capacity_limit),0)||':'||
|
||||
COALESCE(max(allocated_capacity),0)||':'||
|
||||
COALESCE(max(safe_capacity),0)||':'||
|
||||
COALESCE(max(pressure_state),'unknown')
|
||||
FROM gateway_worker_instances
|
||||
WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';")
|
||||
if [[ $allocation == "1:$slots" ]]; then
|
||||
if single_node_capacity_ready "$allocation" "$slots"; then
|
||||
local metrics_ready=false
|
||||
for _ in {1..30}; do
|
||||
if remote_kubectl top node easyai-ningbo --no-headers >/dev/null 2>&1 &&
|
||||
@@ -3520,15 +3527,30 @@ WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';")
|
||||
return 1
|
||||
}
|
||||
active_profile=S$slots
|
||||
echo "single_node_capacity=PASS slots=$slots worker_pool=$worker_pool active_instances=1"
|
||||
local active_instances hard_limit allocated safe_capacity pressure_state
|
||||
IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state <<<"$allocation"
|
||||
echo "single_node_capacity=PASS hard_limit=$slots allocated=$allocated safe_capacity=$safe_capacity pressure_state=$pressure_state worker_pool=$worker_pool active_instances=1"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "single-node Worker allocation did not converge: expected=1:$slots actual=${allocation:-missing}" >&2
|
||||
echo "single-node Worker capacity did not converge: expected=active:1 hard_limit:$slots allocated_and_safe:1..$slots pressure:not-critical actual=${allocation:-missing}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
single_node_capacity_ready() {
|
||||
local state=$1
|
||||
local expected_hard_limit=$2
|
||||
local active_instances hard_limit allocated safe_capacity pressure_state extra
|
||||
IFS=':' read -r active_instances hard_limit allocated safe_capacity pressure_state extra <<<"$state"
|
||||
[[ -z $extra &&
|
||||
$active_instances =~ ^[0-9]+$ && $active_instances -eq 1 &&
|
||||
$hard_limit =~ ^[0-9]+$ && $hard_limit -eq $expected_hard_limit &&
|
||||
$allocated =~ ^[0-9]+$ && $allocated -ge 1 && $allocated -le $expected_hard_limit &&
|
||||
$safe_capacity =~ ^[0-9]+$ && $safe_capacity -ge 1 && $safe_capacity -le $expected_hard_limit &&
|
||||
$pressure_state != critical ]]
|
||||
}
|
||||
|
||||
install_single_node_load_driver() {
|
||||
[[ $remote_load_drivers_installed == false ]] || return 0
|
||||
local binary_path env_file remote_env local_digest remote_digest
|
||||
@@ -3676,6 +3698,17 @@ record_single_node_result() {
|
||||
load:$load[0]
|
||||
}' >"$summary"
|
||||
chmod 0600 "$summary"
|
||||
if (( peak_allocated < slots || peak_running < slots )); then
|
||||
echo "single-node target concurrency was not observed: configured=$slots peak_allocated=$peak_allocated peak_running=$peak_running" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_single_node_kubernetes_resources() {
|
||||
remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
}
|
||||
|
||||
run_single_node_profile() {
|
||||
@@ -3768,10 +3801,7 @@ FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
|
||||
admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response"
|
||||
failure_recorded=true
|
||||
cleanup_single_node_emulator
|
||||
remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
cleanup_single_node_kubernetes_resources
|
||||
}
|
||||
|
||||
run_single_node_acceptance() {
|
||||
|
||||
Reference in New Issue
Block a user