BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(worker): 拆分准入事务并收敛验收失败状态

Browse Source 
将批量准入改为每任务独立事务，避免 P24 压测时 48 条任务共享长事务造成 transactionid 锁潮与 Worker 槽位空转。失败验收在保持 validation 的同时清理未提交任务，并记录精确压力门禁原因。统一生产 0+2 Worker 拓扑、运行时 ConfigMap 与发布配置，补充镜像预检诊断和回归测试。\n\n验证：Go 全量测试、go vet、真实 PostgreSQL 48 任务集成测试、迁移安全检查、发布脚本测试、bash -n、ShellCheck、Compose 配置均通过。
This commit is contained in:
wangbo
2026-08-01 18:23:00 +08:00
parent 215b6b607f
commit edcbbd6a87
11 changed files with 194 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cluster/run-production-acceptance.sh
+33 -8
View File
@@ -1512,6 +1512,9 @@ mark_run_failed() {
--output "$report_root/acceptance-report.partial.json" >/dev/null || true
if admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$body" "$response"; then
failure_recorded=true
if ! wait_for_terminal_acceptance_tasks_to_drain; then
echo 'failed acceptance Run did not drain all safe tasks and outbox work' >&2
fi
fi
}
@@ -2608,15 +2611,25 @@ run_with_pressure_monitor() {
return "$load_status"
}
pressure_row_passes_live_gates() {
pressure_row_live_gate_reason() {
local pressure_row=$1
awk -F',' '
$4 > 900 || $6 <= 0 || $5 * 4 >= $6 * 3 ||
$7 >= 80 || $8 >= 1536 || $14 > 0 || $15 > 0 ||
$16 < 1 || $16 > 2 || $28 >= 1536 {exit 1}
$4 >= 900 {print "oldest_wait_seconds=" $4 " limit=900"; exit}
$6 <= 0 {print "postgres_max_connections_unavailable"; exit}
$5 * 4 >= $6 * 3 {print "postgres_connections=" $5 " limit_75_percent_of=" $6; exit}
$7 >= 80 {print "node_memory_percent=" $7 " limit=80"; exit}
$8 >= 1536 {print "api_pod_memory_mib=" $8 " limit=1536"; exit}
$14 > 0 {print "lease_renewal_failures=" $14; exit}
$15 > 0 {print "lease_lost=" $15; exit}
$16 < 1 || $16 > 2 {print "active_instances=" $16 " expected=1..2"; exit}
$28 >= 1536 {print "worker_memory_mib=" $28 " limit=1536"; exit}
' <<<"$pressure_row"
}
pressure_row_passes_live_gates() {
[[ -z $(pressure_row_live_gate_reason "$1") ]]
}
current_max_pod_memory_mib() {
remote_kubectl top pods -n "$namespace" \
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
@@ -2914,6 +2927,7 @@ WHERE status='active'
sample_pressure() {
local output=$1
local stop_file=$2
local failure_file=${3:-}
local database_state node_memory_percent pod_memory_mib worker_resources pool_state metrics
local consecutive_failures=0 missing_fields metrics_status pressure_row
echo 'timestamp,queued,running,oldest_wait_seconds,db_connections,db_max_connections,node_memory_percent,pod_memory_mib,pool_acquired_max,pool_max,pool_idle_min,empty_acquire_total,canceled_acquire_total,lease_failure_total,lease_lost_total,active_instances_min,allocated_capacity_max,critical_pool_acquired_max,critical_pool_max,critical_pool_idle_min,critical_empty_acquire_total,critical_canceled_acquire_total,river_pool_acquired_max,river_pool_max,river_pool_idle_min,river_empty_acquire_total,river_canceled_acquire_total,worker_memory_mib,worker_cpu_millicores' >"$output"
@@ -3046,7 +3060,12 @@ WHERE acceptance_run_id='$run_id'::uuid;"); then
"$pod_memory_mib" "$pool_state" "$worker_resources"
printf '%s\n' "$pressure_row" >>"$output"
if ! pressure_row_passes_live_gates "$pressure_row"; then
echo 'pressure sampling crossed a live resource or lease hard gate' >&2
local gate_reason
gate_reason=$(pressure_row_live_gate_reason "$pressure_row")
if [[ -n $failure_file ]]; then
printf '%s\n' "$gate_reason" >"$failure_file"
fi
echo "pressure sampling crossed a live resource or lease hard gate: $gate_reason" >&2
return 1
fi
sleep 5
@@ -3174,6 +3193,7 @@ run_capacity_round() {
profile_slug=$(printf '%s' "$profile" | tr '[:upper:]' '[:lower:]')
local prefix=$report_root/$profile_slug-$repetition
local pressure_report=$prefix-pressure.csv
local pressure_failure=$prefix-pressure.failure.txt
local pressure_stop=$temporary_root/pressure-stop
local pressure_pid status=0 pressure_status=0
local baseline_memory_mib
@@ -3185,7 +3205,8 @@ run_capacity_round() {
esac
baseline_memory_mib=$(current_max_pod_memory_mib)
rm -f -- "$pressure_stop"
sample_pressure "$pressure_report" "$pressure_stop" &
rm -f -- "$pressure_failure"
sample_pressure "$pressure_report" "$pressure_stop" "$pressure_failure" &
pressure_pid=$!
active_pressure_pid=$pressure_pid
run_with_pressure_monitor "$pressure_pid" \
@@ -3210,6 +3231,10 @@ run_capacity_round() {
wait "$pressure_pid" || pressure_status=$?
active_pressure_pid=
(( status == 0 && pressure_status == 0 )) || {
if [[ -s $pressure_failure ]]; then
failure_gate_id=pressure_live_gate
failure_reason=$(<"$pressure_failure")
fi
(( status != 0 )) && return "$status"
return "$pressure_status"
}
@@ -3585,8 +3610,8 @@ for profile in P24 P28 P32; do
for repetition in 1 2 3; do
if ! run_capacity_round "$profile" "$repetition"; then
profile_passed=false
failure_gate_id=capacity_round
failure_reason="$profile repetition $repetition failed"
[[ -n $failure_gate_id ]] || failure_gate_id=capacity_round
[[ -n $failure_reason ]] || failure_reason="$profile repetition $repetition failed"
break
fi
done