BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(worker): 实现集群限流与自适应负载

Browse Source 
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
wangbo
2026-08-03 00:13:46 +08:00
parent 9a01fd4657
commit c28bf74230
52 changed files with 3700 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cluster/run-production-acceptance.sh
+556 -23
View File
@@ -15,6 +15,8 @@ Usage:
scripts/cluster/run-production-acceptance.sh \
--execute dist/releases/<SHA>.json \
--skip-local-acceptance
scripts/cluster/run-production-acceptance.sh \
--execute-single-node dist/releases/<SHA>.json
scripts/cluster/run-production-acceptance.sh \
--promote dist/releases/<SHA>.json --run-id <production-run-id>
@@ -29,6 +31,10 @@ The explicit --skip-local-acceptance form records both local stages as skipped
with a user-directed waiver. It never records them as passed and preserves all
online simulation, real canary, resource, consistency, and release CAS gates.
The --execute-single-node form is a Ningbo-only diagnostic. It keeps production
traffic in validation, routes acceptance candidates only to the protocol
emulator, never runs real-canary, and leaves the Run non-promotable.
Required private environment values:
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN, or AI_GATEWAY_ONLINE_ACCOUNT/PASSWORD
@@ -54,6 +60,7 @@ if [[ ${1:-} == --promote ]]; then
fi
skip_local_acceptance=false
single_node_acceptance=false
local_acceptance_report=
if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then
release_manifest=$2
@@ -61,6 +68,10 @@ if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then
elif [[ ${1:-} == --execute && ${3:-} == --skip-local-acceptance && $# -eq 3 ]]; then
release_manifest=$2
skip_local_acceptance=true
elif [[ ${1:-} == --execute-single-node && $# -eq 2 ]]; then
release_manifest=$2
skip_local_acceptance=true
single_node_acceptance=true
else
usage >&2
exit 64
@@ -79,6 +90,18 @@ fi
load_cluster_env
require_commands curl git go jq node openssl sed shasum
if [[ $single_node_acceptance == true ]]; then
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=1
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=0
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=1
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=0
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=1
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG=0
AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS=${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:-16}
AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB=512
AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS=30
fi
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:=}"
@@ -109,15 +132,45 @@ require_commands curl git go jq node openssl sed shasum
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=8}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=8}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS:=96}"
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=8 12 16 24 32 40 48}"
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}"
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}"
: "${AI_GATEWAY_ONLINE_ACCOUNT:=}"
: "${AI_GATEWAY_ONLINE_PASSWORD:=}"
: "${AI_GATEWAY_ONLINE_BASE_URL:=}"
if [[ -z $AI_GATEWAY_ACCEPTANCE_GATEWAYS ]]; then
AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}"
: "${AI_GATEWAY_DEPLOY_DOMAIN:?}"
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN}
if [[ $single_node_acceptance == true ]]; then
AI_GATEWAY_ACCEPTANCE_GATEWAYS='http://10.77.0.1:18089'
else
AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}"
: "${AI_GATEWAY_DEPLOY_DOMAIN:?}"
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN}
fi
fi
single_node_video_slots=()
if [[ $single_node_acceptance == true ]]; then
[[ $AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS -le 128 &&
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS -le 128 &&
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS =~ ^[1-9][0-9]*$ ]] || {
echo 'single-node baseline, GEMINI slots, and GEMINI requests must be positive bounded integers' >&2
exit 1
}
read -r -a single_node_video_slots <<<"$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS"
(( ${#single_node_video_slots[@]} > 0 )) || {
echo 'AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS must contain at least one slot count' >&2
exit 1
}
for single_node_slot in "${single_node_video_slots[@]}"; do
[[ $single_node_slot =~ ^[1-9][0-9]*$ && $single_node_slot -le 128 ]] || {
echo 'single-node video slot counts must be integers between 1 and 128' >&2
exit 1
}
done
fi
[[ $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS =~ ^[1-9][0-9]*$ &&
$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS -le 256 ]] || {
@@ -189,16 +242,27 @@ fi
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[0-9]+$ &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG)) -eq 2 &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG -le 16 ]] || {
echo 'acceptance requires exactly two baseline Workers and ordered 0..16 per-site autoscaling bounds' >&2
echo 'acceptance baseline Worker topology or per-site autoscaling bounds are invalid' >&2
exit 1
}
if [[ $single_node_acceptance == true ]]; then
[[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 1 &&
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 0 ]] || {
echo 'single-node acceptance requires one Ningbo Worker and zero Hong Kong Workers' >&2
exit 1
}
else
(( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) || {
echo 'production acceptance requires exactly two baseline Workers' >&2
exit 1
}
fi
node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
@@ -223,24 +287,30 @@ if [[ $skip_local_acceptance != true ]]; then
exit 1
}
fi
[[ -z $(git -C "$cluster_root" status --short) ]] || {
echo 'production acceptance requires a clean release working copy' >&2
exit 1
}
if [[ $single_node_acceptance != true ]]; then
[[ -z $(git -C "$cluster_root" status --short) ]] || {
echo 'production acceptance requires a clean release working copy' >&2
exit 1
}
else
echo 'single_node_acceptance_tool=working_tree diagnostic_only=true certification=false'
fi
acceptance_tool_sha=$(git -C "$cluster_root" rev-parse HEAD)
if [[ $acceptance_tool_sha != "$release_sha" ]]; then
git -C "$cluster_root" merge-base --is-ancestor "$release_sha" "$acceptance_tool_sha" || {
echo 'production release must be an ancestor of the acceptance tool HEAD' >&2
exit 1
}
acceptance_tool_delta=$(node "$cluster_root/scripts/release-components.mjs" \
"$release_sha" "$acceptance_tool_sha")
[[ $(jq -r '.components' <<<"$acceptance_tool_delta") == none &&
$(jq -r '.migrationsChanged' <<<"$acceptance_tool_delta") == false ]] || {
echo 'acceptance tool HEAD contains runtime or migration changes beyond the production release' >&2
exit 1
}
echo "acceptance_tool_delta=PASS release=$release_sha tool_sha=$acceptance_tool_sha runtime_changes=false"
if [[ $single_node_acceptance != true ]]; then
acceptance_tool_delta=$(node "$cluster_root/scripts/release-components.mjs" \
"$release_sha" "$acceptance_tool_sha")
[[ $(jq -r '.components' <<<"$acceptance_tool_delta") == none &&
$(jq -r '.migrationsChanged' <<<"$acceptance_tool_delta") == false ]] || {
echo 'acceptance tool HEAD contains runtime or migration changes beyond the production release' >&2
exit 1
}
echo "acceptance_tool_delta=PASS release=$release_sha tool_sha=$acceptance_tool_sha runtime_changes=false"
fi
fi
namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai}
@@ -262,6 +332,7 @@ run_id=
report_root=
stable_profile=P24
active_profile=P24
single_node_stable_slots=4
failure_reason=
failure_gate_id=
failure_recorded=false
@@ -272,6 +343,7 @@ acceptance_participants_json='[]'
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
acceptance_load_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-load-linux-amd64
acceptance_emulator_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-emulator-linux-amd64
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
current_production_snapshot=$temporary_root/current-production-snapshot.json
local_snapshot_config_hash=
@@ -291,6 +363,8 @@ video_admitted_throughput=
certified_max_replicas_ningbo=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
runtime_observation_started_at=
single_node_emulator_url=
single_node_emulator_installed=false
cleanup() {
local status=$?
@@ -306,9 +380,16 @@ cleanup() {
if [[ $remote_load_drivers_installed == true ]]; then
cleanup_remote_load_drivers >/dev/null 2>&1 || true
fi
if [[ $single_node_emulator_installed == true ]]; then
cleanup_single_node_emulator >/dev/null 2>&1 || true
fi
if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
set +e
apply_capacity_profile "$stable_profile" >/dev/null 2>&1
if [[ $single_node_acceptance == true ]]; then
apply_single_node_capacity "$single_node_stable_slots" >/dev/null 2>&1
else
apply_capacity_profile "$stable_profile" >/dev/null 2>&1
fi
[[ -n $failure_gate_id ]] || failure_gate_id=workflow_unexpected_exit
mark_run_failed "${failure_reason:-acceptance workflow exited unexpectedly}"
set -e
@@ -361,6 +442,10 @@ bootstrap_acceptance_admin_token
-trimpath -o "$acceptance_load_binary" ./cmd/acceptance-load
env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-trimpath -o "$acceptance_load_linux_binary" ./cmd/acceptance-load
if [[ $single_node_acceptance == true ]]; then
env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-trimpath -o "$acceptance_emulator_linux_binary" ./cmd/acceptance-emulator
fi
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
-trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
)
@@ -1386,14 +1471,93 @@ WHERE NOT EXISTS (
echo "acceptance_model_access=PASS resources=$expected"
}
cleanup_single_node_emulator() {
[[ $run_token =~ ^[0-9a-f]{64}$ ]] || return 0
local suffix=${run_token:0:12}
local binary_path=/root/easyai-acceptance-emulator-$suffix
local pid_path=$binary_path.pid
local log_path=$binary_path.log
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$binary_path" "$pid_path" "$log_path" <<'REMOTE' || true
set -euo pipefail
binary_path=$1
pid_path=$2
log_path=$3
if [[ -f $pid_path && ! -L $pid_path ]]; then
pid=$(<"$pid_path")
if [[ $pid =~ ^[0-9]+$ && -e /proc/$pid/exe && $(readlink -f "/proc/$pid/exe") == "$binary_path" ]]; then
kill -TERM "$pid"
for _ in {1..20}; do
kill -0 "$pid" >/dev/null 2>&1 || break
sleep 0.1
done
fi
fi
for path in "$binary_path" "$pid_path" "$log_path"; do
[[ ! -e $path ]] || unlink "$path"
done
REMOTE
single_node_emulator_installed=false
}
start_single_node_emulator() {
[[ $run_token =~ ^[0-9a-f]{64}$ ]]
local suffix=${run_token:0:12}
local binary_path=/root/easyai-acceptance-emulator-$suffix
local pid_path=$binary_path.pid
local log_path=$binary_path.log
local local_digest remote_digest node_ip
local_digest=$(shasum -a 256 "$acceptance_emulator_linux_binary" | awk '{print $1}')
cluster_scp "$acceptance_emulator_linux_binary" "$CLUSTER_NINGBO_HOST:$binary_path" >/dev/null
remote_digest=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "chmod 0755 '$binary_path'; sha256sum '$binary_path'" | awk '{print $1}')
[[ $remote_digest == "$local_digest" ]]
single_node_emulator_installed=true
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$binary_path" "$pid_path" "$log_path" <<'REMOTE'
set -euo pipefail
binary_path=$1
pid_path=$2
log_path=$3
if ss -ltn '( sport = :18090 )' | tail -n +2 | grep -q .; then
echo 'single-node emulator port 18090 is already in use' >&2
exit 1
fi
nohup env HTTP_ADDR=:18090 "$binary_path" </dev/null >"$log_path" 2>&1 &
pid=$!
printf '%s\n' "$pid" >"$pid_path"
chmod 0600 "$pid_path" "$log_path"
for _ in {1..50}; do
if curl -fsS --max-time 2 http://127.0.0.1:18090/healthz >/dev/null; then
exit 0
fi
kill -0 "$pid" >/dev/null 2>&1 || exit 1
sleep 0.2
done
exit 1
REMOTE
node_ip=$(remote_kubectl get node easyai-ningbo -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')
[[ $node_ip =~ ^[0-9a-fA-F:.]+$ ]]
single_node_emulator_url=http://$node_ip:18090
remote_kubectl exec -n "$namespace" deployment/easyai-api-ningbo -- \
wget -qO- "$single_node_emulator_url/healthz" >/dev/null
echo 'single_node_protocol_emulator=PASS placement=ningbo-host digest_verified=true pod_reachable=true'
}
deploy_protocol_emulator() {
sed "s|image: easyai-api|image: $api_image|" \
"$cluster_root/deploy/kubernetes/acceptance/protocol-emulator.yaml" |
cluster_ssh "$CLUSTER_NINGBO_HOST" 'k3s kubectl apply -f -' >/dev/null
if [[ $single_node_acceptance == true ]]; then
remote_kubectl patch deployment easyai-acceptance-emulator -n "$namespace" \
--type=json -p='[{"op":"replace","path":"/spec/template/spec/nodeSelector","value":{"kubernetes.io/hostname":"easyai-ningbo"}}]' >/dev/null
remote_kubectl patch deployment easyai-acceptance-callback-collector -n "$namespace" \
--type=json -p='[{"op":"replace","path":"/spec/template/spec/nodeSelector","value":{"kubernetes.io/hostname":"easyai-ningbo"}}]' >/dev/null
fi
remote_kubectl rollout status deployment/easyai-acceptance-emulator \
-n "$namespace" --timeout=300s
remote_kubectl rollout status deployment/easyai-acceptance-callback-collector \
-n "$namespace" --timeout=300s
if [[ $single_node_acceptance == true ]]; then
start_single_node_emulator
fi
}
is_release_ancestor() {
@@ -1408,7 +1572,11 @@ create_and_activate_run() {
local activate_response=$temporary_root/activate-run.json
local traffic_response=$temporary_root/pre-activate-traffic.json
local previous_run_response=$temporary_root/previous-run.json
local body
local body emulator_url='http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090'
if [[ $single_node_acceptance == true ]]; then
[[ $single_node_emulator_url == http://* ]]
emulator_url=$single_node_emulator_url
fi
body=$(jq -cn \
--arg releaseSha "$release_sha" \
--arg apiDigest "$api_digest" \
@@ -1416,7 +1584,7 @@ create_and_activate_run() {
--arg apiKeyID "$AI_GATEWAY_ACCEPTANCE_API_KEY_ID" \
--arg userID "$AI_GATEWAY_ACCEPTANCE_USER_ID" \
--arg token "$run_token" \
--arg emulatorURL 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \
--arg emulatorURL "$emulator_url" \
--arg callbackURL 'http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks' \
--argjson participants "$acceptance_participants_json" \
'{
@@ -2271,8 +2439,10 @@ cleanup_remote_load_drivers() {
hongkong_env=$(remote_load_env_path hongkong)
cluster_ssh "$CLUSTER_NINGBO_HOST" \
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$ningbo_env' ]] || unlink '$ningbo_env'" || true
cluster_ssh "$CLUSTER_HONGKONG_HOST" \
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
if [[ $single_node_acceptance != true ]]; then
cluster_ssh "$CLUSTER_HONGKONG_HOST" \
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
fi
remote_load_drivers_installed=false
}
@@ -3268,6 +3438,363 @@ certify_worker_resource_requests() {
}' >"$report_root/certified-worker-resources.json"
}
apply_single_node_capacity() {
local slots=$1
[[ $slots =~ ^[0-9]+$ ]] && (( slots >= 4 && slots <= 48 )) || return 1
local worker_pool=$((slots + 12))
local target_outstanding=$((slots * 2))
local config_patch
config_patch=$(jq -nc \
--arg slots "$slots" \
--arg workerPool "$worker_pool" \
--arg targetOutstanding "$target_outstanding" \
'{data:{
AI_GATEWAY_WORKER_REPLICAS_NINGBO:"1",
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:"0",
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:"1",
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:"0",
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:"1",
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:"0",
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$slots,
AI_GATEWAY_WORKER_DATABASE_MAX_CONNS:$workerPool,
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:$slots,
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:$slots,
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$targetOutstanding
}}')
remote_kubectl label node easyai-ningbo easyai.io/worker=true --overwrite >/dev/null
remote_kubectl scale deployment/easyai-capacity-controller -n "$namespace" --replicas=0 >/dev/null
remote_kubectl scale deployment/easyai-worker-hongkong -n "$namespace" --replicas=0 >/dev/null
remote_kubectl patch configmap easyai-ai-gateway-config -n "$namespace" \
--type=merge -p "$config_patch" >/dev/null
remote_kubectl set env deployment/easyai-worker-ningbo -n "$namespace" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$slots" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$slots" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$slots" \
"AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=$AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$worker_pool" \
'AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4' \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" \
'AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4' \
'AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30' \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$slots" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$slots" \
'AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=2' >/dev/null
remote_kubectl set resources deployment/easyai-worker-ningbo -n "$namespace" \
--containers=worker \
--requests="cpu=${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB}Mi" \
--limits='cpu=2,memory=2Gi' >/dev/null
remote_kubectl scale deployment/easyai-worker-ningbo -n "$namespace" --replicas=1 >/dev/null
remote_kubectl rollout status deployment/easyai-worker-ningbo -n "$namespace" --timeout=300s
remote_kubectl set env deployment/easyai-api-ningbo -n "$namespace" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$slots" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$slots" >/dev/null
remote_kubectl rollout status deployment/easyai-api-ningbo -n "$namespace" --timeout=300s
local deadline=$((SECONDS + 90)) allocation
while (( SECONDS < deadline )); do
allocation=$(database_query "
SELECT count(*)||':'||COALESCE(max(allocated_capacity),0)
FROM gateway_worker_instances
WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';")
if [[ $allocation == "1:$slots" ]]; then
local metrics_ready=false
for _ in {1..30}; do
if remote_kubectl top node easyai-ningbo --no-headers >/dev/null 2>&1 &&
[[ -n $(remote_kubectl top pods -n "$namespace" \
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo' \
--no-headers 2>/dev/null) ]]; then
metrics_ready=true
break
fi
sleep 2
done
[[ $metrics_ready == true ]] || {
echo 'single-node metrics did not become ready after Worker rollout' >&2
return 1
}
active_profile=S$slots
echo "single_node_capacity=PASS slots=$slots worker_pool=$worker_pool active_instances=1"
return 0
fi
sleep 2
done
echo "single-node Worker allocation did not converge: expected=1:$slots actual=${allocation:-missing}" >&2
return 1
}
install_single_node_load_driver() {
[[ $remote_load_drivers_installed == false ]] || return 0
local binary_path env_file remote_env local_digest remote_digest
binary_path=$(remote_load_binary_path)
remote_env=$(remote_load_env_path ningbo)
env_file=$temporary_root/remote-load-ningbo.env
write_remote_load_env "$env_file" "$AI_GATEWAY_ACCEPTANCE_GATEWAYS"
local_digest=$(shasum -a 256 "$acceptance_load_linux_binary" | awk '{print $1}')
cluster_scp "$acceptance_load_linux_binary" "$CLUSTER_NINGBO_HOST:$binary_path" >/dev/null
cluster_scp "$env_file" "$CLUSTER_NINGBO_HOST:$remote_env" >/dev/null
remote_digest=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \
"chmod 0755 '$binary_path'; chmod 0600 '$remote_env'; sha256sum '$binary_path'" | awk '{print $1}')
[[ $remote_digest == "$local_digest" ]]
remote_load_drivers_installed=true
echo 'acceptance_load_driver=PASS sites=1 placement=ningbo-host digest_verified=true'
}
run_single_node_load_profile() {
local profile=$1
local report_path=$2
local requests=$3
install_single_node_load_driver
local binary_path remote_env artifact remote_report status=0
binary_path=$(remote_load_binary_path)
remote_env=$(remote_load_env_path ningbo)
artifact=$(basename "$report_path")
remote_report=/root/easyai-acceptance-load-"$run_id"-"$artifact"
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- \
"$binary_path" "$remote_env" "$profile" "$remote_report" "${artifact%.json}" "$requests" \
>"$temporary_root/$artifact.stdout" <<'REMOTE' || status=$?
set -euo pipefail
binary_path=$1
env_file=$2
profile=$3
report=$4
execution_id=$5
requests=$6
[[ ! -e $report ]] || unlink "$report"
set -a
# shellcheck source=/dev/null
source "$env_file"
set +a
"$binary_path" -profile "$profile" -report "$report" \
-shard-index 0 -shard-count 1 -execution-id "$execution_id" \
-requests "$requests"
REMOTE
cluster_scp "$CLUSTER_NINGBO_HOST:$remote_report" "$report_path" >/dev/null || return 1
cluster_ssh "$CLUSTER_NINGBO_HOST" "[[ ! -e '$remote_report' ]] || unlink '$remote_report'" >/dev/null || true
chmod 0600 "$report_path"
jq -e --arg runId "$run_id" --arg profile "$profile" \
'.schemaVersion == "acceptance-load-report/v1" and .runId == $runId and .profile == $profile and .secretSafe == true' \
"$report_path" >/dev/null || return 1
(( status == 0 )) && jq -e '.passed == true' "$report_path" >/dev/null
}
sample_single_node_pressure() {
local output=$1
local stop_file=$2
local failure_file=$3
echo 'timestamp,queued,running,db_connections,db_max_connections,active_instances,allocated_capacity,node_memory_percent,worker_memory_mib,worker_cpu_millicores,restarts' >"$output"
while [[ ! -f $stop_file ]]; do
local state node_memory worker_resources restarts row
state=$(database_query "
SELECT
count(*) FILTER (WHERE status='queued')||','||
count(*) FILTER (WHERE status='running')||','||
(SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
(SELECT setting FROM pg_settings WHERE name='max_connections')||','||
(SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now()-interval '30 seconds')||','||
(SELECT COALESCE(max(allocated_capacity),0) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now()-interval '30 seconds')
FROM gateway_tasks
WHERE acceptance_run_id='$run_id'::uuid;") || return 1
node_memory=$(remote_kubectl top node easyai-ningbo --no-headers |
awk '{value=$5; sub(/%$/, "", value); print value}') || return 1
worker_resources=$(remote_kubectl top pods -n "$namespace" \
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo' --no-headers |
awk '{
cpu=$2; memory=$3
if (cpu ~ /n$/) {sub(/n$/, "", cpu); cpu/=1000000}
else if (cpu ~ /u$/) {sub(/u$/, "", cpu); cpu/=1000}
else if (cpu ~ /m$/) {sub(/m$/, "", cpu)}
else {cpu*=1000}
if (memory ~ /Gi$/) {sub(/Gi$/, "", memory); memory*=1024}
else if (memory ~ /Mi$/) {sub(/Mi$/, "", memory)}
else if (memory ~ /Ki$/) {sub(/Ki$/, "", memory); memory/=1024}
print int(memory+0) "," int(cpu+0)
}') || return 1
restarts=$(remote_kubectl get pods -n "$namespace" \
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' -o json |
jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0') || return 1
row="$(date -u '+%Y-%m-%dT%H:%M:%SZ'),$state,$node_memory,$worker_resources,$restarts"
printf '%s\n' "$row" >>"$output"
if ! awk -F',' '($5<=0)||($4*4>=$5*3)||($6!=1)||($7<1)||($8>=85)||($9>=1536)||($11>0){exit 1}' <<<"$row"; then
printf '%s\n' "$row" >"$failure_file"
return 1
fi
sleep 1
done
}
record_single_node_result() {
local workload=$1
local slots=$2
local load_report=$3
local pressure_report=$4
local summary=$5
local peaks
peaks=$(awk -F',' '
NR>1 {
samples++
if ($2>queued) queued=$2
if ($3>running) running=$3
if ($4>db) db=$4
if ($7>allocated) allocated=$7
if ($8>node_memory) node_memory=$8
if ($9>worker_memory) worker_memory=$9
if ($10>worker_cpu) worker_cpu=$10
}
END {printf "%d,%d,%d,%d,%d,%d,%d,%d",samples,queued,running,db,allocated,node_memory,worker_memory,worker_cpu}
' "$pressure_report")
IFS=',' read -r samples peak_queued peak_running peak_db peak_allocated peak_node_memory peak_worker_memory peak_worker_cpu <<<"$peaks"
jq -n \
--arg workload "$workload" \
--argjson slots "$slots" \
--argjson samples "$samples" \
--argjson peakQueued "$peak_queued" \
--argjson peakRunning "$peak_running" \
--argjson peakDatabaseConnections "$peak_db" \
--argjson peakAllocatedCapacity "$peak_allocated" \
--argjson peakNodeMemoryPercent "$peak_node_memory" \
--argjson peakWorkerMemoryMiB "$peak_worker_memory" \
--argjson peakWorkerCPUMillicores "$peak_worker_cpu" \
--slurpfile load "$load_report" \
'{
workload:$workload,
configuredSlots:$slots,
samples:$samples,
peakQueued:$peakQueued,
peakRunning:$peakRunning,
peakDatabaseConnections:$peakDatabaseConnections,
peakAllocatedCapacity:$peakAllocatedCapacity,
peakNodeMemoryPercent:$peakNodeMemoryPercent,
peakWorkerMemoryMiB:$peakWorkerMemoryMiB,
peakWorkerCPUMillicores:$peakWorkerCPUMillicores,
load:$load[0]
}' >"$summary"
chmod 0600 "$summary"
}
run_single_node_profile() {
local workload=$1
local profile=$2
local slots=$3
local requests=$4
local prefix=$report_root/single-node-$workload-s$slots
local load_report=$prefix-load.json
local pressure_report=$prefix-pressure.csv
local pressure_failure=$prefix-pressure.failure.csv
local pressure_stop=$temporary_root/single-pressure-stop
local sampler_pid status=0 sampler_status=0
rm -f -- "$pressure_stop" "$pressure_failure"
sample_single_node_pressure "$pressure_report" "$pressure_stop" "$pressure_failure" &
sampler_pid=$!
active_pressure_pid=$sampler_pid
run_with_pressure_monitor "$sampler_pid" \
run_single_node_load_profile "$profile" "$load_report" "$requests" || status=$?
touch "$pressure_stop"
wait "$sampler_pid" || sampler_status=$?
active_pressure_pid=
(( status == 0 && sampler_status == 0 )) || {
[[ -s $pressure_failure ]] && cp "$pressure_failure" "$report_root/last-pressure-failure.csv"
return 1
}
record_single_node_result "$workload" "$slots" "$load_report" "$pressure_report" \
"$prefix-summary.json"
}
finish_single_node_diagnostic() {
local diagnostic_status=$1
local reason=$2
local emulator_report=$report_root/single-node-emulator-report.json
local callback_report=$report_root/single-node-callback-report.json
local finish_response=$temporary_root/single-node-finish.json
if [[ $single_node_emulator_installed == true ]]; then
cluster_ssh "$CLUSTER_NINGBO_HOST" \
'curl -fsS --max-time 10 http://127.0.0.1:18090/report' >"$emulator_report" || true
else
remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-emulator -- \
wget -qO- http://127.0.0.1:8090/report >"$emulator_report" || true
fi
remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-callback-collector -- \
wget -qO- http://127.0.0.1:8091/report >"$callback_report" || true
local task_state
task_state=$(database_query "
SELECT count(*)||':'||count(*) FILTER (WHERE status='succeeded')||':'||count(*) FILTER (WHERE status IN ('queued','running'))
FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
local results_json=$temporary_root/single-node-results.json
local -a result_files=()
shopt -s nullglob
result_files=("$report_root"/single-node-*-summary.json)
shopt -u nullglob
if (( ${#result_files[@]} > 0 )); then
jq -s '.' "${result_files[@]}" >"$results_json"
else
printf '[]\n' >"$results_json"
fi
jq -n \
--arg runId "$run_id" \
--arg releaseSha "$release_sha" \
--arg status "$diagnostic_status" \
--arg reason "$reason" \
--arg taskState "$task_state" \
--argjson stableSlots "$single_node_stable_slots" \
--slurpfile results "$results_json" \
'{
schemaVersion:"acceptance-single-node-diagnostic/v1",
runId:$runId,
releaseSha:$releaseSha,
site:"ningbo",
mode:"acceptance-emulator-only",
realUpstreamRequests:0,
certification:false,
promotable:false,
status:$status,
reason:$reason,
taskState:$taskState,
stableVideoSlots:$stableSlots,
results:$results[0]
}' >"$report_root/single-node-summary.json"
chmod 0600 "$report_root/single-node-summary.json"
local finish_body
finish_body=$(jq -cn \
--arg reason "single-node diagnostic completed; real upstream and certification intentionally skipped" \
--arg status "$diagnostic_status" \
--argjson stableSlots "$single_node_stable_slots" \
'{passed:false,failureReason:$reason,report:{diagnosticStatus:$status,site:"ningbo",realUpstreamRequests:0,certification:false,stableVideoSlots:$stableSlots}}')
admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response"
failure_recorded=true
cleanup_single_node_emulator
remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \
-n "$namespace" --ignore-not-found >/dev/null || true
remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \
-n "$namespace" --ignore-not-found >/dev/null || true
}
run_single_node_acceptance() {
wait_for_existing_tasks_to_drain
single_node_stable_slots=$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS
apply_single_node_capacity "$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS"
if ! run_single_node_profile gemini-multi-image gemini-multi-image \
"$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS" \
"$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS"; then
finish_single_node_diagnostic failed 'GEMINI multi-image diagnostic failed or crossed a resource gate'
return 1
fi
local slots requests
for slots in "${single_node_video_slots[@]}"; do
apply_single_node_capacity "$slots" || break
requests=$((slots * 3))
(( requests < 32 )) && requests=32
if ! run_single_node_profile image-video video-throughput "$slots" "$requests"; then
break
fi
single_node_stable_slots=$slots
done
apply_single_node_capacity "$single_node_stable_slots"
finish_single_node_diagnostic completed 'single-node emulator-only capacity ladder completed'
echo "single_node_acceptance=PASS_DIAGNOSTIC run_id=$run_id stable_video_slots=$single_node_stable_slots traffic_mode=validation certification=false report=$report_root/single-node-summary.json"
}
run_capacity_round() {
local profile=$1
local repetition=$2
@@ -3651,11 +4178,17 @@ verify_release_cas false
bootstrap_acceptance_primary_identity
ensure_acceptance_user_group
ensure_acceptance_identity_shards
ensure_acceptance_real_images
if [[ $single_node_acceptance != true ]]; then
ensure_acceptance_real_images
fi
select_acceptance_models
ensure_acceptance_model_access
deploy_protocol_emulator
create_and_activate_run
if [[ $single_node_acceptance == true ]]; then
run_single_node_acceptance
exit $?
fi
wait_for_existing_tasks_to_drain
snapshot_pre_acceptance_capacity
if ! apply_capacity_profile P24; then