fix(worker): 拆分准入事务并收敛验收失败状态

将批量准入改为每任务独立事务,避免 P24 压测时 48 条任务共享长事务造成 transactionid 锁潮与 Worker 槽位空转。失败验收在保持 validation 的同时清理未提交任务,并记录精确压力门禁原因。统一生产 0+2 Worker 拓扑、运行时 ConfigMap 与发布配置,补充镜像预检诊断和回归测试。\n\n验证:Go 全量测试、go vet、真实 PostgreSQL 48 任务集成测试、迁移安全检查、发布脚本测试、bash -n、ShellCheck、Compose 配置均通过。
This commit is contained in:
2026-08-01 18:23:00 +08:00
parent 215b6b607f
commit edcbbd6a87
11 changed files with 194 additions and 100 deletions
@@ -15,8 +15,8 @@ source "$config_file"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=0}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
@@ -32,10 +32,10 @@ source "$config_file"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=0}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=0}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
@@ -199,8 +199,55 @@ capacity_config_hash() {
}
record_capacity_config() {
local config_hash
config_hash=$(capacity_config_hash)
local config_hash
local runtime_patch
runtime_patch=$(jq -nc \
--arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
--arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
--arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
--arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
--arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
--arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
--arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
--arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
--arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
--arg workerPool "$AI_GATEWAY_DATABASE_MAX_CONNS" \
--arg materialization "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
--arg mediaRequest "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
--arg targetOutstanding "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
--arg scaleUp "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
--arg scaleDown "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
--arg drainTimeout "$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
--arg memoryTarget "$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
--arg memoryHard "$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
--arg cpuTarget "$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
--arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
'{data:{
AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo,
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong,
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo,
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong,
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo,
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong,
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling,
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit,
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit,
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$globalLimit,
AI_GATEWAY_WORKER_DATABASE_MAX_CONNS:$workerPool,
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:$materialization,
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:$mediaRequest,
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$targetOutstanding,
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:$scaleUp,
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:$scaleDown,
AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS:$drainTimeout,
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT:$memoryTarget,
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:$memoryHard,
AI_GATEWAY_NODE_CPU_TARGET_PERCENT:$cpuTarget,
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:$postgresBudget
}}')
"${kubectl[@]}" patch configmap easyai-ai-gateway-config -n "$NAMESPACE" \
--type merge -p "$runtime_patch" >/dev/null
config_hash=$(capacity_config_hash)
"${kubectl[@]}" patch configmap easyai-gateway-release -n "$NAMESPACE" \
--type merge -p "{\"data\":{\"configHash\":\"$config_hash\"}}" >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
@@ -512,9 +559,18 @@ spec:
emptyDir:
sizeLimit: 64Mi
EOF
if ! "${kubectl[@]}" wait --for=condition=Ready "pod/$pod" \
-n "$NAMESPACE" --timeout=120s; then
"${kubectl[@]}" delete pod "$pod" -n "$NAMESPACE" \
if ! "${kubectl[@]}" wait --for=condition=Ready "pod/$pod" \
-n "$NAMESPACE" --timeout=300s; then
"${kubectl[@]}" get pod "$pod" -n "$NAMESPACE" -o json 2>/dev/null | jq -c '
{
phase:(.status.phase // "unknown"),
node:(.spec.nodeName // "unscheduled"),
ready:([.status.containerStatuses[]?.ready] | all),
restarts:([.status.containerStatuses[]?.restartCount] | add // 0),
waitingReasons:[.status.containerStatuses[]?.state.waiting.reason // empty],
terminatedReasons:[.status.containerStatuses[]?.state.terminated.reason // empty]
}' >&2 || true
"${kubectl[@]}" delete pod "$pod" -n "$NAMESPACE" \
--ignore-not-found=true --wait=true >/dev/null || true
echo 'capacity controller exact-image preflight failed' >&2
return 1
@@ -866,7 +922,7 @@ activate_manifest() {
-n "$NAMESPACE" --replicas=0
fi
if { [[ $api_changed != true ]] ||
if { [[ $api_changed != true ]] ||
{ rollout_worker_site hongkong "$api_image" &&
rollout_worker_site ningbo "$api_image" &&
{ [[ $action == rollback ]] ||
@@ -875,16 +931,17 @@ activate_manifest() {
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
:
else
else
if [[ -n $previous_api && -n $previous_web ]]; then
echo '[cluster-release] restoring exact pre-release Deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
fi
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
return 1
fi
record_current "$manifest" "$action" "$started_at"
record_capacity_config
record_current "$manifest" "$action" "$started_at"
rm -f -- "$deployment_snapshot" "$current_file"
echo "production_release=PASS action=$action release=$source_sha"
}
@@ -6,8 +6,8 @@ DESIRED_STATE_DIR=/usr/local/share/easyai-ai-gateway-release/production
# Worker 容量只需修改本发布配置并执行获授权的 `capacity`,不再修改 Go 代码。
# 副本数属于 Deployment 配置,不能由容器内部环境变量改变;发布工具读取这两个变量并写入 replicas。
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
AI_GATEWAY_WORKER_REPLICAS_NINGBO=0
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=2
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
@@ -23,10 +23,10 @@ AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=0
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=0
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=2
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600
@@ -46,12 +46,12 @@ data:
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48"
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "false"
AI_GATEWAY_WORKER_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_REPLICAS_NINGBO: "0"
AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "2"
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "0"
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "0"
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "2"
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48"
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20"
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600"
@@ -434,7 +434,7 @@ metadata:
app.kubernetes.io/component: worker
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
replicas: 1
replicas: 0
strategy:
type: RollingUpdate
rollingUpdate:
@@ -607,7 +607,7 @@ metadata:
app.kubernetes.io/component: worker
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
replicas: 1
replicas: 2
strategy:
type: RollingUpdate
rollingUpdate: