wangbo
c89c56ca65
原因：线上 P24 同构验收中，单任务同步复制提交与全局容量锁串行化，使两个 Worker 的 48 个执行槽只能维持约 10–14 个运行任务，最老等待超过 15 分钟。 影响：新增可配置的 1–32 条准入微批次，默认 8；租约、任务准入与唯一 River job 在一个有界事务内原子提交，并按确定顺序预锁任务和容量范围，避免整窗 48 条大事务和多 Dispatcher 死锁。并容忍 rebind 已被其他 Dispatcher 完成的幂等竞态。 验证：Go 全量测试、go vet、真实 PostgreSQL 跨 Store 集成测试、ShellCheck、迁移安全检查、OpenAPI、前端 lint/test/build、Compose/Kubernetes 渲染及人工发布脚本均通过。
96 lines
2.9 KiB
Bash
Executable File
96 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2034 # Variables are consumed by the dynamically sourced function.
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
|
|
helper=$root/deploy/kubernetes/easyai-ai-gateway-cluster-release
|
|
application=$root/deploy/kubernetes/production/application.yaml
|
|
|
|
awk '
|
|
/^rollout_worker_site\(\)/ { capture = 1 }
|
|
capture && /^rollout_capacity_controller\(\)/ { exit }
|
|
capture { print }
|
|
' "$helper" >"$tmp/rollout-worker-site.sh"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "$tmp/rollout-worker-site.sh"
|
|
|
|
cat >"$tmp/kubectl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
if [[ ${1:-} == rollout && ${2:-} == status ]]; then
|
|
exit 42
|
|
fi
|
|
if [[ ${1:-} == get && $* == *jsonpath* ]]; then
|
|
printf worker
|
|
exit 0
|
|
fi
|
|
if [[ ${1:-} == get && $* == *'-o json'* ]]; then
|
|
printf '{"status":{"readyReplicas":1}}\n'
|
|
exit 0
|
|
fi
|
|
exit 0
|
|
EOF
|
|
chmod +x "$tmp/kubectl"
|
|
|
|
kubectl=("$tmp/kubectl")
|
|
NAMESPACE=easyai
|
|
ACTIVE_RELEASE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
|
|
AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=8
|
|
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
|
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4
|
|
AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=4
|
|
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=2
|
|
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=300
|
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
|
|
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
|
|
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
|
|
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=512
|
|
|
|
grep -Fq "\"AI_GATEWAY_REVISION=\${ACTIVE_RELEASE_SHA:-}\"" "$helper"
|
|
|
|
# activate_manifest invokes this function inside an && condition. A failed
|
|
# rollout must still propagate instead of falling through to a stale Ready Pod.
|
|
if rollout_worker_site hongkong 'registry.example.invalid/api@sha256:deadbeef' && true; then
|
|
echo 'failed Worker rollout was reported as successful' >&2
|
|
exit 1
|
|
fi
|
|
|
|
deployment_strategy() {
|
|
local deployment=$1
|
|
awk -v deployment="$deployment" '
|
|
$0 == " name: " deployment { capture = 1 }
|
|
capture && /^---$/ { exit }
|
|
capture { print }
|
|
' "$application"
|
|
}
|
|
|
|
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
|
strategy=$(deployment_strategy "$deployment")
|
|
grep -Fq ' maxSurge: 0' <<<"$strategy"
|
|
grep -Fq ' maxUnavailable: 1' <<<"$strategy"
|
|
done
|
|
|
|
for deployment in easyai-api-ningbo easyai-api-hongkong; do
|
|
strategy=$(deployment_strategy "$deployment")
|
|
grep -Fq ' maxSurge: 1' <<<"$strategy"
|
|
grep -Fq ' maxUnavailable: 0' <<<"$strategy"
|
|
done
|
|
|
|
echo 'cluster_release_helper_tests=PASS'
|