feat(acceptance): 增加生产同构媒体压力验收模式

引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器,覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。

新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排;Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。

验证:Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。
This commit is contained in:
2026-07-30 23:06:19 +08:00
parent f33d6d64e0
commit 3053ba4925
45 changed files with 5214 additions and 45 deletions
@@ -14,10 +14,40 @@ source "$config_file"
: "${NAMESPACE:=easyai}"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
echo 'invalid cluster release configuration' >&2
exit 1
}
for capacity_value in \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
echo 'worker capacity configuration must contain positive integers' >&2
exit 1
}
done
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
echo 'worker capacity configuration exceeds the release safety bounds' >&2
exit 1
}
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
echo 'invalid public base URL' >&2
exit 1
@@ -87,11 +117,47 @@ verify_site() {
rollout_worker_site() {
local site=$1
local api_image=$2
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"worker=$api_image"
local replicas
case $site in
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
*) return 1 ;;
esac
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--replicas="$replicas"
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"worker=$api_image"
fi
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
}
rollout_api_capacity_site() {
local site=$1
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}
apply_capacity_config() {
rollout_worker_site hongkong ""
rollout_worker_site ningbo ""
rollout_api_capacity_site hongkong
rollout_api_capacity_site ningbo
verify_site hongkong
verify_site ningbo
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
}
rollout_site() {
@@ -101,6 +167,8 @@ rollout_site() {
local api_changed=$4
local web_changed=$5
if [[ $api_changed == true ]]; then
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
"api=$api_image"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
@@ -305,8 +373,12 @@ case ${1:-} in
[[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
activate_manifest "$target" rollback
;;
capacity)
[[ $# -eq 1 ]] || exit 64
apply_capacity_config
;;
*)
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>}' >&2
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity}' >&2
exit 64
;;
esac