feat(acceptance): 增加生产同构媒体压力验收模式
引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器,覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。 新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排;Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。 验证:Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。
This commit is contained in:
@@ -14,10 +14,40 @@ source "$config_file"
|
||||
: "${NAMESPACE:=easyai}"
|
||||
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
|
||||
: "${K3S_BIN:=/usr/local/bin/k3s}"
|
||||
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
|
||||
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
||||
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
|
||||
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
||||
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
||||
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
||||
echo 'invalid cluster release configuration' >&2
|
||||
exit 1
|
||||
}
|
||||
for capacity_value in \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
|
||||
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
|
||||
echo 'worker capacity configuration must contain positive integers' >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
|
||||
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
|
||||
echo 'invalid public base URL' >&2
|
||||
exit 1
|
||||
@@ -87,11 +117,47 @@ verify_site() {
|
||||
rollout_worker_site() {
|
||||
local site=$1
|
||||
local api_image=$2
|
||||
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"worker=$api_image"
|
||||
local replicas
|
||||
case $site in
|
||||
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
|
||||
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
|
||||
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
--replicas="$replicas"
|
||||
if [[ -n $api_image ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"worker=$api_image"
|
||||
fi
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
|
||||
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
|
||||
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
|
||||
}
|
||||
|
||||
rollout_api_capacity_site() {
|
||||
local site=$1
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
}
|
||||
|
||||
apply_capacity_config() {
|
||||
rollout_worker_site hongkong ""
|
||||
rollout_worker_site ningbo ""
|
||||
rollout_api_capacity_site hongkong
|
||||
rollout_api_capacity_site ningbo
|
||||
verify_site hongkong
|
||||
verify_site ningbo
|
||||
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
||||
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
||||
}
|
||||
|
||||
rollout_site() {
|
||||
@@ -101,6 +167,8 @@ rollout_site() {
|
||||
local api_changed=$4
|
||||
local web_changed=$5
|
||||
if [[ $api_changed == true ]]; then
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
|
||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"api=$api_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
@@ -305,8 +373,12 @@ case ${1:-} in
|
||||
[[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
|
||||
activate_manifest "$target" rollback
|
||||
;;
|
||||
capacity)
|
||||
[[ $# -eq 1 ]] || exit 64
|
||||
apply_capacity_config
|
||||
;;
|
||||
*)
|
||||
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>}' >&2
|
||||
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity}' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user