feat(acceptance): 增加生产同构媒体压力验收模式

引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器,覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。

新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排;Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。

验证:Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。
This commit is contained in:
2026-07-30 23:06:19 +08:00
parent f33d6d64e0
commit 3053ba4925
45 changed files with 5214 additions and 45 deletions
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- protocol-emulator.yaml
@@ -0,0 +1,67 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-acceptance-emulator
namespace: easyai
labels:
app.kubernetes.io/name: easyai-acceptance-emulator
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-acceptance-emulator
template:
metadata:
labels:
app.kubernetes.io/name: easyai-acceptance-emulator
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
nodeSelector:
easyai.io/site: ningbo
easyai.io/workload: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: emulator
image: easyai-api
command: ["/app/easyai-ai-gateway-acceptance-emulator"]
env:
- name: HTTP_ADDR
value: ":8090"
ports:
- name: http
containerPort: 8090
readinessProbe:
httpGet:
path: /healthz
port: http
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "2"
memory: 1Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
---
apiVersion: v1
kind: Service
metadata:
name: easyai-acceptance-emulator
namespace: easyai
spec:
selector:
app.kubernetes.io/name: easyai-acceptance-emulator
ports:
- name: http
port: 8090
targetPort: http
@@ -14,10 +14,40 @@ source "$config_file"
: "${NAMESPACE:=easyai}"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
echo 'invalid cluster release configuration' >&2
exit 1
}
for capacity_value in \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
echo 'worker capacity configuration must contain positive integers' >&2
exit 1
}
done
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
echo 'worker capacity configuration exceeds the release safety bounds' >&2
exit 1
}
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
echo 'invalid public base URL' >&2
exit 1
@@ -87,11 +117,47 @@ verify_site() {
rollout_worker_site() {
local site=$1
local api_image=$2
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"worker=$api_image"
local replicas
case $site in
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
*) return 1 ;;
esac
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--replicas="$replicas"
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"worker=$api_image"
fi
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
}
rollout_api_capacity_site() {
local site=$1
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}
apply_capacity_config() {
rollout_worker_site hongkong ""
rollout_worker_site ningbo ""
rollout_api_capacity_site hongkong
rollout_api_capacity_site ningbo
verify_site hongkong
verify_site ningbo
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
}
rollout_site() {
@@ -101,6 +167,8 @@ rollout_site() {
local api_changed=$4
local web_changed=$5
if [[ $api_changed == true ]]; then
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
"api=$api_image"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
@@ -305,8 +373,12 @@ case ${1:-} in
[[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
activate_manifest "$target" rollback
;;
capacity)
[[ $# -eq 1 ]] || exit 64
apply_capacity_config
;;
*)
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>}' >&2
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity}' >&2
exit 64
;;
esac
@@ -2,3 +2,13 @@ RELEASES_DIR=/var/lib/easyai-ai-gateway-cluster-release/releases
NAMESPACE=easyai
PUBLIC_BASE_URL=https://ai.51easyai.com
K3S_BIN=/usr/local/bin/k3s
# Worker 容量只需修改本发布配置并执行获授权的 `capacity`,不再修改 Go 代码。
# 副本数属于 Deployment 配置,不能由容器内部环境变量改变;发布工具读取这两个变量并写入 replicas。
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
AI_GATEWAY_DATABASE_MAX_CONNS=32
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
@@ -37,5 +37,5 @@ data:
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456"
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912"
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "2048"
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "48"
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
@@ -316,6 +316,8 @@ spec:
value: "24"
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
value: "24"
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
value: "24"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
@@ -445,6 +447,8 @@ spec:
value: "24"
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
value: "24"
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
value: "24"
- name: POD_NAMESPACE
valueFrom:
fieldRef: