feat(acceptance): 增加生产同构媒体压力验收模式
引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器,覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。 新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排;Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。 验证:Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- protocol-emulator.yaml
|
||||
@@ -0,0 +1,67 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: easyai-acceptance-emulator
|
||||
namespace: easyai
|
||||
labels:
|
||||
app.kubernetes.io/name: easyai-acceptance-emulator
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: easyai-acceptance-emulator
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: easyai-acceptance-emulator
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
spec:
|
||||
nodeSelector:
|
||||
easyai.io/site: ningbo
|
||||
easyai.io/workload: "true"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: emulator
|
||||
image: easyai-api
|
||||
command: ["/app/easyai-ai-gateway-acceptance-emulator"]
|
||||
env:
|
||||
- name: HTTP_ADDR
|
||||
value: ":8090"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8090
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
readOnlyRootFilesystem: true
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: easyai-acceptance-emulator
|
||||
namespace: easyai
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: easyai-acceptance-emulator
|
||||
ports:
|
||||
- name: http
|
||||
port: 8090
|
||||
targetPort: http
|
||||
@@ -14,10 +14,40 @@ source "$config_file"
|
||||
: "${NAMESPACE:=easyai}"
|
||||
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
|
||||
: "${K3S_BIN:=/usr/local/bin/k3s}"
|
||||
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
|
||||
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
||||
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
||||
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
|
||||
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
||||
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
||||
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
||||
echo 'invalid cluster release configuration' >&2
|
||||
exit 1
|
||||
}
|
||||
for capacity_value in \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
||||
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
|
||||
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
|
||||
echo 'worker capacity configuration must contain positive integers' >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
|
||||
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
|
||||
echo 'invalid public base URL' >&2
|
||||
exit 1
|
||||
@@ -87,11 +117,47 @@ verify_site() {
|
||||
rollout_worker_site() {
|
||||
local site=$1
|
||||
local api_image=$2
|
||||
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"worker=$api_image"
|
||||
local replicas
|
||||
case $site in
|
||||
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
|
||||
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
|
||||
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
--replicas="$replicas"
|
||||
if [[ -n $api_image ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
||||
"worker=$api_image"
|
||||
fi
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
|
||||
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
|
||||
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
|
||||
}
|
||||
|
||||
rollout_api_capacity_site() {
|
||||
local site=$1
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
}
|
||||
|
||||
apply_capacity_config() {
|
||||
rollout_worker_site hongkong ""
|
||||
rollout_worker_site ningbo ""
|
||||
rollout_api_capacity_site hongkong
|
||||
rollout_api_capacity_site ningbo
|
||||
verify_site hongkong
|
||||
verify_site ningbo
|
||||
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
||||
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
|
||||
}
|
||||
|
||||
rollout_site() {
|
||||
@@ -101,6 +167,8 @@ rollout_site() {
|
||||
local api_changed=$4
|
||||
local web_changed=$5
|
||||
if [[ $api_changed == true ]]; then
|
||||
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT"
|
||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"api=$api_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
@@ -305,8 +373,12 @@ case ${1:-} in
|
||||
[[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
|
||||
activate_manifest "$target" rollback
|
||||
;;
|
||||
capacity)
|
||||
[[ $# -eq 1 ]] || exit 64
|
||||
apply_capacity_config
|
||||
;;
|
||||
*)
|
||||
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>}' >&2
|
||||
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity}' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -2,3 +2,13 @@ RELEASES_DIR=/var/lib/easyai-ai-gateway-cluster-release/releases
|
||||
NAMESPACE=easyai
|
||||
PUBLIC_BASE_URL=https://ai.51easyai.com
|
||||
K3S_BIN=/usr/local/bin/k3s
|
||||
|
||||
# Worker 容量只需修改本发布配置并执行获授权的 `capacity`,不再修改 Go 代码。
|
||||
# 副本数属于 Deployment 配置,不能由容器内部环境变量改变;发布工具读取这两个变量并写入 replicas。
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS=32
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
|
||||
|
||||
@@ -37,5 +37,5 @@ data:
|
||||
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456"
|
||||
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912"
|
||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "2048"
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "48"
|
||||
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
|
||||
|
||||
@@ -316,6 +316,8 @@ spec:
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||
value: "24"
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
@@ -445,6 +447,8 @@ spec:
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY
|
||||
value: "24"
|
||||
- name: AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY
|
||||
value: "24"
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
|
||||
Reference in New Issue
Block a user