BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(acceptance): 建立同构验收与弹性容量体系

Browse Source 
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
wangbo
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/kubernetes/acceptance/protocol-emulator.yaml
+68
View File
@@ -65,3 +65,71 @@ spec:
- name: http
port: 8090
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-acceptance-callback-collector
namespace: easyai
labels:
app.kubernetes.io/name: easyai-acceptance-callback-collector
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-acceptance-callback-collector
template:
metadata:
labels:
app.kubernetes.io/name: easyai-acceptance-callback-collector
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
nodeSelector:
easyai.io/site: hongkong
easyai.io/workload: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: collector
image: easyai-api
command: ["/app/easyai-ai-gateway-acceptance-callback-collector"]
env:
- name: HTTP_ADDR
value: ":8091"
ports:
- name: http
containerPort: 8091
readinessProbe:
httpGet:
path: /healthz
port: http
resources:
requests:
cpu: 20m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
---
apiVersion: v1
kind: Service
metadata:
name: easyai-acceptance-callback-collector
namespace: easyai
spec:
selector:
app.kubernetes.io/name: easyai-acceptance-callback-collector
ports:
- name: http
port: 8091
targetPort: http
deploy/kubernetes/easyai-ai-gateway-cluster-release
+413 -22
View File
@@ -14,50 +14,127 @@ source "$config_file"
: "${NAMESPACE:=easyai}"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
: "${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB:=1536}"
: "${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES:=500}"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=64}"
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_RIVER_MAX_CONNS:=8}"
: "${AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS:=30}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
: "${AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS:=600}"
: "${AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT:=75}"
: "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}"
: "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}"
: "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}"
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
echo 'invalid cluster release configuration' >&2
exit 1
}
[[ $DESIRED_STATE_DIR == /* &&
-f $DESIRED_STATE_DIR/kustomization.yaml &&
! -L $DESIRED_STATE_DIR/kustomization.yaml ]] || {
echo 'production desired-state bundle is unavailable' >&2
exit 1
}
for capacity_value in \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
"$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET"; do
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
echo 'worker capacity configuration must contain positive integers' >&2
exit 1
}
done
[[ $AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET =~ ^[1-9][0-9]*$ ]] || {
echo 'PostgreSQL non-Worker connection budget must be a positive integer' >&2
exit 1
}
[[ $AI_GATEWAY_DATABASE_MIN_IDLE_CONNS =~ ^[0-9]+$ ]] || {
echo 'database minimum idle connection configuration must be a non-negative integer' >&2
exit 1
}
[[ $AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == true ||
$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == false ]] || {
echo 'worker autoscaling flag must be true or false' >&2
exit 1
}
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT == AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT &&
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB >= 256 &&
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB <= 1900 &&
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES >= 100 &&
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES <= 1900 &&
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_API_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS <= 3600 &&
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 &&
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT &&
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 &&
AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 &&
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET +
(AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) *
AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || {
echo 'worker capacity configuration exceeds the release safety bounds' >&2
exit 1
}
@@ -65,11 +142,12 @@ done
echo 'invalid public base URL' >&2
exit 1
}
for command in "$K3S_BIN" curl flock install jq node; do
for command in "$K3S_BIN" curl flock install jq node sha256sum; do
command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done
kubectl=("$K3S_BIN" kubectl)
ACTIVE_RELEASE_SHA=
install -d -m 0700 "$RELEASES_DIR"
exec 9>"$RELEASES_DIR/.release.lock"
flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; }
@@ -78,6 +156,57 @@ manifest_get() {
node "$manifest_tool" get "$1" "$2"
}
capacity_config_hash() {
printf '%s\n' \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
"$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
"$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" |
sha256sum | awk '{print $1}'
}
record_capacity_config() {
local config_hash
config_hash=$(capacity_config_hash)
"${kubectl[@]}" patch configmap easyai-gateway-release -n "$NAMESPACE" \
--type merge -p "{\"data\":{\"configHash\":\"$config_hash\"}}" >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
done
}
current_to_file() {
local output=$1
"${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
@@ -127,6 +256,71 @@ verify_site() {
wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
}
apply_desired_state() {
local api_image=$1
local web_image=$2
local capacity_controller_replicas=${3:-2}
local api_repository=${api_image%@*}
local api_digest=${api_image#*@}
local web_repository=${web_image%@*}
local web_digest=${web_image#*@}
local working_directory
working_directory=$(mktemp -d)
cp -a "$DESIRED_STATE_DIR/." "$working_directory/"
sed -i \
-e "/ - name: easyai-api/{n;s| newName:.*| newName: $api_repository|;n;s| digest:.*| digest: $api_digest|;}" \
-e "/ - name: easyai-web/{n;s| newName:.*| newName: $web_repository|;n;s| digest:.*| digest: $web_digest|;}" \
"$working_directory/kustomization.yaml"
if [[ $capacity_controller_replicas == 0 ]]; then
sed -i \
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 2$/ replicas: 0/' \
"$working_directory/application.yaml"
fi
"${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \
--field-manager=easyai-production-preflight \
-k "$working_directory" >/dev/null
"${kubectl[@]}" apply --server-side --force-conflicts \
--field-manager=easyai-production-release \
-k "$working_directory" >/dev/null
rm -rf -- "$working_directory"
}
snapshot_application_deployments() {
local output=$1
"${kubectl[@]}" get deployments -n "$NAMESPACE" \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong \
-o json |
jq '
del(.metadata)
| .items |= map(
del(
.metadata.annotations["deployment.kubernetes.io/revision"],
.metadata.creationTimestamp,
.metadata.generation,
.metadata.managedFields,
.metadata.resourceVersion,
.metadata.uid,
.status
)
)
' >"$output"
}
restore_application_deployments() {
local snapshot=$1
[[ -s $snapshot ]] || return 1
"${kubectl[@]}" apply -f "$snapshot" >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong; do
"${kubectl[@]}" rollout status "deployment/$deployment" \
-n "$NAMESPACE" --timeout=300s || true
done
}
rollout_worker_site() {
local site=$1
local api_image=$2
@@ -139,11 +333,27 @@ rollout_worker_site() {
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA"
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--containers=worker \
--requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
--limits='cpu=2,memory=2Gi'
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--replicas="$replicas"
if [[ -n $api_image ]]; then
@@ -157,25 +367,74 @@ rollout_worker_site() {
-o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
}
rollout_capacity_controller() {
local api_image=${1:-}
"${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"AI_GATEWAY_NODE_CPU_TARGET_PERCENT=$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
"AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" \
"AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
"capacity-controller=$api_image"
fi
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller -n "$NAMESPACE" --timeout=300s
}
rollout_api_capacity_site() {
local site=$1
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}
apply_capacity_config() {
local capacity_controller_enabled
ACTIVE_RELEASE_SHA=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.releaseSha}' 2>/dev/null || true)
capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
[[ $ACTIVE_RELEASE_SHA =~ ^[0-9a-f]{40}$ ]] || {
echo 'current release SHA is unavailable for capacity update' >&2
return 1
}
rollout_worker_site hongkong ""
rollout_worker_site ningbo ""
rollout_api_capacity_site hongkong
rollout_api_capacity_site ningbo
if [[ $capacity_controller_enabled == true ]]; then
rollout_capacity_controller ""
else
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0
fi
verify_site hongkong
verify_site ningbo
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
record_capacity_config
echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
}
rollout_site() {
@@ -187,7 +446,10 @@ rollout_site() {
if [[ $api_changed == true ]]; then
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
@@ -265,11 +527,36 @@ EOF
-n "$NAMESPACE" --timeout=300s
}
verify_migration_version() {
local expected=$1
[[ $expected =~ ^[0-9]{4}_[a-z0-9][a-z0-9_]*$ ]] || {
echo 'release manifest migration version is invalid' >&2
return 1
}
local primary actual
primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
-o jsonpath='{.status.currentPrimary}')
[[ $primary =~ ^easyai-postgres-[0-9]+$ ]] || {
echo 'PostgreSQL primary is unavailable for migration verification' >&2
return 1
}
actual=$("${kubectl[@]}" exec -n "$NAMESPACE" "$primary" -c postgres -- \
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At \
-c 'SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1')
[[ $actual == "$expected" ]] || {
echo "production migration version mismatch: expected=$expected actual=${actual:-missing}" >&2
return 1
}
}
record_current() {
local manifest=$1
local action=$2
local started_at=$3
local source_sha recorded temporary
local source_sha recorded temporary capacity_controller_enabled=true
if [[ $action == rollback ]]; then
capacity_controller_enabled=false
fi
source_sha=$(manifest_get "$manifest" sourceSha)
recorded=$RELEASES_DIR/$source_sha.json
temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX")
@@ -280,23 +567,102 @@ record_current() {
node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null
install -m 0600 "$temporary" "$recorded"
"${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
--from-file=manifest.json="$recorded" --dry-run=client -o yaml |
--from-file=manifest.json="$recorded" \
--from-literal=configHash="$(capacity_config_hash)" \
--from-literal=capacityControllerEnabled="$capacity_controller_enabled" \
--from-literal=releaseSha="$source_sha" \
--dry-run=client -o yaml |
"${kubectl[@]}" apply -f - >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/release=$source_sha" \
"easyai.io/config-hash=$(capacity_config_hash)" \
--overwrite >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller \
easyai-web-ningbo easyai-web-hongkong; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/release=$source_sha" \
"easyai.io/config-hash=$(capacity_config_hash)" \
--overwrite >/dev/null
done
rm -f -- "$temporary"
}
record_certification() {
local run_id=$1
local capacity_profile=$2
local config_hash=$3
local instance_slots=$4
local max_ningbo=$5
local max_hongkong=$6
[[ $run_id =~ ^[0-9a-f-]{36}$ &&
$capacity_profile =~ ^P(24|28|32)$ &&
$config_hash =~ ^[0-9a-f]{64}$ &&
$instance_slots =~ ^[1-9][0-9]*$ &&
$max_ningbo =~ ^[1-9][0-9]*$ &&
$max_hongkong =~ ^[1-9][0-9]*$ ]] || {
echo 'invalid production certification metadata' >&2
return 1
}
local effective_config_hash
effective_config_hash=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.configHash}' 2>/dev/null || true)
[[ $effective_config_hash == "$config_hash" ]] || {
echo 'production certification config hash does not match the deployed release configuration' >&2
return 1
}
local current_file source_sha recorded temporary capacity_controller_enabled
current_file=$(mktemp "$RELEASES_DIR/.certification-current.XXXXXX")
current_to_file "$current_file"
source_sha=$(manifest_get "$current_file" sourceSha)
recorded=$RELEASES_DIR/$source_sha.json
temporary=$(mktemp "$RELEASES_DIR/.certification.XXXXXX")
rm -f -- "$temporary"
RELEASE_CERTIFIED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
RELEASE_ACCEPTANCE_RUN_ID=$run_id \
RELEASE_CONFIG_HASH=$config_hash \
RELEASE_CAPACITY_PROFILE=$capacity_profile \
RELEASE_WORKER_INSTANCE_SLOTS=$instance_slots \
RELEASE_WORKER_MAX_REPLICAS_NINGBO=$max_ningbo \
RELEASE_WORKER_MAX_REPLICAS_HONGKONG=$max_hongkong \
node "$manifest_tool" record-certification "$current_file" "$temporary" >/dev/null
install -m 0600 "$temporary" "$recorded"
capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
"${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
--from-file=manifest.json="$recorded" \
--from-literal=configHash="$config_hash" \
--from-literal=capacityControllerEnabled="${capacity_controller_enabled:-true}" \
--from-literal=releaseSha="$source_sha" \
--from-literal=certifiedRunId="$run_id" \
--from-literal=certifiedCapacityProfile="$capacity_profile" \
--dry-run=client -o yaml |
"${kubectl[@]}" apply -f - >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/certified-run=$run_id" \
"easyai.io/capacity-profile=$capacity_profile" \
--overwrite >/dev/null
rm -f -- "$temporary" "$current_file"
echo "production_certification=PASS release=$source_sha run_id=$run_id capacity_profile=$capacity_profile config_hash=$config_hash"
}
activate_manifest() {
local manifest=$1
local action=${2:-deploy}
local source_sha base_sha api_image web_image migrations_changed components
local api_changed=false web_changed=false current_file previous_api previous_web started_at
local source_sha base_sha api_image web_image migrations_changed migration_version components
local api_changed=false web_changed=false current_file deployment_snapshot previous_api previous_web started_at
local capacity_controller_replicas
[[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
node "$manifest_tool" validate "$manifest" >/dev/null
source_sha=$(manifest_get "$manifest" sourceSha)
ACTIVE_RELEASE_SHA=$source_sha
base_sha=$(manifest_get "$manifest" baseReleaseSha)
api_image=$(manifest_get "$manifest" images.api)
web_image=$(manifest_get "$manifest" images.web)
migrations_changed=$(manifest_get "$manifest" migrationsChanged)
migration_version=$(manifest_get "$manifest" migrationVersion)
components=$(manifest_get "$manifest" components)
[[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || {
echo 'release images must be digest-pinned' >&2
@@ -329,35 +695,54 @@ activate_manifest() {
previous_web=
fi
started_at=$(date +%s)
deployment_snapshot=$(mktemp "$RELEASES_DIR/.deployments.XXXXXX")
snapshot_application_deployments "$deployment_snapshot"
if [[ $action == deploy && $migrations_changed == true ]]; then
run_backup "$source_sha"
run_migrator "$source_sha" "$api_image"
fi
if [[ $action == deploy ]]; then
verify_migration_version "$migration_version"
fi
capacity_controller_replicas=2
if [[ $action == rollback ]]; then
capacity_controller_replicas=0
fi
if ! apply_desired_state "$api_image" "$web_image" "$capacity_controller_replicas"; then
echo '[cluster-release] desired state apply failed; restoring exact deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
if [[ $action == rollback ]]; then
# Historical API images may predate the capacity-controller process role.
# Rollbacks therefore restore static Worker replicas and leave the
# controller at zero instead of ever starting an incompatible image.
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
-n "$NAMESPACE" --replicas=0
fi
if { [[ $api_changed != true ]] ||
{ rollout_worker_site hongkong "$api_image" &&
rollout_worker_site ningbo "$api_image"; }; } &&
rollout_worker_site ningbo "$api_image" &&
{ [[ $action == rollback ]] ||
rollout_capacity_controller "$api_image"; }; }; } &&
rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
:
else
if [[ -n $previous_api && -n $previous_web ]]; then
echo '[cluster-release] restoring previous application images' >&2
if [[ $api_changed == true ]]; then
rollout_worker_site hongkong "$previous_api" || true
rollout_worker_site ningbo "$previous_api" || true
fi
rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
echo '[cluster-release] restoring exact pre-release Deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
fi
rm -f -- "$current_file"
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
record_current "$manifest" "$action" "$started_at"
rm -f -- "$current_file"
rm -f -- "$deployment_snapshot" "$current_file"
echo "production_release=PASS action=$action release=$source_sha"
}
@@ -378,6 +763,8 @@ case ${1:-} in
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
done
[[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]]
[[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
[[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
@@ -398,8 +785,12 @@ case ${1:-} in
[[ $# -eq 1 ]] || exit 64
apply_capacity_config
;;
certify)
[[ $# -eq 7 ]] || exit 64
record_certification "$2" "$3" "$4" "$5" "$6" "$7"
;;
*)
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity}' >&2
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity|certify <run-id> <profile> <config-hash> <slots> <max-ningbo> <max-hongkong>}' >&2
exit 64
;;
esac
deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example
+23 -1
View File
@@ -2,6 +2,7 @@ RELEASES_DIR=/var/lib/easyai-ai-gateway-cluster-release/releases
NAMESPACE=easyai
PUBLIC_BASE_URL=https://ai.51easyai.com
K3S_BIN=/usr/local/bin/k3s
DESIRED_STATE_DIR=/usr/local/share/easyai-ai-gateway-release/production
# Worker 容量只需修改本发布配置并执行获授权的 `capacity`，不再修改 Go 代码。
# 副本数属于 Deployment 配置，不能由容器内部环境变量改变；发布工具读取这两个变量并写入 replicas。
@@ -9,9 +10,30 @@ AI_GATEWAY_WORKER_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_REPLICAS_HONGKONG=1
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=24
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=48
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=48
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB=1536
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES=500
AI_GATEWAY_DATABASE_MAX_CONNS=32
AI_GATEWAY_API_DATABASE_MAX_CONNS=64
AI_GATEWAY_API_DATABASE_MAX_CONNS=32
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4
AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=8
AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS=4
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=24
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=24
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=false
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=1
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=1
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=1
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=48
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=20
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=600
AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=600
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=75
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=85
AI_GATEWAY_NODE_CPU_TARGET_PERCENT=70
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=150
# 2 个 API 池（2×32）加 2 个 capacity-controller 池（2×4）。
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=72
deploy/kubernetes/local-acceptance/database.yaml
+52
View File
@@ -0,0 +1,52 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: easyai-postgres
namespace: easyai
labels:
app.kubernetes.io/name: easyai-postgres
app.kubernetes.io/part-of: easyai-ai-gateway
easyai.io/environment: local-acceptance
spec:
instances: 2
imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c
bootstrap:
initdb:
database: easyai_ai_gateway
owner: easyai
secret:
name: easyai-postgres-app
postInitSQL:
- CREATE EXTENSION IF NOT EXISTS pgcrypto;
storage:
storageClass: local-path
size: 8Gi
walStorage:
storageClass: local-path
size: 2Gi
affinity:
enablePodAntiAffinity: true
podAntiAffinityType: required
topologyKey: easyai.io/site
nodeSelector:
easyai.io/database: "true"
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 3Gi
postgresql:
synchronous:
method: any
number: 1
dataDurability: required
parameters:
archive_mode: "off"
max_connections: "200"
shared_buffers: 512MB
wal_compression: "on"
wal_keep_size: 1GB
monitoring:
enablePodMonitor: false
deploy/kubernetes/local-acceptance/dependencies.lock
+8
View File
@@ -0,0 +1,8 @@
# Local acceptance dependency lock. Checksums are lowercase SHA-256.
K3D_VERSION=v5.9.0
K3D_DARWIN_ARM64_SHA256=fe106541d5d0a3f18debcd4d432a16f8c0ce3e6ddc06f8fbb6f696a122313e00
K3D_DARWIN_AMD64_SHA256=b4aabc37534f95b9c764e7823f2df923f50d57600837aa60a06266cce47db732
K3S_IMAGE=rancher/k3s:v1.36.2-k3s1
CNPG_VERSION=1.29.1
CNPG_MANIFEST_URL=https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.29/releases/cnpg-1.29.1.yaml
CNPG_MANIFEST_SHA256=e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a
deploy/kubernetes/local-acceptance/k3d.yaml
+50
View File
@@ -0,0 +1,50 @@
apiVersion: k3d.io/v1alpha5
kind: Simple
metadata:
name: easyai-acceptance-local
servers: 3
agents: 0
image: rancher/k3s:v1.36.2-k3s1
kubeAPI:
host: 127.0.0.1
hostPort: "16550"
ports:
- port: 18443:31088
nodeFilters:
- server:0
- port: 19443:31089
nodeFilters:
- server:1
volumes:
- volume: EASYAI_ACCEPTANCE_MEDIA_DIR:/var/lib/easyai-acceptance/media
nodeFilters:
- server:*
options:
k3s:
extraArgs:
- arg: --disable=traefik
nodeFilters:
- server:*
- arg: --disable=servicelb
nodeFilters:
- server:*
- arg: --node-label=easyai.io/site=ningbo
nodeFilters:
- server:0
- arg: --node-label=easyai.io/site=hongkong
nodeFilters:
- server:1
- arg: --node-label=easyai.io/site=los-angeles
nodeFilters:
- server:2
- arg: --node-label=easyai.io/workload=true
nodeFilters:
- server:0
- server:1
- arg: --node-label=easyai.io/database=true
nodeFilters:
- server:0
- server:1
- arg: --node-taint=easyai.io/control-plane-only=true:NoSchedule
nodeFilters:
- server:2
deploy/kubernetes/local-acceptance/local-config.yaml
+54
View File
@@ -0,0 +1,54 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: easyai-ai-gateway-config
namespace: easyai
labels:
app.kubernetes.io/part-of: easyai-ai-gateway
easyai.io/environment: local-acceptance
data:
APP_ENV: production
HTTP_ADDR: ":8088"
IDENTITY_MODE: standalone
IDENTITY_SECRET_STORE: file
IDENTITY_SECRET_DIR: /app/data/identity
IDENTITY_SECURITY_EVENTS_HEARTBEAT_INTERVAL_SECONDS: "0"
IDENTITY_SECURITY_EVENTS_STALE_AFTER_SECONDS: "0"
IDENTITY_SECURITY_EVENTS_CLOCK_SKEW_SECONDS: "0"
SERVER_MAIN_BASE_URL: http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090
TASK_PROGRESS_CALLBACK_ENABLED: "true"
TASK_PROGRESS_CALLBACK_URL: http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks
AI_GATEWAY_PUBLIC_BASE_URL: https://gateway.easyai.local
AI_GATEWAY_WEB_BASE_URL: https://gateway.easyai.local
CORS_ALLOWED_ORIGIN: https://gateway.easyai.local
AI_GATEWAY_GENERATED_STORAGE_DIR: /app/data/static/generated
AI_GATEWAY_UPLOADED_STORAGE_DIR: /app/data/static/uploaded
AI_GATEWAY_LOCAL_TEMP_ASSET_TTL_HOURS: "24"
AI_GATEWAY_LOCAL_RESULT_TTL_HOURS: "24"
AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES: "0"
AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456"
AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912"
AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED: "false"
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT: "24"
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "24"
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48"
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
AI_GATEWAY_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS: "60"
AI_GATEWAY_DATABASE_LOCK_TIMEOUT_SECONDS: "30"
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "true"
AI_GATEWAY_WORKER_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "2"
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "2"
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48"
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20"
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600"
AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS: "600"
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT: "75"
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT: "85"
AI_GATEWAY_NODE_CPU_TARGET_PERCENT: "70"
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET: "150"
AI_GATEWAY_WORKER_DATABASE_MAX_CONNS: "32"
deploy/kubernetes/local-acceptance/support-services.yaml
+312
View File
@@ -0,0 +1,312 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-acceptance-emulator
namespace: easyai
labels:
app.kubernetes.io/name: easyai-acceptance-emulator
easyai.io/environment: local-acceptance
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-acceptance-emulator
template:
metadata:
labels:
app.kubernetes.io/name: easyai-acceptance-emulator
easyai.io/environment: local-acceptance
spec:
nodeSelector:
easyai.io/site: ningbo
containers:
- name: emulator
image: easyai-api
command: ["/app/easyai-ai-gateway-acceptance-emulator"]
env:
- name: HTTP_ADDR
value: ":8090"
ports:
- name: http
containerPort: 8090
readinessProbe:
httpGet:
path: /healthz
port: http
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "2"
memory: 1Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
---
apiVersion: v1
kind: Service
metadata:
name: easyai-acceptance-emulator
namespace: easyai
spec:
selector:
app.kubernetes.io/name: easyai-acceptance-emulator
ports:
- name: http
port: 8090
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-acceptance-callback-collector
namespace: easyai
labels:
app.kubernetes.io/name: easyai-acceptance-callback-collector
easyai.io/environment: local-acceptance
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-acceptance-callback-collector
template:
metadata:
labels:
app.kubernetes.io/name: easyai-acceptance-callback-collector
easyai.io/environment: local-acceptance
spec:
nodeSelector:
easyai.io/site: ningbo
containers:
- name: collector
image: easyai-api
command: ["/app/easyai-ai-gateway-acceptance-callback-collector"]
env:
- name: HTTP_ADDR
value: ":8091"
ports:
- name: http
containerPort: 8091
readinessProbe:
httpGet:
path: /healthz
port: http
resources:
requests:
cpu: 20m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
---
apiVersion: v1
kind: Service
metadata:
name: easyai-acceptance-callback-collector
namespace: easyai
spec:
selector:
app.kubernetes.io/name: easyai-acceptance-callback-collector
ports:
- name: http
port: 8091
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-acceptance-upstream-proxy
namespace: easyai
labels:
app.kubernetes.io/name: easyai-acceptance-upstream-proxy
easyai.io/environment: local-acceptance
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-acceptance-upstream-proxy
template:
metadata:
labels:
app.kubernetes.io/name: easyai-acceptance-upstream-proxy
easyai.io/environment: local-acceptance
spec:
nodeSelector:
easyai.io/site: hongkong
containers:
- name: proxy
image: easyai-acceptance-netem
env:
- name: NETEM_LISTEN_PORT
value: "18090"
- name: NETEM_TARGET_HOST
value: easyai-acceptance-emulator.easyai.svc.cluster.local
- name: NETEM_TARGET_PORT
value: "8090"
ports:
- name: proxy
containerPort: 18090
resources:
requests:
cpu: 20m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add: ["NET_ADMIN"]
drop: ["ALL"]
readOnlyRootFilesystem: true
---
apiVersion: v1
kind: Service
metadata:
name: easyai-acceptance-upstream-proxy
namespace: easyai
spec:
selector:
app.kubernetes.io/name: easyai-acceptance-upstream-proxy
ports:
- name: proxy
port: 8090
targetPort: proxy
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-postgres-proxy-ningbo
namespace: easyai
labels:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: ningbo
easyai.io/environment: local-acceptance
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: ningbo
template:
metadata:
labels:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: ningbo
easyai.io/environment: local-acceptance
spec:
nodeSelector:
easyai.io/site: ningbo
containers:
- name: proxy
image: easyai-acceptance-netem
env:
- name: NETEM_LISTEN_PORT
value: "15432"
- name: NETEM_TARGET_HOST
value: easyai-postgres-rw.easyai.svc.cluster.local
- name: NETEM_TARGET_PORT
value: "5432"
ports:
- name: postgres
containerPort: 15432
resources:
requests:
cpu: 20m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add: ["NET_ADMIN"]
drop: ["ALL"]
readOnlyRootFilesystem: true
---
apiVersion: v1
kind: Service
metadata:
name: easyai-postgres-proxy-ningbo
namespace: easyai
spec:
selector:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: ningbo
ports:
- name: postgres
port: 5432
targetPort: postgres
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-postgres-proxy-hongkong
namespace: easyai
labels:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: hongkong
easyai.io/environment: local-acceptance
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: hongkong
template:
metadata:
labels:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: hongkong
easyai.io/environment: local-acceptance
spec:
nodeSelector:
easyai.io/site: hongkong
containers:
- name: proxy
image: easyai-acceptance-netem
env:
- name: NETEM_LISTEN_PORT
value: "15432"
- name: NETEM_TARGET_HOST
value: easyai-postgres-rw.easyai.svc.cluster.local
- name: NETEM_TARGET_PORT
value: "5432"
ports:
- name: postgres
containerPort: 15432
resources:
requests:
cpu: 20m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add: ["NET_ADMIN"]
drop: ["ALL"]
readOnlyRootFilesystem: true
---
apiVersion: v1
kind: Service
metadata:
name: easyai-postgres-proxy-hongkong
namespace: easyai
spec:
selector:
app.kubernetes.io/name: easyai-postgres-proxy
easyai.io/site: hongkong
ports:
- name: postgres
port: 5432
targetPort: postgres
deploy/kubernetes/local-acceptance/tls-edges.yaml
+172
View File
@@ -0,0 +1,172 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: easyai-acceptance-tls-edge
namespace: easyai
data:
default.conf: |
server {
listen 8443 ssl;
server_name gateway.easyai.local;
client_max_body_size 128m;
proxy_request_buffering off;
proxy_read_timeout 900s;
ssl_certificate /etc/acceptance-tls/tls.crt;
ssl_certificate_key /etc/acceptance-tls/tls.key;
location / {
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://LOCAL_API_SERVICE.easyai.svc.cluster.local:8088;
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-acceptance-edge-ningbo
namespace: easyai
labels:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: ningbo
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: ningbo
template:
metadata:
labels:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: ningbo
spec:
nodeSelector:
easyai.io/site: ningbo
containers:
- name: edge
image: easyai-web
command:
- /bin/sh
- -ec
- sed 's/LOCAL_API_SERVICE/api-ningbo-edge/g' /etc/acceptance-edge/default.conf > /etc/nginx/conf.d/default.conf && exec nginx -g 'daemon off;'
ports:
- name: https
containerPort: 8443
volumeMounts:
- name: config
mountPath: /etc/acceptance-edge
readOnly: true
- name: nginx-config
mountPath: /etc/nginx/conf.d
- name: tls
mountPath: /etc/acceptance-tls
readOnly: true
resources:
requests:
cpu: 50m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
volumes:
- name: config
configMap:
name: easyai-acceptance-tls-edge
- name: tls
secret:
secretName: easyai-acceptance-tls
- name: nginx-config
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: easyai-acceptance-edge-ningbo
namespace: easyai
spec:
type: NodePort
externalTrafficPolicy: Local
selector:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: ningbo
ports:
- name: https
port: 8443
targetPort: https
nodePort: 31088
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-acceptance-edge-hongkong
namespace: easyai
labels:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: hongkong
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: hongkong
template:
metadata:
labels:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: hongkong
spec:
nodeSelector:
easyai.io/site: hongkong
containers:
- name: edge
image: easyai-web
command:
- /bin/sh
- -ec
- sed 's/LOCAL_API_SERVICE/api-hongkong-edge/g' /etc/acceptance-edge/default.conf > /etc/nginx/conf.d/default.conf && exec nginx -g 'daemon off;'
ports:
- name: https
containerPort: 8443
volumeMounts:
- name: config
mountPath: /etc/acceptance-edge
readOnly: true
- name: nginx-config
mountPath: /etc/nginx/conf.d
- name: tls
mountPath: /etc/acceptance-tls
readOnly: true
resources:
requests:
cpu: 50m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
volumes:
- name: config
configMap:
name: easyai-acceptance-tls-edge
- name: tls
secret:
secretName: easyai-acceptance-tls
- name: nginx-config
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: easyai-acceptance-edge-hongkong
namespace: easyai
spec:
type: NodePort
externalTrafficPolicy: Local
selector:
app.kubernetes.io/name: easyai-acceptance-edge
easyai.io/site: hongkong
ports:
- name: https
port: 8443
targetPort: https
nodePort: 31089
deploy/kubernetes/production/application-config.yaml
+17
View File
@@ -43,4 +43,21 @@ data:
AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX: easyai-ai-gateway/production/media
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "48"
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48"
AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "false"
AI_GATEWAY_WORKER_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "1"
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "1"
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48"
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20"
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600"
AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS: "600"
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT: "75"
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT: "85"
AI_GATEWAY_NODE_CPU_TARGET_PERCENT: "70"
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET: "150"
AI_GATEWAY_WORKER_DATABASE_MAX_CONNS: "32"
deploy/kubernetes/production/application.yaml
+170 -6
View File
@@ -53,7 +53,11 @@ spec:
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
value: "false"
- name: AI_GATEWAY_DATABASE_MAX_CONNS
value: "64"
value: "32"
- name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -143,6 +147,128 @@ spec:
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-capacity-controller
labels:
app.kubernetes.io/name: easyai-capacity-controller
app.kubernetes.io/component: capacity-controller
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
replicas: 2
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-capacity-controller
template:
metadata:
labels:
app.kubernetes.io/name: easyai-capacity-controller
app.kubernetes.io/component: capacity-controller
app.kubernetes.io/part-of: easyai-ai-gateway
spec:
serviceAccountName: easyai-capacity-controller
terminationGracePeriodSeconds: 30
nodeSelector:
easyai.io/workload: "true"
topologySpreadConstraints:
- maxSkew: 1
topologyKey: easyai.io/site
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: easyai-capacity-controller
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: capacity-controller
image: easyai-api
imagePullPolicy: IfNotPresent
envFrom:
- configMapRef:
name: easyai-ai-gateway-config
env:
- name: AI_GATEWAY_PROCESS_ROLE
value: capacity-controller
- name: AI_GATEWAY_DATABASE_URL
valueFrom:
secretKeyRef:
name: easyai-ai-gateway-runtime
key: AI_GATEWAY_DATABASE_URL
- name: AI_GATEWAY_DATABASE_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
value: "0"
- name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
value: "0"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "1"
- name: AI_GATEWAY_WORKER_DATABASE_MAX_CONNS
value: "32"
- name: AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET
value: "72"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
ports:
- name: health
containerPort: 8088
startupProbe:
httpGet:
path: /healthz
port: health
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 24
readinessProbe:
httpGet:
path: /readyz
port: health
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 6
livenessProbe:
httpGet:
path: /healthz
port: health
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 6
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
volumeMounts:
- name: temporary-files
mountPath: /tmp
volumes:
- name: temporary-files
emptyDir:
sizeLimit: 64Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: easyai-api-hongkong
labels:
@@ -196,7 +322,11 @@ spec:
- name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
value: "false"
- name: AI_GATEWAY_DATABASE_MAX_CONNS
value: "64"
value: "32"
- name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -313,6 +443,14 @@ spec:
spec:
serviceAccountName: easyai-ai-gateway
terminationGracePeriodSeconds: 90
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: easyai-worker
easyai.io/site: ningbo
nodeSelector:
easyai.io/site: ningbo
easyai.io/workload: "true"
@@ -340,6 +478,10 @@ spec:
value: "true"
- name: AI_GATEWAY_DATABASE_MAX_CONNS
value: "32"
- name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
value: "8"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -406,8 +548,8 @@ spec:
command: ["/bin/sh", "-c", "sleep 10"]
resources:
requests:
cpu: 250m
memory: 512Mi
cpu: 500m
memory: 1536Mi
limits:
cpu: "2"
memory: 2Gi
@@ -458,6 +600,14 @@ spec:
spec:
serviceAccountName: easyai-ai-gateway
terminationGracePeriodSeconds: 90
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: easyai-worker
easyai.io/site: hongkong
nodeSelector:
easyai.io/site: hongkong
easyai.io/workload: "true"
@@ -485,6 +635,10 @@ spec:
value: "true"
- name: AI_GATEWAY_DATABASE_MAX_CONNS
value: "32"
- name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
value: "8"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -551,8 +705,8 @@ spec:
command: ["/bin/sh", "-c", "sleep 10"]
resources:
requests:
cpu: 250m
memory: 512Mi
cpu: 500m
memory: 1536Mi
limits:
cpu: "2"
memory: 2Gi
@@ -896,6 +1050,16 @@ spec:
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: easyai-capacity-controller
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: easyai-capacity-controller
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: easyai-worker
spec:
deploy/kubernetes/production/network-policy.yaml
+1 -1
View File
@@ -7,7 +7,7 @@ spec:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values: [easyai-api, easyai-worker, easyai-web]
values: [easyai-api, easyai-worker, easyai-web, easyai-capacity-controller]
policyTypes: [Ingress]
ingress:
- from:
deploy/kubernetes/production/service-account-rbac.yaml
+65
View File
@@ -29,3 +29,68 @@ roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: easyai-gateway-security-events
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: easyai-capacity-controller
labels:
app.kubernetes.io/name: easyai-capacity-controller
app.kubernetes.io/part-of: easyai-ai-gateway
automountServiceAccountToken: true
imagePullSecrets:
- name: easyai-registry
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: easyai-capacity-controller
rules:
- apiGroups: ["apps"]
resources: ["deployments"]
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
verbs: ["get"]
- apiGroups: ["apps"]
resources: ["deployments/scale"]
resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
verbs: ["get", "update", "patch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: easyai-capacity-controller
subjects:
- kind: ServiceAccount
name: easyai-capacity-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: easyai-capacity-controller
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: easyai-capacity-controller
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list"]
- apiGroups: ["metrics.k8s.io"]
resources: ["nodes", "pods"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: easyai-capacity-controller
subjects:
- kind: ServiceAccount
name: easyai-capacity-controller
namespace: easyai
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: easyai-capacity-controller