BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(acceptance): 建立同构验收与弹性容量体系

Browse Source 
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
wangbo
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/kubernetes/easyai-ai-gateway-cluster-release
+413 -22
View File
@@ -14,50 +14,127 @@ source "$config_file"
: "${NAMESPACE:=easyai}"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
: "${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB:=1536}"
: "${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES:=500}"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=64}"
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_RIVER_MAX_CONNS:=8}"
: "${AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS:=30}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
: "${AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS:=600}"
: "${AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT:=75}"
: "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}"
: "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}"
: "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}"
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
echo 'invalid cluster release configuration' >&2
exit 1
}
[[ $DESIRED_STATE_DIR == /* &&
-f $DESIRED_STATE_DIR/kustomization.yaml &&
! -L $DESIRED_STATE_DIR/kustomization.yaml ]] || {
echo 'production desired-state bundle is unavailable' >&2
exit 1
}
for capacity_value in \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
"$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"; do
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET"; do
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
echo 'worker capacity configuration must contain positive integers' >&2
exit 1
}
done
[[ $AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET =~ ^[1-9][0-9]*$ ]] || {
echo 'PostgreSQL non-Worker connection budget must be a positive integer' >&2
exit 1
}
[[ $AI_GATEWAY_DATABASE_MIN_IDLE_CONNS =~ ^[0-9]+$ ]] || {
echo 'database minimum idle connection configuration must be a non-negative integer' >&2
exit 1
}
[[ $AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == true ||
$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == false ]] || {
echo 'worker autoscaling flag must be true or false' >&2
exit 1
}
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT == AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT &&
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB >= 256 &&
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB <= 1900 &&
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES >= 100 &&
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES <= 1900 &&
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_API_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS <= 3600 &&
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 )) || {
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 &&
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT &&
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 &&
AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 &&
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET +
(AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) *
AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || {
echo 'worker capacity configuration exceeds the release safety bounds' >&2
exit 1
}
@@ -65,11 +142,12 @@ done
echo 'invalid public base URL' >&2
exit 1
}
for command in "$K3S_BIN" curl flock install jq node; do
for command in "$K3S_BIN" curl flock install jq node sha256sum; do
command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done
kubectl=("$K3S_BIN" kubectl)
ACTIVE_RELEASE_SHA=
install -d -m 0700 "$RELEASES_DIR"
exec 9>"$RELEASES_DIR/.release.lock"
flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; }
@@ -78,6 +156,57 @@ manifest_get() {
node "$manifest_tool" get "$1" "$2"
}
capacity_config_hash() {
printf '%s\n' \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
"$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
"$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" |
sha256sum | awk '{print $1}'
}
record_capacity_config() {
local config_hash
config_hash=$(capacity_config_hash)
"${kubectl[@]}" patch configmap easyai-gateway-release -n "$NAMESPACE" \
--type merge -p "{\"data\":{\"configHash\":\"$config_hash\"}}" >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
done
}
current_to_file() {
local output=$1
"${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
@@ -127,6 +256,71 @@ verify_site() {
wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
}
apply_desired_state() {
local api_image=$1
local web_image=$2
local capacity_controller_replicas=${3:-2}
local api_repository=${api_image%@*}
local api_digest=${api_image#*@}
local web_repository=${web_image%@*}
local web_digest=${web_image#*@}
local working_directory
working_directory=$(mktemp -d)
cp -a "$DESIRED_STATE_DIR/." "$working_directory/"
sed -i \
-e "/ - name: easyai-api/{n;s| newName:.*| newName: $api_repository|;n;s| digest:.*| digest: $api_digest|;}" \
-e "/ - name: easyai-web/{n;s| newName:.*| newName: $web_repository|;n;s| digest:.*| digest: $web_digest|;}" \
"$working_directory/kustomization.yaml"
if [[ $capacity_controller_replicas == 0 ]]; then
sed -i \
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 2$/ replicas: 0/' \
"$working_directory/application.yaml"
fi
"${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \
--field-manager=easyai-production-preflight \
-k "$working_directory" >/dev/null
"${kubectl[@]}" apply --server-side --force-conflicts \
--field-manager=easyai-production-release \
-k "$working_directory" >/dev/null
rm -rf -- "$working_directory"
}
snapshot_application_deployments() {
local output=$1
"${kubectl[@]}" get deployments -n "$NAMESPACE" \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong \
-o json |
jq '
del(.metadata)
| .items |= map(
del(
.metadata.annotations["deployment.kubernetes.io/revision"],
.metadata.creationTimestamp,
.metadata.generation,
.metadata.managedFields,
.metadata.resourceVersion,
.metadata.uid,
.status
)
)
' >"$output"
}
restore_application_deployments() {
local snapshot=$1
[[ -s $snapshot ]] || return 1
"${kubectl[@]}" apply -f "$snapshot" >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong; do
"${kubectl[@]}" rollout status "deployment/$deployment" \
-n "$NAMESPACE" --timeout=300s || true
done
}
rollout_worker_site() {
local site=$1
local api_image=$2
@@ -139,11 +333,27 @@ rollout_worker_site() {
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY"
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA"
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--containers=worker \
--requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
--limits='cpu=2,memory=2Gi'
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--replicas="$replicas"
if [[ -n $api_image ]]; then
@@ -157,25 +367,74 @@ rollout_worker_site() {
-o jsonpath='{.status.readyReplicas}') == "$replicas" ]]
}
rollout_capacity_controller() {
local api_image=${1:-}
"${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"AI_GATEWAY_NODE_CPU_TARGET_PERCENT=$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
"AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" \
"AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
"capacity-controller=$api_image"
fi
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller -n "$NAMESPACE" --timeout=300s
}
rollout_api_capacity_site() {
local site=$1
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}
apply_capacity_config() {
local capacity_controller_enabled
ACTIVE_RELEASE_SHA=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.releaseSha}' 2>/dev/null || true)
capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
[[ $ACTIVE_RELEASE_SHA =~ ^[0-9a-f]{40}$ ]] || {
echo 'current release SHA is unavailable for capacity update' >&2
return 1
}
rollout_worker_site hongkong ""
rollout_worker_site ningbo ""
rollout_api_capacity_site hongkong
rollout_api_capacity_site ningbo
if [[ $capacity_controller_enabled == true ]]; then
rollout_capacity_controller ""
else
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0
fi
verify_site hongkong
verify_site ningbo
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
echo "production_capacity=PASS ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY"
record_capacity_config
echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
}
rollout_site() {
@@ -187,7 +446,10 @@ rollout_site() {
if [[ $api_changed == true ]]; then
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS"
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
@@ -265,11 +527,36 @@ EOF
-n "$NAMESPACE" --timeout=300s
}
verify_migration_version() {
local expected=$1
[[ $expected =~ ^[0-9]{4}_[a-z0-9][a-z0-9_]*$ ]] || {
echo 'release manifest migration version is invalid' >&2
return 1
}
local primary actual
primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
-o jsonpath='{.status.currentPrimary}')
[[ $primary =~ ^easyai-postgres-[0-9]+$ ]] || {
echo 'PostgreSQL primary is unavailable for migration verification' >&2
return 1
}
actual=$("${kubectl[@]}" exec -n "$NAMESPACE" "$primary" -c postgres -- \
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At \
-c 'SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1')
[[ $actual == "$expected" ]] || {
echo "production migration version mismatch: expected=$expected actual=${actual:-missing}" >&2
return 1
}
}
record_current() {
local manifest=$1
local action=$2
local started_at=$3
local source_sha recorded temporary
local source_sha recorded temporary capacity_controller_enabled=true
if [[ $action == rollback ]]; then
capacity_controller_enabled=false
fi
source_sha=$(manifest_get "$manifest" sourceSha)
recorded=$RELEASES_DIR/$source_sha.json
temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX")
@@ -280,23 +567,102 @@ record_current() {
node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null
install -m 0600 "$temporary" "$recorded"
"${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
--from-file=manifest.json="$recorded" --dry-run=client -o yaml |
--from-file=manifest.json="$recorded" \
--from-literal=configHash="$(capacity_config_hash)" \
--from-literal=capacityControllerEnabled="$capacity_controller_enabled" \
--from-literal=releaseSha="$source_sha" \
--dry-run=client -o yaml |
"${kubectl[@]}" apply -f - >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/release=$source_sha" \
"easyai.io/config-hash=$(capacity_config_hash)" \
--overwrite >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller \
easyai-web-ningbo easyai-web-hongkong; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/release=$source_sha" \
"easyai.io/config-hash=$(capacity_config_hash)" \
--overwrite >/dev/null
done
rm -f -- "$temporary"
}
record_certification() {
local run_id=$1
local capacity_profile=$2
local config_hash=$3
local instance_slots=$4
local max_ningbo=$5
local max_hongkong=$6
[[ $run_id =~ ^[0-9a-f-]{36}$ &&
$capacity_profile =~ ^P(24|28|32)$ &&
$config_hash =~ ^[0-9a-f]{64}$ &&
$instance_slots =~ ^[1-9][0-9]*$ &&
$max_ningbo =~ ^[1-9][0-9]*$ &&
$max_hongkong =~ ^[1-9][0-9]*$ ]] || {
echo 'invalid production certification metadata' >&2
return 1
}
local effective_config_hash
effective_config_hash=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.configHash}' 2>/dev/null || true)
[[ $effective_config_hash == "$config_hash" ]] || {
echo 'production certification config hash does not match the deployed release configuration' >&2
return 1
}
local current_file source_sha recorded temporary capacity_controller_enabled
current_file=$(mktemp "$RELEASES_DIR/.certification-current.XXXXXX")
current_to_file "$current_file"
source_sha=$(manifest_get "$current_file" sourceSha)
recorded=$RELEASES_DIR/$source_sha.json
temporary=$(mktemp "$RELEASES_DIR/.certification.XXXXXX")
rm -f -- "$temporary"
RELEASE_CERTIFIED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
RELEASE_ACCEPTANCE_RUN_ID=$run_id \
RELEASE_CONFIG_HASH=$config_hash \
RELEASE_CAPACITY_PROFILE=$capacity_profile \
RELEASE_WORKER_INSTANCE_SLOTS=$instance_slots \
RELEASE_WORKER_MAX_REPLICAS_NINGBO=$max_ningbo \
RELEASE_WORKER_MAX_REPLICAS_HONGKONG=$max_hongkong \
node "$manifest_tool" record-certification "$current_file" "$temporary" >/dev/null
install -m 0600 "$temporary" "$recorded"
capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
"${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
--from-file=manifest.json="$recorded" \
--from-literal=configHash="$config_hash" \
--from-literal=capacityControllerEnabled="${capacity_controller_enabled:-true}" \
--from-literal=releaseSha="$source_sha" \
--from-literal=certifiedRunId="$run_id" \
--from-literal=certifiedCapacityProfile="$capacity_profile" \
--dry-run=client -o yaml |
"${kubectl[@]}" apply -f - >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/certified-run=$run_id" \
"easyai.io/capacity-profile=$capacity_profile" \
--overwrite >/dev/null
rm -f -- "$temporary" "$current_file"
echo "production_certification=PASS release=$source_sha run_id=$run_id capacity_profile=$capacity_profile config_hash=$config_hash"
}
activate_manifest() {
local manifest=$1
local action=${2:-deploy}
local source_sha base_sha api_image web_image migrations_changed components
local api_changed=false web_changed=false current_file previous_api previous_web started_at
local source_sha base_sha api_image web_image migrations_changed migration_version components
local api_changed=false web_changed=false current_file deployment_snapshot previous_api previous_web started_at
local capacity_controller_replicas
[[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
node "$manifest_tool" validate "$manifest" >/dev/null
source_sha=$(manifest_get "$manifest" sourceSha)
ACTIVE_RELEASE_SHA=$source_sha
base_sha=$(manifest_get "$manifest" baseReleaseSha)
api_image=$(manifest_get "$manifest" images.api)
web_image=$(manifest_get "$manifest" images.web)
migrations_changed=$(manifest_get "$manifest" migrationsChanged)
migration_version=$(manifest_get "$manifest" migrationVersion)
components=$(manifest_get "$manifest" components)
[[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || {
echo 'release images must be digest-pinned' >&2
@@ -329,35 +695,54 @@ activate_manifest() {
previous_web=
fi
started_at=$(date +%s)
deployment_snapshot=$(mktemp "$RELEASES_DIR/.deployments.XXXXXX")
snapshot_application_deployments "$deployment_snapshot"
if [[ $action == deploy && $migrations_changed == true ]]; then
run_backup "$source_sha"
run_migrator "$source_sha" "$api_image"
fi
if [[ $action == deploy ]]; then
verify_migration_version "$migration_version"
fi
capacity_controller_replicas=2
if [[ $action == rollback ]]; then
capacity_controller_replicas=0
fi
if ! apply_desired_state "$api_image" "$web_image" "$capacity_controller_replicas"; then
echo '[cluster-release] desired state apply failed; restoring exact deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
if [[ $action == rollback ]]; then
# Historical API images may predate the capacity-controller process role.
# Rollbacks therefore restore static Worker replicas and leave the
# controller at zero instead of ever starting an incompatible image.
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
-n "$NAMESPACE" --replicas=0
fi
if { [[ $api_changed != true ]] ||
{ rollout_worker_site hongkong "$api_image" &&
rollout_worker_site ningbo "$api_image"; }; } &&
rollout_worker_site ningbo "$api_image" &&
{ [[ $action == rollback ]] ||
rollout_capacity_controller "$api_image"; }; }; } &&
rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
:
else
if [[ -n $previous_api && -n $previous_web ]]; then
echo '[cluster-release] restoring previous application images' >&2
if [[ $api_changed == true ]]; then
rollout_worker_site hongkong "$previous_api" || true
rollout_worker_site ningbo "$previous_api" || true
fi
rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
echo '[cluster-release] restoring exact pre-release Deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
fi
rm -f -- "$current_file"
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
record_current "$manifest" "$action" "$started_at"
rm -f -- "$current_file"
rm -f -- "$deployment_snapshot" "$current_file"
echo "production_release=PASS action=$action release=$source_sha"
}
@@ -378,6 +763,8 @@ case ${1:-} in
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
done
[[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]]
[[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
[[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
@@ -398,8 +785,12 @@ case ${1:-} in
[[ $# -eq 1 ]] || exit 64
apply_capacity_config
;;
certify)
[[ $# -eq 7 ]] || exit 64
record_certification "$2" "$3" "$4" "$5" "$6" "$7"
;;
*)
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity}' >&2
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity|certify <run-id> <profile> <config-hash> <slots> <max-ningbo> <max-hongkong>}' >&2
exit 64
;;
esac