BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
edcbbd6a878315dabcf92c1ef7c995e70d756430
easyai-ai-gateway/deploy/kubernetes/easyai-ai-gateway-cluster-release
T
wangbo edcbbd6a87 fix(worker): 拆分准入事务并收敛验收失败状态 
将批量准入改为每任务独立事务，避免 P24 压测时 48 条任务共享长事务造成 transactionid 锁潮与 Worker 槽位空转。失败验收在保持 validation 的同时清理未提交任务，并记录精确压力门禁原因。统一生产 0+2 Worker 拓扑、运行时 ConfigMap 与发布配置，补充镜像预检诊断和回归测试。\n\n验证：Go 全量测试、go vet、真实 PostgreSQL 48 任务集成测试、迁移安全检查、发布脚本测试、bash -n、ShellCheck、Compose 配置均通过。
2026-08-01 18:23:00 +08:00

997 lines
42 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }
config_file=${AI_GATEWAY_CLUSTER_RELEASE_CONFIG:-/etc/easyai-ai-gateway-cluster-release.conf}
manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
[[ -f $config_file && ! -L $config_file ]] || { echo 'missing cluster release config' >&2; exit 1; }
[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
# shellcheck source=/dev/null
source "$config_file"
: "${RELEASES_DIR:?}"
: "${NAMESPACE:=easyai}"
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
: "${K3S_BIN:=/usr/local/bin/k3s}"
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=0}"
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
: "${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB:=1536}"
: "${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES:=500}"
: "${AI_GATEWAY_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_API_DATABASE_MAX_CONNS:=32}"
: "${AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_RIVER_MAX_CONNS:=8}"
: "${AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MIN_IDLE_CONNS:=4}"
: "${AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS:=30}"
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=0}"
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=0}"
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=2}"
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
: "${AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS:=600}"
: "${AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT:=75}"
: "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}"
: "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}"
: "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}"
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}"
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
echo 'invalid cluster release configuration' >&2
exit 1
}
[[ $DESIRED_STATE_DIR == /* &&
-f $DESIRED_STATE_DIR/kustomization.yaml &&
! -L $DESIRED_STATE_DIR/kustomization.yaml ]] || {
echo 'production desired-state bundle is unavailable' >&2
exit 1
}
for replica_value in \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG"; do
[[ $replica_value =~ ^[0-9]+$ ]] || {
echo 'Worker replica configuration must contain non-negative integers' >&2
exit 1
}
done
for capacity_value in \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
"$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET"; do
[[ $capacity_value =~ ^[1-9][0-9]*$ ]] || {
echo 'worker capacity configuration must contain positive integers' >&2
exit 1
}
done
[[ $AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET =~ ^[1-9][0-9]*$ ]] || {
echo 'PostgreSQL non-Worker connection budget must be a positive integer' >&2
exit 1
}
[[ $AI_GATEWAY_DATABASE_MIN_IDLE_CONNS =~ ^[0-9]+$ ]] || {
echo 'database minimum idle connection configuration must be a non-negative integer' >&2
exit 1
}
[[ $AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == true ||
$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED == false ]] || {
echo 'worker autoscaling flag must be true or false' >&2
exit 1
}
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG >= 1 &&
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 &&
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT == AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT &&
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB >= 256 &&
AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB <= 1900 &&
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES >= 100 &&
AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES <= 1900 &&
AI_GATEWAY_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_API_DATABASE_MAX_CONNS <= 256 &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS + AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS < AI_GATEWAY_API_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MIN_IDLE_CONNS <= AI_GATEWAY_API_DATABASE_MAX_CONNS &&
AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS <= 3600 &&
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY <= 256 &&
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 &&
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO &&
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG &&
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 &&
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 &&
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT &&
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 &&
AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 &&
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET &&
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET +
(AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) *
AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || {
echo 'worker capacity configuration exceeds the release safety bounds' >&2
exit 1
}
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
echo 'invalid public base URL' >&2
exit 1
}
for command in "$K3S_BIN" curl flock install jq node sha256sum; do
command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done
kubectl=("$K3S_BIN" kubectl)
ACTIVE_RELEASE_SHA=
install -d -m 0700 "$RELEASES_DIR"
exec 9>"$RELEASES_DIR/.release.lock"
flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; }
manifest_get() {
node "$manifest_tool" get "$1" "$2"
}
capacity_config_hash() {
printf '%s\n' \
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB" \
"$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES" \
"$AI_GATEWAY_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
"$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" |
sha256sum | awk '{print $1}'
}
record_capacity_config() {
local config_hash
local runtime_patch
runtime_patch=$(jq -nc \
--arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
--arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
--arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
--arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
--arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
--arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
--arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
--arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
--arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
--arg workerPool "$AI_GATEWAY_DATABASE_MAX_CONNS" \
--arg materialization "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
--arg mediaRequest "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
--arg targetOutstanding "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
--arg scaleUp "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
--arg scaleDown "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
--arg drainTimeout "$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
--arg memoryTarget "$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
--arg memoryHard "$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
--arg cpuTarget "$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
--arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
'{data:{
AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo,
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong,
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo,
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong,
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo,
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong,
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling,
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit,
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit,
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$globalLimit,
AI_GATEWAY_WORKER_DATABASE_MAX_CONNS:$workerPool,
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:$materialization,
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:$mediaRequest,
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$targetOutstanding,
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:$scaleUp,
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:$scaleDown,
AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS:$drainTimeout,
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT:$memoryTarget,
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:$memoryHard,
AI_GATEWAY_NODE_CPU_TARGET_PERCENT:$cpuTarget,
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:$postgresBudget
}}')
"${kubectl[@]}" patch configmap easyai-ai-gateway-config -n "$NAMESPACE" \
--type merge -p "$runtime_patch" >/dev/null
config_hash=$(capacity_config_hash)
"${kubectl[@]}" patch configmap easyai-gateway-release -n "$NAMESPACE" \
--type merge -p "{\"data\":{\"configHash\":\"$config_hash\"}}" >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
done
}
current_to_file() {
local output=$1
"${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.manifest\.json}' >"$output"
[[ -s $output ]]
node "$manifest_tool" validate "$output" >/dev/null
}
wait_for_url() {
local label=$1
local url=$2
local expected=${3:-}
local body
for _ in $(seq 1 60); do
if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
if [[ -z $expected || $body == *"$expected"* ]]; then
echo "[cluster-release] verified $label"
return 0
fi
fi
sleep 2
done
echo "release probe failed: $label ($url)" >&2
return 1
}
verify_site() {
local site=$1
local address
local api_port
local web_port
case $site in
hongkong)
address=10.77.0.2
api_port=31089
web_port=31179
;;
ningbo)
address=10.77.0.1
api_port=31088
web_port=31178
;;
*) return 1 ;;
esac
wait_for_url "$site API health" "http://$address:$api_port/api/v1/healthz" easyai-ai-gateway
wait_for_url "$site API readiness" "http://$address:$api_port/api/v1/readyz" '"ok":true'
wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
}
apply_desired_state() {
local api_image=$1
local web_image=$2
local capacity_controller_replicas=${3:-2}
local api_repository=${api_image%@*}
local api_digest=${api_image#*@}
local web_repository=${web_image%@*}
local web_digest=${web_image#*@}
local working_directory
working_directory=$(mktemp -d)
cp -a "$DESIRED_STATE_DIR/." "$working_directory/"
sed -i \
-e "/ - name: easyai-api/{n;s| newName:.*| newName: $api_repository|;n;s| digest:.*| digest: $api_digest|;}" \
-e "/ - name: easyai-web/{n;s| newName:.*| newName: $web_repository|;n;s| digest:.*| digest: $web_digest|;}" \
"$working_directory/kustomization.yaml"
if [[ $capacity_controller_replicas == 0 ]]; then
sed -i \
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 2$/ replicas: 0/' \
"$working_directory/application.yaml"
fi
"${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \
--field-manager=easyai-production-preflight \
-k "$working_directory" >/dev/null
"${kubectl[@]}" apply --server-side --force-conflicts \
--field-manager=easyai-production-release \
-k "$working_directory" >/dev/null
rm -rf -- "$working_directory"
}
snapshot_application_deployments() {
local output=$1
"${kubectl[@]}" get deployments -n "$NAMESPACE" \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong \
--ignore-not-found=true \
-o json |
jq '
del(.metadata)
| .items |= map(
del(
.metadata.annotations["deployment.kubernetes.io/revision"],
.metadata.creationTimestamp,
.metadata.generation,
.metadata.managedFields,
.metadata.resourceVersion,
.metadata.uid,
.status
)
)
' >"$output"
}
restore_application_deployments() {
local snapshot=$1
local deployment
[[ -s $snapshot ]] || return 1
"${kubectl[@]}" apply -f "$snapshot" >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller easyai-web-ningbo easyai-web-hongkong; do
if jq -e --arg name "$deployment" \
'any(.items[]; .metadata.name == $name)' "$snapshot" >/dev/null; then
"${kubectl[@]}" rollout status "deployment/$deployment" \
-n "$NAMESPACE" --timeout=300s || true
else
"${kubectl[@]}" delete deployment "$deployment" -n "$NAMESPACE" \
--ignore-not-found=true --wait=true >/dev/null || true
fi
done
}
rollout_worker_site() {
local site=$1
local api_image=$2
local replicas ready_replicas
case $site in
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
*) return 1 ;;
esac
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--containers=worker \
--requests="cpu=${AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB}Mi" \
--limits='cpu=2,memory=2Gi' || return 1
"${kubectl[@]}" scale "deployment/easyai-worker-$site" -n "$NAMESPACE" \
--replicas="$replicas" || return 1
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
"worker=$api_image" || return 1
fi
"${kubectl[@]}" rollout status "deployment/easyai-worker-$site" \
-n "$NAMESPACE" --timeout=300s || return 1
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]] || return 1
ready_replicas=$("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o json | jq -r '.status.readyReplicas // 0') || return 1
[[ $ready_replicas == "$replicas" ]] || return 1
}
rollout_capacity_controller() {
local api_image=${1:-}
"${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
"AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
"AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
"AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS=$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
"AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT=$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
"AI_GATEWAY_NODE_MEMORY_HARD_PERCENT=$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
"AI_GATEWAY_NODE_CPU_TARGET_PERCENT=$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
"AI_GATEWAY_POSTGRES_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
"AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET=$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET" \
"AI_GATEWAY_WORKER_DATABASE_MAX_CONNS=$AI_GATEWAY_DATABASE_MAX_CONNS" \
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" || return 1
if [[ -n $api_image ]]; then
"${kubectl[@]}" set image deployment/easyai-capacity-controller -n "$NAMESPACE" \
"capacity-controller=$api_image" || return 1
fi
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
-n "$NAMESPACE" --replicas=2 || return 1
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
-n "$NAMESPACE" --timeout=300s || return 1
}
run_capacity_controller_preflight() {
local source_sha=$1
local api_image=$2
local pod=easyai-capacity-controller-preflight-${source_sha:0:12}
"${kubectl[@]}" delete pod "$pod" -n "$NAMESPACE" \
--ignore-not-found=true --wait=true >/dev/null
cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
apiVersion: v1
kind: Pod
metadata:
name: $pod
namespace: $NAMESPACE
labels:
easyai.io/preflight: capacity-controller
easyai.io/release: $source_sha
spec:
restartPolicy: Never
serviceAccountName: easyai-capacity-controller
nodeSelector:
easyai.io/workload: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
imagePullSecrets:
- name: easyai-registry
containers:
- name: capacity-controller
image: $api_image
imagePullPolicy: IfNotPresent
envFrom:
- configMapRef:
name: easyai-ai-gateway-config
env:
- name: AI_GATEWAY_PROCESS_ROLE
value: capacity-controller
- name: AI_GATEWAY_DATABASE_URL
valueFrom:
secretKeyRef:
name: easyai-ai-gateway-runtime
key: AI_GATEWAY_DATABASE_URL
- name: AI_GATEWAY_DATABASE_MAX_CONNS
value: "4"
- name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
value: "0"
- name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
value: "0"
- name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
value: "1"
- name: AI_GATEWAY_WORKER_DATABASE_MAX_CONNS
value: "$AI_GATEWAY_DATABASE_MAX_CONNS"
- name: AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET
value: "$AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET"
- name: AI_GATEWAY_WORKER_AUTOSCALING_ENABLED
value: "false"
- name: AI_GATEWAY_REVISION
value: "$source_sha"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
readinessProbe:
httpGet:
path: /readyz
port: 8088
periodSeconds: 2
timeoutSeconds: 2
failureThreshold: 30
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
volumeMounts:
- name: temporary-files
mountPath: /tmp
volumes:
- name: temporary-files
emptyDir:
sizeLimit: 64Mi
EOF
if ! "${kubectl[@]}" wait --for=condition=Ready "pod/$pod" \
-n "$NAMESPACE" --timeout=300s; then
"${kubectl[@]}" get pod "$pod" -n "$NAMESPACE" -o json 2>/dev/null | jq -c '
{
phase:(.status.phase // "unknown"),
node:(.spec.nodeName // "unscheduled"),
ready:([.status.containerStatuses[]?.ready] | all),
restarts:([.status.containerStatuses[]?.restartCount] | add // 0),
waitingReasons:[.status.containerStatuses[]?.state.waiting.reason // empty],
terminatedReasons:[.status.containerStatuses[]?.state.terminated.reason // empty]
}' >&2 || true
"${kubectl[@]}" delete pod "$pod" -n "$NAMESPACE" \
--ignore-not-found=true --wait=true >/dev/null || true
echo 'capacity controller exact-image preflight failed' >&2
return 1
fi
"${kubectl[@]}" delete pod "$pod" -n "$NAMESPACE" \
--ignore-not-found=true --wait=true >/dev/null
echo "capacity_controller_preflight=PASS release=$source_sha"
}
rollout_api_capacity_site() {
local site=$1
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" \
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}"
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
}
apply_capacity_config() {
local capacity_controller_enabled
ACTIVE_RELEASE_SHA=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.releaseSha}' 2>/dev/null || true)
capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
[[ $ACTIVE_RELEASE_SHA =~ ^[0-9a-f]{40}$ ]] || {
echo 'current release SHA is unavailable for capacity update' >&2
return 1
}
rollout_worker_site hongkong ""
rollout_worker_site ningbo ""
rollout_api_capacity_site hongkong
rollout_api_capacity_site ningbo
if [[ $capacity_controller_enabled == true ]]; then
rollout_capacity_controller ""
else
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0
fi
verify_site hongkong
verify_site ningbo
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
record_capacity_config
echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
}
rollout_site() {
local site=$1
local api_image=$2
local web_image=$3
local api_changed=$4
local web_changed=$5
if [[ $api_changed == true ]]; then
"${kubectl[@]}" set env "deployment/easyai-api-$site" -n "$NAMESPACE" \
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
"AI_GATEWAY_DATABASE_MAX_CONNS=$AI_GATEWAY_API_DATABASE_MAX_CONNS" \
"AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=$AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS" \
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_API_DATABASE_RIVER_MAX_CONNS" \
"AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS" \
"AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS" || return 1
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
"api=$api_image" || return 1
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" \
-n "$NAMESPACE" --timeout=300s || return 1
fi
if [[ $web_changed == true ]]; then
"${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
"web=$web_image" || return 1
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" \
-n "$NAMESPACE" --timeout=300s || return 1
fi
verify_site "$site" || return 1
}
run_backup() {
local source_sha=$1
local backup=easyai-pre-release-${source_sha:0:12}
cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
apiVersion: postgresql.cnpg.io/v1
kind: Backup
metadata:
name: $backup
namespace: $NAMESPACE
spec:
cluster:
name: easyai-postgres
method: plugin
pluginConfiguration:
name: barman-cloud.cloudnative-pg.io
EOF
for _ in $(seq 1 120); do
phase=$("${kubectl[@]}" get backup "$backup" -n "$NAMESPACE" \
-o jsonpath='{.status.phase}' 2>/dev/null || true)
[[ ${phase,,} == completed ]] && return 0
[[ ${phase,,} == failed ]] && break
sleep 5
done
echo "pre-migration backup failed: $backup phase=${phase:-unknown}" >&2
return 1
}
run_migrator() {
local source_sha=$1
local api_image=$2
local job=easyai-migrator-${source_sha:0:12}
"${kubectl[@]}" delete job "$job" -n "$NAMESPACE" \
--ignore-not-found=true --wait=true >/dev/null
cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
name: $job
namespace: $NAMESPACE
labels:
easyai.io/release: $source_sha
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 86400
template:
spec:
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile:
type: RuntimeDefault
imagePullSecrets:
- name: easyai-registry
nodeSelector:
easyai.io/site: ningbo
containers:
- name: migrator
image: $api_image
command: ["/app/easyai-ai-gateway-migrate"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
envFrom:
- secretRef:
name: easyai-ai-gateway-runtime
EOF
"${kubectl[@]}" wait --for=condition=complete "job/$job" \
-n "$NAMESPACE" --timeout=300s
}
verify_migration_version() {
local expected=$1
[[ $expected =~ ^[0-9]{4}_[a-z0-9][a-z0-9_]*$ ]] || {
echo 'release manifest migration version is invalid' >&2
return 1
}
local primary actual
primary=$("${kubectl[@]}" get cluster easyai-postgres -n "$NAMESPACE" \
-o jsonpath='{.status.currentPrimary}')
[[ $primary =~ ^easyai-postgres-[0-9]+$ ]] || {
echo 'PostgreSQL primary is unavailable for migration verification' >&2
return 1
}
actual=$("${kubectl[@]}" exec -n "$NAMESPACE" "$primary" -c postgres -- \
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At \
-c 'SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1')
[[ $actual == "$expected" ]] || {
echo "production migration version mismatch: expected=$expected actual=${actual:-missing}" >&2
return 1
}
}
record_current() {
local manifest=$1
local action=$2
local started_at=$3
local source_sha recorded temporary capacity_controller_enabled=true
if [[ $action == rollback ]]; then
capacity_controller_enabled=false
fi
source_sha=$(manifest_get "$manifest" sourceSha)
recorded=$RELEASES_DIR/$source_sha.json
temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX")
rm -f -- "$temporary"
RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
RELEASE_DEPLOY_ACTION=$action \
node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null
install -m 0600 "$temporary" "$recorded"
"${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
--from-file=manifest.json="$recorded" \
--from-literal=configHash="$(capacity_config_hash)" \
--from-literal=capacityControllerEnabled="$capacity_controller_enabled" \
--from-literal=releaseSha="$source_sha" \
--dry-run=client -o yaml |
"${kubectl[@]}" apply -f - >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/release=$source_sha" \
"easyai.io/config-hash=$(capacity_config_hash)" \
--overwrite >/dev/null
for deployment in \
easyai-api-ningbo easyai-api-hongkong \
easyai-worker-ningbo easyai-worker-hongkong \
easyai-capacity-controller \
easyai-web-ningbo easyai-web-hongkong; do
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
"easyai.io/release=$source_sha" \
"easyai.io/config-hash=$(capacity_config_hash)" \
--overwrite >/dev/null
done
rm -f -- "$temporary"
}
record_certification() {
local run_id=$1
local capacity_profile=$2
local config_hash=$3
local instance_slots=$4
local max_ningbo=$5
local max_hongkong=$6
[[ $run_id =~ ^[0-9a-f-]{36}$ &&
$capacity_profile =~ ^P(24|28|32)$ &&
$config_hash =~ ^[0-9a-f]{64}$ &&
$instance_slots =~ ^[1-9][0-9]*$ &&
$max_ningbo =~ ^[1-9][0-9]*$ &&
$max_hongkong =~ ^[1-9][0-9]*$ ]] || {
echo 'invalid production certification metadata' >&2
return 1
}
local effective_config_hash
effective_config_hash=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.configHash}' 2>/dev/null || true)
[[ $effective_config_hash == "$config_hash" ]] || {
echo 'production certification config hash does not match the deployed release configuration' >&2
return 1
}
local current_file source_sha recorded temporary capacity_controller_enabled
current_file=$(mktemp "$RELEASES_DIR/.certification-current.XXXXXX")
current_to_file "$current_file"
source_sha=$(manifest_get "$current_file" sourceSha)
recorded=$RELEASES_DIR/$source_sha.json
temporary=$(mktemp "$RELEASES_DIR/.certification.XXXXXX")
rm -f -- "$temporary"
RELEASE_CERTIFIED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
RELEASE_ACCEPTANCE_RUN_ID=$run_id \
RELEASE_CONFIG_HASH=$config_hash \
RELEASE_CAPACITY_PROFILE=$capacity_profile \
RELEASE_WORKER_INSTANCE_SLOTS=$instance_slots \
RELEASE_WORKER_MAX_REPLICAS_NINGBO=$max_ningbo \
RELEASE_WORKER_MAX_REPLICAS_HONGKONG=$max_hongkong \
node "$manifest_tool" record-certification "$current_file" "$temporary" >/dev/null
install -m 0600 "$temporary" "$recorded"
capacity_controller_enabled=$("${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
-o jsonpath='{.data.capacityControllerEnabled}' 2>/dev/null || true)
"${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
--from-file=manifest.json="$recorded" \
--from-literal=configHash="$config_hash" \
--from-literal=capacityControllerEnabled="${capacity_controller_enabled:-true}" \
--from-literal=releaseSha="$source_sha" \
--from-literal=certifiedRunId="$run_id" \
--from-literal=certifiedCapacityProfile="$capacity_profile" \
--dry-run=client -o yaml |
"${kubectl[@]}" apply -f - >/dev/null
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
"easyai.io/certified-run=$run_id" \
"easyai.io/capacity-profile=$capacity_profile" \
--overwrite >/dev/null
rm -f -- "$temporary" "$current_file"
echo "production_certification=PASS release=$source_sha run_id=$run_id capacity_profile=$capacity_profile config_hash=$config_hash"
}
activate_manifest() {
local manifest=$1
local action=${2:-deploy}
local source_sha base_sha api_image web_image migrations_changed migration_version components
local api_changed=false web_changed=false current_file deployment_snapshot previous_api previous_web started_at
local capacity_controller_replicas
[[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
node "$manifest_tool" validate "$manifest" >/dev/null
source_sha=$(manifest_get "$manifest" sourceSha)
ACTIVE_RELEASE_SHA=$source_sha
base_sha=$(manifest_get "$manifest" baseReleaseSha)
api_image=$(manifest_get "$manifest" images.api)
web_image=$(manifest_get "$manifest" images.web)
migrations_changed=$(manifest_get "$manifest" migrationsChanged)
migration_version=$(manifest_get "$manifest" migrationVersion)
components=$(manifest_get "$manifest" components)
[[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || {
echo 'release images must be digest-pinned' >&2
return 1
}
[[ $components == *'"api"'* ]] && api_changed=true
[[ $components == *'"web"'* ]] && web_changed=true
if [[ $action == rollback ]]; then
api_changed=true
web_changed=true
fi
current_file=$(mktemp "$RELEASES_DIR/.current.XXXXXX")
if current_to_file "$current_file"; then
current_sha=$(manifest_get "$current_file" sourceSha)
previous_api=$(manifest_get "$current_file" images.api)
previous_web=$(manifest_get "$current_file" images.web)
if [[ $action == deploy && $current_sha != "$base_sha" ]]; then
echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
rm -f -- "$current_file"
return 1
fi
else
if [[ $action == deploy && -n $base_sha ]]; then
echo 'cluster release status is unavailable for a non-bootstrap manifest' >&2
rm -f -- "$current_file"
return 1
fi
previous_api=
previous_web=
fi
started_at=$(date +%s)
deployment_snapshot=$(mktemp "$RELEASES_DIR/.deployments.XXXXXX")
snapshot_application_deployments "$deployment_snapshot"
if [[ $action == deploy && $migrations_changed == true ]]; then
run_backup "$source_sha"
run_migrator "$source_sha" "$api_image"
fi
if [[ $action == deploy ]]; then
verify_migration_version "$migration_version"
fi
capacity_controller_replicas=0
if ! apply_desired_state "$api_image" "$web_image" "$capacity_controller_replicas"; then
echo '[cluster-release] desired state apply failed; restoring exact deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
if [[ $action == deploy && $api_changed == true ]] &&
! run_capacity_controller_preflight "$source_sha" "$api_image"; then
echo '[cluster-release] capacity controller preflight failed; restoring exact deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
if [[ $action == rollback ]]; then
# Historical API images may predate the capacity-controller process role.
# Rollbacks therefore restore static Worker replicas and leave the
# controller at zero instead of ever starting an incompatible image.
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
-n "$NAMESPACE" --replicas=0
fi
if { [[ $api_changed != true ]] ||
{ rollout_worker_site hongkong "$api_image" &&
rollout_worker_site ningbo "$api_image" &&
{ [[ $action == rollback ]] ||
rollout_capacity_controller "$api_image"; }; }; } &&
rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
:
else
if [[ -n $previous_api && -n $previous_web ]]; then
echo '[cluster-release] restoring exact pre-release Deployment snapshot' >&2
restore_application_deployments "$deployment_snapshot" || true
fi
rm -f -- "$deployment_snapshot" "$current_file"
return 1
fi
record_capacity_config
record_current "$manifest" "$action" "$started_at"
rm -f -- "$deployment_snapshot" "$current_file"
echo "production_release=PASS action=$action release=$source_sha"
}
case ${1:-} in
status)
[[ $# -eq 1 ]] || exit 64
status_file=$(mktemp "$RELEASES_DIR/.status.XXXXXX")
current_to_file "$status_file"
cat "$status_file"
rm -f -- "$status_file"
;;
adopt)
[[ $# -eq 2 ]] || exit 64
node "$manifest_tool" validate "$2" >/dev/null
api_image=$(manifest_get "$2" images.api)
web_image=$(manifest_get "$2" images.web)
for site in ningbo hongkong; do
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
done
[[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]]
[[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
[[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="web")].image}') == "$web_image" ]]
record_current "$2" deploy "$(date +%s)"
;;
deploy)
[[ $# -eq 2 ]] || exit 64
activate_manifest "$2" deploy
;;
rollback)
[[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
target=$RELEASES_DIR/$2.json
[[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
activate_manifest "$target" rollback
;;
capacity)
[[ $# -eq 1 ]] || exit 64
apply_capacity_config
;;
certify)
[[ $# -eq 7 ]] || exit 64
record_certification "$2" "$3" "$4" "$5" "$6" "$7"
;;
*)
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>|capacity|certify <run-id> <profile> <config-hash> <slots> <max-ningbo> <max-hongkong>}' >&2
exit 64
;;
esac
Reference in New Issue View Git Blame Copy Permalink