|
|
|
@@ -15,8 +15,7 @@ source "$config_file"
|
|
|
|
|
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
|
|
|
|
|
: "${K3S_BIN:=/usr/local/bin/k3s}"
|
|
|
|
|
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=0}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=2}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
|
|
|
|
|
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
|
|
|
|
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
|
|
|
|
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
|
|
|
|
@@ -33,10 +32,8 @@ source "$config_file"
|
|
|
|
|
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
|
|
|
|
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=0}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=0}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=2}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
|
|
|
|
@@ -45,7 +42,7 @@ source "$config_file"
|
|
|
|
|
: "${AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:=85}"
|
|
|
|
|
: "${AI_GATEWAY_NODE_CPU_TARGET_PERCENT:=70}"
|
|
|
|
|
: "${AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:=150}"
|
|
|
|
|
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS * 2 + 8))}"
|
|
|
|
|
: "${AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET:=$((AI_GATEWAY_API_DATABASE_MAX_CONNS + 4))}"
|
|
|
|
|
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
|
|
|
|
echo 'invalid cluster release configuration' >&2
|
|
|
|
|
exit 1
|
|
|
|
@@ -58,11 +55,8 @@ source "$config_file"
|
|
|
|
|
}
|
|
|
|
|
for replica_value in \
|
|
|
|
|
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG"; do
|
|
|
|
|
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO"; do
|
|
|
|
|
[[ $replica_value =~ ^[0-9]+$ ]] || {
|
|
|
|
|
echo 'Worker replica configuration must contain non-negative integers' >&2
|
|
|
|
|
exit 1
|
|
|
|
@@ -110,8 +104,7 @@ done
|
|
|
|
|
exit 1
|
|
|
|
|
}
|
|
|
|
|
(( AI_GATEWAY_WORKER_REPLICAS_NINGBO <= 16 &&
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= 16 &&
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG >= 1 &&
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO >= 1 &&
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT <= 256 &&
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT <= 512 &&
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT <= 512 &&
|
|
|
|
@@ -133,17 +126,14 @@ done
|
|
|
|
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY <= 1024 &&
|
|
|
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_REPLICAS_NINGBO &&
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO <= AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO &&
|
|
|
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_REPLICAS_HONGKONG &&
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG <= AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG &&
|
|
|
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO <= 16 &&
|
|
|
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG <= 16 &&
|
|
|
|
|
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT < AI_GATEWAY_NODE_MEMORY_HARD_PERCENT &&
|
|
|
|
|
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT <= 95 &&
|
|
|
|
|
AI_GATEWAY_NODE_CPU_TARGET_PERCENT <= 90 &&
|
|
|
|
|
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET <= 150 &&
|
|
|
|
|
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET < AI_GATEWAY_POSTGRES_CONNECTION_BUDGET &&
|
|
|
|
|
AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET +
|
|
|
|
|
(AI_GATEWAY_WORKER_REPLICAS_NINGBO + AI_GATEWAY_WORKER_REPLICAS_HONGKONG) *
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO *
|
|
|
|
|
AI_GATEWAY_DATABASE_MAX_CONNS <= AI_GATEWAY_POSTGRES_CONNECTION_BUDGET )) || {
|
|
|
|
|
echo 'worker capacity configuration exceeds the release safety bounds' >&2
|
|
|
|
|
exit 1
|
|
|
|
@@ -169,7 +159,6 @@ manifest_get() {
|
|
|
|
|
capacity_config_hash() {
|
|
|
|
|
printf '%s\n' \
|
|
|
|
|
"$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
|
|
|
|
"$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
|
|
|
|
"$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT" \
|
|
|
|
|
"$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
|
|
|
|
@@ -187,9 +176,7 @@ capacity_config_hash() {
|
|
|
|
|
"$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
|
|
|
|
|
"$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
|
|
|
|
@@ -207,11 +194,8 @@ record_capacity_config() {
|
|
|
|
|
local runtime_patch
|
|
|
|
|
runtime_patch=$(jq -nc \
|
|
|
|
|
--arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
|
|
|
|
--arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
|
|
|
|
--arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
|
|
|
|
--arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
|
|
|
|
--arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
|
|
|
|
--arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
|
|
|
|
--arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
|
|
|
|
--arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
|
|
|
|
--arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
|
|
|
|
@@ -229,11 +213,11 @@ record_capacity_config() {
|
|
|
|
|
--arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
|
|
|
|
|
'{data:{
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo,
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong,
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:null,
|
|
|
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo,
|
|
|
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong,
|
|
|
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:null,
|
|
|
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo,
|
|
|
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong,
|
|
|
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:null,
|
|
|
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling,
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit,
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit,
|
|
|
|
@@ -259,8 +243,8 @@ record_capacity_config() {
|
|
|
|
|
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
|
|
|
|
|
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
|
|
|
|
|
for deployment in \
|
|
|
|
|
easyai-api-ningbo easyai-api-hongkong \
|
|
|
|
|
easyai-worker-ningbo easyai-worker-hongkong \
|
|
|
|
|
easyai-api-ningbo \
|
|
|
|
|
easyai-worker-ningbo \
|
|
|
|
|
easyai-capacity-controller; do
|
|
|
|
|
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
|
|
|
|
|
"easyai.io/config-hash=$config_hash" --overwrite >/dev/null
|
|
|
|
@@ -299,11 +283,6 @@ verify_site() {
|
|
|
|
|
local api_port
|
|
|
|
|
local web_port
|
|
|
|
|
case $site in
|
|
|
|
|
hongkong)
|
|
|
|
|
address=10.77.0.2
|
|
|
|
|
api_port=31089
|
|
|
|
|
web_port=31179
|
|
|
|
|
;;
|
|
|
|
|
ningbo)
|
|
|
|
|
address=10.77.0.1
|
|
|
|
|
api_port=31088
|
|
|
|
@@ -316,10 +295,19 @@ verify_site() {
|
|
|
|
|
wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
retire_hongkong_site() {
|
|
|
|
|
"${kubectl[@]}" delete deployment \
|
|
|
|
|
easyai-api-hongkong easyai-worker-hongkong easyai-web-hongkong \
|
|
|
|
|
-n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null
|
|
|
|
|
"${kubectl[@]}" delete service \
|
|
|
|
|
api-hongkong-edge web-hongkong-edge \
|
|
|
|
|
-n "$NAMESPACE" --ignore-not-found=true --wait=true >/dev/null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
apply_desired_state() {
|
|
|
|
|
local api_image=$1
|
|
|
|
|
local web_image=$2
|
|
|
|
|
local capacity_controller_replicas=${3:-2}
|
|
|
|
|
local capacity_controller_replicas=${3:-1}
|
|
|
|
|
local api_repository=${api_image%@*}
|
|
|
|
|
local api_digest=${api_image#*@}
|
|
|
|
|
local web_repository=${web_image%@*}
|
|
|
|
@@ -333,7 +321,7 @@ apply_desired_state() {
|
|
|
|
|
"$working_directory/kustomization.yaml"
|
|
|
|
|
if [[ $capacity_controller_replicas == 0 ]]; then
|
|
|
|
|
sed -i \
|
|
|
|
|
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 2$/ replicas: 0/' \
|
|
|
|
|
'/^ name: easyai-capacity-controller$/,/^---$/s/^ replicas: 1$/ replicas: 0/' \
|
|
|
|
|
"$working_directory/application.yaml"
|
|
|
|
|
fi
|
|
|
|
|
"${kubectl[@]}" apply --server-side --dry-run=server --force-conflicts \
|
|
|
|
@@ -395,7 +383,6 @@ rollout_worker_site() {
|
|
|
|
|
local replicas ready_replicas
|
|
|
|
|
case $site in
|
|
|
|
|
ningbo) replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO ;;
|
|
|
|
|
hongkong) replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG ;;
|
|
|
|
|
*) return 1 ;;
|
|
|
|
|
esac
|
|
|
|
|
"${kubectl[@]}" set env "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
|
|
|
@@ -414,11 +401,8 @@ rollout_worker_site() {
|
|
|
|
|
"AI_GATEWAY_REVISION=${ACTIVE_RELEASE_SHA:-}" \
|
|
|
|
|
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
|
|
|
|
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
|
|
|
|
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
|
|
|
|
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" || return 1
|
|
|
|
|
"${kubectl[@]}" set resources "deployment/easyai-worker-$site" -n "$NAMESPACE" \
|
|
|
|
|
--containers=worker \
|
|
|
|
@@ -446,11 +430,8 @@ rollout_capacity_controller() {
|
|
|
|
|
"${kubectl[@]}" set env deployment/easyai-capacity-controller -n "$NAMESPACE" \
|
|
|
|
|
"AI_GATEWAY_WORKER_AUTOSCALING_ENABLED=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
|
|
|
|
"AI_GATEWAY_WORKER_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
|
|
|
|
"AI_GATEWAY_WORKER_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO=$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
|
|
|
|
"AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG=$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
|
|
|
|
"AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA=$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
|
|
|
|
|
"AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS=$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
|
|
|
|
|
"AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS=$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
|
|
|
|
@@ -469,7 +450,7 @@ rollout_capacity_controller() {
|
|
|
|
|
"capacity-controller=$api_image" || return 1
|
|
|
|
|
fi
|
|
|
|
|
"${kubectl[@]}" scale deployment/easyai-capacity-controller \
|
|
|
|
|
-n "$NAMESPACE" --replicas=2 || return 1
|
|
|
|
|
-n "$NAMESPACE" --replicas=1 || return 1
|
|
|
|
|
"${kubectl[@]}" rollout status deployment/easyai-capacity-controller \
|
|
|
|
|
-n "$NAMESPACE" --timeout=300s || return 1
|
|
|
|
|
}
|
|
|
|
@@ -493,6 +474,7 @@ spec:
|
|
|
|
|
restartPolicy: Never
|
|
|
|
|
serviceAccountName: easyai-capacity-controller
|
|
|
|
|
nodeSelector:
|
|
|
|
|
easyai.io/site: ningbo
|
|
|
|
|
easyai.io/workload: "true"
|
|
|
|
|
securityContext:
|
|
|
|
|
runAsNonRoot: true
|
|
|
|
@@ -615,20 +597,17 @@ apply_capacity_config() {
|
|
|
|
|
echo 'current release SHA is unavailable for capacity update' >&2
|
|
|
|
|
return 1
|
|
|
|
|
}
|
|
|
|
|
rollout_worker_site hongkong ""
|
|
|
|
|
rollout_worker_site ningbo ""
|
|
|
|
|
rollout_api_capacity_site hongkong
|
|
|
|
|
rollout_api_capacity_site ningbo
|
|
|
|
|
if [[ $capacity_controller_enabled == true ]]; then
|
|
|
|
|
rollout_capacity_controller ""
|
|
|
|
|
else
|
|
|
|
|
"${kubectl[@]}" scale deployment/easyai-capacity-controller -n "$NAMESPACE" --replicas=0
|
|
|
|
|
fi
|
|
|
|
|
verify_site hongkong
|
|
|
|
|
verify_site ningbo
|
|
|
|
|
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
|
|
|
|
record_capacity_config
|
|
|
|
|
echo "production_capacity=PASS config_hash=$(capacity_config_hash) ningbo_replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO hongkong_replicas=$AI_GATEWAY_WORKER_REPLICAS_HONGKONG instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
|
|
|
|
|
echo "production_capacity=PASS config_hash=$(capacity_config_hash) site=ningbo replicas=$AI_GATEWAY_WORKER_REPLICAS_NINGBO instance_limit=$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT global_limit=$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT admission_microbatch=$AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE worker_database_pool=$AI_GATEWAY_DATABASE_MAX_CONNS api_database_pool=$AI_GATEWAY_API_DATABASE_MAX_CONNS database_min_idle=$AI_GATEWAY_DATABASE_MIN_IDLE_CONNS database_max_idle_seconds=$AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS media_concurrency=$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY worker_memory_request_mib=$AI_GATEWAY_WORKER_MEMORY_REQUEST_MIB worker_cpu_request_millicores=$AI_GATEWAY_WORKER_CPU_REQUEST_MILLICORES autoscaling=$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rollout_site() {
|
|
|
|
@@ -786,10 +765,10 @@ record_current() {
|
|
|
|
|
"easyai.io/config-hash=$(capacity_config_hash)" \
|
|
|
|
|
--overwrite >/dev/null
|
|
|
|
|
for deployment in \
|
|
|
|
|
easyai-api-ningbo easyai-api-hongkong \
|
|
|
|
|
easyai-worker-ningbo easyai-worker-hongkong \
|
|
|
|
|
easyai-api-ningbo \
|
|
|
|
|
easyai-worker-ningbo \
|
|
|
|
|
easyai-capacity-controller \
|
|
|
|
|
easyai-web-ningbo easyai-web-hongkong; do
|
|
|
|
|
easyai-web-ningbo; do
|
|
|
|
|
"${kubectl[@]}" annotate deployment "$deployment" -n "$NAMESPACE" \
|
|
|
|
|
"easyai.io/release=$source_sha" \
|
|
|
|
|
"easyai.io/config-hash=$(capacity_config_hash)" \
|
|
|
|
@@ -936,11 +915,9 @@ activate_manifest() {
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if { [[ $api_changed != true ]] ||
|
|
|
|
|
{ rollout_worker_site hongkong "$api_image" &&
|
|
|
|
|
rollout_worker_site ningbo "$api_image" &&
|
|
|
|
|
{ rollout_worker_site ningbo "$api_image" &&
|
|
|
|
|
{ [[ $action == rollback ]] ||
|
|
|
|
|
rollout_capacity_controller "$api_image"; }; }; } &&
|
|
|
|
|
rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
|
|
|
|
|
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
|
|
|
|
|
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
|
|
|
|
|
:
|
|
|
|
@@ -953,6 +930,7 @@ activate_manifest() {
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
retire_hongkong_site
|
|
|
|
|
record_capacity_config
|
|
|
|
|
record_current "$manifest" "$action" "$started_at"
|
|
|
|
|
rm -f -- "$deployment_snapshot" "$current_file"
|
|
|
|
@@ -972,10 +950,8 @@ case ${1:-} in
|
|
|
|
|
node "$manifest_tool" validate "$2" >/dev/null
|
|
|
|
|
api_image=$(manifest_get "$2" images.api)
|
|
|
|
|
web_image=$(manifest_get "$2" images.web)
|
|
|
|
|
for site in ningbo hongkong; do
|
|
|
|
|
[[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
|
|
|
|
|
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
|
|
|
|
|
done
|
|
|
|
|
[[ $("${kubectl[@]}" get deployment easyai-worker-ningbo -n "$NAMESPACE" \
|
|
|
|
|
-o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
|
|
|
|
|
[[ $("${kubectl[@]}" get deployment easyai-capacity-controller -n "$NAMESPACE" \
|
|
|
|
|
-o jsonpath='{.spec.template.spec.containers[?(@.name=="capacity-controller")].image}') == "$api_image" ]]
|
|
|
|
|
[[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
|
|
|
|
|