|
|
|
@@ -15,8 +15,8 @@ source "$config_file"
|
|
|
|
|
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
|
|
|
|
|
: "${K3S_BIN:=/usr/local/bin/k3s}"
|
|
|
|
|
: "${DESIRED_STATE_DIR:=/usr/local/share/easyai-ai-gateway-release/production}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_REPLICAS_NINGBO:=0}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_REPLICAS_HONGKONG:=2}"
|
|
|
|
|
: "${AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:=24}"
|
|
|
|
|
: "${AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:=48}"
|
|
|
|
|
: "${AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:=$AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT}"
|
|
|
|
@@ -32,10 +32,10 @@ source "$config_file"
|
|
|
|
|
: "${AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:=24}"
|
|
|
|
|
: "${AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:=24}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:=false}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:=0}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=1}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:=0}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:=2}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:=$((AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT * 2))}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:=20}"
|
|
|
|
|
: "${AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:=600}"
|
|
|
|
@@ -199,8 +199,55 @@ capacity_config_hash() {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
record_capacity_config() {
|
|
|
|
|
local config_hash
|
|
|
|
|
config_hash=$(capacity_config_hash)
|
|
|
|
|
local config_hash
|
|
|
|
|
local runtime_patch
|
|
|
|
|
runtime_patch=$(jq -nc \
|
|
|
|
|
--arg replicasNingbo "$AI_GATEWAY_WORKER_REPLICAS_NINGBO" \
|
|
|
|
|
--arg replicasHongkong "$AI_GATEWAY_WORKER_REPLICAS_HONGKONG" \
|
|
|
|
|
--arg minNingbo "$AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO" \
|
|
|
|
|
--arg minHongkong "$AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG" \
|
|
|
|
|
--arg maxNingbo "$AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO" \
|
|
|
|
|
--arg maxHongkong "$AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG" \
|
|
|
|
|
--arg autoscaling "$AI_GATEWAY_WORKER_AUTOSCALING_ENABLED" \
|
|
|
|
|
--arg instanceLimit "$AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT" \
|
|
|
|
|
--arg globalLimit "$AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT" \
|
|
|
|
|
--arg workerPool "$AI_GATEWAY_DATABASE_MAX_CONNS" \
|
|
|
|
|
--arg materialization "$AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY" \
|
|
|
|
|
--arg mediaRequest "$AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY" \
|
|
|
|
|
--arg targetOutstanding "$AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA" \
|
|
|
|
|
--arg scaleUp "$AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS" \
|
|
|
|
|
--arg scaleDown "$AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS" \
|
|
|
|
|
--arg drainTimeout "$AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS" \
|
|
|
|
|
--arg memoryTarget "$AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT" \
|
|
|
|
|
--arg memoryHard "$AI_GATEWAY_NODE_MEMORY_HARD_PERCENT" \
|
|
|
|
|
--arg cpuTarget "$AI_GATEWAY_NODE_CPU_TARGET_PERCENT" \
|
|
|
|
|
--arg postgresBudget "$AI_GATEWAY_POSTGRES_CONNECTION_BUDGET" \
|
|
|
|
|
'{data:{
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_NINGBO:$replicasNingbo,
|
|
|
|
|
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:$replicasHongkong,
|
|
|
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:$minNingbo,
|
|
|
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:$minHongkong,
|
|
|
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:$maxNingbo,
|
|
|
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:$maxHongkong,
|
|
|
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:$autoscaling,
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$instanceLimit,
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$globalLimit,
|
|
|
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$globalLimit,
|
|
|
|
|
AI_GATEWAY_WORKER_DATABASE_MAX_CONNS:$workerPool,
|
|
|
|
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:$materialization,
|
|
|
|
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:$mediaRequest,
|
|
|
|
|
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$targetOutstanding,
|
|
|
|
|
AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS:$scaleUp,
|
|
|
|
|
AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS:$scaleDown,
|
|
|
|
|
AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS:$drainTimeout,
|
|
|
|
|
AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT:$memoryTarget,
|
|
|
|
|
AI_GATEWAY_NODE_MEMORY_HARD_PERCENT:$memoryHard,
|
|
|
|
|
AI_GATEWAY_NODE_CPU_TARGET_PERCENT:$cpuTarget,
|
|
|
|
|
AI_GATEWAY_POSTGRES_CONNECTION_BUDGET:$postgresBudget
|
|
|
|
|
}}')
|
|
|
|
|
"${kubectl[@]}" patch configmap easyai-ai-gateway-config -n "$NAMESPACE" \
|
|
|
|
|
--type merge -p "$runtime_patch" >/dev/null
|
|
|
|
|
config_hash=$(capacity_config_hash)
|
|
|
|
|
"${kubectl[@]}" patch configmap easyai-gateway-release -n "$NAMESPACE" \
|
|
|
|
|
--type merge -p "{\"data\":{\"configHash\":\"$config_hash\"}}" >/dev/null
|
|
|
|
|
"${kubectl[@]}" annotate namespace "$NAMESPACE" \
|
|
|
|
@@ -512,9 +559,18 @@ spec:
|
|
|
|
|
emptyDir:
|
|
|
|
|
sizeLimit: 64Mi
|
|
|
|
|
EOF
|
|
|
|
|
if ! "${kubectl[@]}" wait --for=condition=Ready "pod/$pod" \
|
|
|
|
|
-n "$NAMESPACE" --timeout=120s; then
|
|
|
|
|
"${kubectl[@]}" delete pod "$pod" -n "$NAMESPACE" \
|
|
|
|
|
if ! "${kubectl[@]}" wait --for=condition=Ready "pod/$pod" \
|
|
|
|
|
-n "$NAMESPACE" --timeout=300s; then
|
|
|
|
|
"${kubectl[@]}" get pod "$pod" -n "$NAMESPACE" -o json 2>/dev/null | jq -c '
|
|
|
|
|
{
|
|
|
|
|
phase:(.status.phase // "unknown"),
|
|
|
|
|
node:(.spec.nodeName // "unscheduled"),
|
|
|
|
|
ready:([.status.containerStatuses[]?.ready] | all),
|
|
|
|
|
restarts:([.status.containerStatuses[]?.restartCount] | add // 0),
|
|
|
|
|
waitingReasons:[.status.containerStatuses[]?.state.waiting.reason // empty],
|
|
|
|
|
terminatedReasons:[.status.containerStatuses[]?.state.terminated.reason // empty]
|
|
|
|
|
}' >&2 || true
|
|
|
|
|
"${kubectl[@]}" delete pod "$pod" -n "$NAMESPACE" \
|
|
|
|
|
--ignore-not-found=true --wait=true >/dev/null || true
|
|
|
|
|
echo 'capacity controller exact-image preflight failed' >&2
|
|
|
|
|
return 1
|
|
|
|
@@ -866,7 +922,7 @@ activate_manifest() {
|
|
|
|
|
-n "$NAMESPACE" --replicas=0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if { [[ $api_changed != true ]] ||
|
|
|
|
|
if { [[ $api_changed != true ]] ||
|
|
|
|
|
{ rollout_worker_site hongkong "$api_image" &&
|
|
|
|
|
rollout_worker_site ningbo "$api_image" &&
|
|
|
|
|
{ [[ $action == rollback ]] ||
|
|
|
|
@@ -875,16 +931,17 @@ activate_manifest() {
|
|
|
|
|
rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
|
|
|
|
|
wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
|
|
|
|
|
:
|
|
|
|
|
else
|
|
|
|
|
else
|
|
|
|
|
if [[ -n $previous_api && -n $previous_web ]]; then
|
|
|
|
|
echo '[cluster-release] restoring exact pre-release Deployment snapshot' >&2
|
|
|
|
|
restore_application_deployments "$deployment_snapshot" || true
|
|
|
|
|
fi
|
|
|
|
|
rm -f -- "$deployment_snapshot" "$current_file"
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
record_current "$manifest" "$action" "$started_at"
|
|
|
|
|
record_capacity_config
|
|
|
|
|
record_current "$manifest" "$action" "$started_at"
|
|
|
|
|
rm -f -- "$deployment_snapshot" "$current_file"
|
|
|
|
|
echo "production_release=PASS action=$action release=$source_sha"
|
|
|
|
|
}
|
|
|
|
|