easyai-ai-gateway/scripts/cluster/migrate-production.sh

#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=scripts/cluster/common.sh
source "$script_dir/common.sh"

action=${1:-preflight}
release_manifest=${2:-}
[[ $action == preflight || $action == cutover ]] || {
  echo 'usage: migrate-production.sh {preflight|cutover} dist/releases/<sha>.json' >&2
  exit 64
}
[[ -f $release_manifest && ! -L $release_manifest ]] || {
  echo 'release manifest must be a regular file' >&2
  exit 1
}

load_cluster_env
node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
source_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)

if [[ $action == cutover && ${AI_GATEWAY_CUTOVER_WINDOW_OVERRIDE:-} != I_ACCEPT_PRODUCTION_RISK ]]; then
  shanghai_hour=$(TZ=Asia/Shanghai date '+%H')
  if (( 10#$shanghai_hour < 2 || 10#$shanghai_hour >= 4 )); then
    echo 'cutover is restricted to 02:00-04:00 Asia/Shanghai' >&2
    exit 1
  fi
fi

echo '[cutover] checking Docker, K3s, CNPG, disk and backup prerequisites'
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
docker inspect easyai-ai-gateway-api-1 easyai-ai-gateway-postgres-1 >/dev/null
docker exec easyai-ai-gateway-postgres-1 pg_isready >/dev/null
k3s kubectl get --raw=/readyz >/dev/null
[[ $(k3s kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
[[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
[[ $primary =~ ^easyai-postgres-[0-9]+$ ]]
for pod in $(k3s kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
  -o jsonpath='{.items[*].metadata.name}'); do
  k3s kubectl exec -n easyai "$pod" -c postgres -- \
    test -r /projected/barman-aws/config
done
archived_before=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
  psql -X -U postgres -d postgres -At -c \
  'SELECT archived_count FROM pg_stat_archiver;')
archived_after=$(k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
  bash -s -- "$archived_before" <<'POD'
set -euo pipefail
archived_before=$1
psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres -At \
  -c "SELECT pg_logical_emit_message(true, 'easyai-ha-preflight', clock_timestamp()::text);" \
  -c 'SELECT pg_switch_wal();' >/dev/null
for _ in $(seq 1 45); do
  archived_after=$(psql -X -U postgres -d postgres -At -c \
    'SELECT archived_count FROM pg_stat_archiver;')
  if (( archived_after > archived_before )); then
    printf '%s\n' "$archived_after"
    exit 0
  fi
  sleep 2
done
exit 1
POD
)
(( archived_after > archived_before ))
[[ $(k3s kubectl get cluster easyai-postgres -n easyai \
  -o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}') == True ]]
latest_completed_backup=$(k3s kubectl get backups -n easyai -o json |
  jq -r '[.items[] | select(
    .spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
  ) | .status.stoppedAt] | sort | last // empty')
[[ -n $latest_completed_backup ]]
latest_completed_epoch=$(date -d "$latest_completed_backup" '+%s')
(( $(date '+%s') - latest_completed_epoch < 172800 ))
[[ $(df --output=avail -B1 /var/lib/docker | tail -1) -gt 2147483648 ]]
[[ -f /etc/nginx/sites-available/ai.51easyai.com-maintenance ]]
[[ -f /etc/nginx/sites-available/ai.51easyai.com-cluster ]]
nginx -t
REMOTE

if [[ $action == preflight ]]; then
  echo "production_migration_preflight=PASS release=$source_sha"
  exit 0
fi

working_directory=$(mktemp -d "$cluster_root/.local-secrets/cluster/cutover.XXXXXX")
trap '[[ $working_directory == "$cluster_root"/.local-secrets/cluster/cutover.* ]] && rm -rf -- "$working_directory"' EXIT HUP INT TERM
cluster_scp "$script_dir/oss-object.mjs" "$CLUSTER_NINGBO_HOST:/root/easyai-oss-object.mjs"

remote_cutover_script="$working_directory/remote-cutover.sh"
umask 077
cp /dev/null "$remote_cutover_script"
chmod 0700 "$remote_cutover_script"
cat >"$remote_cutover_script" <<'REMOTE'
#!/usr/bin/env bash
set -euo pipefail

source_sha=$1
api_image=$2
active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf}
timestamp=$(date -u '+%Y%m%dT%H%M%SZ')
backup_directory=/var/backups/easyai-ai-gateway/pre-cutover-"$timestamp"
old_site_backup=$backup_directory/nginx-site.conf
cutover_started=$(date +%s)
traffic_committed=false

rollback_before_commit() {
  status=$?
  trap - ERR
  if [[ $traffic_committed == false ]]; then
    echo '[cutover] rolling back to the frozen Docker database' >&2
    k3s kubectl scale deployment -n easyai \
      easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \
      --replicas=0 >/dev/null 2>&1 || true
    docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null 2>&1 || true
    if [[ -f $old_site_backup ]]; then
      install -m 0644 "$old_site_backup" "$active_site"
      nginx -t && systemctl reload nginx
    fi
  else
    echo '[cutover] traffic is already committed; automatic database rollback is unsafe' >&2
  fi
  exit "$status"
}
trap rollback_before_commit ERR

install -d -m 0700 "$backup_directory"
[[ -e $active_site || -L $active_site ]] || {
  echo "active NGINX site is unavailable: $active_site" >&2
  exit 1
}
cp -L "$active_site" "$old_site_backup"
chmod 0600 "$old_site_backup"
install -m 0644 /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site"
nginx -t
systemctl reload nginx
maintenance_code=
for _ in $(seq 1 15); do
  maintenance_code=$(curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \
    -o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz || true)
  [[ $maintenance_code == 503 ]] && break
  sleep 1
done
[[ $maintenance_code == 503 ]] || {
  echo "maintenance NGINX did not become active: HTTP $maintenance_code" >&2
  exit 1
}

docker stop --time 60 easyai-ai-gateway-api-1 >/dev/null
postgres_user=$(docker inspect easyai-ai-gateway-postgres-1 \
  --format '{{range .Config.Env}}{{println .}}{{end}}' |
  awk -F= '$1 == "POSTGRES_USER" { print substr($0, index($0, "=") + 1) }')
postgres_database=$(docker inspect easyai-ai-gateway-postgres-1 \
  --format '{{range .Config.Env}}{{println .}}{{end}}' |
  awk -F= '$1 == "POSTGRES_DB" { print substr($0, index($0, "=") + 1) }')
[[ $postgres_user =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]
[[ $postgres_database =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]

snapshot_sql=$backup_directory/acceptance-snapshot.sql
cat >"$snapshot_sql" <<'SQL'
\pset tuples_only on
\pset format unaligned
SELECT 'server_version_num=' || current_setting('server_version_num');
SELECT 'extensions=' || COALESCE(string_agg(extname || ':' || extversion, ',' ORDER BY extname), '')
  FROM pg_extension WHERE extname <> 'plpgsql';
SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
  WHERE c.relkind = 'r' AND n.nspname = 'public';
SELECT 'sequences=' || COALESCE(string_agg(schemaname || '.' || sequencename || ':' ||
  COALESCE(last_value::text, 'null'), ',' ORDER BY schemaname, sequencename), '')
  FROM pg_sequences WHERE schemaname = 'public';
SELECT 'catalog=' || md5(COALESCE(string_agg(definition, E'\n' ORDER BY definition), ''))
FROM (
  SELECT 'column:' || table_name || ':' || column_name || ':' || data_type || ':' ||
    is_nullable || ':' || COALESCE(column_default, '') AS definition
  FROM information_schema.columns WHERE table_schema = 'public'
  UNION ALL
  SELECT 'constraint:' || conrelid::regclass::text || ':' || conname || ':' ||
    pg_get_constraintdef(oid) FROM pg_constraint WHERE connamespace = 'public'::regnamespace
  UNION ALL
  SELECT 'index:' || indexrelid::regclass::text || ':' || pg_get_indexdef(indexrelid)
    FROM pg_index WHERE indrelid IN (
      SELECT oid FROM pg_class WHERE relnamespace = 'public'::regnamespace
    )
) definitions;
SELECT 'gateway_tasks=' || count(*) FROM gateway_tasks;
SELECT 'gateway_task_attempts=' || count(*) FROM gateway_task_attempts;
SELECT 'gateway_wallet_transactions=' || count(*) FROM gateway_wallet_transactions;
SELECT 'gateway_upload_assets=' || count(*) FROM gateway_upload_assets;
SELECT 'gateway_users=' || count(*) FROM gateway_users;
SELECT 'gateway_api_keys=' || count(*) FROM gateway_api_keys;
SELECT 'model_catalog_providers=' || count(*) FROM model_catalog_providers;
SELECT 'task_sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id), ''))
  FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) sample;
SQL

docker exec -i easyai-ai-gateway-postgres-1 \
  psql -X -v ON_ERROR_STOP=1 -U "$postgres_user" -d "$postgres_database" \
  <"$snapshot_sql" >"$backup_directory/source.snapshot"
docker exec easyai-ai-gateway-postgres-1 \
  pg_dumpall -U "$postgres_user" --globals-only \
  >"$backup_directory/globals.sql"
docker exec easyai-ai-gateway-postgres-1 \
  pg_dump -U "$postgres_user" -d "$postgres_database" \
  --format=custom --compress=6 \
  >"$backup_directory/database.dump"
[[ -s $backup_directory/database.dump && -s $backup_directory/globals.sql ]]
docker exec -i easyai-ai-gateway-postgres-1 pg_restore -l \
  <"$backup_directory/database.dump" >/dev/null
sha256sum "$backup_directory/database.dump" "$backup_directory/globals.sql" \
  "$backup_directory/source.snapshot" >"$backup_directory/SHA256SUMS"
sha256sum -c "$backup_directory/SHA256SUMS"

export ALI_KEY
export ALI_SECRET
ALI_KEY=$(k3s kubectl get secret easyai-oss-backup -n easyai \
  -o jsonpath='{.data.ACCESS_KEY_ID}' | base64 -d)
ALI_SECRET=$(k3s kubectl get secret easyai-oss-backup -n easyai \
  -o jsonpath='{.data.ACCESS_SECRET_KEY}' | base64 -d)
export ALI_REGION=oss-cn-shanghai
export ALI_BUCKET=wangbo0808
object_prefix=easyai-ai-gateway/production/pre-cutover/"$timestamp"
for file in database.dump globals.sql source.snapshot SHA256SUMS; do
  node /root/easyai-oss-object.mjs put "$object_prefix/$file" "$backup_directory/$file"
done
node /root/easyai-oss-object.mjs head "$object_prefix/database.dump"

primary=$(k3s kubectl get cluster easyai-postgres -n easyai \
  -o jsonpath='{.status.currentPrimary}')
[[ $primary =~ ^easyai-postgres-[0-9]+$ ]]
k3s kubectl exec -n easyai "$primary" -c postgres -- \
  psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres \
  -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'easyai_ai_gateway' AND pid <> pg_backend_pid();" \
  -c 'DROP DATABASE IF EXISTS easyai_ai_gateway;' \
  -c 'CREATE DATABASE easyai_ai_gateway OWNER easyai;'
k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
  pg_restore --exit-on-error --no-owner --role=easyai \
  -U postgres -d easyai_ai_gateway <"$backup_directory/database.dump"
k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \
  <"$snapshot_sql" >"$backup_directory/restored.snapshot"
diff -u "$backup_directory/source.snapshot" "$backup_directory/restored.snapshot"

cat >"$backup_directory/migrator-job.yaml" <<EOF
apiVersion: batch/v1
kind: Job
metadata:
  name: easyai-migrator-${source_sha:0:12}
  namespace: easyai
  labels:
    app.kubernetes.io/part-of: easyai-ai-gateway
    easyai.io/release: ${source_sha}
spec:
  backoffLimit: 0
  ttlSecondsAfterFinished: 86400
  template:
    metadata:
      labels:
        app.kubernetes.io/name: easyai-migrator
    spec:
      restartPolicy: Never
      imagePullSecrets:
        - name: easyai-registry
      nodeSelector:
        easyai.io/site: ningbo
      containers:
        - name: migrator
          image: ${api_image}
          command: ["/app/easyai-ai-gateway-migrate"]
          envFrom:
            - secretRef:
                name: easyai-ai-gateway-runtime
EOF
k3s kubectl delete job "easyai-migrator-${source_sha:0:12}" -n easyai \
  --ignore-not-found=true --wait=true >/dev/null
k3s kubectl apply -f "$backup_directory/migrator-job.yaml" >/dev/null
k3s kubectl wait --for=condition=complete \
  "job/easyai-migrator-${source_sha:0:12}" -n easyai --timeout=300s
k3s kubectl exec -n easyai "$primary" -c postgres -- \
  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
  "SELECT count(*) FROM schema_migrations WHERE version='0089_file_storage_request_asset_scene';" |
  grep -qx 1
k3s kubectl exec -n easyai "$primary" -c postgres -- \
  psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c \
  "SELECT count(*) FROM file_storage_channels WHERE deleted_at IS NULL AND provider='server_main_openapi' AND config->'scenes' ? 'request_asset';" |
  grep -Eq '^[1-9][0-9]*$'

elapsed=$(( $(date +%s) - cutover_started ))
if (( elapsed > 600 )); then
  echo "cutover exceeded ten-minute rollback boundary: ${elapsed}s" >&2
  exit 1
fi
printf '%s\n' "$backup_directory" >/root/.easyai-pre-cutover-backup
printf 'database_restore=PASS seconds=%s backup=%s\n' "$elapsed" "$backup_directory"
REMOTE

cluster_scp "$remote_cutover_script" "$CLUSTER_NINGBO_HOST:/root/easyai-remote-cutover.sh"
cluster_ssh "$CLUSTER_NINGBO_HOST" \
  "chmod 0700 /root/easyai-remote-cutover.sh && /root/easyai-remote-cutover.sh '$source_sha' '$api_image'"

traffic_committed=false
rollback_local_before_commit() {
  status=$?
  trap - ERR
  if [[ $traffic_committed == false ]]; then
    echo '[cutover] local activation failed; restoring Docker before public traffic commit' >&2
    cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true
set -euo pipefail
backup_directory=$(cat /root/.easyai-pre-cutover-backup)
[[ $backup_directory == /var/backups/easyai-ai-gateway/pre-cutover-* ]]
k3s kubectl scale deployment -n easyai \
  easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \
  --replicas=0 >/dev/null 2>&1 || true
docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null
install -m 0644 "$backup_directory/nginx-site.conf" \
  /etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf
nginx -t
systemctl reload nginx
REMOTE
  fi
  exit "$status"
}
trap rollback_local_before_commit ERR

echo '[cutover] starting both application sites with River workers frozen'
AI_GATEWAY_ACTIVATE_WORKERS=false \
  "$script_dir/bootstrap-platform.sh" activate "$release_manifest"

for site_endpoint in 10.77.0.1:31088:31178 10.77.0.2:31089:31179; do
  IFS=: read -r site_ip api_port web_port <<<"$site_endpoint"
  cluster_ssh "$CLUSTER_NINGBO_HOST" \
    "curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/healthz >/dev/null"
  cluster_ssh "$CLUSTER_NINGBO_HOST" \
    "curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/readyz | grep -q '\"ok\":true'"
  cluster_ssh "$CLUSTER_NINGBO_HOST" \
    "curl -fsS --max-time 10 http://$site_ip:$web_port/ | grep -q 'EasyAI AI Gateway'"
done

"$script_dir/install-nginx-edges.sh" activate
for edge_host in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
  curl -fsS --resolve "ai.51easyai.com:443:$edge_host" \
    https://ai.51easyai.com/api/v1/healthz >/dev/null
  curl -fsS --resolve "ai.51easyai.com:443:$edge_host" \
    https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'
done

traffic_committed=true
echo '[cutover] traffic committed; enabling Hong Kong worker, then Ningbo worker'
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
kubectl='k3s kubectl'
$kubectl set env deployment/easyai-api-hongkong -n easyai \
  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true
$kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
$kubectl set env deployment/easyai-api-ningbo -n easyai \
  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true
$kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
backup_resource=$(cat <<'EOF' | $kubectl create -f - -o name
apiVersion: postgresql.cnpg.io/v1
kind: Backup
metadata:
  generateName: easyai-post-cutover-
  namespace: easyai
spec:
  cluster:
    name: easyai-postgres
  method: plugin
  pluginConfiguration:
    name: barman-cloud.cloudnative-pg.io
EOF
)
backup_name=${backup_resource#*/}
backup_phase=
for _ in $(seq 1 240); do
  backup_phase=$($kubectl get backup "$backup_name" -n easyai \
    -o jsonpath='{.status.phase}')
  [[ $backup_phase == completed ]] && break
  if [[ $backup_phase == failed ]]; then
    echo "post-cutover CNPG backup failed: $backup_name" >&2
    exit 1
  fi
  sleep 5
done
[[ $backup_phase == completed ]] || {
  echo "post-cutover CNPG backup did not complete in time: $backup_name" >&2
  exit 1
}
docker update --restart=no easyai-ai-gateway-api-1 \
  easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
docker stop --time 30 easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
rm -f -- /root/easyai-remote-cutover.sh /root/easyai-oss-object.mjs
echo "post_cutover_backup=PASS name=$backup_name"
REMOTE

"$script_dir/install-release-helper.sh"
remote_manifest=/tmp/easyai-cutover-release-"$source_sha".json
cluster_scp "$release_manifest" "$CLUSTER_NINGBO_HOST:$remote_manifest"
cluster_ssh "$CLUSTER_NINGBO_HOST" \
  "/usr/local/sbin/easyai-ai-gateway-cluster-release adopt '$remote_manifest' && rm -f '$remote_manifest'"

set_local_environment() {
  local key=$1
  local value=$2
  local temporary
  temporary=$(mktemp "$cluster_env_file.next.XXXXXX")
  chmod 0600 "$temporary"
  awk -v key="$key" -v value="$value" -F= '
    $1 == key { if (!written) print key "=" value; written = 1; next }
    { print }
    END { if (!written) print key "=" value }
  ' "$cluster_env_file" >"$temporary"
  mv "$temporary" "$cluster_env_file"
}
set_local_environment AI_GATEWAY_DEPLOY_MODE kubernetes
set_local_environment AI_GATEWAY_REMOTE_RELEASE_HELPER /usr/local/sbin/easyai-ai-gateway-cluster-release
set_local_environment AI_GATEWAY_PRODUCTION_SSH_KEY "$cluster_ssh_key"

trap - ERR
echo "production_cutover=PASS release=$source_sha old_postgres_retained=true"