easyai-ai-gateway/scripts/cluster/bootstrap-platform.sh

#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=scripts/cluster/common.sh
source "$script_dir/common.sh"

mode=${1:-}
release_manifest=${2:-}
[[ $mode == prepare || $mode == activate ]] || {
  echo 'usage: bootstrap-platform.sh {prepare|activate} dist/releases/<sha>.json' >&2
  exit 64
}
[[ -f $release_manifest && ! -L $release_manifest ]] || {
  echo 'release manifest must be a regular file' >&2
  exit 1
}

load_cluster_env
require_commands curl kubectl node openssl rg shasum

node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
web_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.web)
source_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
activate_workers=${AI_GATEWAY_ACTIVATE_WORKERS:-true}
[[ $activate_workers == true || $activate_workers == false ]] || {
  echo 'AI_GATEWAY_ACTIVATE_WORKERS must be true or false' >&2
  exit 64
}
[[ $api_image =~ ^registry\.cn-shanghai\.aliyuncs\.com/easyaigc/ai-gateway@sha256:[0-9a-f]{64}$ ]]
[[ $web_image =~ ^registry\.cn-shanghai\.aliyuncs\.com/easyaigc/ai-gateway-web@sha256:[0-9a-f]{64}$ ]]

secret_root="$cluster_root/.local-secrets/cluster"
postgres_environment="$secret_root/postgres.env"
working_directory=$(mktemp -d "$secret_root/platform.XXXXXX")
trap '[[ $working_directory == "$secret_root"/platform.* ]] && rm -rf -- "$working_directory"' EXIT HUP INT TERM
install -d -m 0700 "$secret_root"

if [[ ! -f $postgres_environment ]]; then
  umask 077
  printf 'POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 32)" >"$postgres_environment"
  chmod 0600 "$postgres_environment"
fi
assert_private_file "$postgres_environment"
# shellcheck source=/dev/null
source "$postgres_environment"
: "${POSTGRES_PASSWORD:?}"
[[ $POSTGRES_PASSWORD =~ ^[0-9a-f]{64}$ ]] || {
  echo 'invalid generated PostgreSQL password' >&2
  exit 1
}

operators=(cert-manager.yaml cnpg.yaml barman.yaml)
operator_urls=(
  https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml
  https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.29/releases/cnpg-1.29.1.yaml
  https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/v0.13.0/manifest.yaml
)
operator_checksums=(
  6e499c3f1ab356abe79a7853911f80cb09c213885bfdf81092fdff142ba63c4a
  e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a
  d2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0
)
if [[ $mode == prepare ]]; then
  for index in 0 1 2; do
    operator=${operators[$index]}
    curl -fsSL "${operator_urls[$index]}" -o "$working_directory/$operator"
    actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}')
    [[ $actual_checksum == "${operator_checksums[$index]}" ]] || {
      echo "operator manifest checksum mismatch: $operator" >&2
      exit 1
    }
  done
fi

cp -R "$cluster_root/deploy/kubernetes/production" "$working_directory/production"
api_digest=${api_image##*@}
web_digest=${web_image##*@}
awk -v api_digest="$api_digest" -v web_digest="$web_digest" '
  $3 == "easyai-api" { image = "api" }
  $3 == "easyai-web" { image = "web" }
  $1 == "digest:" && image == "api" { print "    digest: " api_digest; image = ""; next }
  $1 == "digest:" && image == "web" { print "    digest: " web_digest; image = ""; next }
  { print }
' "$working_directory/production/kustomization.yaml" \
  >"$working_directory/production/kustomization.yaml.next"
mv "$working_directory/production/kustomization.yaml.next" \
  "$working_directory/production/kustomization.yaml"

if [[ $mode == prepare ]]; then
  sed -i.bak 's/^  replicas: 1$/  replicas: 0/' "$working_directory/production/application.yaml"
  rm -f -- "$working_directory/production/application.yaml.bak"
fi
if [[ $activate_workers == false ]]; then
  awk '
    /^---$/ {
      document_kind = ""
      worker_deployment = 0
      print
      next
    }
    /^kind: / {
      document_kind = $2
    }
    document_kind == "Deployment" && /^  name: easyai-worker-/ {
      worker_deployment = 1
    }
    worker_deployment && /^  replicas: 1$/ {
      sub(/1$/, "0")
    }
    { print }
  ' "$working_directory/production/application.yaml" \
    >"$working_directory/production/application.yaml.next"
  mv "$working_directory/production/application.yaml.next" \
    "$working_directory/production/application.yaml"
fi
kubectl kustomize "$working_directory/production" >"$working_directory/easyai-production.yaml"
if rg -q 'sha256:0{64}|image: (easyai-api|easyai-web)$' "$working_directory/easyai-production.yaml"; then
  echo 'application images were not digest-pinned during rendering' >&2
  exit 1
fi
if rg -i -q '^[[:space:]]*(password|secret|token|credential|private.?key):[[:space:]]+[^<{[:space:]]' \
    "$working_directory/easyai-production.yaml"; then
  echo 'rendered production manifests appear to contain a literal secret' >&2
  exit 1
fi

umask 077
cat >"$working_directory/bootstrap-secrets.yaml" <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: easyai-postgres-app
  namespace: easyai
type: kubernetes.io/basic-auth
stringData:
  username: easyai
  password: ${POSTGRES_PASSWORD}
---
apiVersion: v1
kind: Secret
metadata:
  name: easyai-oss-backup
  namespace: easyai
type: Opaque
stringData:
  ACCESS_KEY_ID: ${ALI_KEY}
  ACCESS_SECRET_KEY: ${ALI_SECRET}
EOF
chmod 0600 "$working_directory/bootstrap-secrets.yaml"

platform_files=(easyai-production.yaml bootstrap-secrets.yaml)
if [[ $mode == prepare ]]; then
  platform_files=(cert-manager.yaml cnpg.yaml barman.yaml "${platform_files[@]}")
fi
for file in "${platform_files[@]}"; do
  cluster_scp "$working_directory/$file" "$CLUSTER_NINGBO_HOST:/root/$file"
done

cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- \
  "$source_sha" "$mode" "$activate_workers" <<'REMOTE'
set -euo pipefail
source_sha=$1
mode=$2
activate_workers=$3
kubectl='k3s kubectl'
runtime_environment=
cleanup_bootstrap_files() {
  if [[ -n ${runtime_environment:-} && -f $runtime_environment &&
        ! -L $runtime_environment ]]; then
    shred -u "$runtime_environment"
  fi
  if [[ -f /root/bootstrap-secrets.yaml && ! -L /root/bootstrap-secrets.yaml ]]; then
    shred -u /root/bootstrap-secrets.yaml
  fi
  for path in /root/cert-manager.yaml /root/cnpg.yaml /root/barman.yaml \
    /root/easyai-production.yaml; do
    [[ ! -e $path && ! -L $path ]] || unlink "$path"
  done
}
trap cleanup_bootstrap_files EXIT HUP INT TERM

if [[ $mode == prepare ]]; then
  $kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null
  $kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
  $kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s
  $kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s
  $kubectl apply --server-side --force-conflicts -f /root/cnpg.yaml >/dev/null
  $kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=600s
  $kubectl apply --server-side --force-conflicts -f /root/barman.yaml >/dev/null
  $kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=600s
fi

$kubectl create namespace easyai --dry-run=client -o yaml | $kubectl apply -f - >/dev/null
$kubectl apply -f /root/bootstrap-secrets.yaml >/dev/null
postgres_password=$($kubectl get secret easyai-postgres-app -n easyai \
  -o jsonpath='{.data.password}' | base64 -d)

runtime_environment=$(mktemp)
chmod 0600 "$runtime_environment"
docker inspect easyai-ai-gateway-api-1 \
  --format '{{range .Config.Env}}{{println .}}{{end}}' |
  grep -E '^(CONFIG_JWT_SECRET|SERVER_MAIN_INTERNAL_TOKEN|SERVER_MAIN_INTERNAL_KEY|SERVER_MAIN_INTERNAL_SECRET)=' \
  >"$runtime_environment"
printf 'AI_GATEWAY_DATABASE_URL=postgresql://easyai:%s@easyai-postgres-rw:5432/easyai_ai_gateway?sslmode=require\n' \
  "$postgres_password" >>"$runtime_environment"
[[ $(grep -c '^[A-Z0-9_]*=' "$runtime_environment") -eq 5 ]] || {
  echo 'runtime Secret allowlist is incomplete' >&2
  exit 1
}
$kubectl create secret generic easyai-ai-gateway-runtime -n easyai \
  --from-env-file="$runtime_environment" --dry-run=client -o yaml |
  $kubectl apply -f - >/dev/null

identity_directory=/var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/identity/secrets
[[ -d $identity_directory && ! -L $identity_directory ]] || {
  echo 'legacy identity SecretStore directory is unavailable' >&2
  exit 1
}
mapfile -d '' identity_files < <(find "$identity_directory" -maxdepth 1 -type f -print0)
[[ ${#identity_files[@]} -eq 4 ]] || {
  echo 'identity SecretStore file count changed from the accepted baseline' >&2
  exit 1
}
identity_bytes=0
identity_arguments=()
for identity_file in "${identity_files[@]}"; do
  identity_name=$(basename "$identity_file")
  [[ $identity_name =~ ^[A-Za-z0-9._-]+$ ]] || {
    echo 'identity SecretStore contains an invalid key name' >&2
    exit 1
  }
  identity_bytes=$((identity_bytes + $(stat -c '%s' "$identity_file")))
  identity_arguments+=("--from-file=$identity_name=$identity_file")
done
[[ $identity_bytes -eq 247 ]] || {
  echo 'identity SecretStore byte count changed from the accepted baseline' >&2
  exit 1
}
$kubectl create secret generic easyai-gateway-security-events -n easyai \
  "${identity_arguments[@]}" --dry-run=client -o yaml |
  $kubectl apply -f - >/dev/null

registry_config=/root/.docker/config.json
[[ -f $registry_config && ! -L $registry_config ]] || {
  echo 'Docker registry config is unavailable on Ningbo' >&2
  exit 1
}
$kubectl create secret generic easyai-registry -n easyai \
  --type=kubernetes.io/dockerconfigjson \
  --from-file=.dockerconfigjson="$registry_config" --dry-run=client -o yaml |
  $kubectl apply -f - >/dev/null

$kubectl apply --server-side --dry-run=server --force-conflicts \
  --field-manager=easyai-production-preflight \
  -f /root/easyai-production.yaml >/dev/null
$kubectl apply -f /root/easyai-production.yaml >/dev/null
$kubectl annotate namespace easyai easyai.io/release="$source_sha" --overwrite >/dev/null
if [[ $mode == activate ]]; then
  if [[ $activate_workers == true ]]; then
    $kubectl rollout status deployment/easyai-worker-hongkong -n easyai --timeout=300s
    $kubectl rollout status deployment/easyai-worker-ningbo -n easyai --timeout=300s
  fi
  $kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
  $kubectl rollout status deployment/easyai-web-hongkong -n easyai --timeout=300s
  $kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
  $kubectl rollout status deployment/easyai-web-ningbo -n easyai --timeout=300s
fi

REMOTE

if [[ $mode == prepare ]]; then
  cluster_ssh "$CLUSTER_NINGBO_HOST" \
    'k3s kubectl wait --for=condition=Ready cluster/easyai-postgres -n easyai --timeout=600s'
fi

echo "platform_bootstrap=PASS mode=$mode release=$source_sha"