#!/usr/bin/env bash set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) repository_root=$(cd "$script_dir/../.." && pwd) manifest_root="$repository_root/deploy/kubernetes/local-acceptance" lock_file="$manifest_root/dependencies.lock" private_root="$repository_root/.local-secrets/acceptance" tools_root="$private_root/tools" state_root="$private_root/state" media_root="$private_root/media" cluster_name=easyai-acceptance-local context="k3d-$cluster_name" namespace=easyai usage() { cat <<'EOF' Usage: scripts/acceptance/local-cluster.sh install-tools scripts/acceptance/local-cluster.sh preflight scripts/acceptance/local-cluster.sh up --snapshot scripts/acceptance/local-cluster.sh status scripts/acceptance/local-cluster.sh down --confirm The full cluster requires Docker Desktop memory >= 24 GiB. Failed `up` runs are kept intact for diagnosis. `down` is the only destructive lifecycle action. EOF } load_lock() { [[ -f $lock_file && ! -L $lock_file ]] || { echo "missing dependency lock: $lock_file" >&2 exit 1 } # shellcheck source=/dev/null source "$lock_file" : "${K3D_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}" } require_command() { command -v "$1" >/dev/null 2>&1 || { echo "$1 is required" >&2 exit 1 } } private_file() { local path=$1 [[ -f $path && ! -L $path ]] || return 1 local mode if [[ $(uname -s) == Darwin ]]; then mode=$(stat -f '%Lp' "$path") else mode=$(stat -c '%a' "$path") fi [[ $mode == 600 ]] } install_tools() { load_lock require_command curl require_command shasum install -d -m 0700 "$tools_root" local os arch checksum url temporary os=$(uname -s | tr '[:upper:]' '[:lower:]') arch=$(uname -m) [[ $os == darwin ]] || { echo 'the pinned local acceptance installer currently supports macOS only' >&2 exit 1 } case $arch in arm64) arch=arm64 checksum=$K3D_DARWIN_ARM64_SHA256 ;; x86_64) arch=amd64 checksum=$K3D_DARWIN_AMD64_SHA256 ;; *) echo "unsupported host architecture: $arch" >&2 exit 1 ;; esac url="https://github.com/k3d-io/k3d/releases/download/$K3D_VERSION/k3d-darwin-$arch" temporary="$tools_root/k3d.download" curl -fsSL "$url" -o "$temporary" [[ $(shasum -a 256 "$temporary" | awk '{print $1}') == "$checksum" ]] || { echo 'k3d checksum mismatch' >&2 exit 1 } chmod 0555 "$temporary" mv "$temporary" "$tools_root/k3d" echo "local_acceptance_tools=PASS k3d=$K3D_VERSION path=$tools_root/k3d" } k3d_binary() { if [[ -x $tools_root/k3d ]]; then printf '%s\n' "$tools_root/k3d" return fi command -v k3d } docker_memory_bytes() { docker info --format '{{.MemTotal}}' } preflight() { load_lock require_command docker require_command kubectl require_command jq require_command openssl require_command sed require_command shasum require_command go require_command node require_command curl local k3d memory_bytes architecture k3d=$(k3d_binary) || { echo 'k3d is missing; run install-tools' >&2 exit 1 } "$k3d" version | grep -Fq "$K3D_VERSION" || { echo "k3d must be pinned to $K3D_VERSION" >&2 exit 1 } docker info >/dev/null memory_bytes=$(docker_memory_bytes) [[ $memory_bytes =~ ^[0-9]+$ && $memory_bytes -ge 25769803776 ]] || { memory_gib=$(awk -v bytes="${memory_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}') echo "Docker Desktop memory is ${memory_gib} GiB; local acceptance requires at least 24 GiB" >&2 exit 1 } architecture=$(docker info --format '{{.Architecture}}') [[ $architecture == aarch64 || $architecture == arm64 || $architecture == x86_64 ]] || { echo "unsupported Docker architecture: $architecture" >&2 exit 1 } docker buildx version >/dev/null echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes architecture=$architecture k3d=$K3D_VERSION" } ensure_private_material() { install -d -m 0700 "$private_root" "$state_root" "$media_root" local env_file="$state_root/local.env" if [[ ! -e $env_file ]]; then umask 077 { printf 'LOCAL_CLUSTER_ID=%s\n' "easyai-local-$(openssl rand -hex 12)" printf 'POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 32)" printf 'CONFIG_JWT_SECRET=%s\n' "$(openssl rand -hex 32)" printf 'SERVER_MAIN_INTERNAL_TOKEN=%s\n' "$(openssl rand -hex 32)" printf 'SERVER_MAIN_INTERNAL_KEY=local-acceptance\n' printf 'SERVER_MAIN_INTERNAL_SECRET=%s\n' "$(openssl rand -hex 32)" } >"$env_file" chmod 0600 "$env_file" fi private_file "$env_file" || { echo "local acceptance state must be a 0600 regular file: $env_file" >&2 exit 1 } } generate_tls() { local ca_key="$state_root/ca.key" ca_crt="$state_root/ca.crt" local tls_key="$state_root/tls.key" tls_csr="$state_root/tls.csr" local tls_crt="$state_root/tls.crt" extensions="$state_root/tls.ext" if [[ ! -e $ca_key ]]; then umask 077 openssl genrsa -out "$ca_key" 3072 >/dev/null 2>&1 openssl req -x509 -new -key "$ca_key" -sha256 -days 30 \ -subj '/CN=EasyAI local acceptance CA' -out "$ca_crt" >/dev/null 2>&1 openssl genrsa -out "$tls_key" 2048 >/dev/null 2>&1 openssl req -new -key "$tls_key" -subj '/CN=gateway.easyai.local' \ -out "$tls_csr" >/dev/null 2>&1 printf 'subjectAltName=DNS:gateway.easyai.local\nextendedKeyUsage=serverAuth\n' >"$extensions" openssl x509 -req -in "$tls_csr" -CA "$ca_crt" -CAkey "$ca_key" \ -CAcreateserial -out "$tls_crt" -days 30 -sha256 -extfile "$extensions" \ >/dev/null 2>&1 chmod 0600 "$ca_key" "$ca_crt" "$tls_key" "$tls_csr" "$tls_crt" "$extensions" fi private_file "$ca_crt" && private_file "$tls_key" && private_file "$tls_crt" } render_k3d_config() { local output=$1 [[ $media_root != *'|'* ]] sed "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" "$manifest_root/k3d.yaml" >"$output" } wait_rollout() { kubectl --context "$context" -n "$namespace" rollout status "$1" --timeout="${2:-10m}" } apply_runtime_secrets() { # shellcheck source=/dev/null source "$state_root/local.env" local database_ningbo database_hongkong database_direct database_ningbo="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-ningbo.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require" database_hongkong="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-hongkong.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require" database_direct="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-rw.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require" kubectl --context "$context" create namespace "$namespace" --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" create secret generic easyai-postgres-app \ --type=kubernetes.io/basic-auth \ --from-literal=username=easyai --from-literal=password="$POSTGRES_PASSWORD" \ --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" create secret generic easyai-ai-gateway-runtime \ --from-literal=AI_GATEWAY_DATABASE_URL="$database_direct" \ --from-literal=CONFIG_JWT_SECRET="$CONFIG_JWT_SECRET" \ --from-literal=SERVER_MAIN_INTERNAL_TOKEN="$SERVER_MAIN_INTERNAL_TOKEN" \ --from-literal=SERVER_MAIN_INTERNAL_KEY="$SERVER_MAIN_INTERNAL_KEY" \ --from-literal=SERVER_MAIN_INTERNAL_SECRET="$SERVER_MAIN_INTERNAL_SECRET" \ --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" create secret generic easyai-database-ningbo \ --from-literal=AI_GATEWAY_DATABASE_URL="$database_ningbo" \ --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" create secret generic easyai-database-hongkong \ --from-literal=AI_GATEWAY_DATABASE_URL="$database_hongkong" \ --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" create secret generic easyai-oss-backup \ --from-literal=ACCESS_KEY_ID=local-disabled \ --from-literal=ACCESS_SECRET_KEY=local-disabled \ --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" create secret tls easyai-acceptance-tls \ --cert="$state_root/tls.crt" --key="$state_root/tls.key" \ --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null } run_migrations() { local api_image=$1 kubectl --context "$context" -n "$namespace" delete job easyai-local-migrate \ --ignore-not-found --wait=true >/dev/null cat </dev/null apiVersion: batch/v1 kind: Job metadata: name: easyai-local-migrate namespace: easyai spec: backoffLimit: 0 template: spec: restartPolicy: Never containers: - name: migrate image: $api_image command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"] envFrom: - secretRef: name: easyai-ai-gateway-runtime EOF kubectl --context "$context" -n "$namespace" wait \ --for=condition=complete job/easyai-local-migrate --timeout=10m >/dev/null } mark_local_database() { # shellcheck source=/dev/null source "$state_root/local.env" local primary primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \ -o 'jsonpath={.status.currentPrimary}') [[ $LOCAL_CLUSTER_ID =~ ^easyai-local-[0-9a-f]{24}$ ]] printf '%s\n' \ "INSERT INTO system_settings(setting_key,value) VALUES('acceptance_local_cluster_id',jsonb_build_object('clusterId', :'cluster_id')) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" | kubectl --context "$context" -n "$namespace" exec -i "$primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \ -v cluster_id="$LOCAL_CLUSTER_ID" >/dev/null } import_snapshot() { local api_image=$1 snapshot=$2 # shellcheck source=/dev/null source "$state_root/local.env" kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-snapshot \ --from-file=snapshot.json="$snapshot" --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" delete job easyai-local-snapshot-import \ --ignore-not-found --wait=true >/dev/null cat </dev/null apiVersion: batch/v1 kind: Job metadata: name: easyai-local-snapshot-import namespace: easyai spec: backoffLimit: 0 template: spec: restartPolicy: Never securityContext: fsGroup: 10001 fsGroupChangePolicy: OnRootMismatch initContainers: - name: stage-snapshot image: $api_image command: - /bin/sh - -ec - umask 077; cp /snapshot-source/snapshot.json /snapshot/snapshot.json volumeMounts: - name: snapshot-source mountPath: /snapshot-source readOnly: true - name: snapshot mountPath: /snapshot containers: - name: import image: $api_image command: - /app/easyai-ai-gateway-acceptance-snapshot - import - --input - /snapshot/snapshot.json - --local-cluster-id - $LOCAL_CLUSTER_ID envFrom: - secretRef: name: easyai-ai-gateway-runtime volumeMounts: - name: snapshot mountPath: /snapshot readOnly: true volumes: - name: snapshot-source configMap: name: easyai-acceptance-snapshot - name: snapshot emptyDir: {} EOF kubectl --context "$context" -n "$namespace" wait \ --for=condition=complete job/easyai-local-snapshot-import --timeout=5m >/dev/null } render_and_apply_application() { local api_image=$1 web_image=$2 rendered=$state_root/application.rendered.yaml sed \ -e "s|image: easyai-api|image: $api_image|g" \ -e "s|image: easyai-web|image: $web_image|g" \ -e 's/nodePort: 31088/nodePort: 32088/g' \ -e 's/nodePort: 31089/nodePort: 32089/g' \ "$repository_root/deploy/kubernetes/production/application.yaml" >"$rendered" chmod 0600 "$rendered" kubectl --context "$context" -n "$namespace" apply \ -f "$repository_root/deploy/kubernetes/production/service-account-rbac.yaml" >/dev/null kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/local-config.yaml" >/dev/null kubectl --context "$context" -n "$namespace" apply -f "$rendered" >/dev/null for workload in easyai-api-ningbo easyai-worker-ningbo; do kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \ --from=secret/easyai-database-ningbo >/dev/null done for workload in easyai-api-hongkong easyai-worker-hongkong; do kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \ --from=secret/easyai-database-hongkong >/dev/null done for workload in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do kubectl --context "$context" -n "$namespace" patch deployment "$workload" --type=json \ -p='[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"application-data","hostPath":{"path":"/var/lib/easyai-acceptance/media","type":"DirectoryOrCreate"}}}]' \ >/dev/null done kubectl --context "$context" -n "$namespace" scale \ deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null } bootstrap_runtime() { local api_image=$1 api_digest=$2 snapshot=$3 # shellcheck source=/dev/null source "$state_root/local.env" local snapshot_hash release_sha runtime_file snapshot_hash=$(jq -r '.snapshotSha256' "$snapshot") release_sha=$(git -C "$repository_root" rev-parse HEAD) runtime_file="$state_root/runtime-$snapshot_hash.json" kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \ --ignore-not-found --wait=true >/dev/null cat </dev/null apiVersion: v1 kind: Pod metadata: name: easyai-local-bootstrap namespace: easyai labels: easyai.io/environment: local-acceptance spec: restartPolicy: Never containers: - name: bootstrap image: $api_image command: - /app/easyai-ai-gateway-acceptance-bootstrap - --local-cluster-id - $LOCAL_CLUSTER_ID - --release-sha - $release_sha - --api-image-digest - $api_digest - --worker-image-digest - $api_digest - --emulator-base-url - http://easyai-acceptance-upstream-proxy.easyai.svc.cluster.local:8090 - --callback-url - http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks - --identity-shards - "32" - --output - /tmp/runtime.json envFrom: - secretRef: name: easyai-ai-gateway-runtime EOF kubectl --context "$context" -n "$namespace" wait \ --for=jsonpath='{.status.phase}'=Succeeded pod/easyai-local-bootstrap --timeout=5m >/dev/null umask 077 kubectl --context "$context" -n "$namespace" cp \ easyai-local-bootstrap:/tmp/runtime.json "$runtime_file" >/dev/null chmod 0600 "$runtime_file" kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \ --wait=true >/dev/null printf '%s\n' "$runtime_file" >"$state_root/current-runtime-path" chmod 0600 "$state_root/current-runtime-path" } up_cluster() { [[ ${1:-} == --snapshot && $# -eq 2 ]] || { usage >&2 exit 64 } local snapshot_input=$2 snapshot_dir snapshot [[ -f $snapshot_input && ! -L $snapshot_input ]] || { echo 'snapshot must be a regular non-symlink file' >&2 exit 1 } snapshot_dir=$(cd "$(dirname "$snapshot_input")" && pwd -P) snapshot="$snapshot_dir/$(basename "$snapshot_input")" [[ -z $(git -C "$repository_root" status --short) ]] || { echo 'local acceptance requires a clean source tree so report and image source are reproducible' >&2 exit 1 } preflight ensure_private_material generate_tls ( cd "$repository_root/apps/api" go run ./cmd/acceptance-snapshot validate --input "$snapshot" ) cp "$snapshot" "$state_root/snapshot.json" chmod 0600 "$state_root/snapshot.json" local k3d config native_arch release_sha api_image web_image netem_image api_digest k3d=$(k3d_binary) if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then echo "cluster $cluster_name already exists; use status or explicit down --confirm" >&2 exit 1 fi native_arch=$(docker info --format '{{.Architecture}}') case $native_arch in aarch64|arm64) native_arch=arm64 ;; x86_64) native_arch=amd64 ;; esac release_sha=$(git -C "$repository_root" rev-parse HEAD) api_image="easyai-acceptance-api:$release_sha" web_image="easyai-acceptance-web:$release_sha" netem_image="easyai-acceptance-netem:$release_sha" docker buildx build --load --platform "linux/$native_arch" --target api \ -t "$api_image" "$repository_root" docker buildx build --load --platform "linux/$native_arch" --target web \ -t "$web_image" "$repository_root" docker buildx build --load --platform "linux/$native_arch" --target acceptance-netem \ -t "$netem_image" "$repository_root" api_digest=$(docker image inspect "$api_image" --format '{{.Id}}') [[ $api_digest =~ ^sha256:[0-9a-f]{64}$ ]] config="$state_root/k3d.rendered.yaml" render_k3d_config "$config" EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config" docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null "$k3d" image import -c "$cluster_name" "$api_image" "$web_image" "$netem_image" local cnpg_manifest="$state_root/cnpg.yaml" curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest" [[ $(shasum -a 256 "$cnpg_manifest" | awk '{print $1}') == "$CNPG_MANIFEST_SHA256" ]] || { echo 'CNPG manifest checksum mismatch' >&2 exit 1 } chmod 0600 "$cnpg_manifest" kubectl --context "$context" apply --server-side --force-conflicts -f "$cnpg_manifest" >/dev/null kubectl --context "$context" -n cnpg-system rollout status \ deployment/cnpg-controller-manager --timeout=10m apply_runtime_secrets kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/database.yaml" >/dev/null kubectl --context "$context" -n "$namespace" wait --for=condition=Ready \ cluster/easyai-postgres --timeout=15m >/dev/null run_migrations "$api_image" mark_local_database import_snapshot "$api_image" "$snapshot" sed "s|image: easyai-api|image: $api_image|g; s|image: easyai-acceptance-netem|image: $netem_image|g" \ "$manifest_root/support-services.yaml" | kubectl --context "$context" -n "$namespace" apply -f - >/dev/null wait_rollout deployment/easyai-acceptance-emulator wait_rollout deployment/easyai-acceptance-callback-collector wait_rollout deployment/easyai-acceptance-upstream-proxy wait_rollout deployment/easyai-postgres-proxy-ningbo wait_rollout deployment/easyai-postgres-proxy-hongkong render_and_apply_application "$api_image" "$web_image" sed "s|image: easyai-web|image: $web_image|g" "$manifest_root/tls-edges.yaml" | kubectl --context "$context" -n "$namespace" apply -f - >/dev/null wait_rollout deployment/easyai-api-ningbo wait_rollout deployment/easyai-api-hongkong wait_rollout deployment/easyai-worker-ningbo wait_rollout deployment/easyai-worker-hongkong wait_rollout deployment/easyai-capacity-controller wait_rollout deployment/easyai-acceptance-edge-ningbo wait_rollout deployment/easyai-acceptance-edge-hongkong bootstrap_runtime "$api_image" "$api_digest" "$snapshot" "$script_dir/network-fault.sh" baseline echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt" } status_cluster() { local k3d k3d=$(k3d_binary) || { echo 'k3d is unavailable' >&2 exit 1 } "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name" || { echo "cluster $cluster_name does not exist" >&2 exit 1 } kubectl --context "$context" get nodes -o wide kubectl --context "$context" -n "$namespace" get cluster,pod,deployment echo "local_acceptance_status=PASS context=$context" } down_cluster() { [[ ${1:-} == --confirm && $# -eq 1 ]] || { echo 'down requires the literal --confirm flag' >&2 exit 64 } local k3d k3d=$(k3d_binary) "$k3d" cluster delete "$cluster_name" echo "local_acceptance_down=PASS preserved_private_root=$private_root" } case ${1:-} in install-tools) [[ $# -eq 1 ]] || { usage >&2; exit 64; } install_tools ;; preflight) [[ $# -eq 1 ]] || { usage >&2; exit 64; } preflight ;; up) shift up_cluster "$@" ;; status) [[ $# -eq 1 ]] || { usage >&2; exit 64; } status_cluster ;; down) shift down_cluster "$@" ;; *) usage >&2 exit 64 ;; esac