#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
repository_root=$(cd "$script_dir/../.." && pwd)
manifest_root="$repository_root/deploy/kubernetes/local-acceptance"
lock_file="$manifest_root/dependencies.lock"
private_root="$repository_root/.local-secrets/acceptance"
tools_root="$private_root/tools"
state_root="$private_root/state"
media_root="$private_root/media"
cluster_name=easyai-acceptance-local
context="k3d-$cluster_name"
namespace=easyai
cluster_network="k3d-$cluster_name"
cluster_subnet=172.30.0.0/16
identity_file="$state_root/control-plane-identity.json"
# shellcheck source=scripts/acceptance/local-control-plane.sh
source "$script_dir/local-control-plane.sh"

usage() {
  cat <<'EOF'
Usage:
  scripts/acceptance/local-cluster.sh install-tools
  scripts/acceptance/local-cluster.sh preflight
  scripts/acceptance/local-cluster.sh up --snapshot <acceptance-snapshot-v1.json>
  scripts/acceptance/local-cluster.sh new-run
  scripts/acceptance/local-cluster.sh status
  scripts/acceptance/local-cluster.sh down --confirm

The full cluster requires Docker Desktop memory >= 24 GiB and host free disk
>= 30 GiB. Failed `up` runs are kept intact for diagnosis. `down` is the only
destructive lifecycle action.
EOF
}

load_lock() {
  [[ -f $lock_file && ! -L $lock_file ]] || {
    echo "missing dependency lock: $lock_file" >&2
    exit 1
  }
  # shellcheck source=/dev/null
  source "$lock_file"
  : "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${K3S_PAUSE_IMAGE:?}" "${K3S_BUSYBOX_IMAGE:?}"
  : "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
  : "${CNPG_CONTROLLER_IMAGE:?}" "${CNPG_POSTGRES_IMAGE:?}" "${K3S_COREDNS_IMAGE:?}"
  : "${K3S_METRICS_SERVER_IMAGE:?}" "${K3S_LOCAL_PATH_IMAGE:?}"
}

require_command() {
  command -v "$1" >/dev/null 2>&1 || {
    echo "$1 is required" >&2
    exit 1
  }
}

private_file() {
  local path=$1
  [[ -f $path && ! -L $path ]] || return 1
  local mode
  if [[ $(uname -s) == Darwin ]]; then
    mode=$(stat -f '%Lp' "$path")
  else
    mode=$(stat -c '%a' "$path")
  fi
  [[ $mode == 600 ]]
}

install_tools() {
  load_lock
  require_command curl
  require_command shasum
  require_command go
  install -d -m 0700 "$tools_root"
  local os arch checksum url temporary
  os=$(uname -s | tr '[:upper:]' '[:lower:]')
  arch=$(uname -m)
  [[ $os == darwin ]] || {
    echo 'the pinned local acceptance installer currently supports macOS only' >&2
    exit 1
  }
  case $arch in
    arm64)
      arch=arm64
      checksum=$K3D_DARWIN_ARM64_SHA256
      ;;
    x86_64)
      arch=amd64
      checksum=$K3D_DARWIN_AMD64_SHA256
      ;;
    *)
      echo "unsupported host architecture: $arch" >&2
      exit 1
      ;;
  esac
  url="https://github.com/k3d-io/k3d/releases/download/$K3D_VERSION/k3d-darwin-$arch"
  temporary="$tools_root/k3d.download"
  curl -fsSL "$url" -o "$temporary"
  [[ $(shasum -a 256 "$temporary" | awk '{print $1}') == "$checksum" ]] || {
    echo 'k3d checksum mismatch' >&2
    exit 1
  }
  chmod 0555 "$temporary"
  mv "$temporary" "$tools_root/k3d"
  GOBIN="$tools_root" go install "github.com/google/go-containerregistry/cmd/crane@$CRANE_VERSION"
  chmod 0555 "$tools_root/crane"
  echo "local_acceptance_tools=PASS k3d=$K3D_VERSION crane=$CRANE_VERSION path=$tools_root"
}

k3d_binary() {
  if [[ -x $tools_root/k3d ]]; then
    printf '%s\n' "$tools_root/k3d"
    return
  fi
  command -v k3d
}

docker_memory_bytes() {
  docker info --format '{{.MemTotal}}'
}

host_available_disk_bytes() {
  df -Pk "$repository_root" | awk 'NR == 2 {printf "%.0f\n", $4 * 1024}'
}

preflight() {
  local required_disk_gib=${1:-30} required_disk_bytes
  [[ $required_disk_gib =~ ^[0-9]+$ && $required_disk_gib -ge 10 ]]
  required_disk_bytes=$((required_disk_gib * 1024 * 1024 * 1024))
  load_lock
  require_command docker
  require_command kubectl
  require_command jq
  require_command openssl
  require_command sed
  require_command shasum
  require_command go
  require_command node
  require_command curl
  local k3d memory_bytes architecture cpu_count available_disk_bytes available_disk_gib
  k3d=$(k3d_binary) || {
    echo 'k3d is missing; run install-tools' >&2
    exit 1
  }
  "$k3d" version | grep -Fq "$K3D_VERSION" || {
    echo "k3d must be pinned to $K3D_VERSION" >&2
    exit 1
  }
  if [[ ! -x $tools_root/crane ]] || ! "$tools_root/crane" version | grep -Fq "${CRANE_VERSION#v}"; then
    echo "crane must be pinned to $CRANE_VERSION; run install-tools" >&2
    exit 1
  fi
  docker info >/dev/null
  memory_bytes=$(docker_memory_bytes)
  [[ $memory_bytes =~ ^[0-9]+$ && $memory_bytes -ge 25769803776 ]] || {
    memory_gib=$(awk -v bytes="${memory_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
    echo "Docker Desktop memory is ${memory_gib} GiB; local acceptance requires at least 24 GiB" >&2
    exit 1
  }
  architecture=$(docker info --format '{{.Architecture}}')
  [[ $architecture == aarch64 || $architecture == arm64 || $architecture == x86_64 ]] || {
    echo "unsupported Docker architecture: $architecture" >&2
    exit 1
  }
  docker buildx version >/dev/null
  cpu_count=$(docker info --format '{{.NCPU}}')
  [[ $cpu_count =~ ^[0-9]+$ && $cpu_count -ge 10 ]] || {
    echo "Docker Desktop CPUs=${cpu_count:-unknown}; local acceptance requires at least 10" >&2
    exit 1
  }
  available_disk_bytes=$(host_available_disk_bytes)
  [[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge $required_disk_bytes ]] || {
    available_disk_gib=$(awk -v bytes="${available_disk_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
    echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least ${required_disk_gib} GiB" >&2
    exit 1
  }
  echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes required_disk_gib=$required_disk_gib architecture=$architecture k3d=$K3D_VERSION"
}

ensure_private_material() {
  install -d -m 0700 "$private_root" "$state_root" "$media_root"
  local env_file="$state_root/local.env"
  if [[ ! -e $env_file ]]; then
    umask 077
    {
      printf 'LOCAL_CLUSTER_ID=%s\n' "easyai-local-$(openssl rand -hex 12)"
      printf 'POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 32)"
      printf 'CONFIG_JWT_SECRET=%s\n' "$(openssl rand -hex 32)"
      printf 'SERVER_MAIN_INTERNAL_TOKEN=%s\n' "$(openssl rand -hex 32)"
      printf 'SERVER_MAIN_INTERNAL_KEY=local-acceptance\n'
      printf 'SERVER_MAIN_INTERNAL_SECRET=%s\n' "$(openssl rand -hex 32)"
    } >"$env_file"
    chmod 0600 "$env_file"
  fi
  private_file "$env_file" || {
    echo "local acceptance state must be a 0600 regular file: $env_file" >&2
    exit 1
  }
}

generate_tls() {
  local ca_key="$state_root/ca.key" ca_crt="$state_root/ca.crt"
  local tls_key="$state_root/tls.key" tls_csr="$state_root/tls.csr"
  local tls_crt="$state_root/tls.crt" extensions="$state_root/tls.ext"
  if [[ ! -e $ca_key ]]; then
    umask 077
    openssl genrsa -out "$ca_key" 3072 >/dev/null 2>&1
    openssl req -x509 -new -key "$ca_key" -sha256 -days 30 \
      -subj '/CN=EasyAI local acceptance CA' -out "$ca_crt" >/dev/null 2>&1
    openssl genrsa -out "$tls_key" 2048 >/dev/null 2>&1
    openssl req -new -key "$tls_key" -subj '/CN=gateway.easyai.local' \
      -out "$tls_csr" >/dev/null 2>&1
    printf 'subjectAltName=DNS:gateway.easyai.local\nextendedKeyUsage=serverAuth\n' >"$extensions"
    openssl x509 -req -in "$tls_csr" -CA "$ca_crt" -CAkey "$ca_key" \
      -CAcreateserial -out "$tls_crt" -days 30 -sha256 -extfile "$extensions" \
      >/dev/null 2>&1
    chmod 0600 "$ca_key" "$ca_crt" "$tls_key" "$tls_csr" "$tls_crt" "$extensions"
  fi
  private_file "$ca_crt" && private_file "$tls_key" && private_file "$tls_crt"
}

render_k3d_config() {
  local output=$1 host_cpu host_memory_bytes host_memory_mib
  local site_system_cpu witness_system_cpu site_system_memory witness_system_memory
  [[ $media_root != *'|'* ]]
  host_cpu=$(docker info --format '{{.NCPU}}')
  host_memory_bytes=$(docker_memory_bytes)
  host_memory_mib=$((host_memory_bytes / 1024 / 1024))
  site_system_cpu="$(((host_cpu - 4) * 1000 + 750))m"
  witness_system_cpu="$(((host_cpu - 2) * 1000 + 750))m"
  site_system_memory="$((host_memory_mib - 8192 + 512))Mi"
  witness_system_memory="$((host_memory_mib - 4096 + 512))Mi"
  sed \
    -e "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" \
    -e "s|EASYAI_ACCEPTANCE_K3S_IMAGE|${K3S_IMAGE%%@*}|g" \
    -e "s|EASYAI_ACCEPTANCE_CLUSTER_SUBNET|$cluster_subnet|g" \
    -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_CPU|$site_system_cpu|g" \
    -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_CPU|$witness_system_cpu|g" \
    -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_MEMORY|$site_system_memory|g" \
    -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_MEMORY|$witness_system_memory|g" \
    "$manifest_root/k3d.yaml" >"$output"
}

cluster_volumes() {
  printf '%s\n' \
    easyai-acceptance-local-server-0-etcd \
    easyai-acceptance-local-server-1-etcd \
    easyai-acceptance-local-server-2-etcd \
    easyai-acceptance-local-server-0-storage \
    easyai-acceptance-local-server-1-storage
}

create_cluster_volumes() {
  local volume
  while read -r volume; do
    if docker volume inspect "$volume" >/dev/null 2>&1; then
      echo "stale local acceptance volume exists: $volume; run explicit down --confirm" >&2
      return 1
    fi
  done < <(cluster_volumes)
  while read -r volume; do
    docker volume create --label easyai.io/environment=local-acceptance "$volume" >/dev/null
  done < <(cluster_volumes)
}

remove_cluster_volumes() {
  local volume
  while read -r volume; do
    docker volume rm "$volume" >/dev/null 2>&1 || true
  done < <(cluster_volumes)
}

reconcile_cluster_network() {
  docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
  local actual_subnet attached_containers
  actual_subnet=$(docker network inspect "$cluster_network" --format '{{(index .IPAM.Config 0).Subnet}}')
  attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
  if [[ $actual_subnet == "$cluster_subnet" ]]; then
    return 0
  fi
  if ((attached_containers != 0)); then
    echo "stale local acceptance network has active containers: network=$cluster_network expected_subnet=$cluster_subnet actual_subnet=$actual_subnet" >&2
    return 1
  fi
  docker network rm "$cluster_network" >/dev/null
}

remove_cluster_network() {
  docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
  local attached_containers
  attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
  if ((attached_containers != 0)); then
    echo "refusing to remove local acceptance network with active containers: network=$cluster_network containers=$attached_containers" >&2
    return 1
  fi
  docker network rm "$cluster_network" >/dev/null
}

pull_dependency_images() {
  local image
  for image in "$K3S_IMAGE" "$K3S_PAUSE_IMAGE" "$K3S_BUSYBOX_IMAGE" \
    "$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
    "$K3S_COREDNS_IMAGE" "$K3S_METRICS_SERVER_IMAGE" "$K3S_LOCAL_PATH_IMAGE"; do
    pull_dependency_image "$image"
  done
}

docker_pull_with_deadline() {
  local image=$1 pid deadline
  docker pull "$image" >/dev/null 2>&1 &
  pid=$!
  deadline=$((SECONDS + 10))
  while kill -0 "$pid" >/dev/null 2>&1; do
    if ((SECONDS >= deadline)); then
      kill "$pid" >/dev/null 2>&1 || true
      wait "$pid" >/dev/null 2>&1 || true
      return 1
    fi
    sleep 1
  done
  wait "$pid"
}

docker_start_with_deadline() {
  local container=$1 pid deadline
  docker start "$container" >/dev/null 2>&1 &
  pid=$!
  deadline=$((SECONDS + 10))
  while kill -0 "$pid" >/dev/null 2>&1; do
    if ((SECONDS >= deadline)); then
      kill "$pid" >/dev/null 2>&1 || true
      wait "$pid" >/dev/null 2>&1 || true
      return 1
    fi
    sleep 1
  done
  wait "$pid"
}

verify_docker_container_start() {
  local image=$1 container="easyai-acceptance-start-probe-$$" exit_code
  docker create --name "$container" --network none --entrypoint /bin/true "$image" >/dev/null
  if ! docker_start_with_deadline "$container"; then
    docker rm -f "$container" >/dev/null 2>&1 || true
    echo 'gate=local_docker_container_start_unavailable Docker cannot start a minimal acceptance container within 10 seconds' >&2
    return 1
  fi
  exit_code=$(docker inspect "$container" --format '{{.State.ExitCode}}')
  docker rm -f "$container" >/dev/null
  [[ $exit_code == 0 ]] || {
    echo "gate=local_docker_container_start_failed minimal acceptance container exit_code=$exit_code" >&2
    return 1
  }
}

pull_dependency_image() {
  local image=$1 name_tag repository digest cache_tag native_arch archive source_tag
  name_tag=${image%%@*}
  repository=${name_tag%:*}
  digest=${image##*@sha256:}
  [[ $digest =~ ^[0-9a-f]{64}$ ]]
  cache_tag="$repository:easyai-lock-${digest:0:20}"
  if docker image inspect "$image" >/dev/null 2>&1; then
    docker tag "$image" "$name_tag"
    docker tag "$image" "$cache_tag"
    return
  fi
  if docker image inspect "$cache_tag" >/dev/null 2>&1; then
    docker tag "$cache_tag" "$name_tag"
    return
  fi
  if docker_pull_with_deadline "$image"; then
    docker tag "$image" "$name_tag"
    docker tag "$name_tag" "$cache_tag"
    return
  fi
  native_arch=$(docker info --format '{{.Architecture}}')
  case $native_arch in
    aarch64|arm64) native_arch=arm64 ;;
    x86_64) native_arch=amd64 ;;
    *) return 1 ;;
  esac
  archive="$state_root/dependency-image.tar"
  [[ ! -L $archive ]]
  umask 077
  "$tools_root/crane" pull --platform "linux/$native_arch" "$image" "$archive"
  docker load -i "$archive" >/dev/null
  : >"$archive"
  source_tag="$repository:i-was-a-digest"
  docker tag "$source_tag" "$name_tag"
  docker tag "$source_tag" "$cache_tag"
}

wait_rollout() {
  kubectl --context "$context" -n "$namespace" rollout status "$1" --timeout="${2:-10m}"
}

wait_internal_gateway() {
  local pod attempt=0
  pod=$(kubectl --context "$context" -n "$namespace" get pod \
    -l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
    -o 'jsonpath={.items[0].metadata.name}')
  until kubectl --context "$context" -n "$namespace" exec "$pod" -c api -- \
    wget -q --spider https://gateway.easyai.local/api/v1/healthz \
    >/dev/null 2>&1; do
    attempt=$((attempt + 1))
    if ((attempt >= 60)); then
      echo 'local acceptance gateway DNS or TLS trust did not become ready' >&2
      return 1
    fi
    sleep 1
  done
}

apply_runtime_secrets() {
  # shellcheck source=/dev/null
  source "$state_root/local.env"
  local database_ningbo database_hongkong database_direct
  database_ningbo="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-ningbo.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
  database_hongkong="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-hongkong.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
  database_direct="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-rw.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"

  kubectl --context "$context" create namespace "$namespace" --dry-run=client -o yaml |
    kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" create secret generic easyai-postgres-app \
    --type=kubernetes.io/basic-auth \
    --from-literal=username=easyai --from-literal=password="$POSTGRES_PASSWORD" \
    --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" create secret generic easyai-ai-gateway-runtime \
    --from-literal=AI_GATEWAY_DATABASE_URL="$database_direct" \
    --from-literal=CONFIG_JWT_SECRET="$CONFIG_JWT_SECRET" \
    --from-literal=SERVER_MAIN_INTERNAL_TOKEN="$SERVER_MAIN_INTERNAL_TOKEN" \
    --from-literal=SERVER_MAIN_INTERNAL_KEY="$SERVER_MAIN_INTERNAL_KEY" \
    --from-literal=SERVER_MAIN_INTERNAL_SECRET="$SERVER_MAIN_INTERNAL_SECRET" \
    --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" create secret generic easyai-database-ningbo \
    --from-literal=AI_GATEWAY_DATABASE_URL="$database_ningbo" \
    --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" create secret generic easyai-database-hongkong \
    --from-literal=AI_GATEWAY_DATABASE_URL="$database_hongkong" \
    --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" create secret generic easyai-oss-backup \
    --from-literal=ACCESS_KEY_ID=local-disabled \
    --from-literal=ACCESS_SECRET_KEY=local-disabled \
    --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" create secret tls easyai-acceptance-tls \
    --cert="$state_root/tls.crt" --key="$state_root/tls.key" \
    --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-ca \
    --from-file=ca.crt="$state_root/ca.crt" --dry-run=client -o yaml |
    kubectl --context "$context" apply -f - >/dev/null
}

run_migrations() {
  local api_image=$1
  kubectl --context "$context" -n "$namespace" delete job easyai-local-migrate \
    --ignore-not-found --wait=true >/dev/null
  cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
  name: easyai-local-migrate
  namespace: easyai
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: migrate
          image: $api_image
          command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
          envFrom:
            - secretRef:
                name: easyai-ai-gateway-runtime
EOF
  kubectl --context "$context" -n "$namespace" wait \
    --for=condition=complete job/easyai-local-migrate --timeout=10m >/dev/null
}

mark_local_database() {
  # shellcheck source=/dev/null
  source "$state_root/local.env"
  local primary
  primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
    -o 'jsonpath={.status.currentPrimary}')
  [[ $LOCAL_CLUSTER_ID =~ ^easyai-local-[0-9a-f]{24}$ ]]
  printf '%s\n' \
    "INSERT INTO system_settings(setting_key,value)
     VALUES('acceptance_local_cluster_id',jsonb_build_object('clusterId', :'cluster_id'))
     ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" |
    kubectl --context "$context" -n "$namespace" exec -i "$primary" -c postgres -- \
      psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \
        -v cluster_id="$LOCAL_CLUSTER_ID" >/dev/null
}

import_snapshot() {
  local api_image=$1 snapshot=$2
  # shellcheck source=/dev/null
  source "$state_root/local.env"
  kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-snapshot \
    --from-file=snapshot.json="$snapshot" --dry-run=client -o yaml |
    kubectl --context "$context" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" delete job easyai-local-snapshot-import \
    --ignore-not-found --wait=true >/dev/null
  cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
  name: easyai-local-snapshot-import
  namespace: easyai
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      securityContext:
        fsGroup: 10001
        fsGroupChangePolicy: OnRootMismatch
      initContainers:
        - name: stage-snapshot
          image: $api_image
          command:
            - /bin/sh
            - -ec
            - umask 077; cp /snapshot-source/snapshot.json /snapshot/snapshot.json
          volumeMounts:
            - name: snapshot-source
              mountPath: /snapshot-source
              readOnly: true
            - name: snapshot
              mountPath: /snapshot
      containers:
        - name: import
          image: $api_image
          command:
            - /app/easyai-ai-gateway-acceptance-snapshot
            - import
            - --input
            - /snapshot/snapshot.json
            - --local-cluster-id
            - $LOCAL_CLUSTER_ID
          envFrom:
            - secretRef:
                name: easyai-ai-gateway-runtime
          volumeMounts:
            - name: snapshot
              mountPath: /snapshot
              readOnly: true
      volumes:
        - name: snapshot-source
          configMap:
            name: easyai-acceptance-snapshot
        - name: snapshot
          emptyDir: {}
EOF
  kubectl --context "$context" -n "$namespace" wait \
    --for=condition=complete job/easyai-local-snapshot-import --timeout=5m >/dev/null
}

replay_acceptance_import_migrations() {
  local api_image=$1
  kubectl --context "$context" -n "$namespace" delete job easyai-local-migration-replay \
    --ignore-not-found --wait=true >/dev/null
  cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
  name: easyai-local-migration-replay
  namespace: easyai
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: migrate
          image: $api_image
          command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
          env:
            - name: AI_GATEWAY_MIGRATION_REAPPLY_ACCEPTANCE_IMPORT
              value: "true"
          envFrom:
            - secretRef:
                name: easyai-ai-gateway-runtime
EOF
  kubectl --context "$context" -n "$namespace" wait \
    --for=condition=complete job/easyai-local-migration-replay --timeout=5m >/dev/null
}

render_and_apply_application() {
  local api_image=$1 web_image=$2 rendered=$state_root/application.rendered.yaml
  sed \
    -e "s|image: easyai-api|image: $api_image|g" \
    -e "s|image: easyai-web|image: $web_image|g" \
    -e 's/nodePort: 31088/nodePort: 32088/g' \
    -e 's/nodePort: 31089/nodePort: 32089/g' \
    "$repository_root/deploy/kubernetes/production/application.yaml" >"$rendered"
  chmod 0600 "$rendered"
  kubectl --context "$context" -n "$namespace" apply \
    -f "$repository_root/deploy/kubernetes/production/service-account-rbac.yaml" >/dev/null
  kubectl --context "$context" -n "$namespace" apply \
    -f "$manifest_root/capacity-controller-rbac.yaml" >/dev/null
  kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/local-config.yaml" >/dev/null
  kubectl --context "$context" -n "$namespace" apply -f "$rendered" >/dev/null
  kubectl --context "$context" -n "$namespace" scale \
    deployment/easyai-worker-hongkong --replicas=0 >/dev/null
  kubectl --context "$context" -n "$namespace" rollout status \
    deployment/easyai-worker-hongkong --timeout=5m >/dev/null
  for workload in easyai-worker-ningbo easyai-worker-hongkong; do
    kubectl --context "$context" -n "$namespace" patch deployment "$workload" \
      --type=merge -p='{"spec":{"template":{"spec":{"affinity":null}}}}' >/dev/null
  done
  kubectl --context "$context" -n "$namespace" scale \
    deployment/easyai-worker-hongkong --replicas=2 >/dev/null

  for workload in easyai-api-ningbo easyai-worker-ningbo; do
    kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
      --from=secret/easyai-database-ningbo >/dev/null
  done
  for workload in easyai-api-hongkong easyai-worker-hongkong; do
    kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
      --from=secret/easyai-database-hongkong >/dev/null
  done
  for workload in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do
    kubectl --context "$context" -n "$namespace" patch deployment "$workload" --type=json \
      -p='[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"application-data","hostPath":{"path":"/var/lib/easyai-acceptance/media","type":"DirectoryOrCreate"}}}]' \
      >/dev/null
    local container=worker ca_patch
    [[ $workload == easyai-api-* ]] && container=api
    ca_patch=$(jq -cn --arg container "$container" '{
      spec:{template:{spec:{
        containers:[{
          name:$container,
          env:[{name:"SSL_CERT_FILE",value:"/etc/easyai-acceptance-ca/ca.crt"}],
          volumeMounts:[{name:"acceptance-ca",mountPath:"/etc/easyai-acceptance-ca",readOnly:true}]
        }],
        volumes:[{name:"acceptance-ca",configMap:{name:"easyai-acceptance-ca",defaultMode:292}}]
      }}}
    }')
    kubectl --context "$context" -n "$namespace" patch deployment "$workload" \
      --type=strategic -p="$ca_patch" >/dev/null
  done
  kubectl --context "$context" -n "$namespace" scale \
    deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null
  kubectl --context "$context" -n "$namespace" set resources deployment/easyai-api-ningbo \
    deployment/easyai-api-hongkong --containers=api \
    --requests=cpu=250m,memory=512Mi --limits=cpu=750m,memory=2Gi >/dev/null
  kubectl --context "$context" -n "$namespace" set resources deployment/easyai-worker-ningbo \
    deployment/easyai-worker-hongkong --containers=worker \
    --requests=cpu=750m,memory=1536Mi --limits=cpu=1500m,memory=2Gi >/dev/null
  kubectl --context "$context" -n "$namespace" set env deployment/easyai-worker-ningbo \
    deployment/easyai-worker-hongkong \
    AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
}

bootstrap_runtime() {
  local api_image=$1 api_digest=$2 snapshot=$3
  # shellcheck source=/dev/null
  source "$state_root/local.env"
  local snapshot_hash release_sha runtime_file
  snapshot_hash=$(jq -r '.snapshotSha256' "$snapshot")
  release_sha=$(git -C "$repository_root" rev-parse HEAD)
  runtime_file="$state_root/runtime-$snapshot_hash-$(date -u '+%Y%m%dT%H%M%SZ')-$$.json"
  kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
    --ignore-not-found --wait=true >/dev/null
  cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
apiVersion: v1
kind: Pod
metadata:
  name: easyai-local-bootstrap
  namespace: easyai
  labels:
    easyai.io/environment: local-acceptance
spec:
  activeDeadlineSeconds: 600
  restartPolicy: Never
  securityContext:
    fsGroup: 10001
    fsGroupChangePolicy: OnRootMismatch
  containers:
    - name: bootstrap
      image: $api_image
      command:
        - /app/easyai-ai-gateway-acceptance-bootstrap
        - --local-cluster-id
        - $LOCAL_CLUSTER_ID
        - --release-sha
        - $release_sha
        - --api-image-digest
        - $api_digest
        - --worker-image-digest
        - $api_digest
        - --emulator-base-url
        - http://easyai-acceptance-upstream-proxy.easyai.svc.cluster.local:8090
        - --callback-url
        - http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks
        - --identity-shards
        - "32"
        - --output
        - /runtime/runtime.json
      envFrom:
        - secretRef:
            name: easyai-ai-gateway-runtime
      volumeMounts:
        - name: runtime
          mountPath: /runtime
    - name: runtime-export
      image: $api_image
      command: ["/bin/sh", "-ec", "sleep 600"]
      volumeMounts:
        - name: runtime
          mountPath: /runtime
          readOnly: true
  volumes:
    - name: runtime
      emptyDir: {}
EOF
  local bootstrap_exit='' deadline=$((SECONDS + 300))
  while ((SECONDS < deadline)); do
    bootstrap_exit=$(kubectl --context "$context" -n "$namespace" get \
      pod/easyai-local-bootstrap -o json |
      jq -r '.status.containerStatuses[]? | select(.name == "bootstrap") |
        if .state.terminated then (.state.terminated.exitCode | tostring) else empty end')
    [[ -z $bootstrap_exit ]] || break
    sleep 1
  done
  if [[ $bootstrap_exit != 0 ]]; then
    echo "local acceptance bootstrap failed or timed out: exit_code=${bootstrap_exit:-unknown}" >&2
    kubectl --context "$context" -n "$namespace" logs \
      pod/easyai-local-bootstrap -c bootstrap >&2 || true
    exit 1
  fi
  umask 077
  kubectl --context "$context" -n "$namespace" cp \
    -c runtime-export easyai-local-bootstrap:/runtime/runtime.json "$runtime_file" >/dev/null
  chmod 0600 "$runtime_file"
  kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
    --wait=true >/dev/null
  printf '%s\n' "$runtime_file" >"$state_root/current-runtime-path"
  chmod 0600 "$state_root/current-runtime-path"
}

up_cluster() {
  [[ ${1:-} == --snapshot && $# -eq 2 ]] || {
    usage >&2
    exit 64
  }
  local snapshot_input=$2 snapshot_dir snapshot
  [[ -f $snapshot_input && ! -L $snapshot_input ]] || {
    echo 'snapshot must be a regular non-symlink file' >&2
    exit 1
  }
  snapshot_dir=$(cd "$(dirname "$snapshot_input")" && pwd -P)
  snapshot="$snapshot_dir/$(basename "$snapshot_input")"
  [[ -z $(git -C "$repository_root" status --short) ]] || {
    echo 'local acceptance requires a clean source tree so report and image source are reproducible' >&2
    exit 1
  }
  preflight
  ensure_private_material
  generate_tls
  (
    cd "$repository_root/apps/api"
    go run ./cmd/acceptance-snapshot validate --input "$snapshot"
  )
  if [[ ! -e $state_root/snapshot.json || ! $snapshot -ef $state_root/snapshot.json ]]; then
    cp "$snapshot" "$state_root/snapshot.json"
  fi
  chmod 0600 "$state_root/snapshot.json"
  local k3d config native_arch release_sha api_image web_image netem_image api_digest
  k3d=$(k3d_binary)
  if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then
    echo "cluster $cluster_name already exists; use status or explicit down --confirm" >&2
    exit 1
  fi
  native_arch=$(docker info --format '{{.Architecture}}')
  case $native_arch in
    aarch64|arm64) native_arch=arm64 ;;
    x86_64) native_arch=amd64 ;;
  esac
  release_sha=$(git -C "$repository_root" rev-parse HEAD)
  api_image="easyai-acceptance-api:$release_sha"
  web_image="easyai-acceptance-web:$release_sha"
  netem_image="easyai-acceptance-netem:$release_sha"
  pull_dependency_images
  verify_docker_container_start "${K3S_IMAGE%%@*}"
  docker buildx build --load --platform "linux/$native_arch" --target api \
    -t "$api_image" "$repository_root"
  docker buildx build --load --platform "linux/$native_arch" --target web \
    -t "$web_image" "$repository_root"
  docker buildx build --load --platform "linux/$native_arch" --target acceptance-netem \
    -t "$netem_image" "$repository_root"
  api_digest=$(docker image inspect "$api_image" --format '{{.Id}}')
  [[ $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
  verify_docker_container_start "$netem_image"

  reconcile_cluster_network
  create_cluster_volumes
  config="$state_root/k3d.rendered.yaml"
  render_k3d_config "$config"
  EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config"
  kubectl --context "$context" apply -f "$manifest_root/cluster-dns.yaml" >/dev/null
  docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null
  docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null
  docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null
  verify_local_node_capacity "$context" || {
    echo 'local K3s node allocatable capacity does not match 4/8, 4/8, 2/4 resource envelope' >&2
    exit 1
  }
  local cluster_image
  for cluster_image in \
    "$api_image" "$web_image" "$netem_image" \
    "${K3S_PAUSE_IMAGE%%@*}" "${K3S_BUSYBOX_IMAGE%%@*}" \
    "${CNPG_CONTROLLER_IMAGE%%@*}" "${CNPG_POSTGRES_IMAGE%%@*}" \
    "${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"; do
    "$k3d" image import -c "$cluster_name" "$cluster_image"
  done

  local cnpg_manifest="$state_root/cnpg.yaml"
  curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest"
  [[ $(shasum -a 256 "$cnpg_manifest" | awk '{print $1}') == "$CNPG_MANIFEST_SHA256" ]] || {
    echo 'CNPG manifest checksum mismatch' >&2
    exit 1
  }
  chmod 0600 "$cnpg_manifest"
  kubectl --context "$context" apply --server-side --force-conflicts -f "$cnpg_manifest" >/dev/null
  kubectl --context "$context" -n cnpg-system rollout status \
    deployment/cnpg-controller-manager --timeout=10m
  apply_runtime_secrets
  sed "s|imageName: .*|imageName: ${CNPG_POSTGRES_IMAGE%%@*}|" "$manifest_root/database.yaml" |
    kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
  kubectl --context "$context" -n "$namespace" wait --for=condition=Ready \
    cluster/easyai-postgres --timeout=15m >/dev/null
  run_migrations "$api_image"
  mark_local_database
  import_snapshot "$api_image" "$snapshot"
  replay_acceptance_import_migrations "$api_image"

  sed "s|image: easyai-api|image: $api_image|g; s|image: easyai-acceptance-netem|image: $netem_image|g" \
    "$manifest_root/support-services.yaml" |
    kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
  wait_rollout deployment/easyai-acceptance-emulator
  wait_rollout deployment/easyai-acceptance-callback-collector
  wait_rollout deployment/easyai-acceptance-upstream-proxy
  wait_rollout deployment/easyai-postgres-proxy-ningbo
  wait_rollout deployment/easyai-postgres-proxy-hongkong
  render_and_apply_application "$api_image" "$web_image"
  sed "s|image: easyai-web|image: $web_image|g" "$manifest_root/tls-edges.yaml" |
    kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
  wait_rollout deployment/easyai-api-ningbo
  wait_rollout deployment/easyai-api-hongkong
  wait_rollout deployment/easyai-worker-ningbo
  wait_rollout deployment/easyai-worker-hongkong
  wait_rollout deployment/easyai-capacity-controller
  wait_rollout deployment/easyai-acceptance-edge-ningbo
  wait_rollout deployment/easyai-acceptance-edge-hongkong
  bootstrap_runtime "$api_image" "$api_digest" "$snapshot"
  wait_internal_gateway
  "$script_dir/network-fault.sh" baseline
  sleep 1
  record_local_control_plane_identity "$context" "$cluster_name" "$identity_file"
  echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt"
}

new_run() {
  load_lock
  preflight 10
  [[ -z $(git -C "$repository_root" status --short) ]] || {
    echo 'local acceptance requires a clean source tree for a fresh Run ID' >&2
    exit 1
  }
  if ! private_file "$state_root/current-runtime-path" || ! private_file "$state_root/snapshot.json"; then
    echo 'local acceptance runtime or snapshot is unavailable' >&2
    exit 1
  fi
  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
    echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild the cluster' >&2
    exit 1
  }
  verify_local_node_capacity "$context" || {
    echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
    exit 1
  }
  verify_local_apiserver_ready "$context" || {
    echo 'gate=local_control_plane_unavailable local Kubernetes API is not ready' >&2
    exit 1
  }
  local previous_runtime api_image api_digest previous_release current_release
  previous_runtime=$(<"$state_root/current-runtime-path")
  private_file "$previous_runtime" || {
    echo 'local acceptance current runtime is unavailable' >&2
    exit 1
  }
  previous_release=$(jq -r '.releaseSha' "$previous_runtime")
  current_release=$(git -C "$repository_root" rev-parse HEAD)
  [[ $previous_release == "$current_release" ]] || {
    echo 'local acceptance source changed after cluster creation; rebuild the cluster' >&2
    exit 1
  }
  api_digest=$(jq -r '.apiImageDigest' "$previous_runtime")
  api_image=$(kubectl --context "$context" -n "$namespace" get deployment easyai-api-ningbo \
    -o 'jsonpath={.spec.template.spec.containers[?(@.name=="api")].image}')
  [[ -n $api_image && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
  bootstrap_runtime "$api_image" "$api_digest" "$state_root/snapshot.json"
  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
    echo 'gate=local_control_plane_restarted local K3s server changed while creating a fresh run' >&2
    exit 1
  }
  echo "local_acceptance_new_run=PASS runtime_path=$state_root/current-runtime-path"
}

status_cluster() {
  local k3d
  k3d=$(k3d_binary) || {
    echo 'k3d is unavailable' >&2
    exit 1
  }
  "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name" || {
    echo "cluster $cluster_name does not exist" >&2
    exit 1
  }
  kubectl --context "$context" get nodes -o wide
  kubectl --context "$context" -n "$namespace" get cluster,pod,deployment
  verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
    echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild required' >&2
    exit 1
  }
  verify_local_node_capacity "$context" || {
    echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
    exit 1
  }
  echo "local_acceptance_status=PASS context=$context"
}

down_cluster() {
  [[ ${1:-} == --confirm && $# -eq 1 ]] || {
    echo 'down requires the literal --confirm flag' >&2
    exit 64
  }
  local k3d
  k3d=$(k3d_binary)
  if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then
    "$k3d" cluster delete "$cluster_name"
  fi
  remove_cluster_volumes
  remove_cluster_network
  echo "local_acceptance_down=PASS preserved_private_root=$private_root"
}

case ${1:-} in
  install-tools)
    [[ $# -eq 1 ]] || { usage >&2; exit 64; }
    install_tools
    ;;
  preflight)
    [[ $# -eq 1 ]] || { usage >&2; exit 64; }
    preflight
    ;;
  up)
    shift
    up_cluster "$@"
    ;;
  new-run)
    [[ $# -eq 1 ]] || { usage >&2; exit 64; }
    new_run
    ;;
  status)
    [[ $# -eq 1 ]] || { usage >&2; exit 64; }
    status_cluster
    ;;
  down)
    shift
    down_cluster "$@"
    ;;
  *)
    usage >&2
    exit 64
    ;;
esac
