wangbo
a95184b5b6
本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
960 lines
36 KiB
Bash
Executable File
960 lines
36 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
repository_root=$(cd "$script_dir/../.." && pwd)
|
|
manifest_root="$repository_root/deploy/kubernetes/local-acceptance"
|
|
lock_file="$manifest_root/dependencies.lock"
|
|
private_root="$repository_root/.local-secrets/acceptance"
|
|
tools_root="$private_root/tools"
|
|
state_root="$private_root/state"
|
|
media_root="$private_root/media"
|
|
cluster_name=easyai-acceptance-local
|
|
context="k3d-$cluster_name"
|
|
namespace=easyai
|
|
cluster_network="k3d-$cluster_name"
|
|
cluster_subnet=172.30.0.0/16
|
|
identity_file="$state_root/control-plane-identity.json"
|
|
# shellcheck source=scripts/acceptance/local-control-plane.sh
|
|
source "$script_dir/local-control-plane.sh"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
scripts/acceptance/local-cluster.sh install-tools
|
|
scripts/acceptance/local-cluster.sh preflight
|
|
scripts/acceptance/local-cluster.sh up --snapshot <acceptance-snapshot-v1.json>
|
|
scripts/acceptance/local-cluster.sh new-run
|
|
scripts/acceptance/local-cluster.sh status
|
|
scripts/acceptance/local-cluster.sh down --confirm
|
|
|
|
The full cluster requires Docker Desktop memory >= 24 GiB and host free disk
|
|
>= 30 GiB. Failed `up` runs are kept intact for diagnosis. `down` is the only
|
|
destructive lifecycle action.
|
|
EOF
|
|
}
|
|
|
|
load_lock() {
|
|
[[ -f $lock_file && ! -L $lock_file ]] || {
|
|
echo "missing dependency lock: $lock_file" >&2
|
|
exit 1
|
|
}
|
|
# shellcheck source=/dev/null
|
|
source "$lock_file"
|
|
: "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
|
|
: "${CNPG_CONTROLLER_IMAGE:?}" "${CNPG_POSTGRES_IMAGE:?}" "${K3S_COREDNS_IMAGE:?}"
|
|
: "${K3S_METRICS_SERVER_IMAGE:?}" "${K3S_LOCAL_PATH_IMAGE:?}"
|
|
}
|
|
|
|
require_command() {
|
|
command -v "$1" >/dev/null 2>&1 || {
|
|
echo "$1 is required" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
private_file() {
|
|
local path=$1
|
|
[[ -f $path && ! -L $path ]] || return 1
|
|
local mode
|
|
if [[ $(uname -s) == Darwin ]]; then
|
|
mode=$(stat -f '%Lp' "$path")
|
|
else
|
|
mode=$(stat -c '%a' "$path")
|
|
fi
|
|
[[ $mode == 600 ]]
|
|
}
|
|
|
|
install_tools() {
|
|
load_lock
|
|
require_command curl
|
|
require_command shasum
|
|
require_command go
|
|
install -d -m 0700 "$tools_root"
|
|
local os arch checksum url temporary
|
|
os=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
arch=$(uname -m)
|
|
[[ $os == darwin ]] || {
|
|
echo 'the pinned local acceptance installer currently supports macOS only' >&2
|
|
exit 1
|
|
}
|
|
case $arch in
|
|
arm64)
|
|
arch=arm64
|
|
checksum=$K3D_DARWIN_ARM64_SHA256
|
|
;;
|
|
x86_64)
|
|
arch=amd64
|
|
checksum=$K3D_DARWIN_AMD64_SHA256
|
|
;;
|
|
*)
|
|
echo "unsupported host architecture: $arch" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
url="https://github.com/k3d-io/k3d/releases/download/$K3D_VERSION/k3d-darwin-$arch"
|
|
temporary="$tools_root/k3d.download"
|
|
curl -fsSL "$url" -o "$temporary"
|
|
[[ $(shasum -a 256 "$temporary" | awk '{print $1}') == "$checksum" ]] || {
|
|
echo 'k3d checksum mismatch' >&2
|
|
exit 1
|
|
}
|
|
chmod 0555 "$temporary"
|
|
mv "$temporary" "$tools_root/k3d"
|
|
GOBIN="$tools_root" go install "github.com/google/go-containerregistry/cmd/crane@$CRANE_VERSION"
|
|
chmod 0555 "$tools_root/crane"
|
|
echo "local_acceptance_tools=PASS k3d=$K3D_VERSION crane=$CRANE_VERSION path=$tools_root"
|
|
}
|
|
|
|
k3d_binary() {
|
|
if [[ -x $tools_root/k3d ]]; then
|
|
printf '%s\n' "$tools_root/k3d"
|
|
return
|
|
fi
|
|
command -v k3d
|
|
}
|
|
|
|
docker_memory_bytes() {
|
|
docker info --format '{{.MemTotal}}'
|
|
}
|
|
|
|
host_available_disk_bytes() {
|
|
df -Pk "$repository_root" | awk 'NR == 2 {printf "%.0f\n", $4 * 1024}'
|
|
}
|
|
|
|
preflight() {
|
|
load_lock
|
|
require_command docker
|
|
require_command kubectl
|
|
require_command jq
|
|
require_command openssl
|
|
require_command sed
|
|
require_command shasum
|
|
require_command go
|
|
require_command node
|
|
require_command curl
|
|
local k3d memory_bytes architecture cpu_count available_disk_bytes available_disk_gib
|
|
k3d=$(k3d_binary) || {
|
|
echo 'k3d is missing; run install-tools' >&2
|
|
exit 1
|
|
}
|
|
"$k3d" version | grep -Fq "$K3D_VERSION" || {
|
|
echo "k3d must be pinned to $K3D_VERSION" >&2
|
|
exit 1
|
|
}
|
|
if [[ ! -x $tools_root/crane ]] || ! "$tools_root/crane" version | grep -Fq "${CRANE_VERSION#v}"; then
|
|
echo "crane must be pinned to $CRANE_VERSION; run install-tools" >&2
|
|
exit 1
|
|
fi
|
|
docker info >/dev/null
|
|
memory_bytes=$(docker_memory_bytes)
|
|
[[ $memory_bytes =~ ^[0-9]+$ && $memory_bytes -ge 25769803776 ]] || {
|
|
memory_gib=$(awk -v bytes="${memory_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
|
|
echo "Docker Desktop memory is ${memory_gib} GiB; local acceptance requires at least 24 GiB" >&2
|
|
exit 1
|
|
}
|
|
architecture=$(docker info --format '{{.Architecture}}')
|
|
[[ $architecture == aarch64 || $architecture == arm64 || $architecture == x86_64 ]] || {
|
|
echo "unsupported Docker architecture: $architecture" >&2
|
|
exit 1
|
|
}
|
|
docker buildx version >/dev/null
|
|
cpu_count=$(docker info --format '{{.NCPU}}')
|
|
[[ $cpu_count =~ ^[0-9]+$ && $cpu_count -ge 10 ]] || {
|
|
echo "Docker Desktop CPUs=${cpu_count:-unknown}; local acceptance requires at least 10" >&2
|
|
exit 1
|
|
}
|
|
available_disk_bytes=$(host_available_disk_bytes)
|
|
[[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge 32212254720 ]] || {
|
|
available_disk_gib=$(awk -v bytes="${available_disk_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
|
|
echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least 30 GiB" >&2
|
|
exit 1
|
|
}
|
|
echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes architecture=$architecture k3d=$K3D_VERSION"
|
|
}
|
|
|
|
ensure_private_material() {
|
|
install -d -m 0700 "$private_root" "$state_root" "$media_root"
|
|
local env_file="$state_root/local.env"
|
|
if [[ ! -e $env_file ]]; then
|
|
umask 077
|
|
{
|
|
printf 'LOCAL_CLUSTER_ID=%s\n' "easyai-local-$(openssl rand -hex 12)"
|
|
printf 'POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 32)"
|
|
printf 'CONFIG_JWT_SECRET=%s\n' "$(openssl rand -hex 32)"
|
|
printf 'SERVER_MAIN_INTERNAL_TOKEN=%s\n' "$(openssl rand -hex 32)"
|
|
printf 'SERVER_MAIN_INTERNAL_KEY=local-acceptance\n'
|
|
printf 'SERVER_MAIN_INTERNAL_SECRET=%s\n' "$(openssl rand -hex 32)"
|
|
} >"$env_file"
|
|
chmod 0600 "$env_file"
|
|
fi
|
|
private_file "$env_file" || {
|
|
echo "local acceptance state must be a 0600 regular file: $env_file" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
generate_tls() {
|
|
local ca_key="$state_root/ca.key" ca_crt="$state_root/ca.crt"
|
|
local tls_key="$state_root/tls.key" tls_csr="$state_root/tls.csr"
|
|
local tls_crt="$state_root/tls.crt" extensions="$state_root/tls.ext"
|
|
if [[ ! -e $ca_key ]]; then
|
|
umask 077
|
|
openssl genrsa -out "$ca_key" 3072 >/dev/null 2>&1
|
|
openssl req -x509 -new -key "$ca_key" -sha256 -days 30 \
|
|
-subj '/CN=EasyAI local acceptance CA' -out "$ca_crt" >/dev/null 2>&1
|
|
openssl genrsa -out "$tls_key" 2048 >/dev/null 2>&1
|
|
openssl req -new -key "$tls_key" -subj '/CN=gateway.easyai.local' \
|
|
-out "$tls_csr" >/dev/null 2>&1
|
|
printf 'subjectAltName=DNS:gateway.easyai.local\nextendedKeyUsage=serverAuth\n' >"$extensions"
|
|
openssl x509 -req -in "$tls_csr" -CA "$ca_crt" -CAkey "$ca_key" \
|
|
-CAcreateserial -out "$tls_crt" -days 30 -sha256 -extfile "$extensions" \
|
|
>/dev/null 2>&1
|
|
chmod 0600 "$ca_key" "$ca_crt" "$tls_key" "$tls_csr" "$tls_crt" "$extensions"
|
|
fi
|
|
private_file "$ca_crt" && private_file "$tls_key" && private_file "$tls_crt"
|
|
}
|
|
|
|
render_k3d_config() {
|
|
local output=$1 host_cpu host_memory_bytes host_memory_mib
|
|
local site_system_cpu witness_system_cpu site_system_memory witness_system_memory
|
|
[[ $media_root != *'|'* ]]
|
|
host_cpu=$(docker info --format '{{.NCPU}}')
|
|
host_memory_bytes=$(docker_memory_bytes)
|
|
host_memory_mib=$((host_memory_bytes / 1024 / 1024))
|
|
site_system_cpu="$(((host_cpu - 4) * 1000 + 750))m"
|
|
witness_system_cpu="$(((host_cpu - 2) * 1000 + 750))m"
|
|
site_system_memory="$((host_memory_mib - 8192 + 1024))Mi"
|
|
witness_system_memory="$((host_memory_mib - 4096 + 1024))Mi"
|
|
sed \
|
|
-e "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" \
|
|
-e "s|EASYAI_ACCEPTANCE_K3S_IMAGE|${K3S_IMAGE%%@*}|g" \
|
|
-e "s|EASYAI_ACCEPTANCE_CLUSTER_SUBNET|$cluster_subnet|g" \
|
|
-e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_CPU|$site_system_cpu|g" \
|
|
-e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_CPU|$witness_system_cpu|g" \
|
|
-e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_MEMORY|$site_system_memory|g" \
|
|
-e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_MEMORY|$witness_system_memory|g" \
|
|
"$manifest_root/k3d.yaml" >"$output"
|
|
}
|
|
|
|
cluster_volumes() {
|
|
printf '%s\n' \
|
|
easyai-acceptance-local-server-0-etcd \
|
|
easyai-acceptance-local-server-1-etcd \
|
|
easyai-acceptance-local-server-2-etcd \
|
|
easyai-acceptance-local-server-0-storage \
|
|
easyai-acceptance-local-server-1-storage
|
|
}
|
|
|
|
create_cluster_volumes() {
|
|
local volume
|
|
while read -r volume; do
|
|
if docker volume inspect "$volume" >/dev/null 2>&1; then
|
|
echo "stale local acceptance volume exists: $volume; run explicit down --confirm" >&2
|
|
return 1
|
|
fi
|
|
done < <(cluster_volumes)
|
|
while read -r volume; do
|
|
docker volume create --label easyai.io/environment=local-acceptance "$volume" >/dev/null
|
|
done < <(cluster_volumes)
|
|
}
|
|
|
|
remove_cluster_volumes() {
|
|
local volume
|
|
while read -r volume; do
|
|
docker volume rm "$volume" >/dev/null 2>&1 || true
|
|
done < <(cluster_volumes)
|
|
}
|
|
|
|
reconcile_cluster_network() {
|
|
docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
|
|
local actual_subnet attached_containers
|
|
actual_subnet=$(docker network inspect "$cluster_network" --format '{{(index .IPAM.Config 0).Subnet}}')
|
|
attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
|
|
if [[ $actual_subnet == "$cluster_subnet" ]]; then
|
|
return 0
|
|
fi
|
|
if ((attached_containers != 0)); then
|
|
echo "stale local acceptance network has active containers: network=$cluster_network expected_subnet=$cluster_subnet actual_subnet=$actual_subnet" >&2
|
|
return 1
|
|
fi
|
|
docker network rm "$cluster_network" >/dev/null
|
|
}
|
|
|
|
remove_cluster_network() {
|
|
docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0
|
|
local attached_containers
|
|
attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}')
|
|
if ((attached_containers != 0)); then
|
|
echo "refusing to remove local acceptance network with active containers: network=$cluster_network containers=$attached_containers" >&2
|
|
return 1
|
|
fi
|
|
docker network rm "$cluster_network" >/dev/null
|
|
}
|
|
|
|
pull_dependency_images() {
|
|
local image
|
|
for image in "$K3S_IMAGE" "$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
|
|
"$K3S_COREDNS_IMAGE" "$K3S_METRICS_SERVER_IMAGE" "$K3S_LOCAL_PATH_IMAGE"; do
|
|
pull_dependency_image "$image"
|
|
done
|
|
}
|
|
|
|
docker_pull_with_deadline() {
|
|
local image=$1 pid deadline
|
|
docker pull "$image" >/dev/null 2>&1 &
|
|
pid=$!
|
|
deadline=$((SECONDS + 10))
|
|
while kill -0 "$pid" >/dev/null 2>&1; do
|
|
if ((SECONDS >= deadline)); then
|
|
kill "$pid" >/dev/null 2>&1 || true
|
|
wait "$pid" >/dev/null 2>&1 || true
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
wait "$pid"
|
|
}
|
|
|
|
docker_start_with_deadline() {
|
|
local container=$1 pid deadline
|
|
docker start "$container" >/dev/null 2>&1 &
|
|
pid=$!
|
|
deadline=$((SECONDS + 10))
|
|
while kill -0 "$pid" >/dev/null 2>&1; do
|
|
if ((SECONDS >= deadline)); then
|
|
kill "$pid" >/dev/null 2>&1 || true
|
|
wait "$pid" >/dev/null 2>&1 || true
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
wait "$pid"
|
|
}
|
|
|
|
verify_docker_container_start() {
|
|
local image=$1 container="easyai-acceptance-start-probe-$$" exit_code
|
|
docker create --name "$container" --network none --entrypoint /bin/true "$image" >/dev/null
|
|
if ! docker_start_with_deadline "$container"; then
|
|
docker rm -f "$container" >/dev/null 2>&1 || true
|
|
echo 'gate=local_docker_container_start_unavailable Docker cannot start a minimal acceptance container within 10 seconds' >&2
|
|
return 1
|
|
fi
|
|
exit_code=$(docker inspect "$container" --format '{{.State.ExitCode}}')
|
|
docker rm -f "$container" >/dev/null
|
|
[[ $exit_code == 0 ]] || {
|
|
echo "gate=local_docker_container_start_failed minimal acceptance container exit_code=$exit_code" >&2
|
|
return 1
|
|
}
|
|
}
|
|
|
|
pull_dependency_image() {
|
|
local image=$1 name_tag repository digest cache_tag native_arch archive source_tag
|
|
name_tag=${image%%@*}
|
|
repository=${name_tag%:*}
|
|
digest=${image##*@sha256:}
|
|
[[ $digest =~ ^[0-9a-f]{64}$ ]]
|
|
cache_tag="$repository:easyai-lock-${digest:0:20}"
|
|
if docker image inspect "$image" >/dev/null 2>&1; then
|
|
docker tag "$image" "$name_tag"
|
|
docker tag "$image" "$cache_tag"
|
|
return
|
|
fi
|
|
if docker image inspect "$cache_tag" >/dev/null 2>&1; then
|
|
docker tag "$cache_tag" "$name_tag"
|
|
return
|
|
fi
|
|
if docker_pull_with_deadline "$image"; then
|
|
docker tag "$image" "$name_tag"
|
|
docker tag "$name_tag" "$cache_tag"
|
|
return
|
|
fi
|
|
native_arch=$(docker info --format '{{.Architecture}}')
|
|
case $native_arch in
|
|
aarch64|arm64) native_arch=arm64 ;;
|
|
x86_64) native_arch=amd64 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
archive="$state_root/dependency-image.tar"
|
|
[[ ! -L $archive ]]
|
|
umask 077
|
|
"$tools_root/crane" pull --platform "linux/$native_arch" "$image" "$archive"
|
|
docker load -i "$archive" >/dev/null
|
|
: >"$archive"
|
|
source_tag="$repository:i-was-a-digest"
|
|
docker tag "$source_tag" "$name_tag"
|
|
docker tag "$source_tag" "$cache_tag"
|
|
}
|
|
|
|
wait_rollout() {
|
|
kubectl --context "$context" -n "$namespace" rollout status "$1" --timeout="${2:-10m}"
|
|
}
|
|
|
|
wait_internal_gateway() {
|
|
local pod attempt=0
|
|
pod=$(kubectl --context "$context" -n "$namespace" get pod \
|
|
-l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
|
|
-o 'jsonpath={.items[0].metadata.name}')
|
|
until kubectl --context "$context" -n "$namespace" exec "$pod" -c api -- \
|
|
wget -q --spider https://gateway.easyai.local/api/v1/healthz \
|
|
>/dev/null 2>&1; do
|
|
attempt=$((attempt + 1))
|
|
if ((attempt >= 60)); then
|
|
echo 'local acceptance gateway DNS or TLS trust did not become ready' >&2
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
apply_runtime_secrets() {
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
local database_ningbo database_hongkong database_direct
|
|
database_ningbo="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-ningbo.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
|
|
database_hongkong="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-hongkong.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
|
|
database_direct="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-rw.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
|
|
|
|
kubectl --context "$context" create namespace "$namespace" --dry-run=client -o yaml |
|
|
kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-postgres-app \
|
|
--type=kubernetes.io/basic-auth \
|
|
--from-literal=username=easyai --from-literal=password="$POSTGRES_PASSWORD" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-ai-gateway-runtime \
|
|
--from-literal=AI_GATEWAY_DATABASE_URL="$database_direct" \
|
|
--from-literal=CONFIG_JWT_SECRET="$CONFIG_JWT_SECRET" \
|
|
--from-literal=SERVER_MAIN_INTERNAL_TOKEN="$SERVER_MAIN_INTERNAL_TOKEN" \
|
|
--from-literal=SERVER_MAIN_INTERNAL_KEY="$SERVER_MAIN_INTERNAL_KEY" \
|
|
--from-literal=SERVER_MAIN_INTERNAL_SECRET="$SERVER_MAIN_INTERNAL_SECRET" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-database-ningbo \
|
|
--from-literal=AI_GATEWAY_DATABASE_URL="$database_ningbo" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-database-hongkong \
|
|
--from-literal=AI_GATEWAY_DATABASE_URL="$database_hongkong" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-oss-backup \
|
|
--from-literal=ACCESS_KEY_ID=local-disabled \
|
|
--from-literal=ACCESS_SECRET_KEY=local-disabled \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret tls easyai-acceptance-tls \
|
|
--cert="$state_root/tls.crt" --key="$state_root/tls.key" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-ca \
|
|
--from-file=ca.crt="$state_root/ca.crt" --dry-run=client -o yaml |
|
|
kubectl --context "$context" apply -f - >/dev/null
|
|
}
|
|
|
|
run_migrations() {
|
|
local api_image=$1
|
|
kubectl --context "$context" -n "$namespace" delete job easyai-local-migrate \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: easyai-local-migrate
|
|
namespace: easyai
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: migrate
|
|
image: $api_image
|
|
command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
EOF
|
|
kubectl --context "$context" -n "$namespace" wait \
|
|
--for=condition=complete job/easyai-local-migrate --timeout=10m >/dev/null
|
|
}
|
|
|
|
mark_local_database() {
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
local primary
|
|
primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
|
|
-o 'jsonpath={.status.currentPrimary}')
|
|
[[ $LOCAL_CLUSTER_ID =~ ^easyai-local-[0-9a-f]{24}$ ]]
|
|
printf '%s\n' \
|
|
"INSERT INTO system_settings(setting_key,value)
|
|
VALUES('acceptance_local_cluster_id',jsonb_build_object('clusterId', :'cluster_id'))
|
|
ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" |
|
|
kubectl --context "$context" -n "$namespace" exec -i "$primary" -c postgres -- \
|
|
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \
|
|
-v cluster_id="$LOCAL_CLUSTER_ID" >/dev/null
|
|
}
|
|
|
|
import_snapshot() {
|
|
local api_image=$1 snapshot=$2
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-snapshot \
|
|
--from-file=snapshot.json="$snapshot" --dry-run=client -o yaml |
|
|
kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" delete job easyai-local-snapshot-import \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: easyai-local-snapshot-import
|
|
namespace: easyai
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 10001
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
initContainers:
|
|
- name: stage-snapshot
|
|
image: $api_image
|
|
command:
|
|
- /bin/sh
|
|
- -ec
|
|
- umask 077; cp /snapshot-source/snapshot.json /snapshot/snapshot.json
|
|
volumeMounts:
|
|
- name: snapshot-source
|
|
mountPath: /snapshot-source
|
|
readOnly: true
|
|
- name: snapshot
|
|
mountPath: /snapshot
|
|
containers:
|
|
- name: import
|
|
image: $api_image
|
|
command:
|
|
- /app/easyai-ai-gateway-acceptance-snapshot
|
|
- import
|
|
- --input
|
|
- /snapshot/snapshot.json
|
|
- --local-cluster-id
|
|
- $LOCAL_CLUSTER_ID
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
volumeMounts:
|
|
- name: snapshot
|
|
mountPath: /snapshot
|
|
readOnly: true
|
|
volumes:
|
|
- name: snapshot-source
|
|
configMap:
|
|
name: easyai-acceptance-snapshot
|
|
- name: snapshot
|
|
emptyDir: {}
|
|
EOF
|
|
kubectl --context "$context" -n "$namespace" wait \
|
|
--for=condition=complete job/easyai-local-snapshot-import --timeout=5m >/dev/null
|
|
}
|
|
|
|
replay_acceptance_import_migrations() {
|
|
local api_image=$1
|
|
kubectl --context "$context" -n "$namespace" delete job easyai-local-migration-replay \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: easyai-local-migration-replay
|
|
namespace: easyai
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: migrate
|
|
image: $api_image
|
|
command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
|
|
env:
|
|
- name: AI_GATEWAY_MIGRATION_REAPPLY_ACCEPTANCE_IMPORT
|
|
value: "true"
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
EOF
|
|
kubectl --context "$context" -n "$namespace" wait \
|
|
--for=condition=complete job/easyai-local-migration-replay --timeout=5m >/dev/null
|
|
}
|
|
|
|
render_and_apply_application() {
|
|
local api_image=$1 web_image=$2 rendered=$state_root/application.rendered.yaml
|
|
sed \
|
|
-e "s|image: easyai-api|image: $api_image|g" \
|
|
-e "s|image: easyai-web|image: $web_image|g" \
|
|
-e 's/nodePort: 31088/nodePort: 32088/g' \
|
|
-e 's/nodePort: 31089/nodePort: 32089/g' \
|
|
"$repository_root/deploy/kubernetes/production/application.yaml" >"$rendered"
|
|
chmod 0600 "$rendered"
|
|
kubectl --context "$context" -n "$namespace" apply \
|
|
-f "$repository_root/deploy/kubernetes/production/service-account-rbac.yaml" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/local-config.yaml" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" apply -f "$rendered" >/dev/null
|
|
|
|
for workload in easyai-api-ningbo easyai-worker-ningbo; do
|
|
kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
|
|
--from=secret/easyai-database-ningbo >/dev/null
|
|
done
|
|
for workload in easyai-api-hongkong easyai-worker-hongkong; do
|
|
kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
|
|
--from=secret/easyai-database-hongkong >/dev/null
|
|
done
|
|
for workload in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do
|
|
kubectl --context "$context" -n "$namespace" patch deployment "$workload" --type=json \
|
|
-p='[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"application-data","hostPath":{"path":"/var/lib/easyai-acceptance/media","type":"DirectoryOrCreate"}}}]' \
|
|
>/dev/null
|
|
local container=worker ca_patch
|
|
[[ $workload == easyai-api-* ]] && container=api
|
|
ca_patch=$(jq -cn --arg container "$container" '{
|
|
spec:{template:{spec:{
|
|
containers:[{
|
|
name:$container,
|
|
env:[{name:"SSL_CERT_FILE",value:"/etc/easyai-acceptance-ca/ca.crt"}],
|
|
volumeMounts:[{name:"acceptance-ca",mountPath:"/etc/easyai-acceptance-ca",readOnly:true}]
|
|
}],
|
|
volumes:[{name:"acceptance-ca",configMap:{name:"easyai-acceptance-ca",defaultMode:292}}]
|
|
}}}
|
|
}')
|
|
kubectl --context "$context" -n "$namespace" patch deployment "$workload" \
|
|
--type=strategic -p="$ca_patch" >/dev/null
|
|
done
|
|
kubectl --context "$context" -n "$namespace" scale \
|
|
deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null
|
|
kubectl --context "$context" -n "$namespace" set resources deployment/easyai-api-ningbo \
|
|
deployment/easyai-api-hongkong --containers=api \
|
|
--requests=cpu=250m,memory=512Mi --limits=cpu=750m,memory=2Gi >/dev/null
|
|
kubectl --context "$context" -n "$namespace" set resources deployment/easyai-worker-ningbo \
|
|
deployment/easyai-worker-hongkong --containers=worker \
|
|
--requests=cpu=750m,memory=1536Mi --limits=cpu=1500m,memory=2Gi >/dev/null
|
|
kubectl --context "$context" -n "$namespace" set env deployment/easyai-worker-ningbo \
|
|
deployment/easyai-worker-hongkong \
|
|
AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
|
|
}
|
|
|
|
bootstrap_runtime() {
|
|
local api_image=$1 api_digest=$2 snapshot=$3
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
local snapshot_hash release_sha runtime_file
|
|
snapshot_hash=$(jq -r '.snapshotSha256' "$snapshot")
|
|
release_sha=$(git -C "$repository_root" rev-parse HEAD)
|
|
runtime_file="$state_root/runtime-$snapshot_hash-$(date -u '+%Y%m%dT%H%M%SZ')-$$.json"
|
|
kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: easyai-local-bootstrap
|
|
namespace: easyai
|
|
labels:
|
|
easyai.io/environment: local-acceptance
|
|
spec:
|
|
activeDeadlineSeconds: 600
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 10001
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
containers:
|
|
- name: bootstrap
|
|
image: $api_image
|
|
command:
|
|
- /app/easyai-ai-gateway-acceptance-bootstrap
|
|
- --local-cluster-id
|
|
- $LOCAL_CLUSTER_ID
|
|
- --release-sha
|
|
- $release_sha
|
|
- --api-image-digest
|
|
- $api_digest
|
|
- --worker-image-digest
|
|
- $api_digest
|
|
- --emulator-base-url
|
|
- http://easyai-acceptance-upstream-proxy.easyai.svc.cluster.local:8090
|
|
- --callback-url
|
|
- http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks
|
|
- --identity-shards
|
|
- "32"
|
|
- --output
|
|
- /runtime/runtime.json
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
volumeMounts:
|
|
- name: runtime
|
|
mountPath: /runtime
|
|
- name: runtime-export
|
|
image: $api_image
|
|
command: ["/bin/sh", "-ec", "sleep 600"]
|
|
volumeMounts:
|
|
- name: runtime
|
|
mountPath: /runtime
|
|
readOnly: true
|
|
volumes:
|
|
- name: runtime
|
|
emptyDir: {}
|
|
EOF
|
|
local bootstrap_exit='' deadline=$((SECONDS + 300))
|
|
while ((SECONDS < deadline)); do
|
|
bootstrap_exit=$(kubectl --context "$context" -n "$namespace" get \
|
|
pod/easyai-local-bootstrap -o json |
|
|
jq -r '.status.containerStatuses[]? | select(.name == "bootstrap") |
|
|
if .state.terminated then (.state.terminated.exitCode | tostring) else empty end')
|
|
[[ -z $bootstrap_exit ]] || break
|
|
sleep 1
|
|
done
|
|
if [[ $bootstrap_exit != 0 ]]; then
|
|
echo "local acceptance bootstrap failed or timed out: exit_code=${bootstrap_exit:-unknown}" >&2
|
|
kubectl --context "$context" -n "$namespace" logs \
|
|
pod/easyai-local-bootstrap -c bootstrap >&2 || true
|
|
exit 1
|
|
fi
|
|
umask 077
|
|
kubectl --context "$context" -n "$namespace" cp \
|
|
-c runtime-export easyai-local-bootstrap:/runtime/runtime.json "$runtime_file" >/dev/null
|
|
chmod 0600 "$runtime_file"
|
|
kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
|
|
--wait=true >/dev/null
|
|
printf '%s\n' "$runtime_file" >"$state_root/current-runtime-path"
|
|
chmod 0600 "$state_root/current-runtime-path"
|
|
}
|
|
|
|
up_cluster() {
|
|
[[ ${1:-} == --snapshot && $# -eq 2 ]] || {
|
|
usage >&2
|
|
exit 64
|
|
}
|
|
local snapshot_input=$2 snapshot_dir snapshot
|
|
[[ -f $snapshot_input && ! -L $snapshot_input ]] || {
|
|
echo 'snapshot must be a regular non-symlink file' >&2
|
|
exit 1
|
|
}
|
|
snapshot_dir=$(cd "$(dirname "$snapshot_input")" && pwd -P)
|
|
snapshot="$snapshot_dir/$(basename "$snapshot_input")"
|
|
[[ -z $(git -C "$repository_root" status --short) ]] || {
|
|
echo 'local acceptance requires a clean source tree so report and image source are reproducible' >&2
|
|
exit 1
|
|
}
|
|
preflight
|
|
ensure_private_material
|
|
generate_tls
|
|
(
|
|
cd "$repository_root/apps/api"
|
|
go run ./cmd/acceptance-snapshot validate --input "$snapshot"
|
|
)
|
|
if [[ ! -e $state_root/snapshot.json || ! $snapshot -ef $state_root/snapshot.json ]]; then
|
|
cp "$snapshot" "$state_root/snapshot.json"
|
|
fi
|
|
chmod 0600 "$state_root/snapshot.json"
|
|
local k3d config native_arch release_sha api_image web_image netem_image api_digest
|
|
k3d=$(k3d_binary)
|
|
if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then
|
|
echo "cluster $cluster_name already exists; use status or explicit down --confirm" >&2
|
|
exit 1
|
|
fi
|
|
native_arch=$(docker info --format '{{.Architecture}}')
|
|
case $native_arch in
|
|
aarch64|arm64) native_arch=arm64 ;;
|
|
x86_64) native_arch=amd64 ;;
|
|
esac
|
|
release_sha=$(git -C "$repository_root" rev-parse HEAD)
|
|
api_image="easyai-acceptance-api:$release_sha"
|
|
web_image="easyai-acceptance-web:$release_sha"
|
|
netem_image="easyai-acceptance-netem:$release_sha"
|
|
pull_dependency_images
|
|
verify_docker_container_start "${K3S_IMAGE%%@*}"
|
|
docker buildx build --load --platform "linux/$native_arch" --target api \
|
|
-t "$api_image" "$repository_root"
|
|
docker buildx build --load --platform "linux/$native_arch" --target web \
|
|
-t "$web_image" "$repository_root"
|
|
docker buildx build --load --platform "linux/$native_arch" --target acceptance-netem \
|
|
-t "$netem_image" "$repository_root"
|
|
api_digest=$(docker image inspect "$api_image" --format '{{.Id}}')
|
|
[[ $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
|
|
verify_docker_container_start "$netem_image"
|
|
|
|
reconcile_cluster_network
|
|
create_cluster_volumes
|
|
config="$state_root/k3d.rendered.yaml"
|
|
render_k3d_config "$config"
|
|
EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config"
|
|
kubectl --context "$context" apply -f "$manifest_root/cluster-dns.yaml" >/dev/null
|
|
docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null
|
|
docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null
|
|
docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null
|
|
verify_local_node_capacity "$context" || {
|
|
echo 'local K3s node allocatable capacity does not match 4/8, 4/8, 2/4 resource envelope' >&2
|
|
exit 1
|
|
}
|
|
"$k3d" image import -c "$cluster_name" \
|
|
"$api_image" "$web_image" "$netem_image" \
|
|
"${CNPG_CONTROLLER_IMAGE%%@*}" "${CNPG_POSTGRES_IMAGE%%@*}" \
|
|
"${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"
|
|
|
|
local cnpg_manifest="$state_root/cnpg.yaml"
|
|
curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest"
|
|
[[ $(shasum -a 256 "$cnpg_manifest" | awk '{print $1}') == "$CNPG_MANIFEST_SHA256" ]] || {
|
|
echo 'CNPG manifest checksum mismatch' >&2
|
|
exit 1
|
|
}
|
|
chmod 0600 "$cnpg_manifest"
|
|
kubectl --context "$context" apply --server-side --force-conflicts -f "$cnpg_manifest" >/dev/null
|
|
kubectl --context "$context" -n cnpg-system rollout status \
|
|
deployment/cnpg-controller-manager --timeout=10m
|
|
apply_runtime_secrets
|
|
sed "s|imageName: .*|imageName: ${CNPG_POSTGRES_IMAGE%%@*}|" "$manifest_root/database.yaml" |
|
|
kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" wait --for=condition=Ready \
|
|
cluster/easyai-postgres --timeout=15m >/dev/null
|
|
run_migrations "$api_image"
|
|
mark_local_database
|
|
import_snapshot "$api_image" "$snapshot"
|
|
replay_acceptance_import_migrations "$api_image"
|
|
|
|
sed "s|image: easyai-api|image: $api_image|g; s|image: easyai-acceptance-netem|image: $netem_image|g" \
|
|
"$manifest_root/support-services.yaml" |
|
|
kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
|
|
wait_rollout deployment/easyai-acceptance-emulator
|
|
wait_rollout deployment/easyai-acceptance-callback-collector
|
|
wait_rollout deployment/easyai-acceptance-upstream-proxy
|
|
wait_rollout deployment/easyai-postgres-proxy-ningbo
|
|
wait_rollout deployment/easyai-postgres-proxy-hongkong
|
|
render_and_apply_application "$api_image" "$web_image"
|
|
sed "s|image: easyai-web|image: $web_image|g" "$manifest_root/tls-edges.yaml" |
|
|
kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
|
|
wait_rollout deployment/easyai-api-ningbo
|
|
wait_rollout deployment/easyai-api-hongkong
|
|
wait_rollout deployment/easyai-worker-ningbo
|
|
wait_rollout deployment/easyai-worker-hongkong
|
|
wait_rollout deployment/easyai-capacity-controller
|
|
wait_rollout deployment/easyai-acceptance-edge-ningbo
|
|
wait_rollout deployment/easyai-acceptance-edge-hongkong
|
|
bootstrap_runtime "$api_image" "$api_digest" "$snapshot"
|
|
wait_internal_gateway
|
|
"$script_dir/network-fault.sh" baseline
|
|
sleep 1
|
|
record_local_control_plane_identity "$context" "$cluster_name" "$identity_file"
|
|
echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt"
|
|
}
|
|
|
|
new_run() {
|
|
load_lock
|
|
preflight
|
|
[[ -z $(git -C "$repository_root" status --short) ]] || {
|
|
echo 'local acceptance requires a clean source tree for a fresh Run ID' >&2
|
|
exit 1
|
|
}
|
|
if ! private_file "$state_root/current-runtime-path" || ! private_file "$state_root/snapshot.json"; then
|
|
echo 'local acceptance runtime or snapshot is unavailable' >&2
|
|
exit 1
|
|
fi
|
|
verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
|
|
echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild the cluster' >&2
|
|
exit 1
|
|
}
|
|
verify_local_node_capacity "$context" || {
|
|
echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
|
|
exit 1
|
|
}
|
|
verify_local_apiserver_ready "$context" || {
|
|
echo 'gate=local_control_plane_unavailable local Kubernetes API is not ready' >&2
|
|
exit 1
|
|
}
|
|
local previous_runtime api_image api_digest previous_release current_release
|
|
previous_runtime=$(<"$state_root/current-runtime-path")
|
|
private_file "$previous_runtime" || {
|
|
echo 'local acceptance current runtime is unavailable' >&2
|
|
exit 1
|
|
}
|
|
previous_release=$(jq -r '.releaseSha' "$previous_runtime")
|
|
current_release=$(git -C "$repository_root" rev-parse HEAD)
|
|
[[ $previous_release == "$current_release" ]] || {
|
|
echo 'local acceptance source changed after cluster creation; rebuild the cluster' >&2
|
|
exit 1
|
|
}
|
|
api_digest=$(jq -r '.apiImageDigest' "$previous_runtime")
|
|
api_image=$(kubectl --context "$context" -n "$namespace" get deployment easyai-api-ningbo \
|
|
-o 'jsonpath={.spec.template.spec.containers[?(@.name=="api")].image}')
|
|
[[ -n $api_image && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
|
|
bootstrap_runtime "$api_image" "$api_digest" "$state_root/snapshot.json"
|
|
verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
|
|
echo 'gate=local_control_plane_restarted local K3s server changed while creating a fresh run' >&2
|
|
exit 1
|
|
}
|
|
echo "local_acceptance_new_run=PASS runtime_path=$state_root/current-runtime-path"
|
|
}
|
|
|
|
status_cluster() {
|
|
local k3d
|
|
k3d=$(k3d_binary) || {
|
|
echo 'k3d is unavailable' >&2
|
|
exit 1
|
|
}
|
|
"$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name" || {
|
|
echo "cluster $cluster_name does not exist" >&2
|
|
exit 1
|
|
}
|
|
kubectl --context "$context" get nodes -o wide
|
|
kubectl --context "$context" -n "$namespace" get cluster,pod,deployment
|
|
verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || {
|
|
echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild required' >&2
|
|
exit 1
|
|
}
|
|
verify_local_node_capacity "$context" || {
|
|
echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2
|
|
exit 1
|
|
}
|
|
echo "local_acceptance_status=PASS context=$context"
|
|
}
|
|
|
|
down_cluster() {
|
|
[[ ${1:-} == --confirm && $# -eq 1 ]] || {
|
|
echo 'down requires the literal --confirm flag' >&2
|
|
exit 64
|
|
}
|
|
local k3d
|
|
k3d=$(k3d_binary)
|
|
if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then
|
|
"$k3d" cluster delete "$cluster_name"
|
|
fi
|
|
remove_cluster_volumes
|
|
remove_cluster_network
|
|
echo "local_acceptance_down=PASS preserved_private_root=$private_root"
|
|
}
|
|
|
|
case ${1:-} in
|
|
install-tools)
|
|
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
|
install_tools
|
|
;;
|
|
preflight)
|
|
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
|
preflight
|
|
;;
|
|
up)
|
|
shift
|
|
up_cluster "$@"
|
|
;;
|
|
new-run)
|
|
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
|
new_run
|
|
;;
|
|
status)
|
|
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
|
status_cluster
|
|
;;
|
|
down)
|
|
shift
|
|
down_cluster "$@"
|
|
;;
|
|
*)
|
|
usage >&2
|
|
exit 64
|
|
;;
|
|
esac
|