wangbo
f190af00be
本地同构验收在全量迁移后导入生产快照，导致本次发布的数据迁移被旧能力覆盖；增加仅允许严格本地集群标记启用的导入后迁移重放，并新增幂等 Seedance 约束校准。\n\n批量异步派发改为在事务开始按全局顺序预锁全部任务与容量作用域，同时对 PostgreSQL 死锁和序列化失败做退避重试，避免双 API 重叠批次形成环形等待。\n\n验证：Go 全量测试、gofmt、bash -n、ShellCheck、迁移安全检查。
675 lines
25 KiB
Bash
Executable File
675 lines
25 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
repository_root=$(cd "$script_dir/../.." && pwd)
|
|
manifest_root="$repository_root/deploy/kubernetes/local-acceptance"
|
|
lock_file="$manifest_root/dependencies.lock"
|
|
private_root="$repository_root/.local-secrets/acceptance"
|
|
tools_root="$private_root/tools"
|
|
state_root="$private_root/state"
|
|
media_root="$private_root/media"
|
|
cluster_name=easyai-acceptance-local
|
|
context="k3d-$cluster_name"
|
|
namespace=easyai
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
scripts/acceptance/local-cluster.sh install-tools
|
|
scripts/acceptance/local-cluster.sh preflight
|
|
scripts/acceptance/local-cluster.sh up --snapshot <acceptance-snapshot-v1.json>
|
|
scripts/acceptance/local-cluster.sh status
|
|
scripts/acceptance/local-cluster.sh down --confirm
|
|
|
|
The full cluster requires Docker Desktop memory >= 24 GiB. Failed `up` runs are
|
|
kept intact for diagnosis. `down` is the only destructive lifecycle action.
|
|
EOF
|
|
}
|
|
|
|
load_lock() {
|
|
[[ -f $lock_file && ! -L $lock_file ]] || {
|
|
echo "missing dependency lock: $lock_file" >&2
|
|
exit 1
|
|
}
|
|
# shellcheck source=/dev/null
|
|
source "$lock_file"
|
|
: "${K3D_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
|
|
}
|
|
|
|
require_command() {
|
|
command -v "$1" >/dev/null 2>&1 || {
|
|
echo "$1 is required" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
private_file() {
|
|
local path=$1
|
|
[[ -f $path && ! -L $path ]] || return 1
|
|
local mode
|
|
if [[ $(uname -s) == Darwin ]]; then
|
|
mode=$(stat -f '%Lp' "$path")
|
|
else
|
|
mode=$(stat -c '%a' "$path")
|
|
fi
|
|
[[ $mode == 600 ]]
|
|
}
|
|
|
|
install_tools() {
|
|
load_lock
|
|
require_command curl
|
|
require_command shasum
|
|
install -d -m 0700 "$tools_root"
|
|
local os arch checksum url temporary
|
|
os=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
arch=$(uname -m)
|
|
[[ $os == darwin ]] || {
|
|
echo 'the pinned local acceptance installer currently supports macOS only' >&2
|
|
exit 1
|
|
}
|
|
case $arch in
|
|
arm64)
|
|
arch=arm64
|
|
checksum=$K3D_DARWIN_ARM64_SHA256
|
|
;;
|
|
x86_64)
|
|
arch=amd64
|
|
checksum=$K3D_DARWIN_AMD64_SHA256
|
|
;;
|
|
*)
|
|
echo "unsupported host architecture: $arch" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
url="https://github.com/k3d-io/k3d/releases/download/$K3D_VERSION/k3d-darwin-$arch"
|
|
temporary="$tools_root/k3d.download"
|
|
curl -fsSL "$url" -o "$temporary"
|
|
[[ $(shasum -a 256 "$temporary" | awk '{print $1}') == "$checksum" ]] || {
|
|
echo 'k3d checksum mismatch' >&2
|
|
exit 1
|
|
}
|
|
chmod 0555 "$temporary"
|
|
mv "$temporary" "$tools_root/k3d"
|
|
echo "local_acceptance_tools=PASS k3d=$K3D_VERSION path=$tools_root/k3d"
|
|
}
|
|
|
|
k3d_binary() {
|
|
if [[ -x $tools_root/k3d ]]; then
|
|
printf '%s\n' "$tools_root/k3d"
|
|
return
|
|
fi
|
|
command -v k3d
|
|
}
|
|
|
|
docker_memory_bytes() {
|
|
docker info --format '{{.MemTotal}}'
|
|
}
|
|
|
|
preflight() {
|
|
load_lock
|
|
require_command docker
|
|
require_command kubectl
|
|
require_command jq
|
|
require_command openssl
|
|
require_command sed
|
|
require_command shasum
|
|
require_command go
|
|
require_command node
|
|
require_command curl
|
|
local k3d memory_bytes architecture
|
|
k3d=$(k3d_binary) || {
|
|
echo 'k3d is missing; run install-tools' >&2
|
|
exit 1
|
|
}
|
|
"$k3d" version | grep -Fq "$K3D_VERSION" || {
|
|
echo "k3d must be pinned to $K3D_VERSION" >&2
|
|
exit 1
|
|
}
|
|
docker info >/dev/null
|
|
memory_bytes=$(docker_memory_bytes)
|
|
[[ $memory_bytes =~ ^[0-9]+$ && $memory_bytes -ge 25769803776 ]] || {
|
|
memory_gib=$(awk -v bytes="${memory_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
|
|
echo "Docker Desktop memory is ${memory_gib} GiB; local acceptance requires at least 24 GiB" >&2
|
|
exit 1
|
|
}
|
|
architecture=$(docker info --format '{{.Architecture}}')
|
|
[[ $architecture == aarch64 || $architecture == arm64 || $architecture == x86_64 ]] || {
|
|
echo "unsupported Docker architecture: $architecture" >&2
|
|
exit 1
|
|
}
|
|
docker buildx version >/dev/null
|
|
echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes architecture=$architecture k3d=$K3D_VERSION"
|
|
}
|
|
|
|
ensure_private_material() {
|
|
install -d -m 0700 "$private_root" "$state_root" "$media_root"
|
|
local env_file="$state_root/local.env"
|
|
if [[ ! -e $env_file ]]; then
|
|
umask 077
|
|
{
|
|
printf 'LOCAL_CLUSTER_ID=%s\n' "easyai-local-$(openssl rand -hex 12)"
|
|
printf 'POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 32)"
|
|
printf 'CONFIG_JWT_SECRET=%s\n' "$(openssl rand -hex 32)"
|
|
printf 'SERVER_MAIN_INTERNAL_TOKEN=%s\n' "$(openssl rand -hex 32)"
|
|
printf 'SERVER_MAIN_INTERNAL_KEY=local-acceptance\n'
|
|
printf 'SERVER_MAIN_INTERNAL_SECRET=%s\n' "$(openssl rand -hex 32)"
|
|
} >"$env_file"
|
|
chmod 0600 "$env_file"
|
|
fi
|
|
private_file "$env_file" || {
|
|
echo "local acceptance state must be a 0600 regular file: $env_file" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
generate_tls() {
|
|
local ca_key="$state_root/ca.key" ca_crt="$state_root/ca.crt"
|
|
local tls_key="$state_root/tls.key" tls_csr="$state_root/tls.csr"
|
|
local tls_crt="$state_root/tls.crt" extensions="$state_root/tls.ext"
|
|
if [[ ! -e $ca_key ]]; then
|
|
umask 077
|
|
openssl genrsa -out "$ca_key" 3072 >/dev/null 2>&1
|
|
openssl req -x509 -new -key "$ca_key" -sha256 -days 30 \
|
|
-subj '/CN=EasyAI local acceptance CA' -out "$ca_crt" >/dev/null 2>&1
|
|
openssl genrsa -out "$tls_key" 2048 >/dev/null 2>&1
|
|
openssl req -new -key "$tls_key" -subj '/CN=gateway.easyai.local' \
|
|
-out "$tls_csr" >/dev/null 2>&1
|
|
printf 'subjectAltName=DNS:gateway.easyai.local\nextendedKeyUsage=serverAuth\n' >"$extensions"
|
|
openssl x509 -req -in "$tls_csr" -CA "$ca_crt" -CAkey "$ca_key" \
|
|
-CAcreateserial -out "$tls_crt" -days 30 -sha256 -extfile "$extensions" \
|
|
>/dev/null 2>&1
|
|
chmod 0600 "$ca_key" "$ca_crt" "$tls_key" "$tls_csr" "$tls_crt" "$extensions"
|
|
fi
|
|
private_file "$ca_crt" && private_file "$tls_key" && private_file "$tls_crt"
|
|
}
|
|
|
|
render_k3d_config() {
|
|
local output=$1
|
|
[[ $media_root != *'|'* ]]
|
|
sed "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" "$manifest_root/k3d.yaml" >"$output"
|
|
}
|
|
|
|
wait_rollout() {
|
|
kubectl --context "$context" -n "$namespace" rollout status "$1" --timeout="${2:-10m}"
|
|
}
|
|
|
|
wait_internal_gateway() {
|
|
local pod attempt=0
|
|
pod=$(kubectl --context "$context" -n "$namespace" get pod \
|
|
-l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \
|
|
-o 'jsonpath={.items[0].metadata.name}')
|
|
until kubectl --context "$context" -n "$namespace" exec "$pod" -c api -- \
|
|
wget -q --spider https://gateway.easyai.local/api/v1/healthz \
|
|
>/dev/null 2>&1; do
|
|
attempt=$((attempt + 1))
|
|
if ((attempt >= 60)); then
|
|
echo 'local acceptance gateway DNS or TLS trust did not become ready' >&2
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
apply_runtime_secrets() {
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
local database_ningbo database_hongkong database_direct
|
|
database_ningbo="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-ningbo.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
|
|
database_hongkong="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-proxy-hongkong.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
|
|
database_direct="postgresql://easyai:${POSTGRES_PASSWORD}@easyai-postgres-rw.easyai.svc.cluster.local:5432/easyai_ai_gateway?sslmode=require"
|
|
|
|
kubectl --context "$context" create namespace "$namespace" --dry-run=client -o yaml |
|
|
kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-postgres-app \
|
|
--type=kubernetes.io/basic-auth \
|
|
--from-literal=username=easyai --from-literal=password="$POSTGRES_PASSWORD" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-ai-gateway-runtime \
|
|
--from-literal=AI_GATEWAY_DATABASE_URL="$database_direct" \
|
|
--from-literal=CONFIG_JWT_SECRET="$CONFIG_JWT_SECRET" \
|
|
--from-literal=SERVER_MAIN_INTERNAL_TOKEN="$SERVER_MAIN_INTERNAL_TOKEN" \
|
|
--from-literal=SERVER_MAIN_INTERNAL_KEY="$SERVER_MAIN_INTERNAL_KEY" \
|
|
--from-literal=SERVER_MAIN_INTERNAL_SECRET="$SERVER_MAIN_INTERNAL_SECRET" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-database-ningbo \
|
|
--from-literal=AI_GATEWAY_DATABASE_URL="$database_ningbo" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-database-hongkong \
|
|
--from-literal=AI_GATEWAY_DATABASE_URL="$database_hongkong" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret generic easyai-oss-backup \
|
|
--from-literal=ACCESS_KEY_ID=local-disabled \
|
|
--from-literal=ACCESS_SECRET_KEY=local-disabled \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create secret tls easyai-acceptance-tls \
|
|
--cert="$state_root/tls.crt" --key="$state_root/tls.key" \
|
|
--dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-ca \
|
|
--from-file=ca.crt="$state_root/ca.crt" --dry-run=client -o yaml |
|
|
kubectl --context "$context" apply -f - >/dev/null
|
|
}
|
|
|
|
run_migrations() {
|
|
local api_image=$1
|
|
kubectl --context "$context" -n "$namespace" delete job easyai-local-migrate \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: easyai-local-migrate
|
|
namespace: easyai
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: migrate
|
|
image: $api_image
|
|
command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
EOF
|
|
kubectl --context "$context" -n "$namespace" wait \
|
|
--for=condition=complete job/easyai-local-migrate --timeout=10m >/dev/null
|
|
}
|
|
|
|
mark_local_database() {
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
local primary
|
|
primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
|
|
-o 'jsonpath={.status.currentPrimary}')
|
|
[[ $LOCAL_CLUSTER_ID =~ ^easyai-local-[0-9a-f]{24}$ ]]
|
|
printf '%s\n' \
|
|
"INSERT INTO system_settings(setting_key,value)
|
|
VALUES('acceptance_local_cluster_id',jsonb_build_object('clusterId', :'cluster_id'))
|
|
ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" |
|
|
kubectl --context "$context" -n "$namespace" exec -i "$primary" -c postgres -- \
|
|
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway \
|
|
-v cluster_id="$LOCAL_CLUSTER_ID" >/dev/null
|
|
}
|
|
|
|
import_snapshot() {
|
|
local api_image=$1 snapshot=$2
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-snapshot \
|
|
--from-file=snapshot.json="$snapshot" --dry-run=client -o yaml |
|
|
kubectl --context "$context" apply -f - >/dev/null
|
|
kubectl --context "$context" -n "$namespace" delete job easyai-local-snapshot-import \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: easyai-local-snapshot-import
|
|
namespace: easyai
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 10001
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
initContainers:
|
|
- name: stage-snapshot
|
|
image: $api_image
|
|
command:
|
|
- /bin/sh
|
|
- -ec
|
|
- umask 077; cp /snapshot-source/snapshot.json /snapshot/snapshot.json
|
|
volumeMounts:
|
|
- name: snapshot-source
|
|
mountPath: /snapshot-source
|
|
readOnly: true
|
|
- name: snapshot
|
|
mountPath: /snapshot
|
|
containers:
|
|
- name: import
|
|
image: $api_image
|
|
command:
|
|
- /app/easyai-ai-gateway-acceptance-snapshot
|
|
- import
|
|
- --input
|
|
- /snapshot/snapshot.json
|
|
- --local-cluster-id
|
|
- $LOCAL_CLUSTER_ID
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
volumeMounts:
|
|
- name: snapshot
|
|
mountPath: /snapshot
|
|
readOnly: true
|
|
volumes:
|
|
- name: snapshot-source
|
|
configMap:
|
|
name: easyai-acceptance-snapshot
|
|
- name: snapshot
|
|
emptyDir: {}
|
|
EOF
|
|
kubectl --context "$context" -n "$namespace" wait \
|
|
--for=condition=complete job/easyai-local-snapshot-import --timeout=5m >/dev/null
|
|
}
|
|
|
|
replay_acceptance_import_migrations() {
|
|
local api_image=$1
|
|
kubectl --context "$context" -n "$namespace" delete job easyai-local-migration-replay \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: easyai-local-migration-replay
|
|
namespace: easyai
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: migrate
|
|
image: $api_image
|
|
command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
|
|
env:
|
|
- name: AI_GATEWAY_MIGRATION_REAPPLY_ACCEPTANCE_IMPORT
|
|
value: "true"
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
EOF
|
|
kubectl --context "$context" -n "$namespace" wait \
|
|
--for=condition=complete job/easyai-local-migration-replay --timeout=5m >/dev/null
|
|
}
|
|
|
|
render_and_apply_application() {
|
|
local api_image=$1 web_image=$2 rendered=$state_root/application.rendered.yaml
|
|
sed \
|
|
-e "s|image: easyai-api|image: $api_image|g" \
|
|
-e "s|image: easyai-web|image: $web_image|g" \
|
|
-e 's/nodePort: 31088/nodePort: 32088/g' \
|
|
-e 's/nodePort: 31089/nodePort: 32089/g' \
|
|
"$repository_root/deploy/kubernetes/production/application.yaml" >"$rendered"
|
|
chmod 0600 "$rendered"
|
|
kubectl --context "$context" -n "$namespace" apply \
|
|
-f "$repository_root/deploy/kubernetes/production/service-account-rbac.yaml" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/local-config.yaml" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" apply -f "$rendered" >/dev/null
|
|
|
|
for workload in easyai-api-ningbo easyai-worker-ningbo; do
|
|
kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
|
|
--from=secret/easyai-database-ningbo >/dev/null
|
|
done
|
|
for workload in easyai-api-hongkong easyai-worker-hongkong; do
|
|
kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
|
|
--from=secret/easyai-database-hongkong >/dev/null
|
|
done
|
|
for workload in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do
|
|
kubectl --context "$context" -n "$namespace" patch deployment "$workload" --type=json \
|
|
-p='[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"application-data","hostPath":{"path":"/var/lib/easyai-acceptance/media","type":"DirectoryOrCreate"}}}]' \
|
|
>/dev/null
|
|
local container=worker ca_patch
|
|
[[ $workload == easyai-api-* ]] && container=api
|
|
ca_patch=$(jq -cn --arg container "$container" '{
|
|
spec:{template:{spec:{
|
|
containers:[{
|
|
name:$container,
|
|
env:[{name:"SSL_CERT_FILE",value:"/etc/easyai-acceptance-ca/ca.crt"}],
|
|
volumeMounts:[{name:"acceptance-ca",mountPath:"/etc/easyai-acceptance-ca",readOnly:true}]
|
|
}],
|
|
volumes:[{name:"acceptance-ca",configMap:{name:"easyai-acceptance-ca",defaultMode:292}}]
|
|
}}}
|
|
}')
|
|
kubectl --context "$context" -n "$namespace" patch deployment "$workload" \
|
|
--type=strategic -p="$ca_patch" >/dev/null
|
|
done
|
|
kubectl --context "$context" -n "$namespace" scale \
|
|
deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null
|
|
}
|
|
|
|
bootstrap_runtime() {
|
|
local api_image=$1 api_digest=$2 snapshot=$3
|
|
# shellcheck source=/dev/null
|
|
source "$state_root/local.env"
|
|
local snapshot_hash release_sha runtime_file
|
|
snapshot_hash=$(jq -r '.snapshotSha256' "$snapshot")
|
|
release_sha=$(git -C "$repository_root" rev-parse HEAD)
|
|
runtime_file="$state_root/runtime-$snapshot_hash.json"
|
|
kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: easyai-local-bootstrap
|
|
namespace: easyai
|
|
labels:
|
|
easyai.io/environment: local-acceptance
|
|
spec:
|
|
activeDeadlineSeconds: 600
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 10001
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
containers:
|
|
- name: bootstrap
|
|
image: $api_image
|
|
command:
|
|
- /app/easyai-ai-gateway-acceptance-bootstrap
|
|
- --local-cluster-id
|
|
- $LOCAL_CLUSTER_ID
|
|
- --release-sha
|
|
- $release_sha
|
|
- --api-image-digest
|
|
- $api_digest
|
|
- --worker-image-digest
|
|
- $api_digest
|
|
- --emulator-base-url
|
|
- http://easyai-acceptance-upstream-proxy.easyai.svc.cluster.local:8090
|
|
- --callback-url
|
|
- http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks
|
|
- --identity-shards
|
|
- "32"
|
|
- --output
|
|
- /runtime/runtime.json
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
volumeMounts:
|
|
- name: runtime
|
|
mountPath: /runtime
|
|
- name: runtime-export
|
|
image: $api_image
|
|
command: ["/bin/sh", "-ec", "sleep 600"]
|
|
volumeMounts:
|
|
- name: runtime
|
|
mountPath: /runtime
|
|
readOnly: true
|
|
volumes:
|
|
- name: runtime
|
|
emptyDir: {}
|
|
EOF
|
|
local bootstrap_exit='' deadline=$((SECONDS + 300))
|
|
while ((SECONDS < deadline)); do
|
|
bootstrap_exit=$(kubectl --context "$context" -n "$namespace" get \
|
|
pod/easyai-local-bootstrap -o json |
|
|
jq -r '.status.containerStatuses[]? | select(.name == "bootstrap") |
|
|
if .state.terminated then (.state.terminated.exitCode | tostring) else empty end')
|
|
[[ -z $bootstrap_exit ]] || break
|
|
sleep 1
|
|
done
|
|
if [[ $bootstrap_exit != 0 ]]; then
|
|
echo "local acceptance bootstrap failed or timed out: exit_code=${bootstrap_exit:-unknown}" >&2
|
|
kubectl --context "$context" -n "$namespace" logs \
|
|
pod/easyai-local-bootstrap -c bootstrap >&2 || true
|
|
exit 1
|
|
fi
|
|
umask 077
|
|
kubectl --context "$context" -n "$namespace" cp \
|
|
-c runtime-export easyai-local-bootstrap:/runtime/runtime.json "$runtime_file" >/dev/null
|
|
chmod 0600 "$runtime_file"
|
|
kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \
|
|
--wait=true >/dev/null
|
|
printf '%s\n' "$runtime_file" >"$state_root/current-runtime-path"
|
|
chmod 0600 "$state_root/current-runtime-path"
|
|
}
|
|
|
|
up_cluster() {
|
|
[[ ${1:-} == --snapshot && $# -eq 2 ]] || {
|
|
usage >&2
|
|
exit 64
|
|
}
|
|
local snapshot_input=$2 snapshot_dir snapshot
|
|
[[ -f $snapshot_input && ! -L $snapshot_input ]] || {
|
|
echo 'snapshot must be a regular non-symlink file' >&2
|
|
exit 1
|
|
}
|
|
snapshot_dir=$(cd "$(dirname "$snapshot_input")" && pwd -P)
|
|
snapshot="$snapshot_dir/$(basename "$snapshot_input")"
|
|
[[ -z $(git -C "$repository_root" status --short) ]] || {
|
|
echo 'local acceptance requires a clean source tree so report and image source are reproducible' >&2
|
|
exit 1
|
|
}
|
|
preflight
|
|
ensure_private_material
|
|
generate_tls
|
|
(
|
|
cd "$repository_root/apps/api"
|
|
go run ./cmd/acceptance-snapshot validate --input "$snapshot"
|
|
)
|
|
cp "$snapshot" "$state_root/snapshot.json"
|
|
chmod 0600 "$state_root/snapshot.json"
|
|
local k3d config native_arch release_sha api_image web_image netem_image api_digest
|
|
k3d=$(k3d_binary)
|
|
if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then
|
|
echo "cluster $cluster_name already exists; use status or explicit down --confirm" >&2
|
|
exit 1
|
|
fi
|
|
native_arch=$(docker info --format '{{.Architecture}}')
|
|
case $native_arch in
|
|
aarch64|arm64) native_arch=arm64 ;;
|
|
x86_64) native_arch=amd64 ;;
|
|
esac
|
|
release_sha=$(git -C "$repository_root" rev-parse HEAD)
|
|
api_image="easyai-acceptance-api:$release_sha"
|
|
web_image="easyai-acceptance-web:$release_sha"
|
|
netem_image="easyai-acceptance-netem:$release_sha"
|
|
docker buildx build --load --platform "linux/$native_arch" --target api \
|
|
-t "$api_image" "$repository_root"
|
|
docker buildx build --load --platform "linux/$native_arch" --target web \
|
|
-t "$web_image" "$repository_root"
|
|
docker buildx build --load --platform "linux/$native_arch" --target acceptance-netem \
|
|
-t "$netem_image" "$repository_root"
|
|
api_digest=$(docker image inspect "$api_image" --format '{{.Id}}')
|
|
[[ $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
|
|
|
|
config="$state_root/k3d.rendered.yaml"
|
|
render_k3d_config "$config"
|
|
EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config"
|
|
kubectl --context "$context" apply -f "$manifest_root/cluster-dns.yaml" >/dev/null
|
|
docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null
|
|
docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null
|
|
docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null
|
|
"$k3d" image import -c "$cluster_name" "$api_image" "$web_image" "$netem_image"
|
|
|
|
local cnpg_manifest="$state_root/cnpg.yaml"
|
|
curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest"
|
|
[[ $(shasum -a 256 "$cnpg_manifest" | awk '{print $1}') == "$CNPG_MANIFEST_SHA256" ]] || {
|
|
echo 'CNPG manifest checksum mismatch' >&2
|
|
exit 1
|
|
}
|
|
chmod 0600 "$cnpg_manifest"
|
|
kubectl --context "$context" apply --server-side --force-conflicts -f "$cnpg_manifest" >/dev/null
|
|
kubectl --context "$context" -n cnpg-system rollout status \
|
|
deployment/cnpg-controller-manager --timeout=10m
|
|
apply_runtime_secrets
|
|
kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/database.yaml" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" wait --for=condition=Ready \
|
|
cluster/easyai-postgres --timeout=15m >/dev/null
|
|
run_migrations "$api_image"
|
|
mark_local_database
|
|
import_snapshot "$api_image" "$snapshot"
|
|
replay_acceptance_import_migrations "$api_image"
|
|
|
|
sed "s|image: easyai-api|image: $api_image|g; s|image: easyai-acceptance-netem|image: $netem_image|g" \
|
|
"$manifest_root/support-services.yaml" |
|
|
kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
|
|
wait_rollout deployment/easyai-acceptance-emulator
|
|
wait_rollout deployment/easyai-acceptance-callback-collector
|
|
wait_rollout deployment/easyai-acceptance-upstream-proxy
|
|
wait_rollout deployment/easyai-postgres-proxy-ningbo
|
|
wait_rollout deployment/easyai-postgres-proxy-hongkong
|
|
render_and_apply_application "$api_image" "$web_image"
|
|
sed "s|image: easyai-web|image: $web_image|g" "$manifest_root/tls-edges.yaml" |
|
|
kubectl --context "$context" -n "$namespace" apply -f - >/dev/null
|
|
wait_rollout deployment/easyai-api-ningbo
|
|
wait_rollout deployment/easyai-api-hongkong
|
|
wait_rollout deployment/easyai-worker-ningbo
|
|
wait_rollout deployment/easyai-worker-hongkong
|
|
wait_rollout deployment/easyai-capacity-controller
|
|
wait_rollout deployment/easyai-acceptance-edge-ningbo
|
|
wait_rollout deployment/easyai-acceptance-edge-hongkong
|
|
bootstrap_runtime "$api_image" "$api_digest" "$snapshot"
|
|
wait_internal_gateway
|
|
"$script_dir/network-fault.sh" baseline
|
|
echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt"
|
|
}
|
|
|
|
status_cluster() {
|
|
local k3d
|
|
k3d=$(k3d_binary) || {
|
|
echo 'k3d is unavailable' >&2
|
|
exit 1
|
|
}
|
|
"$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name" || {
|
|
echo "cluster $cluster_name does not exist" >&2
|
|
exit 1
|
|
}
|
|
kubectl --context "$context" get nodes -o wide
|
|
kubectl --context "$context" -n "$namespace" get cluster,pod,deployment
|
|
echo "local_acceptance_status=PASS context=$context"
|
|
}
|
|
|
|
down_cluster() {
|
|
[[ ${1:-} == --confirm && $# -eq 1 ]] || {
|
|
echo 'down requires the literal --confirm flag' >&2
|
|
exit 64
|
|
}
|
|
local k3d
|
|
k3d=$(k3d_binary)
|
|
"$k3d" cluster delete "$cluster_name"
|
|
echo "local_acceptance_down=PASS preserved_private_root=$private_root"
|
|
}
|
|
|
|
case ${1:-} in
|
|
install-tools)
|
|
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
|
install_tools
|
|
;;
|
|
preflight)
|
|
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
|
preflight
|
|
;;
|
|
up)
|
|
shift
|
|
up_cluster "$@"
|
|
;;
|
|
status)
|
|
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
|
status_cluster
|
|
;;
|
|
down)
|
|
shift
|
|
down_cluster "$@"
|
|
;;
|
|
*)
|
|
usage >&2
|
|
exit 64
|
|
;;
|
|
esac
|