feat(deploy): 增加三节点 K3s 高可用迁移能力
新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。 新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。 所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。 验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。
This commit is contained in:
+279
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }
|
||||
|
||||
config_file=${AI_GATEWAY_CLUSTER_RELEASE_CONFIG:-/etc/easyai-ai-gateway-cluster-release.conf}
|
||||
manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
|
||||
[[ -f $config_file && ! -L $config_file ]] || { echo 'missing cluster release config' >&2; exit 1; }
|
||||
[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
|
||||
# shellcheck source=/dev/null
|
||||
source "$config_file"
|
||||
|
||||
: "${RELEASES_DIR:?}"
|
||||
: "${NAMESPACE:=easyai}"
|
||||
: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
|
||||
: "${K3S_BIN:=/usr/local/bin/k3s}"
|
||||
[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
|
||||
echo 'invalid cluster release configuration' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
|
||||
echo 'invalid public base URL' >&2
|
||||
exit 1
|
||||
}
|
||||
for command in "$K3S_BIN" curl flock install jq node; do
|
||||
command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
|
||||
done
|
||||
|
||||
kubectl=("$K3S_BIN" kubectl)
|
||||
install -d -m 0700 "$RELEASES_DIR"
|
||||
exec 9>"$RELEASES_DIR/.release.lock"
|
||||
flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; }
|
||||
|
||||
manifest_get() {
|
||||
node "$manifest_tool" get "$1" "$2"
|
||||
}
|
||||
|
||||
current_to_file() {
|
||||
local output=$1
|
||||
"${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
|
||||
-o jsonpath='{.data.manifest\.json}' >"$output"
|
||||
[[ -s $output ]]
|
||||
node "$manifest_tool" validate "$output" >/dev/null
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
local label=$1
|
||||
local url=$2
|
||||
local expected=${3:-}
|
||||
local body
|
||||
for _ in $(seq 1 60); do
|
||||
if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
|
||||
if [[ -z $expected || $body == *"$expected"* ]]; then
|
||||
echo "[cluster-release] verified $label"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "release probe failed: $label ($url)" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
verify_site() {
|
||||
local site=$1
|
||||
local address
|
||||
case $site in
|
||||
hongkong) address=10.77.0.2 ;;
|
||||
ningbo) address=10.77.0.1 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
wait_for_url "$site API health" "http://$address:30088/api/v1/healthz" easyai-ai-gateway
|
||||
wait_for_url "$site API readiness" "http://$address:30088/api/v1/readyz" '"ok":true'
|
||||
wait_for_url "$site Web" "http://$address:30178/" 'EasyAI AI Gateway'
|
||||
}
|
||||
|
||||
rollout_site() {
|
||||
local site=$1
|
||||
local api_image=$2
|
||||
local web_image=$3
|
||||
local api_changed=$4
|
||||
local web_changed=$5
|
||||
if [[ $api_changed == true ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
|
||||
"api=$api_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
|
||||
fi
|
||||
if [[ $web_changed == true ]]; then
|
||||
"${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
|
||||
"web=$web_image"
|
||||
"${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
|
||||
fi
|
||||
verify_site "$site"
|
||||
}
|
||||
|
||||
run_backup() {
|
||||
local source_sha=$1
|
||||
local backup=easyai-pre-release-${source_sha:0:12}
|
||||
cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Backup
|
||||
metadata:
|
||||
name: $backup
|
||||
namespace: $NAMESPACE
|
||||
spec:
|
||||
cluster:
|
||||
name: easyai-postgres
|
||||
method: plugin
|
||||
pluginConfiguration:
|
||||
name: barman-cloud.cloudnative-pg.io
|
||||
EOF
|
||||
for _ in $(seq 1 120); do
|
||||
phase=$("${kubectl[@]}" get backup "$backup" -n "$NAMESPACE" \
|
||||
-o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
[[ ${phase,,} == completed ]] && return 0
|
||||
[[ ${phase,,} == failed ]] && break
|
||||
sleep 5
|
||||
done
|
||||
echo "pre-migration backup failed: $backup phase=${phase:-unknown}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_migrator() {
|
||||
local source_sha=$1
|
||||
local api_image=$2
|
||||
local job=easyai-migrator-${source_sha:0:12}
|
||||
"${kubectl[@]}" delete job "$job" -n "$NAMESPACE" \
|
||||
--ignore-not-found=true --wait=true >/dev/null
|
||||
cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: $job
|
||||
namespace: $NAMESPACE
|
||||
labels:
|
||||
easyai.io/release: $source_sha
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
imagePullSecrets:
|
||||
- name: easyai-registry
|
||||
nodeSelector:
|
||||
easyai.io/site: ningbo
|
||||
containers:
|
||||
- name: migrator
|
||||
image: $api_image
|
||||
command: ["/app/easyai-ai-gateway-migrate"]
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: easyai-ai-gateway-runtime
|
||||
EOF
|
||||
"${kubectl[@]}" wait --for=condition=complete "job/$job" \
|
||||
-n "$NAMESPACE" --timeout=300s
|
||||
}
|
||||
|
||||
record_current() {
|
||||
local manifest=$1
|
||||
local action=$2
|
||||
local started_at=$3
|
||||
local source_sha recorded temporary
|
||||
source_sha=$(manifest_get "$manifest" sourceSha)
|
||||
recorded=$RELEASES_DIR/$source_sha.json
|
||||
temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX")
|
||||
rm -f -- "$temporary"
|
||||
RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
|
||||
RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
|
||||
RELEASE_DEPLOY_ACTION=$action \
|
||||
node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null
|
||||
install -m 0600 "$temporary" "$recorded"
|
||||
"${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
|
||||
--from-file=manifest.json="$recorded" --dry-run=client -o yaml |
|
||||
"${kubectl[@]}" apply -f - >/dev/null
|
||||
rm -f -- "$temporary"
|
||||
}
|
||||
|
||||
activate_manifest() {
|
||||
local manifest=$1
|
||||
local action=${2:-deploy}
|
||||
local source_sha base_sha api_image web_image migrations_changed components
|
||||
local api_changed=false web_changed=false current_file previous_api previous_web started_at
|
||||
[[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
|
||||
node "$manifest_tool" validate "$manifest" >/dev/null
|
||||
source_sha=$(manifest_get "$manifest" sourceSha)
|
||||
base_sha=$(manifest_get "$manifest" baseReleaseSha)
|
||||
api_image=$(manifest_get "$manifest" images.api)
|
||||
web_image=$(manifest_get "$manifest" images.web)
|
||||
migrations_changed=$(manifest_get "$manifest" migrationsChanged)
|
||||
components=$(manifest_get "$manifest" components)
|
||||
[[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || {
|
||||
echo 'release images must be digest-pinned' >&2
|
||||
return 1
|
||||
}
|
||||
[[ $components == *'"api"'* ]] && api_changed=true
|
||||
[[ $components == *'"web"'* ]] && web_changed=true
|
||||
if [[ $action == rollback ]]; then
|
||||
api_changed=true
|
||||
web_changed=true
|
||||
fi
|
||||
|
||||
current_file=$(mktemp "$RELEASES_DIR/.current.XXXXXX")
|
||||
if current_to_file "$current_file"; then
|
||||
current_sha=$(manifest_get "$current_file" sourceSha)
|
||||
previous_api=$(manifest_get "$current_file" images.api)
|
||||
previous_web=$(manifest_get "$current_file" images.web)
|
||||
if [[ $action == deploy && $current_sha != "$base_sha" ]]; then
|
||||
echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
|
||||
rm -f -- "$current_file"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
if [[ $action == deploy && -n $base_sha ]]; then
|
||||
echo 'cluster release status is unavailable for a non-bootstrap manifest' >&2
|
||||
rm -f -- "$current_file"
|
||||
return 1
|
||||
fi
|
||||
previous_api=
|
||||
previous_web=
|
||||
fi
|
||||
started_at=$(date +%s)
|
||||
|
||||
if [[ $action == deploy && $migrations_changed == true ]]; then
|
||||
run_backup "$source_sha"
|
||||
run_migrator "$source_sha" "$api_image"
|
||||
fi
|
||||
|
||||
if ! rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" ||
|
||||
! rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" ||
|
||||
! wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
|
||||
if [[ -n $previous_api && -n $previous_web ]]; then
|
||||
echo '[cluster-release] restoring previous application images' >&2
|
||||
rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
|
||||
rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
|
||||
fi
|
||||
rm -f -- "$current_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
record_current "$manifest" "$action" "$started_at"
|
||||
rm -f -- "$current_file"
|
||||
echo "production_release=PASS action=$action release=$source_sha"
|
||||
}
|
||||
|
||||
case ${1:-} in
|
||||
status)
|
||||
[[ $# -eq 1 ]] || exit 64
|
||||
status_file=$(mktemp "$RELEASES_DIR/.status.XXXXXX")
|
||||
current_to_file "$status_file"
|
||||
cat "$status_file"
|
||||
rm -f -- "$status_file"
|
||||
;;
|
||||
adopt)
|
||||
[[ $# -eq 2 ]] || exit 64
|
||||
node "$manifest_tool" validate "$2" >/dev/null
|
||||
api_image=$(manifest_get "$2" images.api)
|
||||
web_image=$(manifest_get "$2" images.web)
|
||||
[[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
|
||||
[[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
|
||||
-o jsonpath='{.spec.template.spec.containers[?(@.name=="web")].image}') == "$web_image" ]]
|
||||
record_current "$2" deploy "$(date +%s)"
|
||||
;;
|
||||
deploy)
|
||||
[[ $# -eq 2 ]] || exit 64
|
||||
activate_manifest "$2" deploy
|
||||
;;
|
||||
rollback)
|
||||
[[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
|
||||
target=$RELEASES_DIR/$2.json
|
||||
[[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
|
||||
activate_manifest "$target" rollback
|
||||
;;
|
||||
*)
|
||||
echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>}' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user