feat(deploy): 增加三节点 K3s 高可用迁移能力
新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。 新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。 所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。 验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。
This commit is contained in:
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
load_cluster_env
|
||||
require_commands base64 jq node
|
||||
|
||||
: "${AI_GATEWAY_ONLINE_ACCOUNT:?}"
|
||||
: "${AI_GATEWAY_ONLINE_PASSWORD:?}"
|
||||
|
||||
restore_workers() {
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true
|
||||
set -euo pipefail
|
||||
k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
|
||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
|
||||
k3s kubectl set env deployment/easyai-api-hongkong -n easyai \
|
||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
|
||||
k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
|
||||
k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
|
||||
REMOTE
|
||||
}
|
||||
trap restore_workers EXIT HUP INT TERM
|
||||
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
k3s kubectl set env deployment/easyai-api-hongkong -n easyai \
|
||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
|
||||
k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
|
||||
k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
|
||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null
|
||||
k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
|
||||
curl -fsS http://10.77.0.1:30088/api/v1/readyz | grep -q '"ok":true'
|
||||
curl -fsS http://10.77.0.2:30088/api/v1/readyz | grep -q '"ok":true'
|
||||
REMOTE
|
||||
|
||||
credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX")
|
||||
trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM
|
||||
chmod 0600 "$credentials_file"
|
||||
printf 'E2E_ACCOUNT_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" \
|
||||
>"$credentials_file"
|
||||
printf 'E2E_PASSWORD_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" \
|
||||
>>"$credentials_file"
|
||||
printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" \
|
||||
>>"$credentials_file"
|
||||
cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env"
|
||||
cluster_scp "$script_dir/cross-node-file-e2e.mjs" \
|
||||
"$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs"
|
||||
|
||||
result=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
chmod 0600 /root/easyai-cross-node-e2e.env
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source /root/easyai-cross-node-e2e.env
|
||||
set +a
|
||||
node /root/easyai-cross-node-file-e2e.mjs
|
||||
rm -f -- /root/easyai-cross-node-e2e.env /root/easyai-cross-node-file-e2e.mjs
|
||||
REMOTE
|
||||
)
|
||||
task_id=$(jq -r '.taskId' <<<"$result")
|
||||
source_sha=$(jq -r '.sourceSHA256' <<<"$result")
|
||||
result_url=$(jq -r '.resultURL' <<<"$result")
|
||||
[[ $task_id =~ ^[0-9a-f-]{36}$ && $source_sha =~ ^[0-9a-f]{64}$ && $result_url == https://* ]]
|
||||
|
||||
database_evidence=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$task_id" "$source_sha" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
task_id=$1
|
||||
source_sha=$2
|
||||
primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
|
||||
query() {
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At "$@"
|
||||
}
|
||||
attempted_by=$(query -c \
|
||||
"SELECT attempted_by::text FROM river_job WHERE id=(SELECT river_job_id FROM gateway_tasks WHERE id='$task_id'::uuid);")
|
||||
[[ $attempted_by == *easyai-api-hongkong* ]] || {
|
||||
echo "River attempted_by does not identify Hong Kong: $attempted_by" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid AND status='succeeded';") -eq 1 ]]
|
||||
[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid;") -eq 1 ]]
|
||||
[[ $(query -c "SELECT count(*) FROM settlement_outbox WHERE task_id='$task_id'::uuid AND action='settle';") -eq 1 ]]
|
||||
[[ $(query -c "SELECT count(*) FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id='$task_id' AND transaction_type='task_billing';") -eq 1 ]]
|
||||
callback_counts=$(query -F ' ' -c \
|
||||
"SELECT count(*), count(DISTINCT (seq, callback_url)) FROM gateway_task_callback_outbox WHERE task_id='$task_id'::uuid;")
|
||||
read -r callbacks unique_callbacks <<<"$callback_counts"
|
||||
[[ $callbacks -eq $unique_callbacks ]]
|
||||
source_url=$(query -c \
|
||||
"SELECT url FROM gateway_request_assets WHERE sha256='$source_sha' ORDER BY created_at DESC LIMIT 1;")
|
||||
[[ $source_url == https://* ]]
|
||||
printf '%s\n' "$source_url"
|
||||
REMOTE
|
||||
)
|
||||
source_url=$(tail -1 <<<"$database_evidence")
|
||||
|
||||
compare_url_from_nodes() {
|
||||
local label=$1
|
||||
local url=$2
|
||||
local expected_hash=
|
||||
local expected_size=
|
||||
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
|
||||
evidence=$(cluster_ssh "$host" bash -s -- "$url" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
url=$1
|
||||
temporary=$(mktemp)
|
||||
trap 'rm -f -- "$temporary"' EXIT
|
||||
curl -fsSL --max-time 300 "$url" -o "$temporary"
|
||||
printf '%s %s\n' "$(sha256sum "$temporary" | awk '{print $1}')" "$(stat -c '%s' "$temporary")"
|
||||
REMOTE
|
||||
)
|
||||
read -r hash size <<<"$evidence"
|
||||
[[ $hash =~ ^[0-9a-f]{64}$ && $size -gt 0 ]]
|
||||
if [[ -z $expected_hash ]]; then
|
||||
expected_hash=$hash
|
||||
expected_size=$size
|
||||
else
|
||||
[[ $hash == "$expected_hash" && $size == "$expected_size" ]]
|
||||
fi
|
||||
done
|
||||
printf '%s_url=PASS sha256=%s bytes=%s\n' "$label" "$expected_hash" "$expected_size"
|
||||
}
|
||||
|
||||
compare_url_from_nodes request_asset "$source_url"
|
||||
compare_url_from_nodes generated_result "$result_url"
|
||||
restore_workers
|
||||
trap - EXIT HUP INT TERM
|
||||
rm -f -- "$credentials_file"
|
||||
echo "cross_node_file_e2e=PASS task=$task_id worker=hongkong"
|
||||
Reference in New Issue
Block a user