wangbo
9e4fc7362d
使用 PostgreSQL 统一同步与异步非文本任务的并发准入、持久化等待和 Worker 容量分配，并将生产 API 与独立 Worker 角色拆分。 补充策略管理、共享契约、OpenAPI、Kubernetes 双节点 Worker 清单及跨节点验收脚本；未默认启用任何生产 queue_size 策略。 已在原基线完成 Go、前端、迁移、Shell、Kustomize 与长任务容量验收；合入最新主干后将重新执行发布门禁。
147 lines
6.9 KiB
Bash
Executable File
147 lines
6.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
# shellcheck source=scripts/cluster/common.sh
|
|
source "$script_dir/common.sh"
|
|
|
|
mode=${1:-precutover}
|
|
[[ $mode == precutover || $mode == postcutover ]] || {
|
|
echo 'usage: verify-cluster.sh [precutover|postcutover]' >&2
|
|
exit 64
|
|
}
|
|
load_cluster_env
|
|
|
|
hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
|
|
wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
|
|
for source_index in 0 1 2; do
|
|
handshakes=$(cluster_ssh "${hosts[$source_index]}" \
|
|
"wg show wg0 latest-handshakes | awk '\$2 > 0 && systime() - \$2 < 180 { count++ } END { print count + 0 }'")
|
|
[[ $handshakes -eq 2 ]] || {
|
|
echo "stale WireGuard handshake on ${hosts[$source_index]}" >&2
|
|
exit 1
|
|
}
|
|
for target_index in 0 1 2; do
|
|
[[ $source_index -eq $target_index ]] && continue
|
|
ping_output=$(cluster_ssh "${hosts[$source_index]}" \
|
|
"ping -c 10 -W 2 ${wireguard_ips[$target_index]}")
|
|
packet_loss=$(sed -nE 's/.* ([0-9.]+)% packet loss.*/\1/p' <<<"$ping_output")
|
|
average_rtt=$(sed -nE 's#.* = [0-9.]+/([0-9.]+)/.*#\1#p' <<<"$ping_output")
|
|
awk -v loss="$packet_loss" 'BEGIN { exit !(loss < 1) }'
|
|
if [[ $source_index -eq 2 || $target_index -eq 2 ]]; then
|
|
awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 300) }'
|
|
else
|
|
awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 80) }'
|
|
fi
|
|
done
|
|
done
|
|
echo 'wireguard_acceptance=PASS handshakes=6 packet_loss_lt_1=true'
|
|
|
|
cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
|
set -euo pipefail
|
|
kubectl='k3s kubectl'
|
|
[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
|
|
$kubectl get --raw='/readyz?verbose' | grep -q '\[+\]etcd ok'
|
|
[[ $($kubectl get node easyai-los-angeles -o json |
|
|
jq '[.spec.taints[]? | select(.key=="easyai.io/witness" and .effect=="NoSchedule")] | length') -eq 1 ]]
|
|
[[ $($kubectl get pods -A -o json |
|
|
jq '[.items[] | select(.spec.nodeName=="easyai-los-angeles") |
|
|
select(.metadata.namespace != "kube-system" and .metadata.namespace != "cnpg-system" and .metadata.namespace != "cert-manager")] | length') -eq 0 ]]
|
|
printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)"
|
|
REMOTE
|
|
)
|
|
[[ $cluster_state == nodes=3 ]]
|
|
echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0'
|
|
|
|
database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
|
set -euo pipefail
|
|
kubectl='k3s kubectl'
|
|
[[ $($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
|
|
primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
|
|
[[ $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-ningbo ||
|
|
$($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-hongkong ]]
|
|
replication=$($kubectl exec -n easyai "$primary" -c postgres -- \
|
|
psql -X -U postgres -d easyai_ai_gateway -At -c \
|
|
"SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
|
|
[[ $replication -eq 1 ]]
|
|
archived_count=$($kubectl exec -n easyai "$primary" -c postgres -- \
|
|
psql -X -U postgres -d postgres -At -c \
|
|
'SELECT archived_count FROM pg_stat_archiver;')
|
|
(( archived_count > 0 ))
|
|
[[ $($kubectl get cluster easyai-postgres -n easyai \
|
|
-o jsonpath='{.status.conditions[?(@.type=="ContinuousArchiving")].status}') == True ]]
|
|
completed_backups=$($kubectl get backups -n easyai -o json |
|
|
jq '[.items[] | select(
|
|
.spec.cluster.name == "easyai-postgres" and .status.phase == "completed"
|
|
)] | length')
|
|
(( completed_backups > 0 ))
|
|
for pod in $($kubectl get pods -n easyai -l cnpg.io/cluster=easyai-postgres \
|
|
-o jsonpath='{.items[*].metadata.name}'); do
|
|
$kubectl exec -n easyai "$pod" -c postgres -- \
|
|
test -r /projected/barman-aws/config
|
|
done
|
|
printf 'primary=%s archived=%s backups=%s\n' \
|
|
"$primary" "$archived_count" "$completed_backups"
|
|
REMOTE
|
|
)
|
|
[[ $database_state == primary=easyai-postgres-* ]]
|
|
echo "postgres_preflight=PASS $database_state synchronous_replicas=1"
|
|
|
|
if [[ $mode == postcutover ]]; then
|
|
application_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
|
set -euo pipefail
|
|
kubectl='k3s kubectl'
|
|
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api \
|
|
--field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
|
|
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-web \
|
|
--field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
|
|
[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-worker \
|
|
--field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
|
|
for site in ningbo hongkong; do
|
|
[[ $($kubectl get deployment "easyai-api-$site" -n easyai \
|
|
-o jsonpath='{.status.readyReplicas}') -eq 1 ]]
|
|
[[ $($kubectl get deployment "easyai-worker-$site" -n easyai \
|
|
-o jsonpath='{.status.readyReplicas}') -eq 1 ]]
|
|
[[ $($kubectl get deployment "easyai-api-$site" -n easyai \
|
|
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
|
|
[[ $($kubectl get deployment "easyai-worker-$site" -n easyai \
|
|
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
|
|
done
|
|
images=$($kubectl get deployments -n easyai -o json |
|
|
jq -r '.items[] | select(.metadata.name | startswith("easyai-")) |
|
|
.spec.template.spec.containers[].image')
|
|
if grep -Ev '@sha256:[0-9a-f]{64}$' <<<"$images" | grep -q .; then
|
|
echo 'one or more application images are not digest-pinned' >&2
|
|
exit 1
|
|
fi
|
|
[[ $($kubectl get secret easyai-gateway-security-events -n easyai \
|
|
-o json | jq '.data | length') -eq 4 ]]
|
|
[[ $($kubectl auth can-i update secret/easyai-gateway-security-events -n easyai \
|
|
--as=system:serviceaccount:easyai:easyai-ai-gateway) == yes ]]
|
|
[[ $($kubectl auth can-i update secret/easyai-ai-gateway-runtime -n easyai \
|
|
--as=system:serviceaccount:easyai:easyai-ai-gateway) == no ]]
|
|
printf 'applications=ready\n'
|
|
REMOTE
|
|
)
|
|
[[ $application_state == applications=ready ]]
|
|
echo 'application_acceptance=PASS api=2 worker=2 web=2 roles=isolated digests=pinned'
|
|
|
|
for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
|
|
health_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
|
https://ai.51easyai.com/api/v1/healthz)
|
|
grep -q easyai-ai-gateway <<<"$health_body"
|
|
ready_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
|
https://ai.51easyai.com/api/v1/readyz)
|
|
grep -q '"ok":true' <<<"$ready_body"
|
|
openapi_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
|
https://ai.51easyai.com/api/v1/openapi.json)
|
|
grep -q '"/api/v1/chat/completions"' <<<"$openapi_body"
|
|
web_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
|
https://ai.51easyai.com/)
|
|
grep -q 'EasyAI AI Gateway' <<<"$web_body"
|
|
done
|
|
echo 'nginx_edges=PASS ningbo=true hongkong=true dns_changed=false'
|
|
fi
|
|
|
|
echo "cluster_verification=PASS mode=$mode"
|