diff --git a/scripts/cluster/failure-drill.sh b/scripts/cluster/failure-drill.sh index 88bf7e5..edb2a27 100755 --- a/scripts/cluster/failure-drill.sh +++ b/scripts/cluster/failure-drill.sh @@ -51,10 +51,16 @@ case $drill in cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$new_primary" <<'REMOTE' set -euo pipefail primary=$1 +cleanup_probe() { + k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -U postgres -d easyai_ai_gateway -c \ + "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null 2>&1 || true +} +trap cleanup_probe EXIT HUP INT TERM k3s kubectl exec -n easyai "$primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \ "INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null -for _ in $(seq 1 180); do +for _ in $(seq 1 900); do if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \ psql -X -U postgres -d easyai_ai_gateway -At -c \ @@ -67,6 +73,7 @@ done k3s kubectl exec -n easyai "$primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \ "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null +trap - EXIT HUP INT TERM REMOTE echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary" ;;