easyai-ai-gateway/scripts/cluster/run-cross-node-file-e2e.sh

#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=scripts/cluster/common.sh
source "$script_dir/common.sh"
load_cluster_env
require_commands base64 jq node

: "${AI_GATEWAY_ONLINE_ACCOUNT:?}"
: "${AI_GATEWAY_ONLINE_PASSWORD:?}"

restore_workers() {
  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true
set -euo pipefail
k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
k3s kubectl set env deployment/easyai-api-hongkong -n easyai \
  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
REMOTE
}
trap restore_workers EXIT HUP INT TERM

cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
k3s kubectl set env deployment/easyai-api-hongkong -n easyai \
  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null
k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
curl -fsS http://10.77.0.1:31088/api/v1/readyz | grep -q '"ok":true'
curl -fsS http://10.77.0.2:31089/api/v1/readyz | grep -q '"ok":true'
REMOTE

credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX")
trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM
chmod 0600 "$credentials_file"
printf 'E2E_ACCOUNT_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" \
  >"$credentials_file"
printf 'E2E_PASSWORD_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" \
  >>"$credentials_file"
printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" \
  >>"$credentials_file"
cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env"
cluster_scp "$script_dir/cross-node-file-e2e.mjs" \
  "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs"

result=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
set -euo pipefail
chmod 0600 /root/easyai-cross-node-e2e.env
set -a
# shellcheck source=/dev/null
source /root/easyai-cross-node-e2e.env
set +a
node /root/easyai-cross-node-file-e2e.mjs
rm -f -- /root/easyai-cross-node-e2e.env /root/easyai-cross-node-file-e2e.mjs
REMOTE
)
task_id=$(jq -r '.taskId' <<<"$result")
source_sha=$(jq -r '.sourceSHA256' <<<"$result")
result_url=$(jq -r '.resultURL' <<<"$result")
[[ $task_id =~ ^[0-9a-f-]{36}$ && $source_sha =~ ^[0-9a-f]{64}$ && $result_url == https://* ]]

database_evidence=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$task_id" "$source_sha" <<'REMOTE'
set -euo pipefail
task_id=$1
source_sha=$2
primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
query() {
  k3s kubectl exec -n easyai "$primary" -c postgres -- \
    psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At "$@"
}
attempted_by=$(query -c \
  "SELECT attempted_by::text FROM river_job WHERE id=(SELECT river_job_id FROM gateway_tasks WHERE id='$task_id'::uuid);")
[[ $attempted_by == *easyai-api-hongkong* ]] || {
  echo "River attempted_by does not identify Hong Kong: $attempted_by" >&2
  exit 1
}
[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid AND status='succeeded';") -eq 1 ]]
[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid;") -eq 1 ]]
[[ $(query -c "SELECT count(*) FROM settlement_outbox WHERE task_id='$task_id'::uuid AND action='settle';") -eq 1 ]]
[[ $(query -c "SELECT count(*) FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id='$task_id' AND transaction_type='task_billing';") -eq 1 ]]
callback_counts=$(query -F ' ' -c \
  "SELECT count(*), count(DISTINCT (seq, callback_url)) FROM gateway_task_callback_outbox WHERE task_id='$task_id'::uuid;")
read -r callbacks unique_callbacks <<<"$callback_counts"
[[ $callbacks -eq $unique_callbacks ]]
source_url=$(query -c \
  "SELECT url FROM gateway_request_assets WHERE sha256='$source_sha' ORDER BY created_at DESC LIMIT 1;")
[[ $source_url == https://* ]]
printf '%s\n' "$source_url"
REMOTE
)
source_url=$(tail -1 <<<"$database_evidence")

compare_url_from_nodes() {
  local label=$1
  local url=$2
  local expected_hash=
  local expected_size=
  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
    evidence=$(cluster_ssh "$host" bash -s -- "$url" <<'REMOTE'
set -euo pipefail
url=$1
temporary=$(mktemp)
trap 'rm -f -- "$temporary"' EXIT
curl -fsSL --max-time 300 "$url" -o "$temporary"
printf '%s %s\n' "$(sha256sum "$temporary" | awk '{print $1}')" "$(stat -c '%s' "$temporary")"
REMOTE
)
    read -r hash size <<<"$evidence"
    [[ $hash =~ ^[0-9a-f]{64}$ && $size -gt 0 ]]
    if [[ -z $expected_hash ]]; then
      expected_hash=$hash
      expected_size=$size
    else
      [[ $hash == "$expected_hash" && $size == "$expected_size" ]]
    fi
  done
  printf '%s_url=PASS sha256=%s bytes=%s\n' "$label" "$expected_hash" "$expected_size"
}

compare_url_from_nodes request_asset "$source_url"
compare_url_from_nodes generated_result "$result_url"
restore_workers
trap - EXIT HUP INT TERM
rm -f -- "$credentials_file"
echo "cross_node_file_e2e=PASS task=$task_id worker=hongkong"