wangbo
9e4fc7362d
使用 PostgreSQL 统一同步与异步非文本任务的并发准入、持久化等待和 Worker 容量分配，并将生产 API 与独立 Worker 角色拆分。 补充策略管理、共享契约、OpenAPI、Kubernetes 双节点 Worker 清单及跨节点验收脚本；未默认启用任何生产 queue_size 策略。 已在原基线完成 Go、前端、迁移、Shell、Kustomize 与长任务容量验收；合入最新主干后将重新执行发布门禁。
152 lines
6.3 KiB
Bash
Executable File
152 lines
6.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
# shellcheck source=scripts/cluster/common.sh
|
|
source "$script_dir/common.sh"
|
|
load_cluster_env
|
|
require_commands base64 jq node
|
|
|
|
: "${AI_GATEWAY_ONLINE_ACCOUNT:?}"
|
|
: "${AI_GATEWAY_ONLINE_PASSWORD:?}"
|
|
|
|
restore_workers() {
|
|
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true
|
|
set -euo pipefail
|
|
k3s kubectl scale deployment/easyai-worker-ningbo -n easyai --replicas=1 >/dev/null
|
|
k3s kubectl scale deployment/easyai-worker-hongkong -n easyai --replicas=1 >/dev/null
|
|
k3s kubectl rollout status deployment/easyai-worker-ningbo -n easyai --timeout=300s
|
|
k3s kubectl rollout status deployment/easyai-worker-hongkong -n easyai --timeout=300s
|
|
REMOTE
|
|
}
|
|
trap restore_workers EXIT HUP INT TERM
|
|
|
|
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
|
set -euo pipefail
|
|
k3s kubectl scale deployment/easyai-worker-hongkong -n easyai --replicas=1 >/dev/null
|
|
k3s kubectl rollout status deployment/easyai-worker-hongkong -n easyai --timeout=300s
|
|
k3s kubectl scale deployment/easyai-worker-ningbo -n easyai --replicas=0 >/dev/null
|
|
for _ in $(seq 1 60); do
|
|
pods=$(k3s kubectl get pods -n easyai \
|
|
-l app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
|
pod_count=$(wc -w <<<"$pods" | tr -d ' ')
|
|
if [[ $pod_count == 1 ]]; then
|
|
pod=$pods
|
|
process_role=$(k3s kubectl get pod "$pod" -n easyai \
|
|
-o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}')
|
|
pod_ready=$(k3s kubectl get pod "$pod" -n easyai \
|
|
-o jsonpath='{.status.containerStatuses[0].ready}')
|
|
if [[ $process_role == worker && $pod_ready == true ]]; then
|
|
break
|
|
fi
|
|
fi
|
|
sleep 1
|
|
done
|
|
[[ ${pod_count:-0} == 1 && ${process_role:-} == worker && ${pod_ready:-} == true ]]
|
|
[[ $(k3s kubectl get deployment easyai-api-ningbo -n easyai \
|
|
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
|
|
[[ $(k3s kubectl get deployment easyai-worker-ningbo -n easyai \
|
|
-o jsonpath='{.status.readyReplicas}') == 0 ]]
|
|
curl -fsS http://10.77.0.1:31088/api/v1/readyz | grep -q '"ok":true'
|
|
curl -fsS http://10.77.0.2:31089/api/v1/readyz | grep -q '"ok":true'
|
|
REMOTE
|
|
|
|
credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX")
|
|
trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM
|
|
chmod 0600 "$credentials_file"
|
|
{
|
|
printf 'E2E_ACCOUNT_B64=%s\n' \
|
|
"$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')"
|
|
printf 'E2E_PASSWORD_B64=%s\n' \
|
|
"$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')"
|
|
printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' \
|
|
"${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}"
|
|
printf 'AI_GATEWAY_E2E_IMAGE_SIZE=%s\n' \
|
|
"${AI_GATEWAY_E2E_IMAGE_SIZE:-2048x2048}"
|
|
} >"$credentials_file"
|
|
cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env"
|
|
cluster_scp "$script_dir/cross-node-file-e2e.mjs" \
|
|
"$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs"
|
|
|
|
result=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
|
set -euo pipefail
|
|
chmod 0600 /root/easyai-cross-node-e2e.env
|
|
set -a
|
|
# shellcheck source=/dev/null
|
|
source /root/easyai-cross-node-e2e.env
|
|
set +a
|
|
node /root/easyai-cross-node-file-e2e.mjs
|
|
rm -f -- /root/easyai-cross-node-e2e.env /root/easyai-cross-node-file-e2e.mjs
|
|
REMOTE
|
|
)
|
|
task_id=$(jq -r '.taskId' <<<"$result")
|
|
source_sha=$(jq -r '.sourceSHA256' <<<"$result")
|
|
result_url=$(jq -r '.resultURL' <<<"$result")
|
|
[[ $task_id =~ ^[0-9a-f-]{36}$ && $source_sha =~ ^[0-9a-f]{64}$ && $result_url == https://* ]]
|
|
|
|
database_evidence=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$task_id" "$source_sha" <<'REMOTE'
|
|
set -euo pipefail
|
|
task_id=$1
|
|
source_sha=$2
|
|
primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
|
|
query() {
|
|
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
|
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At "$@"
|
|
}
|
|
attempted_by=$(query -c \
|
|
"SELECT attempted_by::text FROM river_job WHERE id=(SELECT river_job_id FROM gateway_tasks WHERE id='$task_id'::uuid);")
|
|
[[ $attempted_by == *easyai-worker-hongkong* ]] || {
|
|
echo "River attempted_by does not identify Hong Kong: $attempted_by" >&2
|
|
exit 1
|
|
}
|
|
[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid AND status='succeeded';") -eq 1 ]]
|
|
[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid;") -eq 1 ]]
|
|
[[ $(query -c "SELECT count(*) FROM settlement_outbox WHERE task_id='$task_id'::uuid AND action='settle';") -eq 1 ]]
|
|
[[ $(query -c "SELECT count(*) FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id='$task_id' AND transaction_type='task_billing';") -eq 1 ]]
|
|
callback_counts=$(query -F ' ' -c \
|
|
"SELECT count(*), count(DISTINCT (seq, callback_url)) FROM gateway_task_callback_outbox WHERE task_id='$task_id'::uuid;")
|
|
read -r callbacks unique_callbacks <<<"$callback_counts"
|
|
[[ $callbacks -eq $unique_callbacks ]]
|
|
source_url=$(query -c \
|
|
"SELECT url FROM gateway_request_assets WHERE sha256='$source_sha' ORDER BY created_at DESC LIMIT 1;")
|
|
[[ $source_url == https://* ]]
|
|
printf '%s\n' "$source_url"
|
|
REMOTE
|
|
)
|
|
source_url=$(tail -1 <<<"$database_evidence")
|
|
|
|
compare_url_from_nodes() {
|
|
local label=$1
|
|
local url=$2
|
|
local expected_hash=
|
|
local expected_size=
|
|
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
|
|
evidence=$(cluster_ssh "$host" bash -s -- "$url" <<'REMOTE'
|
|
set -euo pipefail
|
|
url=$1
|
|
temporary=$(mktemp)
|
|
trap 'rm -f -- "$temporary"' EXIT
|
|
curl -fsSL --max-time 300 "$url" -o "$temporary"
|
|
printf '%s %s\n' "$(sha256sum "$temporary" | awk '{print $1}')" "$(stat -c '%s' "$temporary")"
|
|
REMOTE
|
|
)
|
|
read -r hash size <<<"$evidence"
|
|
[[ $hash =~ ^[0-9a-f]{64}$ && $size -gt 0 ]]
|
|
if [[ -z $expected_hash ]]; then
|
|
expected_hash=$hash
|
|
expected_size=$size
|
|
else
|
|
[[ $hash == "$expected_hash" && $size == "$expected_size" ]]
|
|
fi
|
|
done
|
|
printf '%s_url=PASS sha256=%s bytes=%s\n' "$label" "$expected_hash" "$expected_size"
|
|
}
|
|
|
|
compare_url_from_nodes request_asset "$source_url"
|
|
compare_url_from_nodes generated_result "$result_url"
|
|
restore_workers
|
|
trap - EXIT HUP INT TERM
|
|
rm -f -- "$credentials_file"
|
|
echo "cross_node_file_e2e=PASS task=$task_id worker=hongkong"
|