feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
@@ -7,13 +7,19 @@ source "$script_dir/common.sh"
|
||||
load_cluster_env
|
||||
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
'install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release /usr/local/share/easyai-ai-gateway-release'
|
||||
'install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release /usr/local/share/easyai-ai-gateway-release/production'
|
||||
cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release" \
|
||||
"$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-ai-gateway-cluster-release"
|
||||
cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example" \
|
||||
"$CLUSTER_NINGBO_HOST:/usr/local/share/easyai-ai-gateway-release/easyai-ai-gateway-cluster-release.conf.example"
|
||||
cluster_scp "$cluster_root/scripts/release-manifest.mjs" \
|
||||
"$CLUSTER_NINGBO_HOST:/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs"
|
||||
for desired_state_file in \
|
||||
namespace.yaml service-account-rbac.yaml application-config.yaml application.yaml \
|
||||
database.yaml network-policy.yaml kustomization.yaml; do
|
||||
cluster_scp "$cluster_root/deploy/kubernetes/production/$desired_state_file" \
|
||||
"$CLUSTER_NINGBO_HOST:/usr/local/share/easyai-ai-gateway-release/production/$desired_state_file"
|
||||
done
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
if [[ ! -e /etc/easyai-ai-gateway-cluster-release.conf ]]; then
|
||||
@@ -24,5 +30,6 @@ fi
|
||||
chmod 0750 /usr/local/sbin/easyai-ai-gateway-cluster-release
|
||||
chmod 0640 /etc/easyai-ai-gateway-cluster-release.conf
|
||||
chmod 0755 /usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs
|
||||
find /usr/local/share/easyai-ai-gateway-release/production -type f -exec chmod 0644 {} +
|
||||
REMOTE
|
||||
echo 'cluster_release_helper_install=PASS'
|
||||
|
||||
+352
@@ -0,0 +1,352 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
cluster_root=${cluster_root:?}
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/cluster/monitor-production-release.sh \
|
||||
--run-id <production-run-id> --release-manifest dist/releases/<SHA>.json
|
||||
|
||||
Runs a finite 24-hour post-promotion monitor: 10-second samples for the first
|
||||
2 hours, then 60-second samples. Test-only duration overrides are available via
|
||||
AI_GATEWAY_RELEASE_MONITOR_DENSE_SECONDS and
|
||||
AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS.
|
||||
EOF
|
||||
}
|
||||
|
||||
[[ ${1:-} == --run-id && ${3:-} == --release-manifest && $# -eq 4 ]] || {
|
||||
usage >&2
|
||||
exit 64
|
||||
}
|
||||
run_id=$2
|
||||
release_manifest=$4
|
||||
[[ $run_id =~ ^[0-9a-f-]{36}$ && -f $release_manifest && ! -L $release_manifest ]] || {
|
||||
echo 'invalid monitor Run ID or release manifest' >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
load_cluster_env
|
||||
require_commands curl jq node openssl
|
||||
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}"
|
||||
: "${AI_GATEWAY_RELEASE_MONITOR_DENSE_SECONDS:=7200}"
|
||||
: "${AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS:=86400}"
|
||||
: "${AI_GATEWAY_RELEASE_MONITOR_DENSE_INTERVAL_SECONDS:=10}"
|
||||
: "${AI_GATEWAY_RELEASE_MONITOR_SPARSE_INTERVAL_SECONDS:=60}"
|
||||
[[ $AI_GATEWAY_RELEASE_MONITOR_DENSE_SECONDS =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_RELEASE_MONITOR_DENSE_INTERVAL_SECONDS =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_RELEASE_MONITOR_SPARSE_INTERVAL_SECONDS =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_RELEASE_MONITOR_DENSE_SECONDS -le $AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS ]] || {
|
||||
echo 'invalid monitor duration configuration' >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
|
||||
release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
|
||||
api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
|
||||
api_digest=${api_image##*@}
|
||||
namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai}
|
||||
report_root=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"}
|
||||
overall_report=$report_root/acceptance-report.json
|
||||
monitor_report=$report_root/post-promotion-monitor.json
|
||||
observations=$report_root/post-promotion-observations.csv
|
||||
capacity_snapshot=$report_root/pre-acceptance-capacity.json
|
||||
[[ -f $overall_report && ! -L $overall_report &&
|
||||
-f $capacity_snapshot && ! -L $capacity_snapshot ]] || {
|
||||
echo 'monitor requires the overall report and pre-acceptance capacity snapshot' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
temporary_root=$(mktemp -d)
|
||||
chmod 0700 "$temporary_root"
|
||||
trap '[[ $temporary_root == /tmp/* || $temporary_root == /var/folders/* ]] && rm -rf -- "$temporary_root"' EXIT
|
||||
|
||||
remote_kubectl() {
|
||||
local command_text
|
||||
printf -v command_text '%q ' "$@"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" "k3s kubectl $command_text"
|
||||
}
|
||||
|
||||
database_query() {
|
||||
local sql=$1 primary
|
||||
primary=$(remote_kubectl get cluster easyai-postgres -n "$namespace" \
|
||||
-o 'jsonpath={.status.currentPrimary}')
|
||||
remote_kubectl exec -n "$namespace" "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql"
|
||||
}
|
||||
|
||||
admin_request() {
|
||||
local method=$1 path=$2 body=${3:-} output=$4
|
||||
local service_ip service_port token_encoded body_encoded response status payload script command
|
||||
service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" -o 'jsonpath={.spec.clusterIP}')
|
||||
service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" -o 'jsonpath={.spec.ports[0].port}')
|
||||
token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | openssl base64 -A)
|
||||
body_encoded=$(printf '%s' "$body" | openssl base64 -A)
|
||||
# shellcheck disable=SC2016
|
||||
script='
|
||||
set -euo pipefail
|
||||
token=$(printf "%s" "$4" | base64 -d)
|
||||
body=$(printf "%s" "$5" | base64 -d)
|
||||
args=(-sS --max-time 30 -w "\n%{http_code}" -X "$1" -H "Authorization: Bearer $token")
|
||||
if [[ -n $body ]]; then args+=(-H "Content-Type: application/json" --data-binary "$body"); fi
|
||||
curl "${args[@]}" "http://$2:$3$6"
|
||||
'
|
||||
printf -v command 'bash -c %q -- %q %q %q %q %q %q' \
|
||||
"$script" "$method" "$service_ip" "$service_port" "$token_encoded" "$body_encoded" "$path"
|
||||
response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$command")
|
||||
status=${response##*$'\n'}
|
||||
payload=${response%$'\n'*}
|
||||
printf '%s' "$payload" >"$output"
|
||||
[[ $status =~ ^2[0-9][0-9]$ ]]
|
||||
}
|
||||
|
||||
metric_sum() {
|
||||
local pattern=$1 sum=0 pod value metrics
|
||||
while read -r pod; do
|
||||
[[ -n $pod ]] || continue
|
||||
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
||||
wget -qO- http://127.0.0.1:8088/metrics)
|
||||
value=$(awk -v pattern="$pattern" '$0 ~ pattern {sum += $NF} END {print sum+0}' <<<"$metrics")
|
||||
sum=$(awk -v left="$sum" -v right="$value" 'BEGIN {print left+right}')
|
||||
done < <(remote_kubectl get pods -n "$namespace" \
|
||||
-l app.kubernetes.io/name=easyai-worker \
|
||||
-o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
printf '%s\n' "$sum"
|
||||
}
|
||||
|
||||
pool_saturated_pods() {
|
||||
local pod metrics max acquired idle saturated=0
|
||||
while read -r pod; do
|
||||
[[ -n $pod ]] || continue
|
||||
metrics=$(remote_kubectl exec -n "$namespace" "$pod" -- \
|
||||
wget -qO- http://127.0.0.1:8088/metrics)
|
||||
max=$(awk '$1=="easyai_gateway_postgres_pool_max_connections" {print int($2)}' <<<"$metrics")
|
||||
acquired=$(awk '$1=="easyai_gateway_postgres_pool_acquired_connections" {print int($2)}' <<<"$metrics")
|
||||
idle=$(awk '$1=="easyai_gateway_postgres_pool_idle_connections" {print int($2)}' <<<"$metrics")
|
||||
[[ -z $max || -z $acquired || -z $idle ]] || {
|
||||
(( acquired == max && idle == 0 )) && saturated=$((saturated + 1))
|
||||
}
|
||||
done < <(remote_kubectl get pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' \
|
||||
-o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
printf '%s\n' "$saturated"
|
||||
}
|
||||
|
||||
check_wireguard() {
|
||||
local -a hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
|
||||
local -a ips=(10.77.0.1 10.77.0.2 10.77.0.3)
|
||||
local source target output loss average handshakes
|
||||
for source in 0 1 2; do
|
||||
handshakes=$(cluster_ssh "${hosts[$source]}" \
|
||||
"wg show wg0 latest-handshakes | awk '\$2 > 0 && systime()-\$2 < 180 {count++} END {print count+0}'")
|
||||
(( handshakes == 2 )) || return 1
|
||||
for target in 0 1 2; do
|
||||
(( source == target )) && continue
|
||||
output=$(cluster_ssh "${hosts[$source]}" "ping -q -c 10 -W 2 ${ips[$target]}") || return 1
|
||||
loss=$(awk -F',' '/packet loss/ {gsub(/[^0-9.]/,"",$3); print $3}' <<<"$output")
|
||||
average=$(awk -F'=' '/min\\/avg\\/max/ {split($2,v,"/"); gsub(/ /,"",v[2]); print v[2]}' <<<"$output")
|
||||
if (( source == 2 || target == 2 )); then
|
||||
awk -v loss="$loss" -v average="$average" 'BEGIN {exit !(loss < 1 && average < 300)}' || return 1
|
||||
else
|
||||
awk -v loss="$loss" -v average="$average" 'BEGIN {exit !(loss < 1 && average < 80)}' || return 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
restore_previous_capacity() {
|
||||
local remote_file="/root/easyai-pre-acceptance-$run_id.json"
|
||||
cluster_scp "$capacity_snapshot" "$CLUSTER_NINGBO_HOST:$remote_file" >/dev/null
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
"k3s kubectl apply -f '$remote_file' >/dev/null && unlink '$remote_file'"
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo \
|
||||
easyai-worker-hongkong easyai-capacity-controller; do
|
||||
remote_kubectl rollout status deployment/"$deployment" -n "$namespace" --timeout=600s
|
||||
done
|
||||
}
|
||||
|
||||
pause_and_restore() {
|
||||
local gate_id=$1 mode_file=$temporary_root/mode.json pause_file=$temporary_root/pause.json body
|
||||
admin_request GET /api/admin/system/acceptance/traffic-mode '' "$mode_file"
|
||||
body=$(jq -cn \
|
||||
--argjson revision "$(jq -r '.revision' "$mode_file")" \
|
||||
--arg releaseSha "$release_sha" \
|
||||
--arg apiDigest "$api_digest" \
|
||||
--arg reason "$gate_id" \
|
||||
'{
|
||||
revision:$revision,
|
||||
releaseSha:$releaseSha,
|
||||
apiImageDigest:$apiDigest,
|
||||
workerImageDigest:$apiDigest,
|
||||
reason:$reason
|
||||
}')
|
||||
admin_request POST /api/admin/system/acceptance/traffic-mode/pause "$body" "$pause_file"
|
||||
[[ $(jq -r '.mode' "$pause_file") == validation ]]
|
||||
restore_previous_capacity
|
||||
}
|
||||
|
||||
started_epoch=$(date +%s)
|
||||
started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
monitor_since=$(date -u '+%Y-%m-%d %H:%M:%S+00')
|
||||
baseline_lease_failure=$(metric_sum 'outcome="failure"')
|
||||
baseline_lease_lost=$(metric_sum 'outcome="lost"')
|
||||
samples=0
|
||||
queue_growth_streak=0
|
||||
pool_saturation_streak=0
|
||||
previous_queue=0
|
||||
failure_gate_id=
|
||||
last_wireguard_epoch=0
|
||||
printf 'timestamp,queue_depth,oldest_wait_seconds,connections,max_connections,saturated_pods,node_max_memory_percent,lease_failure,lease_lost\n' >"$observations"
|
||||
|
||||
while (( $(date +%s) - started_epoch < AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS )); do
|
||||
now_epoch=$(date +%s)
|
||||
elapsed=$((now_epoch - started_epoch))
|
||||
interval=$AI_GATEWAY_RELEASE_MONITOR_SPARSE_INTERVAL_SECONDS
|
||||
(( elapsed < AI_GATEWAY_RELEASE_MONITOR_DENSE_SECONDS )) &&
|
||||
interval=$AI_GATEWAY_RELEASE_MONITOR_DENSE_INTERVAL_SECONDS
|
||||
samples=$((samples + 1))
|
||||
|
||||
if ! curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/healthz >/dev/null ||
|
||||
! curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz >/dev/null; then
|
||||
failure_gate_id=public_health
|
||||
fi
|
||||
ready_nodes=$(remote_kubectl get nodes -o json |
|
||||
jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length')
|
||||
pressure_nodes=$(remote_kubectl get nodes -o json |
|
||||
jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length')
|
||||
[[ $ready_nodes == 3 && $pressure_nodes == 0 ]] || failure_gate_id=${failure_gate_id:-node_health}
|
||||
node_max_memory=$(remote_kubectl top nodes --no-headers |
|
||||
awk '{gsub(/%/,"",$5); if ($5>max) max=$5} END {print max+0}')
|
||||
(( node_max_memory < 85 )) || failure_gate_id=${failure_gate_id:-node_memory_hard}
|
||||
unhealthy=$(remote_kubectl get pods -n "$namespace" -o json |
|
||||
jq '[.items[]
|
||||
| select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
|
||||
.metadata.labels["app.kubernetes.io/name"] == "easyai-worker")
|
||||
| .status.containerStatuses[]?
|
||||
| select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or
|
||||
.state.terminated.reason == "OOMKilled")] | length')
|
||||
[[ $unhealthy == 0 ]] || failure_gate_id=${failure_gate_id:-pod_restart_or_oom}
|
||||
pod_max_memory=$(remote_kubectl top pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers |
|
||||
awk '{
|
||||
value=$3
|
||||
if (value ~ /Gi$/) {sub(/Gi$/,"",value); value*=1024}
|
||||
else if (value ~ /Mi$/) {sub(/Mi$/,"",value)}
|
||||
else if (value ~ /Ki$/) {sub(/Ki$/,"",value); value/=1024}
|
||||
if (value>max) max=value
|
||||
} END {printf "%.0f",max}')
|
||||
(( pod_max_memory < 1536 )) || failure_gate_id=${failure_gate_id:-pod_memory_hard}
|
||||
[[ $(remote_kubectl get cluster easyai-postgres -n "$namespace" \
|
||||
-o 'jsonpath={.status.readyInstances}') == 2 ]] || failure_gate_id=${failure_gate_id:-postgres_ready}
|
||||
sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
|
||||
[[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] ||
|
||||
failure_gate_id=${failure_gate_id:-postgres_replication}
|
||||
connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
|
||||
max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
|
||||
(( connections < 150 && connections * 4 < max_connections * 3 )) ||
|
||||
failure_gate_id=${failure_gate_id:-postgres_connections}
|
||||
queue_state=$(database_query "
|
||||
SELECT count(*) FILTER (WHERE status='queued')||':'||
|
||||
COALESCE(EXTRACT(EPOCH FROM now()-MIN(created_at)) FILTER (WHERE status='queued'),0)::bigint
|
||||
FROM gateway_tasks
|
||||
WHERE run_mode='production';")
|
||||
IFS=: read -r queue_depth oldest_wait <<<"$queue_state"
|
||||
if (( queue_depth > previous_queue && queue_depth > 0 )); then
|
||||
queue_growth_streak=$((queue_growth_streak + 1))
|
||||
else
|
||||
queue_growth_streak=0
|
||||
fi
|
||||
previous_queue=$queue_depth
|
||||
(( oldest_wait <= 900 && queue_growth_streak < 4 )) ||
|
||||
failure_gate_id=${failure_gate_id:-queue_growth_or_oldest_wait}
|
||||
saturated=$(pool_saturated_pods)
|
||||
if (( saturated > 0 )); then
|
||||
pool_saturation_streak=$((pool_saturation_streak + 1))
|
||||
else
|
||||
pool_saturation_streak=0
|
||||
fi
|
||||
(( pool_saturation_streak * interval < 30 )) ||
|
||||
failure_gate_id=${failure_gate_id:-postgres_pool_saturation}
|
||||
lease_failure=$(metric_sum 'outcome="failure"')
|
||||
lease_lost=$(metric_sum 'outcome="lost"')
|
||||
awk -v current="$lease_failure" -v baseline="$baseline_lease_failure" 'BEGIN {exit !(current<=baseline)}' ||
|
||||
failure_gate_id=${failure_gate_id:-lease_renewal_failure}
|
||||
awk -v current="$lease_lost" -v baseline="$baseline_lease_lost" 'BEGIN {exit !(current<=baseline)}' ||
|
||||
failure_gate_id=${failure_gate_id:-lease_lost}
|
||||
duplicates=$(database_query "
|
||||
SELECT
|
||||
(SELECT count(*) FROM (
|
||||
SELECT remote_task_id
|
||||
FROM gateway_tasks
|
||||
WHERE run_mode='production' AND created_at >= '$monitor_since'::timestamptz
|
||||
AND remote_task_id IS NOT NULL
|
||||
GROUP BY remote_task_id HAVING count(*)>1
|
||||
) d)
|
||||
+
|
||||
(SELECT count(*) FROM (
|
||||
SELECT reference_id,transaction_type
|
||||
FROM gateway_wallet_transactions
|
||||
WHERE created_at >= '$monitor_since'::timestamptz
|
||||
AND reference_type='gateway_task'
|
||||
GROUP BY reference_id,transaction_type HAVING count(*)>1
|
||||
) d);")
|
||||
[[ $duplicates == 0 ]] || failure_gate_id=${failure_gate_id:-duplicate_execution_or_billing}
|
||||
if (( now_epoch - last_wireguard_epoch >= 300 )); then
|
||||
check_wireguard || failure_gate_id=${failure_gate_id:-wireguard}
|
||||
last_wireguard_epoch=$now_epoch
|
||||
fi
|
||||
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
|
||||
"$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$queue_depth" "$oldest_wait" \
|
||||
"$connections" "$max_connections" "$saturated" "$node_max_memory" \
|
||||
"$lease_failure" "$lease_lost" >>"$observations"
|
||||
[[ -z $failure_gate_id ]] || break
|
||||
sleep "$interval"
|
||||
done
|
||||
|
||||
finished_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
passed=true
|
||||
traffic_mode=live
|
||||
restored=false
|
||||
if [[ -n $failure_gate_id ]]; then
|
||||
passed=false
|
||||
pause_and_restore "$failure_gate_id"
|
||||
traffic_mode=validation
|
||||
restored=true
|
||||
fi
|
||||
jq -n \
|
||||
--arg runId "$run_id" \
|
||||
--arg releaseSha "$release_sha" \
|
||||
--arg startedAt "$started_at" \
|
||||
--arg finishedAt "$finished_at" \
|
||||
--arg failureGateId "$failure_gate_id" \
|
||||
--arg trafficMode "$traffic_mode" \
|
||||
--argjson samples "$samples" \
|
||||
--argjson passed "$passed" \
|
||||
--argjson restored "$restored" \
|
||||
'{
|
||||
schemaVersion:"acceptance-monitor-report/v1",
|
||||
runId:$runId,
|
||||
releaseSha:$releaseSha,
|
||||
startedAt:$startedAt,
|
||||
finishedAt:$finishedAt,
|
||||
samples:$samples,
|
||||
passed:$passed,
|
||||
failureGateId:($failureGateId | if length>0 then . else null end),
|
||||
trafficMode:$trafficMode,
|
||||
previousCapacityRestored:$restored,
|
||||
secretSafe:true
|
||||
}' >"$monitor_report"
|
||||
chmod 0600 "$monitor_report" "$observations"
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" attach-monitor \
|
||||
--input "$overall_report" --monitor "$monitor_report" --output "$overall_report" >/dev/null
|
||||
if [[ $passed == true ]]; then
|
||||
echo "production_release_monitor=PASS run_id=$run_id samples=$samples duration_seconds=$AI_GATEWAY_RELEASE_MONITOR_TOTAL_SECONDS traffic_mode=live"
|
||||
else
|
||||
echo "production_release_monitor=FAIL run_id=$run_id gate_id=$failure_gate_id traffic_mode=validation previous_capacity_restored=true" >&2
|
||||
exit 1
|
||||
fi
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
cluster_root=${cluster_root:?}
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--promote dist/releases/<SHA>.json --run-id <production-run-id>
|
||||
|
||||
This is the only acceptance action that switches production traffic from
|
||||
validation to live. It requires an already-passed overall acceptance report and
|
||||
rechecks the release, digest, config, Run, and traffic-mode CAS fields.
|
||||
EOF
|
||||
}
|
||||
|
||||
[[ ${1:-} == --promote && ${3:-} == --run-id && $# -eq 4 ]] || {
|
||||
usage >&2
|
||||
exit 64
|
||||
}
|
||||
release_manifest=$2
|
||||
run_id=$4
|
||||
[[ -f $release_manifest && ! -L $release_manifest ]] || {
|
||||
echo 'release manifest must be a regular non-symlink file' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $run_id =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || {
|
||||
echo 'run ID must be a lowercase UUID' >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
load_cluster_env
|
||||
require_commands curl git jq node openssl
|
||||
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}"
|
||||
|
||||
node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
|
||||
release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
|
||||
api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
|
||||
api_digest=${api_image##*@}
|
||||
[[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" &&
|
||||
-z $(git -C "$cluster_root" status --short) ]] || {
|
||||
echo 'manual promotion requires the clean release source at HEAD' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
report=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"}/acceptance-report.json
|
||||
[[ -f $report && ! -L $report ]] || {
|
||||
echo "passed overall acceptance report is missing: $report" >&2
|
||||
exit 1
|
||||
}
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" validate --input "$report" >/dev/null
|
||||
[[ $(jq -r '.runId' "$report") == "$run_id" &&
|
||||
$(jq -r '.release.sourceSha' "$report") == "$release_sha" &&
|
||||
$(jq -r '.release.apiImageDigest' "$report") == "$api_digest" &&
|
||||
$(jq -r '.stages.onlineSimulation.status' "$report") == passed &&
|
||||
$(jq -r '.stages.realCanary.status' "$report") == passed &&
|
||||
$(jq -r '.promotion.ready' "$report") == true &&
|
||||
$(jq -r '.promotion.trafficMode' "$report") == validation ]] || {
|
||||
echo 'overall acceptance report is not ready for this manual promotion' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai}
|
||||
remote_release_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-cluster-release}
|
||||
temporary_root=$(mktemp -d)
|
||||
chmod 0700 "$temporary_root"
|
||||
trap '[[ $temporary_root == /tmp/* || $temporary_root == /var/folders/* ]] && rm -rf -- "$temporary_root"' EXIT
|
||||
|
||||
remote_kubectl() {
|
||||
local command_text
|
||||
printf -v command_text '%q ' "$@"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" "k3s kubectl $command_text"
|
||||
}
|
||||
|
||||
admin_request() {
|
||||
local method=$1 path=$2 body=${3:-} output=$4
|
||||
local service_ip service_port token_encoded body_encoded response status payload remote_script command
|
||||
service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.clusterIP}')
|
||||
service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.ports[0].port}')
|
||||
token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | openssl base64 -A)
|
||||
body_encoded=$(printf '%s' "$body" | openssl base64 -A)
|
||||
# shellcheck disable=SC2016
|
||||
remote_script='
|
||||
set -euo pipefail
|
||||
token=$(printf "%s" "$4" | base64 -d)
|
||||
body=$(printf "%s" "$5" | base64 -d)
|
||||
args=(-sS --max-time 30 -w "\n%{http_code}" -X "$1" -H "Authorization: Bearer $token")
|
||||
if [[ -n $body ]]; then
|
||||
args+=(-H "Content-Type: application/json" --data-binary "$body")
|
||||
fi
|
||||
curl "${args[@]}" "http://$2:$3$6"
|
||||
'
|
||||
printf -v command 'bash -c %q -- %q %q %q %q %q %q' \
|
||||
"$remote_script" "$method" "$service_ip" "$service_port" \
|
||||
"$token_encoded" "$body_encoded" "$path"
|
||||
response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$command")
|
||||
status=${response##*$'\n'}
|
||||
payload=${response%$'\n'*}
|
||||
printf '%s' "$payload" >"$output"
|
||||
[[ $status =~ ^2[0-9][0-9]$ ]] || {
|
||||
echo "acceptance control API failed: method=$method path=$path status=$status" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
current_manifest=$temporary_root/current.json
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_release_helper status" >"$current_manifest"
|
||||
node "$cluster_root/scripts/release-manifest.mjs" validate "$current_manifest" >/dev/null
|
||||
[[ $(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" sourceSha) == "$release_sha" &&
|
||||
$(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" images.api) == "$api_image" ]] || {
|
||||
echo 'production release changed after acceptance' >&2
|
||||
exit 1
|
||||
}
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
[[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \
|
||||
-o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
|
||||
done
|
||||
|
||||
mode_file=$temporary_root/mode.json
|
||||
run_file=$temporary_root/run.json
|
||||
promotion_file=$temporary_root/promotion.json
|
||||
admin_request GET /api/admin/system/acceptance/traffic-mode '' "$mode_file"
|
||||
admin_request GET "/api/admin/system/acceptance/runs/$run_id" '' "$run_file"
|
||||
[[ $(jq -r '.mode' "$mode_file") == validation &&
|
||||
$(jq -r '.runId' "$mode_file") == "$run_id" &&
|
||||
$(jq -r '.releaseSha' "$mode_file") == "$release_sha" &&
|
||||
$(jq -r '.apiImageDigest' "$mode_file") == "$api_digest" &&
|
||||
$(jq -r '.status' "$run_file") == passed ]] || {
|
||||
echo 'production acceptance Run or traffic-mode CAS changed before promotion' >&2
|
||||
exit 1
|
||||
}
|
||||
config_hash=$(jq -r '.config.validationConfigHash' "$run_file")
|
||||
profiles=$(jq -c '.config.validationCapacityProfiles' "$run_file")
|
||||
[[ $config_hash =~ ^[0-9a-f]{64}$ && $(jq 'type == "array" and length > 0' <<<"$profiles") == true ]]
|
||||
[[ $(remote_kubectl get configmap easyai-gateway-release -n "$namespace" \
|
||||
-o 'jsonpath={.data.configHash}') == "$config_hash" ]] || {
|
||||
echo 'production configuration changed after acceptance' >&2
|
||||
exit 1
|
||||
}
|
||||
body=$(jq -cn \
|
||||
--argjson revision "$(jq -r '.revision' "$mode_file")" \
|
||||
--arg releaseSha "$release_sha" \
|
||||
--arg apiDigest "$api_digest" \
|
||||
--arg configHash "$config_hash" \
|
||||
--argjson capacityProfiles "$profiles" \
|
||||
'{
|
||||
revision:$revision,
|
||||
releaseSha:$releaseSha,
|
||||
apiImageDigest:$apiDigest,
|
||||
workerImageDigest:$apiDigest,
|
||||
configHash:$configHash,
|
||||
capacityProfiles:$capacityProfiles
|
||||
}')
|
||||
admin_request POST "/api/admin/system/acceptance/runs/$run_id/promote" "$body" "$promotion_file"
|
||||
[[ $(jq -r '.mode' "$promotion_file") == live &&
|
||||
$(jq -r '.runId' "$promotion_file") == "$run_id" ]] || {
|
||||
echo 'live promotion CAS did not complete' >&2
|
||||
exit 1
|
||||
}
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" mark-promoted \
|
||||
--input "$report" --output "$report" >/dev/null
|
||||
echo "production_acceptance_promotion=PASS run_id=$run_id release=$release_sha traffic_mode=live"
|
||||
exec "$script_dir/monitor-production-release.sh" \
|
||||
--run-id "$run_id" --release-manifest "$release_manifest"
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user