feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
+170
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=scripts/cluster/common.sh
|
||||
source "$script_dir/common.sh"
|
||||
cluster_root=${cluster_root:?}
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--promote dist/releases/<SHA>.json --run-id <production-run-id>
|
||||
|
||||
This is the only acceptance action that switches production traffic from
|
||||
validation to live. It requires an already-passed overall acceptance report and
|
||||
rechecks the release, digest, config, Run, and traffic-mode CAS fields.
|
||||
EOF
|
||||
}
|
||||
|
||||
[[ ${1:-} == --promote && ${3:-} == --run-id && $# -eq 4 ]] || {
|
||||
usage >&2
|
||||
exit 64
|
||||
}
|
||||
release_manifest=$2
|
||||
run_id=$4
|
||||
[[ -f $release_manifest && ! -L $release_manifest ]] || {
|
||||
echo 'release manifest must be a regular non-symlink file' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $run_id =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || {
|
||||
echo 'run ID must be a lowercase UUID' >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
load_cluster_env
|
||||
require_commands curl git jq node openssl
|
||||
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:?}"
|
||||
|
||||
node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
|
||||
release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
|
||||
api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
|
||||
api_digest=${api_image##*@}
|
||||
[[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" &&
|
||||
-z $(git -C "$cluster_root" status --short) ]] || {
|
||||
echo 'manual promotion requires the clean release source at HEAD' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
report=${AI_GATEWAY_ACCEPTANCE_REPORT_DIR:-"$cluster_root/dist/acceptance/$run_id"}/acceptance-report.json
|
||||
[[ -f $report && ! -L $report ]] || {
|
||||
echo "passed overall acceptance report is missing: $report" >&2
|
||||
exit 1
|
||||
}
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" validate --input "$report" >/dev/null
|
||||
[[ $(jq -r '.runId' "$report") == "$run_id" &&
|
||||
$(jq -r '.release.sourceSha' "$report") == "$release_sha" &&
|
||||
$(jq -r '.release.apiImageDigest' "$report") == "$api_digest" &&
|
||||
$(jq -r '.stages.onlineSimulation.status' "$report") == passed &&
|
||||
$(jq -r '.stages.realCanary.status' "$report") == passed &&
|
||||
$(jq -r '.promotion.ready' "$report") == true &&
|
||||
$(jq -r '.promotion.trafficMode' "$report") == validation ]] || {
|
||||
echo 'overall acceptance report is not ready for this manual promotion' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai}
|
||||
remote_release_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-cluster-release}
|
||||
temporary_root=$(mktemp -d)
|
||||
chmod 0700 "$temporary_root"
|
||||
trap '[[ $temporary_root == /tmp/* || $temporary_root == /var/folders/* ]] && rm -rf -- "$temporary_root"' EXIT
|
||||
|
||||
remote_kubectl() {
|
||||
local command_text
|
||||
printf -v command_text '%q ' "$@"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" "k3s kubectl $command_text"
|
||||
}
|
||||
|
||||
admin_request() {
|
||||
local method=$1 path=$2 body=${3:-} output=$4
|
||||
local service_ip service_port token_encoded body_encoded response status payload remote_script command
|
||||
service_ip=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.clusterIP}')
|
||||
service_port=$(remote_kubectl get service api-ningbo-edge -n "$namespace" \
|
||||
-o 'jsonpath={.spec.ports[0].port}')
|
||||
token_encoded=$(printf '%s' "$AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN" | openssl base64 -A)
|
||||
body_encoded=$(printf '%s' "$body" | openssl base64 -A)
|
||||
# shellcheck disable=SC2016
|
||||
remote_script='
|
||||
set -euo pipefail
|
||||
token=$(printf "%s" "$4" | base64 -d)
|
||||
body=$(printf "%s" "$5" | base64 -d)
|
||||
args=(-sS --max-time 30 -w "\n%{http_code}" -X "$1" -H "Authorization: Bearer $token")
|
||||
if [[ -n $body ]]; then
|
||||
args+=(-H "Content-Type: application/json" --data-binary "$body")
|
||||
fi
|
||||
curl "${args[@]}" "http://$2:$3$6"
|
||||
'
|
||||
printf -v command 'bash -c %q -- %q %q %q %q %q %q' \
|
||||
"$remote_script" "$method" "$service_ip" "$service_port" \
|
||||
"$token_encoded" "$body_encoded" "$path"
|
||||
response=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "$command")
|
||||
status=${response##*$'\n'}
|
||||
payload=${response%$'\n'*}
|
||||
printf '%s' "$payload" >"$output"
|
||||
[[ $status =~ ^2[0-9][0-9]$ ]] || {
|
||||
echo "acceptance control API failed: method=$method path=$path status=$status" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
current_manifest=$temporary_root/current.json
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" "$remote_release_helper status" >"$current_manifest"
|
||||
node "$cluster_root/scripts/release-manifest.mjs" validate "$current_manifest" >/dev/null
|
||||
[[ $(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" sourceSha) == "$release_sha" &&
|
||||
$(node "$cluster_root/scripts/release-manifest.mjs" get "$current_manifest" images.api) == "$api_image" ]] || {
|
||||
echo 'production release changed after acceptance' >&2
|
||||
exit 1
|
||||
}
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
[[ $(remote_kubectl get deployment "$deployment" -n "$namespace" \
|
||||
-o 'jsonpath={.spec.template.spec.containers[0].image}') == "$api_image" ]]
|
||||
done
|
||||
|
||||
mode_file=$temporary_root/mode.json
|
||||
run_file=$temporary_root/run.json
|
||||
promotion_file=$temporary_root/promotion.json
|
||||
admin_request GET /api/admin/system/acceptance/traffic-mode '' "$mode_file"
|
||||
admin_request GET "/api/admin/system/acceptance/runs/$run_id" '' "$run_file"
|
||||
[[ $(jq -r '.mode' "$mode_file") == validation &&
|
||||
$(jq -r '.runId' "$mode_file") == "$run_id" &&
|
||||
$(jq -r '.releaseSha' "$mode_file") == "$release_sha" &&
|
||||
$(jq -r '.apiImageDigest' "$mode_file") == "$api_digest" &&
|
||||
$(jq -r '.status' "$run_file") == passed ]] || {
|
||||
echo 'production acceptance Run or traffic-mode CAS changed before promotion' >&2
|
||||
exit 1
|
||||
}
|
||||
config_hash=$(jq -r '.config.validationConfigHash' "$run_file")
|
||||
profiles=$(jq -c '.config.validationCapacityProfiles' "$run_file")
|
||||
[[ $config_hash =~ ^[0-9a-f]{64}$ && $(jq 'type == "array" and length > 0' <<<"$profiles") == true ]]
|
||||
[[ $(remote_kubectl get configmap easyai-gateway-release -n "$namespace" \
|
||||
-o 'jsonpath={.data.configHash}') == "$config_hash" ]] || {
|
||||
echo 'production configuration changed after acceptance' >&2
|
||||
exit 1
|
||||
}
|
||||
body=$(jq -cn \
|
||||
--argjson revision "$(jq -r '.revision' "$mode_file")" \
|
||||
--arg releaseSha "$release_sha" \
|
||||
--arg apiDigest "$api_digest" \
|
||||
--arg configHash "$config_hash" \
|
||||
--argjson capacityProfiles "$profiles" \
|
||||
'{
|
||||
revision:$revision,
|
||||
releaseSha:$releaseSha,
|
||||
apiImageDigest:$apiDigest,
|
||||
workerImageDigest:$apiDigest,
|
||||
configHash:$configHash,
|
||||
capacityProfiles:$capacityProfiles
|
||||
}')
|
||||
admin_request POST "/api/admin/system/acceptance/runs/$run_id/promote" "$body" "$promotion_file"
|
||||
[[ $(jq -r '.mode' "$promotion_file") == live &&
|
||||
$(jq -r '.runId' "$promotion_file") == "$run_id" ]] || {
|
||||
echo 'live promotion CAS did not complete' >&2
|
||||
exit 1
|
||||
}
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" mark-promoted \
|
||||
--input "$report" --output "$report" >/dev/null
|
||||
echo "production_acceptance_promotion=PASS run_id=$run_id release=$release_sha traffic_mode=live"
|
||||
exec "$script_dir/monitor-production-release.sh" \
|
||||
--run-id "$run_id" --release-manifest "$release_manifest"
|
||||
Reference in New Issue
Block a user