feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
Executable
+536
@@ -0,0 +1,536 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
repository_root=$(cd "$script_dir/../.." && pwd)
|
||||
private_root="$repository_root/.local-secrets/acceptance"
|
||||
state_root="$private_root/state"
|
||||
context=k3d-easyai-acceptance-local
|
||||
namespace=easyai
|
||||
runtime_path_file="$state_root/current-runtime-path"
|
||||
snapshot="$state_root/snapshot.json"
|
||||
load_binary="$state_root/easyai-ai-gateway-acceptance-load"
|
||||
active_load_pid=
|
||||
netem_active=false
|
||||
current_phase=initialization
|
||||
stable_profile=
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/acceptance/run-local-acceptance.sh quick
|
||||
scripts/acceptance/run-local-acceptance.sh full --release-manifest dist/releases/<SHA>.json
|
||||
scripts/acceptance/run-local-acceptance.sh artifact-smoke --release-manifest dist/releases/<SHA>.json
|
||||
|
||||
`full` executes P24/P28/P32 three times, the fault matrix, autoscaling/drain,
|
||||
80% soak, 120% overload, and exact linux/amd64 artifact smoke. The load process
|
||||
runs outside K3s and splits requests 50/50 across both TLS entrances.
|
||||
EOF
|
||||
}
|
||||
|
||||
private_file() {
|
||||
local path=$1 mode
|
||||
[[ -f $path && ! -L $path ]] || return 1
|
||||
if [[ $(uname -s) == Darwin ]]; then
|
||||
mode=$(stat -f '%Lp' "$path")
|
||||
else
|
||||
mode=$(stat -c '%a' "$path")
|
||||
fi
|
||||
[[ $mode == 600 ]]
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT
|
||||
if [[ -n $active_load_pid ]]; then
|
||||
kill "$active_load_pid" >/dev/null 2>&1 || true
|
||||
wait "$active_load_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $netem_active == true ]]; then
|
||||
"$script_dir/network-fault.sh" reset >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then
|
||||
restore_profile_best_effort "${stable_profile:-P24}"
|
||||
local partial_args=(
|
||||
"$script_dir/report.mjs" build-local-partial
|
||||
--runtime "$runtime" \
|
||||
--snapshot "$snapshot" \
|
||||
--reports "$report_root" \
|
||||
--failure-phase "$current_phase" \
|
||||
--output "$report_root/acceptance-report.partial.json"
|
||||
)
|
||||
if [[ -n $stable_profile ]]; then
|
||||
partial_args+=(--certified-profile "$stable_profile")
|
||||
fi
|
||||
if node "${partial_args[@]}" >/dev/null 2>&1; then
|
||||
echo "local_acceptance=FAILED phase=$current_phase partial_report=$report_root/acceptance-report.partial.json" >&2
|
||||
else
|
||||
echo "local_acceptance=FAILED phase=$current_phase partial_report=unavailable" >&2
|
||||
fi
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'current_phase=signal_interrupted; exit 130' HUP INT TERM
|
||||
|
||||
require_local_cluster() {
|
||||
command -v kubectl >/dev/null 2>&1
|
||||
command -v jq >/dev/null 2>&1
|
||||
command -v node >/dev/null 2>&1
|
||||
[[ $(kubectl config get-contexts "$context" -o name) == "$context" ]]
|
||||
private_file "$runtime_path_file" && private_file "$snapshot"
|
||||
runtime=$(<"$runtime_path_file")
|
||||
private_file "$runtime" || {
|
||||
echo 'local acceptance runtime file is missing or not mode 0600' >&2
|
||||
exit 1
|
||||
}
|
||||
jq -e '.schemaVersion == "acceptance-runtime/v1"' "$runtime" >/dev/null
|
||||
run_id=$(jq -r '.runId' "$runtime")
|
||||
release_sha=$(jq -r '.releaseSha' "$runtime")
|
||||
report_root="$repository_root/dist/acceptance/local/$run_id"
|
||||
install -d -m 0700 "$report_root"
|
||||
(
|
||||
cd "$repository_root/apps/api"
|
||||
env -u AI_GATEWAY_TEST_DATABASE_URL go build -trimpath \
|
||||
-o "$load_binary" ./cmd/acceptance-load
|
||||
)
|
||||
}
|
||||
|
||||
database_query() {
|
||||
local sql=$1 primary
|
||||
primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
|
||||
-o 'jsonpath={.status.currentPrimary}')
|
||||
kubectl --context "$context" -n "$namespace" exec "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql"
|
||||
}
|
||||
|
||||
load_environment() {
|
||||
acceptance_api_keys=$(jq -r '.apiKeys | join(",")' "$runtime")
|
||||
acceptance_run_token=$(jq -r '.runToken' "$runtime")
|
||||
acceptance_gemini_model=$(jq -r '.geminiModel' "$runtime")
|
||||
acceptance_video_model=$(jq -r '.videoModel' "$runtime")
|
||||
acceptance_emulator_url=$(jq -r '.emulatorBaseUrl' "$runtime")
|
||||
}
|
||||
|
||||
run_load() {
|
||||
local profile=$1 report_path=$2
|
||||
shift 2
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAYS='https://127.0.0.1:18443,https://127.0.0.1:19443' \
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=gateway.easyai.local \
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE="$state_root/ca.crt" \
|
||||
AI_GATEWAY_ACCEPTANCE_EMULATOR_URL="$acceptance_emulator_url" \
|
||||
AI_GATEWAY_ACCEPTANCE_API_KEYS="$acceptance_api_keys" \
|
||||
AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \
|
||||
AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$acceptance_run_token" \
|
||||
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
|
||||
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
|
||||
"$load_binary" -profile "$profile" -report "$report_path" "$@" \
|
||||
>"$report_path.stdout"
|
||||
chmod 0600 "$report_path" "$report_path.stdout"
|
||||
jq -e '.schemaVersion == "acceptance-load-report/v1" and .secretSafe == true and .passed == true' \
|
||||
"$report_path" >/dev/null
|
||||
}
|
||||
|
||||
sample_resources() {
|
||||
local output=$1 stop_file=$2
|
||||
printf 'timestamp,scope,name,cpu,memory\n' >"$output"
|
||||
while [[ ! -e $stop_file ]]; do
|
||||
kubectl --context "$context" top nodes --no-headers 2>/dev/null |
|
||||
awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
||||
'{print timestamp ",node," $1 "," $3 "," $5}' >>"$output" || true
|
||||
kubectl --context "$context" -n "$namespace" top pods --no-headers 2>/dev/null |
|
||||
awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
||||
'{print timestamp ",pod," $1 "," $2 "," $3}' >>"$output" || true
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
verify_hard_gates() {
|
||||
local unhealthy ready sync_state connections max_connections queue duplicate_remote duplicate_billing
|
||||
ready=$(kubectl --context "$context" get nodes -o json |
|
||||
jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length')
|
||||
[[ $ready == 3 ]]
|
||||
[[ $(kubectl --context "$context" get nodes -o json |
|
||||
jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]]
|
||||
[[ $(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
|
||||
-o 'jsonpath={.status.readyInstances}') == 2 ]]
|
||||
sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
|
||||
[[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]]
|
||||
unhealthy=$(kubectl --context "$context" -n "$namespace" get pods -o json |
|
||||
jq '[.items[]
|
||||
| select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
|
||||
.metadata.labels["app.kubernetes.io/name"] == "easyai-worker")
|
||||
| .status.containerStatuses[]?
|
||||
| select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or
|
||||
.state.terminated.reason == "OOMKilled")] | length')
|
||||
[[ $unhealthy == 0 ]]
|
||||
while read -r _node _cpu _cpu_percent _memory memory_percent; do
|
||||
memory_percent=${memory_percent%\%}
|
||||
[[ $memory_percent =~ ^[0-9]+$ ]]
|
||||
(( memory_percent < 80 ))
|
||||
done < <(kubectl --context "$context" top nodes --no-headers)
|
||||
while read -r _pod _cpu memory; do
|
||||
case $memory in
|
||||
*Gi) memory=$(awk -v value="${memory%Gi}" 'BEGIN {printf "%.0f", value*1024}') ;;
|
||||
*Mi) memory=${memory%Mi} ;;
|
||||
*Ki) memory=$(awk -v value="${memory%Ki}" 'BEGIN {printf "%.0f", value/1024}') ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
(( memory < 1536 ))
|
||||
done < <(kubectl --context "$context" -n "$namespace" top pods \
|
||||
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers)
|
||||
connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
|
||||
max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
|
||||
(( connections < 150 && connections * 4 < max_connections * 3 ))
|
||||
queue=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
|
||||
[[ $queue == 0:0 ]]
|
||||
duplicate_remote=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) d;")
|
||||
duplicate_billing=$(database_query "SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) d;")
|
||||
[[ $duplicate_remote == 0 && $duplicate_billing == 0 ]]
|
||||
kubectl --context "$context" -n "$namespace" exec deployment/easyai-acceptance-callback-collector -- \
|
||||
wget -qO- http://127.0.0.1:8091/report |
|
||||
jq -e '.duplicates == 0 and .invalid == 0' >/dev/null
|
||||
}
|
||||
|
||||
apply_profile() {
|
||||
local profile=$1 slots pool
|
||||
case $profile in
|
||||
P24) slots=24; pool=32 ;;
|
||||
P28) slots=28; pool=36 ;;
|
||||
P32) slots=32; pool=40 ;;
|
||||
*) return 64 ;;
|
||||
esac
|
||||
local global=$((slots * 2))
|
||||
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
||||
--type=merge -p "$(jq -cn \
|
||||
--arg slots "$slots" --arg global "$global" \
|
||||
'{data:{
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global,
|
||||
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global
|
||||
}}')" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" scale \
|
||||
deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong --replicas=1 >/dev/null
|
||||
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
||||
done
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-capacity-controller; do
|
||||
kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
||||
done
|
||||
}
|
||||
|
||||
restore_profile_best_effort() {
|
||||
local profile=$1 slots pool global
|
||||
case $profile in
|
||||
P24) slots=24; pool=32 ;;
|
||||
P28) slots=28; pool=36 ;;
|
||||
P32) slots=32; pool=40 ;;
|
||||
*) return ;;
|
||||
esac
|
||||
global=$((slots * 2))
|
||||
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
||||
--type=merge -p "$(jq -cn \
|
||||
--arg slots "$slots" --arg global "$global" \
|
||||
'{data:{
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global,
|
||||
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global
|
||||
}}')" >/dev/null 2>&1 || true
|
||||
kubectl --context "$context" -n "$namespace" scale \
|
||||
deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong \
|
||||
--replicas=1 >/dev/null 2>&1 || true
|
||||
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
|
||||
AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null 2>&1 || true
|
||||
done
|
||||
}
|
||||
|
||||
run_profile_round() {
|
||||
local profile=$1 repetition=$2 prefix
|
||||
prefix="$report_root/${profile,,}-$repetition"
|
||||
local stop_file="$prefix.resources.stop"
|
||||
sample_resources "$prefix.resources.csv" "$stop_file" &
|
||||
local sampler=$!
|
||||
run_load gemini-baseline "$prefix-gemini-baseline.json"
|
||||
run_load gemini-large "$prefix-gemini-large.json"
|
||||
run_load gemini-peak "$prefix-gemini-peak.json"
|
||||
run_load video-throughput "$prefix-video-throughput.json"
|
||||
run_recovery "$prefix-video-recovery.json"
|
||||
touch "$stop_file"
|
||||
wait "$sampler"
|
||||
verify_hard_gates
|
||||
}
|
||||
|
||||
wait_for_recovery_owner() {
|
||||
local deadline=$((SECONDS + 120)) owner
|
||||
while (( SECONDS < deadline )); do
|
||||
owner=$(database_query "
|
||||
SELECT task.id::text||','||worker.pod_name||','||worker.site
|
||||
FROM gateway_tasks task
|
||||
JOIN gateway_worker_instances worker ON worker.instance_id=task.locked_by
|
||||
WHERE task.acceptance_run_id='$run_id'::uuid
|
||||
AND task.status='running'
|
||||
AND task.request::text LIKE '%acceptance-long-recovery%'
|
||||
AND worker.status='active'
|
||||
ORDER BY task.updated_at
|
||||
LIMIT 1;")
|
||||
[[ -z $owner ]] || { printf '%s\n' "$owner"; return; }
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
run_recovery() {
|
||||
local report_path=$1 owner task_id pod site
|
||||
run_load video-recovery "$report_path" &
|
||||
active_load_pid=$!
|
||||
owner=$(wait_for_recovery_owner)
|
||||
IFS=',' read -r task_id pod site <<<"$owner"
|
||||
[[ $task_id =~ ^[0-9a-f-]{36}$ && $pod == easyai-worker-"$site"-* ]]
|
||||
kubectl --context "$context" -n "$namespace" delete pod "$pod" \
|
||||
--force --grace-period=0 --wait=false >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status \
|
||||
deployment/easyai-worker-"$site" --timeout=10m
|
||||
wait "$active_load_pid"
|
||||
active_load_pid=
|
||||
}
|
||||
|
||||
run_fault_matrix() {
|
||||
local duration=${AI_GATEWAY_LOCAL_WEAK_LINK_DURATION:-5m}
|
||||
"$script_dir/network-fault.sh" weak-link
|
||||
netem_active=true
|
||||
run_load mixed-soak "$report_root/fault-weak-link.json" \
|
||||
-duration "$duration" -image-rate 1 -video-rate 1
|
||||
"$script_dir/network-fault.sh" reset
|
||||
netem_active=false
|
||||
|
||||
run_load video-throughput "$report_root/fault-upstream-outage.json" &
|
||||
active_load_pid=$!
|
||||
sleep 5
|
||||
"$script_dir/network-fault.sh" upstream-outage
|
||||
netem_active=true
|
||||
sleep 10
|
||||
"$script_dir/network-fault.sh" reset
|
||||
netem_active=false
|
||||
wait "$active_load_pid"
|
||||
active_load_pid=
|
||||
|
||||
run_load video-recovery "$report_root/fault-database-outage.json" &
|
||||
active_load_pid=$!
|
||||
sleep 10
|
||||
"$script_dir/network-fault.sh" database-outage hongkong
|
||||
netem_active=true
|
||||
sleep 30
|
||||
"$script_dir/network-fault.sh" reset
|
||||
netem_active=false
|
||||
wait "$active_load_pid"
|
||||
active_load_pid=
|
||||
|
||||
local leader
|
||||
leader=$(kubectl --context "$context" -n "$namespace" get pods \
|
||||
-l app.kubernetes.io/name=easyai-capacity-controller -o name |
|
||||
while read -r pod; do
|
||||
status=$(kubectl --context "$context" -n "$namespace" exec "$pod" -- \
|
||||
wget -qO- http://127.0.0.1:8088/status)
|
||||
[[ $(jq -r '.leader' <<<"$status") == true ]] && { printf '%s\n' "${pod#pod/}"; break; }
|
||||
done)
|
||||
[[ -n $leader ]]
|
||||
kubectl --context "$context" -n "$namespace" delete pod "$leader" --wait=false >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status \
|
||||
deployment/easyai-capacity-controller --timeout=5m
|
||||
verify_hard_gates
|
||||
}
|
||||
|
||||
run_autoscaling() {
|
||||
local profile=$1 slots
|
||||
slots=${profile#P}
|
||||
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
||||
--type=merge -p "$(jq -cn --arg global "$((slots * 4))" \
|
||||
'{data:{
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"true",
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:"1",
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:"1",
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:"2",
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:"2",
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global
|
||||
}}')" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout restart \
|
||||
deployment/easyai-capacity-controller >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status \
|
||||
deployment/easyai-capacity-controller --timeout=5m
|
||||
run_load video-throughput "$report_root/autoscaling-load.json" &
|
||||
active_load_pid=$!
|
||||
local deadline=$((SECONDS + 420)) total=2
|
||||
while (( SECONDS < deadline )); do
|
||||
total=$(kubectl --context "$context" -n "$namespace" get deployment \
|
||||
easyai-worker-ningbo easyai-worker-hongkong \
|
||||
-o json | jq '[.items[].spec.replicas] | add')
|
||||
(( total >= 3 )) && break
|
||||
sleep 5
|
||||
done
|
||||
(( total >= 3 ))
|
||||
wait "$active_load_pid"
|
||||
active_load_pid=
|
||||
deadline=$((SECONDS + 780))
|
||||
while (( SECONDS < deadline )); do
|
||||
total=$(kubectl --context "$context" -n "$namespace" get deployment \
|
||||
easyai-worker-ningbo easyai-worker-hongkong \
|
||||
-o json | jq '[.items[].spec.replicas] | add')
|
||||
(( total == 2 )) && break
|
||||
sleep 10
|
||||
done
|
||||
(( total == 2 ))
|
||||
verify_hard_gates
|
||||
}
|
||||
|
||||
artifact_smoke() {
|
||||
local manifest=$1
|
||||
node "$repository_root/scripts/release-manifest.mjs" validate "$manifest" >/dev/null
|
||||
local manifest_sha api_image web_image api_digest artifact_report
|
||||
manifest_sha=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" sourceSha)
|
||||
api_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api)
|
||||
web_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.web)
|
||||
api_digest=${api_image##*@}
|
||||
[[ $manifest_sha == "$release_sha" && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
docker pull --platform linux/amd64 "$api_image" >/dev/null
|
||||
docker pull --platform linux/amd64 "$web_image" >/dev/null
|
||||
"$private_root/tools/k3d" image import -c easyai-acceptance-local "$api_image" "$web_image"
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo \
|
||||
easyai-worker-hongkong easyai-capacity-controller easyai-acceptance-emulator \
|
||||
easyai-acceptance-callback-collector; do
|
||||
kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \
|
||||
"*=$api_image" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=15m
|
||||
done
|
||||
for deployment in easyai-acceptance-edge-ningbo easyai-acceptance-edge-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \
|
||||
"*=$web_image" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
||||
done
|
||||
kubectl --context "$context" -n "$namespace" delete job easyai-artifact-migrate \
|
||||
--ignore-not-found --wait=true >/dev/null
|
||||
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: easyai-artifact-migrate
|
||||
namespace: easyai
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: migrate
|
||||
image: $api_image
|
||||
command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: easyai-ai-gateway-runtime
|
||||
EOF
|
||||
kubectl --context "$context" -n "$namespace" wait \
|
||||
--for=condition=complete job/easyai-artifact-migrate --timeout=10m >/dev/null
|
||||
run_load simulated-smoke "$report_root/artifact-simulated-smoke.json"
|
||||
verify_hard_gates
|
||||
artifact_report="$report_root/artifact-smoke.json"
|
||||
jq -n \
|
||||
--arg releaseSha "$manifest_sha" \
|
||||
--arg apiImageDigest "$api_digest" \
|
||||
--arg webImageDigest "${web_image##*@}" \
|
||||
'{
|
||||
schemaVersion:"acceptance-artifact-smoke/v1",
|
||||
releaseSha:$releaseSha,
|
||||
apiImageDigest:$apiImageDigest,
|
||||
webImageDigest:$webImageDigest,
|
||||
architecture:"linux/amd64",
|
||||
migrationSmoke:true,
|
||||
startupSmoke:true,
|
||||
mediaSmoke:true,
|
||||
passed:true,
|
||||
secretSafe:true
|
||||
}' >"$artifact_report"
|
||||
chmod 0600 "$artifact_report"
|
||||
}
|
||||
|
||||
quick() {
|
||||
current_phase=quick
|
||||
run_load simulated-smoke "$report_root/quick.json"
|
||||
verify_hard_gates
|
||||
echo "local_acceptance_quick=PASS run_id=$run_id report=$report_root/quick.json"
|
||||
}
|
||||
|
||||
full() {
|
||||
local manifest=$1 profile repetition
|
||||
quick
|
||||
for profile in P24 P28 P32; do
|
||||
current_phase="capacity_${profile,,}_apply"
|
||||
apply_profile "$profile"
|
||||
for repetition in 1 2 3; do
|
||||
current_phase="capacity_${profile,,}_round_$repetition"
|
||||
run_profile_round "$profile" "$repetition"
|
||||
done
|
||||
stable_profile=$profile
|
||||
done
|
||||
current_phase=fault_matrix
|
||||
run_fault_matrix
|
||||
current_phase=autoscaling_and_drain
|
||||
run_autoscaling "$stable_profile"
|
||||
current_phase=certified_soak
|
||||
run_load mixed-soak "$report_root/mixed-soak.json" \
|
||||
-duration "${AI_GATEWAY_LOCAL_SOAK_DURATION:-2h}" \
|
||||
-image-rate "${AI_GATEWAY_LOCAL_CERTIFIED_IMAGE_RATE:-1}" \
|
||||
-video-rate "${AI_GATEWAY_LOCAL_CERTIFIED_VIDEO_RATE:-1}"
|
||||
current_phase=overload_shedding
|
||||
run_load mixed-overload "$report_root/mixed-overload.json" \
|
||||
-duration "${AI_GATEWAY_LOCAL_OVERLOAD_DURATION:-10m}" \
|
||||
-image-rate "${AI_GATEWAY_LOCAL_OVERLOAD_IMAGE_RATE:-2}" \
|
||||
-video-rate "${AI_GATEWAY_LOCAL_OVERLOAD_VIDEO_RATE:-2}"
|
||||
current_phase=amd64_artifact_smoke
|
||||
artifact_smoke "$manifest"
|
||||
current_phase=final_report
|
||||
node "$script_dir/report.mjs" build-local \
|
||||
--runtime "$runtime" \
|
||||
--snapshot "$snapshot" \
|
||||
--reports "$report_root" \
|
||||
--artifact "$report_root/artifact-smoke.json" \
|
||||
--api-digest "$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api | sed 's/.*@//')" \
|
||||
--certified-profile "$stable_profile" \
|
||||
--output "$report_root/acceptance-report.json"
|
||||
echo "local_acceptance_full=PASS run_id=$run_id certified_profile=$stable_profile report=$report_root/acceptance-report.json"
|
||||
}
|
||||
|
||||
command=${1:-}
|
||||
shift || true
|
||||
require_local_cluster
|
||||
load_environment
|
||||
case $command in
|
||||
quick)
|
||||
[[ $# -eq 0 ]] || { usage >&2; exit 64; }
|
||||
quick
|
||||
;;
|
||||
artifact-smoke)
|
||||
[[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
|
||||
artifact_smoke "$2"
|
||||
;;
|
||||
full)
|
||||
[[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
|
||||
full "$2"
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user