wangbo
a95184b5b6
本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
637 lines
26 KiB
Bash
Executable File
637 lines
26 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
repository_root=$(cd "$script_dir/../.." && pwd)
|
|
private_root="$repository_root/.local-secrets/acceptance"
|
|
state_root="$private_root/state"
|
|
context=k3d-easyai-acceptance-local
|
|
namespace=easyai
|
|
cluster_name=easyai-acceptance-local
|
|
runtime_path_file="$state_root/current-runtime-path"
|
|
snapshot="$state_root/snapshot.json"
|
|
load_binary="$state_root/easyai-ai-gateway-acceptance-load"
|
|
identity_file="$state_root/control-plane-identity.json"
|
|
active_load_pid=
|
|
netem_active=false
|
|
current_phase=initialization
|
|
stable_profile=
|
|
failure_gate_id=local_execution_incomplete
|
|
# shellcheck source=scripts/acceptance/local-control-plane.sh
|
|
source "$script_dir/local-control-plane.sh"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
scripts/acceptance/run-local-acceptance.sh quick
|
|
scripts/acceptance/run-local-acceptance.sh full --release-manifest dist/releases/<SHA>.json
|
|
scripts/acceptance/run-local-acceptance.sh artifact-smoke --release-manifest dist/releases/<SHA>.json
|
|
|
|
`full` executes P24/P28/P32 three times, the fault matrix, autoscaling/drain,
|
|
80% soak, 120% overload, and exact linux/amd64 artifact smoke. The load process
|
|
runs outside K3s and splits requests 50/50 across both TLS entrances.
|
|
EOF
|
|
}
|
|
|
|
private_file() {
|
|
local path=$1 mode
|
|
[[ -f $path && ! -L $path ]] || return 1
|
|
if [[ $(uname -s) == Darwin ]]; then
|
|
mode=$(stat -f '%Lp' "$path")
|
|
else
|
|
mode=$(stat -c '%a' "$path")
|
|
fi
|
|
[[ $mode == 600 ]]
|
|
}
|
|
|
|
cleanup() {
|
|
local status=$?
|
|
trap - EXIT
|
|
if [[ -n $active_load_pid ]]; then
|
|
kill "$active_load_pid" >/dev/null 2>&1 || true
|
|
wait "$active_load_pid" >/dev/null 2>&1 || true
|
|
fi
|
|
if [[ $netem_active == true ]]; then
|
|
"$script_dir/network-fault.sh" reset >/dev/null 2>&1 || true
|
|
fi
|
|
if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then
|
|
restore_profile_best_effort "${stable_profile:-P24}"
|
|
if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then
|
|
failure_gate_id=$(jq -r '.id' "$failure_gate_file" 2>/dev/null || printf '%s' "$failure_gate_id")
|
|
fi
|
|
local partial_args=(
|
|
"$script_dir/report.mjs" build-local-partial
|
|
--runtime "$runtime" \
|
|
--snapshot "$snapshot" \
|
|
--reports "$report_root" \
|
|
--failure-phase "$current_phase" \
|
|
--failure-gate "$failure_gate_id" \
|
|
--output "$report_root/acceptance-report.partial.json"
|
|
)
|
|
if [[ -n $stable_profile ]]; then
|
|
partial_args+=(--certified-profile "$stable_profile")
|
|
fi
|
|
if node "${partial_args[@]}" >/dev/null 2>&1; then
|
|
echo "local_acceptance=FAILED phase=$current_phase partial_report=$report_root/acceptance-report.partial.json" >&2
|
|
else
|
|
echo "local_acceptance=FAILED phase=$current_phase partial_report=unavailable" >&2
|
|
fi
|
|
fi
|
|
exit "$status"
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'current_phase=signal_interrupted; exit 130' HUP INT TERM
|
|
|
|
require_local_cluster() {
|
|
command -v kubectl >/dev/null 2>&1
|
|
command -v jq >/dev/null 2>&1
|
|
command -v node >/dev/null 2>&1
|
|
[[ $(kubectl config get-contexts "$context" -o name) == "$context" ]]
|
|
private_file "$runtime_path_file" && private_file "$snapshot"
|
|
runtime=$(<"$runtime_path_file")
|
|
private_file "$runtime" || {
|
|
echo 'local acceptance runtime file is missing or not mode 0600' >&2
|
|
exit 1
|
|
}
|
|
jq -e '.schemaVersion == "acceptance-runtime/v1"' "$runtime" >/dev/null
|
|
run_id=$(jq -r '.runId' "$runtime")
|
|
release_sha=$(jq -r '.releaseSha' "$runtime")
|
|
report_root="$repository_root/dist/acceptance/local/$run_id"
|
|
install -d -m 0700 "$report_root"
|
|
failure_gate_file="$report_root/failure-gate.json"
|
|
(
|
|
cd "$repository_root/apps/api"
|
|
env -u AI_GATEWAY_TEST_DATABASE_URL go build -trimpath \
|
|
-o "$load_binary" ./cmd/acceptance-load
|
|
)
|
|
}
|
|
|
|
record_failure_gate() {
|
|
local id=$1 detail=$2 temporary
|
|
[[ $id =~ ^[a-z0-9_]+$ ]]
|
|
failure_gate_id=$id
|
|
[[ -n ${failure_gate_file:-} ]] || return 0
|
|
if [[ ! -e $failure_gate_file ]]; then
|
|
temporary="${failure_gate_file}.tmp.$$"
|
|
jq -n --arg id "$id" --arg detail "$detail" \
|
|
'{schemaVersion:"acceptance-failure-gate/v1",id:$id,detail:$detail}' >"$temporary"
|
|
chmod 0600 "$temporary"
|
|
mv "$temporary" "$failure_gate_file"
|
|
fi
|
|
}
|
|
|
|
fail_gate() {
|
|
local id=$1 detail=$2
|
|
record_failure_gate "$id" "$detail"
|
|
echo "gate=$id $detail" >&2
|
|
return 1
|
|
}
|
|
|
|
database_query() {
|
|
local sql=$1 primary
|
|
primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
|
|
-o 'jsonpath={.status.currentPrimary}')
|
|
kubectl --context "$context" -n "$namespace" exec "$primary" -c postgres -- \
|
|
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -At -c "$sql"
|
|
}
|
|
|
|
load_environment() {
|
|
acceptance_api_keys=$(jq -r '.apiKeys | join(",")' "$runtime")
|
|
acceptance_run_token=$(jq -r '.runToken' "$runtime")
|
|
acceptance_gemini_model=$(jq -r '.geminiModel' "$runtime")
|
|
acceptance_video_model=$(jq -r '.videoModel' "$runtime")
|
|
acceptance_emulator_url=$(jq -r '.emulatorBaseUrl' "$runtime")
|
|
}
|
|
|
|
run_load() {
|
|
local profile=$1 report_path=$2
|
|
shift 2
|
|
local stdout_path="$report_path.stdout" stdout_temporary="${report_path}.stdout.tmp.$$"
|
|
local load_pid load_status=0 unavailable_samples=0 infrastructure_failure=false
|
|
[[ ! -e $report_path && ! -e $stdout_path && ! -e $stdout_temporary ]] ||
|
|
fail_gate acceptance_report_exists "refusing to overwrite an existing load report"
|
|
verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
|
|
fail_gate local_control_plane_restarted "local K3s server identity changed before load"
|
|
AI_GATEWAY_ACCEPTANCE_GATEWAYS='https://127.0.0.1:18443,https://127.0.0.1:19443' \
|
|
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=gateway.easyai.local \
|
|
AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE="$state_root/ca.crt" \
|
|
AI_GATEWAY_ACCEPTANCE_EMULATOR_URL="$acceptance_emulator_url" \
|
|
AI_GATEWAY_ACCEPTANCE_API_KEYS="$acceptance_api_keys" \
|
|
AI_GATEWAY_ACCEPTANCE_RUN_ID="$run_id" \
|
|
AI_GATEWAY_ACCEPTANCE_RUN_TOKEN="$acceptance_run_token" \
|
|
AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \
|
|
AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \
|
|
"$load_binary" -profile "$profile" -report "$report_path" "$@" \
|
|
>"$stdout_temporary" &
|
|
load_pid=$!
|
|
active_load_pid=$load_pid
|
|
while kill -0 "$load_pid" >/dev/null 2>&1; do
|
|
if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
|
|
record_failure_gate local_control_plane_restarted "local K3s server restarted during load"
|
|
infrastructure_failure=true
|
|
kill "$load_pid" >/dev/null 2>&1 || true
|
|
break
|
|
fi
|
|
if verify_local_apiserver_ready "$context"; then
|
|
unavailable_samples=0
|
|
else
|
|
unavailable_samples=$((unavailable_samples + 1))
|
|
if ((unavailable_samples >= 3)); then
|
|
record_failure_gate local_control_plane_unavailable "Kubernetes API readiness failed for three consecutive samples"
|
|
infrastructure_failure=true
|
|
kill "$load_pid" >/dev/null 2>&1 || true
|
|
break
|
|
fi
|
|
fi
|
|
sleep 2
|
|
done
|
|
if wait "$load_pid"; then
|
|
load_status=0
|
|
else
|
|
load_status=$?
|
|
fi
|
|
active_load_pid=
|
|
if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then
|
|
record_failure_gate local_control_plane_restarted "local K3s server identity changed at load completion"
|
|
infrastructure_failure=true
|
|
fi
|
|
[[ ! -e $stdout_path ]] || fail_gate acceptance_report_exists "load stdout report already exists"
|
|
mv "$stdout_temporary" "$stdout_path"
|
|
chmod 0600 "$stdout_path"
|
|
if [[ -f $report_path && ! -L $report_path ]]; then
|
|
chmod 0600 "$report_path"
|
|
fi
|
|
if [[ $infrastructure_failure == true ]]; then
|
|
return 1
|
|
fi
|
|
((load_status == 0)) || return "$load_status"
|
|
jq -e '.schemaVersion == "acceptance-load-report/v1" and .secretSafe == true and .passed == true' \
|
|
"$report_path" >/dev/null
|
|
}
|
|
|
|
sample_resources() {
|
|
local output=$1 stop_file=$2
|
|
printf 'timestamp,scope,name,cpu,memory\n' >"$output"
|
|
while [[ ! -e $stop_file ]]; do
|
|
kubectl --context "$context" top nodes --no-headers 2>/dev/null |
|
|
awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
|
'{print timestamp ",node," $1 "," $3 "," $5}' >>"$output" || true
|
|
kubectl --context "$context" -n "$namespace" top pods --no-headers 2>/dev/null |
|
|
awk -v timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
|
'{print timestamp ",pod," $1 "," $2 "," $3}' >>"$output" || true
|
|
sleep 5
|
|
done
|
|
}
|
|
|
|
verify_hard_gates() {
|
|
local unhealthy ready sync_state connections max_connections queue duplicate_remote duplicate_billing
|
|
verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
|
|
fail_gate local_control_plane_restarted "local K3s server identity changed"
|
|
verify_local_apiserver_ready "$context" ||
|
|
fail_gate local_control_plane_unavailable "Kubernetes API readiness failed"
|
|
verify_local_node_capacity "$context" ||
|
|
fail_gate local_node_capacity_mismatch "node allocatable resources drifted from the acceptance envelope"
|
|
verify_local_etcd_runtime_logs "$cluster_name" "$identity_file" ||
|
|
fail_gate local_etcd_latency "etcd emitted a severe latency or timeout signal during the run"
|
|
ready=$(kubectl --context "$context" get nodes -o json |
|
|
jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length')
|
|
[[ $ready == 3 ]] || fail_gate local_nodes_not_ready "not all three local K3s nodes are Ready"
|
|
[[ $(kubectl --context "$context" get nodes -o json |
|
|
jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] ||
|
|
fail_gate local_node_memory_pressure "a local K3s node reports MemoryPressure"
|
|
[[ $(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \
|
|
-o 'jsonpath={.status.readyInstances}') == 2 ]] ||
|
|
fail_gate local_postgres_not_ready "local PostgreSQL is not 2/2 Ready"
|
|
sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;")
|
|
[[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] ||
|
|
fail_gate local_postgres_not_synchronous "local PostgreSQL lost its synchronous replica"
|
|
unhealthy=$(kubectl --context "$context" -n "$namespace" get pods -o json |
|
|
jq '[.items[]
|
|
| select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or
|
|
.metadata.labels["app.kubernetes.io/name"] == "easyai-worker")
|
|
| .status.containerStatuses[]?
|
|
| select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or
|
|
.state.terminated.reason == "OOMKilled")] | length')
|
|
[[ $unhealthy == 0 ]] || fail_gate gateway_pod_restarted "an API or Worker container restarted or was OOMKilled"
|
|
while read -r _node _cpu _cpu_percent _memory memory_percent; do
|
|
memory_percent=${memory_percent%\%}
|
|
[[ $memory_percent =~ ^[0-9]+$ ]]
|
|
(( memory_percent < 80 )) || fail_gate local_node_memory_target "local node memory reached the 80 percent target"
|
|
done < <(kubectl --context "$context" top nodes --no-headers)
|
|
while read -r _pod _cpu memory; do
|
|
case $memory in
|
|
*Gi) memory=$(awk -v value="${memory%Gi}" 'BEGIN {printf "%.0f", value*1024}') ;;
|
|
*Mi) memory=${memory%Mi} ;;
|
|
*Ki) memory=$(awk -v value="${memory%Ki}" 'BEGIN {printf "%.0f", value/1024}') ;;
|
|
*) return 1 ;;
|
|
esac
|
|
(( memory < 1536 )) || fail_gate gateway_pod_memory_limit "an API or Worker Pod reached 1.5 GiB RSS"
|
|
done < <(kubectl --context "$context" -n "$namespace" top pods \
|
|
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers)
|
|
connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';")
|
|
max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';")
|
|
(( connections < 150 && connections * 4 < max_connections * 3 )) ||
|
|
fail_gate postgres_connection_budget "PostgreSQL client sessions exceeded the acceptance budget"
|
|
queue=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
|
|
[[ $queue == 0:0 ]] || fail_gate acceptance_queue_not_drained "acceptance queue did not return to zero"
|
|
duplicate_remote=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) d;")
|
|
duplicate_billing=$(database_query "SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) d;")
|
|
[[ $duplicate_remote == 0 ]] || fail_gate duplicate_upstream_submission "duplicate remote task IDs were detected"
|
|
[[ $duplicate_billing == 0 ]] || fail_gate duplicate_billing "duplicate billing transactions were detected"
|
|
kubectl --context "$context" -n "$namespace" exec deployment/easyai-acceptance-callback-collector -- \
|
|
wget -qO- http://127.0.0.1:8091/report |
|
|
jq -e '.duplicates == 0 and .invalid == 0' >/dev/null ||
|
|
fail_gate callback_validation_failed "callback collector detected duplicate or invalid callbacks"
|
|
}
|
|
|
|
apply_profile() {
|
|
local profile=$1 slots pool
|
|
case $profile in
|
|
P24) slots=24; pool=32 ;;
|
|
P28) slots=28; pool=36 ;;
|
|
P32) slots=32; pool=40 ;;
|
|
*) return 64 ;;
|
|
esac
|
|
local global=$((slots * 2))
|
|
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
|
--type=merge -p "$(jq -cn \
|
|
--arg slots "$slots" --arg global "$global" \
|
|
'{data:{
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global,
|
|
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global
|
|
}}')" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" scale \
|
|
deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong --replicas=1 >/dev/null
|
|
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
|
kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
|
|
AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
|
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
|
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
|
|
AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
|
done
|
|
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-capacity-controller; do
|
|
kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
|
done
|
|
}
|
|
|
|
restore_profile_best_effort() {
|
|
local profile=$1 slots pool global
|
|
case $profile in
|
|
P24) slots=24; pool=32 ;;
|
|
P28) slots=28; pool=36 ;;
|
|
P32) slots=32; pool=40 ;;
|
|
*) return ;;
|
|
esac
|
|
global=$((slots * 2))
|
|
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
|
--type=merge -p "$(jq -cn \
|
|
--arg slots "$slots" --arg global "$global" \
|
|
'{data:{
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
|
|
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global,
|
|
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$global
|
|
}}')" >/dev/null 2>&1 || true
|
|
kubectl --context "$context" -n "$namespace" scale \
|
|
deployment/easyai-worker-ningbo deployment/easyai-worker-hongkong \
|
|
--replicas=1 >/dev/null 2>&1 || true
|
|
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
|
kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
|
|
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \
|
|
AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \
|
|
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \
|
|
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \
|
|
AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
run_profile_round() {
|
|
local profile=$1 repetition=$2 prefix
|
|
prefix="$report_root/${profile,,}-$repetition"
|
|
local stop_file="$prefix.resources.stop"
|
|
sample_resources "$prefix.resources.csv" "$stop_file" &
|
|
local sampler=$!
|
|
run_load gemini-baseline "$prefix-gemini-baseline.json"
|
|
run_load gemini-large "$prefix-gemini-large.json"
|
|
run_load gemini-peak "$prefix-gemini-peak.json"
|
|
run_load video-throughput "$prefix-video-throughput.json"
|
|
run_recovery "$prefix-video-recovery.json"
|
|
touch "$stop_file"
|
|
wait "$sampler"
|
|
verify_hard_gates
|
|
}
|
|
|
|
wait_for_recovery_owner() {
|
|
local deadline=$((SECONDS + 120)) owner
|
|
while (( SECONDS < deadline )); do
|
|
owner=$(database_query "
|
|
SELECT task.id::text||','||worker.pod_name||','||worker.site
|
|
FROM gateway_tasks task
|
|
JOIN gateway_worker_instances worker ON worker.instance_id=task.locked_by
|
|
WHERE task.acceptance_run_id='$run_id'::uuid
|
|
AND task.status='running'
|
|
AND task.request::text LIKE '%acceptance-long-recovery%'
|
|
AND worker.status='active'
|
|
ORDER BY task.updated_at
|
|
LIMIT 1;")
|
|
[[ -z $owner ]] || { printf '%s\n' "$owner"; return; }
|
|
sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
run_recovery() {
|
|
local report_path=$1 owner task_id pod site
|
|
run_load video-recovery "$report_path" &
|
|
active_load_pid=$!
|
|
owner=$(wait_for_recovery_owner)
|
|
IFS=',' read -r task_id pod site <<<"$owner"
|
|
[[ $task_id =~ ^[0-9a-f-]{36}$ && $pod == easyai-worker-"$site"-* ]]
|
|
kubectl --context "$context" -n "$namespace" delete pod "$pod" \
|
|
--force --grace-period=0 --wait=false >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout status \
|
|
deployment/easyai-worker-"$site" --timeout=10m
|
|
wait "$active_load_pid"
|
|
active_load_pid=
|
|
}
|
|
|
|
run_fault_matrix() {
|
|
local duration=${AI_GATEWAY_LOCAL_WEAK_LINK_DURATION:-5m}
|
|
"$script_dir/network-fault.sh" weak-link
|
|
netem_active=true
|
|
run_load mixed-soak "$report_root/fault-weak-link.json" \
|
|
-duration "$duration" -image-rate 1 -video-rate 1
|
|
"$script_dir/network-fault.sh" reset
|
|
netem_active=false
|
|
|
|
run_load video-throughput "$report_root/fault-upstream-outage.json" &
|
|
active_load_pid=$!
|
|
sleep 5
|
|
"$script_dir/network-fault.sh" upstream-outage
|
|
netem_active=true
|
|
sleep 10
|
|
"$script_dir/network-fault.sh" reset
|
|
netem_active=false
|
|
wait "$active_load_pid"
|
|
active_load_pid=
|
|
|
|
run_load video-recovery "$report_root/fault-database-outage.json" &
|
|
active_load_pid=$!
|
|
sleep 10
|
|
"$script_dir/network-fault.sh" database-outage hongkong
|
|
netem_active=true
|
|
sleep 30
|
|
"$script_dir/network-fault.sh" reset
|
|
netem_active=false
|
|
wait "$active_load_pid"
|
|
active_load_pid=
|
|
|
|
local leader
|
|
leader=$(kubectl --context "$context" -n "$namespace" get pods \
|
|
-l app.kubernetes.io/name=easyai-capacity-controller -o name |
|
|
while read -r pod; do
|
|
status=$(kubectl --context "$context" -n "$namespace" exec "$pod" -- \
|
|
wget -qO- http://127.0.0.1:8088/status)
|
|
[[ $(jq -r '.leader' <<<"$status") == true ]] && { printf '%s\n' "${pod#pod/}"; break; }
|
|
done)
|
|
[[ -n $leader ]]
|
|
kubectl --context "$context" -n "$namespace" delete pod "$leader" --wait=false >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout status \
|
|
deployment/easyai-capacity-controller --timeout=5m
|
|
verify_hard_gates
|
|
}
|
|
|
|
run_autoscaling() {
|
|
local profile=$1 slots
|
|
slots=${profile#P}
|
|
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
|
--type=merge -p "$(jq -cn --arg global "$((slots * 4))" \
|
|
'{data:{
|
|
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"true",
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:"1",
|
|
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:"1",
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:"2",
|
|
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:"2",
|
|
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global
|
|
}}')" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout restart \
|
|
deployment/easyai-capacity-controller >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout status \
|
|
deployment/easyai-capacity-controller --timeout=5m
|
|
run_load video-throughput "$report_root/autoscaling-load.json" &
|
|
active_load_pid=$!
|
|
local deadline=$((SECONDS + 420)) total=2
|
|
while (( SECONDS < deadline )); do
|
|
total=$(kubectl --context "$context" -n "$namespace" get deployment \
|
|
easyai-worker-ningbo easyai-worker-hongkong \
|
|
-o json | jq '[.items[].spec.replicas] | add')
|
|
(( total >= 3 )) && break
|
|
sleep 5
|
|
done
|
|
(( total >= 3 ))
|
|
wait "$active_load_pid"
|
|
active_load_pid=
|
|
deadline=$((SECONDS + 780))
|
|
while (( SECONDS < deadline )); do
|
|
total=$(kubectl --context "$context" -n "$namespace" get deployment \
|
|
easyai-worker-ningbo easyai-worker-hongkong \
|
|
-o json | jq '[.items[].spec.replicas] | add')
|
|
(( total == 2 )) && break
|
|
sleep 10
|
|
done
|
|
(( total == 2 ))
|
|
verify_hard_gates
|
|
}
|
|
|
|
artifact_smoke() {
|
|
local manifest=$1
|
|
node "$repository_root/scripts/release-manifest.mjs" validate "$manifest" >/dev/null
|
|
local manifest_sha api_image web_image api_digest artifact_report
|
|
manifest_sha=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" sourceSha)
|
|
api_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api)
|
|
web_image=$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.web)
|
|
api_digest=${api_image##*@}
|
|
[[ $manifest_sha == "$release_sha" && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]]
|
|
docker pull --platform linux/amd64 "$api_image" >/dev/null
|
|
docker pull --platform linux/amd64 "$web_image" >/dev/null
|
|
"$private_root/tools/k3d" image import -c easyai-acceptance-local "$api_image" "$web_image"
|
|
for deployment in easyai-api-ningbo easyai-api-hongkong easyai-worker-ningbo \
|
|
easyai-worker-hongkong easyai-capacity-controller easyai-acceptance-emulator \
|
|
easyai-acceptance-callback-collector; do
|
|
kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \
|
|
"*=$api_image" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=15m
|
|
done
|
|
for deployment in easyai-acceptance-edge-ningbo easyai-acceptance-edge-hongkong; do
|
|
kubectl --context "$context" -n "$namespace" set image deployment/"$deployment" \
|
|
"*=$web_image" >/dev/null
|
|
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
|
done
|
|
kubectl --context "$context" -n "$namespace" delete job easyai-artifact-migrate \
|
|
--ignore-not-found --wait=true >/dev/null
|
|
cat <<EOF | kubectl --context "$context" apply -f - >/dev/null
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: easyai-artifact-migrate
|
|
namespace: easyai
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: migrate
|
|
image: $api_image
|
|
command: ["/bin/sh", "-ec", "cd /app && exec /app/easyai-ai-gateway-migrate"]
|
|
envFrom:
|
|
- secretRef:
|
|
name: easyai-ai-gateway-runtime
|
|
EOF
|
|
kubectl --context "$context" -n "$namespace" wait \
|
|
--for=condition=complete job/easyai-artifact-migrate --timeout=10m >/dev/null
|
|
run_load simulated-smoke "$report_root/artifact-simulated-smoke.json"
|
|
verify_hard_gates
|
|
artifact_report="$report_root/artifact-smoke.json"
|
|
jq -n \
|
|
--arg releaseSha "$manifest_sha" \
|
|
--arg apiImageDigest "$api_digest" \
|
|
--arg webImageDigest "${web_image##*@}" \
|
|
'{
|
|
schemaVersion:"acceptance-artifact-smoke/v1",
|
|
releaseSha:$releaseSha,
|
|
apiImageDigest:$apiImageDigest,
|
|
webImageDigest:$webImageDigest,
|
|
architecture:"linux/amd64",
|
|
migrationSmoke:true,
|
|
startupSmoke:true,
|
|
mediaSmoke:true,
|
|
passed:true,
|
|
secretSafe:true
|
|
}' >"$artifact_report"
|
|
chmod 0600 "$artifact_report"
|
|
}
|
|
|
|
quick() {
|
|
current_phase=quick
|
|
run_load simulated-smoke "$report_root/quick.json"
|
|
verify_hard_gates
|
|
echo "local_acceptance_quick=PASS run_id=$run_id report=$report_root/quick.json"
|
|
}
|
|
|
|
full() {
|
|
local manifest=$1 profile repetition
|
|
quick
|
|
for profile in P24 P28 P32; do
|
|
current_phase="capacity_${profile,,}_apply"
|
|
apply_profile "$profile"
|
|
for repetition in 1 2 3; do
|
|
current_phase="capacity_${profile,,}_round_$repetition"
|
|
run_profile_round "$profile" "$repetition"
|
|
done
|
|
stable_profile=$profile
|
|
done
|
|
current_phase=fault_matrix
|
|
run_fault_matrix
|
|
current_phase=autoscaling_and_drain
|
|
run_autoscaling "$stable_profile"
|
|
current_phase=certified_soak
|
|
run_load mixed-soak "$report_root/mixed-soak.json" \
|
|
-duration "${AI_GATEWAY_LOCAL_SOAK_DURATION:-2h}" \
|
|
-image-rate "${AI_GATEWAY_LOCAL_CERTIFIED_IMAGE_RATE:-1}" \
|
|
-video-rate "${AI_GATEWAY_LOCAL_CERTIFIED_VIDEO_RATE:-1}"
|
|
current_phase=overload_shedding
|
|
run_load mixed-overload "$report_root/mixed-overload.json" \
|
|
-duration "${AI_GATEWAY_LOCAL_OVERLOAD_DURATION:-10m}" \
|
|
-image-rate "${AI_GATEWAY_LOCAL_OVERLOAD_IMAGE_RATE:-2}" \
|
|
-video-rate "${AI_GATEWAY_LOCAL_OVERLOAD_VIDEO_RATE:-2}"
|
|
current_phase=amd64_artifact_smoke
|
|
artifact_smoke "$manifest"
|
|
current_phase=final_report
|
|
node "$script_dir/report.mjs" build-local \
|
|
--runtime "$runtime" \
|
|
--snapshot "$snapshot" \
|
|
--reports "$report_root" \
|
|
--artifact "$report_root/artifact-smoke.json" \
|
|
--api-digest "$(node "$repository_root/scripts/release-manifest.mjs" get "$manifest" images.api | sed 's/.*@//')" \
|
|
--certified-profile "$stable_profile" \
|
|
--output "$report_root/acceptance-report.json"
|
|
echo "local_acceptance_full=PASS run_id=$run_id certified_profile=$stable_profile report=$report_root/acceptance-report.json"
|
|
}
|
|
|
|
command=${1:-}
|
|
shift || true
|
|
case $command in
|
|
quick)
|
|
[[ $# -eq 0 ]] || { usage >&2; exit 64; }
|
|
;;
|
|
artifact-smoke|full)
|
|
[[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; }
|
|
;;
|
|
*)
|
|
usage >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
"$script_dir/local-cluster.sh" new-run
|
|
require_local_cluster
|
|
load_environment
|
|
case $command in
|
|
quick)
|
|
quick
|
|
;;
|
|
artifact-smoke)
|
|
artifact_smoke "$2"
|
|
;;
|
|
full)
|
|
full "$2"
|
|
;;
|
|
esac
|