wangbo
e05922b0f4
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
97 lines
2.9 KiB
Bash
Executable File
97 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
scripts/acceptance/network-fault.sh baseline
|
|
scripts/acceptance/network-fault.sh weak-link
|
|
scripts/acceptance/network-fault.sh upstream-outage
|
|
scripts/acceptance/network-fault.sh database-outage <ningbo|hongkong>
|
|
scripts/acceptance/network-fault.sh reset
|
|
|
|
Only operates on Pods carrying easyai.io/environment=local-acceptance in the
|
|
easyai-acceptance-local k3d context.
|
|
EOF
|
|
}
|
|
|
|
profile=${1:-}
|
|
site=${2:-}
|
|
namespace=${AI_GATEWAY_LOCAL_ACCEPTANCE_NAMESPACE:-easyai}
|
|
context=${AI_GATEWAY_LOCAL_ACCEPTANCE_CONTEXT:-k3d-easyai-acceptance-local}
|
|
|
|
command -v kubectl >/dev/null 2>&1 || {
|
|
echo 'kubectl is required' >&2
|
|
exit 1
|
|
}
|
|
[[ $(kubectl config get-contexts "$context" -o name) == "$context" ]] || {
|
|
echo "refusing netem because context $context does not exist" >&2
|
|
exit 1
|
|
}
|
|
|
|
pod_for() {
|
|
local name=$1
|
|
kubectl --context "$context" -n "$namespace" get pod \
|
|
-l "app.kubernetes.io/name=$name,easyai.io/environment=local-acceptance" \
|
|
-o 'jsonpath={.items[0].metadata.name}'
|
|
}
|
|
|
|
apply_qdisc() {
|
|
local pod=$1
|
|
shift
|
|
[[ -n $pod ]] || {
|
|
echo 'local acceptance proxy Pod was not found' >&2
|
|
exit 1
|
|
}
|
|
kubectl --context "$context" -n "$namespace" exec "$pod" -- \
|
|
tc qdisc replace dev eth0 root netem "$@"
|
|
}
|
|
|
|
reset_qdisc() {
|
|
local pod=$1
|
|
[[ -z $pod ]] || kubectl --context "$context" -n "$namespace" exec "$pod" -- \
|
|
tc qdisc del dev eth0 root >/dev/null 2>&1 || true
|
|
}
|
|
|
|
upstream_pod=$(pod_for easyai-acceptance-upstream-proxy)
|
|
ningbo_db_pod=$(kubectl --context "$context" -n "$namespace" get pod \
|
|
-l 'app.kubernetes.io/name=easyai-postgres-proxy,easyai.io/site=ningbo,easyai.io/environment=local-acceptance' \
|
|
-o 'jsonpath={.items[0].metadata.name}')
|
|
hongkong_db_pod=$(kubectl --context "$context" -n "$namespace" get pod \
|
|
-l 'app.kubernetes.io/name=easyai-postgres-proxy,easyai.io/site=hongkong,easyai.io/environment=local-acceptance' \
|
|
-o 'jsonpath={.items[0].metadata.name}')
|
|
|
|
case $profile in
|
|
baseline)
|
|
apply_qdisc "$upstream_pod" delay 20ms
|
|
apply_qdisc "$ningbo_db_pod" delay 20ms
|
|
apply_qdisc "$hongkong_db_pod" delay 20ms
|
|
;;
|
|
weak-link)
|
|
apply_qdisc "$upstream_pod" delay 20ms 10ms distribution normal loss 0.5%
|
|
apply_qdisc "$ningbo_db_pod" delay 20ms 10ms distribution normal loss 0.5%
|
|
apply_qdisc "$hongkong_db_pod" delay 20ms 10ms distribution normal loss 0.5%
|
|
;;
|
|
upstream-outage)
|
|
apply_qdisc "$upstream_pod" loss 100%
|
|
;;
|
|
database-outage)
|
|
case $site in
|
|
ningbo) apply_qdisc "$ningbo_db_pod" loss 100% ;;
|
|
hongkong) apply_qdisc "$hongkong_db_pod" loss 100% ;;
|
|
*) usage >&2; exit 64 ;;
|
|
esac
|
|
;;
|
|
reset)
|
|
reset_qdisc "$upstream_pod"
|
|
reset_qdisc "$ningbo_db_pod"
|
|
reset_qdisc "$hongkong_db_pod"
|
|
;;
|
|
*)
|
|
usage >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
|
|
echo "local_acceptance_netem=PASS profile=$profile${site:+ site=$site}"
|