wangbo
a95184b5b6
本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
95 lines
3.8 KiB
Bash
95 lines
3.8 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Shared read-only guards for the local three-server K3s acceptance cluster.
|
|
# The caller owns shell options and failure reporting.
|
|
|
|
capture_local_control_plane_identity() {
|
|
local context=$1 cluster_name=$2 containers nodes
|
|
containers=$(docker inspect \
|
|
"k3d-${cluster_name}-server-0" \
|
|
"k3d-${cluster_name}-server-1" \
|
|
"k3d-${cluster_name}-server-2" |
|
|
jq -c --arg network "k3d-$cluster_name" '[.[] | {
|
|
name:(.Name | ltrimstr("/")),
|
|
id:.Id,
|
|
restartCount:.RestartCount,
|
|
startedAt:.State.StartedAt,
|
|
ip:.NetworkSettings.Networks[$network].IPAddress
|
|
}] | sort_by(.name)') || return 1
|
|
nodes=$(kubectl --context "$context" get nodes -o json |
|
|
jq -c '[.items[] | {
|
|
name:.metadata.name,
|
|
uid:.metadata.uid,
|
|
site:.metadata.labels["easyai.io/site"],
|
|
cpu:.status.allocatable.cpu,
|
|
memory:.status.allocatable.memory
|
|
}] | sort_by(.name)') || return 1
|
|
jq -cn --argjson containers "$containers" --argjson nodes "$nodes" \
|
|
'{containers:$containers,nodes:$nodes}'
|
|
}
|
|
|
|
record_local_control_plane_identity() {
|
|
local context=$1 cluster_name=$2 output=$3 current temporary
|
|
current=$(capture_local_control_plane_identity "$context" "$cluster_name") || return 1
|
|
temporary="${output}.tmp.$$"
|
|
jq -cn \
|
|
--arg recordedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
|
--argjson current "$current" \
|
|
'{schemaVersion:"local-control-plane-identity/v1",recordedAt:$recordedAt} + $current' \
|
|
>"$temporary"
|
|
chmod 0600 "$temporary"
|
|
mv "$temporary" "$output"
|
|
}
|
|
|
|
verify_local_control_plane_identity() {
|
|
local context=$1 cluster_name=$2 baseline=$3 expected current
|
|
[[ -f $baseline && ! -L $baseline ]] || return 1
|
|
expected=$(jq -cS '{containers,nodes}' "$baseline") || return 1
|
|
current=$(capture_local_control_plane_identity "$context" "$cluster_name" |
|
|
jq -cS '{containers,nodes}') || return 1
|
|
[[ $current == "$expected" ]]
|
|
}
|
|
|
|
verify_local_node_capacity() {
|
|
local context=$1
|
|
kubectl --context "$context" get nodes -o json | jq -e '
|
|
def cpu_m:
|
|
if endswith("m") then rtrimstr("m") | tonumber else tonumber * 1000 end;
|
|
def memory_mi:
|
|
if endswith("Ki") then rtrimstr("Ki") | tonumber / 1024
|
|
elif endswith("Mi") then rtrimstr("Mi") | tonumber
|
|
elif endswith("Gi") then rtrimstr("Gi") | tonumber * 1024
|
|
else 0 end;
|
|
([.items[] | select(.metadata.labels["easyai.io/workload"] == "true") |
|
|
select((.status.allocatable.cpu | cpu_m) == 3000 and
|
|
(.status.allocatable.memory | memory_mi) == 6656)] | length) == 2 and
|
|
([.items[] | select(.metadata.labels["easyai.io/site"] == "los-angeles") |
|
|
select((.status.allocatable.cpu | cpu_m) == 1000 and
|
|
(.status.allocatable.memory | memory_mi) == 2560)] | length) == 1 and
|
|
([.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length) == 3 and
|
|
([.items[] | select(any(.status.conditions[]; .type == "MemoryPressure" and .status == "True"))] | length) == 0
|
|
' >/dev/null
|
|
}
|
|
|
|
verify_local_apiserver_ready() {
|
|
local context=$1
|
|
kubectl --context "$context" get --raw='/readyz?verbose' 2>/dev/null |
|
|
grep -Fq 'readyz check passed'
|
|
}
|
|
|
|
verify_local_etcd_runtime_logs() {
|
|
local cluster_name=$1 baseline=$2 recorded_at name
|
|
[[ -f $baseline && ! -L $baseline ]] || return 1
|
|
recorded_at=$(jq -r '.recordedAt' "$baseline") || return 1
|
|
[[ $recorded_at == ????-??-??T*Z ]]
|
|
for name in \
|
|
"k3d-${cluster_name}-server-0" \
|
|
"k3d-${cluster_name}-server-1" \
|
|
"k3d-${cluster_name}-server-2"; do
|
|
if docker logs --since "$recorded_at" "$name" 2>&1 |
|
|
grep -Eqi 'heartbeat interval is too long|apply request took too long|slow fdatasync|failed to check local etcd status|etcdserver: request timed out|waiting for ReadIndex response took too long'; then
|
|
return 1
|
|
fi
|
|
done
|
|
}
|