wangbo
c28bf74230
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
96 lines
3.9 KiB
Bash
96 lines
3.9 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Shared read-only guards for the local three-server K3s acceptance cluster.
|
|
# The caller owns shell options and failure reporting.
|
|
|
|
capture_local_control_plane_identity() {
|
|
local context=$1 cluster_name=$2 containers nodes
|
|
containers=$(docker inspect \
|
|
"k3d-${cluster_name}-server-0" \
|
|
"k3d-${cluster_name}-server-1" \
|
|
"k3d-${cluster_name}-server-2" |
|
|
jq -c --arg network "k3d-$cluster_name" '[.[] | {
|
|
name:(.Name | ltrimstr("/")),
|
|
id:.Id,
|
|
restartCount:.RestartCount,
|
|
startedAt:.State.StartedAt,
|
|
ip:.NetworkSettings.Networks[$network].IPAddress
|
|
}] | sort_by(.name)') || return 1
|
|
nodes=$(kubectl --context "$context" get nodes -o json |
|
|
jq -c '[.items[] | {
|
|
name:.metadata.name,
|
|
uid:.metadata.uid,
|
|
site:.metadata.labels["easyai.io/site"],
|
|
cpu:.status.allocatable.cpu,
|
|
memory:.status.allocatable.memory
|
|
}] | sort_by(.name)') || return 1
|
|
jq -cn --argjson containers "$containers" --argjson nodes "$nodes" \
|
|
'{containers:$containers,nodes:$nodes}'
|
|
}
|
|
|
|
record_local_control_plane_identity() {
|
|
local context=$1 cluster_name=$2 output=$3 current temporary
|
|
current=$(capture_local_control_plane_identity "$context" "$cluster_name") || return 1
|
|
temporary="${output}.tmp.$$"
|
|
jq -cn \
|
|
--arg recordedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
|
--argjson current "$current" \
|
|
'{schemaVersion:"local-control-plane-identity/v1",recordedAt:$recordedAt} + $current' \
|
|
>"$temporary"
|
|
chmod 0600 "$temporary"
|
|
mv "$temporary" "$output"
|
|
}
|
|
|
|
verify_local_control_plane_identity() {
|
|
local context=$1 cluster_name=$2 baseline=$3 expected current
|
|
[[ -f $baseline && ! -L $baseline ]] || return 1
|
|
expected=$(jq -cS '{containers,nodes}' "$baseline") || return 1
|
|
current=$(capture_local_control_plane_identity "$context" "$cluster_name" |
|
|
jq -cS '{containers,nodes}') || return 1
|
|
[[ $current == "$expected" ]]
|
|
}
|
|
|
|
verify_local_node_capacity() {
|
|
local context=$1
|
|
kubectl --context "$context" get nodes -o json | jq -e '
|
|
def cpu_m:
|
|
if endswith("m") then rtrimstr("m") | tonumber else tonumber * 1000 end;
|
|
def memory_mi:
|
|
if endswith("Ki") then rtrimstr("Ki") | tonumber / 1024
|
|
elif endswith("Mi") then rtrimstr("Mi") | tonumber
|
|
elif endswith("Gi") then rtrimstr("Gi") | tonumber * 1024
|
|
else 0 end;
|
|
([.items[] | select(.metadata.labels["easyai.io/workload"] == "true") |
|
|
select(.metadata.labels["easyai.io/worker"] == "true") |
|
|
select((.status.allocatable.cpu | cpu_m) == 3000 and
|
|
(.status.allocatable.memory | memory_mi | floor) == 6656)] | length) == 2 and
|
|
([.items[] | select(.metadata.labels["easyai.io/site"] == "los-angeles") |
|
|
select((.status.allocatable.cpu | cpu_m) == 1000 and
|
|
(.status.allocatable.memory | memory_mi | floor) == 2560)] | length) == 1 and
|
|
([.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length) == 3 and
|
|
([.items[] | select(any(.status.conditions[]; .type == "MemoryPressure" and .status == "True"))] | length) == 0
|
|
' >/dev/null
|
|
}
|
|
|
|
verify_local_apiserver_ready() {
|
|
local context=$1
|
|
kubectl --context "$context" get --raw='/readyz?verbose' 2>/dev/null |
|
|
grep -Fq 'readyz check passed'
|
|
}
|
|
|
|
verify_local_etcd_runtime_logs() {
|
|
local cluster_name=$1 baseline=$2 recorded_at name
|
|
[[ -f $baseline && ! -L $baseline ]] || return 1
|
|
recorded_at=$(jq -r '.recordedAt' "$baseline") || return 1
|
|
[[ $recorded_at == ????-??-??T*Z ]]
|
|
for name in \
|
|
"k3d-${cluster_name}-server-0" \
|
|
"k3d-${cluster_name}-server-1" \
|
|
"k3d-${cluster_name}-server-2"; do
|
|
if docker logs --since "$recorded_at" "$name" 2>&1 |
|
|
grep -Eqi 'heartbeat interval is too long|apply request took too long|slow fdatasync|failed to check local etcd status|etcdserver: request timed out|waiting for ReadIndex response took too long'; then
|
|
return 1
|
|
fi
|
|
done
|
|
}
|