feat(worker): 实现集群限流与自适应负载

保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
2026-08-03 00:13:46 +08:00
parent 9a01fd4657
commit c28bf74230
52 changed files with 3700 additions and 272 deletions
+23 -10
View File
@@ -41,7 +41,8 @@ load_lock() {
}
# shellcheck source=/dev/null
source "$lock_file"
: "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
: "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${K3S_PAUSE_IMAGE:?}" "${K3S_BUSYBOX_IMAGE:?}"
: "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
: "${CNPG_CONTROLLER_IMAGE:?}" "${CNPG_POSTGRES_IMAGE:?}" "${K3S_COREDNS_IMAGE:?}"
: "${K3S_METRICS_SERVER_IMAGE:?}" "${K3S_LOCAL_PATH_IMAGE:?}"
}
@@ -123,6 +124,9 @@ host_available_disk_bytes() {
}
preflight() {
local required_disk_gib=${1:-30} required_disk_bytes
[[ $required_disk_gib =~ ^[0-9]+$ && $required_disk_gib -ge 10 ]]
required_disk_bytes=$((required_disk_gib * 1024 * 1024 * 1024))
load_lock
require_command docker
require_command kubectl
@@ -165,12 +169,12 @@ preflight() {
exit 1
}
available_disk_bytes=$(host_available_disk_bytes)
[[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge 32212254720 ]] || {
[[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge $required_disk_bytes ]] || {
available_disk_gib=$(awk -v bytes="${available_disk_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least 30 GiB" >&2
echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least ${required_disk_gib} GiB" >&2
exit 1
}
echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes architecture=$architecture k3d=$K3D_VERSION"
echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes required_disk_gib=$required_disk_gib architecture=$architecture k3d=$K3D_VERSION"
}
ensure_private_material() {
@@ -224,8 +228,8 @@ render_k3d_config() {
host_memory_mib=$((host_memory_bytes / 1024 / 1024))
site_system_cpu="$(((host_cpu - 4) * 1000 + 750))m"
witness_system_cpu="$(((host_cpu - 2) * 1000 + 750))m"
site_system_memory="$((host_memory_mib - 8192 + 1024))Mi"
witness_system_memory="$((host_memory_mib - 4096 + 1024))Mi"
site_system_memory="$((host_memory_mib - 8192 + 512))Mi"
witness_system_memory="$((host_memory_mib - 4096 + 512))Mi"
sed \
-e "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" \
-e "s|EASYAI_ACCEPTANCE_K3S_IMAGE|${K3S_IMAGE%%@*}|g" \
@@ -294,7 +298,8 @@ remove_cluster_network() {
pull_dependency_images() {
local image
for image in "$K3S_IMAGE" "$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
for image in "$K3S_IMAGE" "$K3S_PAUSE_IMAGE" "$K3S_BUSYBOX_IMAGE" \
"$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
"$K3S_COREDNS_IMAGE" "$K3S_METRICS_SERVER_IMAGE" "$K3S_LOCAL_PATH_IMAGE"; do
pull_dependency_image "$image"
done
@@ -596,6 +601,10 @@ render_and_apply_application() {
-f "$repository_root/deploy/kubernetes/production/service-account-rbac.yaml" >/dev/null
kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/local-config.yaml" >/dev/null
kubectl --context "$context" -n "$namespace" apply -f "$rendered" >/dev/null
for workload in easyai-worker-ningbo easyai-worker-hongkong; do
kubectl --context "$context" -n "$namespace" patch deployment "$workload" \
--type=merge -p='{"spec":{"template":{"spec":{"affinity":null}}}}' >/dev/null
done
for workload in easyai-api-ningbo easyai-worker-ningbo; do
kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
@@ -791,10 +800,14 @@ up_cluster() {
echo 'local K3s node allocatable capacity does not match 4/8, 4/8, 2/4 resource envelope' >&2
exit 1
}
"$k3d" image import -c "$cluster_name" \
local cluster_image
for cluster_image in \
"$api_image" "$web_image" "$netem_image" \
"${K3S_PAUSE_IMAGE%%@*}" "${K3S_BUSYBOX_IMAGE%%@*}" \
"${CNPG_CONTROLLER_IMAGE%%@*}" "${CNPG_POSTGRES_IMAGE%%@*}" \
"${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"
"${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"; do
"$k3d" image import -c "$cluster_name" "$cluster_image"
done
local cnpg_manifest="$state_root/cnpg.yaml"
curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest"
@@ -844,7 +857,7 @@ up_cluster() {
new_run() {
load_lock
preflight
preflight 10
[[ -z $(git -C "$repository_root" status --short) ]] || {
echo 'local acceptance requires a clean source tree for a fresh Run ID' >&2
exit 1