From e20688bbe2bcac622ac98407d2b523a072595fe0 Mon Sep 17 00:00:00 2001 From: wangbo Date: Sun, 2 Aug 2026 10:39:55 +0800 Subject: [PATCH] =?UTF-8?q?fix(k3s):=20=E6=8C=81=E4=B9=85=E5=8C=96?= =?UTF-8?q?=E9=A6=99=E6=B8=AF=E6=8E=A7=E5=88=B6=E9=9D=A2=E7=A3=81=E7=9B=98?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E7=BC=93=E8=A7=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因：香港虚拟磁盘在低吞吐下仍出现高同步写延迟，触发 etcd slow apply、Handler timeout 和 PostgreSQL API Server 连通性异常。\n\n影响：新增仅针对香港根块设备的 mq-deadline 配置脚本，应用时不重启 K3s，并提供显式 rollback 恢复 none；运行手册明确该项仅为缓解，不能替代高性能云盘。\n\n验证：新香港节点同步写探针从 13.28ms 降至 6.25ms；bash -n、ShellCheck、manual-release-test.sh 通过。 --- docs/operations/k3s-ha-runbook.md | 24 ++++++ scripts/cluster/configure-control-plane-io.sh | 84 +++++++++++++++++++ tests/release/manual-release-test.sh | 4 + 3 files changed, 112 insertions(+) create mode 100755 scripts/cluster/configure-control-plane-io.sh diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md index e65fcbb..14ac1eb 100644 --- a/docs/operations/k3s-ha-runbook.md +++ b/docs/operations/k3s-ha-runbook.md @@ -96,6 +96,30 @@ K3s containerd 同时配置 Docker Hub、GHCR 与 Quay 的镜像端点；已有 server 都保持 etcd quorum。宁波停机可能令香港数据库副本晋升；首次切换前如需恢复宁波 初始主库，显式执行 `promote-cnpg-primary.sh`，故障恢复流程不得自动回切。 +#### 香港控制面云盘延迟缓解 + +生产同构验收前必须用 etcd metrics 的增量窗口检查 backend commit 与 WAL fsync，不能只看 +节点 Ready。香港虚拟磁盘若使用 `none` 调度器并出现持续 `Handler timeout`、 +`etcdserver: request timed out` 或 PostgreSQL `API server connectivity issue`，可以先应用已验证、 +可回滚的 `mq-deadline` 缓解： + +```bash +./scripts/cluster/configure-control-plane-io.sh status +./scripts/cluster/configure-control-plane-io.sh apply +``` + +该脚本只修改香港根块设备调度器，写入一个早于 `k3s.service` 执行的 systemd oneshot；应用时 +不重启 K3s。回退命令会删除该 unit 并恢复 `none`： + +```bash +./scripts/cluster/configure-control-plane-io.sh rollback +``` + +这只是尾延迟缓解，不是慢云盘的容量认证。任何 60 秒窗口中 backend commit 平均值达到 +50 ms、WAL fsync 达到 15 ms，或仍出现上述控制面硬错误，都必须停止验收。最终修复应升级 +为具有稳定同步写延迟的高性能云盘，或按 etcd 成员替换流程迁移香港控制节点；禁止以 +`unsafe-no-fsync`、双成员 etcd 或放宽硬门禁代替存储修复。 + ### 3. 平台和入口预置 ```bash diff --git a/scripts/cluster/configure-control-plane-io.sh b/scripts/cluster/configure-control-plane-io.sh new file mode 100755 index 0000000..3209faf --- /dev/null +++ b/scripts/cluster/configure-control-plane-io.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +action=${1:-status} +[[ $action == status || $action == apply || $action == rollback ]] || { + echo 'usage: configure-control-plane-io.sh {status|apply|rollback}' >&2 + exit 64 +} + +load_cluster_env +unit_name=easyai-k3s-io-scheduler.service +unit_path=/etc/systemd/system/$unit_name + +cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- "$action" "$unit_name" "$unit_path" <<'REMOTE' +set -euo pipefail +action=$1 +unit_name=$2 +unit_path=$3 + +root_source=$(findmnt -n -o SOURCE /) +root_partition=${root_source#/dev/} +device=$(lsblk -no PKNAME "/dev/$root_partition" | head -n 1) +[[ $device =~ ^[a-zA-Z0-9._-]+$ && -f /sys/block/$device/queue/scheduler ]] + +current_scheduler() { + sed -n 's/.*\[\([^]]*\)\].*/\1/p' "/sys/block/$device/queue/scheduler" +} + +case $action in + status) + configured=false + systemctl is-enabled --quiet "$unit_name" 2>/dev/null && configured=true + printf 'control_plane_io=STATUS node=hongkong device=%s scheduler=%s configured=%s\n' \ + "$device" "$(current_scheduler)" "$configured" + ;; + apply) + grep -qw mq-deadline "/sys/block/$device/queue/scheduler" || { + echo 'mq-deadline is not supported by the Hong Kong root block device' >&2 + exit 1 + } + temporary=$(mktemp /etc/systemd/system/.easyai-k3s-io-scheduler.XXXXXX) + cleanup() { + [[ -f ${temporary:-} ]] && rm -f -- "$temporary" || true + } + trap cleanup EXIT + cat >"$temporary" < /sys/block/$device/queue/scheduler' +ExecStop=/bin/sh -c 'printf "%s\\n" none > /sys/block/$device/queue/scheduler' +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +EOF + chmod 0644 "$temporary" + install -o root -g root -m 0644 "$temporary" "$unit_path" + systemctl daemon-reload + systemctl enable --now "$unit_name" >/dev/null + [[ $(current_scheduler) == mq-deadline ]] + printf 'control_plane_io=APPLIED node=hongkong device=%s scheduler=mq-deadline restart=false\n' "$device" + ;; + rollback) + if [[ -f $unit_path ]]; then + systemctl disable --now "$unit_name" >/dev/null 2>&1 || true + rm -f -- "$unit_path" + systemctl daemon-reload + else + printf '%s\n' none >"/sys/block/$device/queue/scheduler" + fi + [[ $(current_scheduler) == none ]] + printf 'control_plane_io=ROLLED_BACK node=hongkong device=%s scheduler=none\n' "$device" + ;; +esac +REMOTE diff --git a/tests/release/manual-release-test.sh b/tests/release/manual-release-test.sh index d92448c..46fc326 100755 --- a/tests/release/manual-release-test.sh +++ b/tests/release/manual-release-test.sh @@ -307,6 +307,10 @@ if grep -Fq 'archive_timeout: 60s' "$root/deploy/kubernetes/production/database. fi grep -Fq 'wal_sender_timeout: 30s' "$root/deploy/kubernetes/production/database.yaml" grep -Fq 'wal_receiver_timeout: 30s' "$root/deploy/kubernetes/production/database.yaml" +bash -n "$root/scripts/cluster/configure-control-plane-io.sh" +grep -Fq 'Before=k3s.service' "$root/scripts/cluster/configure-control-plane-io.sh" +grep -Fq 'restart=false' "$root/scripts/cluster/configure-control-plane-io.sh" +grep -Fq 'configure-control-plane-io.sh apply' "$root/docs/operations/k3s-ha-runbook.md" "$root/tests/release/cluster-release-helper-test.sh" >/dev/null "$root/tests/release/production-acceptance-script-test.sh" >/dev/null