From 95ddfb2a04bc9c3c4b9694eda3e977240a175cd9 Mon Sep 17 00:00:00 2001
From: wangbo <wangbolhr@163.com>
Date: Tue, 28 Jul 2026 05:05:33 +0800
Subject: [PATCH] =?UTF-8?q?fix(deploy):=20=E4=B8=BA=20K3s=20=E9=85=8D?=
 =?UTF-8?q?=E7=BD=AE=E5=AE=B9=E5=99=A8=E9=95=9C=E5=83=8F=E7=AB=AF=E7=82=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

三节点 containerd 增加 Docker Hub、GHCR 与 Quay 的 DaoCloud 镜像端点，解决生产节点直连公共 Registry 超时导致 Pod sandbox 无法创建的问题。

新增逐节点滚动刷新脚本，每次等待节点重新 Ready 并实际拉取 pause 镜像后才继续，避免同时重启 etcd 成员。初始 K3s 安装也会预置同一 registries.yaml。

验证：三节点镜像端点均返回 Registry 认证响应，bash -n、ShellCheck。
---
 deploy/kubernetes/registries.yaml           | 10 +++++
 docs/operations/k3s-ha-runbook.md           |  5 +++
 scripts/cluster/bootstrap-k3s.sh            |  7 +++-
 scripts/cluster/configure-k3s-registries.sh | 45 +++++++++++++++++++++
 4 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 deploy/kubernetes/registries.yaml
 create mode 100755 scripts/cluster/configure-k3s-registries.sh

diff --git a/deploy/kubernetes/registries.yaml b/deploy/kubernetes/registries.yaml
new file mode 100644
index 0000000..1044263
--- /dev/null
+++ b/deploy/kubernetes/registries.yaml
@@ -0,0 +1,10 @@
+mirrors:
+  docker.io:
+    endpoint:
+      - https://docker.m.daocloud.io
+  ghcr.io:
+    endpoint:
+      - https://ghcr.m.daocloud.io
+  quay.io:
+    endpoint:
+      - https://quay.m.daocloud.io
diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md
index e5109c6..e9cd9fe 100644
--- a/docs/operations/k3s-ha-runbook.md
+++ b/docs/operations/k3s-ha-runbook.md
@@ -51,6 +51,11 @@ node scripts/ci-validate-migrations.mjs <当前线上完整GitSHA>
 `backup` 会在卸载旧 K3s 前，将 SQLite、manifests 和 staging 资源归档并校验后上传
 OSS `easyai-ai-gateway/production/pre-ha/`。`install` 固定
 `v1.36.2+k3s1`，使用 embedded etcd、静态 Secret 加密、500ms heartbeat 和 5s election timeout。
+K3s containerd 同时配置 Docker Hub、GHCR 与 Quay 的镜像端点；已有集群需要滚动刷新时执行：
+
+```bash
+./scripts/cluster/configure-k3s-registries.sh
+```
 
 ### 3. 平台和入口预置
 
diff --git a/scripts/cluster/bootstrap-k3s.sh b/scripts/cluster/bootstrap-k3s.sh
index 1a9148b..84fe19f 100755
--- a/scripts/cluster/bootstrap-k3s.sh
+++ b/scripts/cluster/bootstrap-k3s.sh
@@ -155,6 +155,8 @@ install_cluster() {
   write_node_config "$configuration_directory/hongkong.yaml" easyai-hongkong 10.77.0.2 join
   write_node_config "$configuration_directory/los-angeles.yaml" easyai-los-angeles 10.77.0.3 join \
     'easyai.io/witness=true:NoSchedule'
+  cp "$cluster_root/deploy/kubernetes/registries.yaml" \
+    "$configuration_directory/registries.yaml"
   umask 077
   cat >"$configuration_directory/etcd-s3-secret.yaml" <<EOF
 apiVersion: v1
@@ -202,7 +204,10 @@ REMOTE
     printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$host" \
       'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token'
     cluster_scp "$configuration_directory/$node_name.yaml" "$host:/etc/rancher/k3s/config.yaml"
-    cluster_ssh "$host" 'chmod 0600 /etc/rancher/k3s/config.yaml'
+    cluster_scp "$configuration_directory/registries.yaml" \
+      "$host:/etc/rancher/k3s/registries.yaml"
+    cluster_ssh "$host" \
+      'chmod 0600 /etc/rancher/k3s/config.yaml /etc/rancher/k3s/registries.yaml'
   done
 
   for index in 0 1 2; do
diff --git a/scripts/cluster/configure-k3s-registries.sh b/scripts/cluster/configure-k3s-registries.sh
new file mode 100755
index 0000000..03e134c
--- /dev/null
+++ b/scripts/cluster/configure-k3s-registries.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+load_cluster_env
+
+registry_config="$cluster_root/deploy/kubernetes/registries.yaml"
+[[ -f $registry_config && ! -L $registry_config ]] || {
+  echo 'K3s registry mirror configuration is unavailable' >&2
+  exit 1
+}
+
+restart_node() {
+  local host=$1
+  local node=$2
+  local observer=$3
+  local ready=false
+  cluster_scp "$registry_config" "$host:/etc/rancher/k3s/registries.yaml"
+  cluster_ssh "$host" \
+    'chmod 0600 /etc/rancher/k3s/registries.yaml && systemctl restart k3s'
+  for _ in $(seq 1 60); do
+    if cluster_ssh "$observer" \
+      "k3s kubectl wait --for=condition=Ready node/$node --timeout=5s >/dev/null 2>&1"; then
+      ready=true
+      break
+    fi
+    sleep 2
+  done
+  [[ $ready == true ]] || {
+    echo "K3s node did not return Ready after registry mirror update: $node" >&2
+    return 1
+  }
+  cluster_ssh "$host" \
+    "grep -Fq 'https://docker.m.daocloud.io' /var/lib/rancher/k3s/agent/etc/containerd/config.toml && k3s crictl pull docker.io/rancher/mirrored-pause:3.6 >/dev/null"
+  echo "[registry] $node ready"
+}
+
+# Restart one etcd member at a time and observe it from a different server.
+restart_node "$CLUSTER_LOS_ANGELES_HOST" easyai-los-angeles "$CLUSTER_NINGBO_HOST"
+restart_node "$CLUSTER_HONGKONG_HOST" easyai-hongkong "$CLUSTER_NINGBO_HOST"
+restart_node "$CLUSTER_NINGBO_HOST" easyai-ningbo "$CLUSTER_HONGKONG_HOST"
+
+echo 'k3s_registry_mirrors=PASS nodes=3 quorum_preserved=true'