From 60d16b11ba36be4bc1242134d58d840ea09de4be Mon Sep 17 00:00:00 2001 From: wangbo Date: Tue, 4 Aug 2026 20:54:25 +0800 Subject: [PATCH] =?UTF-8?q?fix(capacity):=20=E5=BF=BD=E7=95=A5=E6=9C=AA?= =?UTF-8?q?=E5=B0=B1=E7=BB=AA=E7=9A=84=20Worker=20=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 香港站点保留了已离线的旧 control-plane 节点标签，容量控制器仍尝试读取其 metrics，导致 404 并阻断 readiness。容量采集现在只纳入 Ready=True 的候选节点，实际 Worker Pod 所在的就绪节点继续参与容量计算。 验证：容量控制器聚焦测试与 API 全量 go test 均通过，并覆盖同站点 NotReady 旧节点与 Ready Worker 节点并存场景。 --- .../internal/capacitycontroller/kubernetes.go | 16 +++++++++++++- .../capacitycontroller/kubernetes_test.go | 21 +++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/apps/api/internal/capacitycontroller/kubernetes.go b/apps/api/internal/capacitycontroller/kubernetes.go index e6732ac..caa001a 100644 --- a/apps/api/internal/capacitycontroller/kubernetes.go +++ b/apps/api/internal/capacitycontroller/kubernetes.go @@ -178,6 +178,20 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub if err := client.getJSON(ctx, nodePath, &nodes); err != nil { return KubernetesSiteState{}, err } + readyNodes := nodes.Items[:0] + for _, node := range nodes.Items { + ready := false + for _, condition := range node.Status.Conditions { + if condition.Type == "Ready" && condition.Status == "True" { + ready = true + break + } + } + if ready { + readyNodes = append(readyNodes, node) + } + } + nodes.Items = readyNodes if len(nodes.Items) == 0 { // A zero-replica Deployment with no eligible nodes represents an // explicitly disabled site. It must remain visible to the planner, but it @@ -186,7 +200,7 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub if state.CurrentReplicas == 0 { return state, nil } - return KubernetesSiteState{}, fmt.Errorf("site %s has no matching nodes", site) + return KubernetesSiteState{}, fmt.Errorf("site %s has no ready matching nodes", site) } var pods struct { Items []struct { diff --git a/apps/api/internal/capacitycontroller/kubernetes_test.go b/apps/api/internal/capacitycontroller/kubernetes_test.go index 8890c1a..6db6a5f 100644 --- a/apps/api/internal/capacitycontroller/kubernetes_test.go +++ b/apps/api/internal/capacitycontroller/kubernetes_test.go @@ -31,15 +31,24 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) { http.Error(w, "unexpected Worker node selector", http.StatusBadRequest) return } - _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"easyai-hongkong"},"status":{ + _, _ = w.Write([]byte(`{"items":[ + {"metadata":{"name":"easyai-hongkong"},"status":{ "allocatable":{"memory":"8Gi","cpu":"4"}, - "conditions":[{"type":"MemoryPressure","status":"False"}] + "conditions":[{"type":"Ready","status":"False"}] + }}, + {"metadata":{"name":"easyai-hongkong-worker-2"},"status":{ + "allocatable":{"memory":"8Gi","cpu":"4"}, + "conditions":[{"type":"Ready","status":"True"},{"type":"MemoryPressure","status":"False"}] }}]}`)) case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/api/v1/namespaces/easyai/pods"): - _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong"}}]}`)) + _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong-worker-2"}}]}`)) + case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && + strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong-worker-2"): + _, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`)) case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong"): - _, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`)) + t.Error("NotReady node metrics must not be queried") + http.Error(w, "NotReady node", http.StatusInternalServerError) case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && strings.HasSuffix(request.URL.Path, "/pods"): _, _ = w.Write([]byte(`{"items":[{ @@ -69,7 +78,7 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) { if err != nil { t.Fatal(err) } - if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong" || + if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong-worker-2" || state.WorkerRequestMemoryBytes != 512<<20 || state.WorkerRequestMilliCPU != 250 || state.AllocatableMemoryBytes != 8<<30 || state.UsedMemoryBytes != 3<<30 || state.Nodes[0].WorkerUsedMemoryBytes != 384<<20 || state.Nodes[0].WorkerUsedMilliCPU != 125 { @@ -148,7 +157,7 @@ func TestKubernetesClientRejectsActiveSiteWithoutWorkerNodes(t *testing.T) { t.Fatal(err) } if _, err := client.SiteState(context.Background(), "ningbo"); err == nil || - !strings.Contains(err.Error(), "site ningbo has no matching nodes") { + !strings.Contains(err.Error(), "site ningbo has no ready matching nodes") { t.Fatalf("expected missing Worker node error, got %v", err) } }