fix(capacity): 忽略未就绪的 Worker 节点

香港站点保留了已离线的旧 control-plane 节点标签,容量控制器仍尝试读取其 metrics,导致 404 并阻断 readiness。容量采集现在只纳入 Ready=True 的候选节点,实际 Worker Pod 所在的就绪节点继续参与容量计算。

验证:容量控制器聚焦测试与 API 全量 go test 均通过,并覆盖同站点 NotReady 旧节点与 Ready Worker 节点并存场景。
This commit is contained in:
2026-08-04 20:54:25 +08:00
parent a5dd7f36f5
commit 60d16b11ba
2 changed files with 30 additions and 7 deletions
@@ -178,6 +178,20 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
if err := client.getJSON(ctx, nodePath, &nodes); err != nil { if err := client.getJSON(ctx, nodePath, &nodes); err != nil {
return KubernetesSiteState{}, err return KubernetesSiteState{}, err
} }
readyNodes := nodes.Items[:0]
for _, node := range nodes.Items {
ready := false
for _, condition := range node.Status.Conditions {
if condition.Type == "Ready" && condition.Status == "True" {
ready = true
break
}
}
if ready {
readyNodes = append(readyNodes, node)
}
}
nodes.Items = readyNodes
if len(nodes.Items) == 0 { if len(nodes.Items) == 0 {
// A zero-replica Deployment with no eligible nodes represents an // A zero-replica Deployment with no eligible nodes represents an
// explicitly disabled site. It must remain visible to the planner, but it // explicitly disabled site. It must remain visible to the planner, but it
@@ -186,7 +200,7 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
if state.CurrentReplicas == 0 { if state.CurrentReplicas == 0 {
return state, nil return state, nil
} }
return KubernetesSiteState{}, fmt.Errorf("site %s has no matching nodes", site) return KubernetesSiteState{}, fmt.Errorf("site %s has no ready matching nodes", site)
} }
var pods struct { var pods struct {
Items []struct { Items []struct {
@@ -31,15 +31,24 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) {
http.Error(w, "unexpected Worker node selector", http.StatusBadRequest) http.Error(w, "unexpected Worker node selector", http.StatusBadRequest)
return return
} }
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"easyai-hongkong"},"status":{ _, _ = w.Write([]byte(`{"items":[
{"metadata":{"name":"easyai-hongkong"},"status":{
"allocatable":{"memory":"8Gi","cpu":"4"}, "allocatable":{"memory":"8Gi","cpu":"4"},
"conditions":[{"type":"MemoryPressure","status":"False"}] "conditions":[{"type":"Ready","status":"False"}]
}},
{"metadata":{"name":"easyai-hongkong-worker-2"},"status":{
"allocatable":{"memory":"8Gi","cpu":"4"},
"conditions":[{"type":"Ready","status":"True"},{"type":"MemoryPressure","status":"False"}]
}}]}`)) }}]}`))
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/api/v1/namespaces/easyai/pods"): case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/api/v1/namespaces/easyai/pods"):
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong"}}]}`)) _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong-worker-2"}}]}`))
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong-worker-2"):
_, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`))
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong"): strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong"):
_, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`)) t.Error("NotReady node metrics must not be queried")
http.Error(w, "NotReady node", http.StatusInternalServerError)
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
strings.HasSuffix(request.URL.Path, "/pods"): strings.HasSuffix(request.URL.Path, "/pods"):
_, _ = w.Write([]byte(`{"items":[{ _, _ = w.Write([]byte(`{"items":[{
@@ -69,7 +78,7 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong" || if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong-worker-2" ||
state.WorkerRequestMemoryBytes != 512<<20 || state.WorkerRequestMilliCPU != 250 || state.WorkerRequestMemoryBytes != 512<<20 || state.WorkerRequestMilliCPU != 250 ||
state.AllocatableMemoryBytes != 8<<30 || state.UsedMemoryBytes != 3<<30 || state.AllocatableMemoryBytes != 8<<30 || state.UsedMemoryBytes != 3<<30 ||
state.Nodes[0].WorkerUsedMemoryBytes != 384<<20 || state.Nodes[0].WorkerUsedMilliCPU != 125 { state.Nodes[0].WorkerUsedMemoryBytes != 384<<20 || state.Nodes[0].WorkerUsedMilliCPU != 125 {
@@ -148,7 +157,7 @@ func TestKubernetesClientRejectsActiveSiteWithoutWorkerNodes(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if _, err := client.SiteState(context.Background(), "ningbo"); err == nil || if _, err := client.SiteState(context.Background(), "ningbo"); err == nil ||
!strings.Contains(err.Error(), "site ningbo has no matching nodes") { !strings.Contains(err.Error(), "site ningbo has no ready matching nodes") {
t.Fatalf("expected missing Worker node error, got %v", err) t.Fatalf("expected missing Worker node error, got %v", err)
} }
} }