fix(capacity): 忽略未就绪的 Worker 节点
香港站点保留了已离线的旧 control-plane 节点标签,容量控制器仍尝试读取其 metrics,导致 404 并阻断 readiness。容量采集现在只纳入 Ready=True 的候选节点,实际 Worker Pod 所在的就绪节点继续参与容量计算。 验证:容量控制器聚焦测试与 API 全量 go test 均通过,并覆盖同站点 NotReady 旧节点与 Ready Worker 节点并存场景。
This commit is contained in:
@@ -178,6 +178,20 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
|
||||
if err := client.getJSON(ctx, nodePath, &nodes); err != nil {
|
||||
return KubernetesSiteState{}, err
|
||||
}
|
||||
readyNodes := nodes.Items[:0]
|
||||
for _, node := range nodes.Items {
|
||||
ready := false
|
||||
for _, condition := range node.Status.Conditions {
|
||||
if condition.Type == "Ready" && condition.Status == "True" {
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ready {
|
||||
readyNodes = append(readyNodes, node)
|
||||
}
|
||||
}
|
||||
nodes.Items = readyNodes
|
||||
if len(nodes.Items) == 0 {
|
||||
// A zero-replica Deployment with no eligible nodes represents an
|
||||
// explicitly disabled site. It must remain visible to the planner, but it
|
||||
@@ -186,7 +200,7 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
|
||||
if state.CurrentReplicas == 0 {
|
||||
return state, nil
|
||||
}
|
||||
return KubernetesSiteState{}, fmt.Errorf("site %s has no matching nodes", site)
|
||||
return KubernetesSiteState{}, fmt.Errorf("site %s has no ready matching nodes", site)
|
||||
}
|
||||
var pods struct {
|
||||
Items []struct {
|
||||
|
||||
@@ -31,15 +31,24 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) {
|
||||
http.Error(w, "unexpected Worker node selector", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"easyai-hongkong"},"status":{
|
||||
_, _ = w.Write([]byte(`{"items":[
|
||||
{"metadata":{"name":"easyai-hongkong"},"status":{
|
||||
"allocatable":{"memory":"8Gi","cpu":"4"},
|
||||
"conditions":[{"type":"MemoryPressure","status":"False"}]
|
||||
"conditions":[{"type":"Ready","status":"False"}]
|
||||
}},
|
||||
{"metadata":{"name":"easyai-hongkong-worker-2"},"status":{
|
||||
"allocatable":{"memory":"8Gi","cpu":"4"},
|
||||
"conditions":[{"type":"Ready","status":"True"},{"type":"MemoryPressure","status":"False"}]
|
||||
}}]}`))
|
||||
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/api/v1/namespaces/easyai/pods"):
|
||||
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong"}}]}`))
|
||||
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong-worker-2"}}]}`))
|
||||
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
||||
strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong-worker-2"):
|
||||
_, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`))
|
||||
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
||||
strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong"):
|
||||
_, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`))
|
||||
t.Error("NotReady node metrics must not be queried")
|
||||
http.Error(w, "NotReady node", http.StatusInternalServerError)
|
||||
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
||||
strings.HasSuffix(request.URL.Path, "/pods"):
|
||||
_, _ = w.Write([]byte(`{"items":[{
|
||||
@@ -69,7 +78,7 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong" ||
|
||||
if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong-worker-2" ||
|
||||
state.WorkerRequestMemoryBytes != 512<<20 || state.WorkerRequestMilliCPU != 250 ||
|
||||
state.AllocatableMemoryBytes != 8<<30 || state.UsedMemoryBytes != 3<<30 ||
|
||||
state.Nodes[0].WorkerUsedMemoryBytes != 384<<20 || state.Nodes[0].WorkerUsedMilliCPU != 125 {
|
||||
@@ -148,7 +157,7 @@ func TestKubernetesClientRejectsActiveSiteWithoutWorkerNodes(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := client.SiteState(context.Background(), "ningbo"); err == nil ||
|
||||
!strings.Contains(err.Error(), "site ningbo has no matching nodes") {
|
||||
!strings.Contains(err.Error(), "site ningbo has no ready matching nodes") {
|
||||
t.Fatalf("expected missing Worker node error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user