fix(capacity): 忽略未就绪的 Worker 节点
香港站点保留了已离线的旧 control-plane 节点标签,容量控制器仍尝试读取其 metrics,导致 404 并阻断 readiness。容量采集现在只纳入 Ready=True 的候选节点,实际 Worker Pod 所在的就绪节点继续参与容量计算。 验证:容量控制器聚焦测试与 API 全量 go test 均通过,并覆盖同站点 NotReady 旧节点与 Ready Worker 节点并存场景。
This commit is contained in:
@@ -178,6 +178,20 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
|
|||||||
if err := client.getJSON(ctx, nodePath, &nodes); err != nil {
|
if err := client.getJSON(ctx, nodePath, &nodes); err != nil {
|
||||||
return KubernetesSiteState{}, err
|
return KubernetesSiteState{}, err
|
||||||
}
|
}
|
||||||
|
readyNodes := nodes.Items[:0]
|
||||||
|
for _, node := range nodes.Items {
|
||||||
|
ready := false
|
||||||
|
for _, condition := range node.Status.Conditions {
|
||||||
|
if condition.Type == "Ready" && condition.Status == "True" {
|
||||||
|
ready = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ready {
|
||||||
|
readyNodes = append(readyNodes, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodes.Items = readyNodes
|
||||||
if len(nodes.Items) == 0 {
|
if len(nodes.Items) == 0 {
|
||||||
// A zero-replica Deployment with no eligible nodes represents an
|
// A zero-replica Deployment with no eligible nodes represents an
|
||||||
// explicitly disabled site. It must remain visible to the planner, but it
|
// explicitly disabled site. It must remain visible to the planner, but it
|
||||||
@@ -186,7 +200,7 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub
|
|||||||
if state.CurrentReplicas == 0 {
|
if state.CurrentReplicas == 0 {
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
return KubernetesSiteState{}, fmt.Errorf("site %s has no matching nodes", site)
|
return KubernetesSiteState{}, fmt.Errorf("site %s has no ready matching nodes", site)
|
||||||
}
|
}
|
||||||
var pods struct {
|
var pods struct {
|
||||||
Items []struct {
|
Items []struct {
|
||||||
|
|||||||
@@ -31,15 +31,24 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) {
|
|||||||
http.Error(w, "unexpected Worker node selector", http.StatusBadRequest)
|
http.Error(w, "unexpected Worker node selector", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"easyai-hongkong"},"status":{
|
_, _ = w.Write([]byte(`{"items":[
|
||||||
|
{"metadata":{"name":"easyai-hongkong"},"status":{
|
||||||
"allocatable":{"memory":"8Gi","cpu":"4"},
|
"allocatable":{"memory":"8Gi","cpu":"4"},
|
||||||
"conditions":[{"type":"MemoryPressure","status":"False"}]
|
"conditions":[{"type":"Ready","status":"False"}]
|
||||||
|
}},
|
||||||
|
{"metadata":{"name":"easyai-hongkong-worker-2"},"status":{
|
||||||
|
"allocatable":{"memory":"8Gi","cpu":"4"},
|
||||||
|
"conditions":[{"type":"Ready","status":"True"},{"type":"MemoryPressure","status":"False"}]
|
||||||
}}]}`))
|
}}]}`))
|
||||||
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/api/v1/namespaces/easyai/pods"):
|
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/api/v1/namespaces/easyai/pods"):
|
||||||
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong"}}]}`))
|
_, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong-worker-2"}}]}`))
|
||||||
|
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
||||||
|
strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong-worker-2"):
|
||||||
|
_, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`))
|
||||||
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
||||||
strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong"):
|
strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong"):
|
||||||
_, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`))
|
t.Error("NotReady node metrics must not be queried")
|
||||||
|
http.Error(w, "NotReady node", http.StatusInternalServerError)
|
||||||
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") &&
|
||||||
strings.HasSuffix(request.URL.Path, "/pods"):
|
strings.HasSuffix(request.URL.Path, "/pods"):
|
||||||
_, _ = w.Write([]byte(`{"items":[{
|
_, _ = w.Write([]byte(`{"items":[{
|
||||||
@@ -69,7 +78,7 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong" ||
|
if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong-worker-2" ||
|
||||||
state.WorkerRequestMemoryBytes != 512<<20 || state.WorkerRequestMilliCPU != 250 ||
|
state.WorkerRequestMemoryBytes != 512<<20 || state.WorkerRequestMilliCPU != 250 ||
|
||||||
state.AllocatableMemoryBytes != 8<<30 || state.UsedMemoryBytes != 3<<30 ||
|
state.AllocatableMemoryBytes != 8<<30 || state.UsedMemoryBytes != 3<<30 ||
|
||||||
state.Nodes[0].WorkerUsedMemoryBytes != 384<<20 || state.Nodes[0].WorkerUsedMilliCPU != 125 {
|
state.Nodes[0].WorkerUsedMemoryBytes != 384<<20 || state.Nodes[0].WorkerUsedMilliCPU != 125 {
|
||||||
@@ -148,7 +157,7 @@ func TestKubernetesClientRejectsActiveSiteWithoutWorkerNodes(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := client.SiteState(context.Background(), "ningbo"); err == nil ||
|
if _, err := client.SiteState(context.Background(), "ningbo"); err == nil ||
|
||||||
!strings.Contains(err.Error(), "site ningbo has no matching nodes") {
|
!strings.Contains(err.Error(), "site ningbo has no ready matching nodes") {
|
||||||
t.Fatalf("expected missing Worker node error, got %v", err)
|
t.Fatalf("expected missing Worker node error, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user