diff --git a/apps/api/internal/capacitycontroller/kubernetes.go b/apps/api/internal/capacitycontroller/kubernetes.go index e6732ac..caa001a 100644 --- a/apps/api/internal/capacitycontroller/kubernetes.go +++ b/apps/api/internal/capacitycontroller/kubernetes.go @@ -178,6 +178,20 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub if err := client.getJSON(ctx, nodePath, &nodes); err != nil { return KubernetesSiteState{}, err } + readyNodes := nodes.Items[:0] + for _, node := range nodes.Items { + ready := false + for _, condition := range node.Status.Conditions { + if condition.Type == "Ready" && condition.Status == "True" { + ready = true + break + } + } + if ready { + readyNodes = append(readyNodes, node) + } + } + nodes.Items = readyNodes if len(nodes.Items) == 0 { // A zero-replica Deployment with no eligible nodes represents an // explicitly disabled site. It must remain visible to the planner, but it @@ -186,7 +200,7 @@ func (client *KubernetesClient) SiteState(ctx context.Context, site string) (Kub if state.CurrentReplicas == 0 { return state, nil } - return KubernetesSiteState{}, fmt.Errorf("site %s has no matching nodes", site) + return KubernetesSiteState{}, fmt.Errorf("site %s has no ready matching nodes", site) } var pods struct { Items []struct { diff --git a/apps/api/internal/capacitycontroller/kubernetes_test.go b/apps/api/internal/capacitycontroller/kubernetes_test.go index 8890c1a..6db6a5f 100644 --- a/apps/api/internal/capacitycontroller/kubernetes_test.go +++ b/apps/api/internal/capacitycontroller/kubernetes_test.go @@ -31,15 +31,24 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) { http.Error(w, "unexpected Worker node selector", http.StatusBadRequest) return } - _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"easyai-hongkong"},"status":{ + _, _ = w.Write([]byte(`{"items":[ + {"metadata":{"name":"easyai-hongkong"},"status":{ "allocatable":{"memory":"8Gi","cpu":"4"}, - "conditions":[{"type":"MemoryPressure","status":"False"}] + "conditions":[{"type":"Ready","status":"False"}] + }}, + {"metadata":{"name":"easyai-hongkong-worker-2"},"status":{ + "allocatable":{"memory":"8Gi","cpu":"4"}, + "conditions":[{"type":"Ready","status":"True"},{"type":"MemoryPressure","status":"False"}] }}]}`)) case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/api/v1/namespaces/easyai/pods"): - _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong"}}]}`)) + _, _ = w.Write([]byte(`{"items":[{"metadata":{"name":"worker-hongkong-1"},"spec":{"nodeName":"easyai-hongkong-worker-2"}}]}`)) + case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && + strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong-worker-2"): + _, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`)) case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && strings.HasSuffix(request.URL.Path, "/nodes/easyai-hongkong"): - _, _ = w.Write([]byte(`{"usage":{"memory":"3Gi","cpu":"500m"}}`)) + t.Error("NotReady node metrics must not be queried") + http.Error(w, "NotReady node", http.StatusInternalServerError) case request.Method == http.MethodGet && strings.Contains(request.URL.Path, "/metrics.k8s.io/") && strings.HasSuffix(request.URL.Path, "/pods"): _, _ = w.Write([]byte(`{"items":[{ @@ -69,7 +78,7 @@ func TestKubernetesClientReadsSiteAndMutatesOnlyWorkerScale(t *testing.T) { if err != nil { t.Fatal(err) } - if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong" || + if state.CurrentReplicas != 1 || state.NodeName != "easyai-hongkong-worker-2" || state.WorkerRequestMemoryBytes != 512<<20 || state.WorkerRequestMilliCPU != 250 || state.AllocatableMemoryBytes != 8<<30 || state.UsedMemoryBytes != 3<<30 || state.Nodes[0].WorkerUsedMemoryBytes != 384<<20 || state.Nodes[0].WorkerUsedMilliCPU != 125 { @@ -148,7 +157,7 @@ func TestKubernetesClientRejectsActiveSiteWithoutWorkerNodes(t *testing.T) { t.Fatal(err) } if _, err := client.SiteState(context.Background(), "ningbo"); err == nil || - !strings.Contains(err.Error(), "site ningbo has no matching nodes") { + !strings.Contains(err.Error(), "site ningbo has no ready matching nodes") { t.Fatalf("expected missing Worker node error, got %v", err) } }