diff --git a/test/extended/node/node_e2e/container_runtime_config.go b/test/extended/node/node_e2e/container_runtime_config.go new file mode 100644 index 000000000000..dc9c0d39fb1a --- /dev/null +++ b/test/extended/node/node_e2e/container_runtime_config.go @@ -0,0 +1,287 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/origin/test/extended/imagepolicy" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/utils/ptr" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] ContainerRuntimeConfig", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("ctrcfg") + ) + + g.BeforeEach(func() { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to detect MicroShift cluster") + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster - MachineConfig resources are not available") + } + }) + + // Validates that ContainerRuntimeConfig pidsLimit setting is correctly applied + // by MCO to a single worker node and that manual crio.conf edits are overwritten. + //author: cmaurya@redhat.com + g.It("[OTP] Verify pidsLimit and MCO overwrite behavior [OCP-45351]", func() { + ctx := context.Background() + ctrcfgName := "set-pids-limit" + mcpName := "ctrcfg-pids" + + g.By("Get a ready worker node") + workers, err := exutil.GetReadySchedulableWorkerNodes(ctx, oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get ready schedulable worker nodes") + o.Expect(workers).NotTo(o.BeEmpty(), "No Ready worker nodes found") + workerNode := workers[0].Name + + g.By("Make a manual change to crio.conf on worker node") + _, err = nodeutils.ExecOnNodeWithChroot(oc, workerNode, + "/bin/bash", "-c", `sed -i '/^\[crio\.runtime\]/a log_level = "debug"' /etc/crio/crio.conf`) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to edit crio.conf on node %s", workerNode) + + g.By("Verify the manual crio.conf edit took effect") + editedConf, err := nodeutils.ExecOnNodeWithChroot(oc, workerNode, "cat", "/etc/crio/crio.conf") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read crio.conf on node %s", workerNode) + o.Expect(editedConf).To(o.ContainSubstring(`log_level = "debug"`), + "sed edit did not apply: expected log_level = debug in crio.conf") + + createSingleNodeMCP(ctx, oc, mcpName, workerNode) + + g.DeferCleanup(func() { + g.By("Cleanup: delete ContainerRuntimeConfig") + delErr := oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Delete( + ctx, ctrcfgName, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(delErr) { + o.Expect(delErr).NotTo(o.HaveOccurred(), + "cleanup failed: could not delete ContainerRuntimeConfig %s", ctrcfgName) + } + cleanupSingleNodeMCP(ctx, oc, mcpName, workerNode) + }) + + initialSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, mcpName) + + g.By("Create ContainerRuntimeConfig with pidsLimit 2048") + ctrcfg := &mcfgv1.ContainerRuntimeConfig{ + ObjectMeta: metav1.ObjectMeta{Name: ctrcfgName}, + Spec: mcfgv1.ContainerRuntimeConfigSpec{ + MachineConfigPoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"machineconfiguration.openshift.io/pool": mcpName}, + }, + ContainerRuntimeConfig: &mcfgv1.ContainerRuntimeConfiguration{ + PidsLimit: ptr.To[int64](2048), + }, + }, + } + _, err = oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Create( + ctx, ctrcfg, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ContainerRuntimeConfig") + + g.By("Wait for custom MCP rollout to complete") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, mcpName, initialSpec) + e2e.Logf("Worker node rolled out successfully") + + g.By("Verify pidsLimit and conmon in crio config on worker node") + var crioConfig string + o.Eventually(func() error { + var execErr error + crioConfig, execErr = nodeutils.ExecOnNodeWithChroot(oc, workerNode, + "/bin/bash", "-c", "crio config 2>/dev/null") + return execErr + }, 30*time.Second, 5*time.Second).Should(o.Succeed(), "failed to get crio config on node %s", workerNode) + o.Expect(crioConfig).To(o.ContainSubstring("pids_limit = 2048"), "pidsLimit should be 2048") + o.Expect(crioConfig).To(o.ContainSubstring(`conmon = ""`), "conmon should be empty") + o.Expect(crioConfig).NotTo(o.ContainSubstring(`log_level = "debug"`), + "manual crio.conf edit should be overwritten by MCO") + }) + + // Validates that setting overlaySize in ContainerRuntimeConfig is applied to + // storage.conf on a single worker node and the overlay size is reflected inside a container. + //author: cmaurya@redhat.com + g.It("[OTP] Verify overlaySize is applied to node and container [OCP-46313]", func() { + oc.SetupProject() + ctx := context.Background() + ctrcfgName := "ctrcfg-46313" + mcpName := "ctrcfg-overlay" + overlaySize := "9G" + + g.By("Get a ready worker node") + workers, err := exutil.GetReadySchedulableWorkerNodes(ctx, oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get ready schedulable worker nodes") + o.Expect(workers).NotTo(o.BeEmpty(), "No Ready worker nodes found") + workerNode := workers[0].Name + + createSingleNodeMCP(ctx, oc, mcpName, workerNode) + + g.DeferCleanup(func() { + g.By("Cleanup: delete ContainerRuntimeConfig") + delErr := oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Delete( + ctx, ctrcfgName, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(delErr) { + o.Expect(delErr).NotTo(o.HaveOccurred(), + "cleanup failed: could not delete ContainerRuntimeConfig %s", ctrcfgName) + } + cleanupSingleNodeMCP(ctx, oc, mcpName, workerNode) + }) + + initialSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, mcpName) + + g.By("Create ContainerRuntimeConfig with overlaySize " + overlaySize) + quantity := resource.MustParse(overlaySize) + ctrcfg := &mcfgv1.ContainerRuntimeConfig{ + ObjectMeta: metav1.ObjectMeta{Name: ctrcfgName}, + Spec: mcfgv1.ContainerRuntimeConfigSpec{ + MachineConfigPoolSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"machineconfiguration.openshift.io/pool": mcpName}, + }, + ContainerRuntimeConfig: &mcfgv1.ContainerRuntimeConfiguration{ + OverlaySize: &quantity, + }, + }, + } + _, err = oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().Create( + ctx, ctrcfg, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ContainerRuntimeConfig") + + g.By("Wait for custom MCP rollout to complete") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, mcpName, initialSpec) + e2e.Logf("Worker node rolled out successfully") + + g.By("Check overlaySize takes effect in storage.conf on worker node") + storageConf, err := nodeutils.ExecOnNodeWithChroot(oc, workerNode, + "/bin/bash", "-c", "head -n 7 /etc/containers/storage.conf | grep size") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read storage.conf on node %s", workerNode) + e2e.Logf("storage.conf size line: %s", storageConf) + o.Expect(storageConf).To(o.ContainSubstring(overlaySize), + "storage.conf should contain size = %s", overlaySize) + + g.By("Create a pod on the target node to verify overlay size inside container") + podName := "pod-46313" + ns := oc.Namespace() + err = oc.AsAdmin().WithoutNamespace().Run("run").Args( + podName, "-n", ns, + "--image=quay.io/openshifttest/hello-openshift@sha256:56c354e7885051b6bb4263f9faa58b2c292d44790599b7dde0e49e7c466cf339", + "--restart=Never", + "--overrides", `{"spec":{"nodeName":"`+workerNode+`","securityContext":{"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}},"containers":[{"name":"`+podName+`","image":"quay.io/openshifttest/hello-openshift@sha256:56c354e7885051b6bb4263f9faa58b2c292d44790599b7dde0e49e7c466cf339","command":["/bin/bash","-c","sleep 100000000"],"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}`, + ).Execute() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create pod") + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", podName, "-n", ns, "--ignore-not-found").Execute() + + g.By("Wait for pod to be running") + err = wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) { + phase, pollErr := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "pod", podName, "-n", ns, "-o=jsonpath={.status.phase}").Output() + if pollErr != nil { + return false, nil + } + return phase == "Running", nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "pod did not reach Running state") + + g.By("Check overlay filesystem size inside the container") + dfOutput, err := oc.AsAdmin().WithoutNamespace().Run("rsh").Args( + "-n", ns, podName, "/bin/bash", "-c", "df -h / | grep overlay").Output() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to exec df inside pod") + e2e.Logf("overlay df output: %s", dfOutput) + fields := strings.Fields(dfOutput) + o.Expect(len(fields)).To(o.BeNumerically(">=", 2), "unexpected df output format: %s", dfOutput) + actualSize := strings.Split(strings.TrimSuffix(fields[1], "G"), ".")[0] + "G" + o.Expect(actualSize).To(o.Equal(overlaySize), + "overlay filesystem should show %s, got: %s", overlaySize, actualSize) + }) +}) + +// createSingleNodeMCP creates a custom MachineConfigPool that targets exactly one worker node. +// It labels the node to move it into the custom pool and waits until the pool reports 1 node. +func createSingleNodeMCP(ctx context.Context, oc *exutil.CLI, mcpName, workerNode string) { + nodeLabel := "node-role.kubernetes.io/" + mcpName + + g.By("Create a custom MachineConfigPool targeting a single worker node") + mcp := &mcfgv1.MachineConfigPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: mcpName, + Labels: map[string]string{"machineconfiguration.openshift.io/pool": mcpName}, + }, + Spec: mcfgv1.MachineConfigPoolSpec{ + MachineConfigSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "machineconfiguration.openshift.io/role", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"worker", mcpName}, + }, + }, + }, + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{nodeLabel: ""}, + }, + }, + } + _, err := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigPools().Create(ctx, mcp, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create custom MachineConfigPool %s", mcpName) + + g.By("Label worker node to move it into the custom MCP") + patch := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:""}}}`, nodeLabel)) + _, err = oc.AdminKubeClient().CoreV1().Nodes().Patch(ctx, workerNode, types.MergePatchType, patch, metav1.PatchOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to label node %s", workerNode) + + g.By("Wait for the custom MCP to report the node") + o.Eventually(func() int { + pool, getErr := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigPools().Get(ctx, mcpName, metav1.GetOptions{}) + if getErr != nil { + return 0 + } + return int(pool.Status.MachineCount) + }, 2*time.Minute, 10*time.Second).Should(o.Equal(1), "custom MCP %s should have 1 node", mcpName) +} + +// cleanupSingleNodeMCP removes the node label, waits for the node to transition back to the +// worker pool config, and then deletes the custom MCP. +func cleanupSingleNodeMCP(ctx context.Context, oc *exutil.CLI, mcpName, workerNode string) { + nodeLabel := "node-role.kubernetes.io/" + mcpName + + g.By("Cleanup: remove node label to move node back to worker pool") + patch := []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:null}}}`, nodeLabel)) + _, err := oc.AdminKubeClient().CoreV1().Nodes().Patch(ctx, workerNode, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + e2e.Logf("WARNING: failed to remove label from node %s: %v", workerNode, err) + } + + g.By("Cleanup: wait for node to transition back to worker config") + o.Eventually(func() bool { + node, getErr := oc.AdminKubeClient().CoreV1().Nodes().Get(ctx, workerNode, metav1.GetOptions{}) + if getErr != nil { + e2e.Logf("Error getting node: %v", getErr) + return false + } + currentConfig := node.Annotations["machineconfiguration.openshift.io/currentConfig"] + desiredConfig := node.Annotations["machineconfiguration.openshift.io/desiredConfig"] + isWorkerConfig := currentConfig != "" && !strings.Contains(currentConfig, mcpName) && currentConfig == desiredConfig + if !isWorkerConfig { + e2e.Logf("Node %s still transitioning: current=%s, desired=%s", workerNode, currentConfig, desiredConfig) + } + return isWorkerConfig + }, 15*time.Minute, 15*time.Second).Should(o.BeTrue(), + "node %s should transition back to worker pool config", workerNode) + + g.By("Cleanup: delete custom MachineConfigPool") + delErr := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigPools().Delete(ctx, mcpName, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(delErr) && delErr != nil { + e2e.Logf("WARNING: failed to delete MachineConfigPool %s: %v", mcpName, delErr) + } +} diff --git a/test/extended/node/node_e2e/image_registry_config.go b/test/extended/node/node_e2e/image_registry_config.go new file mode 100644 index 000000000000..ce3a02189813 --- /dev/null +++ b/test/extended/node/node_e2e/image_registry_config.go @@ -0,0 +1,125 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + "github.com/openshift/origin/test/extended/imagepolicy" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + e2e "k8s.io/kubernetes/test/e2e/framework" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" + operator "github.com/openshift/origin/test/extended/util/operator" +) + +var _ = g.Describe("[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] Image registry config", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("imgcfg") + ) + + g.BeforeEach(func() { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to detect cluster type") + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster - MachineConfig resources are not available") + } + }) + + // Verifies that updating image.config.openshift.io/cluster with a new search + // registry triggers an MCO rollout and the change lands on nodes. + //author: cmaurya@redhat.com + g.It("[OTP] change container registry config [OCP-44820]", func() { + ctx := context.Background() + searchRegistry := "qe.quay.io" + + g.By("Save the original image.config for later restore") + originalImageConfig, err := oc.AdminConfigClient().ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get image.config.openshift.io/cluster") + + initialWorkerSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "worker") + initialMasterSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "master") + + g.DeferCleanup(func() { + e2e.Logf("Cleanup: restoring original image.config") + restoreErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + current, getErr := oc.AdminConfigClient().ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{}) + if getErr != nil { + return getErr + } + current.Spec.RegistrySources = originalImageConfig.Spec.RegistrySources + _, updateErr := oc.AdminConfigClient().ConfigV1().Images().Update(ctx, current, metav1.UpdateOptions{}) + return updateErr + }) + o.Expect(restoreErr).NotTo(o.HaveOccurred(), + "cleanup failed: could not restore original image.config") + + cleanupWorkerSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "worker") + cleanupMasterSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "master") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "worker", cleanupWorkerSpec) + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "master", cleanupMasterSpec) + + e2e.Logf("Cleanup: waiting for all cluster operators to settle") + waitErr := operator.WaitForOperatorsToSettle(ctx, oc.AdminConfigClient(), 10) + o.Expect(waitErr).NotTo(o.HaveOccurred(), + "cluster operators did not settle after restore") + }) + + g.By("Update image.config to add search registry and allowed registries") + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + imageConfig, getErr := oc.AdminConfigClient().ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{}) + if getErr != nil { + return getErr + } + imageConfig.Spec.RegistrySources.AllowedRegistries = []string{ + "registry.access.redhat.com", "docker.io", "quay.io", searchRegistry, + "image-registry.openshift-image-registry.svc:5000", + } + imageConfig.Spec.RegistrySources.ContainerRuntimeSearchRegistries = []string{ + "registry.access.redhat.com", "docker.io", "quay.io", searchRegistry, + } + _, updateErr := oc.AdminConfigClient().ConfigV1().Images().Update(ctx, imageConfig, metav1.UpdateOptions{}) + return updateErr + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to update image.config.openshift.io/cluster") + + g.By("Wait for worker and master MCP rollout to complete") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "worker", initialWorkerSpec) + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "master", initialMasterSpec) + + g.By("Verify search registries config on a worker node") + workers, err := exutil.GetReadySchedulableWorkerNodes(ctx, oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get ready schedulable worker nodes") + o.Expect(workers).NotTo(o.BeEmpty(), "no ready worker nodes found") + + var registriesConf string + o.Eventually(func() error { + var execErr error + registriesConf, execErr = nodeutils.ExecOnNodeWithChroot(oc, workers[0].Name, + "cat", "/etc/containers/registries.conf.d/01-image-searchRegistries.conf") + if execErr != nil { + return execErr + } + if !strings.Contains(registriesConf, searchRegistry) { + return fmt.Errorf("search registry %s not yet in config", searchRegistry) + } + return nil + }, 30*time.Second, 5*time.Second).Should(o.Succeed(), + "search registry %s not found in registries config on node %s", searchRegistry, workers[0].Name) + e2e.Logf("Registries config on %s:\n%s", workers[0].Name, registriesConf) + + g.By("Verify policy.json is updated with allowed registries") + policyJSON, err := nodeutils.ExecOnNodeWithChroot(oc, workers[0].Name, + "cat", "/etc/containers/policy.json") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read policy.json on node %s", workers[0].Name) + e2e.Logf("policy.json on %s:\n%s", workers[0].Name, policyJSON) + o.Expect(policyJSON).To(o.ContainSubstring(searchRegistry), + "policy.json should contain allowed registry %s", searchRegistry) + }) +}) diff --git a/test/extended/node/node_e2e/initcontainer.go b/test/extended/node/node_e2e/initcontainer.go new file mode 100644 index 000000000000..6e095fe222d7 --- /dev/null +++ b/test/extended/node/node_e2e/initcontainer.go @@ -0,0 +1,153 @@ +package node + +import ( + "context" + "fmt" + "regexp" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] NODE initContainer policy,volume,readiness,quota", func() { + defer g.GinkgoRecover() + + var ( + oc = exutil.NewCLI("node-initcontainer") + ) + + // Skip all tests on MicroShift clusters as MachineConfig resources are not available + g.BeforeEach(func() { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster - MachineConfig resources are not available") + } + }) + + //author: bgudi@redhat.com + g.It("[OTP] Init containers should not restart when the exited init container is removed from node [OCP-38271]", func() { + g.By("Test for case OCP-38271") + oc.SetupProject() + + podName := "initcon-pod" + namespace := oc.Namespace() + ctx := context.Background() + + g.By("Create a pod with init container") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "inittest", + Image: "image-registry.openshift-image-registry.svc:5000/openshift/tools:latest", + Command: []string{"/bin/sh", "-ec", "echo running >> /mnt/data/test"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/mnt/data", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "hello-test", + Image: "image-registry.openshift-image-registry.svc:5000/openshift/tools:latest", + Command: []string{"/bin/sh", "-c", "sleep 3600"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/mnt/data", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } + + _, err := oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + defer func() { + oc.KubeClient().CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + }() + + g.By("Check pod status") + err = e2epod.WaitForPodRunningInNamespace(ctx, oc.KubeClient(), pod) + o.Expect(err).NotTo(o.HaveOccurred(), "pod is not running") + + g.By("Get pod and verify init container exited normally") + pod, err = oc.KubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + o.Expect(pod.Status.InitContainerStatuses).To(o.ContainElement(o.SatisfyAll( + o.HaveField("Name", "inittest"), + o.HaveField("State.Terminated.ExitCode", o.Equal(int32(0))), + )), "init container 'inittest' should have terminated with exit code 0") + + nodeName := pod.Spec.NodeName + o.Expect(nodeName).NotTo(o.BeEmpty(), "pod node name is empty") + + g.By("Get init container ID from pod status") + var containerID string + for _, status := range pod.Status.InitContainerStatuses { + if status.Name == "inittest" { + containerID = status.ContainerID + break + } + } + o.Expect(containerID).NotTo(o.BeEmpty(), "init container ID is empty") + + // Extract the actual container ID (remove prefix like "cri-o://") + containerIDPattern := regexp.MustCompile(`^[^/]+://(.+)$`) + matches := containerIDPattern.FindStringSubmatch(containerID) + o.Expect(matches).To(o.HaveLen(2), "failed to parse container ID") + actualContainerID := matches[1] + + g.By("Delete init container from node") + output, err := nodeutils.ExecOnNodeWithChroot(oc, nodeName, "crictl", "rm", actualContainerID) + o.Expect(err).NotTo(o.HaveOccurred(), "fail to delete container") + e2e.Logf("Container deletion output: %s", output) + + g.By("Check init container not restart again") + err = wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { + pod, err := oc.KubeClient().CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, status := range pod.Status.InitContainerStatuses { + if status.Name == "inittest" { + if status.RestartCount > 0 { + e2e.Logf("Init container restarted, restart count: %d", status.RestartCount) + return true, fmt.Errorf("init container restarted") + } + } + } + e2e.Logf("Init container has not restarted") + return false, nil + }) + o.Expect(err).To(o.Equal(wait.ErrWaitTimeout), "expected timeout while waiting confirms init container did not restart") + }) +}) diff --git a/test/extended/node/node_e2e/node.go b/test/extended/node/node_e2e/node.go index a7814374b0ca..5f43c93e20af 100644 --- a/test/extended/node/node_e2e/node.go +++ b/test/extended/node/node_e2e/node.go @@ -2,17 +2,23 @@ package node import ( "context" + "fmt" "path/filepath" "strings" "time" g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" - nodeutils "github.com/openshift/origin/test/extended/node" - exutil "github.com/openshift/origin/test/extended/util" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/origin/test/extended/imagepolicy" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/wait" e2e "k8s.io/kubernetes/test/e2e/framework" + + nodeutils "github.com/openshift/origin/test/extended/node" + exutil "github.com/openshift/origin/test/extended/util" ) var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager", func() { @@ -114,13 +120,18 @@ var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager", podName := "pod-devfuse" ns := "devfuse-test" - g.By("Check if the default CRI-O runtime is runc") - ctrcfgList, err := oc.MachineConfigurationClient().MachineconfigurationV1().ContainerRuntimeConfigs().List(context.Background(), metav1.ListOptions{}) + // Skip on runc: io.kubernetes.cri-o.Devices annotation is only in crun's allowed_annotations. + // We query crio config directly as ContainerRuntimeConfig API misses platform-default runc. + g.By("Skip if the default runtime is runc") + node, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "nodes", "-l", "node-role.kubernetes.io/worker", "-o=jsonpath={.items[0].metadata.name}").Output() o.Expect(err).NotTo(o.HaveOccurred()) - for _, cfg := range ctrcfgList.Items { - if cfg.Spec.ContainerRuntimeConfig != nil && cfg.Spec.ContainerRuntimeConfig.DefaultRuntime == "runc" { - g.Skip("Skipping: not applicable to runc runtime") - } + o.Expect(node).NotTo(o.BeEmpty()) + runtime, err := nodeutils.ExecOnNodeWithChroot(oc, node, "/bin/bash", "-c", + "crio status config 2>/dev/null | awk -F'\"' '/default_runtime/{print $2}'") + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.TrimSpace(runtime) == "runc" { + g.Skip("Skipping: not applicable to runc runtime") } g.By("Create a test namespace") @@ -154,3 +165,148 @@ var _ = g.Describe("[sig-node] [Jira:Node/Kubelet] Kubelet, CRI-O, CPU manager", o.Expect(output).To(o.ContainSubstring("fuse"), "dev fuse is not mounted inside pod") }) }) + +// author: asahay@redhat.com +var _ = g.Describe("[sig-node][Suite:openshift/disruptive-longrunning][Disruptive][Serial] ImageTagMirrorSet and ImageDigestMirrorSet", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("image-mirror-set") + ctx = context.Background() + ) + + g.BeforeEach(func() { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster - MachineConfig resources are not available") + } + }) + + g.It("[OTP] Create ImageDigestMirrorSet and ImageTagMirrorSet and verify registries.conf [OCP-57401]", func() { + configClient := oc.AdminConfigClient().ConfigV1() + suffix := utilrand.String(5) + idmsName := fmt.Sprintf("digest-mirror-%s", suffix) + itmsName := fmt.Sprintf("tag-mirror-%s", suffix) + + g.By("Step 1: Create an ImageDigestMirrorSet") + idms := &configv1.ImageDigestMirrorSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: idmsName, + }, + Spec: configv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []configv1.ImageDigestMirrors{ + { + Source: "registry.redhat.io/openshift4", + Mirrors: []configv1.ImageMirror{ + "mirror.example.com/redhat", + }, + MirrorSourcePolicy: configv1.AllowContactingSource, + }, + { + Source: "registry.redhat.io/rhel8", + Mirrors: []configv1.ImageMirror{ + "mirror.example.com/rhel8", + }, + MirrorSourcePolicy: configv1.NeverContactSource, + }, + }, + }, + } + + initialWorkerSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "worker") + initialMasterSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "master") + + createdIDMS, err := configClient.ImageDigestMirrorSets().Create(ctx, idms, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ImageDigestMirrorSet") + e2e.Logf("ImageDigestMirrorSet %q created successfully", createdIDMS.Name) + + g.DeferCleanup(func() { + g.By("Cleanup: Delete IDMS and ITMS resources") + cleanupWorkerSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "worker") + cleanupMasterSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "master") + if delErr := configClient.ImageTagMirrorSets().Delete(ctx, itmsName, metav1.DeleteOptions{}); delErr != nil { + e2e.Logf("Warning: failed to delete ImageTagMirrorSet: %v", delErr) + } + if delErr := configClient.ImageDigestMirrorSets().Delete(ctx, idmsName, metav1.DeleteOptions{}); delErr != nil { + e2e.Logf("Warning: failed to delete ImageDigestMirrorSet: %v", delErr) + } + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "worker", cleanupWorkerSpec) + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "master", cleanupMasterSpec) + }) + + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "worker", initialWorkerSpec) + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "master", initialMasterSpec) + e2e.Logf("IDMS MCP rollout complete") + + g.By("Step 2: Create an ImageTagMirrorSet") + itms := &configv1.ImageTagMirrorSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: itmsName, + }, + Spec: configv1.ImageTagMirrorSetSpec{ + ImageTagMirrors: []configv1.ImageTagMirrors{ + { + Source: "registry.access.redhat.com/ubi8/ubi-minimal", + Mirrors: []configv1.ImageMirror{ + "example.io/example/ubi-minimal", + "example.com/example/ubi-minimal", + }, + MirrorSourcePolicy: configv1.AllowContactingSource, + }, + { + Source: "registry.access.redhat.com/ubi8/ubi-minimal-1", + Mirrors: []configv1.ImageMirror{ + "example.io/example/ubi-minimal", + }, + MirrorSourcePolicy: configv1.NeverContactSource, + }, + }, + }, + } + + itmsWorkerSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "worker") + itmsMasterSpec := imagepolicy.GetMCPCurrentSpecConfigName(oc, "master") + + createdITMS, err := configClient.ImageTagMirrorSets().Create(ctx, itms, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create ImageTagMirrorSet") + e2e.Logf("ImageTagMirrorSet %q created successfully", createdITMS.Name) + + g.By("Step 3: Wait for all nodes to finish rolling out") + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "worker", itmsWorkerSpec) + imagepolicy.WaitForMCPConfigSpecChangeAndUpdated(oc, "master", itmsMasterSpec) + e2e.Logf("All MCPs have finished rolling out") + + g.By("Step 4: Verify /etc/containers/registries.conf on a worker node") + workerNodeName := nodeutils.GetFirstReadyWorkerNode(oc) + o.Expect(workerNodeName).NotTo(o.BeEmpty(), "no ready worker node found") + + registriesConf, err := nodeutils.ExecOnNodeWithChroot(oc, workerNodeName, "cat", "/etc/containers/registries.conf") + o.Expect(err).NotTo(o.HaveOccurred(), "failed to read registries.conf from node %s", workerNodeName) + e2e.Logf("registries.conf content:\n%s", registriesConf) + + g.By("Verify IDMS entries (digest-only mirrors)") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.redhat.io/openshift4"`), + "registries.conf should contain the IDMS source for openshift4") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "mirror.example.com/redhat"`), + "registries.conf should contain the IDMS mirror for openshift4") + o.Expect(registriesConf).To(o.ContainSubstring(`pull-from-mirror = "digest-only"`), + "registries.conf should have pull-from-mirror set to digest-only for IDMS mirrors") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.redhat.io/rhel8"`), + "registries.conf should contain the IDMS source for rhel8") + + g.By("Verify ITMS entries (tag-only mirrors)") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal"`), + "registries.conf should contain the ITMS source for ubi-minimal") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "example.io/example/ubi-minimal"`), + "registries.conf should contain the ITMS mirror location") + o.Expect(registriesConf).To(o.ContainSubstring(`pull-from-mirror = "tag-only"`), + "registries.conf should have pull-from-mirror set to tag-only for ITMS mirrors") + o.Expect(registriesConf).To(o.ContainSubstring(`location = "registry.access.redhat.com/ubi8/ubi-minimal-1"`), + "registries.conf should contain the ITMS source for ubi-minimal-1") + + g.By("Verify NeverContactSource entries are blocked") + o.Expect(registriesConf).To(o.ContainSubstring("location = \"registry.access.redhat.com/ubi8/ubi-minimal-1\"\n blocked = true"), + "registry.access.redhat.com/ubi8/ubi-minimal-1 should be blocked (NeverContactSource)") + o.Expect(registriesConf).To(o.ContainSubstring("location = \"registry.redhat.io/rhel8\"\n blocked = true"), + "registry.redhat.io/rhel8 should be blocked (NeverContactSource)") + }) +}) diff --git a/test/extended/node/node_e2e/pdb_drain.go b/test/extended/node/node_e2e/pdb_drain.go new file mode 100644 index 000000000000..dc1c95ce14d7 --- /dev/null +++ b/test/extended/node/node_e2e/pdb_drain.go @@ -0,0 +1,200 @@ +package node + +import ( + "context" + "fmt" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/utils/ptr" + + exutil "github.com/openshift/origin/test/extended/util" + "github.com/openshift/origin/test/extended/util/operator" +) + +var _ = g.Describe("[Suite:openshift/disruptive-longrunning][sig-node][Disruptive] PodDisruptionBudget", func() { + var ( + oc = exutil.NewCLIWithoutNamespace("pdb-drain") + ) + + g.BeforeEach(func() { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("Skipping test on MicroShift cluster") + } + }) + + //author: bgudi@redhat.com + g.It("[OTP] Node's drain should block when PodDisruptionBudget minAvailable equals 100 percentage and selector is empty [OCP-67564]", ote.Informing(), func() { + ctx := context.Background() + + // Skip on SNO/External topologies where there might not be dedicated worker nodes + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get cluster infrastructure") + if infra.Status.ControlPlaneTopology == "SingleReplica" || infra.Status.ControlPlaneTopology == "External" { + g.Skip("Skipping on SNO/External topology - requires dedicated worker nodes") + } + + oc.SetupProject() + namespace := oc.Namespace() + + g.By("Get a worker node to schedule pods on") + workers, err := exutil.GetReadySchedulableWorkerNodes(ctx, oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get worker nodes") + o.Expect(workers).NotTo(o.BeEmpty(), "no ready schedulable worker nodes found") + workerNode := workers[0].Name + e2e.Logf("Selected worker node: %s", workerNode) + + g.By("Create 6 pods on the selected worker node") + numPods := 6 + podBaseName := "pdb-drain-test-pod" + for i := 0; i < numPods; i++ { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", podBaseName, i), + Namespace: namespace, + Labels: map[string]string{ + "app": "pdb-drain-test", + }, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": workerNode, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + Containers: []corev1.Container{ + { + Name: "test-container", + Image: "quay.io/openshifttest/hello-openshift@sha256:4200f438cf2e9446f6bcff9d67ceea1f69ed07a2f83363b7fb52529f7ddd8a83", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, + }, + }, + } + _, err = oc.KubeClient().CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("failed to create pod %d", i)) + } + + g.By("Wait for all pods to be ready") + err = wait.PollUntilContextTimeout(ctx, 3*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + podList, pollErr := oc.KubeClient().CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=pdb-drain-test", + }) + if pollErr != nil { + e2e.Logf("Error getting pods: %v", pollErr) + return false, nil + } + readyPods := 0 + for _, pod := range podList.Items { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + readyPods++ + break + } + } + } + if readyPods == numPods { + e2e.Logf("All %d pods are ready", readyPods) + return true, nil + } + e2e.Logf("Waiting for pods to be ready: %d/%d", readyPods, numPods) + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "pods did not become ready") + + g.By("Create PodDisruptionBudget with 100% minAvailable and empty selector") + pdb := &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pdb-drain-test", + Namespace: namespace, + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "100%", + }, + Selector: &metav1.LabelSelector{}, + }, + } + _, err = oc.KubeClient().PolicyV1().PodDisruptionBudgets(namespace).Create(ctx, pdb, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create PodDisruptionBudget") + g.DeferCleanup(oc.KubeClient().PolicyV1().PodDisruptionBudgets(namespace).Delete, ctx, "pdb-drain-test", metav1.DeleteOptions{}) + + g.By("Verify all test pods are on the selected worker node") + podList, err := oc.KubeClient().CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=pdb-drain-test", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pods") + podsOnWorker := 0 + for _, pod := range podList.Items { + if pod.Spec.NodeName == workerNode { + podsOnWorker++ + } + } + o.Expect(podsOnWorker).To(o.Equal(numPods), "not all pods are on the selected worker node") + + g.By("Make sure that PDB's DisruptionAllowed condition is False") + var pdbStatus string + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 30*time.Second, true, func(pollCtx context.Context) (bool, error) { + var pollErr error + pdbStatus, pollErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("poddisruptionbudget", "pdb-drain-test", "-n", namespace, "-o=jsonpath={.status.conditions[?(@.type==\"DisruptionAllowed\")].status}").Output() + if pollErr != nil { + e2e.Logf("Error getting PDB status: %v", pollErr) + return false, nil + } + if pdbStatus != "" { + return true, nil + } + e2e.Logf("Waiting for PDB DisruptionAllowed condition to appear") + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "PDB DisruptionAllowed condition not found") + o.Expect(pdbStatus).Should(o.Equal("False"), "PDB DisruptionAllowed should be False") + + g.By("Drain the selected worker node") + g.DeferCleanup(func() { + err := operator.WaitForOperatorsToSettle(ctx, oc.AdminConfigClient(), 10) + o.Expect(err).NotTo(o.HaveOccurred(), "cluster operators failed to return to available state after node drain") + }) + g.DeferCleanup(oc.AsAdmin().WithoutNamespace().Run("adm").Args("uncordon", workerNode).Execute) + + out, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args("drain", workerNode, "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--timeout=30s").Output() + o.Expect(err).To(o.HaveOccurred(), "drain operation should have been blocked but it wasn't") + o.Expect(strings.Contains(out, "Cannot evict pod as it would violate the pod's disruption budget")).Should(o.BeTrue(), "drain output missing PDB violation error message") + o.Expect(strings.Contains(out, "There are pending nodes to be drained")).Should(o.BeTrue(), "drain output missing pending nodes error message") + + g.By("Verify that test pods remain on the node after failed drain") + podsAfterDrain, err := oc.KubeClient().CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=pdb-drain-test", + }) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get pods after drain attempt") + podsStillOnWorker := 0 + for _, pod := range podsAfterDrain.Items { + if pod.Spec.NodeName == workerNode { + podsStillOnWorker++ + } + } + o.Expect(podsStillOnWorker).To(o.Equal(numPods), "all test pods should still be on the worker node") + }) +}) diff --git a/test/extended/node/node_utils.go b/test/extended/node/node_utils.go index 8af84984b0d9..f87a40dcd3fb 100644 --- a/test/extended/node/node_utils.go +++ b/test/extended/node/node_utils.go @@ -741,3 +741,26 @@ func ensureDropInDirectoryExists(ctx context.Context, oc *exutil.CLI, dirPath st return nil } + +// GetFirstReadyWorkerNode returns the name of the first Ready worker node in the cluster. +func GetFirstReadyWorkerNode(oc *exutil.CLI) string { + nodeNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "nodes", "-l", "node-role.kubernetes.io/worker", + "-o=jsonpath={.items[*].metadata.name}", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + workers := strings.Fields(nodeNames) + o.Expect(workers).NotTo(o.BeEmpty(), "no worker nodes found") + + for _, w := range workers { + status, statusErr := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "nodes", w, + "-o=jsonpath={.status.conditions[?(@.type=='Ready')].status}", + ).Output() + if statusErr == nil && status == "True" { + return w + } + } + o.Expect(false).To(o.BeTrue(), "no Ready worker node found among %v", workers) + return "" // unreachable; satisfies compiler +}