diff --git a/pkg/csi/service/common/constants.go b/pkg/csi/service/common/constants.go index 4807a15e94..4ae777bfd5 100644 --- a/pkg/csi/service/common/constants.go +++ b/pkg/csi/service/common/constants.go @@ -490,23 +490,27 @@ const ( // HighPVNodeDensity is an FSS for guest cluster nodes that, when enabled, // raises MAX_VOLUMES_PER_NODE from 59 to 255 in NodeGetInfo responses. HighPVNodeDensity = "high-pv-node-density" + // SupportsPerNamespaceNetworkProviders is a WCP capability indicating that network provider + // is resolved per namespace (NetworkSettings CR) rather than from the global wcp-network-config. + SupportsPerNamespaceNetworkProviders = "supports_per_namespace_network_providers" ) var WCPFeatureStates = map[string]struct{}{ - PodVMOnStretchedSupervisor: {}, - CSIDetachOnSupervisor: {}, - WorkloadDomainIsolation: {}, - VPCCapabilitySupervisor: {}, - VolFromSnapshotOnTargetDs: {}, - SharedDiskFss: {}, - LinkedCloneSupport: {}, - WCPMobilityNonDisruptiveImport: {}, - WCPVMServiceVMSnapshots: {}, - BYOKEncryption: {}, - FCDTransactionSupport: {}, - MultipleClustersPerVsphereZone: {}, - FileVolumesWithVmService: {}, - VsanFileVolumeService: {}, + PodVMOnStretchedSupervisor: {}, + CSIDetachOnSupervisor: {}, + WorkloadDomainIsolation: {}, + VPCCapabilitySupervisor: {}, + VolFromSnapshotOnTargetDs: {}, + SharedDiskFss: {}, + LinkedCloneSupport: {}, + WCPMobilityNonDisruptiveImport: {}, + WCPVMServiceVMSnapshots: {}, + BYOKEncryption: {}, + FCDTransactionSupport: {}, + MultipleClustersPerVsphereZone: {}, + FileVolumesWithVmService: {}, + VsanFileVolumeService: {}, + SupportsPerNamespaceNetworkProviders: {}, } // WCPFeatureStatesSupportsLateEnablement contains capabilities that can be enabled later @@ -514,14 +518,15 @@ var WCPFeatureStates = map[string]struct{}{ // During FSS check if driver detects that the capabilities is disabled in the cached configmap, // it will re-fetch the configmap and update the cached configmap. var WCPFeatureStatesSupportsLateEnablement = map[string]struct{}{ - WorkloadDomainIsolation: {}, - LinkedCloneSupport: {}, - MultipleClustersPerVsphereZone: {}, - WCPVMServiceVMSnapshots: {}, - BYOKEncryption: {}, - SharedDiskFss: {}, - FileVolumesWithVmService: {}, - VsanFileVolumeService: {}, + WorkloadDomainIsolation: {}, + LinkedCloneSupport: {}, + MultipleClustersPerVsphereZone: {}, + WCPVMServiceVMSnapshots: {}, + BYOKEncryption: {}, + SharedDiskFss: {}, + FileVolumesWithVmService: {}, + VsanFileVolumeService: {}, + SupportsPerNamespaceNetworkProviders: {}, } // WCPFeatureAssociatedWithPVCSI contains FSS name used in PVCSI and associated WCP Capability name on a diff --git a/pkg/syncer/cnsoperator/controller/cnsfileaccessconfig/cnsfileaccessconfig_controller.go b/pkg/syncer/cnsoperator/controller/cnsfileaccessconfig/cnsfileaccessconfig_controller.go index 9c635bce1f..71ad5d63a1 100644 --- a/pkg/syncer/cnsoperator/controller/cnsfileaccessconfig/cnsfileaccessconfig_controller.go +++ b/pkg/syncer/cnsoperator/controller/cnsfileaccessconfig/cnsfileaccessconfig_controller.go @@ -768,6 +768,9 @@ func (r *ReconcileCnsFileAccessConfig) configureVolumeACLs(ctx context.Context, // getVMExternalIP helps to fetch the external facing IP for a given TKG VM. func (r *ReconcileCnsFileAccessConfig) getVMExternalIP(ctx context.Context, vm *vmoperatortypes.VirtualMachine) (string, error) { + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.SupportsPerNamespaceNetworkProviders) { + return r.getVMExternalIPFromNetworkSettings(ctx, vm) + } log := logger.GetLogger(ctx) networkProvider, err := util.GetNetworkProvider(ctx) if err != nil { @@ -798,7 +801,21 @@ func (r *ReconcileCnsFileAccessConfig) getVMExternalIP(ctx context.Context, } tkgVMIP, err := util.GetTKGVMIP(ctx, r.vmOperatorClient, - r.dynamicClient, vm.Namespace, vm.Name, networkProvider) + r.dynamicClient, vm.Namespace, vm.Name, networkProvider, false) + if err != nil { + return "", logger.LogNewErrorf(log, "Failed to get external facing IP address for VM %q/%q. Err: %+v", + vm.Namespace, vm.Name, err) + } + log.Infof("Found tkg VMIP %q for VM %q in namespace %q", tkgVMIP, vm.Name, vm.Namespace) + return tkgVMIP, nil +} + +// getVMExternalIPFromNetworkSettings resolves the VM IP when per-namespace NetworkSettings is used. +func (r *ReconcileCnsFileAccessConfig) getVMExternalIPFromNetworkSettings(ctx context.Context, + vm *vmoperatortypes.VirtualMachine) (string, error) { + log := logger.GetLogger(ctx) + tkgVMIP, err := util.GetTKGVMIPFromNetworkSettings(ctx, r.vmOperatorClient, r.dynamicClient, + vm.Namespace, vm.Name) if err != nil { return "", logger.LogNewErrorf(log, "Failed to get external facing IP address for VM %q/%q. Err: %+v", vm.Namespace, vm.Name, err) diff --git a/pkg/syncer/cnsoperator/util/util.go b/pkg/syncer/cnsoperator/util/util.go index e7254c6cb8..ac246fb0ed 100644 --- a/pkg/syncer/cnsoperator/util/util.go +++ b/pkg/syncer/cnsoperator/util/util.go @@ -24,8 +24,10 @@ import ( "strconv" "strings" + vmoperatortypes "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware/govmomi/object" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -62,6 +64,12 @@ var namespaceNetworkInfoGVR = schema.GroupVersionResource{ Resource: "namespacenetworkinfos", } +var networkSettingsGVR = schema.GroupVersionResource{ + Group: "netoperator.vmware.com", + Version: "v1alpha1", + Resource: "networksettings", +} + const ( snatIPAnnotation = "ncp/snat_ip" // Namespace for system resources. @@ -75,10 +83,17 @@ const ( // VDSNetworkProvider holds the network provider name for VDS based setups. VDSNetworkProvider = "VSPHERE_NETWORK" // VPCNetworkProvider holds the network provider name for VPC based setups. - VPCNetworkProvider = "NSX_VPC" - vpcDefaultSnatIp = "defaultSNATIP" + VPCNetworkProvider = "NSX_VPC" + vpcDefaultSnatIp = "defaultSNATIP" + networkSettingsProviderVsphereDistributed = "vsphere-distributed" + networkSettingsProviderNsxTier1 = "nsx-tier1" + networkSettingsProviderVPC = "vpc" ) +// ErrNetworkSettingsUnavailable indicates no NetworkSettings object exists in the namespace or +// status.provider is not set on the sole NetworkSettings object. +var ErrNetworkSettingsUnavailable = errors.New("NetworkSettings CR is unavailable or status.provider is not set") + // GetVolumeID gets the volume ID from the PV that is bound to PVC by pvcName. func GetVolumeID(ctx context.Context, client client.Client, pvcName string, namespace string) (string, error) { log := logger.GetLogger(ctx) @@ -105,11 +120,11 @@ func GetVolumeID(ctx context.Context, client client.Client, pvcName string, name return pv.Spec.CSI.VolumeHandle, nil } -// GetTKGVMIP finds the external facing IP address of a TKG VM object from a -// given Supervisor Namespace based on the networking configuration (NSX-T or -// VDS). +// GetTKGVMIP finds the external facing IP address of a TKG VM in a Supervisor namespace for the given +// networkProviderType (e.g. from GetNetworkProvider / wcp-network-config). snatOptional controls whether NSX-T/VPC +// may fall back to the VM primary IP when SNAT is absent (true for per-namespace NetworkSettings workflow). func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.Interface, - vmNamespace, vmName string, network_provider_type string) (string, error) { + vmNamespace, vmName, networkProviderType string, snatOptional bool) (string, error) { log := logger.GetLogger(ctx) log.Infof("Determining external IP Address of VM: %s/%s", vmNamespace, vmName) vmKey := apitypes.NamespacedName{ @@ -127,107 +142,225 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic. common.FileVolumesWithVmService) var networkNames []string - for _, networkInterface := range virtualMachineInstance.Spec.Network.Interfaces { - if !isFileVolumesWithVmServiceVmSupported { - networkNames = append(networkNames, networkInterface.Network.Name) - } else if networkInterface.Network.Name != "" { - networkNames = append(networkNames, networkInterface.Network.Name) + if virtualMachineInstance.Spec.Network != nil { + for _, networkInterface := range virtualMachineInstance.Spec.Network.Interfaces { + if !isFileVolumesWithVmServiceVmSupported { + networkNames = append(networkNames, networkInterface.Network.Name) + } else if networkInterface.Network.Name != "" { + networkNames = append(networkNames, networkInterface.Network.Name) + } } } log.Debugf("VirtualMachine %s/%s is configured with networks %v", vmNamespace, vmName, networkNames) var ip string - var exists bool - if network_provider_type == NSXTNetworkProvider { - for _, networkName := range networkNames { - virtualNetworkInstance, err := dc.Resource(virtualNetworkGVR).Namespace(vmNamespace).Get(ctx, - networkName, metav1.GetOptions{}) - if err != nil { - return "", err - } - log.Debugf("Got VirtualNetwork instance %s/%s with annotations %v", - vmNamespace, virtualNetworkInstance.GetName(), virtualNetworkInstance.GetAnnotations()) - ip, exists = virtualNetworkInstance.GetAnnotations()[snatIPAnnotation] - // Pick the network interface which has the snatIPAnnotation - if exists && ip != "" { - break - } + switch networkProviderType { + case NSXTNetworkProvider: + ip, err = resolveNSXTExternalIP(ctx, dc, vmNamespace, vmName, networkNames, virtualMachineInstance, + isFileVolumesWithVmServiceVmSupported, snatOptional) + if err != nil { + return "", err } - if ip == "" { - if !isFileVolumesWithVmServiceVmSupported { - return "", fmt.Errorf("failed to get SNAT IP annotation from VirtualMachine %s/%s", - vmNamespace, vmName) - } - if len(networkNames) != 0 { - // If networkNames for VirtualNetwork were found on the VM, - // then some error happened in getting the SNAT IP from VirtualNetwork CR. - return "", fmt.Errorf("failed to get SNAT IP annotation for VirtualMachine %s/%s "+ - "from VirtualNetwrok", - vmNamespace, vmName) - } - // It is likely an NSX setup with VM service VMs. - // For TKG service VMs, virtual network CR will always be present. - ip, err = getSnatIpFromNamespaceNetworkInfo(ctx, dc, vmNamespace, vmName) - if err != nil { - log.Errorf("failed to get SNAT IP from NameSpaceNetworkInfo. Err %s", err) - return "", fmt.Errorf("failed to get SNAT IP from NameSpaceNetworkInfo %s/%s", - vmNamespace, vmName) - } - log.Infof("Obtained SNAT IP %s from NamespaceNetworkInfo for VirtualMachine %s/%s", - ip, vmNamespace, vmName) + case VDSNetworkProvider: + ip, err = vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) + if err != nil { + return "", err } - } else if network_provider_type == VDSNetworkProvider { + case VPCNetworkProvider: + ip, err = resolveVPCExternalIP(ctx, dc, vmNamespace, vmName, virtualMachineInstance, snatOptional) + if err != nil { + return "", err + } + default: + return "", fmt.Errorf("unknown network provider %q", networkProviderType) + } + log.Infof("Found external IP Address %s for VirtualMachine %s/%s", ip, vmNamespace, vmName) + return ip, nil +} + +// GetTKGVMIPFromNetworkSettings finds the external facing IP for a TKG VM when per-namespace network +// providers are enabled: provider is read from the sole NetworkSettings object in the namespace (status.provider). +func GetTKGVMIPFromNetworkSettings(ctx context.Context, vmOperatorClient client.Client, dc dynamic.Interface, + vmNamespace, vmName string) (string, error) { + if dc == nil { + return "", fmt.Errorf("dynamic client is required when %s is enabled", + common.SupportsPerNamespaceNetworkProviders) + } + networkProviderType, err := getNetworkProviderFromNetworkSettings(ctx, dc, vmNamespace) + if err != nil { + return "", err + } + return GetTKGVMIP(ctx, vmOperatorClient, dc, vmNamespace, vmName, networkProviderType, true) +} + +func mapNetworkSettingsProviderToNetworkProvider(provider string) (string, error) { + switch strings.TrimSpace(strings.ToLower(provider)) { + case networkSettingsProviderVsphereDistributed: + return VDSNetworkProvider, nil + case networkSettingsProviderNsxTier1: + return NSXTNetworkProvider, nil + case networkSettingsProviderVPC: + return VPCNetworkProvider, nil + default: + return "", fmt.Errorf("unknown NetworkSettings status.provider value %q", provider) + } +} - if virtualMachineInstance.Status.Network == nil { - log.Errorf("virtualMachineInstance.Status.Network is nil for VM %s", vmName) - return "", fmt.Errorf("virtualMachineInstance.Status.Network is nil for VM %s", vmName) +func getNetworkProviderFromNetworkSettings(ctx context.Context, dc dynamic.Interface, + namespace string) (string, error) { + list, err := dc.Resource(networkSettingsGVR).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return "", err + } + switch len(list.Items) { + case 0: + return "", ErrNetworkSettingsUnavailable + case 1: + obj := &list.Items[0] + crName := obj.GetName() + provider, found, err := unstructured.NestedString(obj.Object, "status", "provider") + if err != nil || !found || strings.TrimSpace(provider) == "" { + return "", fmt.Errorf("%w: status.provider is empty or missing on NetworkSettings %s/%s", + ErrNetworkSettingsUnavailable, namespace, crName) } + mapped, err := mapNetworkSettingsProviderToNetworkProvider(provider) + if err != nil { + return "", err + } + return mapped, nil + default: + return "", fmt.Errorf("expected exactly one NetworkSettings object in namespace %q, found %d", + namespace, len(list.Items)) + } +} - ip = virtualMachineInstance.Status.Network.PrimaryIP4 +func vmPrimaryIPFromVirtualMachine(ctx context.Context, vm *vmoperatortypes.VirtualMachine, + vmNamespace, vmName string) (string, error) { + log := logger.GetLogger(ctx) + if vm.Status.Network == nil { + log.Errorf("virtualMachineInstance.Status.Network is nil for VM %s", vmName) + return "", fmt.Errorf("virtualMachineInstance.Status.Network is nil for VM %s", vmName) + } + ip := vm.Status.Network.PrimaryIP4 + if ip == "" { + ip = vm.Status.Network.PrimaryIP6 if ip == "" { - ip = virtualMachineInstance.Status.Network.PrimaryIP6 - if ip == "" { - return "", fmt.Errorf("vm.Status.Network.PrimaryIP6 & PrimaryIP4 is not populated for %s/%s", - vmNamespace, vmName) - } + return "", fmt.Errorf("vm.Status.Network.PrimaryIP6 & PrimaryIP4 is not populated for %s/%s", + vmNamespace, vmName) } - } else { - vpcName := vmNamespace - networkInfoInstance, err := dc.Resource(networkInfoGVR).Namespace(vmNamespace).Get(ctx, - vpcName, metav1.GetOptions{}) + } + return ip, nil +} + +func resolveNSXTExternalIP(ctx context.Context, dc dynamic.Interface, vmNamespace, vmName string, + networkNames []string, virtualMachineInstance *vmoperatortypes.VirtualMachine, + isFileVolumesWithVmServiceVmSupported, snatOptional bool) (string, error) { + log := logger.GetLogger(ctx) + if dc == nil { + return "", fmt.Errorf("dynamic client is nil") + } + var ip string + var exists bool + for _, networkName := range networkNames { + virtualNetworkInstance, err := dc.Resource(virtualNetworkGVR).Namespace(vmNamespace).Get(ctx, + networkName, metav1.GetOptions{}) if err != nil { return "", err } - log.Debugf("Got NetworkInfo instance %s/%s", vmNamespace, networkInfoInstance.GetName()) - vpcs, found, err := unstructured.NestedSlice(networkInfoInstance.Object, "vpcs") - if err != nil || !found || len(vpcs) == 0 { - return "", fmt.Errorf("failed to get vpcs from networkinfo %s/%s with error: %v", - vmNamespace, vmName, err) + log.Debugf("Got VirtualNetwork instance %s/%s with annotations %v", + vmNamespace, virtualNetworkInstance.GetName(), virtualNetworkInstance.GetAnnotations()) + ip, exists = virtualNetworkInstance.GetAnnotations()[snatIPAnnotation] + if exists && ip != "" { + break + } + } + if ip != "" { + return ip, nil + } + if !isFileVolumesWithVmServiceVmSupported { + if snatOptional { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) + } + return "", fmt.Errorf("failed to get SNAT IP annotation from VirtualMachine %s/%s", + vmNamespace, vmName) + } + if len(networkNames) != 0 { + if snatOptional { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) + } + return "", fmt.Errorf("failed to get SNAT IP annotation for VirtualMachine %s/%s "+ + "from VirtualNetwrok", + vmNamespace, vmName) + } + ip, err := getSnatIpFromNamespaceNetworkInfo(ctx, dc, vmNamespace, vmName) + if err != nil { + if snatOptional { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) + } + log.Errorf("failed to get SNAT IP from NameSpaceNetworkInfo. Err %s", err) + return "", fmt.Errorf("failed to get SNAT IP from NameSpaceNetworkInfo %s/%s", + vmNamespace, vmName) + } + log.Infof("Obtained SNAT IP %s from NamespaceNetworkInfo for VirtualMachine %s/%s", + ip, vmNamespace, vmName) + return ip, nil +} + +func resolveVPCExternalIP(ctx context.Context, dc dynamic.Interface, vmNamespace, vmName string, + virtualMachineInstance *vmoperatortypes.VirtualMachine, snatOptional bool) (string, error) { + log := logger.GetLogger(ctx) + if dc == nil { + return "", fmt.Errorf("dynamic client is nil") + } + vpcName := vmNamespace + networkInfoInstance, err := dc.Resource(networkInfoGVR).Namespace(vmNamespace).Get(ctx, + vpcName, metav1.GetOptions{}) + if err != nil { + if snatOptional && apierrors.IsNotFound(err) { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) } + return "", err + } + log.Debugf("Got NetworkInfo instance %s/%s", vmNamespace, networkInfoInstance.GetName()) + vpcs, found, err := unstructured.NestedSlice(networkInfoInstance.Object, "vpcs") + if err != nil || !found || len(vpcs) == 0 { + if snatOptional { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) + } + return "", fmt.Errorf("failed to get vpcs from networkinfo %s/%s with error: %v", + vmNamespace, vmName, err) + } - vpc, ok := vpcs[0].(map[string]interface{}) - if !ok { - return "", fmt.Errorf("failed to assert vpc to map[string]interface{} %s/%s", - vmNamespace, vmName) + vpc, ok := vpcs[0].(map[string]interface{}) + if !ok { + if snatOptional { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) } + return "", fmt.Errorf("failed to assert vpc to map[string]interface{} %s/%s", + vmNamespace, vmName) + } - for key, value := range vpc { - if key == vpcDefaultSnatIp { - ip, ok = value.(string) - if !ok { - return "", fmt.Errorf("failed to cast key %s value to string", key) + var ip string + for key, value := range vpc { + if key == vpcDefaultSnatIp { + ip, ok = value.(string) + if !ok { + if snatOptional { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) } - break + return "", fmt.Errorf("failed to cast key %s value to string", key) } - - } - if ip == "" { - return "", fmt.Errorf("spec.vpc.defaultSNATIP is not populated for "+ - "networkinfo %s/%s", vmNamespace, vmName) + break } } - log.Infof("Found external IP Address %s for VirtualMachine %s/%s", ip, vmNamespace, vmName) - return ip, nil + if ip != "" { + return ip, nil + } + if snatOptional { + return vmPrimaryIPFromVirtualMachine(ctx, virtualMachineInstance, vmNamespace, vmName) + } + return "", fmt.Errorf("spec.vpc.defaultSNATIP is not populated for "+ + "networkinfo %s/%s", vmNamespace, vmName) } // getSnatIpFromNamespaceNetworkInfo finds VM's SNAT IP from the namespace's default NamespaceNetworkInfo CR. diff --git a/pkg/syncer/cnsoperator/util/util_test.go b/pkg/syncer/cnsoperator/util/util_test.go index 8d58eaff03..e806d36a52 100644 --- a/pkg/syncer/cnsoperator/util/util_test.go +++ b/pkg/syncer/cnsoperator/util/util_test.go @@ -18,6 +18,7 @@ package util import ( "context" + "errors" "os" "strings" "testing" @@ -40,6 +41,17 @@ import ( k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes" ) +// newFakeDynamicClientForNetworkSettings returns a fake dynamic client that supports List/Get on +// netoperator.vmware.com NetworkSettings (registers NetworkSettingsList for the fake tracker). +func newFakeDynamicClientForNetworkSettings(objects ...runtime.Object) *fake.FakeDynamicClient { + scheme := runtime.NewScheme() + scheme.AddKnownTypes(networkSettingsGVR.GroupVersion()) + gvrToListKind := map[schema.GroupVersionResource]string{ + networkSettingsGVR: "NetworkSettingsList", + } + return fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, objects...) +} + func TestGetSnatIpFromNamespaceNetworkInfo(t *testing.T) { ctx := context.Background() gvr := schema.GroupVersionResource{ @@ -423,6 +435,7 @@ func TestGetTKGVMIP_NetworkStatusNil(t *testing.T) { vmNamespace, vmName, VDSNetworkProvider, + false, ) if err == nil { @@ -437,3 +450,151 @@ func TestGetTKGVMIP_NetworkStatusNil(t *testing.T) { t.Fatalf("unexpected error message: %v", err) } } + +func TestMapNetworkSettingsProviderToNetworkProvider(t *testing.T) { + tests := []struct { + in string + want string + wantErr bool + }{ + {"vsphere-distributed", VDSNetworkProvider, false}, + {"NSX-TIER1", NSXTNetworkProvider, false}, + {"vpc", VPCNetworkProvider, false}, + {"unknown", "", true}, + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + got, err := mapNetworkSettingsProviderToNetworkProvider(tt.in) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestGetTKGVMIP_NetworkSettingsUnavailable(t *testing.T) { + ctx := context.Background() + vmNamespace := "test-ns" + vmName := "test-vm" + + vm := &vmoperatortypes.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: vmNamespace}, + Spec: vmoperatortypes.VirtualMachineSpec{ + ImageName: "test-image", + ClassName: "test-class", + Network: &vmoperatortypes.VirtualMachineNetworkSpec{}, + }, + Status: vmoperatortypes.VirtualMachineStatus{ + Network: &vmoperatortypes.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.0.2.1", + }, + }, + } + scheme := runtime.NewScheme() + err := vmoperatortypes.AddToScheme(scheme) + if err != nil { + t.Fatalf("add scheme: %v", err) + } + vmOpClient := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(vm).Build() + commonco.ContainerOrchestratorUtility, err = unittestcommon.GetFakeContainerOrchestratorInterface(common.Kubernetes) + if err != nil { + t.Fatalf("orchestrator: %v", err) + } + dc := newFakeDynamicClientForNetworkSettings() + + _, err = GetTKGVMIPFromNetworkSettings(ctx, vmOpClient, dc, vmNamespace, vmName) + assert.Error(t, err) + assert.True(t, errors.Is(err, ErrNetworkSettingsUnavailable)) +} + +func TestGetNetworkProviderFromNetworkSettings_MultipleCRs(t *testing.T) { + ctx := context.Background() + ns := "test-ns" + a := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "netoperator.vmware.com/v1alpha1", + "kind": "NetworkSettings", + "metadata": map[string]interface{}{ + "name": "a", "namespace": ns, + }, + "status": map[string]interface{}{"provider": networkSettingsProviderVsphereDistributed}, + }, + } + b := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "netoperator.vmware.com/v1alpha1", + "kind": "NetworkSettings", + "metadata": map[string]interface{}{ + "name": "b", "namespace": ns, + }, + "status": map[string]interface{}{"provider": networkSettingsProviderVsphereDistributed}, + }, + } + dc := newFakeDynamicClientForNetworkSettings() + _, err := dc.Resource(networkSettingsGVR).Namespace(ns).Create(ctx, a, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create a: %v", err) + } + _, err = dc.Resource(networkSettingsGVR).Namespace(ns).Create(ctx, b, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create b: %v", err) + } + _, err = getNetworkProviderFromNetworkSettings(ctx, dc, ns) + assert.Error(t, err) + assert.Contains(t, err.Error(), "expected exactly one NetworkSettings") +} + +func TestGetTKGVMIP_PerNamespaceVDSUsesVMIP(t *testing.T) { + ctx := context.Background() + vmNamespace := "test-ns" + vmName := "test-vm" + + vm := &vmoperatortypes.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: vmNamespace}, + Spec: vmoperatortypes.VirtualMachineSpec{ + ImageName: "test-image", + ClassName: "test-class", + Network: &vmoperatortypes.VirtualMachineNetworkSpec{}, + }, + Status: vmoperatortypes.VirtualMachineStatus{ + Network: &vmoperatortypes.VirtualMachineNetworkStatus{ + PrimaryIP4: "192.0.2.10", + }, + }, + } + scheme := runtime.NewScheme() + err := vmoperatortypes.AddToScheme(scheme) + if err != nil { + t.Fatalf("add scheme: %v", err) + } + vmOpClient := ctrlruntimefake.NewClientBuilder().WithScheme(scheme).WithObjects(vm).Build() + commonco.ContainerOrchestratorUtility, err = unittestcommon.GetFakeContainerOrchestratorInterface(common.Kubernetes) + if err != nil { + t.Fatalf("orchestrator: %v", err) + } + ns := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "netoperator.vmware.com/v1alpha1", + "kind": "NetworkSettings", + "metadata": map[string]interface{}{ + "name": "arbitrary-network-settings-name", + "namespace": vmNamespace, + }, + "status": map[string]interface{}{ + "provider": networkSettingsProviderVsphereDistributed, + }, + }, + } + dc := newFakeDynamicClientForNetworkSettings() + _, err = dc.Resource(networkSettingsGVR).Namespace(vmNamespace).Create(ctx, ns, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create NetworkSettings: %v", err) + } + + ip, err := GetTKGVMIPFromNetworkSettings(ctx, vmOpClient, dc, vmNamespace, vmName) + assert.NoError(t, err) + assert.Equal(t, "192.0.2.10", ip) +} diff --git a/pkg/syncer/metadatasyncer.go b/pkg/syncer/metadatasyncer.go index c480c06166..ab8b62ee16 100644 --- a/pkg/syncer/metadatasyncer.go +++ b/pkg/syncer/metadatasyncer.go @@ -363,6 +363,10 @@ func InitMetadataSyncer(ctx context.Context, clusterFlavor cnstypes.CnsClusterFl go commonco.ContainerOrchestratorUtility.HandleLateEnablementOfCapability(ctx, clusterFlavor, common.VsanFileVolumeService, "", "") } + if !commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.SupportsPerNamespaceNetworkProviders) { + go commonco.ContainerOrchestratorUtility.HandleLateEnablementOfCapability(ctx, clusterFlavor, + common.SupportsPerNamespaceNetworkProviders, "", "") + } } if metadataSyncer.clusterFlavor == cnstypes.CnsClusterFlavorGuest {