diff --git a/package/Dockerfile b/package/Dockerfile index 38eb7d5de..07a3d57ef 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -102,11 +102,11 @@ RUN zypper -n install sg3_utils iproute2 RUN zypper -n install util-linux-systemd nfs-client nfs4-acl-tools RUN zypper clean -a -# Install SPDK dependencies +# Install SPDK dependencies (--rdma pulls libibverbs/librdmacm runtime libs for the NVMe-oF RDMA transport) COPY --from=cbuilder /usr/src/spdk/scripts /usr/src/spdk/scripts COPY --from=cbuilder /usr/src/spdk/include /usr/src/spdk/include RUN for i in {1..10}; do \ - bash /usr/src/spdk/scripts/pkgdep.sh && break || sleep 1; \ + bash /usr/src/spdk/scripts/pkgdep.sh --rdma && break || sleep 1; \ done # Copy pre-built binaries from cbuilder and gobuilder diff --git a/pkg/client/instance.go b/pkg/client/instance.go index a659ee2eb..f6da84332 100644 --- a/pkg/client/instance.go +++ b/pkg/client/instance.go @@ -149,6 +149,11 @@ type InstanceCreateRequest struct { ShardGroup ShardGroupCreateRequest DataLayoutType rpc.DataLayoutType + // TransportType selects the NVMe-oF transport for the internal + // engine<->replica connections of a v2 (SPDK) volume (TCP or RDMA). The + // zero value is TCP, preserving the pre-existing default behavior. + TransportType rpc.TransportType + // Deprecated: replaced by DataEngine. BackendStoreDriver string } @@ -179,13 +184,14 @@ func (c *InstanceServiceClient) InstanceCreate(req *InstanceCreateRequest) (*api switch req.InstanceType { case types.InstanceTypeEngine: spdkInstanceSpec = &rpc.SpdkInstanceSpec{ - Size: req.Size, - ReplicaAddressMap: req.Engine.ReplicaAddressMap, - Frontend: req.Engine.Frontend, - SalvageRequested: req.Engine.SalvageRequested, - SnapshotMaxCount: int32(req.Engine.SnapshotMaxCount), - UblkQueueDepth: int32(req.Engine.UblkQueueDepth), - UblkNumberOfQueue: int32(req.Engine.UblkNumberOfQueue), + Size: req.Size, + ReplicaAddressMap: req.Engine.ReplicaAddressMap, + Frontend: req.Engine.Frontend, + SalvageRequested: req.Engine.SalvageRequested, + SnapshotMaxCount: int32(req.Engine.SnapshotMaxCount), + UblkQueueDepth: int32(req.Engine.UblkQueueDepth), + UblkNumberOfQueue: int32(req.Engine.UblkNumberOfQueue), + TransportType: req.TransportType, } case types.InstanceTypeEngineFrontend: spdkInstanceSpec = &rpc.SpdkInstanceSpec{ @@ -197,11 +203,12 @@ func (c *InstanceServiceClient) InstanceCreate(req *InstanceCreateRequest) (*api } case types.InstanceTypeReplica: spdkInstanceSpec = &rpc.SpdkInstanceSpec{ - Size: req.Size, - DiskName: req.Replica.DiskName, - DiskUuid: req.Replica.DiskUUID, - ExposeRequired: req.Replica.ExposeRequired, - BackingImageName: req.Replica.BackingImageName, + Size: req.Size, + DiskName: req.Replica.DiskName, + DiskUuid: req.Replica.DiskUUID, + ExposeRequired: req.Replica.ExposeRequired, + BackingImageName: req.Replica.BackingImageName, + TransportType: req.TransportType, } case types.InstanceTypeShard: spdkInstanceSpec = &rpc.SpdkInstanceSpec{ diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 18e1af06a..e81c8fc04 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -197,7 +197,8 @@ func (ops V2DataEngineInstanceOps) InstanceCreate(req *rpc.InstanceCreateRequest engine, err := c.EngineCreate(req.Spec.Name, req.Spec.VolumeName, req.Spec.SpdkInstanceSpec.Frontend, req.Spec.SpdkInstanceSpec.Size, req.Spec.SpdkInstanceSpec.ReplicaAddressMap, req.Spec.PortCount, req.Spec.SpdkInstanceSpec.SalvageRequested, req.Spec.SpdkInstanceSpec.SnapshotMaxCount, - convertIMDataLayoutTypeToSPDKDataLayoutType(req.DataLayoutType)) + convertIMDataLayoutTypeToSPDKDataLayoutType(req.DataLayoutType), + convertIMTransportTypeToSPDKTransportType(req.Spec.SpdkInstanceSpec.TransportType)) if err != nil { return nil, err } @@ -210,7 +211,8 @@ func (ops V2DataEngineInstanceOps) InstanceCreate(req *rpc.InstanceCreateRequest } return engineFrontendResponseToInstanceResponse(engineFrontend), nil case types.InstanceTypeReplica: - replica, err := c.ReplicaCreate(req.Spec.Name, req.Spec.SpdkInstanceSpec.DiskName, req.Spec.SpdkInstanceSpec.DiskUuid, req.Spec.SpdkInstanceSpec.Size, req.Spec.PortCount, req.Spec.SpdkInstanceSpec.BackingImageName) + replica, err := c.ReplicaCreate(req.Spec.Name, req.Spec.SpdkInstanceSpec.DiskName, req.Spec.SpdkInstanceSpec.DiskUuid, req.Spec.SpdkInstanceSpec.Size, req.Spec.PortCount, req.Spec.SpdkInstanceSpec.BackingImageName, + convertIMTransportTypeToSPDKTransportType(req.Spec.SpdkInstanceSpec.TransportType)) if err != nil { return nil, err } @@ -1352,6 +1354,20 @@ func convertIMDataLayoutTypeToSPDKDataLayoutType(layout rpc.DataLayoutType) spdk } } +// convertIMTransportTypeToSPDKTransportType maps the imrpc +// NVMe-oF transport selector to its spdkrpc equivalent. The zero value +// (TRANSPORT_TYPE_TCP) maps to TCP so instances created by an older +// control plane - which never sets the field - keep their historical TCP +// behavior. +func convertIMTransportTypeToSPDKTransportType(transport rpc.TransportType) spdkrpc.TransportType { + switch transport { + case rpc.TransportType_TRANSPORT_TYPE_RDMA: + return spdkrpc.TransportType_TRANSPORT_TYPE_RDMA + default: + return spdkrpc.TransportType_TRANSPORT_TYPE_TCP + } +} + func (s *Server) InstanceDeleteTarget(ctx context.Context, req *rpc.InstanceDeleteTargetRequest) (*emptypb.Empty, error) { logrus.WithFields(logrus.Fields{ "name": req.Name,