Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions block/provider/attach_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,21 @@ func (vpcs *VPCSession) AttachVolume(volumeAttachmentRequest provider.VolumeAtta
}
var volumeAttachResult *models.VolumeAttachment
var varp *provider.VolumeAttachmentResponse
// If it is Non IKS environment then remove the IKSVolumeAttachment field from request struct which contains clusterID.
// TO-DO : Enhance this check. Put it in right place
// For non-IKS clusters, strip the IKS cluster ID from the request.
// Also determine whether this request targets a Bare Metal Server.
isBMS := false
if !vpcs.Config.VPCConfig.IsIKS {
volumeAttachmentRequest.IKSVolumeAttachment = nil
isBMS = volumeAttachmentRequest.BMSVolumeAttachment != nil
}
volumeAttachment := models.NewVolumeAttachment(volumeAttachmentRequest)

// Choose the correct attachment manager: BMS SDP API or VSI/IKS API.
attachMgr := vpcs.APIClientVolAttachMgr
if isBMS {
attachMgr = vpcs.getBMSVolAttachMgr()
}

err = vpcs.APIRetry.FlexyRetry(vpcs.Logger, func() (error, bool) {
// First , check if volume is already attached or attaching to given instance
vpcs.Logger.Info("Checking if volume is already attached by other thread")
Expand All @@ -73,13 +81,13 @@ func (vpcs *VPCSession) AttachVolume(volumeAttachmentRequest provider.VolumeAtta
return nil, true // stop retry volume already attached
}
//Try attaching volume if it's not already attached or there is error in getting current volume attachment
vpcs.Logger.Info("Attaching volume from VPC provider...", zap.Bool("IKSEnabled?", vpcs.Config.VPCConfig.IsIKS))
volumeAttachResult, err = vpcs.APIClientVolAttachMgr.AttachVolume(&volumeAttachment, vpcs.Logger)
vpcs.Logger.Info("Attaching volume from VPC provider...", zap.Bool("IKSEnabled?", vpcs.Config.VPCConfig.IsIKS), zap.Bool("isBMS?", isBMS))
volumeAttachResult, err = attachMgr.AttachVolume(&volumeAttachment, vpcs.Logger)
// Keep retry, until we get the proper volumeAttachResult object
if err != nil {
return err, skipRetryForObviousErrors(err, vpcs.Config.VPCConfig.IsIKS)
}
varp = volumeAttachResult.ToVolumeAttachmentResponse(vpcs.Config.VPCConfig.VPCBlockProviderType)
varp = volumeAttachResult.ToVolumeAttachmentResponse(vpcs.Config.VPCConfig.VPCBlockProviderType, isBMS)
return err, true // stop retry as no error
})

Expand Down
112 changes: 112 additions & 0 deletions block/provider/bms_storage_access.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/**
* Copyright 2025 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

// Package provider ...
package provider

// bms_storage_access.go implements the BMS-specific VPC API calls needed by
// the CSI controller's ControllerPublishVolume STEP 4:
//
// 1. RotateBMSEncryptedSecret — POST .../storage_access/encrypted_secret/rotate
// body: {"key":{"id":keyID}}
// VPC re-encrypts the NVMe-oF PSK with the RSA public key registered as
// VPC key {keyID}.
//
// 2. GetBMSServer — GET .../bare_metal_servers/{serverID}
// Returns BOTH the host NQN (nvme_qualified_name) AND the encrypted PSK
// (storage_access.encrypted_secret) in one round-trip, per the VPC BMS
// Block Storage SDP User Guide.
//
// All methods delegate to getBMSVolAttachMgr() which returns the
// lazily-initialised BMS VolumeAttachService.

import (
"time"

"github.com/IBM/ibmcloud-volume-interface/lib/metrics"
"github.com/IBM/ibmcloud-volume-interface/lib/provider"
"go.uber.org/zap"
)

// RotateBMSEncryptedSecret calls
// POST v1/bare_metal_servers/{serverID}/storage_access/encrypted_secret/rotate
// body: {"key":{"id":keyID}}
//
// Tells VPC to re-encrypt the NVMe-oF PSK with the RSA public key registered
// as VPC key {keyID}. Must be called before GetBMSServer so that
// the returned ciphertext is encrypted with this cluster's key.
func (vpcs *VPCSession) RotateBMSEncryptedSecret(serverID, keyID string) error {
vpcs.Logger.Debug("Entry of RotateBMSEncryptedSecret...")
defer vpcs.Logger.Debug("Exit from RotateBMSEncryptedSecret...")
defer metrics.UpdateDurationFromStart(vpcs.Logger, "RotateBMSEncryptedSecret", time.Now())

if err := isValidServiceSession(vpcs); err != nil {
return err
}

vpcs.Logger.Info("RotateBMSEncryptedSecret: calling VPC API", // pragma: allowlist secret
zap.String("serverID", serverID), zap.String("keyID", keyID))

if err := vpcs.getBMSVolAttachMgr().RotateEncryptedSecret(serverID, keyID, vpcs.Logger); err != nil { // pragma: allowlist secret
vpcs.Logger.Error("RotateBMSEncryptedSecret: API call failed", // pragma: allowlist secret
zap.String("serverID", serverID), zap.Error(err))
return err
}

vpcs.Logger.Info("RotateBMSEncryptedSecret: succeeded", zap.String("serverID", serverID)) // pragma: allowlist secret
return nil
}

// GetBMSServer calls GET v1/bare_metal_servers/{serverID} and returns both
// the host NQN and the RSA-encrypted PSK ciphertext in a single round-trip.
//
// Per the VPC BMS Block Storage SDP User Guide:
// nvme_qualified_name → HOSTNQN (initiator NQN of this BMS)

Check failure on line 77 in block/provider/bms_storage_access.go

View workflow job for this annotation

GitHub Actions / test (1.26.6)

File is not properly formatted (gofmt)
// storage_access.encrypted_secret → PSK ciphertext encrypted with this cluster's RSA key
//
// Call RotateBMSEncryptedSecret first so the ciphertext is encrypted with
// this cluster's RSA public key before calling GetBMSServer.
func (vpcs *VPCSession) GetBMSServer(serverID string) (*provider.BMSServerInfo, error) {
vpcs.Logger.Debug("Entry of GetBMSServer...")
defer vpcs.Logger.Debug("Exit from GetBMSServer...")
defer metrics.UpdateDurationFromStart(vpcs.Logger, "GetBMSServer", time.Now())

if err := isValidServiceSession(vpcs); err != nil {
return nil, err
}

vpcs.Logger.Info("GetBMSServer: calling VPC API", zap.String("serverID", serverID))

resp, err := vpcs.getBMSVolAttachMgr().GetBMSServer(serverID, vpcs.Logger)
if err != nil {
vpcs.Logger.Error("GetBMSServer: API call failed",
zap.String("serverID", serverID), zap.Error(err))
return nil, err
}

info := &provider.BMSServerInfo{
HostNQN: resp.NVMeQualifiedName,
}
if resp.StorageAccess != nil {
info.EncryptedSecret = resp.StorageAccess.EncryptedSecret // pragma: allowlist secret
}

vpcs.Logger.Info("GetBMSServer: succeeded",
zap.String("serverID", serverID),
zap.String("hostNQN", info.HostNQN),
zap.Bool("encryptedSecretPresent", info.EncryptedSecret != ""))
return info, nil
}
13 changes: 11 additions & 2 deletions block/provider/detach_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ func (vpcs *VPCSession) DetachVolume(volumeAttachmentTemplate provider.VolumeAtt
return nil, err
}

// Determine whether this request targets a Bare Metal Server.
isBMS := !vpcs.Config.VPCConfig.IsIKS && volumeAttachmentTemplate.BMSVolumeAttachment != nil

// Choose the correct attachment manager.
detachMgr := vpcs.APIClientVolAttachMgr
if isBMS {
detachMgr = vpcs.getBMSVolAttachMgr()
}

var response *http.Response
var volumeAttachment models.VolumeAttachment

Expand All @@ -59,8 +68,8 @@ func (vpcs *VPCSession) DetachVolume(volumeAttachmentTemplate provider.VolumeAtt
vpcs.Logger.Info("Found volume attachment", zap.Reflect("currentVolAttachment", currentVolAttachment))
volumeAttachment := models.NewVolumeAttachment(volumeAttachmentTemplate)
volumeAttachment.ID = currentVolAttachment.VPCVolumeAttachment.ID
vpcs.Logger.Info("Detaching volume from VPC provider...")
response, err = vpcs.APIClientVolAttachMgr.DetachVolume(&volumeAttachment, vpcs.Logger) //nolint:bodyclose
vpcs.Logger.Info("Detaching volume from VPC provider...", zap.Bool("isBMS?", isBMS))
response, err = detachMgr.DetachVolume(&volumeAttachment, vpcs.Logger) //nolint:bodyclose

if err != nil {
return err, skipRetryForObviousErrors(err, vpcs.Config.VPCConfig.IsIKS) // Retry in case of all errors
Expand Down
31 changes: 18 additions & 13 deletions block/provider/get_volume_attachment.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (

"github.com/IBM/ibmcloud-volume-interface/lib/provider"
userError "github.com/IBM/ibmcloud-volume-vpc/common/messages"
"github.com/IBM/ibmcloud-volume-vpc/common/vpcclient/instances"
"github.com/IBM/ibmcloud-volume-vpc/common/vpcclient/models"
"go.uber.org/zap"
)
Expand All @@ -42,32 +43,36 @@ func (vpcs *VPCSession) GetVolumeAttachment(volumeAttachmentRequest provider.Vol
if err != nil {
return nil, err
}

// Determine manager and BMS flag the same way as attach/detach.
isBMS := !vpcs.Config.VPCConfig.IsIKS && volumeAttachmentRequest.BMSVolumeAttachment != nil
mgr := vpcs.APIClientVolAttachMgr
if isBMS {
mgr = vpcs.getBMSVolAttachMgr()
}

var volumeAttachmentResponse *provider.VolumeAttachmentResponse
volumeAttachment := models.NewVolumeAttachment(volumeAttachmentRequest)
if len(volumeAttachment.ID) > 0 {
//Get volume attachments by ID if it is specified
volumeAttachmentResponse, err = vpcs.getVolumeAttachmentByID(volumeAttachment)
volumeAttachmentResponse, err = vpcs.getVolumeAttachmentByID(volumeAttachment, mgr, isBMS)
} else {
// Get volume attachment by Volume ID. This is inefficient operation which requires iteration over volume attachment list
volumeAttachmentResponse, err = vpcs.getVolumeAttachmentByVolumeID(volumeAttachment)
volumeAttachmentResponse, err = vpcs.getVolumeAttachmentByVolumeID(volumeAttachment, mgr, isBMS)
}
vpcs.Logger.Info("Volume attachment response", zap.Reflect("volumeAttachmentResponse", volumeAttachmentResponse), zap.Error(err))
return volumeAttachmentResponse, err
}

func (vpcs *VPCSession) getVolumeAttachmentByID(volumeAttachmentRequest models.VolumeAttachment) (*provider.VolumeAttachmentResponse, error) {
func (vpcs *VPCSession) getVolumeAttachmentByID(volumeAttachmentRequest models.VolumeAttachment, mgr instances.VolumeAttachManager, isBMS bool) (*provider.VolumeAttachmentResponse, error) {
vpcs.Logger.Debug("Entry of getVolumeAttachmentByID()")
defer vpcs.Logger.Debug("Exit from getVolumeAttachmentByID()")
vpcs.Logger.Info("Getting VolumeAttachment from VPC provider...")
var err error
var volumeAttachmentResult *models.VolumeAttachment
/*err = retry(vpcs.Logger, func() error {
volumeAttachmentResult, err = vpcs.APIClientVolAttachMgr.GetVolumeAttachment(&volumeAttachmentRequest, vpcs.Logger)
return err
})*/

err = vpcs.APIRetry.FlexyRetry(vpcs.Logger, func() (error, bool) {
volumeAttachmentResult, err = vpcs.APIClientVolAttachMgr.GetVolumeAttachment(&volumeAttachmentRequest, vpcs.Logger)
volumeAttachmentResult, err = mgr.GetVolumeAttachment(&volumeAttachmentRequest, vpcs.Logger)
// Keep retry, until we get the proper volumeAttachmentRequest object
if err != nil {
return err, skipRetryForObviousErrors(err, vpcs.Config.VPCConfig.IsIKS)
Expand All @@ -81,19 +86,19 @@ func (vpcs *VPCSession) getVolumeAttachmentByID(volumeAttachmentRequest models.V
return nil, userErr
}

volumeAttachmentResponse := volumeAttachmentResult.ToVolumeAttachmentResponse(vpcs.Config.VPCConfig.VPCBlockProviderType)
volumeAttachmentResponse := volumeAttachmentResult.ToVolumeAttachmentResponse(vpcs.Config.VPCConfig.VPCBlockProviderType, isBMS)
vpcs.Logger.Info("Successfully retrieved volume attachment", zap.Reflect("volumeAttachmentResponse", volumeAttachmentResponse))
return volumeAttachmentResponse, err
}

func (vpcs *VPCSession) getVolumeAttachmentByVolumeID(volumeAttachmentRequest models.VolumeAttachment) (*provider.VolumeAttachmentResponse, error) {
func (vpcs *VPCSession) getVolumeAttachmentByVolumeID(volumeAttachmentRequest models.VolumeAttachment, mgr instances.VolumeAttachManager, isBMS bool) (*provider.VolumeAttachmentResponse, error) {
vpcs.Logger.Debug("Entry of getVolumeAttachmentByVolumeID()")
defer vpcs.Logger.Debug("Exit from getVolumeAttachmentByVolumeID()")
vpcs.Logger.Info("Getting VolumeAttachmentList from VPC provider...")
var volumeAttachmentList *models.VolumeAttachmentList
var err error
err = vpcs.APIRetry.FlexyRetry(vpcs.Logger, func() (error, bool) {
volumeAttachmentList, err = vpcs.APIClientVolAttachMgr.ListVolumeAttachments(&volumeAttachmentRequest, vpcs.Logger)
volumeAttachmentList, err = mgr.ListVolumeAttachments(&volumeAttachmentRequest, vpcs.Logger)
// Keep retry, until we get the proper volumeAttachmentRequest object
if err != nil {
return err, skipRetryForObviousErrors(err, vpcs.Config.VPCConfig.IsIKS)
Expand All @@ -111,12 +116,12 @@ func (vpcs *VPCSession) getVolumeAttachmentByVolumeID(volumeAttachmentRequest mo
// Check if volume ID is matching with requested volume ID
if volumeAttachmentItem.Volume.ID == volumeAttachmentRequest.Volume.ID {
vpcs.Logger.Info("Successfully found volume attachment", zap.Reflect("volumeAttachment", volumeAttachmentItem))
volumeResponse := volumeAttachmentItem.ToVolumeAttachmentResponse(vpcs.Config.VPCConfig.VPCBlockProviderType)
volumeResponse := volumeAttachmentItem.ToVolumeAttachmentResponse(vpcs.Config.VPCConfig.VPCBlockProviderType, isBMS)
vpcs.Logger.Info("Successfully fetched volume attachment from VPC provider", zap.Reflect("volumeResponse", volumeResponse))
return volumeResponse, nil
}
}
// No volume attahment found in the list. So return error
// No volume attachment found in the list. So return error
userErr := userError.GetUserError(string(userError.VolumeAttachFindFailed), errors.New("no VolumeAttachment Found"), volumeAttachmentRequest.Volume.ID, *volumeAttachmentRequest.InstanceID)
vpcs.Logger.Error("Volume attachment not found", zap.Error(err))
return nil, userErr
Expand Down
27 changes: 27 additions & 0 deletions block/provider/session.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,23 @@
package provider

import (
"sync"

"github.com/IBM/ibmcloud-volume-interface/lib/provider"
vpcconfig "github.com/IBM/ibmcloud-volume-vpc/block/vpcconfig"
"github.com/IBM/ibmcloud-volume-vpc/common/vpcclient/instances"
"github.com/IBM/ibmcloud-volume-vpc/common/vpcclient/riaas"
"go.uber.org/zap"
)

// bmsOnceState holds the sync.Once state for lazy BMS manager init.
// It is heap-allocated so that VPCSession remains copyable (sync.Once must
// not be copied after first use, but VPCSession is copied in the IKS path).
type bmsOnceState struct {
once sync.Once
mgr instances.VolumeAttachManager
}

// VPCSession implements lib.Session
type VPCSession struct {
provider.DefaultVolumeProvider
Expand All @@ -39,6 +49,23 @@ type VPCSession struct {
Logger *zap.Logger
APIRetry FlexyRetry
SessionError error

// bmsState is heap-allocated so that copying VPCSession (as the IKS path
// does) does not violate the sync.Once noCopy constraint.
// Only initialised on the first BMS request; nil on IKS clusters.
bmsState *bmsOnceState
}

// getBMSVolAttachMgr returns the lazily-initialised BMS volume-attachment
// manager. It is safe for concurrent use via sync.Once.
func (vpcs *VPCSession) getBMSVolAttachMgr() instances.VolumeAttachManager {
if vpcs.bmsState == nil {
vpcs.bmsState = &bmsOnceState{}
}
vpcs.bmsState.once.Do(func() {
vpcs.bmsState.mgr = vpcs.Apiclient.BMSVolumeAttachService()
})
return vpcs.bmsState.mgr
}

const (
Expand Down
19 changes: 14 additions & 5 deletions block/provider/wait_for_attach_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ import (
"github.com/IBM/ibmcloud-volume-interface/lib/metrics"
"github.com/IBM/ibmcloud-volume-interface/lib/provider"
userError "github.com/IBM/ibmcloud-volume-vpc/common/messages"
"github.com/IBM/ibmcloud-volume-vpc/common/vpcclient/models"
"go.uber.org/zap"
)

// WaitForAttachVolume waits for volume to be attached to node. e.g waits till status becomes attached
// WaitForAttachVolume waits for volume to be attached to node. e.g waits till status becomes attached.
// For BMS SDP volumes the terminal status is "available"; for VSI volumes it is "attached".
func (vpcs *VPCSession) WaitForAttachVolume(volumeAttachmentTemplate provider.VolumeAttachmentRequest) (*provider.VolumeAttachmentResponse, error) {
vpcs.Logger.Debug("Entry of WaitForAttachVolume method...")
defer vpcs.Logger.Debug("Exit from WaitForAttachVolume method...")
Expand All @@ -44,6 +46,13 @@ func (vpcs *VPCSession) WaitForAttachVolume(volumeAttachmentTemplate provider.Vo
return nil, err
}

// BMS SDP volumes report "available" when ready; VSI volumes report "attached".
isBMS := !vpcs.Config.VPCConfig.IsIKS && volumeAttachmentTemplate.BMSVolumeAttachment != nil
terminalStatus := StatusAttached
if isBMS {
terminalStatus = models.VolumeAvailable
}

var currentVolAttachment *provider.VolumeAttachmentResponse
err = vpcs.APIRetry.FlexyRetryWithCustomGap(vpcs.Logger, func() (error, bool) {
currentVolAttachment, err = vpcs.GetVolumeAttachment(volumeAttachmentTemplate)
Expand All @@ -52,11 +61,11 @@ func (vpcs *VPCSession) WaitForAttachVolume(volumeAttachmentTemplate provider.Vo
// considering that vpcs.GetVolumeAttachment already re-tried
return err, true
}
// Stop retry in case of volume is attached
return err, currentVolAttachment != nil && currentVolAttachment.Status == StatusAttached
// Stop retry once the volume reaches its terminal attached status
return err, currentVolAttachment != nil && currentVolAttachment.Status == terminalStatus
})
// Success case, checks are required in case of timeout happened and volume is still not attached state
if err == nil && (currentVolAttachment != nil && currentVolAttachment.Status == StatusAttached) {
// Success case, checks are required in case of timeout happened and volume is still not in attached state
if err == nil && (currentVolAttachment != nil && currentVolAttachment.Status == terminalStatus) {
return currentVolAttachment, nil
}

Expand Down
Loading
Loading