feat(migration): adding support to migrate the PV to a new node (#304)

Usecase: A node in the Kubernetes cluster is replaced with a new node. The 
new node gets a different `kubernetes.io/hostname`. The storage devices
that were attached to the old node are re-attached to the new node. 

Fix: Instead of using the default `kubenetes.io/hostname` as the node affinity 
label, this commit changes to use `openebs.io/nodeid`. The ZFS LocalPV driver 
will pick the value from the nodes and set the affinity.

Once the old node is removed from the cluster, the K8s scheduler will continue 
to schedule applications on the old node only.

User can now modify the value of `openebs.io/nodeid` on the new node to the same
value that was available on the old node. This will make sure the pods/volumes are 
scheduled to the node now. 


Note: Now to migrate the PV to the other node, we have to move the disks to the other node
and remove the old node from the cluster and set the same label on the new node using
the same key, which will let k8s scheduler to schedule the pods to that node.

Other updates: 
* adding faq doc
* renaming the config variable to nodename

Signed-off-by: Pawan <pawan@mayadata.io>
Co-authored-by: Akhil Mohan <akhilerm@gmail.com>

* Update docs/faq.md

Co-authored-by: Akhil Mohan <akhilerm@gmail.com>

pkg/apis/openebs.io/zfs/v1/zfsvolume.go

@@ -29,7 +29,7 @@ import (
 // +kubebuilder:storageversion
 // +kubebuilder:resource:scope=Namespaced,shortName=zfsvol;zv
 // +kubebuilder:printcolumn:name="ZPool",type=string,JSONPath=`.spec.poolName`,description="ZFS Pool where the volume is created"
-// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=`.spec.ownerNodeID`,description="Node where the volume is created"
+// +kubebuilder:printcolumn:name="NodeID",type=string,JSONPath=`.spec.ownerNodeID`,description="Node where the volume is created"
 // +kubebuilder:printcolumn:name="Size",type=string,JSONPath=`.spec.capacity`,description="Size of the volume"
 // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,description="Status of the volume"
 // +kubebuilder:printcolumn:name="Filesystem",type=string,JSONPath=`.spec.fsType`,description="filesystem created on the volume"

pkg/builder/volbuilder/build.go

@@ -136,9 +136,9 @@ func (b *Builder) WithThinProv(thinprov string) *Builder {
 	return b
 }
 
-// WithOwnerNode sets owner node for the ZFSVolume where the volume should be provisioned
-func (b *Builder) WithOwnerNode(host string) *Builder {
-	b.volume.Object.Spec.OwnerNodeID = host
+// WithOwnerNodeID sets owner nodeid for the ZFSVolume where the volume should be provisioned
+func (b *Builder) WithOwnerNodeID(nodeid string) *Builder {
+	b.volume.Object.Spec.OwnerNodeID = nodeid
 	return b
 }
 

pkg/config/config.go

@@ -37,11 +37,10 @@ type Config struct {
 	//  - This will be a unix based socket
 	Endpoint string
 
-	// NodeID helps in differentiating the nodes on
-	// which node drivers are running. This is useful
-	// in case of topologies and publishing or
-	// unpublishing volumes on nodes
-	NodeID string
+	// Nodename helps in differentiating the nodes on
+	// which node drivers are running. This is used
+	// to set the topologies for the driver
+	Nodename string
 }
 
 // Default returns a new instance of config

pkg/driver/agent.go

@@ -204,9 +204,9 @@ func (ns *node) NodeGetInfo(
 	req *csi.NodeGetInfoRequest,
 ) (*csi.NodeGetInfoResponse, error) {
 
-	node, err := k8sapi.GetNode(ns.driver.config.NodeID)
+	node, err := k8sapi.GetNode(ns.driver.config.Nodename)
 	if err != nil {
-		klog.Errorf("failed to get the node %s", ns.driver.config.NodeID)
+		klog.Errorf("failed to get the node %s", ns.driver.config.Nodename)
 		return nil, err
 	}
 	/*
@@ -229,11 +229,13 @@ func (ns *node) NodeGetInfo(
 	// support all the keys that node has
 	topology := node.Labels
 
-	// add driver's topology key
-	topology[zfs.ZFSTopologyKey] = ns.driver.config.NodeID
+	// add driver's topology key if not labelled already
+	if _, ok := topology[zfs.ZFSTopologyKey]; !ok {
+		topology[zfs.ZFSTopologyKey] = ns.driver.config.Nodename
+	}
 
 	return &csi.NodeGetInfoResponse{
-		NodeId: ns.driver.config.NodeID,
+		NodeId: ns.driver.config.Nodename,
 		AccessibleTopology: &csi.Topology{
 			Segments: topology,
 		},

pkg/driver/controller.go

@@ -227,7 +227,12 @@ func CreateZFSVolume(ctx context.Context, req *csi.CreateVolumeRequest) (string,
 
 	// try volume creation sequentially on all nodes
 	for _, node := range prfList {
-		vol, _ := volbuilder.BuildFrom(volObj).WithOwnerNode(node).WithVolumeStatus(zfs.ZFSStatusPending).Build()
+		nodeid, err := zfs.GetNodeID(node)
+		if err != nil {
+			continue
+		}
+
+		vol, _ := volbuilder.BuildFrom(volObj).WithOwnerNodeID(nodeid).WithVolumeStatus(zfs.ZFSStatusPending).Build()
 
 		timeout := false
 
@@ -392,7 +397,12 @@ func (cs *controller) CreateVolume(
 
 	sendEventOrIgnore(pvcName, volName, strconv.FormatInt(int64(size), 10), "zfs-localpv", analytics.VolumeProvision)
 
-	topology := map[string]string{zfs.ZFSTopologyKey: selected}
+	nodeid, err := zfs.GetNodeID(selected)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "GetNodeID failed : %s", err.Error())
+	}
+
+	topology := map[string]string{zfs.ZFSTopologyKey: nodeid}
 	cntx := map[string]string{zfs.PoolNameKey: pool}
 
 	return csipayload.NewCreateVolumeResponseBuilder().

pkg/zfs/volume.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"time"
 
+	k8sapi "github.com/openebs/lib-csi/pkg/client/k8s"
 	apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/zfs/v1"
 	"github.com/openebs/zfs-localpv/pkg/builder/bkpbuilder"
 	"github.com/openebs/zfs-localpv/pkg/builder/restorebuilder"
@@ -49,7 +50,7 @@ const (
 	// ZFSNodeKey will be used to insert Label in ZfsVolume CR
 	ZFSNodeKey string = "kubernetes.io/nodename"
 	// ZFSTopologyKey is supported topology key for the zfs driver
-	ZFSTopologyKey string = "openebs.io/nodename"
+	ZFSTopologyKey string = "openebs.io/nodeid"
 	// ZFSStatusPending shows object has not handled yet
 	ZFSStatusPending string = "Pending"
 	// ZFSStatusFailed shows object operation has failed
@@ -70,19 +71,45 @@ var (
 )
 
 func init() {
+	var err error
 
 	OpenEBSNamespace = os.Getenv(OpenEBSNamespaceKey)
-	if OpenEBSNamespace == "" && os.Getenv("OPENEBS_NODE_DRIVER") != "" {
-		klog.Fatalf("OPENEBS_NAMESPACE environment variable not set")
-	}
-	NodeID = os.Getenv("OPENEBS_NODE_ID")
-	if NodeID == "" && os.Getenv("OPENEBS_NODE_DRIVER") != "" {
-		klog.Fatalf("NodeID environment variable not set")
+
+	if os.Getenv("OPENEBS_NODE_DRIVER") != "" {
+		if OpenEBSNamespace == "" {
+			klog.Fatalf("OPENEBS_NAMESPACE environment variable not set for daemonset")
+		}
+		nodename := os.Getenv("OPENEBS_NODE_NAME")
+		if nodename == "" {
+			klog.Fatalf("OPENEBS_NODE_NAME environment variable not set")
+		}
+		if NodeID, err = GetNodeID(nodename); err != nil {
+			klog.Fatalf("GetNodeID failed for node=%s err: %s", nodename, err.Error())
+		}
+		klog.Infof("zfs: node(%s) has node affinity %s=%s", nodename, ZFSTopologyKey, NodeID)
+	} else if os.Getenv("OPENEBS_CONTROLLER_DRIVER") != "" {
+		if OpenEBSNamespace == "" {
+			klog.Fatalf("OPENEBS_NAMESPACE environment variable not set for controller")
+		}
 	}
 
 	GoogleAnalyticsEnabled = os.Getenv(GoogleAnalyticsKey)
 }
 
+func GetNodeID(nodename string) (string, error) {
+	node, err := k8sapi.GetNode(nodename)
+	if err != nil {
+		return "", fmt.Errorf("failed to get the node %s", nodename)
+	}
+
+	nodeid, ok := node.Labels[ZFSTopologyKey]
+	if !ok {
+		// node is not labelled, use node name as nodeid
+		return nodename, nil
+	}
+	return nodeid, nil
+}
+
 func checkVolCreation(ctx context.Context, volname string) (bool, error) {
 	timeout := time.After(10 * time.Second)
 	for {
@@ -104,7 +131,7 @@ func checkVolCreation(ctx context.Context, volname string) (bool, error) {
 				return false, fmt.Errorf("zfs: volume creation failed")
 			}
 
-			klog.Infof("zfs: waiting for volume %s/%s to be created on node %s",
+			klog.Infof("zfs: waiting for volume %s/%s to be created on nodeid %s",
 				vol.Spec.PoolName, volname, vol.Spec.OwnerNodeID)
 
 			time.Sleep(time.Second)
@@ -135,7 +162,7 @@ func ProvisionVolume(
 	}
 
 	if err != nil {
-		klog.Infof("zfs: volume %s/%s provisioning failed on node %s err: %s",
+		klog.Infof("zfs: volume %s/%s provisioning failed on nodeid %s err: %s",
 			vol.Spec.PoolName, vol.Name, vol.Spec.OwnerNodeID, err.Error())
 	}