feat(zfs-localpv): initial commit

provisioning and deprovisioning of
the volumes on the node where zfs pool
has already been setup. Pool name and the volume
parameters has to be given in storage class
which will be used to provision the volume.

Signed-off-by: Pawan <pawan@mayadata.io>

pkg/zfs/mount.go

@@ -0,0 +1,145 @@
+package zfs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/Sirupsen/logrus"
+	apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+// FormatAndMountZvol formats and mounts the created volume to the desired mount path
+func FormatAndMountZvol(devicePath string, mountInfo *apis.MountInfo) error {
+	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()}
+
+	err := mounter.FormatAndMount(devicePath, mountInfo.MountPath, mountInfo.FSType, mountInfo.MountOptions)
+	if err != nil {
+		logrus.Errorf(
+			"zfspv: failed to mount volume %s [%s] to %s, error %v",
+			devicePath, mountInfo.FSType, mountInfo.MountPath, err,
+		)
+		return err
+	}
+
+	logrus.Infof("created zvol %v and mounted %v fs %v", devicePath, mountInfo.MountPath, mountInfo.FSType)
+	return nil
+}
+
+// UmountVolume unmounts the volume and the corresponding mount path is removed
+func UmountVolume(vol *apis.ZFSVolume, targetPath string,
+) error {
+	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()}
+
+	_, _, err := mount.GetDeviceNameFromMount(mounter, targetPath)
+	if err != nil {
+		logrus.Errorf(
+			"zfspv umount volume: failed to get device from mnt: %s\nError: %v",
+			targetPath, err,
+		)
+		return err
+	}
+
+	if pathExists, pathErr := mount.PathExists(targetPath); pathErr != nil {
+		return fmt.Errorf("Error checking if path exists: %v", pathErr)
+	} else if !pathExists {
+		logrus.Warningf(
+			"Warning: Unmount skipped because path does not exist: %v",
+			targetPath,
+		)
+		return nil
+	}
+
+	if err = mounter.Unmount(targetPath); err != nil {
+		logrus.Errorf(
+			"zfspv umount volume: failed to unmount: %s\nError: %v",
+			targetPath, err,
+		)
+		return err
+	}
+
+	if err := os.RemoveAll(targetPath); err != nil {
+		logrus.Errorf("zfspv: failed to remove mount path Error: %v", err)
+		return err
+	}
+
+	logrus.Infof("umount done path %v", targetPath)
+
+	return nil
+}
+
+// GetMounts gets mountpoints for the specified volume
+func GetMounts(devicepath string) ([]string, error) {
+
+	var (
+		currentMounts []string
+		err           error
+		mountList     []mount.MountPoint
+	)
+
+	dev, err := filepath.EvalSymlinks(devicepath)
+	if err != nil {
+		return nil, err
+	}
+	mounter := mount.New("")
+	// Get list of mounted paths present with the node
+	if mountList, err = mounter.List(); err != nil {
+		return nil, err
+	}
+	for _, mntInfo := range mountList {
+		if mntInfo.Device == dev {
+			currentMounts = append(currentMounts, mntInfo.Path)
+		}
+	}
+	return currentMounts, nil
+}
+
+// CreateAndMountZvol creates the zfs Volume
+// and mounts the disk to the specified path
+func CreateAndMountZvol(vol *apis.ZFSVolume, mount *apis.MountInfo) error {
+	if len(mount.MountPath) == 0 {
+		return status.Error(codes.InvalidArgument, "mount path missing in request")
+	}
+
+	if len(vol.Spec.OwnerNodeID) > 0 &&
+		vol.Spec.OwnerNodeID != NodeID {
+		return status.Error(codes.Internal, "volume is owned by different node")
+	}
+
+	devicePath, err := createZvol(vol)
+	if err != nil {
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	err = UpdateZvolInfo(vol)
+	if err != nil {
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	/*
+	 * This check is the famous *Wall Of North*
+	 * It will not let the volume to be mounted
+	 * at more than two places. The volume should
+	 * be unmounted before proceeding to the mount
+	 * operation.
+	 */
+	currentMounts, err := GetMounts(devicePath)
+	if err != nil {
+		return err
+	} else if len(currentMounts) >= 1 {
+		logrus.Errorf(
+			"can not mount, more than one mounts for volume:%s dev %s mounts: %v",
+			vol.Name, devicePath, currentMounts,
+		)
+		return status.Error(codes.Internal, "device already mounted")
+	}
+	err = FormatAndMountZvol(devicePath, mount)
+	if err != nil {
+		return status.Error(codes.Internal, err.Error())
+	}
+
+	return err
+}

pkg/zfs/volume.go

@@ -0,0 +1,138 @@
+// Copyright © 2019 The OpenEBS Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package zfs
+
+import (
+	"github.com/Sirupsen/logrus"
+	"os"
+
+	apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1"
+	"github.com/openebs/zfs-localpv/pkg/builder"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// OpenEBSNamespace is the environment variable to get openebs namespace
+	//
+	// This environment variable is set via kubernetes downward API
+	OpenEBSNamespaceKey string = "OPENEBS_NAMESPACE"
+	// ZFSFinalizer for the ZfsVolume CR
+	ZFSFinalizer string = "zfs.openebs.io/finalizer"
+	// ZFSNodeKey will be used to insert Label
+	// in ZfsVolume CR
+	ZFSNodeKey string = "kubernetes.io/nodename"
+)
+
+var (
+	// OpenEBSNamespace is openebs system namespace
+	OpenEBSNamespace string
+
+	// NodeID is the NodeID of the node on which the pod is present
+	NodeID string
+)
+
+func init() {
+
+	OpenEBSNamespace = os.Getenv(OpenEBSNamespaceKey)
+	if OpenEBSNamespace == "" {
+		logrus.Fatalf("OPENEBS_NAMESPACE environment variable not set")
+	}
+	NodeID = os.Getenv("OPENEBS_NODE_ID")
+	if NodeID == "" && os.Getenv("OPENEBS_NODE_DRIVER") != "" {
+		logrus.Fatalf("NodeID environment variable not set")
+	}
+}
+
+// ProvisionVolume creates a ZFSVolume(zv) CR,
+// watcher for zvc is present in CSI agent
+func ProvisionVolume(
+	size int64,
+	vol *apis.ZFSVolume,
+) error {
+
+	_, err := builder.NewKubeclient().WithNamespace(OpenEBSNamespace).Create(vol)
+	if err == nil {
+		logrus.Infof("provisioned volume %s", vol.Name)
+	}
+
+	return err
+}
+
+// GetVolume the corresponding ZFSVolume CR
+func GetVolume(volumeID string) (*apis.ZFSVolume, error) {
+	return builder.NewKubeclient().
+		WithNamespace(OpenEBSNamespace).
+		Get(volumeID, metav1.GetOptions{})
+}
+
+// DeleteVolume deletes the corresponding ZFSVol CR
+func DeleteVolume(volumeID string) (err error) {
+	err = builder.NewKubeclient().WithNamespace(OpenEBSNamespace).Delete(volumeID)
+	if err == nil {
+		logrus.Infof("deprovisioned volume %s", volumeID)
+	}
+
+	return
+}
+
+// GetVolList fetches the current Published Volume list
+func GetVolList(volumeID string) (*apis.ZFSVolumeList, error) {
+	listOptions := v1.ListOptions{
+		LabelSelector: ZFSNodeKey + "=" + NodeID,
+	}
+
+	return builder.NewKubeclient().
+		WithNamespace(OpenEBSNamespace).List(listOptions)
+
+}
+
+// GetZFSVolume fetches the current Published csi Volume
+func GetZFSVolume(volumeID string) (*apis.ZFSVolume, error) {
+	getOptions := metav1.GetOptions{}
+	vol, err := builder.NewKubeclient().
+		WithNamespace(OpenEBSNamespace).Get(volumeID, getOptions)
+	return vol, err
+}
+
+// UpdateZvolInfo updates ZFSVolume CR with node id and finalizer
+func UpdateZvolInfo(vol *apis.ZFSVolume) error {
+	finalizers := []string{ZFSFinalizer}
+	labels := map[string]string{ZFSNodeKey: NodeID}
+
+	if vol.Finalizers != nil {
+		return nil
+	}
+
+	newVol, err := builder.BuildFrom(vol).
+		WithNodename(NodeID).
+		WithFinalizer(finalizers).
+		WithLabels(labels).Build()
+
+	if err != nil {
+		return err
+	}
+
+	_, err = builder.NewKubeclient().WithNamespace(OpenEBSNamespace).Update(newVol)
+	return err
+}
+
+// RemoveZvolFinalizer adds finalizer to ZFSVolume CR
+func RemoveZvolFinalizer(vol *apis.ZFSVolume) error {
+	vol.Finalizers = nil
+
+	_, err := builder.NewKubeclient().WithNamespace(OpenEBSNamespace).Update(vol)
+	return err
+}

pkg/zfs/zfs_util.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package zfs
+
+import (
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1"
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+const (
+	ZFS_DEVPATH = "/dev/zvol/"
+)
+
+func PropertyChanged(oldVol *apis.ZFSVolume, newVol *apis.ZFSVolume) bool {
+	return oldVol.Spec.Compression != newVol.Spec.Compression ||
+		oldVol.Spec.Dedup != newVol.Spec.Dedup ||
+		oldVol.Spec.Capacity != newVol.Spec.Capacity
+}
+
+// createZvol creates the zvol and returns the corresponding diskPath
+// of the volume which gets created on the node
+func createZvol(vol *apis.ZFSVolume) (string, error) {
+	var out []byte
+	zvol := vol.Spec.PoolName + "/" + vol.Name
+	devicePath := ZFS_DEVPATH + zvol
+
+	if _, err := os.Stat(devicePath); os.IsNotExist(err) {
+		if vol.Spec.ThinProvision == "yes" {
+			out, err = mount.NewOsExec().Run(
+				"zfs", "create",
+				"-s",
+				"-V", vol.Spec.Capacity,
+				"-b", vol.Spec.BlockSize,
+				"-o", "compression="+vol.Spec.Compression,
+				"-o", "dedup="+vol.Spec.Dedup,
+				zvol,
+			)
+		} else {
+			out, err = mount.NewOsExec().Run(
+				"zfs", "create",
+				"-V", vol.Spec.Capacity,
+				"-b", vol.Spec.BlockSize,
+				"-o", "compression="+vol.Spec.Compression,
+				"-o", "dedup="+vol.Spec.Dedup,
+				zvol,
+			)
+		}
+
+		if err != nil {
+			logrus.Errorf(
+				"zfs: could not create zvol %v vol %v error: %s", zvol, vol, string(out),
+			)
+			return "", err
+		}
+		logrus.Infof("created zvol %s", zvol)
+	} else if err == nil {
+		logrus.Infof("using existing zvol %v", zvol)
+	} else {
+		return "", err
+	}
+
+	return devicePath, nil
+}
+
+// SetZvolProp sets the zvol property
+func SetZvolProp(vol *apis.ZFSVolume) error {
+	var out []byte
+	var err error
+	zvol := vol.Spec.PoolName + "/" + vol.Name
+	devicePath := ZFS_DEVPATH + zvol
+
+	if _, err = os.Stat(devicePath); err == nil {
+		// TODO(pawan) need to find a way to identify
+		// which property has changed
+		out, err = mount.NewOsExec().Run(
+			"zfs", "set",
+			"volsize="+vol.Spec.Capacity,
+			"compression="+vol.Spec.Compression,
+			"dedup="+vol.Spec.Dedup,
+			zvol,
+		)
+		if err != nil {
+			logrus.Errorf(
+				"zfs: could not set property on zvol %v vol %v error: %s", zvol, vol, string(out),
+			)
+			return err
+		}
+		logrus.Infof("property set on zvol %s", zvol)
+	}
+
+	return err
+}
+
+// DestroyZvol deletes the zvol
+func DestroyZvol(vol *apis.ZFSVolume) error {
+	var out []byte
+	zvol := vol.Spec.PoolName + "/" + vol.Name
+	devicePath := ZFS_DEVPATH + zvol
+
+	if _, err := os.Stat(devicePath); err == nil {
+		out, err = mount.NewOsExec().Run(
+			"zfs", "destroy",
+			"-R",
+			zvol,
+		)
+		if err != nil {
+			logrus.Errorf(
+				"zfs: could not destroy zvol %v vol %v error: %s", zvol, vol, string(out),
+			)
+			return err
+		}
+		logrus.Infof("destroyed zvol %s", zvol)
+	}
+
+	return nil
+}