feat(ZFSPV): adding encryption in ZFSVolume CR (#6)

Adding support for enabling encryption using a custom key. 

Also, adding support to inherit the properties from ZPOOL
which are not listed in the storage class, ZFS driver will
not pass default values while creating the volume. Those
properties will be inherited from the ZPOOL.

we can use the encryption option in storage class 
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: openebs-zfspv
allowVolumeExpansion: true
parameters:
  blocksize: "4k"
  compression: "on"
  dedup: "on"
  thinprovision: "yes"
  encryption: "on"
  keyformat: "raw"
  keylocation: "file:///home/keys/key"
  poolname: "zfspv-pool"
provisioner: openebs.io/zfs
```

Just a note, the key file should be mounted inside the node-agent container so that we can use that file while provisioning the volume. keyformat can be raw, hex or passphrase.

Signed-off-by: Pawan <pawan@mayadata.io>

deploy/sample/fio.yaml

@@ -8,6 +8,9 @@ parameters:
   compression: "on"
   dedup: "on"
   thinprovision: "yes"
+  #encryption: "on"
+  #keyformat: "raw"
+  #keylocation: "file:///home/pawan/key"
   poolname: "zfspv-pool"
 provisioner: openebs.io/zfs
 volumeBindingMode: WaitForFirstConsumer

deploy/sample/percona.yaml

@@ -0,0 +1,131 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: openebs-zfspv
+allowVolumeExpansion: true
+parameters:
+  blocksize: "4k"
+  compression: "on"
+  dedup: "on"
+  thinprovision: "yes"
+  poolname: "zfspv-pool"
+provisioner: openebs.io/zfs
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: csi-zfspv
+spec:
+  storageClassName: openebs-zfspv
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 4Gi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  annotations:
+  name: sqltest
+  namespace: default
+data:
+  sql-test.sh: |
+    #!/bin/bash
+
+    DB_PREFIX="Inventory"
+    DB_SUFFIX=`echo $(mktemp) | cut -d '.' -f 2`
+    DB_NAME="${DB_PREFIX}_${DB_SUFFIX}"
+
+
+    echo -e "\nWaiting for mysql server to start accepting connections.."
+    retries=10;wait_retry=30
+    for i in `seq 1 $retries`; do
+      mysql -uroot -pk8sDem0 -e 'status' > /dev/null 2>&1
+      rc=$?
+      [ $rc -eq 0 ] && break
+      sleep $wait_retry
+    done
+
+    if [ $rc -ne 0 ];
+    then
+      echo -e "\nFailed to connect to db server after trying for $(($retries * $wait_retry))s, exiting\n"
+      exit 1
+    fi
+    mysql -uroot -pk8sDem0 -e "CREATE DATABASE $DB_NAME;"
+    mysql -uroot -pk8sDem0 -e "CREATE TABLE Hardware (id INTEGER, name VARCHAR(20), owner VARCHAR(20),description VARCHAR(20));" $DB_NAME
+    mysql -uroot -pk8sDem0 -e "INSERT INTO Hardware (id, name, owner, description) values (1, "dellserver", "basavaraj", "controller");" $DB_NAME
+    mysql -uroot -pk8sDem0 -e "DROP DATABASE $DB_NAME;"
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: percona
+  labels:
+    name: percona
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: percona
+  template:
+    metadata:
+      labels:
+        name: percona
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: In
+                values:
+                - gke-pawan-zfspv-default-pool-26f2b9a9-5fqd
+      containers:
+        - resources:
+          name: percona
+          image: openebs/tests-custom-percona:latest
+          imagePullPolicy: IfNotPresent
+          args:
+            - "--ignore-db-dir"
+            - "lost+found"
+          env:
+            - name: MYSQL_ROOT_PASSWORD
+              value: k8sDem0
+          ports:
+            - containerPort: 3306
+              name: percona
+          volumeMounts:
+            - mountPath: /var/lib/mysql
+              name: demo-vol1
+            - mountPath: /sql-test.sh
+              subPath: sql-test.sh
+              name: sqltest-configmap
+          livenessProbe:
+            exec:
+              command: ["bash", "sql-test.sh"]
+            initialDelaySeconds: 30
+            periodSeconds: 1
+            timeoutSeconds: 10
+      volumes:
+        - name: demo-vol1
+          persistentVolumeClaim:
+            claimName: csi-zfspv
+        - name: sqltest-configmap
+          configMap:
+            name: sqltest
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: percona-mysql
+  labels:
+    name: percona-mysql
+spec:
+  ports:
+    - port: 3306
+      targetPort: 3306
+  selector:
+      name: percona

deploy/zfs-operator.yaml

@@ -396,6 +396,8 @@ spec:
               mountPath: /plugin
             - name: device-dir
               mountPath: /dev
+            - name: encr-keys
+              mountPath: /home/keys
             - name: zfs-bin
               mountPath: /sbin/zfs
             - name: libzpool
@@ -418,6 +420,10 @@ spec:
           hostPath:
             path: /dev
             type: Directory
+        - name: encr-keys
+          hostPath:
+            path: /home/keys
+            type: DirectoryOrCreate
         - name: zfs-bin
           hostPath:
             path: /sbin/zfs

pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go

@@ -91,9 +91,21 @@ type VolumeInfo struct {
 	Compression string `json:"compression"`
 
 	// Dedup specifies the deduplication
-	// should be enabledd on the zvol
+	// should be enabled on the zvol
 	Dedup string `json:"dedup"`
 
+	// Encryption specifies the encryption
+	// should be enabled on the zvol
+	Encryption string `json:"encryption"`
+
+	// KeyLocation is the location of key
+	// for the encryption
+	KeyLocation string `json:"keylocation"`
+
+	// KeyFormat specifies format of the
+	// encryption key
+	KeyFormat string `json:"keyformat"`
+
 	// Thinprovision specifies if we should
 	// thin provisioned the volume or not
 	ThinProvision string `json:"thinProvison"`

pkg/builder/build.go

@@ -54,7 +54,7 @@ func BuildFrom(volume *apis.ZFSVolume) *Builder {
 	}
 }
 
-// WithNamespace sets the namespace of csi volume
+// WithNamespace sets the namespace of  ZFSVolume
 func (b *Builder) WithNamespace(namespace string) *Builder {
 	if namespace == "" {
 		b.errs = append(
@@ -69,7 +69,7 @@ func (b *Builder) WithNamespace(namespace string) *Builder {
 	return b
 }
 
-// WithName sets the name of csi volume
+// WithName sets the name of ZFSVolume
 func (b *Builder) WithName(name string) *Builder {
 	if name == "" {
 		b.errs = append(
@@ -100,42 +100,44 @@ func (b *Builder) WithCapacity(capacity string) *Builder {
 	return b
 }
 
-// WithCompression sets compression of CStorVolumeClaim
+// WithEncryption sets the encryption on ZFSVolume
+func (b *Builder) WithEncryption(encr string) *Builder {
+	b.volume.Object.Spec.Encryption = encr
+	return b
+}
+
+// WithKeyLocation sets the encryption key location on ZFSVolume
+func (b *Builder) WithKeyLocation(kl string) *Builder {
+	b.volume.Object.Spec.KeyLocation = kl
+	return b
+}
+
+// WithKeyFormat sets the encryption key format on ZFSVolume
+func (b *Builder) WithKeyFormat(kf string) *Builder {
+	b.volume.Object.Spec.KeyFormat = kf
+	return b
+}
+
+// WithCompression sets compression of ZFSVolume
 func (b *Builder) WithCompression(compression string) *Builder {
-
+	b.volume.Object.Spec.Compression = compression
-	comp := "off"
-	if compression == "on" {
-		comp = "on"
-	}
-	b.volume.Object.Spec.Compression = comp
 	return b
 }
 
-// WithDedup sets compression of CStorVolumeClaim
+// WithDedup sets dedup property of ZFSVolume
 func (b *Builder) WithDedup(dedup string) *Builder {
-
+	b.volume.Object.Spec.Dedup = dedup
-	dp := "off"
-	if dedup == "on" {
-		dp = "on"
-	}
-	b.volume.Object.Spec.Dedup = dp
 	return b
 }
 
-// WithThinProv sets compression of CStorVolumeClaim
+// WithThinProv sets if ZFSVolume needs to be thin provisioned
 func (b *Builder) WithThinProv(thinprov string) *Builder {
-
+	b.volume.Object.Spec.ThinProvision = thinprov
-	tp := "no"
-	if thinprov == "yes" {
-		tp = "yes"
-	}
-	b.volume.Object.Spec.ThinProvision = tp
 	return b
 }
 
-// WithBlockSize sets blocksize of CStorVolumeClaim
+// WithBlockSize sets blocksize of ZFSVolume
 func (b *Builder) WithBlockSize(blockSize string) *Builder {
-
 	bs := "4k"
 	if len(blockSize) > 0 {
 		bs = blockSize

pkg/driver/controller.go

@@ -73,6 +73,9 @@ func (cs *controller) CreateVolume(
 	bs := req.GetParameters()["blocksize"]
 	compression := req.GetParameters()["compression"]
 	dedup := req.GetParameters()["dedup"]
+	encr := req.GetParameters()["encryption"]
+	kf := req.GetParameters()["keyformat"]
+	kl := req.GetParameters()["keylocation"]
 	pool := req.GetParameters()["poolname"]
 	tp := req.GetParameters()["thinprovision"]
 
@@ -82,6 +85,9 @@ func (cs *controller) CreateVolume(
 		WithBlockSize(bs).
 		WithPoolName(pool).
 		WithDedup(dedup).
+		WithEncryption(encr).
+		WithKeyFormat(kf).
+		WithKeyLocation(kl).
 		WithThinProv(tp).
 		WithCompression(compression).Build()
 

pkg/zfs/zfs_util.go

@@ -18,14 +18,18 @@ package zfs
 
 import (
 	"os"
+	"os/exec"
 
 	"github.com/Sirupsen/logrus"
 	apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1"
-	"k8s.io/kubernetes/pkg/util/mount"
 )
 
 const (
-	ZFS_DEVPATH = "/dev/zvol/"
+	ZFS_DEVPATH   = "/dev/zvol/"
+	ZFSVolCmd     = "zfs"
+	ZFSCreateArg  = "create"
+	ZFSDestroyArg = "destroy"
+	ZFSSetArg     = "set"
 )
 
 func PropertyChanged(oldVol *apis.ZFSVolume, newVol *apis.ZFSVolume) bool {
@@ -34,38 +38,102 @@ func PropertyChanged(oldVol *apis.ZFSVolume, newVol *apis.ZFSVolume) bool {
 		oldVol.Spec.Capacity != newVol.Spec.Capacity
 }
 
+// builldVolumeCreateArgs returns zvol create command along with attributes as a string array
+func buildVolumeCreateArgs(vol *apis.ZFSVolume) []string {
+	var ZFSVolCmd []string
+
+	zvol := vol.Spec.PoolName + "/" + vol.Name
+
+	ZFSVolCmd = append(ZFSVolCmd, ZFSCreateArg)
+
+	if vol.Spec.ThinProvision == "yes" {
+		ZFSVolCmd = append(ZFSVolCmd, "-s")
+	}
+	if len(vol.Spec.Capacity) != 0 {
+		ZFSVolCmd = append(ZFSVolCmd, "-V", vol.Spec.Capacity)
+	}
+	if len(vol.Spec.BlockSize) != 0 {
+		ZFSVolCmd = append(ZFSVolCmd, "-b", vol.Spec.BlockSize)
+	}
+	if len(vol.Spec.Dedup) != 0 {
+		dedupProperty := "dedup=" + vol.Spec.Dedup
+		ZFSVolCmd = append(ZFSVolCmd, "-o", dedupProperty)
+	}
+	if len(vol.Spec.Compression) != 0 {
+		compressionProperty := "compression=" + vol.Spec.Compression
+		ZFSVolCmd = append(ZFSVolCmd, "-o", compressionProperty)
+	}
+	if len(vol.Spec.Encryption) != 0 {
+		encryptionProperty := "encryption=" + vol.Spec.Encryption
+		ZFSVolCmd = append(ZFSVolCmd, "-o", encryptionProperty)
+	}
+	if len(vol.Spec.KeyLocation) != 0 {
+		keyLocation := "keylocation=" + vol.Spec.KeyLocation
+		ZFSVolCmd = append(ZFSVolCmd, "-o", keyLocation)
+	}
+	if len(vol.Spec.KeyFormat) != 0 {
+		keyFormat := "keyformat=" + vol.Spec.KeyFormat
+		ZFSVolCmd = append(ZFSVolCmd, "-o", keyFormat)
+	}
+
+	ZFSVolCmd = append(ZFSVolCmd, zvol)
+
+	return ZFSVolCmd
+}
+
+// builldVolumeSetArgs returns zvol set command along with attributes as a string array
+// TODO(pawan) need to find a way to identify which property has changed
+func buildVolumeSetArgs(vol *apis.ZFSVolume) []string {
+	var ZFSVolCmd []string
+
+	zvol := vol.Spec.PoolName + "/" + vol.Name
+
+	ZFSVolCmd = append(ZFSVolCmd, ZFSSetArg)
+
+	if len(vol.Spec.Capacity) != 0 {
+		volsize := "volsize=" + vol.Spec.Capacity
+		ZFSVolCmd = append(ZFSVolCmd, volsize)
+	}
+	if len(vol.Spec.Dedup) != 0 {
+		dedupProperty := "dedup=" + vol.Spec.Dedup
+		ZFSVolCmd = append(ZFSVolCmd, dedupProperty)
+	}
+	if len(vol.Spec.Compression) != 0 {
+		compressionProperty := "compression=" + vol.Spec.Compression
+		ZFSVolCmd = append(ZFSVolCmd, compressionProperty)
+	}
+
+	ZFSVolCmd = append(ZFSVolCmd, zvol)
+
+	return ZFSVolCmd
+}
+
+// builldVolumeDestroyArgs returns zvol destroy command along with attributes as a string array
+func buildVolumeDestroyArgs(vol *apis.ZFSVolume) []string {
+	var ZFSVolCmd []string
+
+	zvol := vol.Spec.PoolName + "/" + vol.Name
+
+	ZFSVolCmd = append(ZFSVolCmd, ZFSDestroyArg, "-R", zvol)
+
+	return ZFSVolCmd
+}
+
 // createZvol creates the zvol and returns the corresponding diskPath
 // of the volume which gets created on the node
 func createZvol(vol *apis.ZFSVolume) (string, error) {
-	var out []byte
 	zvol := vol.Spec.PoolName + "/" + vol.Name
 	devicePath := ZFS_DEVPATH + zvol
 
 	if _, err := os.Stat(devicePath); os.IsNotExist(err) {
-		if vol.Spec.ThinProvision == "yes" {
+
-			out, err = mount.NewOsExec().Run(
+		args := buildVolumeCreateArgs(vol)
-				"zfs", "create",
+		cmd := exec.Command(ZFSVolCmd, args...)
-				"-s",
+		out, err := cmd.CombinedOutput()
-				"-V", vol.Spec.Capacity,
-				"-b", vol.Spec.BlockSize,
-				"-o", "compression="+vol.Spec.Compression,
-				"-o", "dedup="+vol.Spec.Dedup,
-				zvol,
-			)
-		} else {
-			out, err = mount.NewOsExec().Run(
-				"zfs", "create",
-				"-V", vol.Spec.Capacity,
-				"-b", vol.Spec.BlockSize,
-				"-o", "compression="+vol.Spec.Compression,
-				"-o", "dedup="+vol.Spec.Dedup,
-				zvol,
-			)
-		}
 
 		if err != nil {
 			logrus.Errorf(
-				"zfs: could not create zvol %v vol %v error: %s", zvol, vol, string(out),
+				"zfs: could not create zvol %v cmd %v error: %s", zvol, args, string(out),
 			)
 			return "", err
 		}
@@ -81,24 +149,18 @@ func createZvol(vol *apis.ZFSVolume) (string, error) {
 
 // SetZvolProp sets the zvol property
 func SetZvolProp(vol *apis.ZFSVolume) error {
-	var out []byte
 	var err error
 	zvol := vol.Spec.PoolName + "/" + vol.Name
 	devicePath := ZFS_DEVPATH + zvol
 
 	if _, err = os.Stat(devicePath); err == nil {
-		// TODO(pawan) need to find a way to identify
+		args := buildVolumeSetArgs(vol)
-		// which property has changed
+		cmd := exec.Command(ZFSVolCmd, args...)
-		out, err = mount.NewOsExec().Run(
+		out, err := cmd.CombinedOutput()
-			"zfs", "set",
+
-			"volsize="+vol.Spec.Capacity,
-			"compression="+vol.Spec.Compression,
-			"dedup="+vol.Spec.Dedup,
-			zvol,
-		)
 		if err != nil {
 			logrus.Errorf(
-				"zfs: could not set property on zvol %v vol %v error: %s", zvol, vol, string(out),
+				"zfs: could not set property on zvol %v cmd %v error: %s", zvol, args, string(out),
 			)
 			return err
 		}
@@ -110,19 +172,17 @@ func SetZvolProp(vol *apis.ZFSVolume) error {
 
 // DestroyZvol deletes the zvol
 func DestroyZvol(vol *apis.ZFSVolume) error {
-	var out []byte
 	zvol := vol.Spec.PoolName + "/" + vol.Name
 	devicePath := ZFS_DEVPATH + zvol
 
 	if _, err := os.Stat(devicePath); err == nil {
-		out, err = mount.NewOsExec().Run(
+		args := buildVolumeDestroyArgs(vol)
-			"zfs", "destroy",
+		cmd := exec.Command(ZFSVolCmd, args...)
-			"-R",
+		out, err := cmd.CombinedOutput()
-			zvol,
+
-		)
 		if err != nil {
 			logrus.Errorf(
-				"zfs: could not destroy zvol %v vol %v error: %s", zvol, vol, string(out),
+				"zfs: could not destroy zvol %v cmd %v error: %s", zvol, args, string(out),
 			)
 			return err
 		}