From 0218dacea0bca37e31a942f5e47df5390ecbcf33 Mon Sep 17 00:00:00 2001 From: Pawan Prakash Sharma Date: Tue, 15 Oct 2019 22:51:48 +0530 Subject: [PATCH] feat(ZFSPV): adding encryption in ZFSVolume CR (#6) Adding support for enabling encryption using a custom key. Also, adding support to inherit the properties from ZPOOL which are not listed in the storage class, ZFS driver will not pass default values while creating the volume. Those properties will be inherited from the ZPOOL. we can use the encryption option in storage class ``` apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: openebs-zfspv allowVolumeExpansion: true parameters: blocksize: "4k" compression: "on" dedup: "on" thinprovision: "yes" encryption: "on" keyformat: "raw" keylocation: "file:///home/keys/key" poolname: "zfspv-pool" provisioner: openebs.io/zfs ``` Just a note, the key file should be mounted inside the node-agent container so that we can use that file while provisioning the volume. keyformat can be raw, hex or passphrase. Signed-off-by: Pawan --- deploy/sample/fio.yaml | 3 + deploy/sample/percona.yaml | 131 ++++++++++++++++ deploy/zfs-operator.yaml | 6 + .../openebs.io/core/v1alpha1/zfsvolume.go | 14 +- pkg/builder/build.go | 52 ++++--- pkg/driver/controller.go | 6 + pkg/zfs/zfs_util.go | 144 +++++++++++++----- 7 files changed, 288 insertions(+), 68 deletions(-) create mode 100644 deploy/sample/percona.yaml diff --git a/deploy/sample/fio.yaml b/deploy/sample/fio.yaml index 041273b..49f5a72 100644 --- a/deploy/sample/fio.yaml +++ b/deploy/sample/fio.yaml @@ -8,6 +8,9 @@ parameters: compression: "on" dedup: "on" thinprovision: "yes" + #encryption: "on" + #keyformat: "raw" + #keylocation: "file:///home/pawan/key" poolname: "zfspv-pool" provisioner: openebs.io/zfs volumeBindingMode: WaitForFirstConsumer diff --git a/deploy/sample/percona.yaml b/deploy/sample/percona.yaml new file mode 100644 index 0000000..104b0eb --- /dev/null +++ b/deploy/sample/percona.yaml @@ -0,0 +1,131 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: openebs-zfspv +allowVolumeExpansion: true +parameters: + blocksize: "4k" + compression: "on" + dedup: "on" + thinprovision: "yes" + poolname: "zfspv-pool" +provisioner: openebs.io/zfs +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: csi-zfspv +spec: + storageClassName: openebs-zfspv + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + name: sqltest + namespace: default +data: + sql-test.sh: | + #!/bin/bash + + DB_PREFIX="Inventory" + DB_SUFFIX=`echo $(mktemp) | cut -d '.' -f 2` + DB_NAME="${DB_PREFIX}_${DB_SUFFIX}" + + + echo -e "\nWaiting for mysql server to start accepting connections.." + retries=10;wait_retry=30 + for i in `seq 1 $retries`; do + mysql -uroot -pk8sDem0 -e 'status' > /dev/null 2>&1 + rc=$? + [ $rc -eq 0 ] && break + sleep $wait_retry + done + + if [ $rc -ne 0 ]; + then + echo -e "\nFailed to connect to db server after trying for $(($retries * $wait_retry))s, exiting\n" + exit 1 + fi + mysql -uroot -pk8sDem0 -e "CREATE DATABASE $DB_NAME;" + mysql -uroot -pk8sDem0 -e "CREATE TABLE Hardware (id INTEGER, name VARCHAR(20), owner VARCHAR(20),description VARCHAR(20));" $DB_NAME + mysql -uroot -pk8sDem0 -e "INSERT INTO Hardware (id, name, owner, description) values (1, "dellserver", "basavaraj", "controller");" $DB_NAME + mysql -uroot -pk8sDem0 -e "DROP DATABASE $DB_NAME;" +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: percona + labels: + name: percona +spec: + replicas: 1 + selector: + matchLabels: + name: percona + template: + metadata: + labels: + name: percona + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - gke-pawan-zfspv-default-pool-26f2b9a9-5fqd + containers: + - resources: + name: percona + image: openebs/tests-custom-percona:latest + imagePullPolicy: IfNotPresent + args: + - "--ignore-db-dir" + - "lost+found" + env: + - name: MYSQL_ROOT_PASSWORD + value: k8sDem0 + ports: + - containerPort: 3306 + name: percona + volumeMounts: + - mountPath: /var/lib/mysql + name: demo-vol1 + - mountPath: /sql-test.sh + subPath: sql-test.sh + name: sqltest-configmap + livenessProbe: + exec: + command: ["bash", "sql-test.sh"] + initialDelaySeconds: 30 + periodSeconds: 1 + timeoutSeconds: 10 + volumes: + - name: demo-vol1 + persistentVolumeClaim: + claimName: csi-zfspv + - name: sqltest-configmap + configMap: + name: sqltest + +--- +apiVersion: v1 +kind: Service +metadata: + name: percona-mysql + labels: + name: percona-mysql +spec: + ports: + - port: 3306 + targetPort: 3306 + selector: + name: percona diff --git a/deploy/zfs-operator.yaml b/deploy/zfs-operator.yaml index 86fc5fa..a86c7cb 100644 --- a/deploy/zfs-operator.yaml +++ b/deploy/zfs-operator.yaml @@ -396,6 +396,8 @@ spec: mountPath: /plugin - name: device-dir mountPath: /dev + - name: encr-keys + mountPath: /home/keys - name: zfs-bin mountPath: /sbin/zfs - name: libzpool @@ -418,6 +420,10 @@ spec: hostPath: path: /dev type: Directory + - name: encr-keys + hostPath: + path: /home/keys + type: DirectoryOrCreate - name: zfs-bin hostPath: path: /sbin/zfs diff --git a/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go b/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go index 337872f..b70630d 100644 --- a/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go +++ b/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go @@ -91,9 +91,21 @@ type VolumeInfo struct { Compression string `json:"compression"` // Dedup specifies the deduplication - // should be enabledd on the zvol + // should be enabled on the zvol Dedup string `json:"dedup"` + // Encryption specifies the encryption + // should be enabled on the zvol + Encryption string `json:"encryption"` + + // KeyLocation is the location of key + // for the encryption + KeyLocation string `json:"keylocation"` + + // KeyFormat specifies format of the + // encryption key + KeyFormat string `json:"keyformat"` + // Thinprovision specifies if we should // thin provisioned the volume or not ThinProvision string `json:"thinProvison"` diff --git a/pkg/builder/build.go b/pkg/builder/build.go index 506a2f8..e47151c 100644 --- a/pkg/builder/build.go +++ b/pkg/builder/build.go @@ -54,7 +54,7 @@ func BuildFrom(volume *apis.ZFSVolume) *Builder { } } -// WithNamespace sets the namespace of csi volume +// WithNamespace sets the namespace of ZFSVolume func (b *Builder) WithNamespace(namespace string) *Builder { if namespace == "" { b.errs = append( @@ -69,7 +69,7 @@ func (b *Builder) WithNamespace(namespace string) *Builder { return b } -// WithName sets the name of csi volume +// WithName sets the name of ZFSVolume func (b *Builder) WithName(name string) *Builder { if name == "" { b.errs = append( @@ -100,42 +100,44 @@ func (b *Builder) WithCapacity(capacity string) *Builder { return b } -// WithCompression sets compression of CStorVolumeClaim +// WithEncryption sets the encryption on ZFSVolume +func (b *Builder) WithEncryption(encr string) *Builder { + b.volume.Object.Spec.Encryption = encr + return b +} + +// WithKeyLocation sets the encryption key location on ZFSVolume +func (b *Builder) WithKeyLocation(kl string) *Builder { + b.volume.Object.Spec.KeyLocation = kl + return b +} + +// WithKeyFormat sets the encryption key format on ZFSVolume +func (b *Builder) WithKeyFormat(kf string) *Builder { + b.volume.Object.Spec.KeyFormat = kf + return b +} + +// WithCompression sets compression of ZFSVolume func (b *Builder) WithCompression(compression string) *Builder { - - comp := "off" - if compression == "on" { - comp = "on" - } - b.volume.Object.Spec.Compression = comp + b.volume.Object.Spec.Compression = compression return b } -// WithDedup sets compression of CStorVolumeClaim +// WithDedup sets dedup property of ZFSVolume func (b *Builder) WithDedup(dedup string) *Builder { - - dp := "off" - if dedup == "on" { - dp = "on" - } - b.volume.Object.Spec.Dedup = dp + b.volume.Object.Spec.Dedup = dedup return b } -// WithThinProv sets compression of CStorVolumeClaim +// WithThinProv sets if ZFSVolume needs to be thin provisioned func (b *Builder) WithThinProv(thinprov string) *Builder { - - tp := "no" - if thinprov == "yes" { - tp = "yes" - } - b.volume.Object.Spec.ThinProvision = tp + b.volume.Object.Spec.ThinProvision = thinprov return b } -// WithBlockSize sets blocksize of CStorVolumeClaim +// WithBlockSize sets blocksize of ZFSVolume func (b *Builder) WithBlockSize(blockSize string) *Builder { - bs := "4k" if len(blockSize) > 0 { bs = blockSize diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 5f87c26..eaa433b 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -73,6 +73,9 @@ func (cs *controller) CreateVolume( bs := req.GetParameters()["blocksize"] compression := req.GetParameters()["compression"] dedup := req.GetParameters()["dedup"] + encr := req.GetParameters()["encryption"] + kf := req.GetParameters()["keyformat"] + kl := req.GetParameters()["keylocation"] pool := req.GetParameters()["poolname"] tp := req.GetParameters()["thinprovision"] @@ -82,6 +85,9 @@ func (cs *controller) CreateVolume( WithBlockSize(bs). WithPoolName(pool). WithDedup(dedup). + WithEncryption(encr). + WithKeyFormat(kf). + WithKeyLocation(kl). WithThinProv(tp). WithCompression(compression).Build() diff --git a/pkg/zfs/zfs_util.go b/pkg/zfs/zfs_util.go index adff3a0..12f40e8 100644 --- a/pkg/zfs/zfs_util.go +++ b/pkg/zfs/zfs_util.go @@ -18,14 +18,18 @@ package zfs import ( "os" + "os/exec" "github.com/Sirupsen/logrus" apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1" - "k8s.io/kubernetes/pkg/util/mount" ) const ( - ZFS_DEVPATH = "/dev/zvol/" + ZFS_DEVPATH = "/dev/zvol/" + ZFSVolCmd = "zfs" + ZFSCreateArg = "create" + ZFSDestroyArg = "destroy" + ZFSSetArg = "set" ) func PropertyChanged(oldVol *apis.ZFSVolume, newVol *apis.ZFSVolume) bool { @@ -34,38 +38,102 @@ func PropertyChanged(oldVol *apis.ZFSVolume, newVol *apis.ZFSVolume) bool { oldVol.Spec.Capacity != newVol.Spec.Capacity } +// builldVolumeCreateArgs returns zvol create command along with attributes as a string array +func buildVolumeCreateArgs(vol *apis.ZFSVolume) []string { + var ZFSVolCmd []string + + zvol := vol.Spec.PoolName + "/" + vol.Name + + ZFSVolCmd = append(ZFSVolCmd, ZFSCreateArg) + + if vol.Spec.ThinProvision == "yes" { + ZFSVolCmd = append(ZFSVolCmd, "-s") + } + if len(vol.Spec.Capacity) != 0 { + ZFSVolCmd = append(ZFSVolCmd, "-V", vol.Spec.Capacity) + } + if len(vol.Spec.BlockSize) != 0 { + ZFSVolCmd = append(ZFSVolCmd, "-b", vol.Spec.BlockSize) + } + if len(vol.Spec.Dedup) != 0 { + dedupProperty := "dedup=" + vol.Spec.Dedup + ZFSVolCmd = append(ZFSVolCmd, "-o", dedupProperty) + } + if len(vol.Spec.Compression) != 0 { + compressionProperty := "compression=" + vol.Spec.Compression + ZFSVolCmd = append(ZFSVolCmd, "-o", compressionProperty) + } + if len(vol.Spec.Encryption) != 0 { + encryptionProperty := "encryption=" + vol.Spec.Encryption + ZFSVolCmd = append(ZFSVolCmd, "-o", encryptionProperty) + } + if len(vol.Spec.KeyLocation) != 0 { + keyLocation := "keylocation=" + vol.Spec.KeyLocation + ZFSVolCmd = append(ZFSVolCmd, "-o", keyLocation) + } + if len(vol.Spec.KeyFormat) != 0 { + keyFormat := "keyformat=" + vol.Spec.KeyFormat + ZFSVolCmd = append(ZFSVolCmd, "-o", keyFormat) + } + + ZFSVolCmd = append(ZFSVolCmd, zvol) + + return ZFSVolCmd +} + +// builldVolumeSetArgs returns zvol set command along with attributes as a string array +// TODO(pawan) need to find a way to identify which property has changed +func buildVolumeSetArgs(vol *apis.ZFSVolume) []string { + var ZFSVolCmd []string + + zvol := vol.Spec.PoolName + "/" + vol.Name + + ZFSVolCmd = append(ZFSVolCmd, ZFSSetArg) + + if len(vol.Spec.Capacity) != 0 { + volsize := "volsize=" + vol.Spec.Capacity + ZFSVolCmd = append(ZFSVolCmd, volsize) + } + if len(vol.Spec.Dedup) != 0 { + dedupProperty := "dedup=" + vol.Spec.Dedup + ZFSVolCmd = append(ZFSVolCmd, dedupProperty) + } + if len(vol.Spec.Compression) != 0 { + compressionProperty := "compression=" + vol.Spec.Compression + ZFSVolCmd = append(ZFSVolCmd, compressionProperty) + } + + ZFSVolCmd = append(ZFSVolCmd, zvol) + + return ZFSVolCmd +} + +// builldVolumeDestroyArgs returns zvol destroy command along with attributes as a string array +func buildVolumeDestroyArgs(vol *apis.ZFSVolume) []string { + var ZFSVolCmd []string + + zvol := vol.Spec.PoolName + "/" + vol.Name + + ZFSVolCmd = append(ZFSVolCmd, ZFSDestroyArg, "-R", zvol) + + return ZFSVolCmd +} + // createZvol creates the zvol and returns the corresponding diskPath // of the volume which gets created on the node func createZvol(vol *apis.ZFSVolume) (string, error) { - var out []byte zvol := vol.Spec.PoolName + "/" + vol.Name devicePath := ZFS_DEVPATH + zvol if _, err := os.Stat(devicePath); os.IsNotExist(err) { - if vol.Spec.ThinProvision == "yes" { - out, err = mount.NewOsExec().Run( - "zfs", "create", - "-s", - "-V", vol.Spec.Capacity, - "-b", vol.Spec.BlockSize, - "-o", "compression="+vol.Spec.Compression, - "-o", "dedup="+vol.Spec.Dedup, - zvol, - ) - } else { - out, err = mount.NewOsExec().Run( - "zfs", "create", - "-V", vol.Spec.Capacity, - "-b", vol.Spec.BlockSize, - "-o", "compression="+vol.Spec.Compression, - "-o", "dedup="+vol.Spec.Dedup, - zvol, - ) - } + + args := buildVolumeCreateArgs(vol) + cmd := exec.Command(ZFSVolCmd, args...) + out, err := cmd.CombinedOutput() if err != nil { logrus.Errorf( - "zfs: could not create zvol %v vol %v error: %s", zvol, vol, string(out), + "zfs: could not create zvol %v cmd %v error: %s", zvol, args, string(out), ) return "", err } @@ -81,24 +149,18 @@ func createZvol(vol *apis.ZFSVolume) (string, error) { // SetZvolProp sets the zvol property func SetZvolProp(vol *apis.ZFSVolume) error { - var out []byte var err error zvol := vol.Spec.PoolName + "/" + vol.Name devicePath := ZFS_DEVPATH + zvol if _, err = os.Stat(devicePath); err == nil { - // TODO(pawan) need to find a way to identify - // which property has changed - out, err = mount.NewOsExec().Run( - "zfs", "set", - "volsize="+vol.Spec.Capacity, - "compression="+vol.Spec.Compression, - "dedup="+vol.Spec.Dedup, - zvol, - ) + args := buildVolumeSetArgs(vol) + cmd := exec.Command(ZFSVolCmd, args...) + out, err := cmd.CombinedOutput() + if err != nil { logrus.Errorf( - "zfs: could not set property on zvol %v vol %v error: %s", zvol, vol, string(out), + "zfs: could not set property on zvol %v cmd %v error: %s", zvol, args, string(out), ) return err } @@ -110,19 +172,17 @@ func SetZvolProp(vol *apis.ZFSVolume) error { // DestroyZvol deletes the zvol func DestroyZvol(vol *apis.ZFSVolume) error { - var out []byte zvol := vol.Spec.PoolName + "/" + vol.Name devicePath := ZFS_DEVPATH + zvol if _, err := os.Stat(devicePath); err == nil { - out, err = mount.NewOsExec().Run( - "zfs", "destroy", - "-R", - zvol, - ) + args := buildVolumeDestroyArgs(vol) + cmd := exec.Command(ZFSVolCmd, args...) + out, err := cmd.CombinedOutput() + if err != nil { logrus.Errorf( - "zfs: could not destroy zvol %v vol %v error: %s", zvol, vol, string(out), + "zfs: could not destroy zvol %v cmd %v error: %s", zvol, args, string(out), ) return err }