From 86e623a369708354faad21aca1727efdce259cf3 Mon Sep 17 00:00:00 2001 From: Pawan Date: Mon, 2 Mar 2020 12:03:03 +0530 Subject: [PATCH] feat(resize): adding Online volume expansion support for ZFSPV We can resize the volume by updating the PVC yaml to the desired size and apply it. The ZFS Driver will take care of updating the quota in case of dataset. If we are using a Zvol and have mounted it as ext4 or xfs filesystem, the driver will take care of expanding the volume via reize2fs/xfs_growfs binaries. For resize, storageclass that provisions the pvc must suppo rt resize. We should have allowVolumeExpansion as true in storageclass ```yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: openebs-zfspv allowVolumeExpansion: true parameters: poolname: "zfspv-pool" provisioner: zfs.csi.openebs.io ``` Signed-off-by: Pawan --- deploy/zfs-operator.yaml | 18 ++++++++- pkg/driver/agent.go | 31 ++++++++++++++- pkg/driver/controller.go | 46 +++++++++++++++++++++- pkg/response/resize.go | 54 ++++++++++++++++++++++++++ pkg/zfs/resize.go | 83 ++++++++++++++++++++++++++++++++++++++++ pkg/zfs/volume.go | 10 +++++ pkg/zfs/zfs_util.go | 39 +++++++++++++++++++ 7 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 pkg/response/resize.go create mode 100644 pkg/zfs/resize.go diff --git a/deploy/zfs-operator.yaml b/deploy/zfs-operator.yaml index ffd8013..60381e7 100644 --- a/deploy/zfs-operator.yaml +++ b/deploy/zfs-operator.yaml @@ -517,10 +517,13 @@ rules: verbs: ["*"] - apiGroups: [""] resources: ["persistentvolumes", "services"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses", "csinodes"] verbs: ["get", "list", "watch"] @@ -580,6 +583,19 @@ spec: priorityClassName: system-cluster-critical serviceAccount: openebs-zfs-controller-sa containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.4.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-snapshotter image: quay.io/k8scsi/csi-snapshotter:v2.0.1 imagePullPolicy: IfNotPresent diff --git a/pkg/driver/agent.go b/pkg/driver/agent.go index 71271d8..b1874a7 100644 --- a/pkg/driver/agent.go +++ b/pkg/driver/agent.go @@ -220,6 +220,13 @@ func (ns *node) NodeGetCapabilities( }, }, }, + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_EXPAND_VOLUME, + }, + }, + }, }, }, nil } @@ -267,7 +274,29 @@ func (ns *node) NodeExpandVolume( req *csi.NodeExpandVolumeRequest, ) (*csi.NodeExpandVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") + volumeID := req.GetVolumeId() + vol, err := zfs.GetZFSVolume(volumeID) + + if err != nil { + return nil, status.Errorf( + codes.Internal, + "failed to handle NodeExpandVolume Request for %s, {%s}", + req.VolumeId, + err.Error(), + ) + } + if err = zfs.ResizeZFSVolume(vol, req.GetVolumePath()); err != nil { + return nil, status.Errorf( + codes.Internal, + "failed to handle NodeExpandVolume Request for %s, {%s}", + req.VolumeId, + err.Error(), + ) + } + + return &csi.NodeExpandVolumeResponse{ + CapacityBytes: req.GetCapacityRange().GetRequiredBytes(), + }, nil } // NodeGetVolumeStats returns statistics for the diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 6e491a0..9dfb919 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -291,7 +291,50 @@ func (cs *controller) ControllerExpandVolume( req *csi.ControllerExpandVolumeRequest, ) (*csi.ControllerExpandVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") + updatedSize := req.GetCapacityRange().GetRequiredBytes() + + vol, err := zfs.GetZFSVolume(req.VolumeId) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "ControllerExpandVolumeRequest: failed to get ZFSVolume in for %s, {%s}", + req.VolumeId, + err.Error(), + ) + } + + volsize, err := strconv.ParseInt(vol.Spec.Capacity, 10, 64) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "ControllerExpandVolumeRequest: failed to parse volsize in for %s, {%s}", + req.VolumeId, + err.Error(), + ) + } + /* + * Controller expand volume must be idempotent. If a volume corresponding + * to the specified volume ID is already larger than or equal to the target + * capacity of the expansion request, the plugin should reply 0 OK. + */ + if volsize >= updatedSize { + return csipayload.NewControllerExpandVolumeResponseBuilder(). + WithCapacityBytes(volsize). + Build(), nil + } + + if err := zfs.ResizeVolume(vol, updatedSize); err != nil { + return nil, status.Errorf( + codes.Internal, + "failed to handle ControllerExpandVolumeRequest for %s, {%s}", + req.VolumeId, + err.Error(), + ) + } + return csipayload.NewControllerExpandVolumeResponseBuilder(). + WithCapacityBytes(updatedSize). + WithNodeExpansionRequired(true). + Build(), nil } // CreateSnapshot creates a snapshot for given volume @@ -521,6 +564,7 @@ func newControllerCapabilities() []*csi.ControllerServiceCapability { csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, + csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, } { capabilities = append(capabilities, fromType(cap)) } diff --git a/pkg/response/resize.go b/pkg/response/resize.go new file mode 100644 index 0000000..2423899 --- /dev/null +++ b/pkg/response/resize.go @@ -0,0 +1,54 @@ +/* +Copyright 2020 The OpenEBS Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "github.com/container-storage-interface/spec/lib/go/csi" +) + +// ControllerExpandVolumeResponseBuilder helps building an +// instance of csi ControllerExpandVolumeResponse +type ControllerExpandVolumeResponseBuilder struct { + response *csi.ControllerExpandVolumeResponse +} + +// NewControllerExpandVolumeResponseBuilder returns a new +// instance of ControllerExpandVolumeResponse +func NewControllerExpandVolumeResponseBuilder() *ControllerExpandVolumeResponseBuilder { + return &ControllerExpandVolumeResponseBuilder{ + response: &csi.ControllerExpandVolumeResponse{}, + } +} + +// WithCapacityBytes sets the CapacityBytes against the +// ControllerExpandVolumeResponse instance +func (b *ControllerExpandVolumeResponseBuilder) WithCapacityBytes( + capacity int64) *ControllerExpandVolumeResponseBuilder { + b.response.CapacityBytes = capacity + return b +} + +// WithNodeExpansionRequired sets the NodeExpansionRequired against the +// ControllerExpandVolumeResponse instance +func (b *ControllerExpandVolumeResponseBuilder) WithNodeExpansionRequired( + nodeExpansionRequired bool) *ControllerExpandVolumeResponseBuilder { + b.response.NodeExpansionRequired = nodeExpansionRequired + return b +} + +// Build returns the constructed instance +// of csi ControllerExpandVolumeResponse +func (b *ControllerExpandVolumeResponseBuilder) Build() *csi.ControllerExpandVolumeResponse { + return b.response +} diff --git a/pkg/zfs/resize.go b/pkg/zfs/resize.go new file mode 100644 index 0000000..4421840 --- /dev/null +++ b/pkg/zfs/resize.go @@ -0,0 +1,83 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package zfs + +import ( + "github.com/Sirupsen/logrus" + apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1" + "k8s.io/kubernetes/pkg/util/mount" + "os/exec" +) + +// ResizeExtn can be used to run a resize command on the ext2/3/4 filesystem +// to expand the filesystem to the actual size of the device +func ResizeExtn(devpath string) error { + cmd := exec.Command("resize2fs", devpath) + out, err := cmd.CombinedOutput() + if err != nil { + logrus.Errorf("zfspv: ResizeExtn failed error: %s", string(out)) + return err + } + return nil +} + +// ResizeXFS can be used to run a resize command on the xfs filesystem +// to expand the filesystem to the actual size of the device +func ResizeXFS(path string) error { + cmd := exec.Command("xfs_growfs", path) + out, err := cmd.CombinedOutput() + if err != nil { + logrus.Errorf("zfspv: ResizeXFS failed error: %s", string(out)) + return err + } + return nil +} + +// handleVolResize resizes the filesystem, it is called after quota +// has been set on the volume. It takes care of expanding the filesystem. +func handleVolResize(vol *apis.ZFSVolume, volumePath string) error { + var err error + + devpath, err := GetVolumeDevPath(vol) + if err != nil { + return err + } + + fsType := vol.Spec.FsType + + mounter := mount.New("") + list, _ := mounter.List() + for _, mpt := range list { + if mpt.Path == volumePath { + switch fsType { + case "xfs": + err = ResizeXFS(volumePath) + case "zfs": + // just setting the quota is suffcient + // nothing to handle here + err = nil + default: + err = ResizeExtn(devpath) + } + if err != nil { + return err + } + break + } + } + return nil +} diff --git a/pkg/zfs/volume.go b/pkg/zfs/volume.go index 70aa6d5..da02f0b 100644 --- a/pkg/zfs/volume.go +++ b/pkg/zfs/volume.go @@ -17,6 +17,7 @@ package zfs import ( "github.com/Sirupsen/logrus" "os" + "strconv" apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1" "github.com/openebs/zfs-localpv/pkg/builder/snapbuilder" @@ -85,6 +86,15 @@ func ProvisionVolume( return err } +// ResizeVolume resizes the zfs volume +func ResizeVolume(vol *apis.ZFSVolume, newSize int64) error { + + vol.Spec.Capacity = strconv.FormatInt(int64(newSize), 10) + + _, err := volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Update(vol) + return err +} + // ProvisionSnapshot creates a ZFSSnapshot CR, // watcher for zvc is present in CSI agent func ProvisionSnapshot( diff --git a/pkg/zfs/zfs_util.go b/pkg/zfs/zfs_util.go index fa0096e..3acad20 100644 --- a/pkg/zfs/zfs_util.go +++ b/pkg/zfs/zfs_util.go @@ -236,6 +236,27 @@ func buildVolumeSetArgs(vol *apis.ZFSVolume) []string { return ZFSVolArg } +// builldVolumeResizeArgs returns volume set for resizing the zfs volume +func buildVolumeResizeArgs(vol *apis.ZFSVolume) []string { + var ZFSVolArg []string + + volume := vol.Spec.PoolName + "/" + vol.Name + + ZFSVolArg = append(ZFSVolArg, ZFSSetArg) + + if vol.Spec.VolumeType == VOLTYPE_DATASET { + quotaProperty := "quota=" + vol.Spec.Capacity + ZFSVolArg = append(ZFSVolArg, quotaProperty) + } else { + volsizeProperty := "volsize=" + vol.Spec.Capacity + ZFSVolArg = append(ZFSVolArg, volsizeProperty) + } + + ZFSVolArg = append(ZFSVolArg, volume) + + return ZFSVolArg +} + // builldVolumeDestroyArgs returns volume destroy command along with attributes as a string array func buildVolumeDestroyArgs(vol *apis.ZFSVolume) []string { var ZFSVolArg []string @@ -496,3 +517,21 @@ func GetVolumeDevPath(vol *apis.ZFSVolume) (string, error) { return dev, nil } + +func ResizeZFSVolume(vol *apis.ZFSVolume, mountpath string) error { + + volume := vol.Spec.PoolName + "/" + vol.Name + args := buildVolumeResizeArgs(vol) + cmd := exec.Command(ZFSVolCmd, args...) + out, err := cmd.CombinedOutput() + + if err != nil { + logrus.Errorf( + "zfs: could not resize the volume %v cmd %v error: %s", volume, args, string(out), + ) + return err + } + + err = handleVolResize(vol, mountpath) + return err +}