diff --git a/deploy/sample/fio-block.yaml b/deploy/sample/fio-block.yaml new file mode 100644 index 0000000..403aafe --- /dev/null +++ b/deploy/sample/fio-block.yaml @@ -0,0 +1,50 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: zfspv-block +allowVolumeExpansion: true +parameters: + poolname: "zfspv-pool" +provisioner: zfs.csi.openebs.io +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: block-claim +spec: + volumeMode: Block + storageClassName: zfspv-block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fiob +spec: + replicas: 1 + selector: + matchLabels: + name: fiob + template: + metadata: + labels: + name: fiob + spec: + containers: + - resources: + name: perfrunner + image: openebs/tests-fio + imagePullPolicy: IfNotPresent + command: ["/bin/bash"] + args: ["-c", "while true ;do sleep 50; done"] + volumeDevices: + - devicePath: /dev/xvda + name: storage + volumes: + - name: storage + persistentVolumeClaim: + claimName: block-claim diff --git a/deploy/yamls/zfs-driver.yaml b/deploy/yamls/zfs-driver.yaml index 6a189d2..d5608a5 100644 --- a/deploy/yamls/zfs-driver.yaml +++ b/deploy/yamls/zfs-driver.yaml @@ -791,7 +791,7 @@ spec: - name: libnvpair mountPath: /lib/libnvpair.so.1 - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods + mountPath: /var/lib/kubelet/ # needed so that any mounts setup inside this container are # propagated back to the host machine. mountPropagation: "Bidirectional" @@ -838,6 +838,6 @@ spec: type: DirectoryOrCreate - name: pods-mount-dir hostPath: - path: /var/lib/kubelet/pods + path: /var/lib/kubelet/ type: Directory --- diff --git a/deploy/zfs-operator.yaml b/deploy/zfs-operator.yaml index 954143b..b202f3f 100644 --- a/deploy/zfs-operator.yaml +++ b/deploy/zfs-operator.yaml @@ -1248,7 +1248,7 @@ spec: - name: libnvpair mountPath: /lib/libnvpair.so.1 - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods + mountPath: /var/lib/kubelet/ # needed so that any mounts setup inside this container are # propagated back to the host machine. mountPropagation: "Bidirectional" @@ -1295,6 +1295,6 @@ spec: type: DirectoryOrCreate - name: pods-mount-dir hostPath: - path: /var/lib/kubelet/pods + path: /var/lib/kubelet/ type: Directory --- diff --git a/pkg/driver/agent.go b/pkg/driver/agent.go index b6e9c2c..d5e9850 100644 --- a/pkg/driver/agent.go +++ b/pkg/driver/agent.go @@ -110,14 +110,18 @@ func (ns *node) NodePublishVolume( vol, mountInfo, err := GetVolAndMountInfo(req) if err != nil { - goto PublishVolumeResponse + return nil, status.Error(codes.Internal, err.Error()) } - // attempt mount operation on the requested path - if err = zfs.MountVolume(vol, mountInfo); err != nil { - goto PublishVolumeResponse + // If the access type is block, do nothing for stage + switch req.GetVolumeCapability().GetAccessType().(type) { + case *csi.VolumeCapability_Block: + // attempt block mount operation on the requested path + err = zfs.MountBlock(vol, mountInfo) + case *csi.VolumeCapability_Mount: + // attempt filesystem mount operation on the requested path + err = zfs.MountFilesystem(vol, mountInfo) } -PublishVolumeResponse: if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/zfs/mount.go b/pkg/zfs/mount.go index a9b8c88..19a543a 100644 --- a/pkg/zfs/mount.go +++ b/pkg/zfs/mount.go @@ -208,7 +208,7 @@ func MountDataset(vol *apis.ZFSVolume, mount *apis.MountInfo) error { } // MountVolume mounts the disk to the specified path -func MountVolume(vol *apis.ZFSVolume, mount *apis.MountInfo) error { +func MountFilesystem(vol *apis.ZFSVolume, mount *apis.MountInfo) error { switch vol.Spec.VolumeType { case VOLTYPE_DATASET: return MountDataset(vol, mount) @@ -216,3 +216,29 @@ func MountVolume(vol *apis.ZFSVolume, mount *apis.MountInfo) error { return MountZvol(vol, mount) } } + +func MountBlock(vol *apis.ZFSVolume, mountinfo *apis.MountInfo) error { + target := mountinfo.MountPath + devicePath := ZFS_DEVPATH + vol.Spec.PoolName + "/" + vol.Name + mountopt := []string{"bind"} + + mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()} + + // Create the mount point as a file since bind mount device node requires it to be a file + err := mounter.MakeFile(target) + if err != nil { + return status.Errorf(codes.Internal, "Could not create target file %q: %v", target, err) + } + + // do the bind mount of the zvol device at the target path + if err := mounter.Mount(devicePath, target, "", mountopt); err != nil { + if removeErr := os.Remove(target); removeErr != nil { + return status.Errorf(codes.Internal, "Could not remove mount target %q: %v", target, removeErr) + } + return status.Errorf(codes.Internal, "mount failed at %v err : %v", target, err) + } + + logrus.Infof("NodePublishVolume mounted block device %s at %s", devicePath, target) + + return nil +} diff --git a/tests/container/container.go b/tests/container/container.go index f707287..106b282 100644 --- a/tests/container/container.go +++ b/tests/container/container.go @@ -254,6 +254,29 @@ func (b *Builder) WithVolumeMountsNew(volumeMounts []corev1.VolumeMount) *Builde return b } +// WithVolumeDevicesNew sets the command arguments of the container +func (b *Builder) WithVolumeDevicesNew(volumeDevices []corev1.VolumeDevice) *Builder { + if volumeDevices == nil { + b.errors = append( + b.errors, + errors.New("failed to build container object: nil volumeDevices"), + ) + return b + } + + if len(volumeDevices) == 0 { + b.errors = append( + b.errors, + errors.New("failed to build container object: missing volumeDevices"), + ) + return b + } + newvolumeDevices := []corev1.VolumeDevice{} + newvolumeDevices = append(newvolumeDevices, volumeDevices...) + b.con.VolumeDevices = newvolumeDevices + return b +} + // WithImagePullPolicy sets the image pull policy of the container func (b *Builder) WithImagePullPolicy(policy corev1.PullPolicy) *Builder { if len(policy) == 0 { diff --git a/tests/provision_test.go b/tests/provision_test.go index 0f79c28..04c98dc 100644 --- a/tests/provision_test.go +++ b/tests/provision_test.go @@ -57,7 +57,20 @@ func zvolCreationTest() { By("Deleting storage class", deleteStorageClass) } +func blockVolCreationTest() { + By("Creating default storage class", createStorageClass) + By("creating and verifying PVC bound status", createAndVerifyBlockPVC) + + By("Creating and deploying app pod", createDeployVerifyBlockApp) + By("verifying ZFSVolume object", VerifyZFSVolume) + By("verifying ZFSVolume property change", VerifyZFSVolumePropEdit) + By("Deleting application deployment", deleteAppDeployment) + By("Deleting pvc", deletePVC) + By("Deleting storage class", deleteStorageClass) +} + func volumeCreationTest() { By("Running dataset creation test", datasetCreationTest) By("Running zvol creation test", zvolCreationTest) + By("Running block volume creation test", blockVolCreationTest) } diff --git a/tests/pvc/build.go b/tests/pvc/build.go index 1d4cee0..5e80702 100644 --- a/tests/pvc/build.go +++ b/tests/pvc/build.go @@ -179,3 +179,13 @@ func (b *Builder) Build() (*corev1.PersistentVolumeClaim, error) { } return b.pvc.object, nil } + +// WithVolumeMode sets the VolumeMode field in PVC with provided arguments +func (b *Builder) WithVolumeMode(volumemode *corev1.PersistentVolumeMode) *Builder { + if volumemode == nil { + b.errs = append(b.errs, errors.New("failed to build PVC object: missing volumemode")) + return b + } + b.pvc.object.Spec.VolumeMode = volumemode + return b +} diff --git a/tests/utils.go b/tests/utils.go index 8fc110d..7281230 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -127,6 +127,27 @@ func createExt4StorageClass() { Expect(err).To(BeNil(), "while creating a ext4 storageclass {%s}", scName) } +func createStorageClass() { + var ( + err error + ) + + parameters := map[string]string{ + "poolname": POOLNAME, + } + + By("building a default storage class") + scObj, err = sc.NewBuilder(). + WithGenerateName(scName). + WithParametersNew(parameters). + WithProvisioner(ZFSProvisioner).Build() + Expect(err).ShouldNot(HaveOccurred(), + "while building default storageclass obj with prefix {%s}", scName) + + scObj, err = SCClient.Create(scObj) + Expect(err).To(BeNil(), "while creating a default storageclass {%s}", scName) +} + func createZfsStorageClass() { var ( err error @@ -322,6 +343,53 @@ func createAndVerifyPVC() { ) } +func createAndVerifyBlockPVC() { + var ( + err error + pvcName = "zfspv-pvc" + ) + + volmode := corev1.PersistentVolumeBlock + + By("building a pvc") + pvcObj, err = pvc.NewBuilder(). + WithName(pvcName). + WithNamespace(OpenEBSNamespace). + WithStorageClass(scObj.Name). + WithAccessModes(accessModes). + WithVolumeMode(&volmode). + WithCapacity(capacity).Build() + Expect(err).ShouldNot( + HaveOccurred(), + "while building pvc {%s} in namespace {%s}", + pvcName, + OpenEBSNamespace, + ) + + By("creating above pvc") + pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Create(pvcObj) + Expect(err).To( + BeNil(), + "while creating pvc {%s} in namespace {%s}", + pvcName, + OpenEBSNamespace, + ) + + By("verifying pvc status as bound") + + status := IsPVCBoundEventually(pvcName) + Expect(status).To(Equal(true), + "while checking status equal to bound") + + pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcObj.Name, metav1.GetOptions{}) + Expect(err).To( + BeNil(), + "while retrieving pvc {%s} in namespace {%s}", + pvcName, + OpenEBSNamespace, + ) +} + func resizeAndVerifyPVC() { var ( err error @@ -427,6 +495,75 @@ func createAndDeployAppPod() { ) } +func createAndDeployBlockAppPod() { + var err error + By("building a busybox app pod deployment using above zfs volume") + deployObj, err = deploy.NewBuilder(). + WithName(appName). + WithNamespace(OpenEBSNamespace). + WithLabelsNew( + map[string]string{ + "app": "busybox", + }, + ). + WithSelectorMatchLabelsNew( + map[string]string{ + "app": "busybox", + }, + ). + WithPodTemplateSpecBuilder( + pts.NewBuilder(). + WithLabelsNew( + map[string]string{ + "app": "busybox", + }, + ). + WithContainerBuilders( + container.NewBuilder(). + WithImage("busybox"). + WithName("busybox"). + WithImagePullPolicy(corev1.PullIfNotPresent). + WithCommandNew( + []string{ + "sh", + "-c", + "date > /mnt/datadir/date.txt; sync; sleep 5; sync; tail -f /dev/null;", + }, + ). + WithVolumeDevicesNew( + []corev1.VolumeDevice{ + corev1.VolumeDevice{ + Name: "datavol1", + DevicePath: "/dev/xvda", + }, + }, + ), + ). + WithVolumeBuilders( + k8svolume.NewBuilder(). + WithName("datavol1"). + WithPVCSource(pvcObj.Name), + ), + ). + Build() + + Expect(err).ShouldNot(HaveOccurred(), "while building app deployement {%s}", appName) + + deployObj, err = DeployClient.WithNamespace(OpenEBSNamespace).Create(deployObj) + Expect(err).ShouldNot( + HaveOccurred(), + "while creating pod {%s} in namespace {%s}", + appName, + OpenEBSNamespace, + ) +} + +func createDeployVerifyBlockApp() { + By("creating and deploying app pod", createAndDeployBlockAppPod) + time.Sleep(30 * time.Second) + By("verifying app pod is running", verifyAppPodRunning) +} + func verifyAppPodRunning() { var err error appPod, err = PodClient.WithNamespace(OpenEBSNamespace). diff --git a/unreleased/102-pawanpraka1 b/unreleased/102-pawanpraka1 new file mode 100644 index 0000000..52366da --- /dev/null +++ b/unreleased/102-pawanpraka1 @@ -0,0 +1 @@ +adding RAW Block Volume support for ZFSPV