zfs-localpv/deploy/zfs-operator.yaml

# This manifest deploys the OpenEBS ZFS control plane components, 
# with associated CRs & RBAC rules.

# Create the OpenEBS namespace
apiVersion: v1
kind: Namespace
metadata:
  name: openebs
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: zfsvolumes.openebs.io
spec:
  group: openebs.io
  version: v1alpha1
  scope: Namespaced
  names:
    plural: zfsvolumes
    singular: zfsvolume
    kind: ZFSVolume
    shortNames:
    - zfsvol
    - zv
  additionalPrinterColumns:
  - JSONPath: .spec.poolName
    name: ZPool
    description: ZFS Pool where the volume is created
    type: string
  - JSONPath: .spec.ownerNodeID
    name: Node
    description: Node where the volume is created
    type: string
  - JSONPath: .spec.capacity
    name: Size
    description: Size of the volume
    type: string
  - JSONPath: .spec.volblocksize
    name: volblocksize
    description: volblocksize for the created zvol
    type: string
  - JSONPath: .spec.recordsize
    name: recordsize
    description: recordsize for the created zfs dataset
    type: string
  - JSONPath: .spec.fsType
    name: Filesystem
    description: filesystem created on the volume
    type: string

---
##############################################
###########                       ############
###########   Controller plugin   ############
###########                       ############
##############################################

kind: ServiceAccount
apiVersion: v1
metadata:
  name: openebs-zfs-controller-sa
  namespace: kube-system

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-provisioner-role
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["persistentvolumes", "services"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses", "csinodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: ["*"]
    resources: ["zfsvolumes"]
    verbs: ["*"]
---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-provisioner-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-provisioner-role
  apiGroup: rbac.authorization.k8s.io

---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: openebs-zfs-controller
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: openebs-zfs-controller
      role: openebs-zfs
  serviceName: "openebs-zfs"
  replicas: 1
  template:
    metadata:
      labels:
        app: openebs-zfs-controller
        role: openebs-zfs
    spec:
      priorityClassName: system-cluster-critical
      serviceAccount: openebs-zfs-controller-sa
      containers:
        - name: csi-provisioner
          image: quay.io/k8scsi/csi-provisioner:v1.4.0
          imagePullPolicy: IfNotPresent
          args:
            - "--csi-address=$(ADDRESS)"
            - "--v=5"
            - "--feature-gates=Topology=true"
            - "--strict-topology"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-attacher
          image: quay.io/k8scsi/csi-attacher:v2.0.0
          imagePullPolicy: IfNotPresent
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-cluster-driver-registrar
          image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1
          args:
            - "--v=5"
            - "--driver-requires-attachment=false"
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: openebs-zfs-plugin
          image: quay.io/openebs/zfs-driver:ci
          imagePullPolicy: Always
          env:
            - name: OPENEBS_CONTROLLER_DRIVER
              value: controller
            - name: OPENEBS_CSI_ENDPOINT
              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
            - name: OPENEBS_NAMESPACE
              value: openebs
          args :
            - "--endpoint=$(OPENEBS_CSI_ENDPOINT)"
            - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)"
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
      volumes:
        - name: socket-dir
          emptyDir: {}
---

############################## CSI- Attacher #######################
# Attacher must be able to work with PVs, nodes and VolumeAttachments

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-attacher-role
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["csi.storage.k8s.io"]
    resources: ["csinodeinfos"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments", "csinodes"]
    verbs: ["get", "list", "watch", "update", "patch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-attacher-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-attacher-role
  apiGroup: rbac.authorization.k8s.io

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-snapshotter-role
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents"]
    verbs: ["create", "get", "list", "watch", "update", "delete"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["apiextensions.k8s.io"]
    resources: ["customresourcedefinitions"]
    verbs: ["create", "list", "watch", "delete"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-snapshotter-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-snapshotter-role
  apiGroup: rbac.authorization.k8s.io

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-cluster-driver-registrar-role
rules:
  - apiGroups: ["csi.storage.k8s.io"]
    resources: ["csidrivers"]
    verbs: ["create", "delete"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-cluster-driver-registrar-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-cluster-driver-registrar-role
  apiGroup: rbac.authorization.k8s.io

---

########################################
###########                 ############
###########   Node plugin   ############
###########                 ############
########################################

apiVersion: v1
kind: ServiceAccount
metadata:
  name: openebs-zfs-node-sa
  namespace: kube-system

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-driver-registrar-role
rules:
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumes", "nodes", "services"]
    verbs: ["get", "list"]
  - apiGroups: ["*"]
    resources: ["zfsvolumes"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-driver-registrar-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-node-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-driver-registrar-role
  apiGroup: rbac.authorization.k8s.io

---

kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: openebs-zfs-node
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: openebs-zfs-node
  template:
    metadata:
      labels:
        app: openebs-zfs-node
        role: openebs-zfs
    spec:
      priorityClassName: system-node-critical
      serviceAccount: openebs-zfs-node-sa
      hostNetwork: true
      containers:
        - name: csi-node-driver-registrar
          image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
          lifecycle:
            preStop:
              exec:
                command: ["/bin/sh", "-c", "rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock"]
          env:
            - name: ADDRESS
              value: /plugin/csi.sock
            - name: DRIVER_REG_SOCK_PATH
              value: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: NODE_DRIVER
              value: openebs-zfs
          volumeMounts:
            - name: plugin-dir
              mountPath: /plugin
            - name: registration-dir
              mountPath: /registration
        - name: openebs-zfs-plugin
          securityContext:
            privileged: true
            capabilities:
              add: ["CAP_MKNOD", "CAP_SYS_ADMIN", "SYS_ADMIN"]
            allowPrivilegeEscalation: true
          image: quay.io/openebs/zfs-driver:ci
          imagePullPolicy: Always
          args:
            - "--nodeid=$(OPENEBS_NODE_ID)"
            - "--endpoint=$(OPENEBS_CSI_ENDPOINT)"
            - "--plugin=$(OPENEBS_NODE_DRIVER)"
          env:
            - name: OPENEBS_NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: OPENEBS_CSI_ENDPOINT
              value: unix:///plugin/csi.sock
            - name: OPENEBS_NODE_DRIVER
              value: agent
            - name: OPENEBS_NAMESPACE
              value: openebs
          volumeMounts:
            - name: plugin-dir
              mountPath: /plugin
            - name: device-dir
              mountPath: /dev
            - name: encr-keys
              mountPath: /home/keys
            - name: zfs-bin
              mountPath: /sbin/zfs
            - name: libzpool
              mountPath: /lib/libzpool.so.2
            - name: libzfscore
              mountPath: /lib/libzfs_core.so.1
            - name: libzfs
              mountPath: /lib/libzfs.so.2
            - name: libuutil
              mountPath: /lib/libuutil.so.1
            - name: libnvpair
              mountPath: /lib/libnvpair.so.1
            - name: pods-mount-dir
              mountPath: /var/lib/kubelet/pods
              # needed so that any mounts setup inside this container are
              # propagated back to the host machine.
              mountPropagation: "Bidirectional"
      volumes:
        - name: device-dir
          hostPath:
            path: /dev
            type: Directory
        - name: encr-keys
          hostPath:
            path: /home/keys
            type: DirectoryOrCreate
        - name: zfs-bin
          hostPath:
            path: /sbin/zfs
            type: File
        - name: libzpool
          hostPath:
            path: /lib/libzpool.so.2.0.0
            type: File
        - name: libzfscore
          hostPath:
            path: /lib/libzfs_core.so.1.0.0
            type: File
        - name: libzfs
          hostPath:
            path: /lib/libzfs.so.2.0.0
            type: File
        - name: libuutil
          hostPath:
            path: /lib/libuutil.so.1.0.1
            type: File
        - name: libnvpair
          hostPath:
            path: /lib/libnvpair.so.1.0.1
            type: File
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry/
            type: DirectoryOrCreate
        - name: plugin-dir
          hostPath:
            path: /var/lib/kubelet/plugins/zfs-localpv/
            type: DirectoryOrCreate
        - name: pods-mount-dir
          hostPath:
            path: /var/lib/kubelet/pods
            type: Directory
---