mirror of
https://github.com/TECHNOFAB11/zfs-localpv.git
synced 2025-12-12 14:30:12 +01:00
a10dedbd5e
This is an initial scheduler implementation for ZFS Local PV.
* adding scheduler as a configurable option
* adding volumeWeightedScheduler as scheduling logic
The volumeWeightedScheduler will go through all the nodes as per
topology information and it will pick the node which has less
volume provisioned in the given pool.
lets say there are 2 nodes node1 and node2 with below pool configuration :-
```
node1
|
|-----> pool1
| |
| |------> pvc1
| |------> pvc2
|-----> pool2
|------> pvc3
node2
|
|-----> pool1
| |
| |------> pvc4
|-----> pool2
|------> pvc5
|------> pvc6
```
So if application is using pool1 as shown in the below storage class, then ZFS driver will schedule it on node2 as it has one volume as compared to node1 which has 2 volumes in pool1.
```yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: openebs-zfspv
provisioner: zfs.csi.openebs.io
parameters:
blocksize: "4k"
compression: "on"
dedup: "on"
thinprovision: "yes"
poolname: "pool1"
```
So if application is using pool2 as shown in the below storage class, then ZFS driver will schedule it on node1 as it has one volume only as compared node2 which has 2 volumes in pool2.
```yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: openebs-zfspv
provisioner: zfs.csi.openebs.io
parameters:
blocksize: "4k"
compression: "on"
dedup: "on"
thinprovision: "yes"
poolname: "pool2"
```
In case of same number of volumes on all the nodes for the given pool, it can pick any node and schedule the PV on that.
Signed-off-by: Pawan <pawan@mayadata.io>
467 lines
13 KiB
YAML
467 lines
13 KiB
YAML
# This manifest deploys the OpenEBS ZFS control plane components,
|
|
# with associated CRs & RBAC rules.
|
|
|
|
# Create the OpenEBS namespace
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: openebs
|
|
---
|
|
apiVersion: apiextensions.k8s.io/v1beta1
|
|
kind: CustomResourceDefinition
|
|
metadata:
|
|
name: zfsvolumes.openebs.io
|
|
spec:
|
|
group: openebs.io
|
|
version: v1alpha1
|
|
scope: Namespaced
|
|
names:
|
|
plural: zfsvolumes
|
|
singular: zfsvolume
|
|
kind: ZFSVolume
|
|
shortNames:
|
|
- zvol
|
|
- zv
|
|
additionalPrinterColumns:
|
|
- JSONPath: .spec.poolName
|
|
name: ZPool
|
|
description: ZFS Pool where the volume is created
|
|
type: string
|
|
- JSONPath: .spec.ownerNodeID
|
|
name: Node
|
|
description: Node where the volume is created
|
|
type: string
|
|
- JSONPath: .spec.capacity
|
|
name: Size
|
|
description: Size of the volume
|
|
type: string
|
|
|
|
---
|
|
##############################################
|
|
########### ############
|
|
########### Controller plugin ############
|
|
########### ############
|
|
##############################################
|
|
|
|
kind: ServiceAccount
|
|
apiVersion: v1
|
|
metadata:
|
|
name: openebs-zfs-controller-sa
|
|
namespace: kube-system
|
|
|
|
---
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-provisioner-role
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumes", "services"]
|
|
verbs: ["get", "list", "watch", "create", "delete"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims"]
|
|
verbs: ["get", "list", "watch", "update"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["storageclasses", "csinodes"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: [""]
|
|
resources: ["events"]
|
|
verbs: ["list", "watch", "create", "update", "patch"]
|
|
- apiGroups: ["*"]
|
|
resources: ["zfsvolumes"]
|
|
verbs: ["*"]
|
|
---
|
|
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-provisioner-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: openebs-zfs-controller-sa
|
|
namespace: kube-system
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: openebs-zfs-provisioner-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
---
|
|
kind: StatefulSet
|
|
apiVersion: apps/v1
|
|
metadata:
|
|
name: openebs-zfs-controller
|
|
namespace: kube-system
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: openebs-zfs-controller
|
|
role: openebs-zfs
|
|
serviceName: "openebs-zfs"
|
|
replicas: 1
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: openebs-zfs-controller
|
|
role: openebs-zfs
|
|
spec:
|
|
priorityClassName: system-cluster-critical
|
|
serviceAccount: openebs-zfs-controller-sa
|
|
containers:
|
|
- name: csi-provisioner
|
|
image: quay.io/k8scsi/csi-provisioner:v1.4.0
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- "--csi-address=$(ADDRESS)"
|
|
- "--v=5"
|
|
- "--feature-gates=Topology=true"
|
|
- "--strict-topology"
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
- name: csi-attacher
|
|
image: quay.io/k8scsi/csi-attacher:v2.0.0
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- "--v=5"
|
|
- "--csi-address=$(ADDRESS)"
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
- name: csi-cluster-driver-registrar
|
|
image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1
|
|
args:
|
|
- "--v=5"
|
|
- "--driver-requires-attachment=false"
|
|
- "--csi-address=$(ADDRESS)"
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
- name: openebs-zfs-plugin
|
|
image: quay.io/openebs/zfs-driver:ci
|
|
imagePullPolicy: Always
|
|
env:
|
|
- name: OPENEBS_CONTROLLER_DRIVER
|
|
value: controller
|
|
- name: OPENEBS_CSI_ENDPOINT
|
|
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
|
|
- name: OPENEBS_NAMESPACE
|
|
value: openebs
|
|
args :
|
|
- "--endpoint=$(OPENEBS_CSI_ENDPOINT)"
|
|
- "--plugin=$(OPENEBS_CONTROLLER_DRIVER)"
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
volumes:
|
|
- name: socket-dir
|
|
emptyDir: {}
|
|
---
|
|
|
|
############################## CSI- Attacher #######################
|
|
# Attacher must be able to work with PVs, nodes and VolumeAttachments
|
|
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-attacher-role
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumes"]
|
|
verbs: ["get", "list", "watch", "update"]
|
|
- apiGroups: [""]
|
|
resources: ["nodes"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["csi.storage.k8s.io"]
|
|
resources: ["csinodeinfos"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["volumeattachments", "csinodes"]
|
|
verbs: ["get", "list", "watch", "update", "patch"]
|
|
|
|
---
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-attacher-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: openebs-zfs-controller-sa
|
|
namespace: kube-system
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: openebs-zfs-attacher-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
---
|
|
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-snapshotter-role
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumes"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["storageclasses"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: [""]
|
|
resources: ["events"]
|
|
verbs: ["list", "watch", "create", "update", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshotclasses"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshotcontents"]
|
|
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshots"]
|
|
verbs: ["get", "list", "watch", "update"]
|
|
- apiGroups: ["apiextensions.k8s.io"]
|
|
resources: ["customresourcedefinitions"]
|
|
verbs: ["create", "list", "watch", "delete"]
|
|
|
|
---
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-snapshotter-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: openebs-zfs-controller-sa
|
|
namespace: kube-system
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: openebs-zfs-snapshotter-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
---
|
|
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-cluster-driver-registrar-role
|
|
rules:
|
|
- apiGroups: ["csi.storage.k8s.io"]
|
|
resources: ["csidrivers"]
|
|
verbs: ["create", "delete"]
|
|
|
|
---
|
|
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-cluster-driver-registrar-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: openebs-zfs-controller-sa
|
|
namespace: kube-system
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: openebs-zfs-cluster-driver-registrar-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
---
|
|
|
|
########################################
|
|
########### ############
|
|
########### Node plugin ############
|
|
########### ############
|
|
########################################
|
|
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: openebs-zfs-node-sa
|
|
namespace: kube-system
|
|
|
|
---
|
|
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-driver-registrar-role
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["events"]
|
|
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumes", "nodes", "services"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: ["*"]
|
|
resources: ["zfsvolumes"]
|
|
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
|
|
|
---
|
|
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: openebs-zfs-driver-registrar-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: openebs-zfs-node-sa
|
|
namespace: kube-system
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: openebs-zfs-driver-registrar-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
---
|
|
|
|
kind: DaemonSet
|
|
apiVersion: apps/v1
|
|
metadata:
|
|
name: openebs-zfs-node
|
|
namespace: kube-system
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: openebs-zfs-node
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: openebs-zfs-node
|
|
role: openebs-zfs
|
|
spec:
|
|
priorityClassName: system-node-critical
|
|
serviceAccount: openebs-zfs-node-sa
|
|
hostNetwork: true
|
|
containers:
|
|
- name: csi-node-driver-registrar
|
|
image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
|
|
args:
|
|
- "--v=5"
|
|
- "--csi-address=$(ADDRESS)"
|
|
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
|
|
lifecycle:
|
|
preStop:
|
|
exec:
|
|
command: ["/bin/sh", "-c", "rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock"]
|
|
env:
|
|
- name: ADDRESS
|
|
value: /plugin/csi.sock
|
|
- name: DRIVER_REG_SOCK_PATH
|
|
value: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
|
|
- name: KUBE_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
- name: NODE_DRIVER
|
|
value: openebs-zfs
|
|
volumeMounts:
|
|
- name: plugin-dir
|
|
mountPath: /plugin
|
|
- name: registration-dir
|
|
mountPath: /registration
|
|
- name: openebs-zfs-plugin
|
|
securityContext:
|
|
privileged: true
|
|
capabilities:
|
|
add: ["CAP_MKNOD", "CAP_SYS_ADMIN", "SYS_ADMIN"]
|
|
allowPrivilegeEscalation: true
|
|
image: quay.io/openebs/zfs-driver:ci
|
|
imagePullPolicy: Always
|
|
args:
|
|
- "--nodeid=$(OPENEBS_NODE_ID)"
|
|
- "--endpoint=$(OPENEBS_CSI_ENDPOINT)"
|
|
- "--plugin=$(OPENEBS_NODE_DRIVER)"
|
|
env:
|
|
- name: OPENEBS_NODE_ID
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
- name: OPENEBS_CSI_ENDPOINT
|
|
value: unix:///plugin/csi.sock
|
|
- name: OPENEBS_NODE_DRIVER
|
|
value: agent
|
|
- name: OPENEBS_NAMESPACE
|
|
value: openebs
|
|
volumeMounts:
|
|
- name: plugin-dir
|
|
mountPath: /plugin
|
|
- name: device-dir
|
|
mountPath: /dev
|
|
- name: encr-keys
|
|
mountPath: /home/keys
|
|
- name: zfs-bin
|
|
mountPath: /sbin/zfs
|
|
- name: libzpool
|
|
mountPath: /lib/libzpool.so.2
|
|
- name: libzfscore
|
|
mountPath: /lib/libzfs_core.so.1
|
|
- name: libzfs
|
|
mountPath: /lib/libzfs.so.2
|
|
- name: libuutil
|
|
mountPath: /lib/libuutil.so.1
|
|
- name: libnvpair
|
|
mountPath: /lib/libnvpair.so.1
|
|
- name: pods-mount-dir
|
|
mountPath: /var/lib/kubelet/pods
|
|
# needed so that any mounts setup inside this container are
|
|
# propagated back to the host machine.
|
|
mountPropagation: "Bidirectional"
|
|
volumes:
|
|
- name: device-dir
|
|
hostPath:
|
|
path: /dev
|
|
type: Directory
|
|
- name: encr-keys
|
|
hostPath:
|
|
path: /home/keys
|
|
type: DirectoryOrCreate
|
|
- name: zfs-bin
|
|
hostPath:
|
|
path: /sbin/zfs
|
|
type: File
|
|
- name: libzpool
|
|
hostPath:
|
|
path: /lib/libzpool.so.2.0.0
|
|
type: File
|
|
- name: libzfscore
|
|
hostPath:
|
|
path: /lib/libzfs_core.so.1.0.0
|
|
type: File
|
|
- name: libzfs
|
|
hostPath:
|
|
path: /lib/libzfs.so.2.0.0
|
|
type: File
|
|
- name: libuutil
|
|
hostPath:
|
|
path: /lib/libuutil.so.1.0.1
|
|
type: File
|
|
- name: libnvpair
|
|
hostPath:
|
|
path: /lib/libnvpair.so.1.0.1
|
|
type: File
|
|
- name: registration-dir
|
|
hostPath:
|
|
path: /var/lib/kubelet/plugins_registry/
|
|
type: DirectoryOrCreate
|
|
- name: plugin-dir
|
|
hostPath:
|
|
path: /var/lib/kubelet/plugins/zfs-localpv/
|
|
type: DirectoryOrCreate
|
|
- name: pods-mount-dir
|
|
hostPath:
|
|
path: /var/lib/kubelet/pods
|
|
type: Directory
|
|
---
|