diff --git a/buildscripts/generate-manifests.sh b/buildscripts/generate-manifests.sh index 41e4a18..ebc574a 100755 --- a/buildscripts/generate-manifests.sh +++ b/buildscripts/generate-manifests.sh @@ -81,8 +81,18 @@ cat deploy/yamls/zfsvolume-crd.yaml >> deploy/zfs-operator.yaml # Add ZFSSnapshot v1alpha1 and v1 CRDs to the Operator yaml cat deploy/yamls/zfssnapshot-crd.yaml >> deploy/zfs-operator.yaml -# Add the driver deployment to the Operator -cat deploy/yamls/zfs-driver.yaml >> deploy/zfs-operator.yaml +# Copy the base Operator yaml to CentOS 7 and 8 Operator yamls +cp deploy/zfs-operator.yaml deploy/operators/centos7/zfs-operator.yaml +cp deploy/zfs-operator.yaml deploy/operators/centos8/zfs-operator.yaml + +# Add the ubuntu driver deployment to the Operator +cat deploy/yamls/ubuntu/zfs-driver.yaml >> deploy/zfs-operator.yaml + +# Add the centos7 driver deployment to the Operator +cat deploy/yamls/centos7/zfs-driver.yaml >> deploy/operators/centos7/zfs-operator.yaml + +# Add the centos8 driver deployment to the Operator +cat deploy/yamls/centos8/zfs-driver.yaml >> deploy/operators/centos8/zfs-operator.yaml # To use your own boilerplate text use: # --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt diff --git a/changelogs/unreleased/149-pawanpraka1 b/changelogs/unreleased/149-pawanpraka1 new file mode 100644 index 0000000..d8eca63 --- /dev/null +++ b/changelogs/unreleased/149-pawanpraka1 @@ -0,0 +1 @@ +adding operator yaml for centos7 and centos8 diff --git a/deploy/operators/centos7/zfs-operator.yaml b/deploy/operators/centos7/zfs-operator.yaml new file mode 100644 index 0000000..0479a04 --- /dev/null +++ b/deploy/operators/centos7/zfs-operator.yaml @@ -0,0 +1,1359 @@ +# This manifest is autogenerated via `make manifests` command +# Do the modification to the zfs-driver.yaml in directory deploy/yamls/ +# and then run `make manifests` command + +# This manifest deploys the OpenEBS ZFS control plane components, +# with associated CRs & RBAC rules. + +# Create the OpenEBS namespace +# This is the default namespace where the ZFS driver will create all +# its resources. If we want to change it to use a different namespace +# modify this to create the new namespace and also modify the OPENEBS_NAMESPACE +# env for the ZFS Driver's controller and agent deployments. +# please note that this should be changed while initial setup, once ZFS Driver +# is deployed with a namespace, we should never modify it as old resources will +# not be available under the new namespace and ZFS Driver looks for all the resources +# in the OPENEBS_NAMESPACE namespace passed as an env. + +apiVersion: v1 +kind: Namespace +metadata: + name: openebs + + +############################################## +########### ############ +########### ZFSVolume CRD ############ +########### ############ +############################################## + +# ZFSVolume CRD is autogenerated via `make manifests` command. +# Do the modification in the code and run the `make manifests` command +# to generate the CRD definition + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.8 + creationTimestamp: null + name: zfsvolumes.zfs.openebs.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.poolName + description: ZFS Pool where the volume is created + name: ZPool + type: string + - JSONPath: .spec.ownerNodeID + description: Node where the volume is created + name: Node + type: string + - JSONPath: .spec.capacity + description: Size of the volume + name: Size + type: string + - JSONPath: .status.state + description: Status of the volume + name: Status + type: string + - JSONPath: .spec.fsType + description: filesystem created on the volume + name: Filesystem + type: string + - JSONPath: .metadata.creationTimestamp + description: Age of the volume + name: Age + type: date + group: zfs.openebs.io + names: + kind: ZFSVolume + listKind: ZFSVolumeList + plural: zfsvolumes + shortNames: + - zfsvol + - zv + singular: zfsvolume + preserveUnknownFields: false + scope: Namespaced + subresources: {} + validation: + openAPIV3Schema: + description: ZFSVolume represents a ZFS based volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeInfo defines ZFS volume parameters for all modes in which + ZFS volumes can be created like - ZFS volume with filesystem, ZFS Volume + exposed as zfs or ZFS volume exposed as raw block device. Some of the + parameters can be only set during creation time (as specified in the details + of the parameter), and a few are editable. In case of Cloned volumes, + the parameters are assigned the same values as the source volume. + properties: + capacity: + description: Capacity of the volume + minLength: 1 + type: string + compression: + description: 'Compression specifies the block-level compression algorithm + to be applied to the ZFS Volume. The value "on" indicates ZFS to use + the default compression algorithm. The default compression algorithm + used by ZFS will be either lzjb or, if the lz4_compress feature is + enabled, lz4. Compression property can be edited after the volume + has been created. The change will only be applied to the newly-written + data. For instance, if the Volume was created with "off" and the next + day the compression was modified to "on", the data written prior to + setting "on" will not be compressed. Default Value: off.' + pattern: ^(on|off|lzjb|gzip|gzip-[1-9]|zle|lz4)$ + type: string + dedup: + description: 'Deduplication is the process for removing redundant data + at the block level, reducing the total amount of data stored. If a + file system has the dedup property enabled, duplicate data blocks + are removed synchronously. The result is that only unique data is + stored and common components are shared among files. Deduplication + can consume significant processing power (CPU) and memory as well + as generate additional disk IO. Before creating a pool with deduplication + enabled, ensure that you have planned your hardware requirements appropriately + and implemented appropriate recovery practices, such as regular backups. + As an alternative to deduplication consider using compression=lz4, + as a less resource-intensive alternative. should be enabled on the + zvol. Dedup property can be edited after the volume has been created. + Default Value: off.' + enum: + - "on" + - "off" + type: string + encryption: + description: 'Enabling the encryption feature allows for the creation + of encrypted filesystems and volumes. ZFS will encrypt file and zvol + data, file attributes, ACLs, permission bits, directory listings, + FUID mappings, and userused / groupused data. ZFS will not encrypt + metadata related to the pool structure, including dataset and snapshot + names, dataset hierarchy, properties, file size, file holes, and deduplication + tables (though the deduplicated data itself is encrypted). Default + Value: off.' + pattern: ^(on|off|aes-128-[c,g]cm|aes-192-[c,g]cm|aes-256-[c,g]cm)$ + type: string + fsType: + description: 'FsType specifies filesystem type for the zfs volume/dataset. + If FsType is provided as "zfs", then the driver will create a ZFS + dataset, formatting is not required as underlying filesystem is ZFS + anyway. If FsType is ext2, ext3, ext4 or xfs, then the driver will + create a ZVOL and format the volume accordingly. FsType can not be + modified once volume has been provisioned. Default Value: ext4.' + type: string + keyformat: + description: KeyFormat specifies format of the encryption key The supported + KeyFormats are passphrase, raw, hex. + enum: + - passphrase + - raw + - hex + type: string + keylocation: + description: KeyLocation is the location of key for the encryption + type: string + ownerNodeID: + description: OwnerNodeID is the Node ID where the ZPOOL is running which + is where the volume has been provisioned. OwnerNodeID can not be edited + after the volume has been provisioned. + minLength: 1 + type: string + poolName: + description: poolName specifies the name of the pool where the volume + has been created. PoolName can not be edited after the volume has + been provisioned. + minLength: 1 + type: string + recordsize: + description: 'Specifies a suggested block size for files in the file + system. The size specified must be a power of two greater than or + equal to 512 and less than or equal to 128 Kbytes. RecordSize property + can be edited after the volume has been created. Changing the file + system''s recordsize affects only files created afterward; existing + files are unaffected. Default Value: 128k.' + minLength: 1 + type: string + snapname: + description: SnapName specifies the name of the snapshot where the volume + has been cloned from. Snapname can not be edited after the volume + has been provisioned. + type: string + thinProvision: + description: 'ThinProvision describes whether space reservation for + the source volume is required or not. The value "yes" indicates that + volume should be thin provisioned and "no" means thick provisioning + of the volume. If thinProvision is set to "yes" then volume can be + provisioned even if the ZPOOL does not have the enough capacity. If + thinProvision is set to "no" then volume can be provisioned only if + the ZPOOL has enough capacity and capacity required by volume can + be reserved. ThinProvision can not be modified once volume has been + provisioned. Default Value: no.' + enum: + - "yes" + - "no" + type: string + volblocksize: + description: 'VolBlockSize specifies the block size for the zvol. The + volsize can only be set to a multiple of volblocksize, and cannot + be zero. VolBlockSize can not be edited after the volume has been + provisioned. Default Value: 8k.' + minLength: 1 + type: string + volumeType: + description: volumeType determines whether the volume is of type "DATASET" + or "ZVOL". If fstype provided in the storageclass is "zfs", a volume + of type dataset will be created. If "ext4", "ext3", "ext2" or "xfs" + is mentioned as fstype in the storageclass, then a volume of type + zvol will be created, which will be further formatted as the fstype + provided in the storageclass. VolumeType can not be modified once + volume has been provisioned. + enum: + - ZVOL + - DATASET + type: string + required: + - capacity + - ownerNodeID + - poolName + - volumeType + type: object + status: + properties: + state: + description: State specifies the current state of the volume provisioning + request. The state "Pending" means that the volume creation request + has not processed yet. The state "Ready" means that the volume has + been created and it is ready for the use. + enum: + - Pending + - Ready + type: string + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1alpha1 + served: true + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + +############################################## +########### ############ +########### ZFSSnapshot CRD ############ +########### ############ +############################################## + +# ZFSSnapshot CRD is autogenerated via `make manifests` command. +# Do the modification in the code and run the `make manifests` command +# to generate the CRD definition + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.8 + creationTimestamp: null + name: zfssnapshots.zfs.openebs.io +spec: + group: zfs.openebs.io + names: + kind: ZFSSnapshot + listKind: ZFSSnapshotList + plural: zfssnapshots + shortNames: + - zfssnap + singular: zfssnapshot + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: ZFSSnapshot represents a ZFS Snapshot of the zfsvolume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeInfo defines ZFS volume parameters for all modes in which + ZFS volumes can be created like - ZFS volume with filesystem, ZFS Volume + exposed as zfs or ZFS volume exposed as raw block device. Some of the + parameters can be only set during creation time (as specified in the details + of the parameter), and a few are editable. In case of Cloned volumes, + the parameters are assigned the same values as the source volume. + properties: + capacity: + description: Capacity of the volume + minLength: 1 + type: string + compression: + description: 'Compression specifies the block-level compression algorithm + to be applied to the ZFS Volume. The value "on" indicates ZFS to use + the default compression algorithm. The default compression algorithm + used by ZFS will be either lzjb or, if the lz4_compress feature is + enabled, lz4. Compression property can be edited after the volume + has been created. The change will only be applied to the newly-written + data. For instance, if the Volume was created with "off" and the next + day the compression was modified to "on", the data written prior to + setting "on" will not be compressed. Default Value: off.' + pattern: ^(on|off|lzjb|gzip|gzip-[1-9]|zle|lz4)$ + type: string + dedup: + description: 'Deduplication is the process for removing redundant data + at the block level, reducing the total amount of data stored. If a + file system has the dedup property enabled, duplicate data blocks + are removed synchronously. The result is that only unique data is + stored and common components are shared among files. Deduplication + can consume significant processing power (CPU) and memory as well + as generate additional disk IO. Before creating a pool with deduplication + enabled, ensure that you have planned your hardware requirements appropriately + and implemented appropriate recovery practices, such as regular backups. + As an alternative to deduplication consider using compression=lz4, + as a less resource-intensive alternative. should be enabled on the + zvol. Dedup property can be edited after the volume has been created. + Default Value: off.' + enum: + - "on" + - "off" + type: string + encryption: + description: 'Enabling the encryption feature allows for the creation + of encrypted filesystems and volumes. ZFS will encrypt file and zvol + data, file attributes, ACLs, permission bits, directory listings, + FUID mappings, and userused / groupused data. ZFS will not encrypt + metadata related to the pool structure, including dataset and snapshot + names, dataset hierarchy, properties, file size, file holes, and deduplication + tables (though the deduplicated data itself is encrypted). Default + Value: off.' + pattern: ^(on|off|aes-128-[c,g]cm|aes-192-[c,g]cm|aes-256-[c,g]cm)$ + type: string + fsType: + description: 'FsType specifies filesystem type for the zfs volume/dataset. + If FsType is provided as "zfs", then the driver will create a ZFS + dataset, formatting is not required as underlying filesystem is ZFS + anyway. If FsType is ext2, ext3, ext4 or xfs, then the driver will + create a ZVOL and format the volume accordingly. FsType can not be + modified once volume has been provisioned. Default Value: ext4.' + type: string + keyformat: + description: KeyFormat specifies format of the encryption key The supported + KeyFormats are passphrase, raw, hex. + enum: + - passphrase + - raw + - hex + type: string + keylocation: + description: KeyLocation is the location of key for the encryption + type: string + ownerNodeID: + description: OwnerNodeID is the Node ID where the ZPOOL is running which + is where the volume has been provisioned. OwnerNodeID can not be edited + after the volume has been provisioned. + minLength: 1 + type: string + poolName: + description: poolName specifies the name of the pool where the volume + has been created. PoolName can not be edited after the volume has + been provisioned. + minLength: 1 + type: string + recordsize: + description: 'Specifies a suggested block size for files in the file + system. The size specified must be a power of two greater than or + equal to 512 and less than or equal to 128 Kbytes. RecordSize property + can be edited after the volume has been created. Changing the file + system''s recordsize affects only files created afterward; existing + files are unaffected. Default Value: 128k.' + minLength: 1 + type: string + snapname: + description: SnapName specifies the name of the snapshot where the volume + has been cloned from. Snapname can not be edited after the volume + has been provisioned. + type: string + thinProvision: + description: 'ThinProvision describes whether space reservation for + the source volume is required or not. The value "yes" indicates that + volume should be thin provisioned and "no" means thick provisioning + of the volume. If thinProvision is set to "yes" then volume can be + provisioned even if the ZPOOL does not have the enough capacity. If + thinProvision is set to "no" then volume can be provisioned only if + the ZPOOL has enough capacity and capacity required by volume can + be reserved. ThinProvision can not be modified once volume has been + provisioned. Default Value: no.' + enum: + - "yes" + - "no" + type: string + volblocksize: + description: 'VolBlockSize specifies the block size for the zvol. The + volsize can only be set to a multiple of volblocksize, and cannot + be zero. VolBlockSize can not be edited after the volume has been + provisioned. Default Value: 8k.' + minLength: 1 + type: string + volumeType: + description: volumeType determines whether the volume is of type "DATASET" + or "ZVOL". If fstype provided in the storageclass is "zfs", a volume + of type dataset will be created. If "ext4", "ext3", "ext2" or "xfs" + is mentioned as fstype in the storageclass, then a volume of type + zvol will be created, which will be further formatted as the fstype + provided in the storageclass. VolumeType can not be modified once + volume has been provisioned. + enum: + - ZVOL + - DATASET + type: string + required: + - capacity + - ownerNodeID + - poolName + - volumeType + type: object + status: + properties: + state: + type: string + type: object + required: + - spec + - status + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1alpha1 + served: true + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- + +# Create the CSI Driver object +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: zfs.csi.openebs.io +spec: + # do not require volumeattachment + attachRequired: false + podInfoOnMount: false + volumeLifecycleModes: + - Persistent +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +############################################## +########### ############ +########### Controller plugin ############ +########### ############ +############################################## + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: openebs-zfs-controller-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["*"] + - apiGroups: [""] + resources: ["persistentvolumes", "services"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["*"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-controller + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-controller + role: openebs-zfs + serviceName: "openebs-zfs" + replicas: 1 + template: + metadata: + labels: + app: openebs-zfs-controller + role: openebs-zfs + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - openebs-zfs-controller + topologyKey: "kubernetes.io/hostname" + priorityClassName: system-cluster-critical + serviceAccount: openebs-zfs-controller-sa + containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.4.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v2.0.1 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: snapshot-controller + image: quay.io/k8scsi/snapshot-controller:v2.0.1 + args: + - "--v=5" + - "--leader-election=true" + imagePullPolicy: IfNotPresent + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--feature-gates=Topology=true" + - "--strict-topology" + - "--enable-leader-election" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: openebs-zfs-plugin + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + env: + - name: OPENEBS_CONTROLLER_DRIVER + value: controller + - name: OPENEBS_CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: OPENEBS_NAMESPACE + value: openebs + - name: OPENEBS_IO_INSTALLER_TYPE + value: "zfs-operator" + - name: OPENEBS_IO_ENABLE_ANALYTICS + value: "true" + args : + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- + +######################################## +########### ############ +########### Node plugin ############ +########### ############ +######################################## + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-zfs-node-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes", "nodes", "services"] + verbs: ["get", "list"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-node + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-node + template: + metadata: + labels: + app: openebs-zfs-node + role: openebs-zfs + spec: + priorityClassName: system-node-critical + serviceAccount: openebs-zfs-node-sa + hostNetwork: true + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock"] + env: + - name: ADDRESS + value: /plugin/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/zfs-localpv/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_DRIVER + value: openebs-zfs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + - name: openebs-zfs-plugin + securityContext: + privileged: true + capabilities: + add: ["CAP_MKNOD", "CAP_SYS_ADMIN", "SYS_ADMIN"] + allowPrivilegeEscalation: true + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + args: + - "--nodeid=$(OPENEBS_NODE_ID)" + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_NODE_DRIVER)" + env: + - name: OPENEBS_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPENEBS_CSI_ENDPOINT + value: unix:///plugin/csi.sock + - name: OPENEBS_NODE_DRIVER + value: agent + - name: OPENEBS_NAMESPACE + value: openebs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: device-dir + mountPath: /dev + - name: encr-keys + mountPath: /home/keys + - name: zfs-bin + mountPath: /sbin/zfs + - name: libzpool + mountPath: /lib/libzpool.so.2 + - name: libzfscore + mountPath: /lib/libzfs_core.so.1 + - name: libzfs + mountPath: /lib/libzfs.so.2 + - name: libuutil + mountPath: /lib/libuutil.so.1 + - name: libnvpair + mountPath: /lib/libnvpair.so.1 + - name: libssl + mountPath: /lib/libssl.so.10 + - name: libcrypto + mountPath: /lib/libcrypto.so.10 + - name: libk5crypto + mountPath: /lib/libk5crypto.so.3 + - name: libgssapi + mountPath: /lib/libgssapi_krb5.so.2 + - name: libkrb + mountPath: /lib/libkrb5.so.3 + - name: libkrb5support + mountPath: /lib/libkrb5support.so.0 + - name: libkeyutils + mountPath: /lib/libkeyutils.so.1 + - name: libtirpc + mountPath: /lib/libtirpc.so.3 + - name: pods-mount-dir + mountPath: /var/lib/kubelet/ + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + volumes: + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: encr-keys + hostPath: + path: /home/keys + type: DirectoryOrCreate + - name: zfs-bin + hostPath: + path: /usr/sbin/zfs + type: File + - name: libzpool + hostPath: + path: /lib64/libzpool.so.2.0.0 + type: File + - name: libzfscore + hostPath: + path: /lib64/libzfs_core.so.1.0.0 + type: File + - name: libzfs + hostPath: + path: /lib64/libzfs.so.2.0.0 + type: File + - name: libuutil + hostPath: + path: /lib64/libuutil.so.1.0.1 + type: File + - name: libnvpair + hostPath: + path: /lib64/libnvpair.so.1.0.1 + type: File + - name: libssl + hostPath: + path: /lib64/libssl.so.1.0.2k + type: FileOrCreate + - name: libcrypto + hostPath: + path: /lib64/libcrypto.so.1.0.2k + type: FileOrCreate + - name: libk5crypto + hostPath: + path: /lib64/libk5crypto.so.3.1 + type: FileOrCreate + - name: libgssapi + hostPath: + path: /lib64/libgssapi_krb5.so.2.2 + type: FileOrCreate + - name: libkrb + hostPath: + path: /lib64/libkrb5.so.3.3 + type: FileOrCreate + - name: libkrb5support + hostPath: + path: /lib64/libkrb5support.so.0.1 + type: FileOrCreate + - name: libkeyutils + hostPath: + path: /lib64/libkeyutils.so.1.5 + type: FileOrCreate + - name: libtirpc + hostPath: + path: /lib64/libtirpc.so.3.0.0 + type: FileOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/zfs-localpv/ + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/ + type: Directory +--- diff --git a/deploy/operators/centos8/zfs-operator.yaml b/deploy/operators/centos8/zfs-operator.yaml new file mode 100644 index 0000000..c80cb88 --- /dev/null +++ b/deploy/operators/centos8/zfs-operator.yaml @@ -0,0 +1,1359 @@ +# This manifest is autogenerated via `make manifests` command +# Do the modification to the zfs-driver.yaml in directory deploy/yamls/ +# and then run `make manifests` command + +# This manifest deploys the OpenEBS ZFS control plane components, +# with associated CRs & RBAC rules. + +# Create the OpenEBS namespace +# This is the default namespace where the ZFS driver will create all +# its resources. If we want to change it to use a different namespace +# modify this to create the new namespace and also modify the OPENEBS_NAMESPACE +# env for the ZFS Driver's controller and agent deployments. +# please note that this should be changed while initial setup, once ZFS Driver +# is deployed with a namespace, we should never modify it as old resources will +# not be available under the new namespace and ZFS Driver looks for all the resources +# in the OPENEBS_NAMESPACE namespace passed as an env. + +apiVersion: v1 +kind: Namespace +metadata: + name: openebs + + +############################################## +########### ############ +########### ZFSVolume CRD ############ +########### ############ +############################################## + +# ZFSVolume CRD is autogenerated via `make manifests` command. +# Do the modification in the code and run the `make manifests` command +# to generate the CRD definition + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.8 + creationTimestamp: null + name: zfsvolumes.zfs.openebs.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.poolName + description: ZFS Pool where the volume is created + name: ZPool + type: string + - JSONPath: .spec.ownerNodeID + description: Node where the volume is created + name: Node + type: string + - JSONPath: .spec.capacity + description: Size of the volume + name: Size + type: string + - JSONPath: .status.state + description: Status of the volume + name: Status + type: string + - JSONPath: .spec.fsType + description: filesystem created on the volume + name: Filesystem + type: string + - JSONPath: .metadata.creationTimestamp + description: Age of the volume + name: Age + type: date + group: zfs.openebs.io + names: + kind: ZFSVolume + listKind: ZFSVolumeList + plural: zfsvolumes + shortNames: + - zfsvol + - zv + singular: zfsvolume + preserveUnknownFields: false + scope: Namespaced + subresources: {} + validation: + openAPIV3Schema: + description: ZFSVolume represents a ZFS based volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeInfo defines ZFS volume parameters for all modes in which + ZFS volumes can be created like - ZFS volume with filesystem, ZFS Volume + exposed as zfs or ZFS volume exposed as raw block device. Some of the + parameters can be only set during creation time (as specified in the details + of the parameter), and a few are editable. In case of Cloned volumes, + the parameters are assigned the same values as the source volume. + properties: + capacity: + description: Capacity of the volume + minLength: 1 + type: string + compression: + description: 'Compression specifies the block-level compression algorithm + to be applied to the ZFS Volume. The value "on" indicates ZFS to use + the default compression algorithm. The default compression algorithm + used by ZFS will be either lzjb or, if the lz4_compress feature is + enabled, lz4. Compression property can be edited after the volume + has been created. The change will only be applied to the newly-written + data. For instance, if the Volume was created with "off" and the next + day the compression was modified to "on", the data written prior to + setting "on" will not be compressed. Default Value: off.' + pattern: ^(on|off|lzjb|gzip|gzip-[1-9]|zle|lz4)$ + type: string + dedup: + description: 'Deduplication is the process for removing redundant data + at the block level, reducing the total amount of data stored. If a + file system has the dedup property enabled, duplicate data blocks + are removed synchronously. The result is that only unique data is + stored and common components are shared among files. Deduplication + can consume significant processing power (CPU) and memory as well + as generate additional disk IO. Before creating a pool with deduplication + enabled, ensure that you have planned your hardware requirements appropriately + and implemented appropriate recovery practices, such as regular backups. + As an alternative to deduplication consider using compression=lz4, + as a less resource-intensive alternative. should be enabled on the + zvol. Dedup property can be edited after the volume has been created. + Default Value: off.' + enum: + - "on" + - "off" + type: string + encryption: + description: 'Enabling the encryption feature allows for the creation + of encrypted filesystems and volumes. ZFS will encrypt file and zvol + data, file attributes, ACLs, permission bits, directory listings, + FUID mappings, and userused / groupused data. ZFS will not encrypt + metadata related to the pool structure, including dataset and snapshot + names, dataset hierarchy, properties, file size, file holes, and deduplication + tables (though the deduplicated data itself is encrypted). Default + Value: off.' + pattern: ^(on|off|aes-128-[c,g]cm|aes-192-[c,g]cm|aes-256-[c,g]cm)$ + type: string + fsType: + description: 'FsType specifies filesystem type for the zfs volume/dataset. + If FsType is provided as "zfs", then the driver will create a ZFS + dataset, formatting is not required as underlying filesystem is ZFS + anyway. If FsType is ext2, ext3, ext4 or xfs, then the driver will + create a ZVOL and format the volume accordingly. FsType can not be + modified once volume has been provisioned. Default Value: ext4.' + type: string + keyformat: + description: KeyFormat specifies format of the encryption key The supported + KeyFormats are passphrase, raw, hex. + enum: + - passphrase + - raw + - hex + type: string + keylocation: + description: KeyLocation is the location of key for the encryption + type: string + ownerNodeID: + description: OwnerNodeID is the Node ID where the ZPOOL is running which + is where the volume has been provisioned. OwnerNodeID can not be edited + after the volume has been provisioned. + minLength: 1 + type: string + poolName: + description: poolName specifies the name of the pool where the volume + has been created. PoolName can not be edited after the volume has + been provisioned. + minLength: 1 + type: string + recordsize: + description: 'Specifies a suggested block size for files in the file + system. The size specified must be a power of two greater than or + equal to 512 and less than or equal to 128 Kbytes. RecordSize property + can be edited after the volume has been created. Changing the file + system''s recordsize affects only files created afterward; existing + files are unaffected. Default Value: 128k.' + minLength: 1 + type: string + snapname: + description: SnapName specifies the name of the snapshot where the volume + has been cloned from. Snapname can not be edited after the volume + has been provisioned. + type: string + thinProvision: + description: 'ThinProvision describes whether space reservation for + the source volume is required or not. The value "yes" indicates that + volume should be thin provisioned and "no" means thick provisioning + of the volume. If thinProvision is set to "yes" then volume can be + provisioned even if the ZPOOL does not have the enough capacity. If + thinProvision is set to "no" then volume can be provisioned only if + the ZPOOL has enough capacity and capacity required by volume can + be reserved. ThinProvision can not be modified once volume has been + provisioned. Default Value: no.' + enum: + - "yes" + - "no" + type: string + volblocksize: + description: 'VolBlockSize specifies the block size for the zvol. The + volsize can only be set to a multiple of volblocksize, and cannot + be zero. VolBlockSize can not be edited after the volume has been + provisioned. Default Value: 8k.' + minLength: 1 + type: string + volumeType: + description: volumeType determines whether the volume is of type "DATASET" + or "ZVOL". If fstype provided in the storageclass is "zfs", a volume + of type dataset will be created. If "ext4", "ext3", "ext2" or "xfs" + is mentioned as fstype in the storageclass, then a volume of type + zvol will be created, which will be further formatted as the fstype + provided in the storageclass. VolumeType can not be modified once + volume has been provisioned. + enum: + - ZVOL + - DATASET + type: string + required: + - capacity + - ownerNodeID + - poolName + - volumeType + type: object + status: + properties: + state: + description: State specifies the current state of the volume provisioning + request. The state "Pending" means that the volume creation request + has not processed yet. The state "Ready" means that the volume has + been created and it is ready for the use. + enum: + - Pending + - Ready + type: string + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1alpha1 + served: true + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + +############################################## +########### ############ +########### ZFSSnapshot CRD ############ +########### ############ +############################################## + +# ZFSSnapshot CRD is autogenerated via `make manifests` command. +# Do the modification in the code and run the `make manifests` command +# to generate the CRD definition + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.8 + creationTimestamp: null + name: zfssnapshots.zfs.openebs.io +spec: + group: zfs.openebs.io + names: + kind: ZFSSnapshot + listKind: ZFSSnapshotList + plural: zfssnapshots + shortNames: + - zfssnap + singular: zfssnapshot + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: ZFSSnapshot represents a ZFS Snapshot of the zfsvolume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeInfo defines ZFS volume parameters for all modes in which + ZFS volumes can be created like - ZFS volume with filesystem, ZFS Volume + exposed as zfs or ZFS volume exposed as raw block device. Some of the + parameters can be only set during creation time (as specified in the details + of the parameter), and a few are editable. In case of Cloned volumes, + the parameters are assigned the same values as the source volume. + properties: + capacity: + description: Capacity of the volume + minLength: 1 + type: string + compression: + description: 'Compression specifies the block-level compression algorithm + to be applied to the ZFS Volume. The value "on" indicates ZFS to use + the default compression algorithm. The default compression algorithm + used by ZFS will be either lzjb or, if the lz4_compress feature is + enabled, lz4. Compression property can be edited after the volume + has been created. The change will only be applied to the newly-written + data. For instance, if the Volume was created with "off" and the next + day the compression was modified to "on", the data written prior to + setting "on" will not be compressed. Default Value: off.' + pattern: ^(on|off|lzjb|gzip|gzip-[1-9]|zle|lz4)$ + type: string + dedup: + description: 'Deduplication is the process for removing redundant data + at the block level, reducing the total amount of data stored. If a + file system has the dedup property enabled, duplicate data blocks + are removed synchronously. The result is that only unique data is + stored and common components are shared among files. Deduplication + can consume significant processing power (CPU) and memory as well + as generate additional disk IO. Before creating a pool with deduplication + enabled, ensure that you have planned your hardware requirements appropriately + and implemented appropriate recovery practices, such as regular backups. + As an alternative to deduplication consider using compression=lz4, + as a less resource-intensive alternative. should be enabled on the + zvol. Dedup property can be edited after the volume has been created. + Default Value: off.' + enum: + - "on" + - "off" + type: string + encryption: + description: 'Enabling the encryption feature allows for the creation + of encrypted filesystems and volumes. ZFS will encrypt file and zvol + data, file attributes, ACLs, permission bits, directory listings, + FUID mappings, and userused / groupused data. ZFS will not encrypt + metadata related to the pool structure, including dataset and snapshot + names, dataset hierarchy, properties, file size, file holes, and deduplication + tables (though the deduplicated data itself is encrypted). Default + Value: off.' + pattern: ^(on|off|aes-128-[c,g]cm|aes-192-[c,g]cm|aes-256-[c,g]cm)$ + type: string + fsType: + description: 'FsType specifies filesystem type for the zfs volume/dataset. + If FsType is provided as "zfs", then the driver will create a ZFS + dataset, formatting is not required as underlying filesystem is ZFS + anyway. If FsType is ext2, ext3, ext4 or xfs, then the driver will + create a ZVOL and format the volume accordingly. FsType can not be + modified once volume has been provisioned. Default Value: ext4.' + type: string + keyformat: + description: KeyFormat specifies format of the encryption key The supported + KeyFormats are passphrase, raw, hex. + enum: + - passphrase + - raw + - hex + type: string + keylocation: + description: KeyLocation is the location of key for the encryption + type: string + ownerNodeID: + description: OwnerNodeID is the Node ID where the ZPOOL is running which + is where the volume has been provisioned. OwnerNodeID can not be edited + after the volume has been provisioned. + minLength: 1 + type: string + poolName: + description: poolName specifies the name of the pool where the volume + has been created. PoolName can not be edited after the volume has + been provisioned. + minLength: 1 + type: string + recordsize: + description: 'Specifies a suggested block size for files in the file + system. The size specified must be a power of two greater than or + equal to 512 and less than or equal to 128 Kbytes. RecordSize property + can be edited after the volume has been created. Changing the file + system''s recordsize affects only files created afterward; existing + files are unaffected. Default Value: 128k.' + minLength: 1 + type: string + snapname: + description: SnapName specifies the name of the snapshot where the volume + has been cloned from. Snapname can not be edited after the volume + has been provisioned. + type: string + thinProvision: + description: 'ThinProvision describes whether space reservation for + the source volume is required or not. The value "yes" indicates that + volume should be thin provisioned and "no" means thick provisioning + of the volume. If thinProvision is set to "yes" then volume can be + provisioned even if the ZPOOL does not have the enough capacity. If + thinProvision is set to "no" then volume can be provisioned only if + the ZPOOL has enough capacity and capacity required by volume can + be reserved. ThinProvision can not be modified once volume has been + provisioned. Default Value: no.' + enum: + - "yes" + - "no" + type: string + volblocksize: + description: 'VolBlockSize specifies the block size for the zvol. The + volsize can only be set to a multiple of volblocksize, and cannot + be zero. VolBlockSize can not be edited after the volume has been + provisioned. Default Value: 8k.' + minLength: 1 + type: string + volumeType: + description: volumeType determines whether the volume is of type "DATASET" + or "ZVOL". If fstype provided in the storageclass is "zfs", a volume + of type dataset will be created. If "ext4", "ext3", "ext2" or "xfs" + is mentioned as fstype in the storageclass, then a volume of type + zvol will be created, which will be further formatted as the fstype + provided in the storageclass. VolumeType can not be modified once + volume has been provisioned. + enum: + - ZVOL + - DATASET + type: string + required: + - capacity + - ownerNodeID + - poolName + - volumeType + type: object + status: + properties: + state: + type: string + type: object + required: + - spec + - status + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1alpha1 + served: true + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- + +# Create the CSI Driver object +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: zfs.csi.openebs.io +spec: + # do not require volumeattachment + attachRequired: false + podInfoOnMount: false + volumeLifecycleModes: + - Persistent +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +############################################## +########### ############ +########### Controller plugin ############ +########### ############ +############################################## + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: openebs-zfs-controller-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["*"] + - apiGroups: [""] + resources: ["persistentvolumes", "services"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["*"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-controller + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-controller + role: openebs-zfs + serviceName: "openebs-zfs" + replicas: 1 + template: + metadata: + labels: + app: openebs-zfs-controller + role: openebs-zfs + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - openebs-zfs-controller + topologyKey: "kubernetes.io/hostname" + priorityClassName: system-cluster-critical + serviceAccount: openebs-zfs-controller-sa + containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.4.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v2.0.1 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: snapshot-controller + image: quay.io/k8scsi/snapshot-controller:v2.0.1 + args: + - "--v=5" + - "--leader-election=true" + imagePullPolicy: IfNotPresent + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--feature-gates=Topology=true" + - "--strict-topology" + - "--enable-leader-election" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: openebs-zfs-plugin + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + env: + - name: OPENEBS_CONTROLLER_DRIVER + value: controller + - name: OPENEBS_CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: OPENEBS_NAMESPACE + value: openebs + - name: OPENEBS_IO_INSTALLER_TYPE + value: "zfs-operator" + - name: OPENEBS_IO_ENABLE_ANALYTICS + value: "true" + args : + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- + +######################################## +########### ############ +########### Node plugin ############ +########### ############ +######################################## + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-zfs-node-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes", "nodes", "services"] + verbs: ["get", "list"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-node + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-node + template: + metadata: + labels: + app: openebs-zfs-node + role: openebs-zfs + spec: + priorityClassName: system-node-critical + serviceAccount: openebs-zfs-node-sa + hostNetwork: true + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock"] + env: + - name: ADDRESS + value: /plugin/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/zfs-localpv/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_DRIVER + value: openebs-zfs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + - name: openebs-zfs-plugin + securityContext: + privileged: true + capabilities: + add: ["CAP_MKNOD", "CAP_SYS_ADMIN", "SYS_ADMIN"] + allowPrivilegeEscalation: true + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + args: + - "--nodeid=$(OPENEBS_NODE_ID)" + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_NODE_DRIVER)" + env: + - name: OPENEBS_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPENEBS_CSI_ENDPOINT + value: unix:///plugin/csi.sock + - name: OPENEBS_NODE_DRIVER + value: agent + - name: OPENEBS_NAMESPACE + value: openebs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: device-dir + mountPath: /dev + - name: encr-keys + mountPath: /home/keys + - name: zfs-bin + mountPath: /sbin/zfs + - name: libzpool + mountPath: /lib/libzpool.so.2 + - name: libzfscore + mountPath: /lib/libzfs_core.so.1 + - name: libzfs + mountPath: /lib/libzfs.so.2 + - name: libuutil + mountPath: /lib/libuutil.so.1 + - name: libnvpair + mountPath: /lib/libnvpair.so.1 + - name: libssl + mountPath: /lib/libssl.so.10 + - name: libcrypto + mountPath: /lib/libcrypto.so.10 + - name: libk5crypto + mountPath: /lib/libk5crypto.so.3 + - name: libgssapi + mountPath: /lib/libgssapi_krb5.so.2 + - name: libkrb + mountPath: /lib/libkrb5.so.3 + - name: libkrb5support + mountPath: /lib/libkrb5support.so.0 + - name: libkeyutils + mountPath: /lib/libkeyutils.so.1 + - name: libtirpc + mountPath: /lib/libtirpc.so.3 + - name: pods-mount-dir + mountPath: /var/lib/kubelet/ + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + volumes: + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: encr-keys + hostPath: + path: /home/keys + type: DirectoryOrCreate + - name: zfs-bin + hostPath: + path: /usr/sbin/zfs + type: File + - name: libzpool + hostPath: + path: /lib64/libzpool.so.2.0.0 + type: File + - name: libzfscore + hostPath: + path: /lib64/libzfs_core.so.1.0.0 + type: File + - name: libzfs + hostPath: + path: /lib64/libzfs.so.2.0.0 + type: File + - name: libuutil + hostPath: + path: /lib64/libuutil.so.1.0.1 + type: File + - name: libnvpair + hostPath: + path: /lib64/libnvpair.so.1.0.1 + type: File + - name: libssl + hostPath: + path: /lib64/libssl.so.1.0.2k + type: FileOrCreate + - name: libcrypto + hostPath: + path: /lib64/libcrypto.so.1.0.2k + type: FileOrCreate + - name: libk5crypto + hostPath: + path: /lib64/libk5crypto.so.3.1 + type: FileOrCreate + - name: libgssapi + hostPath: + path: /lib64/libgssapi_krb5.so.2.2 + type: FileOrCreate + - name: libkrb + hostPath: + path: /lib64/libkrb5.so.3.3 + type: FileOrCreate + - name: libkrb5support + hostPath: + path: /lib64/libkrb5support.so.0.1 + type: FileOrCreate + - name: libkeyutils + hostPath: + path: /lib64/libkeyutils.so.1.6 + type: FileOrCreate + - name: libtirpc + hostPath: + path: /lib64/libtirpc.so.3.0.0 + type: FileOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/zfs-localpv/ + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/ + type: Directory +--- diff --git a/deploy/yamls/centos7/zfs-driver.yaml b/deploy/yamls/centos7/zfs-driver.yaml new file mode 100644 index 0000000..9cff1dd --- /dev/null +++ b/deploy/yamls/centos7/zfs-driver.yaml @@ -0,0 +1,891 @@ + +--- + +# Create the CSI Driver object +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: zfs.csi.openebs.io +spec: + # do not require volumeattachment + attachRequired: false + podInfoOnMount: false + volumeLifecycleModes: + - Persistent +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +############################################## +########### ############ +########### Controller plugin ############ +########### ############ +############################################## + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: openebs-zfs-controller-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["*"] + - apiGroups: [""] + resources: ["persistentvolumes", "services"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["*"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-controller + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-controller + role: openebs-zfs + serviceName: "openebs-zfs" + replicas: 1 + template: + metadata: + labels: + app: openebs-zfs-controller + role: openebs-zfs + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - openebs-zfs-controller + topologyKey: "kubernetes.io/hostname" + priorityClassName: system-cluster-critical + serviceAccount: openebs-zfs-controller-sa + containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.4.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v2.0.1 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: snapshot-controller + image: quay.io/k8scsi/snapshot-controller:v2.0.1 + args: + - "--v=5" + - "--leader-election=true" + imagePullPolicy: IfNotPresent + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--feature-gates=Topology=true" + - "--strict-topology" + - "--enable-leader-election" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: openebs-zfs-plugin + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + env: + - name: OPENEBS_CONTROLLER_DRIVER + value: controller + - name: OPENEBS_CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: OPENEBS_NAMESPACE + value: openebs + - name: OPENEBS_IO_INSTALLER_TYPE + value: "zfs-operator" + - name: OPENEBS_IO_ENABLE_ANALYTICS + value: "true" + args : + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- + +######################################## +########### ############ +########### Node plugin ############ +########### ############ +######################################## + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-zfs-node-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes", "nodes", "services"] + verbs: ["get", "list"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-node + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-node + template: + metadata: + labels: + app: openebs-zfs-node + role: openebs-zfs + spec: + priorityClassName: system-node-critical + serviceAccount: openebs-zfs-node-sa + hostNetwork: true + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock"] + env: + - name: ADDRESS + value: /plugin/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/zfs-localpv/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_DRIVER + value: openebs-zfs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + - name: openebs-zfs-plugin + securityContext: + privileged: true + capabilities: + add: ["CAP_MKNOD", "CAP_SYS_ADMIN", "SYS_ADMIN"] + allowPrivilegeEscalation: true + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + args: + - "--nodeid=$(OPENEBS_NODE_ID)" + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_NODE_DRIVER)" + env: + - name: OPENEBS_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPENEBS_CSI_ENDPOINT + value: unix:///plugin/csi.sock + - name: OPENEBS_NODE_DRIVER + value: agent + - name: OPENEBS_NAMESPACE + value: openebs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: device-dir + mountPath: /dev + - name: encr-keys + mountPath: /home/keys + - name: zfs-bin + mountPath: /sbin/zfs + - name: libzpool + mountPath: /lib/libzpool.so.2 + - name: libzfscore + mountPath: /lib/libzfs_core.so.1 + - name: libzfs + mountPath: /lib/libzfs.so.2 + - name: libuutil + mountPath: /lib/libuutil.so.1 + - name: libnvpair + mountPath: /lib/libnvpair.so.1 + - name: libssl + mountPath: /lib/libssl.so.10 + - name: libcrypto + mountPath: /lib/libcrypto.so.10 + - name: libk5crypto + mountPath: /lib/libk5crypto.so.3 + - name: libgssapi + mountPath: /lib/libgssapi_krb5.so.2 + - name: libkrb + mountPath: /lib/libkrb5.so.3 + - name: libkrb5support + mountPath: /lib/libkrb5support.so.0 + - name: libkeyutils + mountPath: /lib/libkeyutils.so.1 + - name: libtirpc + mountPath: /lib/libtirpc.so.3 + - name: pods-mount-dir + mountPath: /var/lib/kubelet/ + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + volumes: + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: encr-keys + hostPath: + path: /home/keys + type: DirectoryOrCreate + - name: zfs-bin + hostPath: + path: /usr/sbin/zfs + type: File + - name: libzpool + hostPath: + path: /lib64/libzpool.so.2.0.0 + type: File + - name: libzfscore + hostPath: + path: /lib64/libzfs_core.so.1.0.0 + type: File + - name: libzfs + hostPath: + path: /lib64/libzfs.so.2.0.0 + type: File + - name: libuutil + hostPath: + path: /lib64/libuutil.so.1.0.1 + type: File + - name: libnvpair + hostPath: + path: /lib64/libnvpair.so.1.0.1 + type: File + - name: libssl + hostPath: + path: /lib64/libssl.so.1.0.2k + type: FileOrCreate + - name: libcrypto + hostPath: + path: /lib64/libcrypto.so.1.0.2k + type: FileOrCreate + - name: libk5crypto + hostPath: + path: /lib64/libk5crypto.so.3.1 + type: FileOrCreate + - name: libgssapi + hostPath: + path: /lib64/libgssapi_krb5.so.2.2 + type: FileOrCreate + - name: libkrb + hostPath: + path: /lib64/libkrb5.so.3.3 + type: FileOrCreate + - name: libkrb5support + hostPath: + path: /lib64/libkrb5support.so.0.1 + type: FileOrCreate + - name: libkeyutils + hostPath: + path: /lib64/libkeyutils.so.1.5 + type: FileOrCreate + - name: libtirpc + hostPath: + path: /lib64/libtirpc.so.3.0.0 + type: FileOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/zfs-localpv/ + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/ + type: Directory +--- diff --git a/deploy/yamls/centos8/zfs-driver.yaml b/deploy/yamls/centos8/zfs-driver.yaml new file mode 100644 index 0000000..549996a --- /dev/null +++ b/deploy/yamls/centos8/zfs-driver.yaml @@ -0,0 +1,891 @@ + +--- + +# Create the CSI Driver object +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: zfs.csi.openebs.io +spec: + # do not require volumeattachment + attachRequired: false + podInfoOnMount: false + volumeLifecycleModes: + - Persistent +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + subresources: + status: {} + #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/ + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- + +############################################## +########### ############ +########### Controller plugin ############ +########### ############ +############################################## + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: openebs-zfs-controller-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["*"] + - apiGroups: [""] + resources: ["persistentvolumes", "services"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["*"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-provisioner-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-controller + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-controller + role: openebs-zfs + serviceName: "openebs-zfs" + replicas: 1 + template: + metadata: + labels: + app: openebs-zfs-controller + role: openebs-zfs + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - openebs-zfs-controller + topologyKey: "kubernetes.io/hostname" + priorityClassName: system-cluster-critical + serviceAccount: openebs-zfs-controller-sa + containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.4.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v2.0.1 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: snapshot-controller + image: quay.io/k8scsi/snapshot-controller:v2.0.1 + args: + - "--v=5" + - "--leader-election=true" + imagePullPolicy: IfNotPresent + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--feature-gates=Topology=true" + - "--strict-topology" + - "--enable-leader-election" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: openebs-zfs-plugin + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + env: + - name: OPENEBS_CONTROLLER_DRIVER + value: controller + - name: OPENEBS_CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: OPENEBS_NAMESPACE + value: openebs + - name: OPENEBS_IO_INSTALLER_TYPE + value: "zfs-operator" + - name: OPENEBS_IO_ENABLE_ANALYTICS + value: "true" + args : + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-snapshotter-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- + +######################################## +########### ############ +########### Node plugin ############ +########### ############ +######################################## + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-zfs-node-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes", "nodes", "services"] + verbs: ["get", "list"] + - apiGroups: ["*"] + resources: ["zfsvolumes", "zfssnapshots"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-zfs-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: openebs-zfs-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: openebs-zfs-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: openebs-zfs-node + namespace: kube-system +spec: + selector: + matchLabels: + app: openebs-zfs-node + template: + metadata: + labels: + app: openebs-zfs-node + role: openebs-zfs + spec: + priorityClassName: system-node-critical + serviceAccount: openebs-zfs-node-sa + hostNetwork: true + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock"] + env: + - name: ADDRESS + value: /plugin/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/zfs-localpv/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_DRIVER + value: openebs-zfs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + - name: openebs-zfs-plugin + securityContext: + privileged: true + capabilities: + add: ["CAP_MKNOD", "CAP_SYS_ADMIN", "SYS_ADMIN"] + allowPrivilegeEscalation: true + image: quay.io/openebs/zfs-driver:ci + imagePullPolicy: IfNotPresent + args: + - "--nodeid=$(OPENEBS_NODE_ID)" + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_NODE_DRIVER)" + env: + - name: OPENEBS_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPENEBS_CSI_ENDPOINT + value: unix:///plugin/csi.sock + - name: OPENEBS_NODE_DRIVER + value: agent + - name: OPENEBS_NAMESPACE + value: openebs + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: device-dir + mountPath: /dev + - name: encr-keys + mountPath: /home/keys + - name: zfs-bin + mountPath: /sbin/zfs + - name: libzpool + mountPath: /lib/libzpool.so.2 + - name: libzfscore + mountPath: /lib/libzfs_core.so.1 + - name: libzfs + mountPath: /lib/libzfs.so.2 + - name: libuutil + mountPath: /lib/libuutil.so.1 + - name: libnvpair + mountPath: /lib/libnvpair.so.1 + - name: libssl + mountPath: /lib/libssl.so.10 + - name: libcrypto + mountPath: /lib/libcrypto.so.10 + - name: libk5crypto + mountPath: /lib/libk5crypto.so.3 + - name: libgssapi + mountPath: /lib/libgssapi_krb5.so.2 + - name: libkrb + mountPath: /lib/libkrb5.so.3 + - name: libkrb5support + mountPath: /lib/libkrb5support.so.0 + - name: libkeyutils + mountPath: /lib/libkeyutils.so.1 + - name: libtirpc + mountPath: /lib/libtirpc.so.3 + - name: pods-mount-dir + mountPath: /var/lib/kubelet/ + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + volumes: + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: encr-keys + hostPath: + path: /home/keys + type: DirectoryOrCreate + - name: zfs-bin + hostPath: + path: /usr/sbin/zfs + type: File + - name: libzpool + hostPath: + path: /lib64/libzpool.so.2.0.0 + type: File + - name: libzfscore + hostPath: + path: /lib64/libzfs_core.so.1.0.0 + type: File + - name: libzfs + hostPath: + path: /lib64/libzfs.so.2.0.0 + type: File + - name: libuutil + hostPath: + path: /lib64/libuutil.so.1.0.1 + type: File + - name: libnvpair + hostPath: + path: /lib64/libnvpair.so.1.0.1 + type: File + - name: libssl + hostPath: + path: /lib64/libssl.so.1.0.2k + type: FileOrCreate + - name: libcrypto + hostPath: + path: /lib64/libcrypto.so.1.0.2k + type: FileOrCreate + - name: libk5crypto + hostPath: + path: /lib64/libk5crypto.so.3.1 + type: FileOrCreate + - name: libgssapi + hostPath: + path: /lib64/libgssapi_krb5.so.2.2 + type: FileOrCreate + - name: libkrb + hostPath: + path: /lib64/libkrb5.so.3.3 + type: FileOrCreate + - name: libkrb5support + hostPath: + path: /lib64/libkrb5support.so.0.1 + type: FileOrCreate + - name: libkeyutils + hostPath: + path: /lib64/libkeyutils.so.1.6 + type: FileOrCreate + - name: libtirpc + hostPath: + path: /lib64/libtirpc.so.3.0.0 + type: FileOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/zfs-localpv/ + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/ + type: Directory +--- diff --git a/deploy/yamls/zfs-driver.yaml b/deploy/yamls/ubuntu/zfs-driver.yaml similarity index 100% rename from deploy/yamls/zfs-driver.yaml rename to deploy/yamls/ubuntu/zfs-driver.yaml