feat(provision): try volume creation on all the nodes

Currently controller picks one node and the node agent keeps on trying to
create the volume on that node. There might not be enough space available
on that node to create the volume.

The controller can try on all the nodes sequentially and fail
the request if volume creation fails on all the nodes which satisfies the
topology contraints.

Signed-off-by: Pawan <pawan@mayadata.io>

changelogs/unreleased/270-pawanpraka1

@@ -0,0 +1 @@
+try volume creation on all the nodes that satisfy topology contraints

deploy/yamls/zfsvolume-crd.yaml

@@ -231,6 +231,7 @@ spec:
                 enum:
                 - Pending
                 - Ready
+                - Failed
                 type: string
             type: object
         required:

deploy/zfs-operator.yaml

@@ -252,6 +252,7 @@ spec:
                 enum:
                 - Pending
                 - Ready
+                - Failed
                 type: string
             type: object
         required:

go.sum

@@ -317,7 +317,6 @@ github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7ir
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -352,7 +351,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -361,7 +359,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -379,7 +376,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
@@ -398,13 +394,11 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
@@ -423,7 +417,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
@@ -481,7 +474,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2l
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
@@ -529,14 +521,12 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
@@ -563,7 +553,6 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -612,7 +601,6 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9 h1:6XzpBoANz1NqMNfDXzc2QmHmbb1vyMsvRfoP5rM+K1I=
 google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@@ -646,7 +634,6 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -717,7 +704,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhY
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/testing_frameworks v0.1.1 h1:cP2l8fkA3O9vekpy5Ks8mmA0NW/F7yBdXf8brkWhVrs=
 sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U=
-sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

pkg/apis/openebs.io/zfs/v1/zfsvolume.go

@@ -186,6 +186,6 @@ type VolStatus struct {
 	// The state "Pending" means that the volume creation request has not
 	// processed yet. The state "Ready" means that the volume has been created
 	// and it is ready for the use.
-	// +kubebuilder:validation:Enum=Pending;Ready
+	// +kubebuilder:validation:Enum=Pending;Ready;Failed
 	State string `json:"state,omitempty"`
 }

pkg/driver/controller.go

@@ -101,23 +101,6 @@ func getRoundedCapacity(size int64) int64 {
 	return ((size + Mi - 1) / Mi) * Mi
 }
 
-func waitForReadyVolume(volname string) error {
-	for true {
-		vol, err := zfs.GetZFSVolume(volname)
-		if err != nil {
-			return status.Errorf(codes.Internal,
-				"zfs: wait failed, not able to get the volume %s %s", volname, err.Error())
-		}
-
-		switch vol.Status.State {
-		case zfs.ZFSStatusReady:
-			return nil
-		}
-		time.Sleep(time.Second)
-	}
-	return nil
-}
-
 func waitForVolDestroy(volname string) error {
 	for true {
 		_, err := zfs.GetZFSVolume(volname)
@@ -129,6 +112,7 @@ func waitForVolDestroy(volname string) error {
 				"zfs: destroy wait failed, not able to get the volume %s %s", volname, err.Error())
 		}
 		time.Sleep(time.Second)
+		klog.Infof("waiting for volume to be destroyed %s", volname)
 	}
 	return nil
 }
@@ -151,7 +135,7 @@ func waitForReadySnapshot(snapname string) error {
 }
 
 // CreateZFSVolume create new zfs volume from csi volume request
-func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) {
+func CreateZFSVolume(ctx context.Context, req *csi.CreateVolumeRequest) (string, error) {
 	volName := strings.ToLower(req.GetName())
 	size := getRoundedCapacity(req.GetCapacityRange().RequiredBytes)
 
@@ -191,6 +175,10 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) {
 				return "", status.Errorf(codes.AlreadyExists,
 					"volume %s already present", volName)
 			}
+			if vol.Status.State != zfs.ZFSStatusReady {
+				return "", status.Errorf(codes.Aborted,
+					"volume %s request already pending", volName)
+			}
 			return vol.Spec.OwnerNodeID, nil
 		}
 	}
@@ -200,21 +188,19 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) {
 		return "", status.Errorf(codes.Internal, "get node map failed : %s", err.Error())
 	}
 
-	// run the scheduler get the preferred nodelist
+	var prfList []string
-	var selected string
+
-	nodelist := schd.Scheduler(req, nmap)
+	if node, ok := parameters["node"]; ok {
-	if len(nodelist) != 0 {
-		selected = nodelist[0]
-	}
-	if len(selected) == 0 {
 		// (hack): CSI Sanity test does not pass topology information
-		selected = parameters["node"]
+		prfList = append(prfList, node)
-		if len(selected) == 0 {
+	} else {
-			return "", status.Error(codes.Internal, "scheduler failed, not able to select a node to create the PV")
+		// run the scheduler
-		}
+		prfList = schd.Scheduler(req, nmap)
 	}
 
-	klog.Infof("scheduled the volume %s/%s on node %s", pool, volName, selected)
+	if len(prfList) == 0 {
+		return "", status.Error(codes.Internal, "scheduler failed, node list is empty for creating the PV")
+	}
 
 	volObj, err := volbuilder.NewBuilder().
 		WithName(volName).
@@ -227,7 +213,6 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) {
 		WithKeyFormat(kf).
 		WithKeyLocation(kl).
 		WithThinProv(tp).
-		WithOwnerNode(selected).
 		WithVolumeType(vtype).
 		WithVolumeStatus(zfs.ZFSStatusPending).
 		WithFsType(fstype).
@@ -238,17 +223,36 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) {
 		return "", status.Error(codes.Internal, err.Error())
 	}
 
-	err = zfs.ProvisionVolume(volObj)
+	klog.Infof("zfs: trying volume creation %s/%s on node %s", pool, volName, prfList)
-	if err != nil {
+
-		return "", status.Errorf(codes.Internal,
+	// try volume creation sequentially on all nodes
-			"not able to provision the volume %s", err.Error())
+	for _, node := range prfList {
+		vol, _ := volbuilder.BuildFrom(volObj).WithOwnerNode(node).WithVolumeStatus(zfs.ZFSStatusPending).Build()
+
+		timeout := false
+
+		timeout, err = zfs.ProvisionVolume(ctx, vol)
+		if err == nil {
+			return node, nil
+		}
+
+		// if timeout reached, return the error and let csi retry the volume creation
+		if timeout {
+			break
+		}
 	}
 
-	return selected, nil
+	if err != nil {
+		// volume provisioning failed, delete the zfs volume resource
+		zfs.DeleteVolume(volName) // ignore error
+	}
+
+	return "", status.Errorf(codes.Internal,
+		"not able to provision the volume, nodes %v, err : %s", prfList, err.Error())
 }
 
 // CreateVolClone creates the clone from a volume
-func CreateVolClone(req *csi.CreateVolumeRequest, srcVol string) (string, error) {
+func CreateVolClone(ctx context.Context, req *csi.CreateVolumeRequest, srcVol string) (string, error) {
 	volName := strings.ToLower(req.GetName())
 	parameters := req.GetParameters()
 	// lower case keys, cf CreateZFSVolume()
@@ -286,17 +290,17 @@ func CreateVolClone(req *csi.CreateVolumeRequest, srcVol string) (string, error)
 	// use the snapshot name same as new volname
 	volObj.Spec.SnapName = vol.Name + "@" + volName
 
-	err = zfs.ProvisionVolume(volObj)
+	_, err = zfs.ProvisionVolume(ctx, volObj)
 	if err != nil {
 		return "", status.Errorf(codes.Internal,
-			"clone: not able to provision the volume %s", err.Error())
+			"clone: not able to provision the volume err : %s", err.Error())
 	}
 
 	return selected, nil
 }
 
 // CreateSnapClone creates the clone from a snapshot
-func CreateSnapClone(req *csi.CreateVolumeRequest, snapshot string) (string, error) {
+func CreateSnapClone(ctx context.Context, req *csi.CreateVolumeRequest, snapshot string) (string, error) {
 	volName := strings.ToLower(req.GetName())
 	parameters := req.GetParameters()
 	// lower case keys, cf CreateZFSVolume()
@@ -339,10 +343,10 @@ func CreateSnapClone(req *csi.CreateVolumeRequest, snapshot string) (string, err
 	volObj.Spec = snap.Spec
 	volObj.Spec.SnapName = strings.ToLower(snapshot)
 
-	err = zfs.ProvisionVolume(volObj)
+	_, err = zfs.ProvisionVolume(ctx, volObj)
 	if err != nil {
 		return "", status.Errorf(codes.Internal,
-			"not able to provision the clone volume %s", err.Error())
+			"not able to provision the clone volume err : %s", err.Error())
 	}
 
 	return selected, nil
@@ -372,23 +376,19 @@ func (cs *controller) CreateVolume(
 	if contentSource != nil && contentSource.GetSnapshot() != nil {
 		snapshotID := contentSource.GetSnapshot().GetSnapshotId()
 
-		selected, err = CreateSnapClone(req, snapshotID)
+		selected, err = CreateSnapClone(ctx, req, snapshotID)
 	} else if contentSource != nil && contentSource.GetVolume() != nil {
 		srcVol := contentSource.GetVolume().GetVolumeId()
-		selected, err = CreateVolClone(req, srcVol)
+		selected, err = CreateVolClone(ctx, req, srcVol)
 	} else {
-		selected, err = CreateZFSVolume(req)
+		selected, err = CreateZFSVolume(ctx, req)
 	}
 
 	if err != nil {
 		return nil, err
 	}
 
-	if _, ok := parameters["wait"]; ok {
+	klog.Infof("created the volume %s/%s on node %s", pool, volName, selected)
-		if err := waitForReadyVolume(volName); err != nil {
-			return nil, err
-		}
-	}
 
 	sendEventOrIgnore(pvcName, volName, strconv.FormatInt(int64(size), 10), "zfs-localpv", analytics.VolumeProvision)
 
@@ -411,11 +411,7 @@ func (cs *controller) DeleteVolume(
 
 	klog.Infof("received request to delete volume {%s}", req.VolumeId)
 
-	var (
+	if err := cs.validateDeleteVolumeReq(req); err != nil {
-		err error
-	)
-
-	if err = cs.validateDeleteVolumeReq(req); err != nil {
 		return nil, err
 	}
 
@@ -438,6 +434,11 @@ func (cs *controller) DeleteVolume(
 		)
 	}
 
+	// if volume is not ready, create volume will delete it
+	if vol.Status.State != zfs.ZFSStatusReady {
+		return nil, status.Error(codes.Internal, "can not delete, volume creation is in progress")
+	}
+
 	// Delete the corresponding ZV CR
 	err = zfs.DeleteVolume(volumeID)
 	if err != nil {

pkg/mgmt/volume/volume.go

@@ -95,7 +95,9 @@ func (c *ZVController) syncZV(zv *apis.ZFSVolume) error {
 				err = zfs.CreateVolume(zv)
 			}
 			if err == nil {
-				err = zfs.UpdateZvolInfo(zv)
+				err = zfs.UpdateZvolInfo(zv, zfs.ZFSStatusReady)
+			} else {
+				err = zfs.UpdateZvolInfo(zv, zfs.ZFSStatusFailed)
 			}
 		}
 	}
@@ -132,7 +134,8 @@ func (c *ZVController) updateZV(oldObj, newObj interface{}) {
 
 	oldZV, _ := oldObj.(*apis.ZFSVolume)
 	if zfs.PropertyChanged(oldZV, newZV) ||
-		c.isDeletionCandidate(newZV) {
+		c.isDeletionCandidate(newZV) ||
+		newZV.Status.State == zfs.ZFSStatusPending {
 		klog.Infof("Got update event for ZV %s/%s", newZV.Spec.PoolName, newZV.Name)
 		c.enqueueZV(newZV)
 	}

pkg/zfs/volume.go

@@ -15,8 +15,11 @@
 package zfs
 
 import (
+	"context"
+	"fmt"
 	"os"
 	"strconv"
+	"time"
 
 	apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/zfs/v1"
 	"github.com/openebs/zfs-localpv/pkg/builder/bkpbuilder"
@@ -80,18 +83,63 @@ func init() {
 	GoogleAnalyticsEnabled = os.Getenv(GoogleAnalyticsKey)
 }
 
+func checkVolCreation(ctx context.Context, volname string) (bool, error) {
+	timeout := time.After(10 * time.Second)
+	for {
+		select {
+		case <-ctx.Done():
+			return true, fmt.Errorf("zfs: context deadline reached")
+		case <-timeout:
+			return true, fmt.Errorf("zfs: vol creation timeout reached")
+		default:
+			vol, err := GetZFSVolume(volname)
+			if err != nil {
+				return false, fmt.Errorf("zfs: wait failed, not able to get the volume %s %s", volname, err.Error())
+			}
+
+			switch vol.Status.State {
+			case ZFSStatusReady:
+				return false, nil
+			case ZFSStatusFailed:
+				return false, fmt.Errorf("zfs: volume creation failed")
+			}
+
+			klog.Infof("zfs: waiting for volume %s/%s to be created on node %s",
+				vol.Spec.PoolName, volname, vol.Spec.OwnerNodeID)
+
+			time.Sleep(time.Second)
+		}
+	}
+}
+
 // ProvisionVolume creates a ZFSVolume(zv) CR,
 // watcher for zvc is present in CSI agent
 func ProvisionVolume(
+	ctx context.Context,
 	vol *apis.ZFSVolume,
-) error {
+) (bool, error) {
+	timeout := false
+	zv, err := GetZFSVolume(vol.Name)
 
-	_, err := volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Create(vol)
 	if err == nil {
-		klog.Infof("provisioned volume %s", vol.Name)
+		// update the spec and status
+		zv.Spec = vol.Spec
+		zv.Status = vol.Status
+		_, err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Update(zv)
+	} else {
+		_, err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Create(vol)
 	}
 
-	return err
+	if err == nil {
+		timeout, err = checkVolCreation(ctx, vol.Name)
+	}
+
+	if err != nil {
+		klog.Infof("zfs: volume %s/%s provisioning failed on node %s err: %s",
+			vol.Spec.PoolName, vol.Name, vol.Spec.OwnerNodeID, err.Error())
+	}
+
+	return timeout, err
 }
 
 // ResizeVolume resizes the zfs volume
@@ -138,7 +186,9 @@ func GetVolume(volumeID string) (*apis.ZFSVolume, error) {
 func DeleteVolume(volumeID string) (err error) {
 	err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Delete(volumeID)
 	if err == nil {
-		klog.Infof("deprovisioned volume %s", volumeID)
+		klog.Infof("zfs: deleted the volume %s", volumeID)
+	} else {
+		klog.Infof("zfs: volume %s deletion failed %s", volumeID, err.Error())
 	}
 
 	return
@@ -179,17 +229,22 @@ func GetZFSVolumeState(volID string) (string, string, error) {
 }
 
 // UpdateZvolInfo updates ZFSVolume CR with node id and finalizer
-func UpdateZvolInfo(vol *apis.ZFSVolume) error {
+func UpdateZvolInfo(vol *apis.ZFSVolume, status string) error {
-	finalizers := []string{ZFSFinalizer}
+	finalizers := []string{}
 	labels := map[string]string{ZFSNodeKey: NodeID}
 
 	if vol.Finalizers != nil {
 		return nil
 	}
 
+	switch status {
+	case ZFSStatusReady:
+		finalizers = append(finalizers, ZFSFinalizer)
+	}
+
 	newVol, err := volbuilder.BuildFrom(vol).
 		WithFinalizer(finalizers).
-		WithVolumeStatus(ZFSStatusReady).
+		WithVolumeStatus(status).
 		WithLabels(labels).Build()
 
 	if err != nil {